IMBENJI.NET/SweepStore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Add concurrency handling implementation with ticket management and file locking

Browse Source
This commit is contained in:
ImBenji
2025-12-02 14:11:45 +00:00
parent e6ccad87b4
commit eae4d0e24e
16 changed files with 1405 additions and 1551 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
cpp/CMakeLists.txt
View File

@@ -3,13 +3,31 @@ project(BinaryTable)
set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
# Add include directories globally
include_directories(${CMAKE_SOURCE_DIR}/src/Public)
include_directories(${CMAKE_SOURCE_DIR}/src/Private)
# Main executable with integrated binary table implementation # Main executable with integrated binary table implementation
add_executable(main add_executable(main
binary_table.h src/Private/sweepstore/header.cpp
binary_table.cpp src/Public/sweepstore/utils/helpers.h
src/Private/sweepstore/structures.cpp
src/Public/sweepstore/structures.h
src/Private/sweepstore/sweepstore.cpp
src/Public/sweepstore/sweepstore.h
src/Public/sweepstore/concurrency.h
src/Private/sweepstore/concurrency.cpp
src/Public/sweepstore/utils/file_lock.h
src/Public/sweepstore/utils/file_handle.h
src/Public/sweepstore/header.h
src/Private/sweepstore/benchmark.cpp
) )
# Add include directories
target_include_directories(main PRIVATE ${CMAKE_SOURCE_DIR}/src/Public)
# Compiler Settings # Compiler Settings
if(MSVC) if(MSVC)
target_compile_options(main PRIVATE /W4) target_compile_options(main PRIVATE /W4)

1243
cpp/binary_table.cpp
View File

File diff suppressed because it is too large Load Diff

305
cpp/binary_table.h
View File

@@ -1,305 +0,0 @@
/*
/$$$$$$ /$$ /$$ /$$$$$$$ /$$$$$$$$ /$$ /$$ /$$$$$ /$$$$$$ /$$ /$$ /$$$$$$$$ /$$$$$$$$
|_ $$_/| $$$ /$$$| $$__ $$| $$_____/| $$$ | $$ |__ $$|_ $$_/ | $$$ | $$| $$_____/|__ $$__/
| $$ | $$$$ /$$$$| $$ \ $$| $$ | $$$$| $$ | $$ | $$ | $$$$| $$| $$ | $$
| $$ | $$ $$/$$ $$| $$$$$$$ | $$$$$ | $$ $$ $$ | $$ | $$ | $$ $$ $$| $$$$$ | $$
| $$ | $$ $$$| $$| $$__ $$| $$__/ | $$ $$$$ /$$ | $$ | $$ | $$ $$$$| $$__/ | $$
| $$ | $$\ $ | $$| $$ \ $$| $$ | $$\ $$$| $$ | $$ | $$ | $$\ $$$| $$ | $$
/$$$$$$| $$ \/ | $$| $$$$$$$/| $$$$$$$$| $$ \ $$| $$$$$$/ /$$$$$$ /$$| $$ \ $$| $$$$$$$$ | $$
|______/|__/ |__/|_______/ |________/|__/ \__/ \______/ |______/|__/|__/ \__/|________/ |__/
<EFBFBD> 2025-26 by Benjamin Watt of IMBENJI.NET LIMITED - All rights reserved.
Use of this source code is governed by the Business Source License 1.1 that can be found in the LICENSE file.
This file is part of the SweepStore (formerly Binary Table) package for C++.
*/
#pragma once
#include <cstdint>
#include <string>
#include <vector>
#include <unordered_map>
#include <fstream>
#include <variant>
#include <memory>
#include <stdexcept>
#include <type_traits>
#include <functional>
#include <iostream>
// Debug control - comment out this line to disable all debug output
// #define ENABLE_DEBUG 1
#ifdef ENABLE_DEBUG
#define DEBUG_PRINT(x) std::cout << x
#define DEBUG_PRINTLN(x) std::cout << x << std::endl
#else
#define DEBUG_PRINT(x)
#define DEBUG_PRINTLN(x)
#endif
namespace bt {
// Forward declarations
class BinaryTable;
class BT_Reference;
template<typename T> class BT_UniformArray;
// Type enumeration matching Dart version
enum class BT_Type : uint8_t {
POINTER = 0,
ADDRESS_TABLE = 1,
INTEGER = 2,
FLOAT = 3,
STRING = 4,
INTEGER_ARRAY = 5,
FLOAT_ARRAY = 6
};
// Size mapping for types
constexpr int getTypeSize(BT_Type type) {
switch (type) {
case BT_Type::POINTER: return 8;
case BT_Type::ADDRESS_TABLE: return -1;
case BT_Type::INTEGER: return 4;
case BT_Type::FLOAT: return 4;
case BT_Type::STRING: return -1;
case BT_Type::INTEGER_ARRAY: return -1;
case BT_Type::FLOAT_ARRAY: return -1;
}
return -1;
}
// Check if type is array type
constexpr bool isArrayType(BT_Type type) {
return type == BT_Type::INTEGER_ARRAY || type == BT_Type::FLOAT_ARRAY;
}
// Type deduction helpers
template<typename T>
constexpr BT_Type getTypeFromValue() {
if constexpr (std::is_same_v<T, int32_t> || std::is_same_v<T, int>) {
return BT_Type::INTEGER;
} else if constexpr (std::is_same_v<T, float>) {
return BT_Type::FLOAT;
} else if constexpr (std::is_same_v<T, std::string>) {
return BT_Type::STRING;
} else if constexpr (std::is_same_v<T, std::vector<int32_t>> || std::is_same_v<T, std::vector<int>>) {
return BT_Type::INTEGER_ARRAY;
} else if constexpr (std::is_same_v<T, std::vector<float>>) {
return BT_Type::FLOAT_ARRAY;
} else {
static_assert(sizeof(T) == 0, "Unsupported type");
}
}
// Pointer class
class BT_Pointer {
private:
int64_t address_;
public:
explicit BT_Pointer(int64_t address = -1) : address_(address) {}
bool isNull() const { return address_ == -1; }
int64_t address() const { return address_; }
bool operator==(const BT_Pointer& other) const {
return address_ == other.address_;
}
bool operator!=(const BT_Pointer& other) const {
return !(*this == other);
}
};
// Null pointer constant
const BT_Pointer BT_Null{-1};
// Free list entry
struct BT_FreeListEntry {
BT_Pointer pointer;
int32_t size;
BT_FreeListEntry(BT_Pointer ptr, int32_t sz) : pointer(ptr), size(sz) {}
};
// Value encoding functions
std::vector<uint8_t> encodeValue(const int32_t& value);
std::vector<uint8_t> encodeValue(const float& value);
std::vector<uint8_t> encodeValue(const std::string& value);
std::vector<uint8_t> encodeValue(const std::vector<int32_t>& value);
std::vector<uint8_t> encodeValue(const std::vector<float>& value);
// Template wrapper for encoding
template<typename T>
std::vector<uint8_t> encodeValue(const T& value) {
return encodeValue(value);
}
// Reference class for handling stored values
class BT_Reference {
protected:
BinaryTable* table_;
BT_Pointer pointer_;
public:
BT_Reference(BinaryTable* table, BT_Pointer pointer);
template<typename T>
T decodeValue();
int32_t size() const;
BT_Type getType() const;
bool isNull() const { return pointer_.isNull(); }
BT_Pointer getPointer() const { return pointer_; }
};
// Uniform array class template
template<typename T>
class BT_UniformArray : public BT_Reference {
public:
BT_UniformArray(BinaryTable* table, BT_Pointer pointer) : BT_Reference(table, pointer) {}
int32_t length() const;
T operator[](int32_t index) const;
void set(int32_t index, const T& value);
void add(const T& value);
void addAll(const std::vector<T>& values);
std::vector<T> fetchSublist(int32_t start = 0, int32_t end = -1);
};
// Main BinaryTable class
class BinaryTable {
private:
FILE* file_;
std::string filePath_;
// Free list management
bool freeListLifted_;
std::vector<BT_FreeListEntry> freeListCache_;
// Internal methods
std::unordered_map<int64_t, BT_Pointer> getAddressTable();
void setAddressTable(const std::unordered_map<int64_t, BT_Pointer>& table);
std::vector<BT_FreeListEntry> getFreeList();
void setFreeList(const std::vector<BT_FreeListEntry>& list);
int64_t hashString(const std::string& str) const;
void truncateFile(int64_t newSize);
void antiFreeListScope(std::function<void()> fn);
void free(BT_Pointer pointer, int32_t size);
BT_Pointer alloc(int32_t size);
// File I/O helpers
int32_t readInt32(int64_t position);
float readFloat32(int64_t position);
int64_t readInt64(int64_t position);
uint8_t readByte(int64_t position);
std::vector<uint8_t> readBytes(int64_t position, int32_t count);
void writeInt32(int64_t position, int32_t value);
void writeFloat32(int64_t position, float value);
void writeInt64(int64_t position, int64_t value);
void writeByte(int64_t position, uint8_t value);
void writeBytes(int64_t position, const std::vector<uint8_t>& data);
public:
explicit BinaryTable(const std::string& path);
~BinaryTable();
void initialize();
// Memory management
void liftFreeList();
void dropFreeList();
// Data operations
template<typename T>
void set(const std::string& key, const T& value);
template<typename T>
T get(const std::string& key);
BT_Reference getReference(const std::string& key);
template<typename T>
BT_UniformArray<T> getArray(const std::string& key);
void remove(const std::string& key);
void truncate();
// Debug methods
void debugAddressTable(const std::string& context = "");
// File access for reference classes
friend class BT_Reference;
template<typename T> friend class BT_UniformArray;
int64_t getFileLength();
void setFilePosition(int64_t position);
};
// Template specializations for decodeValue
template<> int32_t BT_Reference::decodeValue<int32_t>();
template<> float BT_Reference::decodeValue<float>();
template<> std::string BT_Reference::decodeValue<std::string>();
template<> std::vector<int32_t> BT_Reference::decodeValue<std::vector<int32_t>>();
template<> std::vector<float> BT_Reference::decodeValue<std::vector<float>>();
template<> BT_UniformArray<int32_t> BT_Reference::decodeValue<BT_UniformArray<int32_t>>();
template<> BT_UniformArray<float> BT_Reference::decodeValue<BT_UniformArray<float>>();
// Template method implementations for BinaryTable
template<typename T>
void BinaryTable::set(const std::string& key, const T& value) {
antiFreeListScope([&]() {
auto addressTable = getAddressTable();
int64_t keyHash = hashString(key);
if (addressTable.find(keyHash) != addressTable.end()) {
throw std::runtime_error("Key already exists");
}
auto valueBuffer = encodeValue(value);
BT_Pointer valueAddress = alloc(static_cast<int32_t>(valueBuffer.size()));
writeBytes(valueAddress.address(), valueBuffer);
addressTable[keyHash] = valueAddress;
setAddressTable(addressTable);
});
}
template<typename T>
T BinaryTable::get(const std::string& key) {
auto addressTable = getAddressTable();
int64_t keyHash = hashString(key);
auto it = addressTable.find(keyHash);
if (it == addressTable.end()) {
throw std::runtime_error("Key does not exist");
}
BT_Reference valueRef(this, it->second);
return valueRef.decodeValue<T>();
}
template<typename T>
BT_UniformArray<T> BinaryTable::getArray(const std::string& key) {
auto addressTable = getAddressTable();
int64_t keyHash = hashString(key);
auto it = addressTable.find(keyHash);
if (it == addressTable.end()) {
throw std::runtime_error("Key does not exist");
}
return BT_UniformArray<T>(this, it->second);
}
} // namespace bt

248
cpp/src/Private/sweepstore/concurrency.cpp Normal file
View File

@@ -0,0 +1,248 @@
#include <functional>
#include <iosfwd>
#include "sweepstore/concurrency.h"
#include <iostream>
#include <random>
#include "sweepstore/header.h"
#include "sweepstore/utils/helpers.h"
#include "sweepstore/utils/file_handle.h"
uint64_t getRandomOffset(uint64_t maxValue) {
static std::random_device rd;
static std::mt19937_64 gen(rd());
std::uniform_int_distribution<uint64_t> dist(0, maxValue);
return dist(gen);
}
int randomId() {
// mix timestamp with random for better uniqueness
// keep it positive to avoid signed int issues when storing
auto now = std::chrono::system_clock::now();
auto millis = std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count();
int32_t time = static_cast<int32_t>(millis & 0xFFFFFFFF); // Get lower 32 bits
int32_t random = static_cast<int32_t>(getRandomOffset(0x7FFFFFFF)); // 0 to 0x7FFFFFFF
return (time ^ random) & 0x7FFFFFFF;
}
void SweepstoreConcurrency::spawnTicket(std::string filePath,
const SweepstoreTicketOperation& operation,
const uint32_t keyHash,
const uint32_t targetSize,
const std::function<void()> onApproved,
std::string debugLabel
) {
SweepstoreFileHandle file(filePath, std::ios::binary | std::ios::in | std::ios::out);
/*
Useful Functions
*/
/// Logging function
auto log = [&](const std::string &message) {
std::string prefix = !debugLabel.empty() ? "\033[38;5;208m[Ticket Spawner - " + debugLabel + "]:\033[0m " : "\033[38;5;208m[Ticket Spawner]:\033[0m ";
debugPrint(prefix + message);
};
// Sleep with variance (additive only)
auto varySleep = [&](std::chrono::nanoseconds minSleepDuration, std::chrono::nanoseconds variance) {
if (variance.count() <= 0) {
preciseSleep(minSleepDuration);
} else {
// Generate random duration within variance
uint64_t randomOffset = getRandomOffset(variance.count());
preciseSleep(minSleepDuration + std::chrono::nanoseconds(randomOffset));
}
};
// Exponential sleep
std::unordered_map<std::string, int> expSleepTracker = {};
auto expSleep = [&expSleepTracker](const std::string& label) {
int count = expSleepTracker[label]; // defaults to 0 if not found
int sleepTime = (1 << count); // Exponential backoff
sleepTime = std::max(1, std::min(sleepTime, 1000)); // Clamp between 1ms and 1000ms
preciseSleep(std::chrono::microseconds(sleepTime * 5000));
expSleepTracker[label] = count + 1;
};
// Get the header(s)
SweepstoreHeader header(file);
SweepstoreConcurrencyHeader concurrencyHeader(file);
/*
Ticket Acquisition
*/
auto acquireTicket = [&](uint32_t newIdentifier) -> SweepstoreWorkerTicket {
// Reduce the chance of race condition
varySleep(std::chrono::microseconds(500), std::chrono::microseconds(200));
uint32_t ticketIndex = -1u;
while (true) {
uint32_t concurrentWorkers = concurrencyHeader.readNumberOfWorkers();
for (uint32_t i = 0; i < concurrentWorkers; i++) {
SweepstoreWorkerTicket ticket = SweepstoreWorkerTicket(i, file);
if (!ticket.writable()) {
continue;
}
SweepstoreWorkerTicketSnapshot snapshot = ticket.snapshot();
int identifier = snapshot.identifier;
bool identifier_unassigned = identifier == 0;
bool stale_heartbeat = millisecondsSinceEpoch32() - snapshot.workerHeartbeat > STALE_HEARTBEAT_THRESHOLD_MS;
bool is_free = snapshot.state == SweepstoreTicketState::FREE;
if (identifier_unassigned && stale_heartbeat && is_free) {
snapshot.identifier = newIdentifier;
snapshot.workerHeartbeat = millisecondsSinceEpoch32();
snapshot.state = SweepstoreTicketState::WAITING;
ticket.write(snapshot);
ticketIndex = i;
break;
}
}
preciseSleep(std::chrono::milliseconds(2));
// Ensure we still own the ticket - if not, reset and try again
if (ticketIndex != -1u) {
SweepstoreWorkerTicketSnapshot verifySnapshot = concurrencyHeader[ticketIndex].snapshot();
if (verifySnapshot.identifier != newIdentifier) {
ticketIndex = -1; // Lost the ticket, try again
} else {
log("Acquired ticket " + std::to_string(ticketIndex) + " with identifier " + std::to_string(newIdentifier) + ".");
return concurrencyHeader[ticketIndex];
}
}
expSleep("acquireTicket");
}
};
uint32_t myIdentifier = randomId();
SweepstoreWorkerTicket myTicket = acquireTicket(myIdentifier);
SweepstoreWorkerTicketSnapshot mySnapshot = myTicket.snapshot();
mySnapshot.workerHeartbeat = millisecondsSinceEpoch32();
mySnapshot.state = SweepstoreTicketState::WAITING;
mySnapshot.operation = operation;
mySnapshot.keyHash = keyHash;
mySnapshot.targetSize = targetSize;
myTicket.write(mySnapshot);
// Wait for approval
while (true) {
SweepstoreWorkerTicketSnapshot snapshot = myTicket.snapshot();
// Update heartbeat
uint32_t currentTime = millisecondsSinceEpoch32();
if (currentTime - snapshot.workerHeartbeat > 700) {
snapshot.workerHeartbeat = currentTime;
myTicket.write(snapshot);
}
// Check if we still own the ticket
if (snapshot.identifier != myIdentifier) {
preciseSleep(std::chrono::milliseconds(10));
// Re-verify we lost the ticket
SweepstoreWorkerTicketSnapshot recheckSnapshot = myTicket.snapshot();
if (recheckSnapshot.identifier != myIdentifier) {
log("Lost ownership of ticket " + std::to_string(myTicket.getTicketIndex()) + ", was expecting identifier " + std::to_string(myIdentifier) + " but found " + std::to_string(recheckSnapshot.identifier) + ".");
// ReSharper disable once CppDFAInfiniteRecursion
return spawnTicket(
filePath,
operation,
keyHash,
targetSize,
onApproved,
debugLabel
);
}
// False alarm, continue waiting
log("False alarm, still own ticket " + std::to_string(myTicket.getTicketIndex()) + ".");
snapshot = recheckSnapshot;
}
if (snapshot.state == SweepstoreTicketState::APPROVED) {
snapshot.state = SweepstoreTicketState::EXECUTING;
myTicket.write(snapshot);
onApproved();
snapshot.state = SweepstoreTicketState::COMPLETED;
myTicket.write(snapshot);
break;
}
varySleep(std::chrono::microseconds(500), std::chrono::microseconds(200));
}
// std::cout << "\033[38;5;82m[Ticket Spawner - " << debugLabel << "]:\033[0m Completed ticket " << myTicket.getTicketIndex() << "." << std::endl;
}
void SweepstoreConcurrency::initialiseMaster(std::string filePath) {
auto log = [&](const std::string &message) {
debugPrint("\033[38;5;33m[Concurrency Master]:\033[0m " + message);
};
SweepstoreFileHandle file(filePath, std::ios::binary | std::ios::in | std::ios::out);
SweepstoreHeader header(file);
SweepstoreConcurrencyHeader concurrencyHeader(file);
while (true) {
int concurrentWorkers = concurrencyHeader.readNumberOfWorkers();
for (uint32_t i = 0; i < concurrentWorkers; i++) {
SweepstoreWorkerTicket ticket(i, file);
SweepstoreWorkerTicketSnapshot snapshot = ticket.snapshot();
if (snapshot.state == WAITING) {
log("Found waiting ticket " + std::to_string(i) + "(Key Hash: " + std::to_string(snapshot.keyHash) + ")...");
// Approve the ticket
snapshot.state = APPROVED;
ticket.write(snapshot);
log("Approved ticket " + std::to_string(i) + ".");
} else if (snapshot.state == SweepstoreTicketState::COMPLETED) {
log("Ticket " + std::to_string(i) + " has completed. Resetting...");
// Reset the ticket
SweepstoreWorkerTicketSnapshot cleanSnapshot = SweepstoreWorkerTicketSnapshot();
ticket.write(cleanSnapshot);
log("Reset ticket " + std::to_string(i) + ".");
}
// Handle stale tickets
uint32_t currentTime = millisecondsSinceEpoch32();
}
preciseSleep(std::chrono::milliseconds(1));
}
}

220
cpp/src/Private/sweepstore/header.cpp Normal file
View File

@@ -0,0 +1,220 @@
#include "sweepstore/header.h"
#include "sweepstore/utils/file_lock.h"
#include "sweepstore/utils/helpers.h"
std::string SweepstoreHeader::readMagicNumber() {
file->seekg(0, std::ios::beg);
char buffer[4];
file->read(buffer, 4);
return std::string(buffer, 4);
}
void SweepstoreHeader::writeMagicNumber(const std::string& magicNumber) {
if (magicNumber.size() != 4) {
throw std::invalid_argument("Magic number must be exactly 4 characters long.");
}
file->seekp(0, std::ios::beg);
file->write(magicNumber.c_str(), 4);
}
std::string SweepstoreHeader::readVersion() {
file->seekg(4, std::ios::beg);
char buffer[12];
file->read(buffer, 12);
// Trim leading and trailing spaces
std::string version(buffer, 12);
version = trim(version);
return version;
}
void SweepstoreHeader::writeVersion(const std::string& version) {
if (version.size() > 11) {
throw std::invalid_argument("Version string must be at most 11 characters long.");
}
// Pad 1 space to the beginning
// And pad to end to make it 12 bytes
std::string paddedVersion = " " + version;
paddedVersion.resize(12, ' ');
file->seekp(4, std::ios::beg);
file->write(paddedVersion.c_str(), 12);
}
SweepstorePointer SweepstoreHeader::readAddressTablePointer() {
file->seekg(16, std::ios::beg);
int64_t address;
file->read(reinterpret_cast<char*>(&address), sizeof(address));
return address; // Implicit conversion to SweepstorePointer
}
void SweepstoreHeader::writeAddressTablePointer(const SweepstorePointer& ptr) {
file->seekp(16, std::ios::beg);
int64_t address = ptr;
file->write(reinterpret_cast<const char*>(&address), sizeof(address));
}
uint32_t SweepstoreHeader::readFreeListCount() {
file->seekg(24, std::ios::beg);
uint32_t count;
file->read(reinterpret_cast<char*>(&count), sizeof(count));
return count;
}
void SweepstoreHeader::writeFreeListCount(uint32_t count) {
file->seekp(24, std::ios::beg);
file->write(reinterpret_cast<const char*>(&count), sizeof(count));
}
bool SweepstoreHeader::readIsFreeListLifted() {
file->seekg(28, std::ios::beg);
char flag;
file->read(&flag, sizeof(flag));
return flag != 0;
}
void SweepstoreHeader::writeIsFreeListLifted(bool isLifted) {
file->seekp(28, std::ios::beg);
char flag = isLifted ? 1 : 0;
file->write(&flag, sizeof(flag));
}
void SweepstoreHeader::initialise() {
writeMagicNumber("SWPT");
writeVersion("undefined");
writeAddressTablePointer(SweepstorePointer::NULL_PTR);
writeFreeListCount(0);
writeIsFreeListLifted(false);
file->flush();
}
uint64_t SweepstoreConcurrencyHeader::readMasterIdentifier() {
file->seekg(29, std::ios::beg);
uint64_t identifier;
file->read(reinterpret_cast<char*>(&identifier), sizeof(identifier));
return identifier;
}
void SweepstoreConcurrencyHeader::writeMasterIdentifier(uint64_t identifier) {
file->seekp(29, std::ios::beg);
file->write(reinterpret_cast<const char*>(&identifier), sizeof(identifier));
}
uint32_t SweepstoreConcurrencyHeader::readMasterHeartbeat() {
file->seekg(37, std::ios::beg);
uint32_t heartbeat;
file->read(reinterpret_cast<char*>(&heartbeat), sizeof(heartbeat));
return heartbeat;
}
void SweepstoreConcurrencyHeader::writeMasterHeartbeat(uint32_t heartbeat) {
file->seekp(37, std::ios::beg);
file->write(reinterpret_cast<const char*>(&heartbeat), sizeof(heartbeat));
}
uint32_t SweepstoreConcurrencyHeader::readNumberOfWorkers() {
file->seekg(41, std::ios::beg);
uint32_t numWorkers;
file->read(reinterpret_cast<char*>(&numWorkers), sizeof(numWorkers));
return numWorkers;
}
void SweepstoreConcurrencyHeader::writeNumberOfWorkers(uint32_t numWorkers) {
file->seekp(41, std::ios::beg);
file->write(reinterpret_cast<const char*>(&numWorkers), sizeof(numWorkers));
}
bool SweepstoreConcurrencyHeader::readIsReadAllowed() {
file->seekg(45, std::ios::beg);
char flag;
file->read(&flag, sizeof(flag));
return flag != 0;
}
void SweepstoreConcurrencyHeader::writeIsReadAllowed(bool isAllowed) {
file->seekp(45, std::ios::beg);
char flag = isAllowed ? 1 : 0;
file->write(&flag, sizeof(flag));
}
void SweepstoreConcurrencyHeader::initialise(int concurrentWorkers) {
writeMasterIdentifier(0);
writeMasterHeartbeat(0);
writeNumberOfWorkers(concurrentWorkers);
writeIsReadAllowed(true);
for (uint32_t i = 0; i < readNumberOfWorkers(); i++) {
SweepstoreWorkerTicketSnapshot ticket = SweepstoreWorkerTicketSnapshot();
ticket.identifier = 0;
ticket.workerHeartbeat = 0;
ticket.state = SweepstoreTicketState::FREE;
ticket.operation = SweepstoreTicketOperation::NONE;
ticket.keyHash = 0;
ticket.targetAddress = SweepstorePointer::NULL_PTR;
ticket.targetSize = 0;
SweepstoreWorkerTicket ticketWriter = SweepstoreWorkerTicket(i, file);
ticketWriter.write(ticket);
}
file->flush();
}
void SweepstoreWorkerTicket::write(SweepstoreWorkerTicketSnapshot &snapshot) {
RandomAccessMemory buffer;
SweepstoreFileLock lock(file.getPath(), SweepstoreFileLock::Mode::Exclusive);
SweepstoreFileLock::Scoped scopedLock(lock);
buffer.setPositionSync(0);
buffer.writeIntSync(snapshot.identifier, 4);
buffer.writeIntSync(snapshot.workerHeartbeat, 4);
buffer.writeIntSync(static_cast<uint8_t>(snapshot.state), 1);
buffer.writeIntSync(static_cast<uint8_t>(snapshot.operation), 1);
buffer.writeUIntSync(snapshot.keyHash, 8);
buffer.writePointerSync(snapshot.targetAddress, 8);
buffer.writeUIntSync(snapshot.targetSize, 4);
// Pad the rest with zeros if necessary
while (buffer.length() < TICKET_SIZE) {
buffer.writeIntSync(0, 1);
}
// Prepare data
buffer.setPositionSync(0);
std::vector<uint8_t> data = buffer.readSync(buffer.length());
char* dataPtr = reinterpret_cast<char*>(data.data());
// Write to file
file->seekp(getOffset());
file->write(dataPtr, data.size());
file->flush();
}
bool SweepstoreWorkerTicket::writable() {
SweepstoreFileLock lock(file.getPath(), SweepstoreFileLock::Mode::Exclusive);
return lock.isLocked() == false;
}
SweepstoreWorkerTicketSnapshot SweepstoreWorkerTicket::snapshot() {
SweepstoreFileLock lock(file.getPath(), SweepstoreFileLock::Mode::Shared);
lock.lock();
file->seekg(getOffset());
std::unique_ptr<char[]> buffer(new char[TICKET_SIZE]);
file->read(buffer.get(), TICKET_SIZE);
lock.unlock();
RandomAccessMemory ram(reinterpret_cast<uint8_t*>(buffer.get()), TICKET_SIZE);
SweepstoreWorkerTicketSnapshot snapshot;
ram.setPositionSync(0);
snapshot.identifier = ram.readUIntSync(4);
snapshot.workerHeartbeat = ram.readUIntSync(4);
snapshot.state = static_cast<SweepstoreTicketState>(ram.readUIntSync(1));
snapshot.operation = static_cast<SweepstoreTicketOperation>(ram.readUIntSync(1));
snapshot.keyHash = ram.readUIntSync(8);
snapshot.targetAddress = ram.readPointerSync(8);
snapshot.targetSize = ram.readUIntSync(4);
return snapshot;
}

13
cpp/src/Private/sweepstore/structures.cpp Normal file
View File

@@ -0,0 +1,13 @@
#include "sweepstore/structures.h"
const SweepstorePointer SweepstorePointer::NULL_PTR = SweepstorePointer(UINT64_MAX);
bool SweepstorePointer::operator==(const SweepstorePointer &p) {
if (this->address == p.address) {
return true;
} else {
return false;
}
}

21
cpp/src/Private/sweepstore/sweepstore.cpp Normal file
View File

@@ -0,0 +1,21 @@
//
// Created by Benjamin Watt on 24/11/2025.
//
#include "sweepstore/sweepstore.h"
#include <string>
#include <filesystem>
#include <iostream>
#include <thread>
#include "sweepstore/utils/helpers.h"
#include "sweepstore/utils/file_handle.h"
void Sweepstore::initialise(int concurrentWorkers) {
header->initialise();
header->writeVersion("1.1.0.2");
concurrencyHeader->initialise(concurrentWorkers);
debugPrint("Version: " + header->readVersion());
}

30
cpp/src/Public/sweepstore/concurrency.h Normal file
View File

@@ -0,0 +1,30 @@
#pragma once
#include <cstdint>
#include <functional>
#include <iosfwd>
#include <thread>
#define STALE_HEARTBEAT_THRESHOLD_MS 5000
enum SweepstoreTicketOperation : int;
namespace SweepstoreConcurrency {
void spawnTicket(std::string filePath,
const SweepstoreTicketOperation& operation,
const uint32_t keyHash,
const uint32_t targetSize,
const std::function<void()> onApproved,
std::string debugLabel = ""
);
void initialiseMaster(std::string filePath);
inline void initialiseMasterAsync(std::string filePath) {
std::thread([&filePath]() {
initialiseMaster(filePath);
}).detach();
}
}

143
cpp/src/Public/sweepstore/header.h Normal file
View File

@@ -0,0 +1,143 @@
#pragma once
#include <iosfwd>
#include "structures.h"
#include "utils/file_handle.h"
constexpr int roundToNearest16(int number) {
return (number + 15) & ~15;
}
class SweepstoreHeader {
private:
SweepstoreFileHandle& file;
public:
explicit SweepstoreHeader(SweepstoreFileHandle &fileStream) : file(fileStream) {}
// Offset 0 - 4 bytes
std::string readMagicNumber();
void writeMagicNumber(const std::string& magicNumber);
// Offset 4 - 12 bytes
std::string readVersion();
void writeVersion(const std::string& version);
// Offset 16 - 8 bytes
SweepstorePointer readAddressTablePointer();
void writeAddressTablePointer(const SweepstorePointer& ptr);
// Offset 24 - 4 bytes
uint32_t readFreeListCount();
void writeFreeListCount(uint32_t count);
// Offset 28 - 1 byte
bool readIsFreeListLifted();
void writeIsFreeListLifted(bool isLifted);
/**
* Initialises the header with default values.
*/
void initialise();
};
constexpr int SWEEPSTORE_COMBINED_STATIC_HEADER_SIZE = roundToNearest16(46);
struct SweepstoreWorkerTicketSnapshot {
SweepstoreWorkerTicketSnapshot() :
identifier(0),
workerHeartbeat(0),
state(SweepstoreTicketState::FREE),
operation(SweepstoreTicketOperation::NONE),
keyHash(0),
targetAddress(SweepstorePointer::NULL_PTR),
targetSize(0) {}
// Offset 0 - 4 bytes
uint32_t identifier;
// Offset 4 - 4 bytes
uint32_t workerHeartbeat;
// Offset 8 - 1 byte
SweepstoreTicketState state;
// Offset 9 - 1 byte
SweepstoreTicketOperation operation;
// Offset 10 - 8 bytes
uint64_t keyHash;
// Offset 18 - 8 bytes
SweepstorePointer targetAddress;
// Offset 26 - 4 bytes
uint32_t targetSize;
};
class SweepstoreWorkerTicket {
SweepstoreFileHandle& file;
uint32_t ticketIndex;
uint64_t getOffset() const {
return SWEEPSTORE_COMBINED_STATIC_HEADER_SIZE + (ticketIndex * TICKET_SIZE);
}
public:
static constexpr int TICKET_SIZE = roundToNearest16(29);
SweepstoreWorkerTicket(const uint32_t index, SweepstoreFileHandle& fileStream) :
file(fileStream),
ticketIndex(index) {}
int getTicketIndex() const {
return ticketIndex;
}
void write(SweepstoreWorkerTicketSnapshot &snapshot);
bool writable();
SweepstoreWorkerTicketSnapshot snapshot();
};
class SweepstoreConcurrencyHeader {
private:
SweepstoreFileHandle& file;
public:
explicit SweepstoreConcurrencyHeader(SweepstoreFileHandle &fileStream) : file(fileStream) {}
// Offset 29 - 8 bytes
uint64_t readMasterIdentifier();
void writeMasterIdentifier(uint64_t identifier);
// Offset 37 - 4 bytes
uint32_t readMasterHeartbeat();
void writeMasterHeartbeat(uint32_t heartbeat);
// Offset 41 - 4 bytes
uint32_t readNumberOfWorkers();
void writeNumberOfWorkers(uint32_t numWorkers);
// Offset 45 - 1 byte
bool readIsReadAllowed();
void writeIsReadAllowed(bool isAllowed);
/**
* Initialises the concurrency header with default values.
*/
void initialise(int concurrentWorkers);
SweepstoreWorkerTicket operator[](const uint32_t index) const {
return SweepstoreWorkerTicket(index, file);
}
};

39
cpp/src/Public/sweepstore/structures.h Normal file
View File

@@ -0,0 +1,39 @@
#pragma once
#include <cstdint>
class SweepstorePointer {
private:
uint64_t address;
public:
static const SweepstorePointer NULL_PTR;
SweepstorePointer(int64_t addr) : address(addr) {}
bool isNull() {
return this->address == UINT64_MAX;
}
bool operator==(const SweepstorePointer &p);
// Implicit conversion to uint64_t
operator uint64_t() const {
return address;
}
};
enum SweepstoreTicketState {
FREE,
WAITING,
APPROVED,
EXECUTING,
COMPLETED,
};
enum SweepstoreTicketOperation : int {
NONE,
READ,
MODIFY,
WRITE
};

67
cpp/src/Public/sweepstore/sweepstore.h Normal file
View File

@@ -0,0 +1,67 @@
#pragma once
#include <iosfwd>
#include "concurrency.h"
#include "header.h"
#include "utils/helpers.h"
#include "utils/file_handle.h"
class Sweepstore {
private:
std::string filePath;
SweepstoreFileHandle file;
SweepstoreHeader* header;
SweepstoreConcurrencyHeader* concurrencyHeader;
public:
Sweepstore(const std::string& filePath) : filePath(filePath), file(filePath, std::ios::binary | std::ios::in | std::ios::out | std::ios::trunc) {
header = new SweepstoreHeader(file);
concurrencyHeader = new SweepstoreConcurrencyHeader(file);
}
~Sweepstore() {
delete header;
delete concurrencyHeader;
file.close();
}
void initialise(int concurrentWorkers = 4);
SweepstoreConcurrencyHeader* getConcurrencyHeader() {
return concurrencyHeader;
}
class Proxy {
Sweepstore* sweepstore;
std::string key;
public:
Proxy(Sweepstore* sweepstoreIn, const std::string& keyIn)
: sweepstore(sweepstoreIn), key(keyIn) {}
template <typename T>
operator T() {
// Get value from sweepstore
throw std::runtime_error("Not implemented: Reading values from Sweepstore by key is not implemented.");
}
template <typename T>
void operator=(const T& value) {
SweepstoreConcurrency::spawnTicket(sweepstore->filePath,
SweepstoreTicketOperation::WRITE,
bt_hash(key),
sizeof(T),
[this, key = this->key, &value]() {
}
);
};
};
Proxy operator[](const std::string& key) {
return Proxy(this, key);
}
};

49
cpp/src/Public/sweepstore/utils/file_handle.h Normal file
View File

@@ -0,0 +1,49 @@
#pragma once
#include <fstream>
#include <string>
#include <memory>
class SweepstoreFileHandle {
private:
std::string path;
std::unique_ptr<std::fstream> stream;
public:
SweepstoreFileHandle(const std::string& p, std::ios::openmode mode = std::ios::in | std::ios::out | std::ios::binary)
: path(p), stream(std::make_unique<std::fstream>(p, mode)) {
if (!stream->is_open()) {
throw std::runtime_error("Failed to open file: " + path);
}
}
const std::string& getPath() const { return path; }
std::fstream& getStream() { return *stream; }
const std::fstream& getStream() const { return *stream; }
// Smart pointer-like interface
std::fstream* operator->() { return stream.get(); }
const std::fstream* operator->() const { return stream.get(); }
std::fstream& operator*() { return *stream; }
const std::fstream& operator*() const { return *stream; }
bool isOpen() const { return stream && stream->is_open(); }
void close() {
if (stream) {
stream->close();
}
}
SweepstoreFileHandle(SweepstoreFileHandle&&) noexcept = default;
SweepstoreFileHandle& operator=(SweepstoreFileHandle&&) noexcept = default;
SweepstoreFileHandle(const SweepstoreFileHandle&) = delete;
SweepstoreFileHandle& operator=(const SweepstoreFileHandle&) = delete;
~SweepstoreFileHandle() {
close();
}
};

169
cpp/src/Public/sweepstore/utils/file_lock.h Normal file
View File

@@ -0,0 +1,169 @@
#pragma once
#include <string>
#ifdef _WIN32
#include <windows.h>
#else
#include <sys/file.h>
#include <fcntl.h>
#include <unistd.h>
#endif
class SweepstoreFileLock {
public:
enum class Mode {
Shared,
Exclusive
};
private:
#ifdef _WIN32
HANDLE handle;
OVERLAPPED overlapped;
#else
int fd;
#endif
std::string path;
Mode mode;
bool locked;
void acquire() {
#ifdef _WIN32
handle = CreateFileA(path.c_str(), GENERIC_READ | GENERIC_WRITE,
FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING,
FILE_ATTRIBUTE_NORMAL, NULL);
if (handle == INVALID_HANDLE_VALUE) {
throw std::runtime_error("Failed to open file");
}
memset(&overlapped, 0, sizeof(overlapped));
DWORD flags = (mode == Mode::Exclusive) ? LOCKFILE_EXCLUSIVE_LOCK : 0;
if (!LockFileEx(handle, flags, 0, MAXDWORD, MAXDWORD, &overlapped)) {
CloseHandle(handle);
throw std::runtime_error("Failed to lock");
}
#else
fd = open(path.c_str(), O_RDWR);
if (fd == -1) throw std::runtime_error("Failed to open file");
int op = (mode == Mode::Exclusive) ? LOCK_EX : LOCK_SH;
if (flock(fd, op) != 0) {
close(fd);
throw std::runtime_error("Failed to lock");
}
#endif
locked = true;
}
void release() {
if (locked) {
#ifdef _WIN32
UnlockFileEx(handle, 0, MAXDWORD, MAXDWORD, &overlapped);
CloseHandle(handle);
#else
flock(fd, LOCK_UN);
close(fd);
#endif
locked = false;
}
}
public:
SweepstoreFileLock(const std::string& p, Mode m)
: path(p), mode(m), locked(false) {
#ifdef _WIN32
handle = INVALID_HANDLE_VALUE;
memset(&overlapped, 0, sizeof(overlapped));
#else
fd = -1;
#endif
}
~SweepstoreFileLock() {
release();
}
void lock() {
if (!locked) {
acquire();
}
}
void unlock() {
release();
}
// Check if THIS instance holds the lock
bool holdsLock() const { return locked; }
// Check if the file is locked by ANYONE (including this instance)
bool isLocked() const {
#ifdef _WIN32
HANDLE testHandle = CreateFileA(path.c_str(), GENERIC_READ | GENERIC_WRITE,
FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING,
FILE_ATTRIBUTE_NORMAL, NULL);
if (testHandle == INVALID_HANDLE_VALUE) {
return false; // Can't even open file
}
OVERLAPPED testOverlapped;
memset(&testOverlapped, 0, sizeof(testOverlapped));
DWORD flags = (mode == Mode::Exclusive) ? LOCKFILE_EXCLUSIVE_LOCK : 0;
flags |= LOCKFILE_FAIL_IMMEDIATELY; // Non-blocking
bool isLocked = !LockFileEx(testHandle, flags, 0, MAXDWORD, MAXDWORD, &testOverlapped);
if (!isLocked) {
// We got the lock, release it
UnlockFileEx(testHandle, 0, MAXDWORD, MAXDWORD, &testOverlapped);
}
CloseHandle(testHandle);
return isLocked;
#else
int testFd = open(path.c_str(), O_RDWR);
if (testFd == -1) {
return false; // Can't open file
}
int op = (mode == Mode::Exclusive) ? LOCK_EX : LOCK_SH;
op |= LOCK_NB; // Non-blocking
bool isLocked = (flock(testFd, op) != 0);
if (!isLocked) {
// We got the lock, release it
flock(testFd, LOCK_UN);
}
close(testFd);
return isLocked;
#endif
}
SweepstoreFileLock(const SweepstoreFileLock&) = delete;
SweepstoreFileLock& operator=(const SweepstoreFileLock&) = delete;
SweepstoreFileLock(SweepstoreFileLock&&) = default;
SweepstoreFileLock& operator=(SweepstoreFileLock&&) = default;
class Scoped {
private:
SweepstoreFileLock& lock;
public:
Scoped(SweepstoreFileLock& l) : lock(l) {
lock.lock();
}
~Scoped() {
lock.unlock();
}
Scoped(const Scoped&) = delete;
Scoped& operator=(const Scoped&) = delete;
};
};

381
cpp/src/Public/sweepstore/utils/helpers.h Normal file
View File

@@ -0,0 +1,381 @@
#pragma once
#include <iostream>
#include <string>
#include <sstream>
#include <iomanip>
#include <vector>
#include <cstdint>
#include <fstream>
#include <chrono>
#include <thread>
// Toggleable debug printing via preprocessor
#ifndef SWEEPSTORE_DEBUG
#define SWEEPSTORE_DEBUG 0
#endif
#if SWEEPSTORE_DEBUG
#define debugPrint(msg) std::cout << msg << std::endl
#else
#define debugPrint(msg) ((void)0)
#endif
inline void print(const char* message) {
// Print the message to the console
std::cout << message << std::endl;
}
inline std::string trim(const std::string& str) {
size_t start = str.find_first_not_of(" \t\n\r");
if (start == std::string::npos) return ""; // all whitespace
size_t end = str.find_last_not_of(" \t\n\r");
return str.substr(start, end - start + 1);
}
inline std::string binaryDump(const std::vector<uint8_t>& data) {
std::ostringstream buffer;
for (size_t i = 0; i < data.size(); i += 16) {
// Address
buffer << "0x"
<< std::setfill('0') << std::setw(4) << std::uppercase << std::hex << i
<< " (" << std::dec << std::setw(4) << i << ") | ";
// Hex bytes
for (size_t j = 0; j < 16; j++) {
if (i + j < data.size()) {
buffer << std::setfill('0') << std::setw(2) << std::uppercase << std::hex
<< static_cast<int>(data[i + j]) << " ";
} else {
buffer << " ";
}
}
buffer << " | ";
// Integer representation
for (size_t j = 0; j < 16; j++) {
if (i + j < data.size()) {
buffer << std::dec << std::setw(3) << static_cast<int>(data[i + j]) << " ";
} else {
buffer << " ";
}
}
buffer << " | ";
// ASCII representation
for (size_t j = 0; j < 16; j++) {
if (i + j < data.size()) {
uint8_t byte = data[i + j];
if (byte >= 32 && byte <= 126) {
buffer << static_cast<char>(byte);
} else {
buffer << '.';
}
}
}
buffer << " | ";
if (i + 16 < data.size()) buffer << '\n';
}
return buffer.str();
}
inline std::vector<uint8_t> loadFile(const std::string& filename) {
std::ifstream file(filename, std::ios::binary | std::ios::ate);
if (!file) {
throw std::runtime_error("Failed to open file: " + filename);
}
// Get file size
std::streamsize size = file.tellg();
file.seekg(0, std::ios::beg);
// Pre-allocate vector and read
std::vector<uint8_t> buffer(size);
if (!file.read(reinterpret_cast<char*>(buffer.data()), size)) {
throw std::runtime_error("Failed to read file: " + filename);
}
return buffer;
}
enum class Endian {
Little,
Big
};
class RandomAccessMemory {
private:
std::vector<uint8_t> _buffer;
size_t _position;
public:
// Constructors
RandomAccessMemory() : _position(0) {}
explicit RandomAccessMemory(const std::vector<uint8_t>& initialData)
: _buffer(initialData), _position(0) {}
explicit RandomAccessMemory(const uint8_t* data, size_t size)
: _buffer(data, data + size), _position(0) {}
// Position management
size_t positionSync() const {
return _position;
}
void setPositionSync(size_t position) {
_position = position;
}
size_t length() const {
return _buffer.size();
}
// Read bytes
std::vector<uint8_t> readSync(size_t count) {
if (_position + count > _buffer.size()) {
throw std::range_error("Not enough bytes to read");
}
std::vector<uint8_t> result(_buffer.begin() + _position,
_buffer.begin() + _position + count);
_position += count;
return result;
}
// Write bytes
void writeFromSync(const std::vector<uint8_t>& bytes) {
for (size_t i = 0; i < bytes.size(); i++) {
if (_position + i >= _buffer.size()) {
_buffer.push_back(bytes[i]);
} else {
_buffer[_position + i] = bytes[i];
}
}
_position += bytes.size();
}
// Read/Write Int Dynamic
int64_t readIntSync(int size = 4, Endian endianness = Endian::Little) {
if (size < 1 || size > 8) {
throw std::invalid_argument("Size must be between 1 and 8 bytes");
}
std::vector<uint8_t> bytes = readSync(size);
// Build integer from bytes with proper endianness
int64_t result = 0;
if (endianness == Endian::Little) {
for (int i = size - 1; i >= 0; i--) {
result = (result << 8) | bytes[i];
}
} else {
for (int i = 0; i < size; i++) {
result = (result << 8) | bytes[i];
}
}
// Sign extend if MSB is set
int64_t signBit = 1LL << (size * 8 - 1);
if (result & signBit) {
result -= 1LL << (size * 8);
}
return result;
}
uint64_t readUIntSync(int size = 4, Endian endianness = Endian::Little) {
if (size < 1 || size > 8) {
throw std::invalid_argument("Size must be between 1 and 8 bytes");
}
std::vector<uint8_t> bytes = readSync(size);
// Build integer from bytes with proper endianness
uint64_t result = 0;
if (endianness == Endian::Little) {
for (int i = size - 1; i >= 0; i--) {
result = (result << 8) | bytes[i];
}
} else {
for (int i = 0; i < size; i++) {
result = (result << 8) | bytes[i];
}
}
return result;
}
void writeIntSync(int64_t value, int size = 4, Endian endianness = Endian::Little) {
if (size < 1 || size > 8) {
throw std::invalid_argument("Size must be between 1 and 8 bytes");
}
std::vector<uint8_t> bytes(size, 0);
// Extract bytes with proper endianness
if (endianness == Endian::Little) {
for (int i = 0; i < size; i++) {
bytes[i] = (value >> (i * 8)) & 0xFF;
}
} else {
for (int i = 0; i < size; i++) {
bytes[size - 1 - i] = (value >> (i * 8)) & 0xFF;
}
}
writeFromSync(bytes);
}
void writeUIntSync(uint64_t value, int size = 4, Endian endianness = Endian::Little) {
if (size < 1 || size > 8) {
throw std::invalid_argument("Size must be between 1 and 8 bytes");
}
std::vector<uint8_t> bytes(size, 0);
// Extract bytes with proper endianness
if (endianness == Endian::Little) {
for (int i = 0; i < size; i++) {
bytes[i] = (value >> (i * 8)) & 0xFF;
}
} else {
for (int i = 0; i < size; i++) {
bytes[size - 1 - i] = (value >> (i * 8)) & 0xFF;
}
}
writeFromSync(bytes);
}
// Read/Write Pointers (assuming POINTER size is 8 bytes)
SweepstorePointer readPointerSync(int pointerSize = 8) {
int64_t offset = readUIntSync(pointerSize);
return SweepstorePointer(offset);
}
void writePointerSync(const SweepstorePointer& pointer, int pointerSize = 8) {
writeUIntSync(pointer, pointerSize);
}
// Read/Write Float32
float readFloat32Sync(Endian endianness = Endian::Little) {
std::vector<uint8_t> bytes = readSync(4);
float value;
if (endianness == Endian::Little) {
std::memcpy(&value, bytes.data(), 4);
} else {
std::vector<uint8_t> reversed(bytes.rbegin(), bytes.rend());
std::memcpy(&value, reversed.data(), 4);
}
return value;
}
void writeFloat32Sync(float value, Endian endianness = Endian::Little) {
std::vector<uint8_t> bytes(4);
std::memcpy(bytes.data(), &value, 4);
if (endianness == Endian::Big) {
std::reverse(bytes.begin(), bytes.end());
}
writeFromSync(bytes);
}
// Read/Write Float64 (Double)
double readFloat64Sync(Endian endianness = Endian::Little) {
std::vector<uint8_t> bytes = readSync(8);
double value;
if (endianness == Endian::Little) {
std::memcpy(&value, bytes.data(), 8);
} else {
std::vector<uint8_t> reversed(bytes.rbegin(), bytes.rend());
std::memcpy(&value, reversed.data(), 8);
}
return value;
}
void writeFloat64Sync(double value, Endian endianness = Endian::Little) {
std::vector<uint8_t> bytes(8);
std::memcpy(bytes.data(), &value, 8);
if (endianness == Endian::Big) {
std::reverse(bytes.begin(), bytes.end());
}
writeFromSync(bytes);
}
// Conversion methods
std::vector<uint8_t> toVector() const {
return _buffer;
}
const uint8_t* data() const {
return _buffer.data();
}
uint8_t* data() {
return _buffer.data();
}
};
#ifdef _WIN32
#include <windows.h>
#endif
inline void preciseSleep(std::chrono::nanoseconds duration) {
auto start = std::chrono::high_resolution_clock::now();
#ifdef _WIN32
const auto windowsMinSleepTime = std::chrono::milliseconds(1);
if (duration < windowsMinSleepTime) {
// Pure busy-wait with high-res timer
while (std::chrono::high_resolution_clock::now() - start < duration) {
// Optionally use _mm_pause() or YieldProcessor() to be nicer to hyperthreading
}
} else {
// Hybrid: sleep most of it, busy-wait the remainder
auto sleepDuration = duration - windowsMinSleepTime;
std::this_thread::sleep_for(sleepDuration);
while (std::chrono::high_resolution_clock::now() - start < duration) {}
}
#else
std::this_thread::sleep_for(duration);
#endif
}
inline int32_t millisecondsSinceEpoch32() {
auto now = std::chrono::system_clock::now();
auto millis = std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count();
return static_cast<int32_t>((millis / 1000) & 0xFFFFFFFF);
}
inline int64_t millisecondsSinceEpoch64() {
auto now = std::chrono::system_clock::now();
auto millis = std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count();
return millis;
}
inline uint64_t bt_hash(const std::string& str) {
uint64_t hash = 0xcbf29ce484222325ULL; // FNV offset basis
for (unsigned char byte : str) {
hash ^= byte;
hash *= 0x100000001b3ULL; // FNV prime
}
return hash;
}

6
dart/lib/sweepstore.dart
View File

@@ -72,10 +72,14 @@ Future<void> main() async {
print("Stale Ticket Threshold: ${STALE_HEARTBEAT_THRESHOLD_MS}ms"); print("Stale Ticket Threshold: ${STALE_HEARTBEAT_THRESHOLD_MS}ms");
while (true) { while (true) {
int concurrencyTest = 128; int concurrencyTest = 256;
final receivePort = ReceivePort(); final receivePort = ReceivePort();
int completedJobs = 0; int completedJobs = 0;
if (iteration > 0) {
break;
}
final stopwatch = Stopwatch()..start(); final stopwatch = Stopwatch()..start();
print('\x1B[95mStarting iteration #$iteration with $concurrencyTest concurrent jobs...\x1B[0m'); print('\x1B[95mStarting iteration #$iteration with $concurrencyTest concurrent jobs...\x1B[0m');

BIN
example.bin
View File

Binary file not shown.