Add concurrency handling implementation with ticket management and file locking

cpp/src/Private/sweepstore/concurrency.cpp

@@ -0,0 +1,248 @@
+
+
+#include <functional>
+#include <iosfwd>
+
+#include "sweepstore/concurrency.h"
+
+#include <iostream>
+#include <random>
+
+#include "sweepstore/header.h"
+#include "sweepstore/utils/helpers.h"
+#include "sweepstore/utils/file_handle.h"
+
+
+uint64_t getRandomOffset(uint64_t maxValue) {
+    static std::random_device rd;
+    static std::mt19937_64 gen(rd());
+    std::uniform_int_distribution<uint64_t> dist(0, maxValue);
+    return dist(gen);
+}
+
+int randomId() {
+    // mix timestamp with random for better uniqueness
+    // keep it positive to avoid signed int issues when storing
+    auto now = std::chrono::system_clock::now();
+    auto millis = std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count();
+    int32_t time = static_cast<int32_t>(millis & 0xFFFFFFFF); // Get lower 32 bits
+    int32_t random = static_cast<int32_t>(getRandomOffset(0x7FFFFFFF)); // 0 to 0x7FFFFFFF
+    return (time ^ random) & 0x7FFFFFFF;
+}
+
+void SweepstoreConcurrency::spawnTicket(std::string filePath,
+    const SweepstoreTicketOperation& operation,
+    const uint32_t keyHash,
+    const uint32_t targetSize,
+    const std::function<void()> onApproved,
+    std::string debugLabel
+) {
+
+    SweepstoreFileHandle file(filePath, std::ios::binary | std::ios::in | std::ios::out);
+
+    /*
+    Useful Functions
+   */
+
+    /// Logging function
+    auto log = [&](const std::string &message) {
+        std::string prefix = !debugLabel.empty() ? "\033[38;5;208m[Ticket Spawner - " + debugLabel + "]:\033[0m " : "\033[38;5;208m[Ticket Spawner]:\033[0m ";
+        debugPrint(prefix + message);
+    };
+
+    // Sleep with variance (additive only)
+    auto varySleep = [&](std::chrono::nanoseconds minSleepDuration, std::chrono::nanoseconds variance) {
+        if (variance.count() <= 0) {
+            preciseSleep(minSleepDuration);
+        } else {
+            // Generate random duration within variance
+            uint64_t randomOffset = getRandomOffset(variance.count());
+            preciseSleep(minSleepDuration + std::chrono::nanoseconds(randomOffset));
+        }
+    };
+
+    // Exponential sleep
+    std::unordered_map<std::string, int> expSleepTracker = {};
+    auto expSleep = [&expSleepTracker](const std::string& label) {
+        int count = expSleepTracker[label]; // defaults to 0 if not found
+        int sleepTime = (1 << count); // Exponential backoff
+        sleepTime = std::max(1, std::min(sleepTime, 1000)); // Clamp between 1ms and 1000ms
+        preciseSleep(std::chrono::microseconds(sleepTime * 5000));
+        expSleepTracker[label] = count + 1;
+    };
+
+    // Get the header(s)
+    SweepstoreHeader header(file);
+    SweepstoreConcurrencyHeader concurrencyHeader(file);
+
+    /*
+        Ticket Acquisition
+    */
+    auto acquireTicket = [&](uint32_t newIdentifier) -> SweepstoreWorkerTicket {
+
+        // Reduce the chance of race condition
+        varySleep(std::chrono::microseconds(500), std::chrono::microseconds(200));
+
+        uint32_t ticketIndex = -1u;
+
+        while (true) {
+
+            uint32_t concurrentWorkers = concurrencyHeader.readNumberOfWorkers();
+
+            for (uint32_t i = 0; i < concurrentWorkers; i++) {
+
+                SweepstoreWorkerTicket ticket = SweepstoreWorkerTicket(i, file);
+
+                if (!ticket.writable()) {
+                    continue;
+                }
+
+                SweepstoreWorkerTicketSnapshot snapshot = ticket.snapshot();
+
+                int identifier = snapshot.identifier;
+
+                bool identifier_unassigned = identifier == 0;
+                bool stale_heartbeat = millisecondsSinceEpoch32() - snapshot.workerHeartbeat > STALE_HEARTBEAT_THRESHOLD_MS;
+                bool is_free = snapshot.state == SweepstoreTicketState::FREE;
+
+                if (identifier_unassigned && stale_heartbeat && is_free) {
+                    snapshot.identifier = newIdentifier;
+                    snapshot.workerHeartbeat = millisecondsSinceEpoch32();
+                    snapshot.state = SweepstoreTicketState::WAITING;
+                    ticket.write(snapshot);
+                    ticketIndex = i;
+                    break;
+                }
+            }
+            preciseSleep(std::chrono::milliseconds(2));
+
+            // Ensure we still own the ticket - if not, reset and try again
+            if (ticketIndex != -1u) {
+                SweepstoreWorkerTicketSnapshot verifySnapshot = concurrencyHeader[ticketIndex].snapshot();
+
+                if (verifySnapshot.identifier != newIdentifier) {
+                    ticketIndex = -1; // Lost the ticket, try again
+                } else {
+                    log("Acquired ticket " + std::to_string(ticketIndex) + " with identifier " + std::to_string(newIdentifier) + ".");
+                    return concurrencyHeader[ticketIndex];
+                }
+            }
+
+            expSleep("acquireTicket");
+        }
+
+    };
+
+
+    uint32_t myIdentifier = randomId();
+
+    SweepstoreWorkerTicket myTicket = acquireTicket(myIdentifier);
+    SweepstoreWorkerTicketSnapshot mySnapshot = myTicket.snapshot();
+    mySnapshot.workerHeartbeat = millisecondsSinceEpoch32();
+    mySnapshot.state = SweepstoreTicketState::WAITING;
+    mySnapshot.operation = operation;
+    mySnapshot.keyHash = keyHash;
+    mySnapshot.targetSize = targetSize;
+    myTicket.write(mySnapshot);
+
+    // Wait for approval
+    while (true) {
+
+        SweepstoreWorkerTicketSnapshot snapshot = myTicket.snapshot();
+
+        // Update heartbeat
+        uint32_t currentTime = millisecondsSinceEpoch32();
+        if (currentTime - snapshot.workerHeartbeat > 700) {
+            snapshot.workerHeartbeat = currentTime;
+            myTicket.write(snapshot);
+        }
+
+        // Check if we still own the ticket
+        if (snapshot.identifier != myIdentifier) {
+
+            preciseSleep(std::chrono::milliseconds(10));
+
+            // Re-verify we lost the ticket
+            SweepstoreWorkerTicketSnapshot recheckSnapshot = myTicket.snapshot();
+            if (recheckSnapshot.identifier != myIdentifier) {
+                log("Lost ownership of ticket " + std::to_string(myTicket.getTicketIndex()) + ", was expecting identifier " + std::to_string(myIdentifier) + " but found " + std::to_string(recheckSnapshot.identifier) + ".");
+
+                // ReSharper disable once CppDFAInfiniteRecursion
+                return spawnTicket(
+                    filePath,
+                    operation,
+                    keyHash,
+                    targetSize,
+                    onApproved,
+                    debugLabel
+                );
+            }
+
+            // False alarm, continue waiting
+            log("False alarm, still own ticket " + std::to_string(myTicket.getTicketIndex()) + ".");
+            snapshot = recheckSnapshot;
+        }
+
+        if (snapshot.state == SweepstoreTicketState::APPROVED) {
+            snapshot.state = SweepstoreTicketState::EXECUTING;
+            myTicket.write(snapshot);
+
+            onApproved();
+
+            snapshot.state = SweepstoreTicketState::COMPLETED;
+            myTicket.write(snapshot);
+
+            break;
+        }
+
+        varySleep(std::chrono::microseconds(500), std::chrono::microseconds(200));
+    }
+
+    // std::cout << "\033[38;5;82m[Ticket Spawner - " << debugLabel << "]:\033[0m Completed ticket " << myTicket.getTicketIndex() << "." << std::endl;
+}
+
+void SweepstoreConcurrency::initialiseMaster(std::string filePath) {
+
+    auto log = [&](const std::string &message) {
+        debugPrint("\033[38;5;33m[Concurrency Master]:\033[0m " + message);
+    };
+
+    SweepstoreFileHandle file(filePath, std::ios::binary | std::ios::in | std::ios::out);
+
+    SweepstoreHeader header(file);
+    SweepstoreConcurrencyHeader concurrencyHeader(file);
+
+    while (true) {
+
+        int concurrentWorkers = concurrencyHeader.readNumberOfWorkers();
+
+        for (uint32_t i = 0; i < concurrentWorkers; i++) {
+
+            SweepstoreWorkerTicket ticket(i, file);
+            SweepstoreWorkerTicketSnapshot snapshot = ticket.snapshot();
+
+            if (snapshot.state == WAITING) {
+                log("Found waiting ticket " + std::to_string(i) + "(Key Hash: " + std::to_string(snapshot.keyHash) + ")...");
+
+                // Approve the ticket
+                snapshot.state = APPROVED;
+                ticket.write(snapshot);
+                log("Approved ticket " + std::to_string(i) + ".");
+            } else if (snapshot.state == SweepstoreTicketState::COMPLETED) {
+                log("Ticket " + std::to_string(i) + " has completed. Resetting...");
+
+                // Reset the ticket
+                SweepstoreWorkerTicketSnapshot cleanSnapshot = SweepstoreWorkerTicketSnapshot();
+                ticket.write(cleanSnapshot);
+                log("Reset ticket " + std::to_string(i) + ".");
+            }
+
+            // Handle stale tickets
+            uint32_t currentTime = millisecondsSinceEpoch32();
+        }
+
+        preciseSleep(std::chrono::milliseconds(1));
+    }
+
+}
+

cpp/src/Private/sweepstore/header.cpp

@@ -0,0 +1,220 @@
+
+#include "sweepstore/header.h"
+
+#include "sweepstore/utils/file_lock.h"
+#include "sweepstore/utils/helpers.h"
+
+std::string SweepstoreHeader::readMagicNumber() {
+    file->seekg(0, std::ios::beg);
+    char buffer[4];
+    file->read(buffer, 4);
+    return std::string(buffer, 4);
+}
+
+void SweepstoreHeader::writeMagicNumber(const std::string& magicNumber) {
+    if (magicNumber.size() != 4) {
+        throw std::invalid_argument("Magic number must be exactly 4 characters long.");
+    }
+    file->seekp(0, std::ios::beg);
+    file->write(magicNumber.c_str(), 4);
+}
+
+std::string SweepstoreHeader::readVersion() {
+    file->seekg(4, std::ios::beg);
+    char buffer[12];
+    file->read(buffer, 12);
+
+    // Trim leading and trailing spaces
+    std::string version(buffer, 12);
+    version = trim(version);
+
+    return version;
+}
+
+void SweepstoreHeader::writeVersion(const std::string& version) {
+    if (version.size() > 11) {
+        throw std::invalid_argument("Version string must be at most 11 characters long.");
+    }
+    // Pad 1 space to the beginning
+    // And pad to end to make it 12 bytes
+    std::string paddedVersion = " " + version;
+    paddedVersion.resize(12, ' ');
+
+    file->seekp(4, std::ios::beg);
+    file->write(paddedVersion.c_str(), 12);
+}
+
+SweepstorePointer SweepstoreHeader::readAddressTablePointer() {
+    file->seekg(16, std::ios::beg);
+    int64_t address;
+    file->read(reinterpret_cast<char*>(&address), sizeof(address));
+    return address; // Implicit conversion to SweepstorePointer
+}
+
+void SweepstoreHeader::writeAddressTablePointer(const SweepstorePointer& ptr) {
+    file->seekp(16, std::ios::beg);
+    int64_t address = ptr;
+    file->write(reinterpret_cast<const char*>(&address), sizeof(address));
+}
+
+uint32_t SweepstoreHeader::readFreeListCount() {
+    file->seekg(24, std::ios::beg);
+    uint32_t count;
+    file->read(reinterpret_cast<char*>(&count), sizeof(count));
+    return count;
+}
+
+void SweepstoreHeader::writeFreeListCount(uint32_t count) {
+    file->seekp(24, std::ios::beg);
+    file->write(reinterpret_cast<const char*>(&count), sizeof(count));
+}
+
+bool SweepstoreHeader::readIsFreeListLifted() {
+    file->seekg(28, std::ios::beg);
+    char flag;
+    file->read(&flag, sizeof(flag));
+    return flag != 0;
+}
+
+void SweepstoreHeader::writeIsFreeListLifted(bool isLifted) {
+    file->seekp(28, std::ios::beg);
+    char flag = isLifted ? 1 : 0;
+    file->write(&flag, sizeof(flag));
+}
+
+void SweepstoreHeader::initialise() {
+    writeMagicNumber("SWPT");
+    writeVersion("undefined");
+    writeAddressTablePointer(SweepstorePointer::NULL_PTR);
+    writeFreeListCount(0);
+    writeIsFreeListLifted(false);
+    file->flush();
+}
+
+uint64_t SweepstoreConcurrencyHeader::readMasterIdentifier() {
+    file->seekg(29, std::ios::beg);
+    uint64_t identifier;
+    file->read(reinterpret_cast<char*>(&identifier), sizeof(identifier));
+    return identifier;
+}
+
+void SweepstoreConcurrencyHeader::writeMasterIdentifier(uint64_t identifier) {
+    file->seekp(29, std::ios::beg);
+    file->write(reinterpret_cast<const char*>(&identifier), sizeof(identifier));
+}
+
+uint32_t SweepstoreConcurrencyHeader::readMasterHeartbeat() {
+    file->seekg(37, std::ios::beg);
+    uint32_t heartbeat;
+    file->read(reinterpret_cast<char*>(&heartbeat), sizeof(heartbeat));
+    return heartbeat;
+}
+
+void SweepstoreConcurrencyHeader::writeMasterHeartbeat(uint32_t heartbeat) {
+    file->seekp(37, std::ios::beg);
+    file->write(reinterpret_cast<const char*>(&heartbeat), sizeof(heartbeat));
+}
+
+uint32_t SweepstoreConcurrencyHeader::readNumberOfWorkers() {
+    file->seekg(41, std::ios::beg);
+    uint32_t numWorkers;
+    file->read(reinterpret_cast<char*>(&numWorkers), sizeof(numWorkers));
+    return numWorkers;
+}
+
+void SweepstoreConcurrencyHeader::writeNumberOfWorkers(uint32_t numWorkers) {
+    file->seekp(41, std::ios::beg);
+    file->write(reinterpret_cast<const char*>(&numWorkers), sizeof(numWorkers));
+}
+
+bool SweepstoreConcurrencyHeader::readIsReadAllowed() {
+    file->seekg(45, std::ios::beg);
+    char flag;
+    file->read(&flag, sizeof(flag));
+    return flag != 0;
+}
+
+void SweepstoreConcurrencyHeader::writeIsReadAllowed(bool isAllowed) {
+    file->seekp(45, std::ios::beg);
+    char flag = isAllowed ? 1 : 0;
+    file->write(&flag, sizeof(flag));
+}
+
+void SweepstoreConcurrencyHeader::initialise(int concurrentWorkers) {
+    writeMasterIdentifier(0);
+    writeMasterHeartbeat(0);
+    writeNumberOfWorkers(concurrentWorkers);
+    writeIsReadAllowed(true);
+    for (uint32_t i = 0; i < readNumberOfWorkers(); i++) {
+        SweepstoreWorkerTicketSnapshot ticket = SweepstoreWorkerTicketSnapshot();
+        ticket.identifier = 0;
+        ticket.workerHeartbeat = 0;
+        ticket.state = SweepstoreTicketState::FREE;
+        ticket.operation = SweepstoreTicketOperation::NONE;
+        ticket.keyHash = 0;
+        ticket.targetAddress = SweepstorePointer::NULL_PTR;
+        ticket.targetSize = 0;
+
+        SweepstoreWorkerTicket ticketWriter = SweepstoreWorkerTicket(i, file);
+        ticketWriter.write(ticket);
+    }
+    file->flush();
+}
+
+void SweepstoreWorkerTicket::write(SweepstoreWorkerTicketSnapshot &snapshot) {
+    RandomAccessMemory buffer;
+
+    SweepstoreFileLock lock(file.getPath(), SweepstoreFileLock::Mode::Exclusive);
+    SweepstoreFileLock::Scoped scopedLock(lock);
+
+    buffer.setPositionSync(0);
+    buffer.writeIntSync(snapshot.identifier, 4);
+    buffer.writeIntSync(snapshot.workerHeartbeat, 4);
+    buffer.writeIntSync(static_cast<uint8_t>(snapshot.state), 1);
+    buffer.writeIntSync(static_cast<uint8_t>(snapshot.operation), 1);
+    buffer.writeUIntSync(snapshot.keyHash, 8);
+    buffer.writePointerSync(snapshot.targetAddress, 8);
+    buffer.writeUIntSync(snapshot.targetSize, 4);
+
+    // Pad the rest with zeros if necessary
+    while (buffer.length() < TICKET_SIZE) {
+        buffer.writeIntSync(0, 1);
+    }
+
+    // Prepare data
+    buffer.setPositionSync(0);
+    std::vector<uint8_t> data = buffer.readSync(buffer.length());
+    char* dataPtr = reinterpret_cast<char*>(data.data());
+
+    // Write to file
+    file->seekp(getOffset());
+    file->write(dataPtr, data.size());
+    file->flush();
+}
+
+bool SweepstoreWorkerTicket::writable() {
+    SweepstoreFileLock lock(file.getPath(), SweepstoreFileLock::Mode::Exclusive);
+    return lock.isLocked() == false;
+}
+
+SweepstoreWorkerTicketSnapshot SweepstoreWorkerTicket::snapshot() {
+    SweepstoreFileLock lock(file.getPath(), SweepstoreFileLock::Mode::Shared);
+    lock.lock();
+    file->seekg(getOffset());
+    std::unique_ptr<char[]> buffer(new char[TICKET_SIZE]);
+    file->read(buffer.get(), TICKET_SIZE);
+    lock.unlock();
+    RandomAccessMemory ram(reinterpret_cast<uint8_t*>(buffer.get()), TICKET_SIZE);
+
+    SweepstoreWorkerTicketSnapshot snapshot;
+    ram.setPositionSync(0);
+    snapshot.identifier = ram.readUIntSync(4);
+    snapshot.workerHeartbeat = ram.readUIntSync(4);
+    snapshot.state = static_cast<SweepstoreTicketState>(ram.readUIntSync(1));
+    snapshot.operation = static_cast<SweepstoreTicketOperation>(ram.readUIntSync(1));
+    snapshot.keyHash = ram.readUIntSync(8);
+    snapshot.targetAddress = ram.readPointerSync(8);
+    snapshot.targetSize = ram.readUIntSync(4);
+
+    return snapshot;
+}

cpp/src/Private/sweepstore/structures.cpp

@@ -0,0 +1,13 @@
+
+
+
+#include "sweepstore/structures.h"
+const SweepstorePointer SweepstorePointer::NULL_PTR = SweepstorePointer(UINT64_MAX);
+
+bool SweepstorePointer::operator==(const SweepstorePointer &p) {
+    if (this->address == p.address) {
+        return true;
+    } else {
+        return false;
+    }
+}

cpp/src/Private/sweepstore/sweepstore.cpp

@@ -0,0 +1,21 @@
+//
+// Created by Benjamin Watt on 24/11/2025.
+//
+
+#include "sweepstore/sweepstore.h"
+
+#include <string>
+#include <filesystem>
+#include <iostream>
+#include <thread>
+
+#include "sweepstore/utils/helpers.h"
+#include "sweepstore/utils/file_handle.h"
+
+void Sweepstore::initialise(int concurrentWorkers) {
+    header->initialise();
+    header->writeVersion("1.1.0.2");
+    concurrencyHeader->initialise(concurrentWorkers);
+
+    debugPrint("Version: " + header->readVersion());
+}

cpp/src/Public/sweepstore/concurrency.h

@@ -0,0 +1,30 @@
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <iosfwd>
+#include <thread>
+
+#define STALE_HEARTBEAT_THRESHOLD_MS 5000
+
+enum SweepstoreTicketOperation : int;
+
+namespace SweepstoreConcurrency {
+
+    void spawnTicket(std::string filePath,
+        const SweepstoreTicketOperation& operation,
+        const uint32_t keyHash,
+        const uint32_t targetSize,
+        const std::function<void()> onApproved,
+        std::string debugLabel = ""
+    );
+
+    void initialiseMaster(std::string filePath);
+
+    inline void initialiseMasterAsync(std::string filePath) {
+        std::thread([&filePath]() {
+            initialiseMaster(filePath);
+        }).detach();
+    }
+
+}

cpp/src/Public/sweepstore/header.h

@@ -0,0 +1,143 @@
+#pragma once
+
+#include <iosfwd>
+
+#include "structures.h"
+#include "utils/file_handle.h"
+
+constexpr int roundToNearest16(int number) {
+    return (number + 15) & ~15;
+}
+
+class SweepstoreHeader {
+
+private:
+    SweepstoreFileHandle& file;
+
+public:
+    explicit SweepstoreHeader(SweepstoreFileHandle &fileStream) : file(fileStream) {}
+
+    // Offset 0 - 4 bytes
+    std::string readMagicNumber();
+    void writeMagicNumber(const std::string& magicNumber);
+
+    // Offset 4 - 12 bytes
+    std::string readVersion();
+    void writeVersion(const std::string& version);
+
+    // Offset 16 - 8 bytes
+    SweepstorePointer readAddressTablePointer();
+    void writeAddressTablePointer(const SweepstorePointer& ptr);
+
+    // Offset 24 - 4 bytes
+    uint32_t readFreeListCount();
+    void writeFreeListCount(uint32_t count);
+
+    // Offset 28 - 1 byte
+    bool readIsFreeListLifted();
+    void writeIsFreeListLifted(bool isLifted);
+
+    /**
+     * Initialises the header with default values.
+     */
+    void initialise();
+
+};
+
+constexpr int SWEEPSTORE_COMBINED_STATIC_HEADER_SIZE = roundToNearest16(46);
+
+struct SweepstoreWorkerTicketSnapshot {
+
+    SweepstoreWorkerTicketSnapshot() :
+        identifier(0),
+        workerHeartbeat(0),
+        state(SweepstoreTicketState::FREE),
+        operation(SweepstoreTicketOperation::NONE),
+        keyHash(0),
+        targetAddress(SweepstorePointer::NULL_PTR),
+        targetSize(0) {}
+
+    // Offset 0 - 4 bytes
+    uint32_t identifier;
+
+    // Offset 4 - 4 bytes
+    uint32_t workerHeartbeat;
+
+    // Offset 8 - 1 byte
+    SweepstoreTicketState state;
+
+    // Offset 9 - 1 byte
+    SweepstoreTicketOperation operation;
+
+    // Offset 10 - 8 bytes
+    uint64_t keyHash;
+
+    // Offset 18 - 8 bytes
+    SweepstorePointer targetAddress;
+
+    // Offset 26 - 4 bytes
+    uint32_t targetSize;
+};
+
+class SweepstoreWorkerTicket {
+
+    SweepstoreFileHandle& file;
+    uint32_t ticketIndex;
+
+    uint64_t getOffset() const {
+        return SWEEPSTORE_COMBINED_STATIC_HEADER_SIZE + (ticketIndex * TICKET_SIZE);
+    }
+
+public:
+    static constexpr int TICKET_SIZE = roundToNearest16(29);
+
+    SweepstoreWorkerTicket(const uint32_t index, SweepstoreFileHandle& fileStream) :
+        file(fileStream),
+        ticketIndex(index) {}
+
+    int getTicketIndex() const {
+        return ticketIndex;
+    }
+
+    void write(SweepstoreWorkerTicketSnapshot &snapshot);
+
+    bool writable();
+
+    SweepstoreWorkerTicketSnapshot snapshot();
+
+};
+
+class SweepstoreConcurrencyHeader {
+
+private:
+    SweepstoreFileHandle& file;
+
+public:
+    explicit SweepstoreConcurrencyHeader(SweepstoreFileHandle &fileStream) : file(fileStream) {}
+
+    // Offset 29 - 8 bytes
+    uint64_t readMasterIdentifier();
+    void writeMasterIdentifier(uint64_t identifier);
+
+    // Offset 37 - 4 bytes
+    uint32_t readMasterHeartbeat();
+    void writeMasterHeartbeat(uint32_t heartbeat);
+
+    // Offset 41 - 4 bytes
+    uint32_t readNumberOfWorkers();
+    void writeNumberOfWorkers(uint32_t numWorkers);
+
+    // Offset 45 - 1 byte
+    bool readIsReadAllowed();
+    void writeIsReadAllowed(bool isAllowed);
+
+    /**
+     * Initialises the concurrency header with default values.
+     */
+    void initialise(int concurrentWorkers);
+
+    SweepstoreWorkerTicket operator[](const uint32_t index) const {
+        return SweepstoreWorkerTicket(index, file);
+    }
+
+};

cpp/src/Public/sweepstore/structures.h

@@ -0,0 +1,39 @@
+#pragma once
+#include <cstdint>
+
+class SweepstorePointer {
+
+private:
+    uint64_t address;
+
+public:
+    static const SweepstorePointer NULL_PTR;
+
+    SweepstorePointer(int64_t addr) : address(addr) {}
+
+    bool isNull() {
+        return this->address == UINT64_MAX;
+    }
+
+    bool operator==(const SweepstorePointer &p);
+
+    // Implicit conversion to uint64_t
+    operator uint64_t() const {
+        return address;
+    }
+};
+
+enum SweepstoreTicketState {
+    FREE,
+    WAITING,
+    APPROVED,
+    EXECUTING,
+    COMPLETED,
+};
+
+enum SweepstoreTicketOperation : int {
+    NONE,
+    READ,
+    MODIFY,
+    WRITE
+};

cpp/src/Public/sweepstore/sweepstore.h

@@ -0,0 +1,67 @@
+#pragma once
+
+#include <iosfwd>
+
+#include "concurrency.h"
+#include "header.h"
+#include "utils/helpers.h"
+#include "utils/file_handle.h"
+
+class Sweepstore {
+
+private:
+    std::string filePath;
+    SweepstoreFileHandle file;
+    SweepstoreHeader* header;
+    SweepstoreConcurrencyHeader* concurrencyHeader;
+
+public:
+
+    Sweepstore(const std::string& filePath) : filePath(filePath), file(filePath, std::ios::binary | std::ios::in | std::ios::out | std::ios::trunc) {
+        header = new SweepstoreHeader(file);
+        concurrencyHeader = new SweepstoreConcurrencyHeader(file);
+    }
+
+    ~Sweepstore() {
+        delete header;
+        delete concurrencyHeader;
+        file.close();
+    }
+
+    void initialise(int concurrentWorkers = 4);
+
+    SweepstoreConcurrencyHeader* getConcurrencyHeader() {
+        return concurrencyHeader;
+    }
+
+    class Proxy {
+        Sweepstore* sweepstore;
+        std::string key;
+    public:
+        Proxy(Sweepstore* sweepstoreIn, const std::string& keyIn)
+            : sweepstore(sweepstoreIn), key(keyIn) {}
+
+        template <typename T>
+        operator T() {
+            // Get value from sweepstore
+            throw std::runtime_error("Not implemented: Reading values from Sweepstore by key is not implemented.");
+        }
+
+        template <typename T>
+        void operator=(const T& value) {
+            SweepstoreConcurrency::spawnTicket(sweepstore->filePath,
+                SweepstoreTicketOperation::WRITE,
+                bt_hash(key),
+                sizeof(T),
+                [this, key = this->key, &value]() {
+
+                }
+            );
+        };
+    };
+
+    Proxy operator[](const std::string& key) {
+        return Proxy(this, key);
+    }
+
+};

cpp/src/Public/sweepstore/utils/file_handle.h

@@ -0,0 +1,49 @@
+#pragma once
+
+#include <fstream>
+#include <string>
+#include <memory>
+
+class SweepstoreFileHandle {
+private:
+    std::string path;
+    std::unique_ptr<std::fstream> stream;
+
+public:
+    SweepstoreFileHandle(const std::string& p, std::ios::openmode mode = std::ios::in | std::ios::out | std::ios::binary)
+        : path(p), stream(std::make_unique<std::fstream>(p, mode)) {
+        if (!stream->is_open()) {
+            throw std::runtime_error("Failed to open file: " + path);
+        }
+    }
+
+    const std::string& getPath() const { return path; }
+
+    std::fstream& getStream() { return *stream; }
+    const std::fstream& getStream() const { return *stream; }
+
+    // Smart pointer-like interface
+    std::fstream* operator->() { return stream.get(); }
+    const std::fstream* operator->() const { return stream.get(); }
+
+    std::fstream& operator*() { return *stream; }
+    const std::fstream& operator*() const { return *stream; }
+
+    bool isOpen() const { return stream && stream->is_open(); }
+
+    void close() {
+        if (stream) {
+            stream->close();
+        }
+    }
+
+    SweepstoreFileHandle(SweepstoreFileHandle&&) noexcept = default;
+    SweepstoreFileHandle& operator=(SweepstoreFileHandle&&) noexcept = default;
+
+    SweepstoreFileHandle(const SweepstoreFileHandle&) = delete;
+    SweepstoreFileHandle& operator=(const SweepstoreFileHandle&) = delete;
+
+    ~SweepstoreFileHandle() {
+        close();
+    }
+};

cpp/src/Public/sweepstore/utils/file_lock.h

@@ -0,0 +1,169 @@
+#pragma once
+
+#include <string>
+
+#ifdef _WIN32
+    #include <windows.h>
+#else
+    #include <sys/file.h>
+    #include <fcntl.h>
+    #include <unistd.h>
+#endif
+
+class SweepstoreFileLock {
+public:
+    enum class Mode {
+        Shared,
+        Exclusive
+    };
+
+private:
+#ifdef _WIN32
+    HANDLE handle;
+    OVERLAPPED overlapped;
+#else
+    int fd;
+#endif
+    std::string path;
+    Mode mode;
+    bool locked;
+
+    void acquire() {
+#ifdef _WIN32
+        handle = CreateFileA(path.c_str(), GENERIC_READ | GENERIC_WRITE,
+            FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING,
+            FILE_ATTRIBUTE_NORMAL, NULL);
+
+        if (handle == INVALID_HANDLE_VALUE) {
+            throw std::runtime_error("Failed to open file");
+        }
+
+        memset(&overlapped, 0, sizeof(overlapped));
+        DWORD flags = (mode == Mode::Exclusive) ? LOCKFILE_EXCLUSIVE_LOCK : 0;
+
+        if (!LockFileEx(handle, flags, 0, MAXDWORD, MAXDWORD, &overlapped)) {
+            CloseHandle(handle);
+            throw std::runtime_error("Failed to lock");
+        }
+#else
+        fd = open(path.c_str(), O_RDWR);
+        if (fd == -1) throw std::runtime_error("Failed to open file");
+
+        int op = (mode == Mode::Exclusive) ? LOCK_EX : LOCK_SH;
+        if (flock(fd, op) != 0) {
+            close(fd);
+            throw std::runtime_error("Failed to lock");
+        }
+#endif
+        locked = true;
+    }
+
+    void release() {
+        if (locked) {
+#ifdef _WIN32
+            UnlockFileEx(handle, 0, MAXDWORD, MAXDWORD, &overlapped);
+            CloseHandle(handle);
+#else
+            flock(fd, LOCK_UN);
+            close(fd);
+#endif
+            locked = false;
+        }
+    }
+
+public:
+    SweepstoreFileLock(const std::string& p, Mode m)
+        : path(p), mode(m), locked(false) {
+#ifdef _WIN32
+        handle = INVALID_HANDLE_VALUE;
+        memset(&overlapped, 0, sizeof(overlapped));
+#else
+        fd = -1;
+#endif
+    }
+
+    ~SweepstoreFileLock() {
+        release();
+    }
+
+    void lock() {
+        if (!locked) {
+            acquire();
+        }
+    }
+
+    void unlock() {
+        release();
+    }
+
+    // Check if THIS instance holds the lock
+    bool holdsLock() const { return locked; }
+
+    // Check if the file is locked by ANYONE (including this instance)
+    bool isLocked() const {
+#ifdef _WIN32
+        HANDLE testHandle = CreateFileA(path.c_str(), GENERIC_READ | GENERIC_WRITE,
+            FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING,
+            FILE_ATTRIBUTE_NORMAL, NULL);
+
+        if (testHandle == INVALID_HANDLE_VALUE) {
+            return false; // Can't even open file
+        }
+
+        OVERLAPPED testOverlapped;
+        memset(&testOverlapped, 0, sizeof(testOverlapped));
+        DWORD flags = (mode == Mode::Exclusive) ? LOCKFILE_EXCLUSIVE_LOCK : 0;
+        flags |= LOCKFILE_FAIL_IMMEDIATELY; // Non-blocking
+
+        bool isLocked = !LockFileEx(testHandle, flags, 0, MAXDWORD, MAXDWORD, &testOverlapped);
+
+        if (!isLocked) {
+            // We got the lock, release it
+            UnlockFileEx(testHandle, 0, MAXDWORD, MAXDWORD, &testOverlapped);
+        }
+
+        CloseHandle(testHandle);
+        return isLocked;
+#else
+        int testFd = open(path.c_str(), O_RDWR);
+        if (testFd == -1) {
+            return false; // Can't open file
+        }
+
+        int op = (mode == Mode::Exclusive) ? LOCK_EX : LOCK_SH;
+        op |= LOCK_NB; // Non-blocking
+
+        bool isLocked = (flock(testFd, op) != 0);
+
+        if (!isLocked) {
+            // We got the lock, release it
+            flock(testFd, LOCK_UN);
+        }
+
+        close(testFd);
+        return isLocked;
+#endif
+    }
+
+    SweepstoreFileLock(const SweepstoreFileLock&) = delete;
+    SweepstoreFileLock& operator=(const SweepstoreFileLock&) = delete;
+    SweepstoreFileLock(SweepstoreFileLock&&) = default;
+    SweepstoreFileLock& operator=(SweepstoreFileLock&&) = default;
+
+    class Scoped {
+    private:
+        SweepstoreFileLock& lock;
+
+    public:
+        Scoped(SweepstoreFileLock& l) : lock(l) {
+            lock.lock();
+        }
+
+        ~Scoped() {
+            lock.unlock();
+        }
+
+        Scoped(const Scoped&) = delete;
+        Scoped& operator=(const Scoped&) = delete;
+    };
+};

cpp/src/Public/sweepstore/utils/helpers.h

@@ -0,0 +1,381 @@
+#pragma once
+
+#include <iostream>
+#include <string>
+#include <sstream>
+#include <iomanip>
+#include <vector>
+#include <cstdint>
+#include <fstream>
+#include <chrono>
+#include <thread>
+
+// Toggleable debug printing via preprocessor
+#ifndef SWEEPSTORE_DEBUG
+#define SWEEPSTORE_DEBUG 0
+#endif
+
+#if SWEEPSTORE_DEBUG
+    #define debugPrint(msg) std::cout << msg << std::endl
+#else
+    #define debugPrint(msg) ((void)0)
+#endif
+
+
+inline void print(const char* message) {
+    // Print the message to the console
+    std::cout << message << std::endl;
+}
+
+inline std::string trim(const std::string& str) {
+    size_t start = str.find_first_not_of(" \t\n\r");
+    if (start == std::string::npos) return ""; // all whitespace
+
+    size_t end = str.find_last_not_of(" \t\n\r");
+    return str.substr(start, end - start + 1);
+}
+
+inline std::string binaryDump(const std::vector<uint8_t>& data) {
+    std::ostringstream buffer;
+
+    for (size_t i = 0; i < data.size(); i += 16) {
+        // Address
+        buffer << "0x"
+               << std::setfill('0') << std::setw(4) << std::uppercase << std::hex << i
+               << " (" << std::dec << std::setw(4) << i << ") | ";
+
+        // Hex bytes
+        for (size_t j = 0; j < 16; j++) {
+            if (i + j < data.size()) {
+                buffer << std::setfill('0') << std::setw(2) << std::uppercase << std::hex
+                       << static_cast<int>(data[i + j]) << " ";
+            } else {
+                buffer << "   ";
+            }
+        }
+
+        buffer << " | ";
+
+        // Integer representation
+        for (size_t j = 0; j < 16; j++) {
+            if (i + j < data.size()) {
+                buffer << std::dec << std::setw(3) << static_cast<int>(data[i + j]) << " ";
+            } else {
+                buffer << "    ";
+            }
+        }
+
+        buffer << " | ";
+
+        // ASCII representation
+        for (size_t j = 0; j < 16; j++) {
+            if (i + j < data.size()) {
+                uint8_t byte = data[i + j];
+                if (byte >= 32 && byte <= 126) {
+                    buffer << static_cast<char>(byte);
+                } else {
+                    buffer << '.';
+                }
+            }
+        }
+
+        buffer << " | ";
+        if (i + 16 < data.size()) buffer << '\n';
+    }
+
+    return buffer.str();
+}
+
+inline std::vector<uint8_t> loadFile(const std::string& filename) {
+    std::ifstream file(filename, std::ios::binary | std::ios::ate);
+
+    if (!file) {
+        throw std::runtime_error("Failed to open file: " + filename);
+    }
+
+    // Get file size
+    std::streamsize size = file.tellg();
+    file.seekg(0, std::ios::beg);
+
+    // Pre-allocate vector and read
+    std::vector<uint8_t> buffer(size);
+    if (!file.read(reinterpret_cast<char*>(buffer.data()), size)) {
+        throw std::runtime_error("Failed to read file: " + filename);
+    }
+
+    return buffer;
+}
+
+enum class Endian {
+    Little,
+    Big
+};
+
+class RandomAccessMemory {
+private:
+    std::vector<uint8_t> _buffer;
+    size_t _position;
+
+public:
+    // Constructors
+    RandomAccessMemory() : _position(0) {}
+
+    explicit RandomAccessMemory(const std::vector<uint8_t>& initialData)
+        : _buffer(initialData), _position(0) {}
+
+    explicit RandomAccessMemory(const uint8_t* data, size_t size)
+        : _buffer(data, data + size), _position(0) {}
+
+    // Position management
+    size_t positionSync() const {
+        return _position;
+    }
+
+    void setPositionSync(size_t position) {
+        _position = position;
+    }
+
+    size_t length() const {
+        return _buffer.size();
+    }
+
+    // Read bytes
+    std::vector<uint8_t> readSync(size_t count) {
+        if (_position + count > _buffer.size()) {
+            throw std::range_error("Not enough bytes to read");
+        }
+
+        std::vector<uint8_t> result(_buffer.begin() + _position,
+                                     _buffer.begin() + _position + count);
+        _position += count;
+        return result;
+    }
+
+    // Write bytes
+    void writeFromSync(const std::vector<uint8_t>& bytes) {
+        for (size_t i = 0; i < bytes.size(); i++) {
+            if (_position + i >= _buffer.size()) {
+                _buffer.push_back(bytes[i]);
+            } else {
+                _buffer[_position + i] = bytes[i];
+            }
+        }
+        _position += bytes.size();
+    }
+
+    // Read/Write Int Dynamic
+    int64_t readIntSync(int size = 4, Endian endianness = Endian::Little) {
+        if (size < 1 || size > 8) {
+            throw std::invalid_argument("Size must be between 1 and 8 bytes");
+        }
+
+        std::vector<uint8_t> bytes = readSync(size);
+
+        // Build integer from bytes with proper endianness
+        int64_t result = 0;
+        if (endianness == Endian::Little) {
+            for (int i = size - 1; i >= 0; i--) {
+                result = (result << 8) | bytes[i];
+            }
+        } else {
+            for (int i = 0; i < size; i++) {
+                result = (result << 8) | bytes[i];
+            }
+        }
+
+        // Sign extend if MSB is set
+        int64_t signBit = 1LL << (size * 8 - 1);
+        if (result & signBit) {
+            result -= 1LL << (size * 8);
+        }
+
+        return result;
+    }
+    uint64_t readUIntSync(int size = 4, Endian endianness = Endian::Little) {
+        if (size < 1 || size > 8) {
+            throw std::invalid_argument("Size must be between 1 and 8 bytes");
+        }
+
+        std::vector<uint8_t> bytes = readSync(size);
+
+        // Build integer from bytes with proper endianness
+        uint64_t result = 0;
+        if (endianness == Endian::Little) {
+            for (int i = size - 1; i >= 0; i--) {
+                result = (result << 8) | bytes[i];
+            }
+        } else {
+            for (int i = 0; i < size; i++) {
+                result = (result << 8) | bytes[i];
+            }
+        }
+
+        return result;
+    }
+
+    void writeIntSync(int64_t value, int size = 4, Endian endianness = Endian::Little) {
+        if (size < 1 || size > 8) {
+            throw std::invalid_argument("Size must be between 1 and 8 bytes");
+        }
+
+        std::vector<uint8_t> bytes(size, 0);
+
+        // Extract bytes with proper endianness
+        if (endianness == Endian::Little) {
+            for (int i = 0; i < size; i++) {
+                bytes[i] = (value >> (i * 8)) & 0xFF;
+            }
+        } else {
+            for (int i = 0; i < size; i++) {
+                bytes[size - 1 - i] = (value >> (i * 8)) & 0xFF;
+            }
+        }
+
+        writeFromSync(bytes);
+    }
+    void writeUIntSync(uint64_t value, int size = 4, Endian endianness = Endian::Little) {
+        if (size < 1 || size > 8) {
+            throw std::invalid_argument("Size must be between 1 and 8 bytes");
+        }
+
+        std::vector<uint8_t> bytes(size, 0);
+
+        // Extract bytes with proper endianness
+        if (endianness == Endian::Little) {
+            for (int i = 0; i < size; i++) {
+                bytes[i] = (value >> (i * 8)) & 0xFF;
+            }
+        } else {
+            for (int i = 0; i < size; i++) {
+                bytes[size - 1 - i] = (value >> (i * 8)) & 0xFF;
+            }
+        }
+
+        writeFromSync(bytes);
+    }
+
+    // Read/Write Pointers (assuming POINTER size is 8 bytes)
+    SweepstorePointer readPointerSync(int pointerSize = 8) {
+        int64_t offset = readUIntSync(pointerSize);
+        return SweepstorePointer(offset);
+    }
+
+    void writePointerSync(const SweepstorePointer& pointer, int pointerSize = 8) {
+        writeUIntSync(pointer, pointerSize);
+    }
+
+    // Read/Write Float32
+    float readFloat32Sync(Endian endianness = Endian::Little) {
+        std::vector<uint8_t> bytes = readSync(4);
+        float value;
+
+        if (endianness == Endian::Little) {
+            std::memcpy(&value, bytes.data(), 4);
+        } else {
+            std::vector<uint8_t> reversed(bytes.rbegin(), bytes.rend());
+            std::memcpy(&value, reversed.data(), 4);
+        }
+
+        return value;
+    }
+
+    void writeFloat32Sync(float value, Endian endianness = Endian::Little) {
+        std::vector<uint8_t> bytes(4);
+        std::memcpy(bytes.data(), &value, 4);
+
+        if (endianness == Endian::Big) {
+            std::reverse(bytes.begin(), bytes.end());
+        }
+
+        writeFromSync(bytes);
+    }
+
+    // Read/Write Float64 (Double)
+    double readFloat64Sync(Endian endianness = Endian::Little) {
+        std::vector<uint8_t> bytes = readSync(8);
+        double value;
+
+        if (endianness == Endian::Little) {
+            std::memcpy(&value, bytes.data(), 8);
+        } else {
+            std::vector<uint8_t> reversed(bytes.rbegin(), bytes.rend());
+            std::memcpy(&value, reversed.data(), 8);
+        }
+
+        return value;
+    }
+
+    void writeFloat64Sync(double value, Endian endianness = Endian::Little) {
+        std::vector<uint8_t> bytes(8);
+        std::memcpy(bytes.data(), &value, 8);
+
+        if (endianness == Endian::Big) {
+            std::reverse(bytes.begin(), bytes.end());
+        }
+
+        writeFromSync(bytes);
+    }
+
+    // Conversion methods
+    std::vector<uint8_t> toVector() const {
+        return _buffer;
+    }
+
+    const uint8_t* data() const {
+        return _buffer.data();
+    }
+
+    uint8_t* data() {
+        return _buffer.data();
+    }
+};
+
+
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
+inline void preciseSleep(std::chrono::nanoseconds duration) {
+    auto start = std::chrono::high_resolution_clock::now();
+
+#ifdef _WIN32
+    const auto windowsMinSleepTime = std::chrono::milliseconds(1);
+
+    if (duration < windowsMinSleepTime) {
+        // Pure busy-wait with high-res timer
+        while (std::chrono::high_resolution_clock::now() - start < duration) {
+            // Optionally use _mm_pause() or YieldProcessor() to be nicer to hyperthreading
+        }
+    } else {
+        // Hybrid: sleep most of it, busy-wait the remainder
+        auto sleepDuration = duration - windowsMinSleepTime;
+        std::this_thread::sleep_for(sleepDuration);
+        while (std::chrono::high_resolution_clock::now() - start < duration) {}
+    }
+#else
+    std::this_thread::sleep_for(duration);
+#endif
+}
+
+inline int32_t millisecondsSinceEpoch32() {
+    auto now = std::chrono::system_clock::now();
+    auto millis = std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count();
+    return static_cast<int32_t>((millis / 1000) & 0xFFFFFFFF);
+}
+
+inline int64_t millisecondsSinceEpoch64() {
+    auto now = std::chrono::system_clock::now();
+    auto millis = std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count();
+    return millis;
+}
+
+inline uint64_t bt_hash(const std::string& str) {
+    uint64_t hash = 0xcbf29ce484222325ULL; // FNV offset basis
+
+    for (unsigned char byte : str) {
+        hash ^= byte;
+        hash *= 0x100000001b3ULL; // FNV prime
+    }
+
+    return hash;
+}