Add RandomAccessMemory class for in-memory binary operations

dart/lib/concurrency.dart

@@ -0,0 +1,190 @@
+
+
+import 'dart:io';
+import 'dart:isolate';
+import 'dart:math';
+
+import 'package:sweepstore/header.dart';
+import 'package:sweepstore/helper_extensions.dart';
+import 'package:sweepstore/structures.dart';
+
+int _randomId() {
+  // mix timestamp with random for better uniquness
+  // keep it positive to avoid signed int issues when storing
+  int time = DateTime.now().millisecondsSinceEpoch32();
+  int random = Random().nextInt(0x80000000);
+  return (time ^ random) & 0x7FFFFFFF;
+}
+
+// Spawn a ticket for a worker to perform an operation
+void spawnTicket(RandomAccessFile file, {
+  required SweepstoreTicketOperation operation,
+  required int keyHash,
+  required int writeSize,
+  required void Function() onApproved,
+  String? debugLabel,
+}) {
+
+  void log(String message) {
+
+    String prefix = debugLabel != null ? "\x1B[38;5;208m[Ticket Spawner - $debugLabel]:\x1B[0m " : "\x1B[38;5;208m[Ticket Spawner]:\x1B[0m ";
+    print("$prefix$message");
+
+  }
+  void tickSleep([int microsecondVariance = 10]) {
+    sleep(Duration(microseconds: 100 + Random().nextInt(microsecondVariance)));
+  }
+  Map<String, int> expSleepTracker = {};
+  void expSleep(String label) {
+    int count = expSleepTracker[label] ?? 0;
+    int sleepTime = (1 << count); // Exponential backoff
+    sleep(Duration(milliseconds: sleepTime));
+    expSleepTracker[label] = count + 1;
+  }
+
+  // Reduce the chance of race conditions by adding a small random delay
+  tickSleep(100);
+
+  SweepstoreHeader header = SweepstoreHeader(file);
+  SweepstoreConcurrencyHeader concurrencyHeader = SweepstoreConcurrencyHeader(header);
+
+  int? ticketIndex;
+  int myIdentifier = _randomId();
+
+  // Try to acquire a ticket - (Acquire loop)
+  while (ticketIndex == null) {
+
+    for (int i = 0; i < concurrencyHeader.numberOfWorkers; i++) {
+
+      SweepstoreWorkerTicket ticket = concurrencyHeader[i];
+
+      if (!ticket.writable()) {
+        continue;
+      }
+
+      int identifier = ticket.identifier;
+
+      bool identifier_unassigned = identifier == 0;
+      bool stale_heartbeat = (DateTime.now().millisecondsSinceEpoch32() - ticket.workerHeartbeat) > 2000;
+      bool is_free = ticket.ticketState == SweepstoreTicketState.FREE;
+
+      if (identifier_unassigned && stale_heartbeat && is_free) {
+        ticket.write(
+          identifier: myIdentifier,
+          ticketState: SweepstoreTicketState.WAITING,
+        );
+        ticketIndex = i;
+        log("Acquired ticket $ticketIndex with identifier $myIdentifier.");
+        break;
+
+      }
+
+    }
+
+    expSleep("acquire_loop");
+
+    // Ensure we still own the ticket - if not, reset and try again
+    if (ticketIndex != null) {
+      SweepstoreWorkerTicket ticket = concurrencyHeader[ticketIndex];
+      if (ticket.identifier != myIdentifier) {
+        log("Lost ticket $ticketIndex, retrying...");
+        ticketIndex = null;
+      }
+    }
+  }
+
+  // We have a ticket, set it up
+  SweepstoreWorkerTicket myTicket = concurrencyHeader[ticketIndex];
+  myTicket.write(
+    workerHeartbeat: DateTime.now().millisecondsSinceEpoch32(),
+    ticketState: SweepstoreTicketState.WAITING,
+    ticketOperation: operation,
+    keyHash: keyHash,
+    writeSize: writeSize,
+  );
+
+  // Wait for approval - (Approval loop)
+  while (true) {
+
+    // Check we still own the ticket
+    if (myTicket.identifier != myIdentifier) {
+      String exceptionMessage = "CRITICAL: Lost ownership of ticket $ticketIndex, was expecting identifier $myIdentifier but found ${myTicket.identifier}.";
+      throw Exception(exceptionMessage);
+    }
+
+    if (myTicket.ticketState == SweepstoreTicketState.APPROVED) {
+      myTicket.write(
+        ticketState: SweepstoreTicketState.EXECUTING,
+      );
+      onApproved();
+      myTicket.write(
+        ticketState: SweepstoreTicketState.COMPLETED,
+      );
+
+      break;
+    }
+
+    // randomSleep(10);
+    tickSleep();
+
+    // Update heartbeat
+    if (DateTime.now().millisecondsSinceEpoch32() != myTicket.workerHeartbeat) {
+      myTicket.write(
+        workerHeartbeat: myTicket.workerHeartbeat,
+      );
+    }
+  }
+
+
+
+}
+
+// Master side
+void initialiseMasterListener(RandomAccessFile file) async {
+  String filePath = file.path;
+  Isolate.spawn((_) {
+
+    void log(String message) {
+      print("\x1B[38;5;82m[Master Listener]:\x1B[0m $message");
+    }
+
+    RandomAccessFile file = File(filePath).openSync(mode: FileMode.append);
+
+    SweepstoreHeader header = SweepstoreHeader(file);
+    SweepstoreConcurrencyHeader concurrencyHeader = SweepstoreConcurrencyHeader(header);
+
+    while (true) {
+
+      for (int i = 0; i < concurrencyHeader.numberOfWorkers; i++) {
+
+        SweepstoreWorkerTicket ticket = concurrencyHeader[i];
+
+        if (ticket.ticketState == SweepstoreTicketState.WAITING) {
+          log("Found waiting ticket $i (Key Hash: ${ticket.keyHash})...");
+
+          // Approve the ticket
+          ticket.write(
+            ticketState: SweepstoreTicketState.APPROVED,
+          );
+          log("Approved ticket $i.");
+        } else if (ticket.ticketState == SweepstoreTicketState.COMPLETED) {
+          log("Ticket $i completed. Resetting ticket...");
+          // Reset the ticket
+          ticket.write(
+            identifier: 0,
+            workerHeartbeat: 0,
+            ticketState: SweepstoreTicketState.FREE,
+            ticketOperation: SweepstoreTicketOperation.NONE,
+            keyHash: 0,
+            writeSize: 0,
+          );
+          log("Reset ticket $i.");
+        }
+
+      }
+
+      sleep(Duration(milliseconds: 1));
+
+    }
+  }, null);
+}

dart/lib/debug.dart

@@ -0,0 +1,50 @@
+
+import 'dart:typed_data';
+
+String binaryDump(Uint8List data) {
+  StringBuffer buffer = StringBuffer();
+
+  for (int i = 0; i < data.length; i += 16) {
+    // Address
+    buffer.write('0x${i.toRadixString(16).padLeft(4, '0').toUpperCase()} (${i.toString().padLeft(4)}) | ');
+
+    // Hex bytes
+    for (int j = 0; j < 16; j++) {
+      if (i + j < data.length) {
+        buffer.write('${data[i + j].toRadixString(16).padLeft(2, '0').toUpperCase()} ');
+      } else {
+        buffer.write('   ');
+      }
+    }
+
+    buffer.write(' | ');
+
+    // Integer representation
+    for (int j = 0; j < 16; j++) {
+      if (i + j < data.length) {
+        buffer.write('${data[i + j].toString().padLeft(3)} ');
+      } else {
+        buffer.write('    ');
+      }
+    }
+
+    buffer.write(' | ');
+
+    // ASCII representation
+    for (int j = 0; j < 16; j++) {
+      if (i + j < data.length) {
+        int byte = data[i + j];
+        if (byte >= 32 && byte <= 126) {
+          buffer.write(String.fromCharCode(byte));
+        } else {
+          buffer.write('.');
+        }
+      }
+    }
+
+    buffer.write(' | ');
+    if (i + 16 < data.length) buffer.writeln();
+  }
+
+  return buffer.toString();
+}

dart/lib/dev_tools/watch_dump.dart

@@ -0,0 +1,31 @@
+import 'dart:io';
+import 'dart:async';
+import 'package:sweepstore/debug.dart';
+
+void main() async {
+  int refreshCount = 0;
+  while (true) {
+    // Clear console
+    if (Platform.isWindows) {
+      print(Process.runSync("cls", [], runInShell: true).stdout);
+    } else {
+      print(Process.runSync("clear", [], runInShell: true).stdout);
+    }
+
+    refreshCount++;
+
+    // Read example.bin
+    final file = File('example.bin');
+
+    if (await file.exists()) {
+      final data = await file.readAsBytes();
+      print('Binary dump of example.bin (${data.length} bytes) - Refresh #$refreshCount\n');
+      print(binaryDump(data));
+      print('\n--- Refreshing in 1 seconds ---');
+    } else {
+      print('Error: example.bin not found - Refresh #$refreshCount');
+    }
+
+    await Future.delayed(Duration(seconds: 1));
+  }
+}

dart/lib/dev_tools/watch_tickets.dart

@@ -0,0 +1,84 @@
+import 'dart:io';
+import 'dart:async';
+import 'package:sweepstore/header.dart';
+
+
+void main() async {
+  int refreshCount = 0;
+  int? previousMasterHeartbeat;
+  Map<int, int> previousWorkerHeartbeats = {};
+
+  while (true) {
+    // Clear console
+    if (Platform.isWindows) {
+      print(Process.runSync("cls", [], runInShell: true).stdout);
+    } else {
+      print(Process.runSync("clear", [], runInShell: true).stdout);
+    }
+
+    refreshCount++;
+
+    // Read example.bin
+    final file = File('example.bin');
+
+    if (await file.exists()) {
+      final raf = await file.open(mode: FileMode.read);
+
+
+      try {
+        final header = SweepstoreHeader(raf);
+        final concurrency = header.concurrency;
+
+        int now32 = (DateTime.now().millisecondsSinceEpoch ~/ 1000) & 0xFFFFFFFF;
+
+        print('Sweepstore Tickets - Refresh #$refreshCount');
+        print('Current Time (now32): $now32');
+        print('Master ID: ${concurrency.masterIdentifier}');
+
+        int masterAge = now32 - concurrency.masterHeartbeat;
+        String masterStatus = masterAge > 5 ? "(stale)" : "(active)";
+        String masterPrevious = previousMasterHeartbeat != null ? "(previously $previousMasterHeartbeat)" : "";
+        print('Master Heartbeat: ${concurrency.masterHeartbeat} $masterStatus $masterPrevious');
+
+        print('Workers: ${concurrency.numberOfWorkers}');
+        print('Read Allowed: ${concurrency.isReadAllowed}');
+        print('');
+
+        // display each ticket
+        for (int i = 0; i < concurrency.numberOfWorkers; i++) {
+          final ticket = concurrency[i];
+
+          print('--- Ticket #$i ---');
+          print('  Identifier: ${ticket.identifier}');
+
+          int workerAge = now32 - ticket.workerHeartbeat;
+          String workerStatus = workerAge > 5 ? "(stale)" : "(active)";
+          String workerPrevious = previousWorkerHeartbeats.containsKey(i) ? "(previously ${previousWorkerHeartbeats[i]})" : "";
+          print('  Heartbeat: ${ticket.workerHeartbeat} $workerStatus $workerPrevious');
+
+          print('  State: ${ticket.ticketState.name}');
+          print('  Operation: ${ticket.ticketOperation.name}');
+          print('  Key Hash: ${ticket.keyHash}');
+          print('  Write Ptr: ${ticket.writePointer}');
+          print('  Write Size: ${ticket.writeSize} bytes');
+          print('');
+
+          // update previous heartbeat
+          previousWorkerHeartbeats[i] = ticket.workerHeartbeat;
+
+        }
+
+        // updat previous master heartbeat
+        previousMasterHeartbeat = concurrency.masterHeartbeat;
+
+        print('--- Refreshing in 1 seconds ---');
+      } finally {
+        await raf.close();
+      }
+    } else {
+      print('Error: example.bin not found - Refresh #$refreshCount');
+    }
+
+    await Future.delayed(Duration(seconds: 1));
+  }
+}

dart/lib/header.dart

@@ -0,0 +1,440 @@
+
+import 'dart:convert';
+import 'dart:io';
+import 'package:sweepstore/structures.dart';
+
+import 'helper_extensions.dart';
+
+void initialiseSweepstoreHeader(RandomAccessFile file, {
+  int concurrentWorkers = 4,
+}) {
+
+  SweepstoreHeaderWriter header = SweepstoreHeaderWriter(file);
+
+  if (header.magicNumber == 'SWPT') {
+    throw ArgumentError('Sweepstore file is already initialised.');
+  }
+
+  SweepstoreConcurrencyHeaderWriter concurrencyHeader = SweepstoreConcurrencyHeaderWriter(header);
+
+  header.magicNumber = 'SWPT';
+  header.version = "undefined";
+  header.addressTablePointer = SweepstorePointer.nullptr;
+  header.freeListCount = 0;
+  header.isFreeListLifted = false;
+  concurrencyHeader.masterIdentifier = 0;
+  concurrencyHeader.masterHeartbeat = 0;
+  concurrencyHeader.numberOfWorkers = concurrentWorkers;
+  concurrencyHeader.isReadAllowed = false;
+
+  for (int i = 0; i < concurrentWorkers; i++) {
+    SweepstoreWorkerTicket ticket = concurrencyHeader[i];
+    ticket.write(
+      identifier: 0,
+      workerHeartbeat: 0,
+      ticketState: SweepstoreTicketState.FREE,
+      ticketOperation: SweepstoreTicketOperation.NONE,
+      keyHash: 0,
+      writePointer: SweepstorePointer.nullptr,
+      writeSize: 0,
+    );
+  }
+
+}
+
+class SweepstoreHeader {
+
+  final RandomAccessFile _file;
+
+  SweepstoreHeader(this._file);
+
+  // Offset 0 - 4 bytes
+  String get magicNumber {
+    _file.setPositionSync(0);
+    final bytes = _file.readSync(4);
+    return String.fromCharCodes(bytes);
+  }
+
+  // Offset 4 - 12 bytes
+  String get version {
+    _file.setPositionSync(4);
+    String version = utf8.decode(_file.readSync(12)).trim();
+    return version;
+  }
+
+  // Offset 16 - 8 bytes
+  SweepstorePointer get addressTablePointer {
+    _file.setPositionSync(16);
+    final address = _file.readIntSync(8);
+    return SweepstorePointer(address);
+  }
+
+  // Offset 24 - 4 bytes
+  int get freeListCount {
+    _file.setPositionSync(24);
+    int count = _file.readIntSync(4);
+    return count;
+  }
+
+  // Offset 28 - 1 byte
+  bool get isFreeListLifted {
+    _file.setPositionSync(28);
+    int flag = _file.readIntSync(1);
+    return flag != 0;
+  }
+
+  SweepstoreConcurrencyHeader get concurrency => SweepstoreConcurrencyHeader(this);
+}
+
+class SweepstoreHeaderWriter extends SweepstoreHeader {
+
+  SweepstoreHeaderWriter(RandomAccessFile file) : super(file);
+
+  // Offset 0 - 4 bytes
+  void set magicNumber(String value) {
+    if (value.length != 4) {
+      throw ArgumentError('Magic number must be exactly 4 characters long');
+    }
+    _file.setPositionSync(0);
+    _file.writeFromSync(value.codeUnits);
+  }
+
+  // Offset 4 - 12 bytes
+  void set version(String value) {
+    if (value.length > 11) {
+      throw ArgumentError('Version string must be at most 11 characters long');
+    }
+    _file.setPositionSync(4);
+    _file.writeFromSync(utf8.encode(" " + value.padRight(11, ' ').substring(0, 11)));
+  }
+
+  // Offset 16 - 8 bytes
+  void set addressTablePointer(SweepstorePointer pointer) {
+    _file.setPositionSync(16);
+    _file.writeIntSync(pointer.address, 8);
+  }
+
+  // Offset 24 - 4 bytes
+  void set freeListCount(int value) {
+    _file.setPositionSync(24);
+    _file.writeIntSync(value, 4);
+  }
+
+  // Offset 28 - 1 byte
+  void set isFreeListLifted(bool lifted) {
+    _file.setPositionSync(28);
+    _file.writeIntSync(lifted ? 1 : 0, 1);
+  }
+}
+
+class SweepstoreConcurrencyHeader {
+
+  final SweepstoreHeader _header;
+
+  SweepstoreConcurrencyHeader(this._header);
+
+  // Offset 29 - 8 bytes
+  int get masterIdentifier {
+    _header._file.setPositionSync(29);
+    int id = _header._file.readIntSync(8);
+    return id;
+  }
+
+  // Offset 37 - 4 bytes
+  int get masterHeartbeat {
+    _header._file.setPositionSync(37);
+    int heartbeat = _header._file.readIntSync(4);
+    return heartbeat;
+  }
+
+  // Offset 41 - 4 bytes
+  int get numberOfWorkers {
+    _header._file.setPositionSync(41);
+    int numWorkers = _header._file.readIntSync(4);
+    return numWorkers;
+  }
+
+  // Offset 45 - 1 byte
+  bool get isReadAllowed {
+    _header._file.setPositionSync(45);
+    int flag = _header._file.readIntSync(1);
+    return flag != 0;
+  }
+
+  SweepstoreWorkerTicket operator [](int ticketIndex) {
+    if (ticketIndex < 0 || ticketIndex >= numberOfWorkers) {
+      throw RangeError.index(ticketIndex, this, 'ticketIndex', null, numberOfWorkers);
+    }
+    return SweepstoreWorkerTicket(ticketIndex, this);
+  }
+
+}
+
+class SweepstoreConcurrencyHeaderWriter extends SweepstoreConcurrencyHeader {
+
+  SweepstoreConcurrencyHeaderWriter(SweepstoreHeader header) : super(header);
+
+  // Offset 29 - 8 bytes
+  void set masterIdentifier(int id) {
+    _header._file.setPositionSync(29);
+    _header._file.writeIntSync(id, 8);
+  }
+
+  // Offset 37 - 4 bytes
+  void set masterHeartbeat(int heartbeat) {
+    _header._file.setPositionSync(37);
+    _header._file.writeIntSync(heartbeat, 4);
+  }
+
+  // Offset 41 - 4 bytes
+  void set numberOfWorkers(int numWorkers) {
+    _header._file.setPositionSync(41);
+    _header._file.writeIntSync(numWorkers, 4);
+  }
+
+  // Offset 45 - 1 byte
+  void set isReadAllowed(bool allowed) {
+    _header._file.setPositionSync(45);
+    _header._file.writeIntSync(allowed ? 1 : 0, 1);
+  }
+
+}
+
+const int endOfStaticHeaderOffset = 46;
+
+class SweepstoreWorkerTicket {
+
+  static const int _ticketSize = 30;
+  final SweepstoreConcurrencyHeader _concurrencyHeader;
+  final int ticketIndex;
+
+  SweepstoreWorkerTicket(this.ticketIndex, this._concurrencyHeader);
+
+  // All offsets are relative to the start of the workers ticket
+  int get _baseOffset => endOfStaticHeaderOffset + (ticketIndex * _ticketSize);
+
+  // Offset 0 - 4 bytes
+  int get identifier {
+    _concurrencyHeader._header._file.setPositionSync(_baseOffset);
+    int id = _concurrencyHeader._header._file.readIntSync(4);
+    return id;
+  }
+
+  // Offset 4 - 4 bytes
+  int get workerHeartbeat {
+    _concurrencyHeader._header._file.setPositionSync(_baseOffset + 4);
+    int heartbeat = _concurrencyHeader._header._file.readIntSync(4);
+    return heartbeat;
+  }
+
+  // Offset 8 - 1 byte
+  SweepstoreTicketState get ticketState {
+    _concurrencyHeader._header._file.setPositionSync(_baseOffset + 8);
+    int stateValue = _concurrencyHeader._header._file.readIntSync(1);
+    return SweepstoreTicketState.values[stateValue];
+  }
+
+  // Offset 9 - 1 byte
+  SweepstoreTicketOperation get ticketOperation {
+    _concurrencyHeader._header._file.setPositionSync(_baseOffset + 9);
+    int operationValue = _concurrencyHeader._header._file.readIntSync(1);
+    return SweepstoreTicketOperation.values[operationValue];
+  }
+
+  // Offset 10 - 8 bytes
+  int get keyHash {
+    _concurrencyHeader._header._file.setPositionSync(_baseOffset + 10);
+    int hash = _concurrencyHeader._header._file.readIntSync(8);
+    return hash;
+  }
+
+  // Offset 18 - 8 bytes
+  SweepstorePointer get writePointer {
+    _concurrencyHeader._header._file.setPositionSync(_baseOffset + 18);
+    int address = _concurrencyHeader._header._file.readIntSync(8);
+    return SweepstorePointer(address);
+  }
+
+  // Offset 26 - 4 bytes
+  int get writeSize {
+    _concurrencyHeader._header._file.setPositionSync(_baseOffset + 26);
+    int size = _concurrencyHeader._header._file.readIntSync(4);
+    return size;
+  }
+
+  // Writer
+  void write({
+    int? identifier,
+    int? workerHeartbeat,
+    SweepstoreTicketState? ticketState,
+    SweepstoreTicketOperation? ticketOperation,
+    int? keyHash,
+    SweepstorePointer? writePointer,
+    int? writeSize,
+  }) {
+
+    try {
+
+      _concurrencyHeader._header._file.lockSync(FileLock.blockingExclusive, _baseOffset, _baseOffset + _ticketSize);
+
+      _concurrencyHeader._header._file.setPositionSync(_baseOffset);
+      List<int> existingBuffer = _concurrencyHeader._header._file.readSync(_ticketSize);
+      RandomAccessMemory buffer = RandomAccessMemory(existingBuffer);
+
+      if (identifier != null) {
+        buffer.setPositionSync(0);
+        buffer.writeIntSync(identifier, 4);
+      }
+      if (workerHeartbeat != null) {
+        buffer.setPositionSync(4);
+        buffer.writeIntSync(workerHeartbeat, 4);
+      }
+      if (ticketState != null) {
+        buffer.setPositionSync(8);
+        buffer.writeIntSync(ticketState.index, 1);
+      }
+      if (ticketOperation != null) {
+        buffer.setPositionSync(9);
+        buffer.writeIntSync(ticketOperation.index, 1);
+      }
+      if (keyHash != null) {
+        buffer.setPositionSync(10);
+        buffer.writeIntSync(keyHash, 8);
+      }
+      if (writePointer != null) {
+        buffer.setPositionSync(18);
+        buffer.writeIntSync(writePointer.address, 8);
+      }
+      if (writeSize != null) {
+        buffer.setPositionSync(26);
+        buffer.writeIntSync(writeSize, 4);
+      }
+
+      _concurrencyHeader._header._file.setPositionSync(_baseOffset);
+      _concurrencyHeader._header._file.writeFromSync(buffer.toUint8List());
+
+    } finally {
+      _concurrencyHeader._header._file.unlockSync(_baseOffset, _baseOffset + _ticketSize);
+    }
+  }
+
+  bool writable() {
+    try {
+      _concurrencyHeader._header._file.lockSync(
+          FileLock.blockingExclusive,
+          _baseOffset,
+          _baseOffset + _ticketSize
+      );
+      // Successfully locked - immediately unlock and return true
+      _concurrencyHeader._header._file.unlockSync(_baseOffset, _baseOffset + _ticketSize);
+      return true;
+    } catch (e) {
+      // Lock failed - already held by another process
+      return false;
+    }
+  }
+}
+
+@deprecated
+class _SweepstoreWorkerTicket {
+
+  static const int _ticketSize = 30;
+
+  final SweepstoreConcurrencyHeader _concurrencyHeader;
+  final int ticketIndex;
+
+  _SweepstoreWorkerTicket(this.ticketIndex, this._concurrencyHeader);
+
+  // All offsets are relative to the start of the workers ticket
+  int get _baseOffset => endOfStaticHeaderOffset + (ticketIndex * _ticketSize);
+
+  // Offset 0 - 4 bytes
+  int get identifier {
+    _concurrencyHeader._header._file.setPositionSync(_baseOffset);
+    int id = _concurrencyHeader._header._file.readIntSync(4);
+    return id;
+  }
+  set identifier(int id) {
+    _concurrencyHeader._header._file.setPositionSync(_baseOffset);
+    _concurrencyHeader._header._file.writeIntSync(id, 4);
+    _concurrencyHeader._header._file.flushSync();
+  }
+
+  // Offset 4 - 4 bytes
+  int get workerHeartbeat {
+    _concurrencyHeader._header._file.setPositionSync(_baseOffset + 4);
+    int heartbeat = _concurrencyHeader._header._file.readIntSync(4);
+    return heartbeat;
+  }
+  set workerHeartbeat(int heartbeat) {
+    _concurrencyHeader._header._file.setPositionSync(_baseOffset + 4);
+    _concurrencyHeader._header._file.writeIntSync(heartbeat, 4);
+  }
+
+  // Offset 8 - 1 byte
+  SweepstoreTicketState get ticketState {
+    _concurrencyHeader._header._file.setPositionSync(_baseOffset + 8);
+    int stateValue = _concurrencyHeader._header._file.readIntSync(1);
+    return SweepstoreTicketState.values[stateValue];
+  }
+  set ticketState(SweepstoreTicketState state) {
+    _concurrencyHeader._header._file.setPositionSync(_baseOffset + 8);
+    _concurrencyHeader._header._file.writeIntSync(state.index, 1);
+    _concurrencyHeader._header._file.flushSync();
+  }
+
+  // Offset 9 - 1 byte
+  SweepstoreTicketOperation get ticketOperation {
+    _concurrencyHeader._header._file.setPositionSync(_baseOffset + 9);
+    int operationValue = _concurrencyHeader._header._file.readIntSync(1);
+    return SweepstoreTicketOperation.values[operationValue];
+  }
+  set ticketOperation(SweepstoreTicketOperation operation) {
+    _concurrencyHeader._header._file.setPositionSync(_baseOffset + 9);
+    _concurrencyHeader._header._file.writeIntSync(operation.index, 1);
+  }
+
+  // Offset 10 - 8 bytes
+  int get keyHash {
+    _concurrencyHeader._header._file.setPositionSync(_baseOffset + 10);
+    int hash = _concurrencyHeader._header._file.readIntSync(8);
+    return hash;
+  }
+  set keyHash(int hash) {
+    _concurrencyHeader._header._file.setPositionSync(_baseOffset + 10);
+    _concurrencyHeader._header._file.writeIntSync(hash, 8);
+  }
+
+  // Offset 18 - 8 bytes
+  SweepstorePointer get writePointer {
+    _concurrencyHeader._header._file.setPositionSync(_baseOffset + 18);
+    int address = _concurrencyHeader._header._file.readIntSync(8);
+    return SweepstorePointer(address);
+  }
+  set writePointer(SweepstorePointer pointer) {
+    _concurrencyHeader._header._file.setPositionSync(_baseOffset + 18);
+    _concurrencyHeader._header._file.writeIntSync(pointer.address, 8);
+  }
+
+  // Offset 26 - 4 bytes
+  int get writeSize {
+    _concurrencyHeader._header._file.setPositionSync(_baseOffset + 26);
+    int size = _concurrencyHeader._header._file.readIntSync(4);
+    return size;
+  }
+  set writeSize(int size) {
+    _concurrencyHeader._header._file.setPositionSync(_baseOffset + 26);
+    _concurrencyHeader._header._file.writeIntSync(size, 4);
+  }
+
+  // Helpers
+
+  void lock() {
+    _concurrencyHeader._header._file.lockSync(FileLock.exclusive, _baseOffset, _baseOffset + _ticketSize);
+  }
+
+  void unlock() {
+    _concurrencyHeader._header._file.unlockSync(_baseOffset, _baseOffset + _ticketSize);
+  }
+
+}

dart/lib/helper_extensions.dart

@@ -0,0 +1,228 @@
+
+import 'dart:io';
+import 'dart:math';
+import 'dart:typed_data';
+
+import 'package:sweepstore/structures.dart';
+
+
+class RandomAccessMemory {
+  List<int> _buffer;
+  int _position = 0;
+
+  RandomAccessMemory([List<int>? initialData]) : _buffer = initialData != null ? List<int>.from(initialData) : [];
+
+  // Position management
+  int positionSync() => _position;
+
+  void setPositionSync(int position) {
+    _position = position;
+  }
+
+  int length() => _buffer.length;
+
+  // Read bytes
+  List<int> readSync(int count) {
+    if (_position + count > _buffer.length) {
+      throw RangeError('Not enough bytes to read');
+    }
+    List<int> result = _buffer.sublist(_position, _position + count);
+    _position += count;
+    return result;
+  }
+
+  // Write bytes
+  void writeFromSync(List<int> bytes) {
+    for (int i = 0; i < bytes.length; i++) {
+      if (_position + i >= _buffer.length) {
+        _buffer.add(bytes[i]);
+      } else {
+        _buffer[_position + i] = bytes[i];
+      }
+    }
+    _position += bytes.length;
+  }
+
+  // Read/Write Int Dynamic
+  int readIntSync([int size = 4, Endian endianness = Endian.little]) {
+    if (size < 1 || size > 8) {
+      throw ArgumentError('Size must be between 1 and 8 bytes');
+    }
+
+    List<int> bytes = readSync(size);
+
+    // Build integer from bytes with proper endianness
+    int result = 0;
+    if (endianness == Endian.little) {
+      for (int i = size - 1; i >= 0; i--) {
+        result = (result << 8) | bytes[i];
+      }
+    } else {
+      for (int i = 0; i < size; i++) {
+        result = (result << 8) | bytes[i];
+      }
+    }
+
+    // Sign extend if MSB is set
+    int signBit = 1 << (size * 8 - 1);
+    if (result & signBit != 0) {
+      result -= 1 << (size * 8);
+    }
+
+    return result;
+  }
+
+  void writeIntSync(int value, [int size = 4, Endian endianness = Endian.little]) {
+    if (size < 1 || size > 8) {
+      throw ArgumentError('Size must be between 1 and 8 bytes');
+    }
+
+    List<int> bytes = List.filled(size, 0);
+
+    // Extract bytes with proper endianness
+    if (endianness == Endian.little) {
+      for (int i = 0; i < size; i++) {
+        bytes[i] = (value >> (i * 8)) & 0xFF;
+      }
+    } else {
+      for (int i = 0; i < size; i++) {
+        bytes[size - 1 - i] = (value >> (i * 8)) & 0xFF;
+      }
+    }
+
+    writeFromSync(bytes);
+  }
+
+  // Read/Write Pointers
+  SweepstorePointer readPointerSync() {
+    int offset = readIntSync(SweepstorePrimitives.POINTER.size);
+    return SweepstorePointer(offset);
+  }
+
+  void writePointerSync(SweepstorePointer pointer) {
+    writeIntSync(pointer.address, SweepstorePrimitives.POINTER.size);
+  }
+
+  // Read/Write Float32
+  double readFloat32Sync([Endian endianness = Endian.little]) {
+    List<int> bytes = readSync(4);
+    return ByteData.sublistView(Uint8List.fromList(bytes)).getFloat32(0, endianness);
+  }
+
+  void writeFloat32Sync(double value, [Endian endianness = Endian.little]) {
+    ByteData byteData = ByteData(4);
+    byteData.setFloat32(0, value, endianness);
+    writeFromSync(byteData.buffer.asUint8List());
+  }
+
+  // Read/Write Float64 (Double)
+  double readFloat64Sync([Endian endianness = Endian.little]) {
+    List<int> bytes = readSync(8);
+    return ByteData.sublistView(Uint8List.fromList(bytes)).getFloat64(0, endianness);
+  }
+
+  void writeFloat64Sync(double value, [Endian endianness = Endian.little]) {
+    ByteData byteData = ByteData(8);
+    byteData.setFloat64(0, value, endianness);
+    writeFromSync(byteData.buffer.asUint8List());
+  }
+
+  // Conversion methods
+  List<int> toList() => List<int>.from(_buffer);
+
+  Uint8List toUint8List() => Uint8List.fromList(_buffer);
+}
+
+extension SweepstoreRandomAccessFileHelper on RandomAccessFile {
+
+  // Read/Write Int Dynamic - Can specify size in bytes, does not have to align to 1, 2, 4, or 8 bytes. Default is 4 bytes (Int32)
+  int readIntSync([int size = 4, Endian endianness = Endian.little]) {
+    if (size < 1 || size > 8) {
+      throw ArgumentError('Size must be between 1 and 8 bytes');
+    }
+
+    List<int> bytes = readSync(size);
+
+    // Build integer from bytes with proper endianness
+    int result = 0;
+    if (endianness == Endian.little) {
+      for (int i = size - 1; i >= 0; i--) {
+        result = (result << 8) | bytes[i];
+      }
+    } else {
+      for (int i = 0; i < size; i++) {
+        result = (result << 8) | bytes[i];
+      }
+    }
+
+    // Sign extend if MSB is set
+    int signBit = 1 << (size * 8 - 1);
+    if (result & signBit != 0) {
+      result -= 1 << (size * 8);
+    }
+
+    return result;
+  }
+
+  void writeIntSync(int value, [int size = 4, Endian endianness = Endian.little]) {
+    if (size < 1 || size > 8) {
+      throw ArgumentError('Size must be between 1 and 8 bytes');
+    }
+
+    List<int> bytes = List.filled(size, 0);
+
+    // Extract bytes with proper endianness
+    if (endianness == Endian.little) {
+      for (int i = 0; i < size; i++) {
+        bytes[i] = (value >> (i * 8)) & 0xFF;
+      }
+    } else {
+      for (int i = 0; i < size; i++) {
+        bytes[size - 1 - i] = (value >> (i * 8)) & 0xFF;
+      }
+    }
+
+    writeFromSync(bytes);
+  }
+
+  // Read/Write Pointers
+  SweepstorePointer readPointerSync() {
+    int offset = readIntSync(SweepstorePrimitives.POINTER.size);
+    return SweepstorePointer(offset);
+  }
+  void writePointerSync(SweepstorePointer pointer) {
+    writeIntSync(pointer.address, SweepstorePrimitives.POINTER.size);
+  }
+
+  // Read/Write Float32
+  double readFloat32Sync([Endian endianness = Endian.little]) {
+    List<int> bytes = readSync(4);
+    return ByteData.sublistView(Uint8List.fromList(bytes)).getFloat32(0, endianness);
+  }
+  void writeFloat32Sync(double value, [Endian endianness = Endian.little]) {
+    ByteData byteData = ByteData(4);
+    byteData.setFloat32(0, value, endianness);
+    writeFromSync(byteData.buffer.asUint8List());
+  }
+
+  // Read/Write Float64 (Double)
+  double readFloat64Sync([Endian endianness = Endian.little]) {
+    List<int> bytes = readSync(8);
+    return ByteData.sublistView(Uint8List.fromList(bytes)).getFloat64(0, endianness);
+  }
+  void writeFloat64Sync(double value, [Endian endianness = Endian.little]) {
+    ByteData byteData = ByteData(8);
+    byteData.setFloat64(0, value, endianness);
+    writeFromSync(byteData.buffer.asUint8List());
+  }
+
+}
+
+extension SweepstoreDateTimeHelper on DateTime {
+  int millisecondsSinceEpoch32() {
+    return (millisecondsSinceEpoch ~/ 1000) & 0xFFFFFFFF;
+  }
+  int millisecondsSinceEpoch64() {
+    return millisecondsSinceEpoch;
+  }
+}

dart/lib/structures.dart

@@ -0,0 +1,48 @@
+
+enum SweepstorePrimitives {
+
+  POINTER       (8),
+  ADDRESS_TABLE (-1);
+
+  final int size;
+  final bool arrayType;
+  const SweepstorePrimitives(this.size, {
+    this.arrayType = false
+  });
+
+}
+
+class SweepstorePointer {
+
+  static const SweepstorePointer nullptr = SweepstorePointer(-1);
+
+  final int address;
+
+  const SweepstorePointer(this.address);
+
+  bool get isNull => address == -1;
+
+  operator ==(Object other) {
+    if (identical(this, other)) return true;
+    if (other is! SweepstorePointer) return false;
+    return address == other.address;
+  }
+
+  @override
+  String toString() => '0x${address.toRadixString(16)} ($address)';
+}
+
+enum SweepstoreTicketState {
+  FREE,
+  WAITING,
+  APPROVED,
+  EXECUTING,
+  COMPLETED,
+}
+
+enum SweepstoreTicketOperation {
+  NONE,
+  READ,
+  MODIFY,
+  WRITE,
+}

dart/lib/sweepstore.dart

@@ -0,0 +1,105 @@
+
+import 'dart:io';
+import 'dart:isolate';
+import 'dart:math';
+import 'package:sweepstore/debug.dart';
+import 'package:sweepstore/header.dart';
+import 'package:sweepstore/structures.dart';
+import 'package:sweepstore/concurrency.dart';
+
+class Sweepstore {
+
+  final RandomAccessFile _file;
+
+  Sweepstore(String filePath)
+      : _file = File(filePath).openSync(mode: FileMode.append)
+  {
+    _header = SweepstoreHeaderWriter(_file);
+  }
+
+  late final SweepstoreHeaderWriter _header;
+  late final SweepstoreConcurrencyHeaderWriter _concurrencyHeader = SweepstoreConcurrencyHeaderWriter(_header);
+
+  void initialise({
+      int concurrentWorkers = 4,
+  }) {
+    
+    initialiseSweepstoreHeader(_file,
+      concurrentWorkers: concurrentWorkers,
+    );
+
+    _header.version = "1.1.0.1";
+    print("Version: ${_header.version}");
+    
+  }
+
+  void operator []=(String key, dynamic value) {
+
+    spawnTicket(_file,
+      operation: SweepstoreTicketOperation.WRITE,
+      keyHash: key.hashCode,
+      writeSize: 0, // Placeholder
+      onApproved: () {
+        print("Writing key: $key with hash ${key.hashCode} and value: $value");
+      },
+      debugLabel: key
+    );
+
+  }
+}
+
+Future<void> main() async {
+
+  String filePath = '../example.bin';
+
+  File file = File(filePath);
+  if (file.existsSync()) {
+    file.deleteSync();
+  }
+  file.createSync();
+
+  Sweepstore store = Sweepstore(filePath);
+  store.initialise(
+    concurrentWorkers: 20
+  );
+  initialiseMasterListener(file.openSync(mode: FileMode.append));
+
+  print(binaryDump(file.readAsBytesSync()));
+
+  int iteration = 1;
+
+  for (int j = 0; j < iteration; j++) {
+    int concurrencyTest = 256;
+    final receivePort = ReceivePort();
+    int completedJobs = 0;
+
+    final stopwatch = Stopwatch()..start();
+
+    for (int i = 0; i < concurrencyTest; i++) {
+      await Isolate.spawn((message) {
+        final index = message['index'] as int;
+        final sendPort = message['sendPort'] as SendPort;
+
+        Sweepstore store = Sweepstore(filePath);
+        store['key_$index'] = 'value_$index';
+
+        sendPort.send('done');
+      }, {'index': i, 'sendPort': receivePort.sendPort});
+    }
+
+
+    // wait for all jobs to finish
+    await for (var msg in receivePort) {
+      completedJobs++;
+      if (completedJobs >= concurrencyTest) {
+        receivePort.close();
+        break;
+      }
+    }
+
+    stopwatch.stop();
+
+    print('\x1B[95mAll jobs completed!\x1B[0m');
+    print('\x1B[95mTime taken: ${stopwatch.elapsedMilliseconds}ms (${stopwatch.elapsed.inSeconds}s)\x1B[0m');
+  }
+}