Add concurrency handling implementation with ticket management and file locking
This commit is contained in:
ImBenji
@@ -3,13 +3,31 @@ project(BinaryTable)
|
||||
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
||||
|
||||
# Add include directories globally
|
||||
include_directories(${CMAKE_SOURCE_DIR}/src/Public)
|
||||
include_directories(${CMAKE_SOURCE_DIR}/src/Private)
|
||||
|
||||
# Main executable with integrated binary table implementation
|
||||
add_executable(main
|
||||
binary_table.h
|
||||
binary_table.cpp
|
||||
src/Private/sweepstore/header.cpp
|
||||
src/Public/sweepstore/utils/helpers.h
|
||||
src/Private/sweepstore/structures.cpp
|
||||
src/Public/sweepstore/structures.h
|
||||
src/Private/sweepstore/sweepstore.cpp
|
||||
src/Public/sweepstore/sweepstore.h
|
||||
src/Public/sweepstore/concurrency.h
|
||||
src/Private/sweepstore/concurrency.cpp
|
||||
src/Public/sweepstore/utils/file_lock.h
|
||||
src/Public/sweepstore/utils/file_handle.h
|
||||
src/Public/sweepstore/header.h
|
||||
src/Private/sweepstore/benchmark.cpp
|
||||
)
|
||||
|
||||
# Add include directories
|
||||
target_include_directories(main PRIVATE ${CMAKE_SOURCE_DIR}/src/Public)
|
||||
|
||||
# Compiler Settings
|
||||
if(MSVC)
|
||||
target_compile_options(main PRIVATE /W4)
|
||||
|
||||
1243
cpp/binary_table.cpp
1243
cpp/binary_table.cpp
File diff suppressed because it is too large
Load Diff
@@ -1,305 +0,0 @@
|
||||
/*
|
||||
|
||||
/$$$$$$ /$$ /$$ /$$$$$$$ /$$$$$$$$ /$$ /$$ /$$$$$ /$$$$$$ /$$ /$$ /$$$$$$$$ /$$$$$$$$
|
||||
|_ $$_/| $$$ /$$$| $$__ $$| $$_____/| $$$ | $$ |__ $$|_ $$_/ | $$$ | $$| $$_____/|__ $$__/
|
||||
| $$ | $$$$ /$$$$| $$ \ $$| $$ | $$$$| $$ | $$ | $$ | $$$$| $$| $$ | $$
|
||||
| $$ | $$ $$/$$ $$| $$$$$$$ | $$$$$ | $$ $$ $$ | $$ | $$ | $$ $$ $$| $$$$$ | $$
|
||||
| $$ | $$ $$$| $$| $$__ $$| $$__/ | $$ $$$$ /$$ | $$ | $$ | $$ $$$$| $$__/ | $$
|
||||
| $$ | $$\ $ | $$| $$ \ $$| $$ | $$\ $$$| $$ | $$ | $$ | $$\ $$$| $$ | $$
|
||||
/$$$$$$| $$ \/ | $$| $$$$$$$/| $$$$$$$$| $$ \ $$| $$$$$$/ /$$$$$$ /$$| $$ \ $$| $$$$$$$$ | $$
|
||||
|______/|__/ |__/|_______/ |________/|__/ \__/ \______/ |______/|__/|__/ \__/|________/ |__/
|
||||
|
||||
<EFBFBD> 2025-26 by Benjamin Watt of IMBENJI.NET LIMITED - All rights reserved.
|
||||
|
||||
Use of this source code is governed by the Business Source License 1.1 that can be found in the LICENSE file.
|
||||
|
||||
This file is part of the SweepStore (formerly Binary Table) package for C++.
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <unordered_map>
|
||||
#include <fstream>
|
||||
#include <variant>
|
||||
#include <memory>
|
||||
#include <stdexcept>
|
||||
#include <type_traits>
|
||||
#include <functional>
|
||||
#include <iostream>
|
||||
|
||||
// Debug control - comment out this line to disable all debug output
|
||||
// #define ENABLE_DEBUG 1
|
||||
|
||||
#ifdef ENABLE_DEBUG
|
||||
#define DEBUG_PRINT(x) std::cout << x
|
||||
#define DEBUG_PRINTLN(x) std::cout << x << std::endl
|
||||
#else
|
||||
#define DEBUG_PRINT(x)
|
||||
#define DEBUG_PRINTLN(x)
|
||||
#endif
|
||||
|
||||
namespace bt {
|
||||
|
||||
// Forward declarations
|
||||
class BinaryTable;
|
||||
class BT_Reference;
|
||||
template<typename T> class BT_UniformArray;
|
||||
|
||||
// Type enumeration matching Dart version
|
||||
enum class BT_Type : uint8_t {
|
||||
POINTER = 0,
|
||||
ADDRESS_TABLE = 1,
|
||||
INTEGER = 2,
|
||||
FLOAT = 3,
|
||||
STRING = 4,
|
||||
INTEGER_ARRAY = 5,
|
||||
FLOAT_ARRAY = 6
|
||||
};
|
||||
|
||||
// Size mapping for types
|
||||
constexpr int getTypeSize(BT_Type type) {
|
||||
switch (type) {
|
||||
case BT_Type::POINTER: return 8;
|
||||
case BT_Type::ADDRESS_TABLE: return -1;
|
||||
case BT_Type::INTEGER: return 4;
|
||||
case BT_Type::FLOAT: return 4;
|
||||
case BT_Type::STRING: return -1;
|
||||
case BT_Type::INTEGER_ARRAY: return -1;
|
||||
case BT_Type::FLOAT_ARRAY: return -1;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Check if type is array type
|
||||
constexpr bool isArrayType(BT_Type type) {
|
||||
return type == BT_Type::INTEGER_ARRAY || type == BT_Type::FLOAT_ARRAY;
|
||||
}
|
||||
|
||||
// Type deduction helpers
|
||||
template<typename T>
|
||||
constexpr BT_Type getTypeFromValue() {
|
||||
if constexpr (std::is_same_v<T, int32_t> || std::is_same_v<T, int>) {
|
||||
return BT_Type::INTEGER;
|
||||
} else if constexpr (std::is_same_v<T, float>) {
|
||||
return BT_Type::FLOAT;
|
||||
} else if constexpr (std::is_same_v<T, std::string>) {
|
||||
return BT_Type::STRING;
|
||||
} else if constexpr (std::is_same_v<T, std::vector<int32_t>> || std::is_same_v<T, std::vector<int>>) {
|
||||
return BT_Type::INTEGER_ARRAY;
|
||||
} else if constexpr (std::is_same_v<T, std::vector<float>>) {
|
||||
return BT_Type::FLOAT_ARRAY;
|
||||
} else {
|
||||
static_assert(sizeof(T) == 0, "Unsupported type");
|
||||
}
|
||||
}
|
||||
|
||||
// Pointer class
|
||||
class BT_Pointer {
|
||||
private:
|
||||
int64_t address_;
|
||||
|
||||
public:
|
||||
explicit BT_Pointer(int64_t address = -1) : address_(address) {}
|
||||
|
||||
bool isNull() const { return address_ == -1; }
|
||||
int64_t address() const { return address_; }
|
||||
|
||||
bool operator==(const BT_Pointer& other) const {
|
||||
return address_ == other.address_;
|
||||
}
|
||||
|
||||
bool operator!=(const BT_Pointer& other) const {
|
||||
return !(*this == other);
|
||||
}
|
||||
};
|
||||
|
||||
// Null pointer constant
|
||||
const BT_Pointer BT_Null{-1};
|
||||
|
||||
// Free list entry
|
||||
struct BT_FreeListEntry {
|
||||
BT_Pointer pointer;
|
||||
int32_t size;
|
||||
|
||||
BT_FreeListEntry(BT_Pointer ptr, int32_t sz) : pointer(ptr), size(sz) {}
|
||||
};
|
||||
|
||||
// Value encoding functions
|
||||
std::vector<uint8_t> encodeValue(const int32_t& value);
|
||||
std::vector<uint8_t> encodeValue(const float& value);
|
||||
std::vector<uint8_t> encodeValue(const std::string& value);
|
||||
std::vector<uint8_t> encodeValue(const std::vector<int32_t>& value);
|
||||
std::vector<uint8_t> encodeValue(const std::vector<float>& value);
|
||||
|
||||
// Template wrapper for encoding
|
||||
template<typename T>
|
||||
std::vector<uint8_t> encodeValue(const T& value) {
|
||||
return encodeValue(value);
|
||||
}
|
||||
|
||||
// Reference class for handling stored values
|
||||
class BT_Reference {
|
||||
protected:
|
||||
BinaryTable* table_;
|
||||
BT_Pointer pointer_;
|
||||
|
||||
public:
|
||||
BT_Reference(BinaryTable* table, BT_Pointer pointer);
|
||||
|
||||
template<typename T>
|
||||
T decodeValue();
|
||||
|
||||
int32_t size() const;
|
||||
BT_Type getType() const;
|
||||
bool isNull() const { return pointer_.isNull(); }
|
||||
BT_Pointer getPointer() const { return pointer_; }
|
||||
};
|
||||
|
||||
// Uniform array class template
|
||||
template<typename T>
|
||||
class BT_UniformArray : public BT_Reference {
|
||||
public:
|
||||
BT_UniformArray(BinaryTable* table, BT_Pointer pointer) : BT_Reference(table, pointer) {}
|
||||
|
||||
int32_t length() const;
|
||||
T operator[](int32_t index) const;
|
||||
void set(int32_t index, const T& value);
|
||||
void add(const T& value);
|
||||
void addAll(const std::vector<T>& values);
|
||||
std::vector<T> fetchSublist(int32_t start = 0, int32_t end = -1);
|
||||
};
|
||||
|
||||
// Main BinaryTable class
|
||||
class BinaryTable {
|
||||
private:
|
||||
FILE* file_;
|
||||
std::string filePath_;
|
||||
|
||||
// Free list management
|
||||
bool freeListLifted_;
|
||||
std::vector<BT_FreeListEntry> freeListCache_;
|
||||
|
||||
|
||||
// Internal methods
|
||||
std::unordered_map<int64_t, BT_Pointer> getAddressTable();
|
||||
void setAddressTable(const std::unordered_map<int64_t, BT_Pointer>& table);
|
||||
std::vector<BT_FreeListEntry> getFreeList();
|
||||
void setFreeList(const std::vector<BT_FreeListEntry>& list);
|
||||
int64_t hashString(const std::string& str) const;
|
||||
|
||||
void truncateFile(int64_t newSize);
|
||||
void antiFreeListScope(std::function<void()> fn);
|
||||
void free(BT_Pointer pointer, int32_t size);
|
||||
BT_Pointer alloc(int32_t size);
|
||||
|
||||
// File I/O helpers
|
||||
int32_t readInt32(int64_t position);
|
||||
float readFloat32(int64_t position);
|
||||
int64_t readInt64(int64_t position);
|
||||
uint8_t readByte(int64_t position);
|
||||
std::vector<uint8_t> readBytes(int64_t position, int32_t count);
|
||||
|
||||
void writeInt32(int64_t position, int32_t value);
|
||||
void writeFloat32(int64_t position, float value);
|
||||
void writeInt64(int64_t position, int64_t value);
|
||||
void writeByte(int64_t position, uint8_t value);
|
||||
void writeBytes(int64_t position, const std::vector<uint8_t>& data);
|
||||
|
||||
public:
|
||||
explicit BinaryTable(const std::string& path);
|
||||
~BinaryTable();
|
||||
|
||||
void initialize();
|
||||
|
||||
// Memory management
|
||||
void liftFreeList();
|
||||
void dropFreeList();
|
||||
|
||||
// Data operations
|
||||
template<typename T>
|
||||
void set(const std::string& key, const T& value);
|
||||
|
||||
template<typename T>
|
||||
T get(const std::string& key);
|
||||
|
||||
BT_Reference getReference(const std::string& key);
|
||||
|
||||
template<typename T>
|
||||
BT_UniformArray<T> getArray(const std::string& key);
|
||||
|
||||
void remove(const std::string& key);
|
||||
void truncate();
|
||||
|
||||
// Debug methods
|
||||
void debugAddressTable(const std::string& context = "");
|
||||
|
||||
// File access for reference classes
|
||||
friend class BT_Reference;
|
||||
template<typename T> friend class BT_UniformArray;
|
||||
|
||||
int64_t getFileLength();
|
||||
void setFilePosition(int64_t position);
|
||||
};
|
||||
|
||||
// Template specializations for decodeValue
|
||||
template<> int32_t BT_Reference::decodeValue<int32_t>();
|
||||
template<> float BT_Reference::decodeValue<float>();
|
||||
template<> std::string BT_Reference::decodeValue<std::string>();
|
||||
template<> std::vector<int32_t> BT_Reference::decodeValue<std::vector<int32_t>>();
|
||||
template<> std::vector<float> BT_Reference::decodeValue<std::vector<float>>();
|
||||
template<> BT_UniformArray<int32_t> BT_Reference::decodeValue<BT_UniformArray<int32_t>>();
|
||||
template<> BT_UniformArray<float> BT_Reference::decodeValue<BT_UniformArray<float>>();
|
||||
|
||||
// Template method implementations for BinaryTable
|
||||
template<typename T>
|
||||
void BinaryTable::set(const std::string& key, const T& value) {
|
||||
antiFreeListScope([&]() {
|
||||
auto addressTable = getAddressTable();
|
||||
int64_t keyHash = hashString(key);
|
||||
|
||||
if (addressTable.find(keyHash) != addressTable.end()) {
|
||||
throw std::runtime_error("Key already exists");
|
||||
}
|
||||
|
||||
auto valueBuffer = encodeValue(value);
|
||||
BT_Pointer valueAddress = alloc(static_cast<int32_t>(valueBuffer.size()));
|
||||
|
||||
writeBytes(valueAddress.address(), valueBuffer);
|
||||
|
||||
addressTable[keyHash] = valueAddress;
|
||||
setAddressTable(addressTable);
|
||||
});
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
T BinaryTable::get(const std::string& key) {
|
||||
auto addressTable = getAddressTable();
|
||||
int64_t keyHash = hashString(key);
|
||||
|
||||
auto it = addressTable.find(keyHash);
|
||||
if (it == addressTable.end()) {
|
||||
throw std::runtime_error("Key does not exist");
|
||||
}
|
||||
|
||||
BT_Reference valueRef(this, it->second);
|
||||
return valueRef.decodeValue<T>();
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
BT_UniformArray<T> BinaryTable::getArray(const std::string& key) {
|
||||
auto addressTable = getAddressTable();
|
||||
int64_t keyHash = hashString(key);
|
||||
|
||||
auto it = addressTable.find(keyHash);
|
||||
if (it == addressTable.end()) {
|
||||
throw std::runtime_error("Key does not exist");
|
||||
}
|
||||
|
||||
return BT_UniformArray<T>(this, it->second);
|
||||
}
|
||||
|
||||
} // namespace bt
|
||||
248
cpp/src/Private/sweepstore/concurrency.cpp
Normal file
248
cpp/src/Private/sweepstore/concurrency.cpp
Normal file
@@ -0,0 +1,248 @@
|
||||
|
||||
|
||||
#include <functional>
|
||||
#include <iosfwd>
|
||||
|
||||
#include "sweepstore/concurrency.h"
|
||||
|
||||
#include <iostream>
|
||||
#include <random>
|
||||
|
||||
#include "sweepstore/header.h"
|
||||
#include "sweepstore/utils/helpers.h"
|
||||
#include "sweepstore/utils/file_handle.h"
|
||||
|
||||
|
||||
uint64_t getRandomOffset(uint64_t maxValue) {
|
||||
static std::random_device rd;
|
||||
static std::mt19937_64 gen(rd());
|
||||
std::uniform_int_distribution<uint64_t> dist(0, maxValue);
|
||||
return dist(gen);
|
||||
}
|
||||
|
||||
int randomId() {
|
||||
// mix timestamp with random for better uniqueness
|
||||
// keep it positive to avoid signed int issues when storing
|
||||
auto now = std::chrono::system_clock::now();
|
||||
auto millis = std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count();
|
||||
int32_t time = static_cast<int32_t>(millis & 0xFFFFFFFF); // Get lower 32 bits
|
||||
int32_t random = static_cast<int32_t>(getRandomOffset(0x7FFFFFFF)); // 0 to 0x7FFFFFFF
|
||||
return (time ^ random) & 0x7FFFFFFF;
|
||||
}
|
||||
|
||||
void SweepstoreConcurrency::spawnTicket(std::string filePath,
|
||||
const SweepstoreTicketOperation& operation,
|
||||
const uint32_t keyHash,
|
||||
const uint32_t targetSize,
|
||||
const std::function<void()> onApproved,
|
||||
std::string debugLabel
|
||||
) {
|
||||
|
||||
SweepstoreFileHandle file(filePath, std::ios::binary | std::ios::in | std::ios::out);
|
||||
|
||||
/*
|
||||
Useful Functions
|
||||
*/
|
||||
|
||||
/// Logging function
|
||||
auto log = [&](const std::string &message) {
|
||||
std::string prefix = !debugLabel.empty() ? "\033[38;5;208m[Ticket Spawner - " + debugLabel + "]:\033[0m " : "\033[38;5;208m[Ticket Spawner]:\033[0m ";
|
||||
debugPrint(prefix + message);
|
||||
};
|
||||
|
||||
// Sleep with variance (additive only)
|
||||
auto varySleep = [&](std::chrono::nanoseconds minSleepDuration, std::chrono::nanoseconds variance) {
|
||||
if (variance.count() <= 0) {
|
||||
preciseSleep(minSleepDuration);
|
||||
} else {
|
||||
// Generate random duration within variance
|
||||
uint64_t randomOffset = getRandomOffset(variance.count());
|
||||
preciseSleep(minSleepDuration + std::chrono::nanoseconds(randomOffset));
|
||||
}
|
||||
};
|
||||
|
||||
// Exponential sleep
|
||||
std::unordered_map<std::string, int> expSleepTracker = {};
|
||||
auto expSleep = [&expSleepTracker](const std::string& label) {
|
||||
int count = expSleepTracker[label]; // defaults to 0 if not found
|
||||
int sleepTime = (1 << count); // Exponential backoff
|
||||
sleepTime = std::max(1, std::min(sleepTime, 1000)); // Clamp between 1ms and 1000ms
|
||||
preciseSleep(std::chrono::microseconds(sleepTime * 5000));
|
||||
expSleepTracker[label] = count + 1;
|
||||
};
|
||||
|
||||
// Get the header(s)
|
||||
SweepstoreHeader header(file);
|
||||
SweepstoreConcurrencyHeader concurrencyHeader(file);
|
||||
|
||||
/*
|
||||
Ticket Acquisition
|
||||
*/
|
||||
auto acquireTicket = [&](uint32_t newIdentifier) -> SweepstoreWorkerTicket {
|
||||
|
||||
// Reduce the chance of race condition
|
||||
varySleep(std::chrono::microseconds(500), std::chrono::microseconds(200));
|
||||
|
||||
uint32_t ticketIndex = -1u;
|
||||
|
||||
while (true) {
|
||||
|
||||
uint32_t concurrentWorkers = concurrencyHeader.readNumberOfWorkers();
|
||||
|
||||
for (uint32_t i = 0; i < concurrentWorkers; i++) {
|
||||
|
||||
SweepstoreWorkerTicket ticket = SweepstoreWorkerTicket(i, file);
|
||||
|
||||
if (!ticket.writable()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
SweepstoreWorkerTicketSnapshot snapshot = ticket.snapshot();
|
||||
|
||||
int identifier = snapshot.identifier;
|
||||
|
||||
bool identifier_unassigned = identifier == 0;
|
||||
bool stale_heartbeat = millisecondsSinceEpoch32() - snapshot.workerHeartbeat > STALE_HEARTBEAT_THRESHOLD_MS;
|
||||
bool is_free = snapshot.state == SweepstoreTicketState::FREE;
|
||||
|
||||
if (identifier_unassigned && stale_heartbeat && is_free) {
|
||||
snapshot.identifier = newIdentifier;
|
||||
snapshot.workerHeartbeat = millisecondsSinceEpoch32();
|
||||
snapshot.state = SweepstoreTicketState::WAITING;
|
||||
ticket.write(snapshot);
|
||||
ticketIndex = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
preciseSleep(std::chrono::milliseconds(2));
|
||||
|
||||
// Ensure we still own the ticket - if not, reset and try again
|
||||
if (ticketIndex != -1u) {
|
||||
SweepstoreWorkerTicketSnapshot verifySnapshot = concurrencyHeader[ticketIndex].snapshot();
|
||||
|
||||
if (verifySnapshot.identifier != newIdentifier) {
|
||||
ticketIndex = -1; // Lost the ticket, try again
|
||||
} else {
|
||||
log("Acquired ticket " + std::to_string(ticketIndex) + " with identifier " + std::to_string(newIdentifier) + ".");
|
||||
return concurrencyHeader[ticketIndex];
|
||||
}
|
||||
}
|
||||
|
||||
expSleep("acquireTicket");
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
|
||||
uint32_t myIdentifier = randomId();
|
||||
|
||||
SweepstoreWorkerTicket myTicket = acquireTicket(myIdentifier);
|
||||
SweepstoreWorkerTicketSnapshot mySnapshot = myTicket.snapshot();
|
||||
mySnapshot.workerHeartbeat = millisecondsSinceEpoch32();
|
||||
mySnapshot.state = SweepstoreTicketState::WAITING;
|
||||
mySnapshot.operation = operation;
|
||||
mySnapshot.keyHash = keyHash;
|
||||
mySnapshot.targetSize = targetSize;
|
||||
myTicket.write(mySnapshot);
|
||||
|
||||
// Wait for approval
|
||||
while (true) {
|
||||
|
||||
SweepstoreWorkerTicketSnapshot snapshot = myTicket.snapshot();
|
||||
|
||||
// Update heartbeat
|
||||
uint32_t currentTime = millisecondsSinceEpoch32();
|
||||
if (currentTime - snapshot.workerHeartbeat > 700) {
|
||||
snapshot.workerHeartbeat = currentTime;
|
||||
myTicket.write(snapshot);
|
||||
}
|
||||
|
||||
// Check if we still own the ticket
|
||||
if (snapshot.identifier != myIdentifier) {
|
||||
|
||||
preciseSleep(std::chrono::milliseconds(10));
|
||||
|
||||
// Re-verify we lost the ticket
|
||||
SweepstoreWorkerTicketSnapshot recheckSnapshot = myTicket.snapshot();
|
||||
if (recheckSnapshot.identifier != myIdentifier) {
|
||||
log("Lost ownership of ticket " + std::to_string(myTicket.getTicketIndex()) + ", was expecting identifier " + std::to_string(myIdentifier) + " but found " + std::to_string(recheckSnapshot.identifier) + ".");
|
||||
|
||||
// ReSharper disable once CppDFAInfiniteRecursion
|
||||
return spawnTicket(
|
||||
filePath,
|
||||
operation,
|
||||
keyHash,
|
||||
targetSize,
|
||||
onApproved,
|
||||
debugLabel
|
||||
);
|
||||
}
|
||||
|
||||
// False alarm, continue waiting
|
||||
log("False alarm, still own ticket " + std::to_string(myTicket.getTicketIndex()) + ".");
|
||||
snapshot = recheckSnapshot;
|
||||
}
|
||||
|
||||
if (snapshot.state == SweepstoreTicketState::APPROVED) {
|
||||
snapshot.state = SweepstoreTicketState::EXECUTING;
|
||||
myTicket.write(snapshot);
|
||||
|
||||
onApproved();
|
||||
|
||||
snapshot.state = SweepstoreTicketState::COMPLETED;
|
||||
myTicket.write(snapshot);
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
varySleep(std::chrono::microseconds(500), std::chrono::microseconds(200));
|
||||
}
|
||||
|
||||
// std::cout << "\033[38;5;82m[Ticket Spawner - " << debugLabel << "]:\033[0m Completed ticket " << myTicket.getTicketIndex() << "." << std::endl;
|
||||
}
|
||||
|
||||
void SweepstoreConcurrency::initialiseMaster(std::string filePath) {
|
||||
|
||||
auto log = [&](const std::string &message) {
|
||||
debugPrint("\033[38;5;33m[Concurrency Master]:\033[0m " + message);
|
||||
};
|
||||
|
||||
SweepstoreFileHandle file(filePath, std::ios::binary | std::ios::in | std::ios::out);
|
||||
|
||||
SweepstoreHeader header(file);
|
||||
SweepstoreConcurrencyHeader concurrencyHeader(file);
|
||||
|
||||
while (true) {
|
||||
|
||||
int concurrentWorkers = concurrencyHeader.readNumberOfWorkers();
|
||||
|
||||
for (uint32_t i = 0; i < concurrentWorkers; i++) {
|
||||
|
||||
SweepstoreWorkerTicket ticket(i, file);
|
||||
SweepstoreWorkerTicketSnapshot snapshot = ticket.snapshot();
|
||||
|
||||
if (snapshot.state == WAITING) {
|
||||
log("Found waiting ticket " + std::to_string(i) + "(Key Hash: " + std::to_string(snapshot.keyHash) + ")...");
|
||||
|
||||
// Approve the ticket
|
||||
snapshot.state = APPROVED;
|
||||
ticket.write(snapshot);
|
||||
log("Approved ticket " + std::to_string(i) + ".");
|
||||
} else if (snapshot.state == SweepstoreTicketState::COMPLETED) {
|
||||
log("Ticket " + std::to_string(i) + " has completed. Resetting...");
|
||||
|
||||
// Reset the ticket
|
||||
SweepstoreWorkerTicketSnapshot cleanSnapshot = SweepstoreWorkerTicketSnapshot();
|
||||
ticket.write(cleanSnapshot);
|
||||
log("Reset ticket " + std::to_string(i) + ".");
|
||||
}
|
||||
|
||||
// Handle stale tickets
|
||||
uint32_t currentTime = millisecondsSinceEpoch32();
|
||||
}
|
||||
|
||||
preciseSleep(std::chrono::milliseconds(1));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
220
cpp/src/Private/sweepstore/header.cpp
Normal file
220
cpp/src/Private/sweepstore/header.cpp
Normal file
@@ -0,0 +1,220 @@
|
||||
|
||||
#include "sweepstore/header.h"
|
||||
|
||||
#include "sweepstore/utils/file_lock.h"
|
||||
#include "sweepstore/utils/helpers.h"
|
||||
|
||||
std::string SweepstoreHeader::readMagicNumber() {
|
||||
file->seekg(0, std::ios::beg);
|
||||
char buffer[4];
|
||||
file->read(buffer, 4);
|
||||
return std::string(buffer, 4);
|
||||
}
|
||||
|
||||
void SweepstoreHeader::writeMagicNumber(const std::string& magicNumber) {
|
||||
if (magicNumber.size() != 4) {
|
||||
throw std::invalid_argument("Magic number must be exactly 4 characters long.");
|
||||
}
|
||||
file->seekp(0, std::ios::beg);
|
||||
file->write(magicNumber.c_str(), 4);
|
||||
}
|
||||
|
||||
std::string SweepstoreHeader::readVersion() {
|
||||
file->seekg(4, std::ios::beg);
|
||||
char buffer[12];
|
||||
file->read(buffer, 12);
|
||||
|
||||
// Trim leading and trailing spaces
|
||||
std::string version(buffer, 12);
|
||||
version = trim(version);
|
||||
|
||||
return version;
|
||||
}
|
||||
|
||||
void SweepstoreHeader::writeVersion(const std::string& version) {
|
||||
if (version.size() > 11) {
|
||||
throw std::invalid_argument("Version string must be at most 11 characters long.");
|
||||
}
|
||||
// Pad 1 space to the beginning
|
||||
// And pad to end to make it 12 bytes
|
||||
std::string paddedVersion = " " + version;
|
||||
paddedVersion.resize(12, ' ');
|
||||
|
||||
file->seekp(4, std::ios::beg);
|
||||
file->write(paddedVersion.c_str(), 12);
|
||||
}
|
||||
|
||||
SweepstorePointer SweepstoreHeader::readAddressTablePointer() {
|
||||
file->seekg(16, std::ios::beg);
|
||||
int64_t address;
|
||||
file->read(reinterpret_cast<char*>(&address), sizeof(address));
|
||||
return address; // Implicit conversion to SweepstorePointer
|
||||
}
|
||||
|
||||
void SweepstoreHeader::writeAddressTablePointer(const SweepstorePointer& ptr) {
|
||||
file->seekp(16, std::ios::beg);
|
||||
int64_t address = ptr;
|
||||
file->write(reinterpret_cast<const char*>(&address), sizeof(address));
|
||||
}
|
||||
|
||||
uint32_t SweepstoreHeader::readFreeListCount() {
|
||||
file->seekg(24, std::ios::beg);
|
||||
uint32_t count;
|
||||
file->read(reinterpret_cast<char*>(&count), sizeof(count));
|
||||
return count;
|
||||
}
|
||||
|
||||
void SweepstoreHeader::writeFreeListCount(uint32_t count) {
|
||||
file->seekp(24, std::ios::beg);
|
||||
file->write(reinterpret_cast<const char*>(&count), sizeof(count));
|
||||
}
|
||||
|
||||
bool SweepstoreHeader::readIsFreeListLifted() {
|
||||
file->seekg(28, std::ios::beg);
|
||||
char flag;
|
||||
file->read(&flag, sizeof(flag));
|
||||
return flag != 0;
|
||||
}
|
||||
|
||||
void SweepstoreHeader::writeIsFreeListLifted(bool isLifted) {
|
||||
file->seekp(28, std::ios::beg);
|
||||
char flag = isLifted ? 1 : 0;
|
||||
file->write(&flag, sizeof(flag));
|
||||
}
|
||||
|
||||
void SweepstoreHeader::initialise() {
|
||||
writeMagicNumber("SWPT");
|
||||
writeVersion("undefined");
|
||||
writeAddressTablePointer(SweepstorePointer::NULL_PTR);
|
||||
writeFreeListCount(0);
|
||||
writeIsFreeListLifted(false);
|
||||
file->flush();
|
||||
}
|
||||
|
||||
uint64_t SweepstoreConcurrencyHeader::readMasterIdentifier() {
|
||||
file->seekg(29, std::ios::beg);
|
||||
uint64_t identifier;
|
||||
file->read(reinterpret_cast<char*>(&identifier), sizeof(identifier));
|
||||
return identifier;
|
||||
}
|
||||
|
||||
void SweepstoreConcurrencyHeader::writeMasterIdentifier(uint64_t identifier) {
|
||||
file->seekp(29, std::ios::beg);
|
||||
file->write(reinterpret_cast<const char*>(&identifier), sizeof(identifier));
|
||||
}
|
||||
|
||||
uint32_t SweepstoreConcurrencyHeader::readMasterHeartbeat() {
|
||||
file->seekg(37, std::ios::beg);
|
||||
uint32_t heartbeat;
|
||||
file->read(reinterpret_cast<char*>(&heartbeat), sizeof(heartbeat));
|
||||
return heartbeat;
|
||||
}
|
||||
|
||||
void SweepstoreConcurrencyHeader::writeMasterHeartbeat(uint32_t heartbeat) {
|
||||
file->seekp(37, std::ios::beg);
|
||||
file->write(reinterpret_cast<const char*>(&heartbeat), sizeof(heartbeat));
|
||||
}
|
||||
|
||||
uint32_t SweepstoreConcurrencyHeader::readNumberOfWorkers() {
|
||||
file->seekg(41, std::ios::beg);
|
||||
uint32_t numWorkers;
|
||||
file->read(reinterpret_cast<char*>(&numWorkers), sizeof(numWorkers));
|
||||
return numWorkers;
|
||||
}
|
||||
|
||||
void SweepstoreConcurrencyHeader::writeNumberOfWorkers(uint32_t numWorkers) {
|
||||
file->seekp(41, std::ios::beg);
|
||||
file->write(reinterpret_cast<const char*>(&numWorkers), sizeof(numWorkers));
|
||||
}
|
||||
|
||||
bool SweepstoreConcurrencyHeader::readIsReadAllowed() {
|
||||
file->seekg(45, std::ios::beg);
|
||||
char flag;
|
||||
file->read(&flag, sizeof(flag));
|
||||
return flag != 0;
|
||||
}
|
||||
|
||||
void SweepstoreConcurrencyHeader::writeIsReadAllowed(bool isAllowed) {
|
||||
file->seekp(45, std::ios::beg);
|
||||
char flag = isAllowed ? 1 : 0;
|
||||
file->write(&flag, sizeof(flag));
|
||||
}
|
||||
|
||||
void SweepstoreConcurrencyHeader::initialise(int concurrentWorkers) {
|
||||
writeMasterIdentifier(0);
|
||||
writeMasterHeartbeat(0);
|
||||
writeNumberOfWorkers(concurrentWorkers);
|
||||
writeIsReadAllowed(true);
|
||||
for (uint32_t i = 0; i < readNumberOfWorkers(); i++) {
|
||||
SweepstoreWorkerTicketSnapshot ticket = SweepstoreWorkerTicketSnapshot();
|
||||
ticket.identifier = 0;
|
||||
ticket.workerHeartbeat = 0;
|
||||
ticket.state = SweepstoreTicketState::FREE;
|
||||
ticket.operation = SweepstoreTicketOperation::NONE;
|
||||
ticket.keyHash = 0;
|
||||
ticket.targetAddress = SweepstorePointer::NULL_PTR;
|
||||
ticket.targetSize = 0;
|
||||
|
||||
SweepstoreWorkerTicket ticketWriter = SweepstoreWorkerTicket(i, file);
|
||||
ticketWriter.write(ticket);
|
||||
}
|
||||
file->flush();
|
||||
}
|
||||
|
||||
void SweepstoreWorkerTicket::write(SweepstoreWorkerTicketSnapshot &snapshot) {
|
||||
RandomAccessMemory buffer;
|
||||
|
||||
SweepstoreFileLock lock(file.getPath(), SweepstoreFileLock::Mode::Exclusive);
|
||||
SweepstoreFileLock::Scoped scopedLock(lock);
|
||||
|
||||
buffer.setPositionSync(0);
|
||||
buffer.writeIntSync(snapshot.identifier, 4);
|
||||
buffer.writeIntSync(snapshot.workerHeartbeat, 4);
|
||||
buffer.writeIntSync(static_cast<uint8_t>(snapshot.state), 1);
|
||||
buffer.writeIntSync(static_cast<uint8_t>(snapshot.operation), 1);
|
||||
buffer.writeUIntSync(snapshot.keyHash, 8);
|
||||
buffer.writePointerSync(snapshot.targetAddress, 8);
|
||||
buffer.writeUIntSync(snapshot.targetSize, 4);
|
||||
|
||||
// Pad the rest with zeros if necessary
|
||||
while (buffer.length() < TICKET_SIZE) {
|
||||
buffer.writeIntSync(0, 1);
|
||||
}
|
||||
|
||||
// Prepare data
|
||||
buffer.setPositionSync(0);
|
||||
std::vector<uint8_t> data = buffer.readSync(buffer.length());
|
||||
char* dataPtr = reinterpret_cast<char*>(data.data());
|
||||
|
||||
// Write to file
|
||||
file->seekp(getOffset());
|
||||
file->write(dataPtr, data.size());
|
||||
file->flush();
|
||||
}
|
||||
|
||||
bool SweepstoreWorkerTicket::writable() {
|
||||
SweepstoreFileLock lock(file.getPath(), SweepstoreFileLock::Mode::Exclusive);
|
||||
return lock.isLocked() == false;
|
||||
}
|
||||
|
||||
SweepstoreWorkerTicketSnapshot SweepstoreWorkerTicket::snapshot() {
|
||||
SweepstoreFileLock lock(file.getPath(), SweepstoreFileLock::Mode::Shared);
|
||||
lock.lock();
|
||||
file->seekg(getOffset());
|
||||
std::unique_ptr<char[]> buffer(new char[TICKET_SIZE]);
|
||||
file->read(buffer.get(), TICKET_SIZE);
|
||||
lock.unlock();
|
||||
RandomAccessMemory ram(reinterpret_cast<uint8_t*>(buffer.get()), TICKET_SIZE);
|
||||
|
||||
SweepstoreWorkerTicketSnapshot snapshot;
|
||||
ram.setPositionSync(0);
|
||||
snapshot.identifier = ram.readUIntSync(4);
|
||||
snapshot.workerHeartbeat = ram.readUIntSync(4);
|
||||
snapshot.state = static_cast<SweepstoreTicketState>(ram.readUIntSync(1));
|
||||
snapshot.operation = static_cast<SweepstoreTicketOperation>(ram.readUIntSync(1));
|
||||
snapshot.keyHash = ram.readUIntSync(8);
|
||||
snapshot.targetAddress = ram.readPointerSync(8);
|
||||
snapshot.targetSize = ram.readUIntSync(4);
|
||||
|
||||
return snapshot;
|
||||
}
|
||||
13
cpp/src/Private/sweepstore/structures.cpp
Normal file
13
cpp/src/Private/sweepstore/structures.cpp
Normal file
@@ -0,0 +1,13 @@
|
||||
|
||||
|
||||
|
||||
#include "sweepstore/structures.h"
|
||||
const SweepstorePointer SweepstorePointer::NULL_PTR = SweepstorePointer(UINT64_MAX);
|
||||
|
||||
bool SweepstorePointer::operator==(const SweepstorePointer &p) {
|
||||
if (this->address == p.address) {
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
21
cpp/src/Private/sweepstore/sweepstore.cpp
Normal file
21
cpp/src/Private/sweepstore/sweepstore.cpp
Normal file
@@ -0,0 +1,21 @@
|
||||
//
|
||||
// Created by Benjamin Watt on 24/11/2025.
|
||||
//
|
||||
|
||||
#include "sweepstore/sweepstore.h"
|
||||
|
||||
#include <string>
|
||||
#include <filesystem>
|
||||
#include <iostream>
|
||||
#include <thread>
|
||||
|
||||
#include "sweepstore/utils/helpers.h"
|
||||
#include "sweepstore/utils/file_handle.h"
|
||||
|
||||
void Sweepstore::initialise(int concurrentWorkers) {
|
||||
header->initialise();
|
||||
header->writeVersion("1.1.0.2");
|
||||
concurrencyHeader->initialise(concurrentWorkers);
|
||||
|
||||
debugPrint("Version: " + header->readVersion());
|
||||
}
|
||||
30
cpp/src/Public/sweepstore/concurrency.h
Normal file
30
cpp/src/Public/sweepstore/concurrency.h
Normal file
@@ -0,0 +1,30 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <iosfwd>
|
||||
#include <thread>
|
||||
|
||||
#define STALE_HEARTBEAT_THRESHOLD_MS 5000
|
||||
|
||||
enum SweepstoreTicketOperation : int;
|
||||
|
||||
namespace SweepstoreConcurrency {
|
||||
|
||||
void spawnTicket(std::string filePath,
|
||||
const SweepstoreTicketOperation& operation,
|
||||
const uint32_t keyHash,
|
||||
const uint32_t targetSize,
|
||||
const std::function<void()> onApproved,
|
||||
std::string debugLabel = ""
|
||||
);
|
||||
|
||||
void initialiseMaster(std::string filePath);
|
||||
|
||||
inline void initialiseMasterAsync(std::string filePath) {
|
||||
std::thread([&filePath]() {
|
||||
initialiseMaster(filePath);
|
||||
}).detach();
|
||||
}
|
||||
|
||||
}
|
||||
143
cpp/src/Public/sweepstore/header.h
Normal file
143
cpp/src/Public/sweepstore/header.h
Normal file
@@ -0,0 +1,143 @@
|
||||
#pragma once
|
||||
|
||||
#include <iosfwd>
|
||||
|
||||
#include "structures.h"
|
||||
#include "utils/file_handle.h"
|
||||
|
||||
constexpr int roundToNearest16(int number) {
|
||||
return (number + 15) & ~15;
|
||||
}
|
||||
|
||||
class SweepstoreHeader {
|
||||
|
||||
private:
|
||||
SweepstoreFileHandle& file;
|
||||
|
||||
public:
|
||||
explicit SweepstoreHeader(SweepstoreFileHandle &fileStream) : file(fileStream) {}
|
||||
|
||||
// Offset 0 - 4 bytes
|
||||
std::string readMagicNumber();
|
||||
void writeMagicNumber(const std::string& magicNumber);
|
||||
|
||||
// Offset 4 - 12 bytes
|
||||
std::string readVersion();
|
||||
void writeVersion(const std::string& version);
|
||||
|
||||
// Offset 16 - 8 bytes
|
||||
SweepstorePointer readAddressTablePointer();
|
||||
void writeAddressTablePointer(const SweepstorePointer& ptr);
|
||||
|
||||
// Offset 24 - 4 bytes
|
||||
uint32_t readFreeListCount();
|
||||
void writeFreeListCount(uint32_t count);
|
||||
|
||||
// Offset 28 - 1 byte
|
||||
bool readIsFreeListLifted();
|
||||
void writeIsFreeListLifted(bool isLifted);
|
||||
|
||||
/**
|
||||
* Initialises the header with default values.
|
||||
*/
|
||||
void initialise();
|
||||
|
||||
};
|
||||
|
||||
constexpr int SWEEPSTORE_COMBINED_STATIC_HEADER_SIZE = roundToNearest16(46);
|
||||
|
||||
struct SweepstoreWorkerTicketSnapshot {
|
||||
|
||||
SweepstoreWorkerTicketSnapshot() :
|
||||
identifier(0),
|
||||
workerHeartbeat(0),
|
||||
state(SweepstoreTicketState::FREE),
|
||||
operation(SweepstoreTicketOperation::NONE),
|
||||
keyHash(0),
|
||||
targetAddress(SweepstorePointer::NULL_PTR),
|
||||
targetSize(0) {}
|
||||
|
||||
// Offset 0 - 4 bytes
|
||||
uint32_t identifier;
|
||||
|
||||
// Offset 4 - 4 bytes
|
||||
uint32_t workerHeartbeat;
|
||||
|
||||
// Offset 8 - 1 byte
|
||||
SweepstoreTicketState state;
|
||||
|
||||
// Offset 9 - 1 byte
|
||||
SweepstoreTicketOperation operation;
|
||||
|
||||
// Offset 10 - 8 bytes
|
||||
uint64_t keyHash;
|
||||
|
||||
// Offset 18 - 8 bytes
|
||||
SweepstorePointer targetAddress;
|
||||
|
||||
// Offset 26 - 4 bytes
|
||||
uint32_t targetSize;
|
||||
};
|
||||
|
||||
class SweepstoreWorkerTicket {
|
||||
|
||||
SweepstoreFileHandle& file;
|
||||
uint32_t ticketIndex;
|
||||
|
||||
uint64_t getOffset() const {
|
||||
return SWEEPSTORE_COMBINED_STATIC_HEADER_SIZE + (ticketIndex * TICKET_SIZE);
|
||||
}
|
||||
|
||||
public:
|
||||
static constexpr int TICKET_SIZE = roundToNearest16(29);
|
||||
|
||||
SweepstoreWorkerTicket(const uint32_t index, SweepstoreFileHandle& fileStream) :
|
||||
file(fileStream),
|
||||
ticketIndex(index) {}
|
||||
|
||||
int getTicketIndex() const {
|
||||
return ticketIndex;
|
||||
}
|
||||
|
||||
void write(SweepstoreWorkerTicketSnapshot &snapshot);
|
||||
|
||||
bool writable();
|
||||
|
||||
SweepstoreWorkerTicketSnapshot snapshot();
|
||||
|
||||
};
|
||||
|
||||
class SweepstoreConcurrencyHeader {
|
||||
|
||||
private:
|
||||
SweepstoreFileHandle& file;
|
||||
|
||||
public:
|
||||
explicit SweepstoreConcurrencyHeader(SweepstoreFileHandle &fileStream) : file(fileStream) {}
|
||||
|
||||
// Offset 29 - 8 bytes
|
||||
uint64_t readMasterIdentifier();
|
||||
void writeMasterIdentifier(uint64_t identifier);
|
||||
|
||||
// Offset 37 - 4 bytes
|
||||
uint32_t readMasterHeartbeat();
|
||||
void writeMasterHeartbeat(uint32_t heartbeat);
|
||||
|
||||
// Offset 41 - 4 bytes
|
||||
uint32_t readNumberOfWorkers();
|
||||
void writeNumberOfWorkers(uint32_t numWorkers);
|
||||
|
||||
// Offset 45 - 1 byte
|
||||
bool readIsReadAllowed();
|
||||
void writeIsReadAllowed(bool isAllowed);
|
||||
|
||||
/**
|
||||
* Initialises the concurrency header with default values.
|
||||
*/
|
||||
void initialise(int concurrentWorkers);
|
||||
|
||||
SweepstoreWorkerTicket operator[](const uint32_t index) const {
|
||||
return SweepstoreWorkerTicket(index, file);
|
||||
}
|
||||
|
||||
};
|
||||
39
cpp/src/Public/sweepstore/structures.h
Normal file
39
cpp/src/Public/sweepstore/structures.h
Normal file
@@ -0,0 +1,39 @@
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
|
||||
class SweepstorePointer {
|
||||
|
||||
private:
|
||||
uint64_t address;
|
||||
|
||||
public:
|
||||
static const SweepstorePointer NULL_PTR;
|
||||
|
||||
SweepstorePointer(int64_t addr) : address(addr) {}
|
||||
|
||||
bool isNull() {
|
||||
return this->address == UINT64_MAX;
|
||||
}
|
||||
|
||||
bool operator==(const SweepstorePointer &p);
|
||||
|
||||
// Implicit conversion to uint64_t
|
||||
operator uint64_t() const {
|
||||
return address;
|
||||
}
|
||||
};
|
||||
|
||||
enum SweepstoreTicketState {
|
||||
FREE,
|
||||
WAITING,
|
||||
APPROVED,
|
||||
EXECUTING,
|
||||
COMPLETED,
|
||||
};
|
||||
|
||||
enum SweepstoreTicketOperation : int {
|
||||
NONE,
|
||||
READ,
|
||||
MODIFY,
|
||||
WRITE
|
||||
};
|
||||
67
cpp/src/Public/sweepstore/sweepstore.h
Normal file
67
cpp/src/Public/sweepstore/sweepstore.h
Normal file
@@ -0,0 +1,67 @@
|
||||
#pragma once
|
||||
|
||||
#include <iosfwd>
|
||||
|
||||
#include "concurrency.h"
|
||||
#include "header.h"
|
||||
#include "utils/helpers.h"
|
||||
#include "utils/file_handle.h"
|
||||
|
||||
class Sweepstore {
|
||||
|
||||
private:
|
||||
std::string filePath;
|
||||
SweepstoreFileHandle file;
|
||||
SweepstoreHeader* header;
|
||||
SweepstoreConcurrencyHeader* concurrencyHeader;
|
||||
|
||||
public:
|
||||
|
||||
Sweepstore(const std::string& filePath) : filePath(filePath), file(filePath, std::ios::binary | std::ios::in | std::ios::out | std::ios::trunc) {
|
||||
header = new SweepstoreHeader(file);
|
||||
concurrencyHeader = new SweepstoreConcurrencyHeader(file);
|
||||
}
|
||||
|
||||
~Sweepstore() {
|
||||
delete header;
|
||||
delete concurrencyHeader;
|
||||
file.close();
|
||||
}
|
||||
|
||||
void initialise(int concurrentWorkers = 4);
|
||||
|
||||
SweepstoreConcurrencyHeader* getConcurrencyHeader() {
|
||||
return concurrencyHeader;
|
||||
}
|
||||
|
||||
class Proxy {
|
||||
Sweepstore* sweepstore;
|
||||
std::string key;
|
||||
public:
|
||||
Proxy(Sweepstore* sweepstoreIn, const std::string& keyIn)
|
||||
: sweepstore(sweepstoreIn), key(keyIn) {}
|
||||
|
||||
template <typename T>
|
||||
operator T() {
|
||||
// Get value from sweepstore
|
||||
throw std::runtime_error("Not implemented: Reading values from Sweepstore by key is not implemented.");
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void operator=(const T& value) {
|
||||
SweepstoreConcurrency::spawnTicket(sweepstore->filePath,
|
||||
SweepstoreTicketOperation::WRITE,
|
||||
bt_hash(key),
|
||||
sizeof(T),
|
||||
[this, key = this->key, &value]() {
|
||||
|
||||
}
|
||||
);
|
||||
};
|
||||
};
|
||||
|
||||
Proxy operator[](const std::string& key) {
|
||||
return Proxy(this, key);
|
||||
}
|
||||
|
||||
};
|
||||
49
cpp/src/Public/sweepstore/utils/file_handle.h
Normal file
49
cpp/src/Public/sweepstore/utils/file_handle.h
Normal file
@@ -0,0 +1,49 @@
|
||||
#pragma once
|
||||
|
||||
#include <fstream>
|
||||
#include <string>
|
||||
#include <memory>
|
||||
|
||||
class SweepstoreFileHandle {
|
||||
private:
|
||||
std::string path;
|
||||
std::unique_ptr<std::fstream> stream;
|
||||
|
||||
public:
|
||||
SweepstoreFileHandle(const std::string& p, std::ios::openmode mode = std::ios::in | std::ios::out | std::ios::binary)
|
||||
: path(p), stream(std::make_unique<std::fstream>(p, mode)) {
|
||||
if (!stream->is_open()) {
|
||||
throw std::runtime_error("Failed to open file: " + path);
|
||||
}
|
||||
}
|
||||
|
||||
const std::string& getPath() const { return path; }
|
||||
|
||||
std::fstream& getStream() { return *stream; }
|
||||
const std::fstream& getStream() const { return *stream; }
|
||||
|
||||
// Smart pointer-like interface
|
||||
std::fstream* operator->() { return stream.get(); }
|
||||
const std::fstream* operator->() const { return stream.get(); }
|
||||
|
||||
std::fstream& operator*() { return *stream; }
|
||||
const std::fstream& operator*() const { return *stream; }
|
||||
|
||||
bool isOpen() const { return stream && stream->is_open(); }
|
||||
|
||||
void close() {
|
||||
if (stream) {
|
||||
stream->close();
|
||||
}
|
||||
}
|
||||
|
||||
SweepstoreFileHandle(SweepstoreFileHandle&&) noexcept = default;
|
||||
SweepstoreFileHandle& operator=(SweepstoreFileHandle&&) noexcept = default;
|
||||
|
||||
SweepstoreFileHandle(const SweepstoreFileHandle&) = delete;
|
||||
SweepstoreFileHandle& operator=(const SweepstoreFileHandle&) = delete;
|
||||
|
||||
~SweepstoreFileHandle() {
|
||||
close();
|
||||
}
|
||||
};
|
||||
169
cpp/src/Public/sweepstore/utils/file_lock.h
Normal file
169
cpp/src/Public/sweepstore/utils/file_lock.h
Normal file
@@ -0,0 +1,169 @@
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
#ifdef _WIN32
|
||||
#include <windows.h>
|
||||
#else
|
||||
#include <sys/file.h>
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
class SweepstoreFileLock {
|
||||
public:
|
||||
enum class Mode {
|
||||
Shared,
|
||||
Exclusive
|
||||
};
|
||||
|
||||
private:
|
||||
#ifdef _WIN32
|
||||
HANDLE handle;
|
||||
OVERLAPPED overlapped;
|
||||
#else
|
||||
int fd;
|
||||
#endif
|
||||
std::string path;
|
||||
Mode mode;
|
||||
bool locked;
|
||||
|
||||
void acquire() {
|
||||
#ifdef _WIN32
|
||||
handle = CreateFileA(path.c_str(), GENERIC_READ | GENERIC_WRITE,
|
||||
FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING,
|
||||
FILE_ATTRIBUTE_NORMAL, NULL);
|
||||
|
||||
if (handle == INVALID_HANDLE_VALUE) {
|
||||
throw std::runtime_error("Failed to open file");
|
||||
}
|
||||
|
||||
memset(&overlapped, 0, sizeof(overlapped));
|
||||
DWORD flags = (mode == Mode::Exclusive) ? LOCKFILE_EXCLUSIVE_LOCK : 0;
|
||||
|
||||
if (!LockFileEx(handle, flags, 0, MAXDWORD, MAXDWORD, &overlapped)) {
|
||||
CloseHandle(handle);
|
||||
throw std::runtime_error("Failed to lock");
|
||||
}
|
||||
#else
|
||||
fd = open(path.c_str(), O_RDWR);
|
||||
if (fd == -1) throw std::runtime_error("Failed to open file");
|
||||
|
||||
int op = (mode == Mode::Exclusive) ? LOCK_EX : LOCK_SH;
|
||||
if (flock(fd, op) != 0) {
|
||||
close(fd);
|
||||
throw std::runtime_error("Failed to lock");
|
||||
}
|
||||
#endif
|
||||
locked = true;
|
||||
}
|
||||
|
||||
void release() {
|
||||
if (locked) {
|
||||
#ifdef _WIN32
|
||||
UnlockFileEx(handle, 0, MAXDWORD, MAXDWORD, &overlapped);
|
||||
CloseHandle(handle);
|
||||
#else
|
||||
flock(fd, LOCK_UN);
|
||||
close(fd);
|
||||
#endif
|
||||
locked = false;
|
||||
}
|
||||
}
|
||||
|
||||
public:
|
||||
SweepstoreFileLock(const std::string& p, Mode m)
|
||||
: path(p), mode(m), locked(false) {
|
||||
#ifdef _WIN32
|
||||
handle = INVALID_HANDLE_VALUE;
|
||||
memset(&overlapped, 0, sizeof(overlapped));
|
||||
#else
|
||||
fd = -1;
|
||||
#endif
|
||||
}
|
||||
|
||||
~SweepstoreFileLock() {
|
||||
release();
|
||||
}
|
||||
|
||||
void lock() {
|
||||
if (!locked) {
|
||||
acquire();
|
||||
}
|
||||
}
|
||||
|
||||
void unlock() {
|
||||
release();
|
||||
}
|
||||
|
||||
// Check if THIS instance holds the lock
|
||||
bool holdsLock() const { return locked; }
|
||||
|
||||
// Check if the file is locked by ANYONE (including this instance)
|
||||
bool isLocked() const {
|
||||
#ifdef _WIN32
|
||||
HANDLE testHandle = CreateFileA(path.c_str(), GENERIC_READ | GENERIC_WRITE,
|
||||
FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING,
|
||||
FILE_ATTRIBUTE_NORMAL, NULL);
|
||||
|
||||
if (testHandle == INVALID_HANDLE_VALUE) {
|
||||
return false; // Can't even open file
|
||||
}
|
||||
|
||||
OVERLAPPED testOverlapped;
|
||||
memset(&testOverlapped, 0, sizeof(testOverlapped));
|
||||
DWORD flags = (mode == Mode::Exclusive) ? LOCKFILE_EXCLUSIVE_LOCK : 0;
|
||||
flags |= LOCKFILE_FAIL_IMMEDIATELY; // Non-blocking
|
||||
|
||||
bool isLocked = !LockFileEx(testHandle, flags, 0, MAXDWORD, MAXDWORD, &testOverlapped);
|
||||
|
||||
if (!isLocked) {
|
||||
// We got the lock, release it
|
||||
UnlockFileEx(testHandle, 0, MAXDWORD, MAXDWORD, &testOverlapped);
|
||||
}
|
||||
|
||||
CloseHandle(testHandle);
|
||||
return isLocked;
|
||||
#else
|
||||
int testFd = open(path.c_str(), O_RDWR);
|
||||
if (testFd == -1) {
|
||||
return false; // Can't open file
|
||||
}
|
||||
|
||||
int op = (mode == Mode::Exclusive) ? LOCK_EX : LOCK_SH;
|
||||
op |= LOCK_NB; // Non-blocking
|
||||
|
||||
bool isLocked = (flock(testFd, op) != 0);
|
||||
|
||||
if (!isLocked) {
|
||||
// We got the lock, release it
|
||||
flock(testFd, LOCK_UN);
|
||||
}
|
||||
|
||||
close(testFd);
|
||||
return isLocked;
|
||||
#endif
|
||||
}
|
||||
|
||||
SweepstoreFileLock(const SweepstoreFileLock&) = delete;
|
||||
SweepstoreFileLock& operator=(const SweepstoreFileLock&) = delete;
|
||||
SweepstoreFileLock(SweepstoreFileLock&&) = default;
|
||||
SweepstoreFileLock& operator=(SweepstoreFileLock&&) = default;
|
||||
|
||||
class Scoped {
|
||||
private:
|
||||
SweepstoreFileLock& lock;
|
||||
|
||||
public:
|
||||
Scoped(SweepstoreFileLock& l) : lock(l) {
|
||||
lock.lock();
|
||||
}
|
||||
|
||||
~Scoped() {
|
||||
lock.unlock();
|
||||
}
|
||||
|
||||
Scoped(const Scoped&) = delete;
|
||||
Scoped& operator=(const Scoped&) = delete;
|
||||
};
|
||||
};
|
||||
381
cpp/src/Public/sweepstore/utils/helpers.h
Normal file
381
cpp/src/Public/sweepstore/utils/helpers.h
Normal file
@@ -0,0 +1,381 @@
|
||||
#pragma once
|
||||
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <sstream>
|
||||
#include <iomanip>
|
||||
#include <vector>
|
||||
#include <cstdint>
|
||||
#include <fstream>
|
||||
#include <chrono>
|
||||
#include <thread>
|
||||
|
||||
// Toggleable debug printing via preprocessor
|
||||
#ifndef SWEEPSTORE_DEBUG
|
||||
#define SWEEPSTORE_DEBUG 0
|
||||
#endif
|
||||
|
||||
#if SWEEPSTORE_DEBUG
|
||||
#define debugPrint(msg) std::cout << msg << std::endl
|
||||
#else
|
||||
#define debugPrint(msg) ((void)0)
|
||||
#endif
|
||||
|
||||
|
||||
inline void print(const char* message) {
|
||||
// Print the message to the console
|
||||
std::cout << message << std::endl;
|
||||
}
|
||||
|
||||
inline std::string trim(const std::string& str) {
|
||||
size_t start = str.find_first_not_of(" \t\n\r");
|
||||
if (start == std::string::npos) return ""; // all whitespace
|
||||
|
||||
size_t end = str.find_last_not_of(" \t\n\r");
|
||||
return str.substr(start, end - start + 1);
|
||||
}
|
||||
|
||||
inline std::string binaryDump(const std::vector<uint8_t>& data) {
|
||||
std::ostringstream buffer;
|
||||
|
||||
for (size_t i = 0; i < data.size(); i += 16) {
|
||||
// Address
|
||||
buffer << "0x"
|
||||
<< std::setfill('0') << std::setw(4) << std::uppercase << std::hex << i
|
||||
<< " (" << std::dec << std::setw(4) << i << ") | ";
|
||||
|
||||
// Hex bytes
|
||||
for (size_t j = 0; j < 16; j++) {
|
||||
if (i + j < data.size()) {
|
||||
buffer << std::setfill('0') << std::setw(2) << std::uppercase << std::hex
|
||||
<< static_cast<int>(data[i + j]) << " ";
|
||||
} else {
|
||||
buffer << " ";
|
||||
}
|
||||
}
|
||||
|
||||
buffer << " | ";
|
||||
|
||||
// Integer representation
|
||||
for (size_t j = 0; j < 16; j++) {
|
||||
if (i + j < data.size()) {
|
||||
buffer << std::dec << std::setw(3) << static_cast<int>(data[i + j]) << " ";
|
||||
} else {
|
||||
buffer << " ";
|
||||
}
|
||||
}
|
||||
|
||||
buffer << " | ";
|
||||
|
||||
// ASCII representation
|
||||
for (size_t j = 0; j < 16; j++) {
|
||||
if (i + j < data.size()) {
|
||||
uint8_t byte = data[i + j];
|
||||
if (byte >= 32 && byte <= 126) {
|
||||
buffer << static_cast<char>(byte);
|
||||
} else {
|
||||
buffer << '.';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
buffer << " | ";
|
||||
if (i + 16 < data.size()) buffer << '\n';
|
||||
}
|
||||
|
||||
return buffer.str();
|
||||
}
|
||||
|
||||
inline std::vector<uint8_t> loadFile(const std::string& filename) {
|
||||
std::ifstream file(filename, std::ios::binary | std::ios::ate);
|
||||
|
||||
if (!file) {
|
||||
throw std::runtime_error("Failed to open file: " + filename);
|
||||
}
|
||||
|
||||
// Get file size
|
||||
std::streamsize size = file.tellg();
|
||||
file.seekg(0, std::ios::beg);
|
||||
|
||||
// Pre-allocate vector and read
|
||||
std::vector<uint8_t> buffer(size);
|
||||
if (!file.read(reinterpret_cast<char*>(buffer.data()), size)) {
|
||||
throw std::runtime_error("Failed to read file: " + filename);
|
||||
}
|
||||
|
||||
return buffer;
|
||||
}
|
||||
|
||||
enum class Endian {
|
||||
Little,
|
||||
Big
|
||||
};
|
||||
|
||||
class RandomAccessMemory {
|
||||
private:
|
||||
std::vector<uint8_t> _buffer;
|
||||
size_t _position;
|
||||
|
||||
public:
|
||||
// Constructors
|
||||
RandomAccessMemory() : _position(0) {}
|
||||
|
||||
explicit RandomAccessMemory(const std::vector<uint8_t>& initialData)
|
||||
: _buffer(initialData), _position(0) {}
|
||||
|
||||
explicit RandomAccessMemory(const uint8_t* data, size_t size)
|
||||
: _buffer(data, data + size), _position(0) {}
|
||||
|
||||
// Position management
|
||||
size_t positionSync() const {
|
||||
return _position;
|
||||
}
|
||||
|
||||
void setPositionSync(size_t position) {
|
||||
_position = position;
|
||||
}
|
||||
|
||||
size_t length() const {
|
||||
return _buffer.size();
|
||||
}
|
||||
|
||||
// Read bytes
|
||||
std::vector<uint8_t> readSync(size_t count) {
|
||||
if (_position + count > _buffer.size()) {
|
||||
throw std::range_error("Not enough bytes to read");
|
||||
}
|
||||
|
||||
std::vector<uint8_t> result(_buffer.begin() + _position,
|
||||
_buffer.begin() + _position + count);
|
||||
_position += count;
|
||||
return result;
|
||||
}
|
||||
|
||||
// Write bytes
|
||||
void writeFromSync(const std::vector<uint8_t>& bytes) {
|
||||
for (size_t i = 0; i < bytes.size(); i++) {
|
||||
if (_position + i >= _buffer.size()) {
|
||||
_buffer.push_back(bytes[i]);
|
||||
} else {
|
||||
_buffer[_position + i] = bytes[i];
|
||||
}
|
||||
}
|
||||
_position += bytes.size();
|
||||
}
|
||||
|
||||
// Read/Write Int Dynamic
|
||||
int64_t readIntSync(int size = 4, Endian endianness = Endian::Little) {
|
||||
if (size < 1 || size > 8) {
|
||||
throw std::invalid_argument("Size must be between 1 and 8 bytes");
|
||||
}
|
||||
|
||||
std::vector<uint8_t> bytes = readSync(size);
|
||||
|
||||
// Build integer from bytes with proper endianness
|
||||
int64_t result = 0;
|
||||
if (endianness == Endian::Little) {
|
||||
for (int i = size - 1; i >= 0; i--) {
|
||||
result = (result << 8) | bytes[i];
|
||||
}
|
||||
} else {
|
||||
for (int i = 0; i < size; i++) {
|
||||
result = (result << 8) | bytes[i];
|
||||
}
|
||||
}
|
||||
|
||||
// Sign extend if MSB is set
|
||||
int64_t signBit = 1LL << (size * 8 - 1);
|
||||
if (result & signBit) {
|
||||
result -= 1LL << (size * 8);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
uint64_t readUIntSync(int size = 4, Endian endianness = Endian::Little) {
|
||||
if (size < 1 || size > 8) {
|
||||
throw std::invalid_argument("Size must be between 1 and 8 bytes");
|
||||
}
|
||||
|
||||
std::vector<uint8_t> bytes = readSync(size);
|
||||
|
||||
// Build integer from bytes with proper endianness
|
||||
uint64_t result = 0;
|
||||
if (endianness == Endian::Little) {
|
||||
for (int i = size - 1; i >= 0; i--) {
|
||||
result = (result << 8) | bytes[i];
|
||||
}
|
||||
} else {
|
||||
for (int i = 0; i < size; i++) {
|
||||
result = (result << 8) | bytes[i];
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
void writeIntSync(int64_t value, int size = 4, Endian endianness = Endian::Little) {
|
||||
if (size < 1 || size > 8) {
|
||||
throw std::invalid_argument("Size must be between 1 and 8 bytes");
|
||||
}
|
||||
|
||||
std::vector<uint8_t> bytes(size, 0);
|
||||
|
||||
// Extract bytes with proper endianness
|
||||
if (endianness == Endian::Little) {
|
||||
for (int i = 0; i < size; i++) {
|
||||
bytes[i] = (value >> (i * 8)) & 0xFF;
|
||||
}
|
||||
} else {
|
||||
for (int i = 0; i < size; i++) {
|
||||
bytes[size - 1 - i] = (value >> (i * 8)) & 0xFF;
|
||||
}
|
||||
}
|
||||
|
||||
writeFromSync(bytes);
|
||||
}
|
||||
void writeUIntSync(uint64_t value, int size = 4, Endian endianness = Endian::Little) {
|
||||
if (size < 1 || size > 8) {
|
||||
throw std::invalid_argument("Size must be between 1 and 8 bytes");
|
||||
}
|
||||
|
||||
std::vector<uint8_t> bytes(size, 0);
|
||||
|
||||
// Extract bytes with proper endianness
|
||||
if (endianness == Endian::Little) {
|
||||
for (int i = 0; i < size; i++) {
|
||||
bytes[i] = (value >> (i * 8)) & 0xFF;
|
||||
}
|
||||
} else {
|
||||
for (int i = 0; i < size; i++) {
|
||||
bytes[size - 1 - i] = (value >> (i * 8)) & 0xFF;
|
||||
}
|
||||
}
|
||||
|
||||
writeFromSync(bytes);
|
||||
}
|
||||
|
||||
// Read/Write Pointers (assuming POINTER size is 8 bytes)
|
||||
SweepstorePointer readPointerSync(int pointerSize = 8) {
|
||||
int64_t offset = readUIntSync(pointerSize);
|
||||
return SweepstorePointer(offset);
|
||||
}
|
||||
|
||||
void writePointerSync(const SweepstorePointer& pointer, int pointerSize = 8) {
|
||||
writeUIntSync(pointer, pointerSize);
|
||||
}
|
||||
|
||||
// Read/Write Float32
|
||||
float readFloat32Sync(Endian endianness = Endian::Little) {
|
||||
std::vector<uint8_t> bytes = readSync(4);
|
||||
float value;
|
||||
|
||||
if (endianness == Endian::Little) {
|
||||
std::memcpy(&value, bytes.data(), 4);
|
||||
} else {
|
||||
std::vector<uint8_t> reversed(bytes.rbegin(), bytes.rend());
|
||||
std::memcpy(&value, reversed.data(), 4);
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
void writeFloat32Sync(float value, Endian endianness = Endian::Little) {
|
||||
std::vector<uint8_t> bytes(4);
|
||||
std::memcpy(bytes.data(), &value, 4);
|
||||
|
||||
if (endianness == Endian::Big) {
|
||||
std::reverse(bytes.begin(), bytes.end());
|
||||
}
|
||||
|
||||
writeFromSync(bytes);
|
||||
}
|
||||
|
||||
// Read/Write Float64 (Double)
|
||||
double readFloat64Sync(Endian endianness = Endian::Little) {
|
||||
std::vector<uint8_t> bytes = readSync(8);
|
||||
double value;
|
||||
|
||||
if (endianness == Endian::Little) {
|
||||
std::memcpy(&value, bytes.data(), 8);
|
||||
} else {
|
||||
std::vector<uint8_t> reversed(bytes.rbegin(), bytes.rend());
|
||||
std::memcpy(&value, reversed.data(), 8);
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
void writeFloat64Sync(double value, Endian endianness = Endian::Little) {
|
||||
std::vector<uint8_t> bytes(8);
|
||||
std::memcpy(bytes.data(), &value, 8);
|
||||
|
||||
if (endianness == Endian::Big) {
|
||||
std::reverse(bytes.begin(), bytes.end());
|
||||
}
|
||||
|
||||
writeFromSync(bytes);
|
||||
}
|
||||
|
||||
// Conversion methods
|
||||
std::vector<uint8_t> toVector() const {
|
||||
return _buffer;
|
||||
}
|
||||
|
||||
const uint8_t* data() const {
|
||||
return _buffer.data();
|
||||
}
|
||||
|
||||
uint8_t* data() {
|
||||
return _buffer.data();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
#ifdef _WIN32
|
||||
#include <windows.h>
|
||||
#endif
|
||||
|
||||
inline void preciseSleep(std::chrono::nanoseconds duration) {
|
||||
auto start = std::chrono::high_resolution_clock::now();
|
||||
|
||||
#ifdef _WIN32
|
||||
const auto windowsMinSleepTime = std::chrono::milliseconds(1);
|
||||
|
||||
if (duration < windowsMinSleepTime) {
|
||||
// Pure busy-wait with high-res timer
|
||||
while (std::chrono::high_resolution_clock::now() - start < duration) {
|
||||
// Optionally use _mm_pause() or YieldProcessor() to be nicer to hyperthreading
|
||||
}
|
||||
} else {
|
||||
// Hybrid: sleep most of it, busy-wait the remainder
|
||||
auto sleepDuration = duration - windowsMinSleepTime;
|
||||
std::this_thread::sleep_for(sleepDuration);
|
||||
while (std::chrono::high_resolution_clock::now() - start < duration) {}
|
||||
}
|
||||
#else
|
||||
std::this_thread::sleep_for(duration);
|
||||
#endif
|
||||
}
|
||||
|
||||
inline int32_t millisecondsSinceEpoch32() {
|
||||
auto now = std::chrono::system_clock::now();
|
||||
auto millis = std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count();
|
||||
return static_cast<int32_t>((millis / 1000) & 0xFFFFFFFF);
|
||||
}
|
||||
|
||||
inline int64_t millisecondsSinceEpoch64() {
|
||||
auto now = std::chrono::system_clock::now();
|
||||
auto millis = std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count();
|
||||
return millis;
|
||||
}
|
||||
|
||||
inline uint64_t bt_hash(const std::string& str) {
|
||||
uint64_t hash = 0xcbf29ce484222325ULL; // FNV offset basis
|
||||
|
||||
for (unsigned char byte : str) {
|
||||
hash ^= byte;
|
||||
hash *= 0x100000001b3ULL; // FNV prime
|
||||
}
|
||||
|
||||
return hash;
|
||||
}
|
||||
@@ -72,10 +72,14 @@ Future<void> main() async {
|
||||
print("Stale Ticket Threshold: ${STALE_HEARTBEAT_THRESHOLD_MS}ms");
|
||||
|
||||
while (true) {
|
||||
int concurrencyTest = 128;
|
||||
int concurrencyTest = 256;
|
||||
final receivePort = ReceivePort();
|
||||
int completedJobs = 0;
|
||||
|
||||
if (iteration > 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
final stopwatch = Stopwatch()..start();
|
||||
|
||||
print('\x1B[95mStarting iteration #$iteration with $concurrencyTest concurrent jobs...\x1B[0m');
|
||||
|
||||
BIN
example.bin
BIN
example.bin
Binary file not shown.
Reference in New Issue
Block a user