IMBENJI.NET/SweepStore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Add RandomAccessMemory class for in-memory binary operations

Browse Source
This commit is contained in:
ImBenji
2025-11-23 05:29:08 +00:00
parent 9216cd1638
commit 4295d119d7
26 changed files with 2124 additions and 949 deletions
Download Patch File Download Diff File Expand all files Collapse all files

304
cpp/binary_table.cpp
View File

@@ -4,6 +4,9 @@
#include <functional>
#include <filesystem>
#include <iostream>
#include <fstream>
#include <sstream>
#include <iomanip>
namespace bt {
@@ -483,8 +486,25 @@ BinaryTable::~BinaryTable() {
void BinaryTable::initialize() {
fseek(file_, 0, SEEK_SET);
writeInt64(0, BT_Null.address()); // Address table pointer (8 bytes)
writeInt32(8, 0); // Free list entry count (4 bytes)
// Magic number "SWPS" (0/4 bytes)
const char magic[] = "SWPS";
fwrite(magic, 1, 4, file_);
// Version (1.0 float16) (4/2 bytes)
uint16_t version = 0x3C00; // 1.0 in float16 format
uint8_t versionBytes[2] = {
static_cast<uint8_t>(version & 0xFF),
static_cast<uint8_t>((version >> 8) & 0xFF)
};
fwrite(versionBytes, 1, 2, file_);
// Address table pointer (null) (6/8 bytes)
writeInt64(6, BT_Null.address());
// Free list count (0) (14/4 bytes)
writeInt32(14, 0);
fflush(file_);
}
@@ -600,7 +620,7 @@ void BinaryTable::setFilePosition(int64_t position) {
// Address table management
std::unordered_map<int64_t, BT_Pointer> BinaryTable::getAddressTable() {
int64_t tableAddress = readInt64(0);
int64_t tableAddress = readInt64(6);
DEBUG_PRINTLN("DEBUG: getAddressTable reading from address " << tableAddress);
if (tableAddress == -1) { // Null pointer
@@ -674,7 +694,7 @@ void BinaryTable::setAddressTable(const std::unordered_map<int64_t, BT_Pointer>&
}
// Read old table pointer FIRST to ensure we can clean it up later
int64_t oldTablePointerAddress = readInt64(0);
int64_t oldTablePointerAddress = readInt64(6);
BT_Pointer oldTablePtr(oldTablePointerAddress);
int32_t oldTableSize = 0;
@@ -729,7 +749,7 @@ void BinaryTable::setAddressTable(const std::unordered_map<int64_t, BT_Pointer>&
fflush(file_);
// Atomically update header to point to new table
writeInt64(0, newTableAddress.address());
writeInt64(6, newTableAddress.address());
fflush(file_);
// Only free old table after new one is successfully committed
@@ -748,19 +768,15 @@ std::vector<BT_FreeListEntry> BinaryTable::getFreeList() {
return freeListCache_;
}
int64_t fileLength = getFileLength();
if (fileLength < 4) {
return {};
}
int32_t entryCount = readInt32(fileLength - 4);
int32_t entryCount = readInt32(14);
if (entryCount == 0) {
return {};
}
int32_t entrySize = 8 + 4; // Pointer + Size
int32_t freeListSize = entryCount * entrySize;
int64_t freeListStart = fileLength - 4 - freeListSize;
int64_t fileLength = getFileLength();
int64_t freeListStart = fileLength - freeListSize;
std::vector<BT_FreeListEntry> freeList;
for (int32_t i = 0; i < entryCount; i++) {
@@ -781,65 +797,44 @@ void BinaryTable::setFreeList(const std::vector<BT_FreeListEntry>& list) {
return;
}
// Always remove old free list first (matching Dart behavior)
int64_t fileLength = getFileLength();
DEBUG_PRINTLN("DEBUG: setFreeList fileLength=" << fileLength);
// Calculate old free list size to remove
int32_t oldEntryCount = 0;
if (fileLength >= 4) {
oldEntryCount = readInt32(fileLength - 4);
}
DEBUG_PRINTLN("DEBUG: setFreeList oldEntryCount=" << oldEntryCount);
// Remove old free list (matching Dart: always truncate first)
// Read OLD count from header (position 14)
int32_t oldEntryCount = readInt32(14);
// Calculate old free list size (entries only, not count)
int32_t oldListSize = oldEntryCount * (8 + 4);
// Remove old free list entries from EOF
if (oldEntryCount > 0) {
int32_t oldListSize = (oldEntryCount * (8 + 4)) + 4; // Entries + Count
int64_t newFileLength = fileLength - oldListSize;
DEBUG_PRINTLN("DEBUG: setFreeList - removing old free list, oldListSize=" << oldListSize << ", truncating to: " << newFileLength);
truncateFile(newFileLength);
fileLength = newFileLength; // Update file length
int64_t currentLength = getFileLength();
truncateFile(currentLength - oldListSize);
}
// If the new free list is empty, we're done (old list already removed)
if (list.empty()) {
DEBUG_PRINTLN("DEBUG: setFreeList - empty list, old list removed, done");
return;
}
// Write new free list at end of file
int64_t newLogicalEnd = fileLength;
// Encode new free list
std::vector<uint8_t> buffer;
// Entries
for (const auto& entry : list) {
// Pointer (8 bytes, little endian)
int64_t addr = entry.pointer.address();
for (int i = 0; i < 8; i++) {
buffer.push_back(static_cast<uint8_t>((addr >> (i * 8)) & 0xFF));
}
// Size (4 bytes, little endian)
int32_t size = entry.size;
for (int i = 0; i < 4; i++) {
buffer.push_back(static_cast<uint8_t>((size >> (i * 8)) & 0xFF));
// Write NEW count to header (position 14)
writeInt32(14, static_cast<int32_t>(list.size()));
// Write NEW entries to EOF (if any)
if (!list.empty()) {
// Encode new free list entries
std::vector<uint8_t> buffer;
for (const auto& entry : list) {
// Pointer (8 bytes, little endian)
int64_t addr = entry.pointer.address();
for (int i = 0; i < 8; i++) {
buffer.push_back(static_cast<uint8_t>((addr >> (i * 8)) & 0xFF));
}
// Size (4 bytes, little endian)
int32_t size = entry.size;
for (int i = 0; i < 4; i++) {
buffer.push_back(static_cast<uint8_t>((size >> (i * 8)) & 0xFF));
}
}
fseek(file_, 0, SEEK_END);
fwrite(buffer.data(), 1, buffer.size(), file_);
}
// Entry count (4 bytes, little endian)
int32_t count = static_cast<int32_t>(list.size());
for (int i = 0; i < 4; i++) {
buffer.push_back(static_cast<uint8_t>((count >> (i * 8)) & 0xFF));
}
// Write at the logical end position
fseek(file_, newLogicalEnd, SEEK_SET);
fwrite(buffer.data(), 1, buffer.size(), file_);
fflush(file_);
// Update logical file length
// File will be extended automatically by write operations
}
void BinaryTable::truncateFile(int64_t newSize) {
@@ -865,29 +860,24 @@ void BinaryTable::liftFreeList() {
throw std::runtime_error("Free list is already lifted");
}
// Cache the free list
freeListCache_ = getFreeList();
// Remove free list from end of file
int64_t fileLength = getFileLength();
int32_t oldEntryCount = (fileLength >= 4) ? readInt32(fileLength - 4) : 0;
// Read count from header (position 14)
int32_t oldEntryCount = readInt32(14);
if (oldEntryCount > 0) {
int32_t oldEntrySize = 8 + 4;
int32_t oldFreeListSize = oldEntryCount * oldEntrySize + 4;
int64_t newFileLength = fileLength - oldFreeListSize;
int32_t oldFreeListSize = oldEntryCount * oldEntrySize; // Just entries, no count
// Store current file position to restore later if needed
long currentPos = ftell(file_);
// Properly truncate the file
truncateFile(newFileLength);
// Restore file position if it's still valid
if (currentPos >= 0 && currentPos < newFileLength) {
fseek(file_, currentPos, SEEK_SET);
}
// Remove free list entries from EOF
int64_t fileLength = getFileLength();
truncateFile(fileLength - oldFreeListSize);
}
// Clear count in header (position 14)
writeInt32(14, 0);
freeListLifted_ = true;
}
@@ -898,9 +888,7 @@ void BinaryTable::dropFreeList() {
}
freeListLifted_ = false;
DEBUG_PRINTLN("DEBUG: About to call setFreeList - this might corrupt the address table!");
setFreeList(freeListCache_);
DEBUG_PRINTLN("DEBUG: setFreeList completed");
setFreeList(freeListCache_); // This now writes count to header and entries to EOF
freeListCache_.clear();
}
@@ -1106,4 +1094,150 @@ void BinaryTable::debugAddressTable(const std::string& context) {
DEBUG_PRINTLN("=========================");
}
} // namespace bt
std::string binaryDump(const std::vector<uint8_t>& data) {
std::ostringstream buffer;
for (size_t i = 0; i < data.size(); i += 16) {
// Address
buffer << "0x" << std::hex << std::setfill('0') << std::setw(4) << std::uppercase << i
<< " (" << std::dec << std::setfill(' ') << std::setw(4) << i << ") | ";
// Hex bytes
for (int j = 0; j < 16; j++) {
if (i + j < data.size()) {
buffer << std::hex << std::setfill('0') << std::setw(2) << std::uppercase
<< static_cast<int>(data[i + j]) << " ";
} else {
buffer << " ";
}
}
buffer << " | ";
// Integer representation
for (int j = 0; j < 16; j++) {
if (i + j < data.size()) {
buffer << std::dec << std::setfill(' ') << std::setw(3)
<< static_cast<int>(data[i + j]) << " ";
} else {
buffer << " ";
}
}
buffer << " | ";
// ASCII representation
for (int j = 0; j < 16; j++) {
if (i + j < data.size()) {
uint8_t byte = data[i + j];
if (byte >= 32 && byte <= 126) {
buffer << static_cast<char>(byte);
} else {
buffer << '.';
}
}
}
buffer << " | ";
if (i + 16 < data.size()) {
buffer << "\n";
}
}
return buffer.str();
}
} // namespace bt
int main() {
// Delete existing file if it exists
if (std::filesystem::exists("example.bin")) {
std::filesystem::remove("example.bin");
}
bt::BinaryTable table("example.bin");
table.initialize();
// Read file for dump
std::ifstream file("example.bin", std::ios::binary);
std::vector<uint8_t> fileData((std::istreambuf_iterator<char>(file)),
std::istreambuf_iterator<char>());
file.close();
std::cout << "File dump:\n";
std::cout << bt::binaryDump(fileData) << "\n";
std::cout << "File size: " << fileData.size() << " bytes\n";
std::cout << " \n";
// Set values
table.set("int_array", std::vector<int32_t>{6, 3, 9, 2, 5});
table.set("float_array", std::vector<float>{1.5f, 2.5f, 3.5f});
table.set("empty", std::vector<int32_t>{});
// Get arrays and modify elements
auto intArray = table.getArray<int32_t>("int_array");
auto floatArray = table.getArray<float>("float_array");
intArray.set(0, 1);
floatArray.set(1, 4.5f);
std::cout << "int_array pointer: " << intArray.getPointer().address() << "\n";
std::cout << "float_array pointer: " << floatArray.getPointer().address() << "\n";
intArray.add(10);
floatArray.add(5.5f);
intArray.addAll({420, 69, 1337, 1738});
floatArray.addAll({6.5f, 7.5f, 8.5f});
// Test the fetchSublist method
auto intArraySublist = intArray.fetchSublist(0, 3);
auto floatArraySublist = floatArray.fetchSublist(0, 2);
std::cout << "Sublist int_array (0-3): ";
for (auto val : intArraySublist) {
std::cout << val << " ";
}
std::cout << "\n";
std::cout << "Sublist float_array (0-2): ";
for (auto val : floatArraySublist) {
std::cout << val << " ";
}
std::cout << "\n";
// Read back values
auto readback1 = table.get<std::vector<int32_t>>("int_array");
auto readback2 = table.get<std::vector<float>>("float_array");
auto readback3 = table.get<std::vector<int32_t>>("empty");
std::cout << "Readback1: ";
for (auto val : readback1) {
std::cout << val << " ";
}
std::cout << "\n";
std::cout << "Readback2: ";
for (auto val : readback2) {
std::cout << val << " ";
}
std::cout << "\n";
std::cout << "Readback3: ";
for (auto val : readback3) {
std::cout << val << " ";
}
std::cout << "\n";
std::cout << " \n";
// Final file dump
std::ifstream finalFile("example.bin", std::ios::binary);
std::vector<uint8_t> finalFileData((std::istreambuf_iterator<char>(finalFile)),
std::istreambuf_iterator<char>());
finalFile.close();
std::cout << "File dump:\n";
std::cout << bt::binaryDump(finalFileData) << "\n";
std::cout << "File size: " << finalFileData.size() << " bytes\n";
return 0;
}