Add initial project files including analysis options, pubspec, and BinaryTable implementation
This commit is contained in:
ImBenji
916
c++/binary_table.h
Normal file
916
c++/binary_table.h
Normal file
@@ -0,0 +1,916 @@
|
||||
/*
|
||||
|
||||
/$$$$$$ /$$ /$$ /$$$$$$$ /$$$$$$$$ /$$ /$$ /$$$$$ /$$$$$$ /$$ /$$ /$$$$$$$$ /$$$$$$$$
|
||||
|_ $$_/| $$$ /$$$| $$__ $$| $$_____/| $$$ | $$ |__ $$|_ $$_/ | $$$ | $$| $$_____/|__ $$__/
|
||||
| $$ | $$$$ /$$$$| $$ \ $$| $$ | $$$$| $$ | $$ | $$ | $$$$| $$| $$ | $$
|
||||
| $$ | $$ $$/$$ $$| $$$$$$$ | $$$$$ | $$ $$ $$ | $$ | $$ | $$ $$ $$| $$$$$ | $$
|
||||
| $$ | $$ $$$| $$| $$__ $$| $$__/ | $$ $$$$ /$$ | $$ | $$ | $$ $$$$| $$__/ | $$
|
||||
| $$ | $$\ $ | $$| $$ \ $$| $$ | $$\ $$$| $$ | $$ | $$ | $$\ $$$| $$ | $$
|
||||
/$$$$$$| $$ \/ | $$| $$$$$$$/| $$$$$$$$| $$ \ $$| $$$$$$/ /$$$$$$ /$$| $$ \ $$| $$$$$$$$ | $$
|
||||
|______/|__/ |__/|_______/ |________/|__/ \__/ \______/ |______/|__/|__/ \__/|________/ |__/
|
||||
|
||||
© 2025-26 by Benjamin Watt of IMBENJI.NET LIMITED - All rights reserved.
|
||||
|
||||
Use of this source code is governed by a MIT license that can be found in the LICENSE file.
|
||||
|
||||
This file is part of the SweepStore (formerly Binary Table) package for C++.
|
||||
This file was generated by ChatGPT based on the Dart implementation. May contain bugs. Only time will tell.
|
||||
*/
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <variant>
|
||||
#include <vector>
|
||||
#include <algorithm>
|
||||
#include <functional>
|
||||
#include <sstream>
|
||||
#include <iomanip>
|
||||
|
||||
using byte = uint8_t;
|
||||
using Bytes = std::vector<byte>;
|
||||
|
||||
// forward
|
||||
struct BinaryTable;
|
||||
struct BT_UniformArray;
|
||||
|
||||
constexpr int POINTER_SIZE = 8; // same as BT_Type.POINTER in Dart
|
||||
|
||||
enum class BT_Type : int {
|
||||
POINTER = 0, // index 0 in Dart code held pointer size 8
|
||||
ADDRESS_TABLE = 1,
|
||||
INTEGER = 2,
|
||||
FLOAT = 3,
|
||||
STRING = 4,
|
||||
INTEGER_ARRAY = 5,
|
||||
FLOAT_ARRAY = 6
|
||||
};
|
||||
|
||||
inline int bt_type_index(BT_Type t) { return static_cast<int>(t); }
|
||||
|
||||
// Value variant type
|
||||
using Value = std::variant<std::monostate, int32_t, float, std::string,
|
||||
std::vector<int32_t>, std::vector<float>,
|
||||
uint64_t /*pointer address*/, std::shared_ptr<BT_UniformArray>>;
|
||||
|
||||
struct BT_Pointer {
|
||||
int64_t address;
|
||||
BT_Pointer(int64_t a = -1) : address(a) {}
|
||||
bool isNull() const { return address == -1; }
|
||||
bool operator==(const BT_Pointer& o) const { return address == o.address; }
|
||||
std::string toString() const {
|
||||
std::ostringstream ss;
|
||||
ss << "0x" << std::hex << address << " (" << std::dec << address << ")";
|
||||
return ss.str();
|
||||
}
|
||||
};
|
||||
static const BT_Pointer BT_Null = BT_Pointer(-1);
|
||||
|
||||
//
|
||||
// Utility: little-endian read/write helpers
|
||||
//
|
||||
int64_t readIntLE(std::fstream &f, int size) {
|
||||
if (size < 1 || size > 8) throw std::invalid_argument("size must be 1..8");
|
||||
std::vector<byte> buf(size);
|
||||
f.read(reinterpret_cast<char*>(buf.data()), size);
|
||||
if (!f) throw std::runtime_error("readIntLE failed");
|
||||
|
||||
uint64_t val = 0;
|
||||
// little-endian
|
||||
for (int i = size-1; i >= 0; --i) {
|
||||
val = (val << 8) | buf[i];
|
||||
}
|
||||
// sign extend
|
||||
uint64_t signBit = (uint64_t)1 << (size*8 - 1);
|
||||
if (val & signBit) {
|
||||
// negative
|
||||
int64_t signedVal = static_cast<int64_t>(val) - (static_cast<int64_t>(1) << (size*8));
|
||||
return signedVal;
|
||||
} else {
|
||||
return static_cast<int64_t>(val);
|
||||
}
|
||||
}
|
||||
|
||||
void writeIntLE(std::fstream &f, int64_t value, int size) {
|
||||
if (size < 1 || size > 8) throw std::invalid_argument("size must be 1..8");
|
||||
std::vector<byte> buf(size, 0);
|
||||
uint64_t v = static_cast<uint64_t>(value);
|
||||
for (int i = 0; i < size; ++i) {
|
||||
buf[i] = static_cast<byte>((v >> (i*8)) & 0xFF);
|
||||
}
|
||||
f.write(reinterpret_cast<char*>(buf.data()), size);
|
||||
if (!f) throw std::runtime_error("writeIntLE failed");
|
||||
}
|
||||
|
||||
float readFloat32LE(std::fstream &f) {
|
||||
uint32_t u = static_cast<uint32_t>(readIntLE(f,4));
|
||||
float val;
|
||||
std::memcpy(&val, &u, sizeof(float));
|
||||
return val;
|
||||
}
|
||||
void writeFloat32LE(std::fstream &f, float v) {
|
||||
uint32_t u;
|
||||
std::memcpy(&u, &v, sizeof(float));
|
||||
writeIntLE(f, static_cast<int64_t>(u), 4);
|
||||
}
|
||||
|
||||
BT_Pointer readPointerLE(std::fstream &f) {
|
||||
int64_t addr = readIntLE(f, POINTER_SIZE);
|
||||
return BT_Pointer(addr);
|
||||
}
|
||||
void writePointerLE(std::fstream &f, const BT_Pointer &p) {
|
||||
writeIntLE(f, p.address, POINTER_SIZE);
|
||||
}
|
||||
|
||||
//
|
||||
// FNV-1a 64-bit hash (like your Dart fnv1a)
|
||||
//
|
||||
uint64_t fnv1a_hash(const std::string &s) {
|
||||
const uint64_t FNV_OFFSET = 0xcbf29ce484222325ULL;
|
||||
const uint64_t FNV_PRIME = 0x100000001b3ULL;
|
||||
uint64_t hash = FNV_OFFSET;
|
||||
for (unsigned char c : s) {
|
||||
hash ^= (uint64_t)c;
|
||||
hash *= FNV_PRIME;
|
||||
}
|
||||
return hash;
|
||||
}
|
||||
|
||||
//
|
||||
// encodeValue: produce bytes like your Dart encodeValue
|
||||
//
|
||||
Bytes encodeValue(const Value &v) {
|
||||
Bytes out;
|
||||
if (std::holds_alternative<int32_t>(v)) {
|
||||
out.push_back(static_cast<byte>(bt_type_index(BT_Type::INTEGER)));
|
||||
int32_t val = std::get<int32_t>(v);
|
||||
for (int i = 0; i < 4; ++i) out.push_back(static_cast<byte>((val >> (i*8)) & 0xFF));
|
||||
} else if (std::holds_alternative<float>(v)) {
|
||||
out.push_back(static_cast<byte>(bt_type_index(BT_Type::FLOAT)));
|
||||
float f = std::get<float>(v);
|
||||
uint32_t u; std::memcpy(&u, &f, sizeof(float));
|
||||
for (int i = 0; i < 4; ++i) out.push_back(static_cast<byte>((u >> (i*8)) & 0xFF));
|
||||
} else if (std::holds_alternative<std::string>(v)) {
|
||||
out.push_back(static_cast<byte>(bt_type_index(BT_Type::STRING)));
|
||||
const std::string &s = std::get<std::string>(v);
|
||||
int32_t len = static_cast<int32_t>(s.size());
|
||||
for (int i = 0; i < 4; ++i) out.push_back(static_cast<byte>((len >> (i*8)) & 0xFF));
|
||||
out.insert(out.end(), s.begin(), s.end());
|
||||
} else if (std::holds_alternative<std::vector<int32_t>>(v)) {
|
||||
out.push_back(static_cast<byte>(bt_type_index(BT_Type::INTEGER_ARRAY)));
|
||||
const auto &vec = std::get<std::vector<int32_t>>(v);
|
||||
int32_t len = static_cast<int32_t>(vec.size());
|
||||
for (int i = 0; i < 4; ++i) out.push_back(static_cast<byte>((len >> (i*8)) & 0xFF));
|
||||
// encode each element with full encodeValue (type byte + data)
|
||||
for (int32_t item : vec) {
|
||||
Value vi = item;
|
||||
Bytes b = encodeValue(vi);
|
||||
out.insert(out.end(), b.begin(), b.end());
|
||||
}
|
||||
} else if (std::holds_alternative<std::vector<float>>(v)) {
|
||||
out.push_back(static_cast<byte>(bt_type_index(BT_Type::FLOAT_ARRAY)));
|
||||
const auto &vec = std::get<std::vector<float>>(v);
|
||||
int32_t len = static_cast<int32_t>(vec.size());
|
||||
for (int i = 0; i < 4; ++i) out.push_back(static_cast<byte>((len >> (i*8)) & 0xFF));
|
||||
for (float item : vec) {
|
||||
Value vf = item;
|
||||
Bytes b = encodeValue(vf);
|
||||
out.insert(out.end(), b.begin(), b.end());
|
||||
}
|
||||
} else if (std::holds_alternative<uint64_t>(v)) {
|
||||
out.push_back(static_cast<byte>(bt_type_index(BT_Type::POINTER)));
|
||||
uint64_t a = std::get<uint64_t>(v);
|
||||
for (int i = 0; i < POINTER_SIZE; ++i) out.push_back(static_cast<byte>((a >> (i*8)) & 0xFF));
|
||||
} else {
|
||||
throw std::runtime_error("Unsupported type in encodeValue");
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
//
|
||||
// Read a Value from a file at current position
|
||||
//
|
||||
Value decodeValue(std::fstream &f) {
|
||||
int typeId = f.get();
|
||||
if (typeId == EOF) throw std::runtime_error("decodeValue: EOF while reading type");
|
||||
BT_Type type = static_cast<BT_Type>(typeId);
|
||||
|
||||
switch (type) {
|
||||
case BT_Type::INTEGER: {
|
||||
int32_t iv = static_cast<int32_t>(readIntLE(f,4));
|
||||
return iv;
|
||||
}
|
||||
case BT_Type::FLOAT: {
|
||||
float fv = readFloat32LE(f);
|
||||
return fv;
|
||||
}
|
||||
case BT_Type::STRING: {
|
||||
int32_t len = static_cast<int32_t>(readIntLE(f,4));
|
||||
std::string s; s.resize(len);
|
||||
f.read(&s[0], len);
|
||||
if (!f) throw std::runtime_error("decodeValue: unable to read string bytes");
|
||||
return s;
|
||||
}
|
||||
case BT_Type::POINTER: {
|
||||
BT_Pointer p = readPointerLE(f);
|
||||
return static_cast<uint64_t>(p.address);
|
||||
}
|
||||
case BT_Type::ADDRESS_TABLE:
|
||||
throw std::runtime_error("Address table decoding not implemented via decodeValue");
|
||||
case BT_Type::INTEGER_ARRAY:
|
||||
case BT_Type::FLOAT_ARRAY: {
|
||||
// return a UniformArray proxy: encode pointer we are at minus 0? We need to return a proxy object that knows the pointer
|
||||
// However decodeValue isn't given a pointer. For usage we typically read pointer from address table and then create BT_Reference/UniformArray
|
||||
// This function is mostly used for direct inline decode; we'll not implement reading arrays inline here.
|
||||
throw std::runtime_error("Decoding array inline is not supported by decodeValue(); use BT_UniformArray via BinaryTable accessors.");
|
||||
}
|
||||
default:
|
||||
throw std::runtime_error("Unsupported BT_Type in decodeValue");
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Free list entry
|
||||
//
|
||||
struct BT_FreeListEntry {
|
||||
BT_Pointer pointer;
|
||||
int32_t size;
|
||||
BT_FreeListEntry() : pointer(-1), size(0) {}
|
||||
BT_FreeListEntry(BT_Pointer p, int32_t s) : pointer(p), size(s) {}
|
||||
};
|
||||
|
||||
//
|
||||
// Proxy: uniform array wrapper
|
||||
//
|
||||
struct BT_UniformArray {
|
||||
BinaryTable *table;
|
||||
BT_Pointer pointer;
|
||||
|
||||
BT_UniformArray(BinaryTable *t, BT_Pointer p) : table(t), pointer(p) {}
|
||||
|
||||
// Get length
|
||||
int32_t length() const;
|
||||
|
||||
// Index access (read)
|
||||
Value get(int index) const;
|
||||
|
||||
// Index write (only supports fixed-size numeric types for now)
|
||||
void set(int index, const Value &v);
|
||||
|
||||
// Add / addAll will reallocate and update pointer in address table by invoking BinaryTable methods
|
||||
void add(const Value &v);
|
||||
void addAll(const std::vector<Value> &vals);
|
||||
|
||||
// element type detection (returns BT_Type if available)
|
||||
BT_Type elementType() const;
|
||||
|
||||
// size in bytes used in file
|
||||
int32_t size() const;
|
||||
};
|
||||
|
||||
//
|
||||
// BinaryTable
|
||||
//
|
||||
struct BinaryTable {
|
||||
std::string path;
|
||||
std::fstream f;
|
||||
|
||||
BinaryTable(const std::string &p) : path(p) {
|
||||
// open in read/write binary mode, create if not exists
|
||||
f.open(path, std::ios::in | std::ios::out | std::ios::binary);
|
||||
if (!f) {
|
||||
// try create
|
||||
std::ofstream create(path, std::ios::binary);
|
||||
create.close();
|
||||
f.open(path, std::ios::in | std::ios::out | std::ios::binary);
|
||||
if (!f) throw std::runtime_error("Failed to open file " + path);
|
||||
}
|
||||
}
|
||||
|
||||
void initialise() {
|
||||
// set file to empty then write header: pointer to address table (pointer), and free list entry count (4 bytes zeros)
|
||||
f.seekp(0, std::ios::beg);
|
||||
writePointerLE(f, BT_Null); // address table pointer
|
||||
writeIntLE(f, 0, 4); // free-list entry count
|
||||
f.flush();
|
||||
}
|
||||
|
||||
// --- Address table get / set ---
|
||||
// addressTable is map<uint64_t hash, BT_Pointer>
|
||||
std::map<uint64_t, BT_Pointer> addressTableGet() {
|
||||
// read table pointer from start
|
||||
f.seekg(0, std::ios::beg);
|
||||
BT_Pointer tablePtr = readPointerLE(f);
|
||||
std::map<uint64_t, BT_Pointer> table;
|
||||
if (tablePtr.isNull()) return table;
|
||||
|
||||
// at tablePtr.address, first byte is type, then 4 bytes count, then entries of 8 (hash) + POINTER_SIZE (pointer)
|
||||
f.seekg(tablePtr.address + 1, std::ios::beg);
|
||||
int32_t count = static_cast<int32_t>(readIntLE(f,4));
|
||||
int entrySize = 8 + POINTER_SIZE;
|
||||
for (int i = 0; i < count; ++i) {
|
||||
// read hash (8 bytes)
|
||||
uint64_t keyHash = static_cast<uint64_t>(readIntLE(f,8));
|
||||
int64_t valPtrAddr = readIntLE(f, POINTER_SIZE);
|
||||
table[keyHash] = BT_Pointer(valPtrAddr);
|
||||
}
|
||||
return table;
|
||||
}
|
||||
|
||||
void addressTableSet(const std::map<uint64_t, BT_Pointer> &table) {
|
||||
// build buffer
|
||||
Bytes buffer;
|
||||
buffer.push_back(static_cast<byte>(bt_type_index(BT_Type::ADDRESS_TABLE)));
|
||||
int32_t count = static_cast<int32_t>(table.size());
|
||||
for (int i = 0; i < 4; ++i) buffer.push_back(static_cast<byte>((count >> (i*8)) & 0xFF));
|
||||
for (const auto &kv : table) {
|
||||
uint64_t key = kv.first;
|
||||
BT_Pointer val = kv.second;
|
||||
for (int i = 0; i < 8; ++i) buffer.push_back(static_cast<byte>((key >> (i*8)) & 0xFF));
|
||||
for (int i = 0; i < POINTER_SIZE; ++i) buffer.push_back(static_cast<byte>((val.address >> (i*8)) & 0xFF));
|
||||
}
|
||||
|
||||
// allocate at end
|
||||
BT_Pointer tableAddress = alloc(static_cast<int32_t>(buffer.size()));
|
||||
// write buffer
|
||||
f.seekp(tableAddress.address, std::ios::beg);
|
||||
f.write(reinterpret_cast<char*>(buffer.data()), buffer.size());
|
||||
f.flush();
|
||||
|
||||
// read old pointer
|
||||
f.seekg(0, std::ios::beg);
|
||||
BT_Pointer oldPtr = readPointerLE(f);
|
||||
|
||||
// update header
|
||||
f.seekp(0, std::ios::beg);
|
||||
writePointerLE(f, tableAddress);
|
||||
f.flush();
|
||||
|
||||
// free old table if exists and not same
|
||||
if (!oldPtr.isNull() && !(oldPtr == tableAddress)) {
|
||||
BT_Pointer p = oldPtr;
|
||||
int32_t s = pointerSize(oldPtr);
|
||||
// must lift free list for free() but here we do simplest: lift, free, drop
|
||||
liftFreeList();
|
||||
free_internal(p, s);
|
||||
dropFreeList();
|
||||
}
|
||||
}
|
||||
|
||||
// --- free list handling ---
|
||||
bool freeListLifted = false;
|
||||
std::vector<BT_FreeListEntry> freeListCache;
|
||||
|
||||
std::vector<BT_FreeListEntry> freeListGet() {
|
||||
if (freeListLifted) return freeListCache;
|
||||
// read last 4 bytes for entry count
|
||||
f.seekg(0, std::ios::end);
|
||||
std::streampos fileLen = f.tellg();
|
||||
if (fileLen < 4) return {};
|
||||
f.seekg(fileLen - 4, std::ios::beg);
|
||||
int32_t entryCount = static_cast<int32_t>(readIntLE(f, 4));
|
||||
if (entryCount == 0) return {};
|
||||
int entrySize = POINTER_SIZE + 4;
|
||||
std::streampos freeListStart = fileLen - 4 - static_cast<std::streamoff>(entryCount * entrySize);
|
||||
f.seekg(freeListStart, std::ios::beg);
|
||||
std::vector<BT_FreeListEntry> list;
|
||||
for (int i = 0; i < entryCount; ++i) {
|
||||
int64_t ptrAddr = readIntLE(f, POINTER_SIZE);
|
||||
int32_t sz = static_cast<int32_t>(readIntLE(f, 4));
|
||||
list.emplace_back(BT_Pointer(ptrAddr), sz);
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
void freeListSet(const std::vector<BT_FreeListEntry> &list) {
|
||||
if (freeListLifted) {
|
||||
freeListCache = list;
|
||||
return;
|
||||
}
|
||||
// erase old free list at end
|
||||
f.seekg(0, std::ios::end);
|
||||
std::streampos fileLen = f.tellg();
|
||||
if (fileLen >= 4) {
|
||||
// read old count
|
||||
f.seekg(fileLen - 4, std::ios::beg);
|
||||
int32_t oldCount = static_cast<int32_t>(readIntLE(f,4));
|
||||
int oldSize = (oldCount * (POINTER_SIZE + 4)) + 4;
|
||||
// truncate: move put to new size (there's no direct truncate with std fstreams; workaround via reopening)
|
||||
std::streampos newLen = fileLen - oldSize;
|
||||
f.close();
|
||||
// platform-dependent truncation via std::ofstream::open + close
|
||||
std::ofstream trunc(path, std::ios::in | std::ios::out | std::ios::binary);
|
||||
trunc.seekp(newLen);
|
||||
trunc.flush();
|
||||
trunc.close();
|
||||
// reopen
|
||||
f.open(path, std::ios::in | std::ios::out | std::ios::binary);
|
||||
}
|
||||
// append new free list
|
||||
f.seekp(0, std::ios::end);
|
||||
for (const auto &e : list) {
|
||||
writeIntLE(f, e.pointer.address, POINTER_SIZE);
|
||||
writeIntLE(f, e.size, 4);
|
||||
}
|
||||
int32_t count = static_cast<int32_t>(list.size());
|
||||
writeIntLE(f, count, 4);
|
||||
f.flush();
|
||||
}
|
||||
|
||||
void liftFreeList() {
|
||||
if (freeListLifted) throw std::runtime_error("free list already lifted");
|
||||
freeListCache = freeListGet();
|
||||
// truncate file to remove free list
|
||||
f.seekg(0, std::ios::end);
|
||||
std::streampos fileLen = f.tellg();
|
||||
if (fileLen < 4) { freeListLifted = true; return; }
|
||||
f.seekg(fileLen - 4, std::ios::beg);
|
||||
int32_t oldCount = static_cast<int32_t>(readIntLE(f,4));
|
||||
int oldEntrySize = POINTER_SIZE + 4;
|
||||
int oldFreeListSize = oldCount * oldEntrySize + 4;
|
||||
std::streampos newLen = fileLen - oldFreeListSize;
|
||||
f.close();
|
||||
// truncate via std::ofstream
|
||||
std::ofstream trunc(path, std::ios::in | std::ios::out | std::ios::binary);
|
||||
trunc.seekp(newLen);
|
||||
trunc.flush();
|
||||
trunc.close();
|
||||
f.open(path, std::ios::in | std::ios::out | std::ios::binary);
|
||||
freeListLifted = true;
|
||||
}
|
||||
|
||||
void dropFreeList() {
|
||||
if (!freeListLifted) throw std::runtime_error("free list not lifted");
|
||||
// append placeholder 4 bytes then use freeListSet to write full encoded list
|
||||
f.seekp(0, std::ios::end);
|
||||
writeIntLE(f, 0, 4); // placeholder count (will be replaced by freeListSet)
|
||||
freeListLifted = false;
|
||||
freeListSet(freeListCache);
|
||||
freeListCache.clear();
|
||||
}
|
||||
|
||||
// internal free function used when freeListLifted==true
|
||||
void free_internal(const BT_Pointer &pointer, int32_t size) {
|
||||
if (!freeListLifted) throw std::runtime_error("free list must be lifted before freeing");
|
||||
if (pointer.isNull() || size <= 0) throw std::invalid_argument("Cannot free null pointer or zero size");
|
||||
auto list = freeListGet();
|
||||
list.emplace_back(pointer, size);
|
||||
|
||||
// merge contiguous blocks
|
||||
std::sort(list.begin(), list.end(), [](const BT_FreeListEntry &a, const BT_FreeListEntry &b){
|
||||
return a.pointer.address < b.pointer.address;
|
||||
});
|
||||
std::vector<BT_FreeListEntry> merged;
|
||||
for (const auto &e : list) {
|
||||
if (merged.empty()) merged.push_back(e);
|
||||
else {
|
||||
BT_FreeListEntry &last = merged.back();
|
||||
if (last.pointer.address + last.size == e.pointer.address) {
|
||||
last.size += e.size;
|
||||
} else {
|
||||
merged.push_back(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
freeListSet(merged);
|
||||
}
|
||||
|
||||
void free(const BT_Pointer &pointer, int32_t size) {
|
||||
free_internal(pointer, size);
|
||||
}
|
||||
|
||||
// allocation expects free list to be lifted
|
||||
BT_Pointer alloc(int32_t size) {
|
||||
if (!freeListLifted) throw std::runtime_error("free list must be lifted before alloc");
|
||||
auto list = freeListGet();
|
||||
if (list.empty()) {
|
||||
// allocate at end of file
|
||||
f.seekg(0, std::ios::end);
|
||||
std::streampos end = f.tellg();
|
||||
return BT_Pointer(static_cast<int64_t>(end));
|
||||
}
|
||||
// find first fit
|
||||
int idx = -1;
|
||||
for (size_t i = 0; i < list.size(); ++i) {
|
||||
if (list[i].size >= size) { idx = static_cast<int>(i); break; }
|
||||
}
|
||||
if (idx == -1) {
|
||||
f.seekg(0, std::ios::end);
|
||||
std::streampos end = f.tellg();
|
||||
return BT_Pointer(static_cast<int64_t>(end));
|
||||
}
|
||||
BT_FreeListEntry chosen = list[idx];
|
||||
if (chosen.size == size) {
|
||||
// exact fit, remove
|
||||
list.erase(list.begin() + idx);
|
||||
freeListSet(list);
|
||||
return chosen.pointer;
|
||||
} else {
|
||||
// split
|
||||
BT_Pointer allocated = chosen.pointer;
|
||||
BT_FreeListEntry remainder(BT_Pointer(chosen.pointer.address + size), chosen.size - size);
|
||||
list.erase(list.begin() + idx);
|
||||
list.push_back(remainder);
|
||||
freeListSet(list);
|
||||
return allocated;
|
||||
}
|
||||
}
|
||||
|
||||
// pointer size helper: compute stored size of object at pointer by reading type and necessary lengths
|
||||
int32_t pointerSize(const BT_Pointer &p) {
|
||||
if (p.isNull()) return 0;
|
||||
f.seekg(p.address, std::ios::beg);
|
||||
int typeId = f.get();
|
||||
if (typeId == EOF) throw std::runtime_error("pointerSize EOF");
|
||||
BT_Type type = static_cast<BT_Type>(typeId);
|
||||
switch (type) {
|
||||
case BT_Type::INTEGER: return 1 + 4;
|
||||
case BT_Type::FLOAT: return 1 + 4;
|
||||
case BT_Type::STRING: {
|
||||
int32_t len = static_cast<int32_t>(readIntLE(f,4));
|
||||
return 1 + 4 + len;
|
||||
}
|
||||
case BT_Type::ADDRESS_TABLE: {
|
||||
int32_t count = static_cast<int32_t>(readIntLE(f,4));
|
||||
return 1 + 4 + count * (8 + POINTER_SIZE);
|
||||
}
|
||||
case BT_Type::POINTER:
|
||||
return 1 + POINTER_SIZE;
|
||||
case BT_Type::INTEGER_ARRAY:
|
||||
case BT_Type::FLOAT_ARRAY: {
|
||||
// read length then need to determine element size by reading first element (type+size)
|
||||
int32_t len = static_cast<int32_t>(readIntLE(f,4));
|
||||
// save position
|
||||
std::streampos pos = f.tellg();
|
||||
// read first element's type
|
||||
if (len == 0) return 1 + 4; // type byte + length only
|
||||
int subTypeId = f.get();
|
||||
f.seekg(pos, std::ios::beg);
|
||||
BT_Type elemType = static_cast<BT_Type>(subTypeId);
|
||||
// for uniform arrays of numeric types we expect element type size defined
|
||||
int elemSize = 0;
|
||||
if (elemType == BT_Type::INTEGER || elemType == BT_Type::FLOAT) elemSize = 1 + 4;
|
||||
else throw std::runtime_error("Variable-size elements in uniform array not supported in pointerSize()");
|
||||
return 1 + 4 + len * elemSize;
|
||||
}
|
||||
default:
|
||||
throw std::runtime_error("Unsupported type in pointerSize");
|
||||
}
|
||||
}
|
||||
|
||||
// operator[] and operator[]= equivalents
|
||||
void set(const std::string &key, const Value &value) {
|
||||
// anti-free-list-scope: lift free list, do op, drop
|
||||
liftFreeList();
|
||||
auto table = addressTableGet();
|
||||
|
||||
uint64_t keyHash = fnv1a_hash(key);
|
||||
if (table.find(keyHash) != table.end()) {
|
||||
dropFreeList();
|
||||
throw std::runtime_error("Key already exists");
|
||||
}
|
||||
Bytes buf = encodeValue(value);
|
||||
BT_Pointer addr = alloc(static_cast<int32_t>(buf.size()));
|
||||
// write value
|
||||
f.seekp(addr.address, std::ios::beg);
|
||||
f.write(reinterpret_cast<char*>(buf.data()), buf.size());
|
||||
f.flush();
|
||||
table[keyHash] = addr;
|
||||
addressTableSet(table);
|
||||
// addressTableSet will free old table and update header by itself (it lifts/drops internally)
|
||||
dropFreeList();
|
||||
}
|
||||
|
||||
// return Value for key
|
||||
Value get(const std::string &key) {
|
||||
auto table = addressTableGet();
|
||||
uint64_t keyHash = fnv1a_hash(key);
|
||||
if (table.find(keyHash) == table.end()) throw std::runtime_error("Key does not exist");
|
||||
BT_Pointer p = table[keyHash];
|
||||
// to decode numeric/string types we can seek to pointer and call decodeValue
|
||||
f.seekg(p.address, std::ios::beg);
|
||||
int typeId = f.get();
|
||||
if (typeId == EOF) throw std::runtime_error("get: EOF");
|
||||
BT_Type type = static_cast<BT_Type>(typeId);
|
||||
if (type == BT_Type::INTEGER) {
|
||||
int32_t v = static_cast<int32_t>(readIntLE(f,4));
|
||||
return v;
|
||||
} else if (type == BT_Type::FLOAT) {
|
||||
float v = readFloat32LE(f);
|
||||
return v;
|
||||
} else if (type == BT_Type::STRING) {
|
||||
int32_t len = static_cast<int32_t>(readIntLE(f,4));
|
||||
std::string s; s.resize(len);
|
||||
f.read(&s[0], len);
|
||||
if (!f) throw std::runtime_error("get: unable to read string");
|
||||
return s;
|
||||
} else if (type == BT_Type::INTEGER_ARRAY || type == BT_Type::FLOAT_ARRAY) {
|
||||
// return a shared_ptr to uniform array proxy
|
||||
auto proxy = std::make_shared<BT_UniformArray>(this, p);
|
||||
return proxy;
|
||||
} else if (type == BT_Type::POINTER) {
|
||||
int64_t addr = readIntLE(f, POINTER_SIZE);
|
||||
return static_cast<uint64_t>(addr);
|
||||
} else {
|
||||
throw std::runtime_error("Unsupported get type");
|
||||
}
|
||||
}
|
||||
|
||||
void deleteKey(const std::string &key) {
|
||||
liftFreeList();
|
||||
auto table = addressTableGet();
|
||||
uint64_t keyHash = fnv1a_hash(key);
|
||||
if (table.find(keyHash) == table.end()) {
|
||||
dropFreeList();
|
||||
throw std::runtime_error("Key does not exist");
|
||||
}
|
||||
BT_Pointer p = table[keyHash];
|
||||
int32_t s = pointerSize(p);
|
||||
free_internal(p, s);
|
||||
table.erase(keyHash);
|
||||
addressTableSet(table);
|
||||
dropFreeList();
|
||||
}
|
||||
|
||||
// relocate address table and try to truncate file
|
||||
void truncate() {
|
||||
liftFreeList();
|
||||
// re-write address table to ensure it's at the end
|
||||
auto table = addressTableGet();
|
||||
addressTableSet(table);
|
||||
|
||||
// check last free block
|
||||
auto list = freeListGet();
|
||||
if (list.empty()) { dropFreeList(); return; }
|
||||
std::sort(list.begin(), list.end(), [](const BT_FreeListEntry &a, const BT_FreeListEntry &b){
|
||||
return a.pointer.address < b.pointer.address;
|
||||
});
|
||||
auto last = list.back();
|
||||
f.seekg(0, std::ios::end);
|
||||
int64_t fileEnd = static_cast<int64_t>(f.tellg());
|
||||
int64_t expected = last.pointer.address + last.size;
|
||||
if (expected != fileEnd) { dropFreeList(); return; }
|
||||
// remove last entry and truncate
|
||||
list.pop_back();
|
||||
freeListSet(list);
|
||||
// truncate file to last.pointer.address
|
||||
f.close();
|
||||
std::ofstream trunc(path, std::ios::in | std::ios::out | std::ios::binary);
|
||||
trunc.seekp(last.pointer.address);
|
||||
trunc.flush();
|
||||
trunc.close();
|
||||
f.open(path, std::ios::in | std::ios::out | std::ios::binary);
|
||||
dropFreeList();
|
||||
}
|
||||
|
||||
}; // BinaryTable
|
||||
|
||||
// BT_UniformArray implementation methods (defined after BinaryTable)
|
||||
int32_t BT_UniformArray::length() const {
|
||||
if (pointer.isNull()) return 0;
|
||||
table->f.seekg(pointer.address, std::ios::beg);
|
||||
int typeId = table->f.get();
|
||||
BT_Type t = static_cast<BT_Type>(typeId);
|
||||
if (!(t == BT_Type::INTEGER_ARRAY || t == BT_Type::FLOAT_ARRAY))
|
||||
throw std::runtime_error("Not an array");
|
||||
int32_t len = static_cast<int32_t>(readIntLE(table->f,4));
|
||||
return len;
|
||||
}
|
||||
BT_Type BT_UniformArray::elementType() const {
|
||||
if (pointer.isNull()) throw std::runtime_error("Null pointer");
|
||||
int32_t len = length();
|
||||
if (len == 0) throw std::runtime_error("Empty array has no elementType");
|
||||
// element type is stored right after type byte + length (1+4)
|
||||
table->f.seekg(pointer.address + 1 + 4, std::ios::beg);
|
||||
int subTypeId = table->f.get();
|
||||
return static_cast<BT_Type>(subTypeId);
|
||||
}
|
||||
Value BT_UniformArray::get(int index) const {
|
||||
if (pointer.isNull()) throw std::runtime_error("Null pointer");
|
||||
int32_t len = length();
|
||||
if (index < 0 || index >= len) throw std::out_of_range("index");
|
||||
BT_Type elemType = elementType();
|
||||
if (elemType == BT_Type::INTEGER) {
|
||||
// element entry size = 1 + 4
|
||||
int64_t itemAddr = pointer.address + 1 + 4 + index * (1 + 4);
|
||||
table->f.seekg(itemAddr, std::ios::beg);
|
||||
int tid = table->f.get();
|
||||
if (static_cast<BT_Type>(tid) != BT_Type::INTEGER) throw std::runtime_error("element type mismatch");
|
||||
int32_t v = static_cast<int32_t>(readIntLE(table->f,4));
|
||||
return v;
|
||||
} else if (elemType == BT_Type::FLOAT) {
|
||||
int64_t itemAddr = pointer.address + 1 + 4 + index * (1 + 4);
|
||||
table->f.seekg(itemAddr, std::ios::beg);
|
||||
int tid = table->f.get();
|
||||
if (static_cast<BT_Type>(tid) != BT_Type::FLOAT) throw std::runtime_error("element type mismatch");
|
||||
float v = readFloat32LE(table->f);
|
||||
return v;
|
||||
} else {
|
||||
throw std::runtime_error("Unsupported element type in get()");
|
||||
}
|
||||
}
|
||||
|
||||
void BT_UniformArray::set(int index, const Value &v) {
|
||||
if (pointer.isNull()) throw std::runtime_error("Null pointer");
|
||||
int32_t len = length();
|
||||
if (index < 0 || index >= len) throw std::out_of_range("index");
|
||||
BT_Type elemType = elementType();
|
||||
// Only support fixed-size numeric types
|
||||
if (elemType == BT_Type::INTEGER && std::holds_alternative<int32_t>(v)) {
|
||||
int64_t itemAddr = pointer.address + 1 + 4 + index * (1 + 4);
|
||||
auto buf = encodeValue(v);
|
||||
table->f.seekp(itemAddr, std::ios::beg);
|
||||
table->f.write(reinterpret_cast<char*>(buf.data()), buf.size());
|
||||
table->f.flush();
|
||||
} else if (elemType == BT_Type::FLOAT && std::holds_alternative<float>(v)) {
|
||||
int64_t itemAddr = pointer.address + 1 + 4 + index * (1 + 4);
|
||||
auto buf = encodeValue(v);
|
||||
table->f.seekp(itemAddr, std::ios::beg);
|
||||
table->f.write(reinterpret_cast<char*>(buf.data()), buf.size());
|
||||
table->f.flush();
|
||||
} else {
|
||||
throw std::runtime_error("Type mismatch or unsupported set()");
|
||||
}
|
||||
}
|
||||
|
||||
void BT_UniformArray::add(const Value &v) {
|
||||
addAll({v});
|
||||
}
|
||||
|
||||
void BT_UniformArray::addAll(const std::vector<Value> &vals) {
|
||||
if (pointer.isNull()) throw std::runtime_error("Null pointer");
|
||||
// anti-free-list
|
||||
table->liftFreeList();
|
||||
BT_Type elemType;
|
||||
if (length() == 0) {
|
||||
// deduce from first
|
||||
const Value &first = vals.at(0);
|
||||
if (std::holds_alternative<int32_t>(first)) elemType = BT_Type::INTEGER;
|
||||
else if (std::holds_alternative<float>(first)) elemType = BT_Type::FLOAT;
|
||||
else throw std::runtime_error("Unsupported element type for uniform array");
|
||||
} else {
|
||||
elemType = elementType();
|
||||
}
|
||||
// validate
|
||||
for (size_t i = 0; i < vals.size(); ++i) {
|
||||
if (elemType == BT_Type::INTEGER && !std::holds_alternative<int32_t>(vals[i])) {
|
||||
table->dropFreeList();
|
||||
throw std::runtime_error("Type mismatch in addAll");
|
||||
} else if (elemType == BT_Type::FLOAT && !std::holds_alternative<float>(vals[i])) {
|
||||
table->dropFreeList();
|
||||
throw std::runtime_error("Type mismatch in addAll");
|
||||
}
|
||||
}
|
||||
// read full buffer into memory
|
||||
int32_t oldLen = length();
|
||||
int elemStoredSize = 1 + 4;
|
||||
int32_t bufferSize = 1 + 4 + oldLen * elemStoredSize;
|
||||
table->f.seekg(pointer.address, std::ios::beg);
|
||||
Bytes fullBuffer(bufferSize);
|
||||
table->f.read(reinterpret_cast<char*>(fullBuffer.data()), bufferSize);
|
||||
// append encoded new elements
|
||||
for (const auto &val : vals) {
|
||||
Bytes b = encodeValue(val);
|
||||
fullBuffer.insert(fullBuffer.end(), b.begin(), b.end());
|
||||
}
|
||||
int32_t newLen = oldLen + static_cast<int32_t>(vals.size());
|
||||
// update length bytes
|
||||
for (int i = 0; i < 4; ++i) fullBuffer[1 + i] = static_cast<byte>((newLen >> (i*8)) & 0xFF);
|
||||
// free old array space
|
||||
int32_t oldSize = 1 + 4 + oldLen * elemStoredSize;
|
||||
table->free_internal(pointer, oldSize);
|
||||
// allocate new space
|
||||
BT_Pointer newPtr = table->alloc(static_cast<int32_t>(fullBuffer.size()));
|
||||
// update address table entries that pointed to old pointer -> update to new pointer
|
||||
// We need to update any keys that pointed to pointer.address; simplest is to rebuild address table and replace
|
||||
auto tableMap = table->addressTableGet();
|
||||
for (auto &kv : tableMap) {
|
||||
if (kv.second == pointer) kv.second = newPtr;
|
||||
}
|
||||
table->addressTableSet(tableMap);
|
||||
// write new buffer to file
|
||||
table->f.seekp(newPtr.address, std::ios::beg);
|
||||
table->f.write(reinterpret_cast<char*>(fullBuffer.data()), fullBuffer.size());
|
||||
table->f.flush();
|
||||
// update this->pointer
|
||||
pointer = newPtr;
|
||||
table->dropFreeList();
|
||||
}
|
||||
|
||||
int32_t BT_UniformArray::size() const {
|
||||
if (pointer.isNull()) return 0;
|
||||
return table->pointerSize(pointer);
|
||||
}
|
||||
|
||||
//
|
||||
// simple binary dump helper
|
||||
//
|
||||
std::string binaryDump(const std::string &path) {
|
||||
std::ifstream in(path, std::ios::binary);
|
||||
if (!in) return "<no file>";
|
||||
std::ostringstream oss;
|
||||
std::vector<byte> data((std::istreambuf_iterator<char>(in)), std::istreambuf_iterator<char>());
|
||||
size_t len = data.size();
|
||||
for (size_t i = 0; i < len; i += 16) {
|
||||
oss << "0x" << std::uppercase << std::hex << std::setw(4) << std::setfill('0') << i << " (" << std::dec << std::setw(4) << i << ") | ";
|
||||
for (int j = 0; j < 16; ++j) {
|
||||
if (i + j < len) {
|
||||
oss << std::setw(2) << std::setfill('0') << std::hex << (int)data[i+j] << " ";
|
||||
} else oss << " ";
|
||||
}
|
||||
oss << " | ";
|
||||
for (int j = 0; j < 16; ++j) {
|
||||
if (i + j < len) oss << std::dec << std::setw(3) << std::setfill(' ') << (int)data[i+j] << " ";
|
||||
else oss << " ";
|
||||
}
|
||||
oss << " | ";
|
||||
for (int j = 0; j < 16; ++j) {
|
||||
if (i + j < len) {
|
||||
byte b = data[i+j];
|
||||
if (b >= 32 && b <= 126) oss << (char)b;
|
||||
else oss << '.';
|
||||
}
|
||||
}
|
||||
if (i + 16 < len) oss << "\n";
|
||||
}
|
||||
return oss.str();
|
||||
}
|
||||
|
||||
int main() {
|
||||
const std::string fname = "example.bin";
|
||||
// remove existing
|
||||
std::remove(fname.c_str());
|
||||
BinaryTable table(fname);
|
||||
table.initialise();
|
||||
std::cout << "File dump:\n" << binaryDump(fname) << "\n";
|
||||
{
|
||||
// set some arrays/values
|
||||
std::vector<int32_t> iarr = {6,3,9,2,5};
|
||||
std::vector<float> farr = {1.5f, 2.5f, 3.5f};
|
||||
std::vector<int32_t> empty;
|
||||
table.set("int_array", iarr);
|
||||
table.set("float_array", farr);
|
||||
table.set("empty", empty);
|
||||
}
|
||||
|
||||
// update first element of int_array
|
||||
{
|
||||
Value v = table.get("int_array");
|
||||
if (auto proxy = std::get_if<std::shared_ptr<BT_UniformArray>>(&v)) {
|
||||
auto arr = *proxy;
|
||||
// but our get returns pointer wrapped in variant; the variant stores shared_ptr<BT_UniformArray>
|
||||
// set element 0 to 1
|
||||
arr->set(0, Value(int32_t(1)));
|
||||
arr->set(1, Value(int32_t(3))); // just demonstrate
|
||||
// add items
|
||||
arr->add(Value(int32_t(10)));
|
||||
arr->addAll({Value(int32_t(420)), Value(int32_t(69)), Value(int32_t(1337)), Value(int32_t(1738))});
|
||||
// print results by reading back
|
||||
int32_t len = arr->length();
|
||||
std::cout << "int_array content: [";
|
||||
for (int i=0;i<len;++i) {
|
||||
Value e = arr->get(i);
|
||||
if (std::holds_alternative<int32_t>(e)) {
|
||||
std::cout << std::get<int32_t>(e);
|
||||
} else std::cout << "?";
|
||||
if (i+1<len) std::cout << ", ";
|
||||
}
|
||||
std::cout << "]\n";
|
||||
}
|
||||
}
|
||||
|
||||
// float array
|
||||
{
|
||||
Value v = table.get("float_array");
|
||||
if (auto proxy = std::get_if<std::shared_ptr<BT_UniformArray>>(&v)) {
|
||||
auto arr = *proxy;
|
||||
arr->set(1, Value(4.5f));
|
||||
arr->add(Value(5.5f));
|
||||
arr->addAll({Value(6.5f), Value(7.5f), Value(8.5f)});
|
||||
int32_t len = arr->length();
|
||||
std::cout << "float_array content: [";
|
||||
for (int i=0;i<len;++i) {
|
||||
Value e = arr->get(i);
|
||||
if (std::holds_alternative<float>(e)) {
|
||||
std::cout << std::get<float>(e);
|
||||
} else std::cout << "?";
|
||||
if (i+1<len) std::cout << ", ";
|
||||
}
|
||||
std::cout << "]\n";
|
||||
}
|
||||
}
|
||||
|
||||
// print file dump & size
|
||||
std::ifstream infile(fname, std::ios::binary | std::ios::ate);
|
||||
infile.seekg(0, std::ios::end);
|
||||
std::cout << "\nFile dump:\n" << binaryDump(fname) << "\n";
|
||||
std::cout << "File size: " << infile.tellg() << " bytes\n";
|
||||
infile.close();
|
||||
|
||||
return 0;
|
||||
}
|
||||
BIN
example.bin
Normal file
BIN
example.bin
Normal file
Binary file not shown.
Reference in New Issue
Block a user