SweepStore/cpp/binary_table.cpp

#include "binary_table.h"
#include <algorithm>
#include <cstring>
#include <functional>
#include <filesystem>
#include <iostream>
#include <fstream>
#include <sstream>
#include <iomanip>

namespace bt {

// FNV-1a hash implementation
int64_t BinaryTable::hashString(const std::string& str) const {
    uint64_t hash = 0xcbf29ce484222325ULL; // FNV offset basis
    for (char c : str) {
        hash ^= static_cast<uint8_t>(c);
        hash *= 0x100000001b3ULL; // FNV prime
    }
    return static_cast<int64_t>(hash);
}

// Value encoding implementations
std::vector<uint8_t> encodeValue(const int32_t& value) {
    std::vector<uint8_t> buffer;
    buffer.push_back(static_cast<uint8_t>(BT_Type::INTEGER));

    // Little endian encoding
    buffer.push_back(value & 0xFF);
    buffer.push_back((value >> 8) & 0xFF);
    buffer.push_back((value >> 16) & 0xFF);
    buffer.push_back((value >> 24) & 0xFF);

    return buffer;
}

std::vector<uint8_t> encodeValue(const float& value) {
    std::vector<uint8_t> buffer;
    buffer.push_back(static_cast<uint8_t>(BT_Type::FLOAT));

    // Convert float to bytes (little endian)
    uint32_t floatBits;
    std::memcpy(&floatBits, &value, sizeof(float));

    buffer.push_back(floatBits & 0xFF);
    buffer.push_back((floatBits >> 8) & 0xFF);
    buffer.push_back((floatBits >> 16) & 0xFF);
    buffer.push_back((floatBits >> 24) & 0xFF);

    return buffer;
}

std::vector<uint8_t> encodeValue(const std::string& value) {
    std::vector<uint8_t> buffer;
    buffer.push_back(static_cast<uint8_t>(BT_Type::STRING));

    // String length (little endian)
    int32_t length = static_cast<int32_t>(value.length());
    buffer.push_back(length & 0xFF);
    buffer.push_back((length >> 8) & 0xFF);
    buffer.push_back((length >> 16) & 0xFF);
    buffer.push_back((length >> 24) & 0xFF);

    // String bytes
    for (char c : value) {
        buffer.push_back(static_cast<uint8_t>(c));
    }

    return buffer;
}

std::vector<uint8_t> encodeValue(const std::vector<int32_t>& value) {
    std::vector<uint8_t> buffer;
    buffer.push_back(static_cast<uint8_t>(BT_Type::INTEGER_ARRAY));

    // Array length (little endian)
    int32_t length = static_cast<int32_t>(value.size());
    buffer.push_back(length & 0xFF);
    buffer.push_back((length >> 8) & 0xFF);
    buffer.push_back((length >> 16) & 0xFF);
    buffer.push_back((length >> 24) & 0xFF);

    // Array elements
    for (const auto& item : value) {
        auto itemBuffer = encodeValue(item);
        buffer.insert(buffer.end(), itemBuffer.begin(), itemBuffer.end());
    }

    return buffer;
}

std::vector<uint8_t> encodeValue(const std::vector<float>& value) {
    std::vector<uint8_t> buffer;
    buffer.push_back(static_cast<uint8_t>(BT_Type::FLOAT_ARRAY));

    // Array length (little endian)
    int32_t length = static_cast<int32_t>(value.size());
    buffer.push_back(length & 0xFF);
    buffer.push_back((length >> 8) & 0xFF);
    buffer.push_back((length >> 16) & 0xFF);
    buffer.push_back((length >> 24) & 0xFF);

    // Array elements
    for (const auto& item : value) {
        auto itemBuffer = encodeValue(item);
        buffer.insert(buffer.end(), itemBuffer.begin(), itemBuffer.end());
    }

    return buffer;
}

// BT_Reference implementation
BT_Reference::BT_Reference(BinaryTable* table, BT_Pointer pointer)
    : table_(table), pointer_(pointer) {}

template<>
int32_t BT_Reference::decodeValue<int32_t>() {
    if (pointer_.isNull()) {
        throw std::runtime_error("Null pointer");
    }

    table_->setFilePosition(pointer_.address());
    uint8_t typeId = table_->readByte(pointer_.address());

    if (static_cast<BT_Type>(typeId) != BT_Type::INTEGER) {
        throw std::runtime_error("Type mismatch");
    }

    return table_->readInt32(pointer_.address() + 1);
}

template<>
float BT_Reference::decodeValue<float>() {
    if (pointer_.isNull()) {
        throw std::runtime_error("Null pointer");
    }

    table_->setFilePosition(pointer_.address());
    uint8_t typeId = table_->readByte(pointer_.address());

    if (static_cast<BT_Type>(typeId) != BT_Type::FLOAT) {
        throw std::runtime_error("Type mismatch");
    }

    return table_->readFloat32(pointer_.address() + 1);
}

template<>
std::string BT_Reference::decodeValue<std::string>() {
    if (pointer_.isNull()) {
        throw std::runtime_error("Null pointer");
    }

    table_->setFilePosition(pointer_.address());
    uint8_t typeId = table_->readByte(pointer_.address());

    if (static_cast<BT_Type>(typeId) != BT_Type::STRING) {
        throw std::runtime_error("Type mismatch");
    }

    int32_t length = table_->readInt32(pointer_.address() + 1);
    auto bytes = table_->readBytes(pointer_.address() + 5, length);

    return std::string(bytes.begin(), bytes.end());
}

template<>
BT_UniformArray<int32_t> BT_Reference::decodeValue<BT_UniformArray<int32_t>>() {
    return BT_UniformArray<int32_t>(table_, pointer_);
}

template<>
BT_UniformArray<float> BT_Reference::decodeValue<BT_UniformArray<float>>() {
    return BT_UniformArray<float>(table_, pointer_);
}

template<>
std::vector<int32_t> BT_Reference::decodeValue<std::vector<int32_t>>() {
    if (pointer_.isNull()) {
        return {};
    }

    uint8_t typeId = table_->readByte(pointer_.address());
    BT_Type type = static_cast<BT_Type>(typeId);

    if (type != BT_Type::INTEGER_ARRAY) {
        throw std::runtime_error("Type mismatch - expected integer array");
    }

    int32_t length = table_->readInt32(pointer_.address() + 1);
    std::vector<int32_t> result;
    result.reserve(length);

    // Each element is: type byte (1) + int32 data (4) = 5 bytes
    int64_t elementPos = pointer_.address() + 1 + 4; // Skip type and length

    for (int32_t i = 0; i < length; i++) {
        // Skip the type byte, read the int32 value
        int32_t value = table_->readInt32(elementPos + 1);
        result.push_back(value);
        elementPos += 5; // Move to next element
    }

    return result;
}

template<>
std::vector<float> BT_Reference::decodeValue<std::vector<float>>() {
    if (pointer_.isNull()) {
        return {};
    }

    uint8_t typeId = table_->readByte(pointer_.address());
    BT_Type type = static_cast<BT_Type>(typeId);

    if (type != BT_Type::FLOAT_ARRAY) {
        throw std::runtime_error("Type mismatch - expected float array");
    }

    int32_t length = table_->readInt32(pointer_.address() + 1);
    std::vector<float> result;
    result.reserve(length);

    // Each element is: type byte (1) + float data (4) = 5 bytes
    int64_t elementPos = pointer_.address() + 1 + 4; // Skip type and length

    for (int32_t i = 0; i < length; i++) {
        // Skip the type byte, read the float value
        float value = table_->readFloat32(elementPos + 1);
        result.push_back(value);
        elementPos += 5; // Move to next element
    }

    return result;
}

int32_t BT_Reference::size() const {
    if (pointer_.isNull()) {
        return 0;
    }

    uint8_t typeId = table_->readByte(pointer_.address());
    BT_Type type = static_cast<BT_Type>(typeId);

    switch (type) {
        case BT_Type::POINTER:
            return 1 + 8; // Type byte + pointer
        case BT_Type::INTEGER:
        case BT_Type::FLOAT:
            return 1 + 4; // Type byte + data
        case BT_Type::STRING: {
            int32_t length = table_->readInt32(pointer_.address() + 1);
            return 1 + 4 + length; // Type + length + string bytes
        }
        case BT_Type::ADDRESS_TABLE: {
            int32_t count = table_->readInt32(pointer_.address() + 1);
            return 1 + 4 + count * (8 + 8); // Type + count + entries
        }
        case BT_Type::INTEGER_ARRAY:
        case BT_Type::FLOAT_ARRAY: {
            int32_t length = table_->readInt32(pointer_.address() + 1);
            int32_t elementSize = (type == BT_Type::INTEGER_ARRAY) ? (1 + 4) : (1 + 4);
            return 1 + 4 + length * elementSize;
        }
    }
    return 0;
}

BT_Type BT_Reference::getType() const {
    if (pointer_.isNull()) {
        throw std::runtime_error("Null pointer");
    }

    uint8_t typeId = table_->readByte(pointer_.address());
    return static_cast<BT_Type>(typeId);
}

// BT_UniformArray template implementations
template<typename T>
int32_t BT_UniformArray<T>::length() const {
    if (this->pointer_.isNull()) {
        return 0;
    }

    try {
        uint8_t typeId = this->table_->readByte(this->pointer_.address());
        BT_Type type = static_cast<BT_Type>(typeId);

        if (!isArrayType(type)) {
            return 0; // Treat non-array as empty array instead of throwing
        }

        return this->table_->readInt32(this->pointer_.address() + 1);
    } catch (...) {
        return 0; // If we can't read, treat as empty
    }
}

template<typename T>
T BT_UniformArray<T>::operator[](int32_t index) const {
    if (this->pointer_.isNull()) {
        throw std::runtime_error("Null pointer");
    }

    int32_t len = length();
    if (index < 0 || index >= len) {
        throw std::out_of_range("Index out of range");
    }

    // Determine element type and size
    uint8_t elementTypeId = this->table_->readByte(this->pointer_.address() + 1 + 4);
    BT_Type elementType = static_cast<BT_Type>(elementTypeId);
    int32_t elementSize = 1 + getTypeSize(elementType);

    int64_t itemAddress = this->pointer_.address() + 1 + 4 + index * elementSize;
    BT_Reference itemRef(this->table_, BT_Pointer(itemAddress));

    return itemRef.decodeValue<T>();
}

template<typename T>
void BT_UniformArray<T>::set(int32_t index, const T& value) {
    if (this->pointer_.isNull()) {
        throw std::runtime_error("Null pointer");
    }

    int32_t len = length();
    if (index < 0 || index >= len) {
        throw std::out_of_range("Index out of range");
    }

    // Validate type compatibility
    BT_Type expectedType = getTypeFromValue<T>();
    uint8_t elementTypeId = this->table_->readByte(this->pointer_.address() + 1 + 4);
    BT_Type elementType = static_cast<BT_Type>(elementTypeId);

    if (expectedType != elementType) {
        throw std::runtime_error("Type mismatch");
    }

    // Encode and write value
    auto valueBuffer = encodeValue(value);
    int32_t elementSize = 1 + getTypeSize(elementType);
    int64_t itemAddress = this->pointer_.address() + 1 + 4 + index * elementSize;

    this->table_->writeBytes(itemAddress, valueBuffer);
}

template<typename T>
void BT_UniformArray<T>::add(const T& value) {
    addAll({value});
}

template<typename T>
void BT_UniformArray<T>::addAll(const std::vector<T>& values) {
    this->table_->antiFreeListScope([&]() {
        // Get current element type or determine from new values
        BT_Type elementType = getTypeFromValue<T>();

        if (length() > 0) {
            uint8_t existingTypeId = this->table_->readByte(this->pointer_.address() + 1 + 4);
            BT_Type existingType = static_cast<BT_Type>(existingTypeId);
            if (existingType != elementType) {
                throw std::runtime_error("Type mismatch");
            }
        }

        // Validate all values are compatible
        for (const auto& value : values) {
            (void)value; // Suppress unused variable warning
            BT_Type valueType = getTypeFromValue<T>();
            if (valueType != elementType) {
                throw std::runtime_error("Type mismatch in values");
            }
            if (getTypeSize(elementType) == -1) {
                throw std::runtime_error("Variable size types not supported in uniform arrays");
            }
        }

        // Read current array buffer
        int32_t currentLength = length();
        int32_t elementSize = 1 + getTypeSize(elementType);
        int32_t currentBufferSize = 1 + 4 + currentLength * elementSize;

        std::vector<uint8_t> fullBuffer;
        if (currentLength > 0) {
            fullBuffer = this->table_->readBytes(this->pointer_.address(), currentBufferSize);
        } else {
            // Empty array, create initial buffer
            fullBuffer.push_back(static_cast<uint8_t>(elementType == BT_Type::INTEGER ? BT_Type::INTEGER_ARRAY : BT_Type::FLOAT_ARRAY));
            fullBuffer.push_back(0); // Length will be updated
            fullBuffer.push_back(0);
            fullBuffer.push_back(0);
            fullBuffer.push_back(0);
        }

        // Add new values to buffer
        for (const auto& value : values) {
            auto valueBuffer = encodeValue(value);
            fullBuffer.insert(fullBuffer.end(), valueBuffer.begin(), valueBuffer.end());
        }

        // Update length in buffer
        int32_t newLength = currentLength + static_cast<int32_t>(values.size());
        fullBuffer[1] = newLength & 0xFF;
        fullBuffer[2] = (newLength >> 8) & 0xFF;
        fullBuffer[3] = (newLength >> 16) & 0xFF;
        fullBuffer[4] = (newLength >> 24) & 0xFF;

        // Free old array if it exists
        if (!this->pointer_.isNull()) {
            this->table_->free(this->pointer_, currentBufferSize);
        }

        // Allocate new space
        BT_Pointer newPointer = this->table_->alloc(static_cast<int32_t>(fullBuffer.size()));

        // Update any references in address table
        auto addressTable = this->table_->getAddressTable();
        for (auto& [key, value] : addressTable) {
            if (value == this->pointer_) {
                value = newPointer;
            }
        }
        this->table_->setAddressTable(addressTable);
        this->pointer_ = newPointer;

        // Write updated buffer
        this->table_->writeBytes(newPointer.address(), fullBuffer);
    });
}

template<typename T>
std::vector<T> BT_UniformArray<T>::fetchSublist(int32_t start, int32_t end) {
    int32_t len = length();
    if (len == 0) {
        return {};
    }

    if (end == -1) {
        end = len;
    }

    if (start < 0 || start >= len || end < start || end > len) {
        throw std::out_of_range("Invalid range");
    }

    uint8_t elementTypeId = this->table_->readByte(this->pointer_.address() + 1 + 4);
    BT_Type elementType = static_cast<BT_Type>(elementTypeId);
    int32_t elementSize = 1 + getTypeSize(elementType);

    if (getTypeSize(elementType) == -1) {
        throw std::runtime_error("Variable size types not supported in uniform arrays");
    }

    std::vector<T> result;
    for (int32_t i = start; i < end; i++) {
        int64_t itemAddress = this->pointer_.address() + 1 + 4 + i * elementSize;
        BT_Reference itemRef(this->table_, BT_Pointer(itemAddress));
        result.push_back(itemRef.decodeValue<T>());
    }

    return result;
}

// Explicit template instantiations
template class BT_UniformArray<int32_t>;
template class BT_UniformArray<float>;

// BinaryTable implementation
BinaryTable::BinaryTable(const std::string& path)
    : filePath_(path), freeListLifted_(false) {
    file_ = fopen(path.c_str(), "r+b");
    if (!file_) {
        // File doesn't exist, create it
        file_ = fopen(path.c_str(), "w+b");
    }

}

BinaryTable::~BinaryTable() {
    if (file_) {
        fclose(file_);
    }
}

void BinaryTable::initialize() {
    fseek(file_, 0, SEEK_SET);
    
    // Magic number "SWPS" (0/4 bytes)
    const char magic[] = "SWPS";
    fwrite(magic, 1, 4, file_);
    
    // Version (1.0 float16) (4/2 bytes) 
    uint16_t version = 0x3C00; // 1.0 in float16 format
    uint8_t versionBytes[2] = {
        static_cast<uint8_t>(version & 0xFF),
        static_cast<uint8_t>((version >> 8) & 0xFF)
    };
    fwrite(versionBytes, 1, 2, file_);
    
    // Address table pointer (null) (6/8 bytes)
    writeInt64(6, BT_Null.address());
    
    // Free list count (0) (14/4 bytes)
    writeInt32(14, 0);
    
    fflush(file_);
}

// File I/O helper implementations
int32_t BinaryTable::readInt32(int64_t position) {
    fseek(file_, position, SEEK_SET);
    uint8_t bytes[4];
    fread(bytes, 1, 4, file_);

    return static_cast<int32_t>(bytes[0]) |
           (static_cast<int32_t>(bytes[1]) << 8) |
           (static_cast<int32_t>(bytes[2]) << 16) |
           (static_cast<int32_t>(bytes[3]) << 24);
}

float BinaryTable::readFloat32(int64_t position) {
    fseek(file_, position, SEEK_SET);
    uint8_t bytes[4];
    fread(bytes, 1, 4, file_);

    uint32_t floatBits = static_cast<uint32_t>(bytes[0]) |
                        (static_cast<uint32_t>(bytes[1]) << 8) |
                        (static_cast<uint32_t>(bytes[2]) << 16) |
                        (static_cast<uint32_t>(bytes[3]) << 24);

    float result;
    std::memcpy(&result, &floatBits, sizeof(float));
    return result;
}

int64_t BinaryTable::readInt64(int64_t position) {
    fseek(file_, position, SEEK_SET);
    uint8_t bytes[8];
    fread(bytes, 1, 8, file_);

    int64_t result = 0;
    for (int i = 0; i < 8; i++) {
        result |= static_cast<int64_t>(bytes[i]) << (i * 8);
    }

    return result;
}

uint8_t BinaryTable::readByte(int64_t position) {
    fseek(file_, position, SEEK_SET);
    uint8_t byte;
    fread(&byte, 1, 1, file_);
    return byte;
}

std::vector<uint8_t> BinaryTable::readBytes(int64_t position, int32_t count) {
    fseek(file_, position, SEEK_SET);
    std::vector<uint8_t> bytes(count);
    fread(bytes.data(), 1, count, file_);
    return bytes;
}

void BinaryTable::writeInt32(int64_t position, int32_t value) {
    fseek(file_, position, SEEK_SET);
    uint8_t bytes[4] = {
        static_cast<uint8_t>(value & 0xFF),
        static_cast<uint8_t>((value >> 8) & 0xFF),
        static_cast<uint8_t>((value >> 16) & 0xFF),
        static_cast<uint8_t>((value >> 24) & 0xFF)
    };
    fwrite(bytes, 1, 4, file_);
}

void BinaryTable::writeFloat32(int64_t position, float value) {
    fseek(file_, position, SEEK_SET);
    uint32_t floatBits;
    std::memcpy(&floatBits, &value, sizeof(float));

    uint8_t bytes[4] = {
        static_cast<uint8_t>(floatBits & 0xFF),
        static_cast<uint8_t>((floatBits >> 8) & 0xFF),
        static_cast<uint8_t>((floatBits >> 16) & 0xFF),
        static_cast<uint8_t>((floatBits >> 24) & 0xFF)
    };
    fwrite(bytes, 1, 4, file_);
}

void BinaryTable::writeInt64(int64_t position, int64_t value) {
    fseek(file_, position, SEEK_SET);
    uint8_t bytes[8];
    for (int i = 0; i < 8; i++) {
        bytes[i] = static_cast<uint8_t>((value >> (i * 8)) & 0xFF);
    }
    fwrite(bytes, 1, 8, file_);
}

void BinaryTable::writeByte(int64_t position, uint8_t value) {
    fseek(file_, position, SEEK_SET);
    fwrite(&value, 1, 1, file_);
}

void BinaryTable::writeBytes(int64_t position, const std::vector<uint8_t>& data) {
    fseek(file_, position, SEEK_SET);
    fwrite(data.data(), 1, data.size(), file_);
}

int64_t BinaryTable::getFileLength() {
    long current = ftell(file_);
    fseek(file_, 0, SEEK_END);
    long length = ftell(file_);
    fseek(file_, current, SEEK_SET);  // Restore position
    return length;
}

void BinaryTable::setFilePosition(int64_t position) {
    fseek(file_, position, SEEK_SET);
}

// Address table management
std::unordered_map<int64_t, BT_Pointer> BinaryTable::getAddressTable() {
    int64_t tableAddress = readInt64(6);
    DEBUG_PRINTLN("DEBUG: getAddressTable reading from address " << tableAddress);

    if (tableAddress == -1) { // Null pointer
        return {};
    }

    // Validate table address is within file bounds
    int64_t fileLength = getFileLength();
    if (tableAddress < 0 || tableAddress >= fileLength) {
        DEBUG_PRINTLN("DEBUG: Address table pointer is out of bounds: " << tableAddress << " (file length: " << fileLength << ")");
        throw std::runtime_error("Address table pointer is corrupted - out of bounds");
    }

    try {
        uint8_t typeId = readByte(tableAddress);

        if (static_cast<BT_Type>(typeId) != BT_Type::ADDRESS_TABLE) {
            DEBUG_PRINTLN("DEBUG: Invalid type ID at address table location: " << (int)typeId);
            // Address table might not be valid yet, return empty
            return {};
        }

        int32_t tableCount = readInt32(tableAddress + 1);

        // Validate table count is reasonable
        if (tableCount < 0 || tableCount > 1000000) { // Arbitrary but reasonable limit
            DEBUG_PRINTLN("DEBUG: Suspicious address table count: " << tableCount);
            throw std::runtime_error("Address table appears corrupted - invalid entry count");
        }

        // Validate the entire table fits within file bounds
        int64_t requiredSize = 1 + 4 + tableCount * (8 + 8); // Type + count + entries
        if (tableAddress + requiredSize > fileLength) {
            DEBUG_PRINTLN("DEBUG: Address table extends beyond file bounds");
            throw std::runtime_error("Address table appears corrupted - extends beyond file");
        }

        std::unordered_map<int64_t, BT_Pointer> addressTable;

        for (int32_t i = 0; i < tableCount; i++) {
            int64_t offset = tableAddress + 1 + 4 + i * (8 + 8);
            int64_t keyHash = readInt64(offset);
            int64_t valueAddress = readInt64(offset + 8);

            // Validate each value address is within bounds (or null)
            if (valueAddress != -1 && (valueAddress < 0 || valueAddress >= fileLength)) {
                DEBUG_PRINTLN("DEBUG: Invalid value address in entry " << i << ": " << valueAddress);
                throw std::runtime_error("Address table entry contains invalid pointer");
            }

            DEBUG_PRINTLN("  Reading entry " << i << ": hash " << keyHash << " -> address " << valueAddress);
            addressTable[keyHash] = BT_Pointer(valueAddress);
        }

        return addressTable;
    } catch (const std::runtime_error&) {
        // Re-throw runtime errors (our validation failures)
        throw;
    } catch (...) {
        // If we can't read the address table for other reasons, return empty
        DEBUG_PRINTLN("DEBUG: Failed to read address table due to I/O error");
        return {};
    }
}

void BinaryTable::setAddressTable(const std::unordered_map<int64_t, BT_Pointer>& table) {
    DEBUG_PRINTLN("DEBUG: setAddressTable called! This should NOT happen during get operations!");
    DEBUG_PRINTLN("DEBUG: setAddressTable writing " << table.size() << " entries");
    for (const auto& [key, value] : table) {
        DEBUG_PRINTLN("  Writing hash " << key << " -> address " << value.address());
    }

    // Read old table pointer FIRST to ensure we can clean it up later
    int64_t oldTablePointerAddress = readInt64(6);
    BT_Pointer oldTablePtr(oldTablePointerAddress);
    int32_t oldTableSize = 0;

    // Calculate old table size if it exists
    if (!oldTablePtr.isNull()) {
        try {
            BT_Reference oldTableRef(this, oldTablePtr);
            oldTableSize = oldTableRef.size();
        } catch (...) {
            // If we can't read the old table, we can't free it safely
            DEBUG_PRINTLN("DEBUG: WARNING - Cannot read old table for cleanup");
            oldTablePtr = BT_Null;
        }
    }

    // Build buffer manually (matching Dart implementation exactly)
    std::vector<uint8_t> buffer;

    // Type byte
    buffer.push_back(static_cast<uint8_t>(BT_Type::ADDRESS_TABLE));

    // Table count (little endian, 4 bytes)
    int32_t count = static_cast<int32_t>(table.size());
    for (int i = 0; i < 4; i++) {
        buffer.push_back(static_cast<uint8_t>((count >> (i * 8)) & 0xFF));
    }

    // Table entries
    for (const auto& [key, value] : table) {
        // Key hash (little endian, 8 bytes)
        for (int i = 0; i < 8; i++) {
            buffer.push_back(static_cast<uint8_t>((key >> (i * 8)) & 0xFF));
        }
        // Value address (little endian, 8 bytes)
        int64_t addr = value.address();
        for (int i = 0; i < 8; i++) {
            buffer.push_back(static_cast<uint8_t>((addr >> (i * 8)) & 0xFF));
        }
    }

    // Allocate and write new address table
    BT_Pointer newTableAddress = alloc(static_cast<int32_t>(buffer.size()));
    setFilePosition(newTableAddress.address());
    size_t written = fwrite(buffer.data(), 1, buffer.size(), file_);
    (void)written; // Suppress unused variable warning in release builds

    if (written != buffer.size()) {
        throw std::runtime_error("Failed to write complete address table");
    }

    // Ensure new table is written to disk before updating header
    fflush(file_);

    // Atomically update header to point to new table
    writeInt64(6, newTableAddress.address());
    fflush(file_);

    // Only free old table after new one is successfully committed
    DEBUG_PRINTLN("DEBUG: oldTablePtr.isNull()=" << oldTablePtr.isNull() << ", oldTablePtr.address()=" << oldTablePtr.address() << ", newTableAddress=" << newTableAddress.address());
    if (!oldTablePtr.isNull() && oldTablePtr != newTableAddress) {
        DEBUG_PRINTLN("DEBUG: Calling free() for old table");
        free(oldTablePtr, oldTableSize);
    } else {
        DEBUG_PRINTLN("DEBUG: NOT calling free() - condition not met");
    }
}

// Free list management
std::vector<BT_FreeListEntry> BinaryTable::getFreeList() {
    if (freeListLifted_) {
        return freeListCache_;
    }

    int32_t entryCount = readInt32(14);
    if (entryCount == 0) {
        return {};
    }

    int32_t entrySize = 8 + 4; // Pointer + Size
    int32_t freeListSize = entryCount * entrySize;
    int64_t fileLength = getFileLength();
    int64_t freeListStart = fileLength - freeListSize;

    std::vector<BT_FreeListEntry> freeList;
    for (int32_t i = 0; i < entryCount; i++) {
        int64_t offset = freeListStart + i * entrySize;
        int64_t pointerAddress = readInt64(offset);
        int32_t size = readInt32(offset + 8);
        freeList.emplace_back(BT_Pointer(pointerAddress), size);
    }

    return freeList;
}

void BinaryTable::setFreeList(const std::vector<BT_FreeListEntry>& list) {
    DEBUG_PRINTLN("DEBUG: setFreeList called with freeListLifted_=" << freeListLifted_ << ", list.size()=" << list.size());
    if (freeListLifted_) {
        freeListCache_ = list;
        DEBUG_PRINTLN("DEBUG: setFreeList early return - just updating cache");
        return;
    }

    // Read OLD count from header (position 14)
    int32_t oldEntryCount = readInt32(14);
    
    // Calculate old free list size (entries only, not count)
    int32_t oldListSize = oldEntryCount * (8 + 4);
    
    // Remove old free list entries from EOF
    if (oldEntryCount > 0) {
        int64_t currentLength = getFileLength();
        truncateFile(currentLength - oldListSize);
    }
    
    // Write NEW count to header (position 14)
    writeInt32(14, static_cast<int32_t>(list.size()));
    
    // Write NEW entries to EOF (if any)
    if (!list.empty()) {
        // Encode new free list entries
        std::vector<uint8_t> buffer;
        
        for (const auto& entry : list) {
            // Pointer (8 bytes, little endian)
            int64_t addr = entry.pointer.address();
            for (int i = 0; i < 8; i++) {
                buffer.push_back(static_cast<uint8_t>((addr >> (i * 8)) & 0xFF));
            }
            // Size (4 bytes, little endian)
            int32_t size = entry.size;
            for (int i = 0; i < 4; i++) {
                buffer.push_back(static_cast<uint8_t>((size >> (i * 8)) & 0xFF));
            }
        }
        
        fseek(file_, 0, SEEK_END);
        fwrite(buffer.data(), 1, buffer.size(), file_);
    }
    
    fflush(file_);
}

void BinaryTable::truncateFile(int64_t newSize) {
    // Actually truncate the file (matching Dart behavior)
    DEBUG_PRINTLN("DEBUG: truncateFile - truncating to " << newSize);
    fclose(file_);

    try {
        std::filesystem::resize_file(filePath_, newSize);
        DEBUG_PRINTLN("DEBUG: truncateFile - resize successful");
    } catch (const std::exception& e) {
        DEBUG_PRINTLN("DEBUG: truncateFile - resize failed: " << e.what());
        (void)e; // Suppress unused variable warning in release builds
    }

    file_ = fopen(filePath_.c_str(), "r+b");
    DEBUG_PRINTLN("DEBUG: truncateFile - reopen: success=" << (file_ != nullptr));
}

void BinaryTable::liftFreeList() {
    DEBUG_PRINTLN("DEBUG: liftFreeList() called - this truncates the file!");
    if (freeListLifted_) {
        throw std::runtime_error("Free list is already lifted");
    }

    // Cache the free list
    freeListCache_ = getFreeList();

    // Read count from header (position 14)
    int32_t oldEntryCount = readInt32(14);

    if (oldEntryCount > 0) {
        int32_t oldEntrySize = 8 + 4;
        int32_t oldFreeListSize = oldEntryCount * oldEntrySize;  // Just entries, no count

        // Remove free list entries from EOF
        int64_t fileLength = getFileLength();
        truncateFile(fileLength - oldFreeListSize);
    }

    // Clear count in header (position 14)
    writeInt32(14, 0);

    freeListLifted_ = true;
}

void BinaryTable::dropFreeList() {
    DEBUG_PRINTLN("DEBUG: dropFreeList() called - this writes data back to file!");
    if (!freeListLifted_) {
        throw std::runtime_error("Free list is not lifted");
    }

    freeListLifted_ = false;
    setFreeList(freeListCache_);  // This now writes count to header and entries to EOF
    freeListCache_.clear();
}

void BinaryTable::antiFreeListScope(std::function<void()> fn) {
    liftFreeList();
    try {
        fn();
    } catch (...) {
        dropFreeList();
        throw;
    }
    dropFreeList();
}

// Memory management
void BinaryTable::free(BT_Pointer pointer, int32_t size) {
    DEBUG_PRINTLN("DEBUG: free() called with freeListLifted_=" << freeListLifted_);
    if (!freeListLifted_) {
        DEBUG_PRINTLN("DEBUG: free() THROWING EXCEPTION - free list not lifted!");
        throw std::runtime_error("Free list must be lifted before freeing memory");
    }

    if (pointer.isNull() || size <= 0) {
        throw std::invalid_argument("Cannot free null pointer or zero size");
    }

    // Fetch current free list (matching Dart exactly)
    std::vector<BT_FreeListEntry> freeList = freeListCache_;

    // Add new free entry
    freeList.emplace_back(pointer, size);

    // Merge contiguous free entries (matching Dart logic exactly)
    auto mergeContiguousFreeBlocks = [](std::vector<BT_FreeListEntry> freeList) -> std::vector<BT_FreeListEntry> {
        if (freeList.empty()) return {};

        // Create a copy and sort by address to check for contiguous blocks
        std::vector<BT_FreeListEntry> sorted = freeList;
        std::sort(sorted.begin(), sorted.end(),
                  [](const BT_FreeListEntry& a, const BT_FreeListEntry& b) {
                      return a.pointer.address() < b.pointer.address();
                  });

        std::vector<BT_FreeListEntry> merged;

        for (const auto& entry : sorted) {
            if (merged.empty()) {
                // First entry, just add it
                merged.emplace_back(entry.pointer, entry.size);
            } else {
                auto& last = merged.back();

                // Check if current entry is contiguous with the last merged entry
                if (last.pointer.address() + last.size == entry.pointer.address()) {
                    // Merge: extend the size of the last entry
                    last.size += entry.size;
                } else {
                    // Not contiguous, add as separate entry
                    merged.emplace_back(entry.pointer, entry.size);
                }
            }
        }

        return merged;
    };

    freeList = mergeContiguousFreeBlocks(freeList);

    // Update free list
    freeListCache_ = freeList;
}

BT_Pointer BinaryTable::alloc(int32_t size) {
    if (!freeListLifted_) {
        throw std::runtime_error("Free list must be lifted before allocation");
    }

    // Find suitable free block
    auto it = std::find_if(freeListCache_.begin(), freeListCache_.end(),
                          [size](const BT_FreeListEntry& entry) {
                              return entry.size >= size;
                          });

    if (it == freeListCache_.end()) {
        // No suitable block, allocate at end of file
        int64_t allocPos = getFileLength();
        return BT_Pointer(allocPos);
    }

    BT_Pointer result = it->pointer;

    if (it->size == size) {
        // Exact fit, remove block
        freeListCache_.erase(it);
    } else {
        // Split block
        it->pointer = BT_Pointer(it->pointer.address() + size);
        it->size -= size;
    }

    return result;
}

// Data operations
BT_Reference BinaryTable::getReference(const std::string& key) {
    auto addressTable = getAddressTable();
    int64_t keyHash = hashString(key);

    auto it = addressTable.find(keyHash);
    if (it == addressTable.end()) {
        throw std::runtime_error("Key does not exist");
    }

    return BT_Reference(this, it->second);
}

void BinaryTable::remove(const std::string& key) {
    antiFreeListScope([&]() {
        auto addressTable = getAddressTable();
        int64_t keyHash = hashString(key);

        auto it = addressTable.find(keyHash);
        if (it == addressTable.end()) {
            throw std::runtime_error("Key does not exist");
        }

        BT_Reference valueRef(this, it->second);
        free(it->second, valueRef.size());

        addressTable.erase(it);
        setAddressTable(addressTable);
    });
}

void BinaryTable::truncate() {
    antiFreeListScope([&]() {
        // Relocate address table
        setAddressTable(getAddressTable());

        // Check if last free block is at end of file
        auto freeList = getFreeList();
        if (freeList.empty()) {
            return;
        }

        std::sort(freeList.begin(), freeList.end(),
                 [](const BT_FreeListEntry& a, const BT_FreeListEntry& b) {
                     return a.pointer.address() < b.pointer.address();
                 });

        const auto& lastEntry = freeList.back();
        int64_t fileEnd = getFileLength();
        int64_t expectedEnd = lastEntry.pointer.address() + lastEntry.size;

        if (expectedEnd == fileEnd) {
            freeList.pop_back();
            setFreeList(freeList);

            // Actually truncate file (matching Dart behavior)
            truncateFile(lastEntry.pointer.address());
        }
    });
}

// Debug methods
void BinaryTable::debugAddressTable(const std::string& context) {
    DEBUG_PRINT("\n=== DEBUG ADDRESS TABLE");
    if (!context.empty()) {
        DEBUG_PRINT(" (" << context << ")");
    }
    DEBUG_PRINTLN(" ===");

    auto addressTable = getAddressTable();
    DEBUG_PRINTLN("Address table has " << addressTable.size() << " entries");

    for (const auto& [hash, pointer] : addressTable) {
        DEBUG_PRINTLN("  Hash " << hash << " -> Address " << pointer.address());

        if (!pointer.isNull()) {
            try {
                uint8_t typeByte = readByte(pointer.address());
                DEBUG_PRINTLN("    Type byte: " << (int)typeByte);

                if (typeByte == 2) { // INTEGER
                    int32_t value = readInt32(pointer.address() + 1);
                    DEBUG_PRINTLN("    Value: " << value);
                    (void)value; // Suppress unused variable warning in release builds
                } else {
                    DEBUG_PRINT("    Raw bytes: ");
                    for (int i = 0; i < 8; i++) {
                        uint8_t byte = readByte(pointer.address() + i);
                        DEBUG_PRINT(std::hex << (int)byte << " ");
                        (void)byte; // Suppress unused variable warning in release builds
                    }
                    DEBUG_PRINTLN(std::dec);
                }
            } catch (const std::exception& e) {
                DEBUG_PRINTLN("    Error reading data: " << e.what());
                (void)e; // Suppress unused variable warning in release builds
            }
        }
    }
    DEBUG_PRINTLN("=========================");
}

std::string binaryDump(const std::vector<uint8_t>& data) {
    std::ostringstream buffer;
    
    for (size_t i = 0; i < data.size(); i += 16) {
        // Address
        buffer << "0x" << std::hex << std::setfill('0') << std::setw(4) << std::uppercase << i 
               << " (" << std::dec << std::setfill(' ') << std::setw(4) << i << ") | ";
        
        // Hex bytes
        for (int j = 0; j < 16; j++) {
            if (i + j < data.size()) {
                buffer << std::hex << std::setfill('0') << std::setw(2) << std::uppercase 
                       << static_cast<int>(data[i + j]) << " ";
            } else {
                buffer << "   ";
            }
        }
        
        buffer << " | ";
        
        // Integer representation
        for (int j = 0; j < 16; j++) {
            if (i + j < data.size()) {
                buffer << std::dec << std::setfill(' ') << std::setw(3) 
                       << static_cast<int>(data[i + j]) << " ";
            } else {
                buffer << "    ";
            }
        }
        
        buffer << " | ";
        
        // ASCII representation
        for (int j = 0; j < 16; j++) {
            if (i + j < data.size()) {
                uint8_t byte = data[i + j];
                if (byte >= 32 && byte <= 126) {
                    buffer << static_cast<char>(byte);
                } else {
                    buffer << '.';
                }
            }
        }
        
        buffer << " | ";
        if (i + 16 < data.size()) {
            buffer << "\n";
        }
    }
    
    return buffer.str();
}

} // namespace bt

int main() {
    // Delete existing file if it exists
    if (std::filesystem::exists("example.bin")) {
        std::filesystem::remove("example.bin");
    }
    
    bt::BinaryTable table("example.bin");
    table.initialize();
    
    // Read file for dump
    std::ifstream file("example.bin", std::ios::binary);
    std::vector<uint8_t> fileData((std::istreambuf_iterator<char>(file)),
                                  std::istreambuf_iterator<char>());
    file.close();
    
    std::cout << "File dump:\n";
    std::cout << bt::binaryDump(fileData) << "\n";
    std::cout << "File size: " << fileData.size() << " bytes\n";
    std::cout << " \n";
    
    // Set values
    table.set("int_array", std::vector<int32_t>{6, 3, 9, 2, 5});
    table.set("float_array", std::vector<float>{1.5f, 2.5f, 3.5f});
    table.set("empty", std::vector<int32_t>{});
    
    // Get arrays and modify elements
    auto intArray = table.getArray<int32_t>("int_array");
    auto floatArray = table.getArray<float>("float_array");
    
    intArray.set(0, 1);
    floatArray.set(1, 4.5f);
    
    std::cout << "int_array pointer: " << intArray.getPointer().address() << "\n";
    std::cout << "float_array pointer: " << floatArray.getPointer().address() << "\n";
    
    intArray.add(10);
    floatArray.add(5.5f);
    
    intArray.addAll({420, 69, 1337, 1738});
    floatArray.addAll({6.5f, 7.5f, 8.5f});
    
    // Test the fetchSublist method
    auto intArraySublist = intArray.fetchSublist(0, 3);
    auto floatArraySublist = floatArray.fetchSublist(0, 2);
    std::cout << "Sublist int_array (0-3): ";
    for (auto val : intArraySublist) {
        std::cout << val << " ";
    }
    std::cout << "\n";
    std::cout << "Sublist float_array (0-2): ";
    for (auto val : floatArraySublist) {
        std::cout << val << " ";
    }
    std::cout << "\n";
    
    // Read back values
    auto readback1 = table.get<std::vector<int32_t>>("int_array");
    auto readback2 = table.get<std::vector<float>>("float_array");
    auto readback3 = table.get<std::vector<int32_t>>("empty");
    
    std::cout << "Readback1: ";
    for (auto val : readback1) {
        std::cout << val << " ";
    }
    std::cout << "\n";
    
    std::cout << "Readback2: ";
    for (auto val : readback2) {
        std::cout << val << " ";
    }
    std::cout << "\n";
    
    std::cout << "Readback3: ";
    for (auto val : readback3) {
        std::cout << val << " ";
    }
    std::cout << "\n";
    
    std::cout << " \n";
    
    // Final file dump
    std::ifstream finalFile("example.bin", std::ios::binary);
    std::vector<uint8_t> finalFileData((std::istreambuf_iterator<char>(finalFile)),
                                       std::istreambuf_iterator<char>());
    finalFile.close();
    
    std::cout << "File dump:\n";
    std::cout << bt::binaryDump(finalFileData) << "\n";
    std::cout << "File size: " << finalFileData.size() << " bytes\n";
    
    return 0;
}