From ea9bf38f8c0448cc7ba39eb6934a0eb7b222c380 Mon Sep 17 00:00:00 2001 From: ImBenji Date: Wed, 17 Sep 2025 23:12:42 +0100 Subject: [PATCH] Add initial implementation of BinaryTable with memory management and random access capabilities --- README.md | 183 ++++++++ analysis_options.yaml | 30 ++ dart/binary_table.dart | 1004 ++++++++++++++++++++++++++++++++++++++++ file_formats.iml | 14 + pubspec.lock | 405 ++++++++++++++++ pubspec.yaml | 16 + 6 files changed, 1652 insertions(+) create mode 100644 README.md create mode 100644 analysis_options.yaml create mode 100644 dart/binary_table.dart create mode 100644 file_formats.iml create mode 100644 pubspec.lock create mode 100644 pubspec.yaml diff --git a/README.md b/README.md new file mode 100644 index 0000000..c6d94f4 --- /dev/null +++ b/README.md @@ -0,0 +1,183 @@ +# SweepStore (formerly BinaryTable) + +A high-performance binary storage format for Dart applications with efficient memory management and random access capabilities. + +## Overview + +BinaryTable is a key-value storage system designed for applications requiring fast, compact data persistence. It features a custom binary format with automatic memory management, making it ideal for game save files, caching systems, and other performance-critical data storage needs. + +## Features + +### Core Storage Types +- **Integers**: 32-bit signed integers +- **Floats**: 32-bit floating-point numbers +- **Strings**: Variable-length UTF-16 strings +- **Uniform Arrays**: Homogeneous arrays of integers or floats with O(1) random access + +### Memory Management +- **Free List System**: Automatic memory reclamation and defragmentation +- **Block Merging**: Contiguous free blocks are automatically merged +- **File Truncation**: Unused space at end of file is reclaimed +- **Lift/Drop Mechanism**: Prevents free list fragmentation during operations + +### Performance Features +- **Random Access**: Direct addressing for all data types +- **Minimal Seeks**: Optimized file layout reduces disk I/O +- **Compact Format**: No padding or alignment overhead +- **Efficient Arrays**: Uniform arrays use offset calculations for O(1) access + +## Usage + +### Basic Operations + +```dart +// Initialize +File file = File('data.bin'); +file.createSync(); +BinaryTable table = BinaryTable(file.path); +table.initialise(); + +// Store values +table["player_level"] = 42; +table["player_name"] = "Hero"; +table["score"] = 1337.5; + +// Retrieve values +int level = table["player_level"]; +String name = table["player_name"]; +double score = table["score"]; + +// Delete entries +table.delete("old_data"); +``` + +### Array Operations + +```dart +// Create arrays +table["inventory"] = [10, 20, 30, 40, 50]; +table["coordinates"] = [1.5, 2.7, 3.9]; + +// Random access +table["inventory"][0] = 15; +double y = table["coordinates"][1]; + +// Dynamic growth +table["inventory"].add(60); +table["inventory"].addAll([70, 80, 90]); + +// Pre-sized arrays (recommended for game data) +table["heightmap"] = List.filled(1024 * 1024, 0.0); +``` + +### Memory Management + +```dart +// Reclaim unused space +table.truncate(); + +// Manual cleanup (usually automatic) +table.delete("unused_key"); +table.truncate(); +``` + +## Version History + +### Version 3.0 - Current +**Added: Uniform Arrays** +- `BT_UniformArray` class for homogeneous array storage +- O(1) random access for integer and float arrays +- Dynamic array growth with automatic reallocation +- Type safety enforcement for array elements +- Support for empty arrays with type inference on first use + +**Improvements:** +- Enhanced memory management for variable-size data +- Optimized file format for array storage +- Better error handling for type mismatches + +### Version 2.0 +**Added: Persistent Free Lists** +- Free list data persists across program restarts +- Automatic block merging for defragmentation +- File truncation to reclaim unused space +- "Read from end" encoding for free list metadata + +**Improvements:** +- Eliminated memory leaks from previous version +- Reduced file size growth over time +- Better space utilization efficiency + +### Version 1.0 +**Core Features:** +- Basic key-value storage (integers, floats, strings) +- Binary file format with type identification +- Address table for O(1) key lookups +- Volatile garbage collection (in-memory only) + +**Initial Implementation:** +- RandomAccessFile-based storage +- Little-endian encoding +- Hash-based key addressing + +## File Format + +The BinaryTable uses a custom binary format optimized for random access: + +### Header Structure +``` +[Address Table Pointer: 8 bytes] +[Free List Entry Count: 4 bytes] +``` + +### Data Layout +- **Address Table**: Hash-to-pointer mappings at variable location +- **Free List**: Memory management metadata at end of file +- **Data Blocks**: Variable-size type-prefixed data scattered throughout + +### Type Encoding +- Each value prefixed with type identifier byte +- Variable-length data includes size information +- Arrays store length followed by elements + +## Performance Characteristics + +### Time Complexity +- **Key Lookup**: O(1) average case +- **Array Access**: O(1) for uniform arrays +- **Insertion**: O(1) amortized +- **Deletion**: O(1) plus defragmentation cost + +### Space Efficiency +- **Overhead**: ~13 bytes per key-value pair minimum +- **Arrays**: 5 bytes + (5 bytes × length) for uniform arrays +- **Fragmentation**: Automatically managed and minimized + +## Use Cases + +### Game Development +- Save file persistence with fast loading +- Level data storage (heightmaps, object placement) +- Player progression and inventory systems +- Settings and configuration data + +### General Applications +- High-performance caching systems +- Embedded database applications +- Configuration file storage +- Any scenario requiring fast binary persistence + +## Limitations + +- **Hash Collisions**: Uses Dart's built-in string hashCode (risk of collisions) +- **File Corruption**: No built-in checksums or validation +- **Concurrent Access**: Single-threaded access only +- **Type System**: Limited compared to JSON (no nested objects, mixed arrays) + +## Future Roadmap + +- Full JSON parity with nested objects and mixed-type arrays +- Compression support for large data blocks +- Checksums and data integrity validation +- Multi-threaded access with file locking +- Query system for complex data retrieval \ No newline at end of file diff --git a/analysis_options.yaml b/analysis_options.yaml new file mode 100644 index 0000000..dee8927 --- /dev/null +++ b/analysis_options.yaml @@ -0,0 +1,30 @@ +# This file configures the static analysis results for your project (errors, +# warnings, and lints). +# +# This enables the 'recommended' set of lints from `package:lints`. +# This set helps identify many issues that may lead to problems when running +# or consuming Dart code, and enforces writing Dart using a single, idiomatic +# style and format. +# +# If you want a smaller set of lints you can change this to specify +# 'package:lints/core.yaml'. These are just the most critical lints +# (the recommended set includes the core lints). +# The core lints are also what is used by pub.dev for scoring packages. + +include: package:lints/recommended.yaml + +# Uncomment the following section to specify additional rules. + +# linter: +# rules: +# - camel_case_types + +# analyzer: +# exclude: +# - path/to/excluded/files/** + +# For more information about the core and recommended set of lints, see +# https://dart.dev/go/core-lints + +# For additional information about configuring this file, see +# https://dart.dev/guides/language/analysis-options diff --git a/dart/binary_table.dart b/dart/binary_table.dart new file mode 100644 index 0000000..f3fd271 --- /dev/null +++ b/dart/binary_table.dart @@ -0,0 +1,1004 @@ +/* + + /$$$$$$ /$$ /$$ /$$$$$$$ /$$$$$$$$ /$$ /$$ /$$$$$ /$$$$$$ /$$ /$$ /$$$$$$$$ /$$$$$$$$ +|_ $$_/| $$$ /$$$| $$__ $$| $$_____/| $$$ | $$ |__ $$|_ $$_/ | $$$ | $$| $$_____/|__ $$__/ + | $$ | $$$$ /$$$$| $$ \ $$| $$ | $$$$| $$ | $$ | $$ | $$$$| $$| $$ | $$ + | $$ | $$ $$/$$ $$| $$$$$$$ | $$$$$ | $$ $$ $$ | $$ | $$ | $$ $$ $$| $$$$$ | $$ + | $$ | $$ $$$| $$| $$__ $$| $$__/ | $$ $$$$ /$$ | $$ | $$ | $$ $$$$| $$__/ | $$ + | $$ | $$\ $ | $$| $$ \ $$| $$ | $$\ $$$| $$ | $$ | $$ | $$\ $$$| $$ | $$ + /$$$$$$| $$ \/ | $$| $$$$$$$/| $$$$$$$$| $$ \ $$| $$$$$$/ /$$$$$$ /$$| $$ \ $$| $$$$$$$$ | $$ +|______/|__/ |__/|_______/ |________/|__/ \__/ \______/ |______/|__/|__/ \__/|________/ |__/ + +© 2025-26 by Benjamin Watt of IMBENJI.NET LIMITED - All rights reserved. + +Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. + + + */ + +import 'dart:io'; +import 'dart:typed_data'; + +enum BT_Type { + + POINTER (8), + ADDRESS_TABLE (-1), + + INTEGER (4), + FLOAT (4), + STRING (-1), + + INTEGER_ARRAY (-1, arrayType: true), + FLOAT_ARRAY (-1, arrayType: true),; + + + final int size; + final bool arrayType; + const BT_Type(this.size, { + this.arrayType = false, + }); + + static BT_Type fromId(int id) { + if (id < 0 || id >= BT_Type.values.length) { + throw ArgumentError('Invalid BT_Type id: $id'); + } + return BT_Type.values[id]; + } + + static BT_Type fromDynamic(dynamic value) { + if (value is int) { + return BT_Type.INTEGER; + } else if (value is double) { + return BT_Type.FLOAT; + } else if (value is String) { + return BT_Type.STRING; + } else if (value is List) { + return BT_Type.INTEGER_ARRAY; + } else if (value is List) { + return BT_Type.FLOAT_ARRAY; + } else { + throw ArgumentError('Unsupported type: ${value.runtimeType}'); + } + } + +} + +List encodeValue(dynamic value) { + + BT_Type valueType = BT_Type.fromDynamic(value); + List valueBuffer = [valueType.index]; + + + if (valueType == BT_Type.INTEGER) { + + valueBuffer.addAll((ByteData(4)..setInt32(0, value as int, Endian.little)).buffer.asUint8List()); + + } else if (valueType == BT_Type.FLOAT) { + + valueBuffer.addAll((ByteData(4)..setFloat32(0, value as double, Endian.little)).buffer.asUint8List()); + + } else if (valueType == BT_Type.STRING) { + // String length + valueBuffer.addAll((ByteData(4) + ..setInt32(0, (value as String).length, Endian.little)).buffer + .asUint8List()); + // String bytes + valueBuffer.addAll(value.codeUnits); + } else if (valueType == BT_Type.INTEGER_ARRAY) { + List list = value as List; + // Length + valueBuffer.addAll((ByteData(4) + ..setInt32(0, list.length, Endian.little)).buffer.asUint8List()); + // Entries + for (var item in list) { + valueBuffer.addAll(encodeValue(item)); + } + } else if (valueType == BT_Type.FLOAT_ARRAY) { + List list = value as List; + // Length + valueBuffer.addAll((ByteData(4) + ..setInt32(0, list.length, Endian.little)).buffer.asUint8List()); + // Entries + for (var item in list) { + valueBuffer.addAll(encodeValue(item)); + } + } else { + + throw ArgumentError('Unsupported type: ${value.runtimeType}'); + + } + + return valueBuffer; +} + +class BT_Pointer { + final int address; + + const BT_Pointer(this.address); + + bool get isNull => address == -1; + + operator ==(Object other) { + if (other is! BT_Pointer) return false; + return address == other.address; + } + + @override + String toString() => '0x${address.toRadixString(16)} ($address)'; +} +const BT_Pointer BT_Null = BT_Pointer(-1); + +class BT_Reference { + + BinaryTable _table; + BT_Pointer _pointer; + + BT_Reference(this._table, this._pointer); + + dynamic decodeValue() { + + if (_pointer.isNull) { + return null; + } + + _table._file.setPositionSync(_pointer.address); + int typeId = _table._file.readByteSync(); + BT_Type type = BT_Type.fromId(typeId); + + if (type == BT_Type.INTEGER) { + + return _table._file.readIntSync(4); + + } else if (type == BT_Type.FLOAT) { + + return _table._file.readFloat32Sync(); + + } else if (type == BT_Type.STRING) { + int length = _table._file.readIntSync(4); + List bytes = _table._file.readSync(length); + return String.fromCharCodes(bytes); + + } else if (type == BT_Type.ADDRESS_TABLE) { + // Address table decoding not implemented + throw UnimplementedError('Address table decoding not implemented'); + } else if (type == BT_Type.POINTER) { + return _table._file.readPointerSync(); + } else if (type == BT_Type.INTEGER_ARRAY) { + return BT_UniformArray(_table, _pointer); + } else if (type == BT_Type.FLOAT_ARRAY) { + return BT_UniformArray(_table, _pointer); + } else { + throw Exception('Unsupported type: $type'); + } + + } + + /// Determine the size in storage in bytes with the least amount of reads possible + int get size { + if (_pointer.isNull) { + return 0; + } + + _table._file.setPositionSync(_pointer.address); + int typeId = _table._file.readByteSync(); + BT_Type type = BT_Type.fromId(typeId); + + if (type == BT_Type.INTEGER) { + + return 1 + 4; // Type byte + Int32 + + } else if (type == BT_Type.FLOAT) { + + return 1 + 4; // Type byte + Float32 + + } else if (type == BT_Type.STRING) { + + int length = _table._file.readIntSync(4); + return 1 + 4 + length; // Type byte + Length Int32 + String bytes + + } else if (type == BT_Type.ADDRESS_TABLE) { + + // Address table size is variable, need to read the count + int count = _table._file.readIntSync(4); + return 1 + 4 + count * + (8 + BT_Type.POINTER.size); // Type byte + Count Int32 + Entries + + } else { + throw Exception('Unsupported type: $type'); + } + } + + @override + String toString() => _pointer.toString(); +} + +class BT_FreeListEntry { + BT_Pointer pointer; + int size; + + BT_FreeListEntry(this.pointer, this.size); +} + +/// Wrapper for interacting with uniform arrays in the binary table. A uniform array is an array where all elements are of the same type, allowing for more efficient storage and retrieval using random access. +class BT_UniformArray extends BT_Reference { + + BT_UniformArray(super._table, super._pointer); + + /// Get the length of the array - Don't mix up with size. + int get length { + if (_pointer.isNull) { + return 0; + } + + _table._file.setPositionSync(_pointer.address); + int typeId = _table._file.readByteSync(); + BT_Type type = BT_Type.fromId(typeId); + + if (!type.arrayType) { + throw Exception('Not an array'); + } + + return _table._file.readIntSync(4); + } + + dynamic operator [](int index) { + if (_pointer.isNull) { + throw Exception('Null pointer'); + } + + int len = length; + if (index < 0 || index >= len) { + throw RangeError.index(index, this, 'index', null, len); + } + + // Determine the type of the array by reading the first items type + _table._file.setPositionSync(_pointer.address + 1 + 4); + int typeId = _table._file.readByteSync(); + BT_Type type = BT_Type.fromId(typeId); + + int itemOffset = index * (1 + type.size); // Type byte + data size + BT_Reference itemRef = BT_Reference(_table, BT_Pointer((_pointer.address + 1 + 4) + itemOffset)); + return itemRef.decodeValue(); + } + + void operator []=(int index, dynamic value) { + if (_pointer.isNull) { + throw Exception('Null pointer'); + } + + int len = length; + if (index < 0 || index >= len) { + throw RangeError.index(index, this, 'index', null, len); + } + + // Determine the type of the array by reading the first items type + _table._file.setPositionSync(_pointer.address + 1 + 4); + int typeId = _table._file.readByteSync(); + BT_Type type = BT_Type.fromId(typeId); + if (type.size == -1) { + throw Exception("Types with variable size are not supported in uniform arrays. Use a non-uniform array instead."); + } + + // Ensure the new value is of the correct type + BT_Type newValueType = BT_Type.fromDynamic(value); + if (newValueType != type) { + throw Exception('Type mismatch. Expected $type but got $newValueType'); + } + + // Calculate the offset of the item to update + int itemOffset = index * (1 + type.size); // Type byte + data size + BT_Pointer itemPointer = BT_Pointer((_pointer.address + 1 + 4) + itemOffset); + + // Encode the new value + List valueBuffer = encodeValue(value); + + // Place the new value in the file + _table._file.setPositionSync(itemPointer.address); + _table._file.writeFromSync(valueBuffer); + + } + + void addAll(Iterable values) { + + _table.antiFreeListScope(() { + // Determine the type of the array by reading the first items type + BT_Type type = elementType ?? BT_Type.fromDynamic(values.first); + + // Validate all new values are of the correct type + for (int i = 0; i < values.length; i++) { + BT_Type newValueType = BT_Type.fromDynamic(values.elementAt(i)); + if (newValueType != type) { + throw Exception('Type mismatch at index $i. Expected $type but got $newValueType'); + } else if (newValueType.size == -1) { + throw Exception("Types with variable size are not supported in uniform arrays. Use a non-uniform array instead."); + } + } + + // Read the full array by loading the full buffer + List fullBuffer = []; + int bufferSize = 1 + 4 + length * (1 + type.size); + _table._file.setPositionSync(_pointer.address); + fullBuffer = _table._file.readSync(bufferSize).toList(); + + // Encode the new values and add them to the buffer + for (var value in values) { + fullBuffer.addAll(encodeValue(value)); + } + + // Update the length in the buffer + int newLength = length + values.length; + List lengthBytes = (ByteData(4)..setInt32(0, newLength, Endian.little)).buffer.asUint8List(); + fullBuffer.replaceRange(1, 5, lengthBytes); + + // Free the old array + _table.free(_pointer, size); + + // Allocate new space for the updated array + BT_Pointer newPointer = _table.alloc(fullBuffer.length); + + // Replace any references to the old pointer with the new one + Map addressTable = _table._addressTable; + addressTable.updateAll((key, value) { + if (value == _pointer) { + print('Updating address table entry for key $key from $value to $newPointer'); + return newPointer; + } + return value; + }); + _table._addressTable = addressTable; + _pointer = newPointer; + + // Write the updated buffer to the new location + _table._file.setPositionSync(newPointer.address); + _table._file.writeFromSync(fullBuffer); + + print('Array resized to new length $newLength at $newPointer'); + }); + + } + + void add(dynamic value) { + addAll([value]); + } + + @override + int get size { + + if (length == 0) { + return 1 + 4; // Type byte + Length Int32 + } + + _table._file.setPositionSync(_pointer.address); + int typeId = _table._file.readByteSync(); + BT_Type type = BT_Type.fromId(typeId); + + if (type.arrayType) { + + _table._file.setPositionSync(_pointer.address); + int bufferSize = 1 + 4 + length * (1 + elementType!.size); + + return bufferSize; + } + + return super.size; + } + + BT_Type? get elementType { + if (length == 0) { + return null; + } else { + _table._file.setPositionSync(_pointer.address + 1 + 4); + int typeId = _table._file.readByteSync(); + BT_Type type = BT_Type.fromId(typeId); + return type; + } + } + + @override + String toString([bool readValues = false]) { + + if (readValues) { + return 'Uniform Array of length $length'; + } + + List preview = []; + int len = length; + for (int i = 0; i < length; i++) { + preview.add(this[i]); + } + + return 'Uniform Array: $preview'; + + } + +} + +extension FreeList on List { + + void removePointer(BT_Pointer pointer) { + removeWhere((entry) => entry.pointer == pointer); + } + + List bt_encode() { + List buffer = []; + + /* + + New encoding should reflect a "read from the end" approach. + So it should look like: + - Entries (variable) + - Entry count (4 bytes) + + Entry: + - Pointer (8 bytes) + - Size (4 bytes) + + This means that when reading, we can always assume that the last 4 bytes in the file are the entry count for the free list. + The entries however, can be read from front to back. Once we know the count, we know where to start reading. + */ + + for (var entry in this) { + // Pointer + buffer.addAll((ByteData(BT_Type.POINTER.size)..setInt64(0, entry.pointer.address, Endian.little)).buffer.asUint8List()); + // Size + buffer.addAll((ByteData(4)..setInt32(0, entry.size, Endian.little)).buffer.asUint8List()); + } + + // Entry count + buffer.addAll((ByteData(4)..setInt32(0, length, Endian.little)).buffer.asUint8List()); + + return buffer; + } + +} + +class BinaryTable { + + RandomAccessFile _file; + + BinaryTable(String path) : _file = File(path).openSync(mode: FileMode.append); + + void initialise() { + _file.setPositionSync(0); + _file.writePointerSync(BT_Null); // Address table pointer + _file.writeIntSync(0, 4); // Free list entry count + } + + Map get _addressTable { + + _file.setPositionSync(0); + BT_Reference tableRef = BT_Reference(this, _file.readPointerSync()); + + if (tableRef._pointer.isNull) { + return {}; + } + + _file.setPositionSync(tableRef._pointer.address + 1); + int fileSize = _file.lengthSync(); + int tableCount = _file.readIntSync(4); + int tableSize = tableCount * (8 + BT_Type.POINTER.size); + List buffer = _file.readSync(tableSize).toList(); + + Map addressTable = {}; + + for (int i = 0; i < tableCount; i++) { + + int offset = i * (8 + BT_Type.POINTER.size); + + // Key Hash + List keyHashBytes = buffer.sublist(offset, offset + 8); + int keyHash = ByteData.sublistView(Uint8List.fromList(keyHashBytes)).getInt64(0, Endian.little); + + // Value Pointer + List valuePointerBytes = buffer.sublist(offset + 8, offset + 8 + BT_Type.POINTER.size); + int valuePointerAddress = ByteData.sublistView(Uint8List.fromList(valuePointerBytes)).getInt64(0, Endian.little); + BT_Pointer valuePointer = BT_Pointer(valuePointerAddress); + + // Add to address table + addressTable[keyHash] = valuePointer; + } + + return addressTable; + + } + set _addressTable(Map table) { + List buffer = [ + BT_Type.ADDRESS_TABLE.index + ]; + + buffer.addAll((ByteData(4)..setInt32(0, table.length, Endian.little)).buffer.asUint8List()); + table.forEach((key, value) { + buffer.addAll((ByteData(8)..setInt64(0, key, Endian.little)).buffer.asUint8List()); + buffer.addAll((ByteData(BT_Type.POINTER.size)..setInt64(0, value.address, Endian.little)).buffer.asUint8List()); + }); + + // Write new address table at end of file + BT_Pointer tableAddress = alloc(buffer.length); + _file.setPositionSync(tableAddress.address); + _file.writeFromSync(buffer); + + // Read old table pointer before updating + _file.setPositionSync(0); + BT_Reference oldTableRef = BT_Reference(this, _file.readPointerSync()); + + // Update header to point to new table + _file.setPositionSync(0); + _file.writePointerSync(tableAddress); + + // Now free the old table if it exists and is not the same as the new one + if (!oldTableRef._pointer.isNull && oldTableRef._pointer != tableAddress) { + free(oldTableRef._pointer, oldTableRef.size); + } + } + + bool freeListLifted = false; + List? _freeListCache; + + List get _freeList { + + if (freeListLifted) { + return _freeListCache ?? []; + } + + _file.setPositionSync(_file.lengthSync() - 4); + int entryCount = _file.readIntSync(4); + if (entryCount == 0) { + return []; + } + + int entrySize = BT_Type.POINTER.size + 4; // Pointer + Size + int freeListSize = entryCount * entrySize; + _file.setPositionSync(_file.lengthSync() - 4 - freeListSize); + List buffer = _file.readSync(freeListSize); + + List freeList = []; + for (int i = 0; i < entryCount; i++) { + int offset = i * entrySize; + + // Pointer + List pointerBytes = buffer.sublist(offset, offset + BT_Type.POINTER.size); + int pointerAddress = ByteData.sublistView(Uint8List.fromList(pointerBytes)).getInt64(0, Endian.little); + BT_Pointer pointer = BT_Pointer(pointerAddress); + + // Size + List sizeBytes = buffer.sublist(offset + BT_Type.POINTER.size, offset + entrySize); + int size = ByteData.sublistView(Uint8List.fromList(sizeBytes)).getInt32(0, Endian.little); + + freeList.add(BT_FreeListEntry(pointer, size)); + } + + return freeList; + } + set _freeList(List list) { + + if (freeListLifted) { + _freeListCache = list; + return; + } + + _file.setPositionSync(_file.lengthSync() - 4); + int oldEntryCount = _file.readIntSync(4); + int oldListSize = (oldEntryCount * (BT_Type.POINTER.size + 4)) + 4; // Entries + Count + _file.truncateSync(_file.lengthSync() - oldListSize); + + List buffer = list.bt_encode(); + _file.setPositionSync(_file.lengthSync()); + _file.writeFromSync(buffer); + } + + /// Caches the free list in memory, and removed it from the file. + void liftFreeList() { + if (freeListLifted) { + throw StateError('Free list is already lifted'); + } + + _freeListCache = _freeList; + + _file.setPositionSync(_file.lengthSync() - 4); + int oldEntryCount = _file.readIntSync(4); + int oldEntrySize = BT_Type.POINTER.size + 4; // Pointer + Size + int oldFreeListSize = oldEntryCount * oldEntrySize + 4; // +4 for entry count + _file.truncateSync(_file.lengthSync() - oldFreeListSize); + + freeListLifted = true; + } + /// Appends the cached free list back to the file, and clears the cache. + void dropFreeList() { + if (!freeListLifted) { + throw StateError('Free list is not lifted'); + } + + _file.setPositionSync(_file.lengthSync()); + _file.writeIntSync(0, 4); // Placeholder for entry count + + freeListLifted = false; + _freeList = _freeListCache!; + _freeListCache = null; + } + + void antiFreeListScope(void Function() fn) { + liftFreeList(); + try { + fn(); + } finally { + dropFreeList(); + } + } + + void free(BT_Pointer pointer, int size) { + + if (!freeListLifted) { + throw StateError('Free list must be lifted before freeing memory'); + } + + if (pointer.isNull || size <= 0) { + throw ArgumentError('Cannot free null pointer or zero size'); + } + + // Fetch current free list + List freeList = _freeList; + + // Add new free entry + freeList.add(BT_FreeListEntry(pointer, size)); + + // Merge contiguous free entries + List mergeContiguousFreeBlocks(List freeList) { + if (freeList.isEmpty) return []; + + // Create a copy and sort by address to check for contiguous blocks + List sorted = List.from(freeList); + sorted.sort((a, b) => a.pointer.address.compareTo(b.pointer.address)); + + List merged = []; + + for (var entry in sorted) { + if (merged.isEmpty) { + // First entry, just add it + merged.add(BT_FreeListEntry(entry.pointer, entry.size)); + } else { + var last = merged.last; + + // Check if current entry is contiguous with the last merged entry + if (last.pointer.address + last.size == entry.pointer.address) { + // Merge: extend the size of the last entry + last.size += entry.size; + } else { + // Not contiguous, add as separate entry + merged.add(BT_FreeListEntry(entry.pointer, entry.size)); + } + } + } + + return merged; + } + + freeList = mergeContiguousFreeBlocks(freeList); + + // Update free list + _freeList = freeList; + } + BT_Pointer alloc(int size) { + + if (!freeListLifted) { + throw StateError('Free list must be lifted before allocation'); + } + + // Fetch current free list + List freeList = _freeList; + + // If its empty, allocate at end of file + if (freeList.isEmpty) { + // "[ALLOC] CODE 1: No free blocks available, allocating at end of file".yellow.log(); + return BT_Pointer(_file.lengthSync()); + } + + // Find a free block that fits the size (exact fit or larger) + BT_FreeListEntry? bestFit; + for (var entry in freeList) { + if (entry.size >= size) { + bestFit = entry; + break; + } + } + + if (bestFit == null) { + // "[ALLOC] CODE 2: No suitable free block found, allocating at end of file".yellow.log(); + return BT_Pointer(_file.lengthSync()); + } + + bool exactFit = bestFit.size == size; + if (exactFit) { + // "[ALLOC] CODE 3: Found exact fit in free block at ${bestFit.pointer} of size ${bestFit.size} bytes".green.log(); + // Remove from free list + freeList.removePointer(bestFit.pointer); + _freeList = freeList; + return bestFit.pointer; + } else { + // "[ALLOC] CODE 4: Found larger free block at ${bestFit.pointer} of size ${bestFit.size} bytes, allocating ${size} bytes. Splitting block, remaining size ${bestFit.size - size} bytes".green.log(); + // Allocate from start of block, reduce size and move pointer + BT_Pointer allocatedPointer = bestFit.pointer; + bestFit = BT_FreeListEntry(BT_Pointer(bestFit.pointer.address + size), bestFit.size - size); + // Update free list + freeList.removePointer(allocatedPointer); + freeList.add(bestFit); + _freeList = freeList; + return allocatedPointer; + } + } + + operator []=(String key, dynamic value) { + + antiFreeListScope(() { + Map addressTable = _addressTable; + + int keyHash = key.hashCode; + + if (addressTable.containsKey(keyHash)) { + throw Exception('Key already exists'); + } + + + + List valueBuffer = encodeValue(value); + + // Write value to file + BT_Pointer valueAddress = alloc(valueBuffer.length); + + _file.setPositionSync(valueAddress.address); + _file.writeFromSync(valueBuffer); + + // Update address table + addressTable[keyHash] = valueAddress; + _addressTable = addressTable; + }); + + } + operator [](String key) { + Map addressTable = _addressTable; + + int keyHash = key.hashCode; + + if (!addressTable.containsKey(keyHash)) { + throw Exception('Key does not exist'); + } + + BT_Pointer valuePointer = addressTable[keyHash]!; + BT_Reference valueRef = BT_Reference(this, valuePointer); + + return valueRef.decodeValue(); + } + + void delete(String key) { + + antiFreeListScope(() { + Map addressTable = _addressTable; + + int keyHash = key.hashCode; + + if (!addressTable.containsKey(keyHash)) { + throw Exception('Key does not exist'); + } + + BT_Pointer valuePointer = addressTable[keyHash]!; + BT_Reference valueRef = BT_Reference(this, valuePointer); + + // Free the value + free(valuePointer, valueRef.size); + + // Remove from address table + addressTable.remove(keyHash); + _addressTable = addressTable; + }); + + } + + void truncate() { + + antiFreeListScope(() { + // Relocate the address table if possible + _addressTable = _addressTable; + + // Check if the last free block is at the end of the file + List freeList = _freeList; + freeList.sort((a, b) => a.pointer.address.compareTo(b.pointer.address)); + + if (freeList.isEmpty) { + return; + } + + BT_FreeListEntry lastEntry = freeList.last; + int fileEnd = _file.lengthSync(); + int expectedEnd = lastEntry.pointer.address + lastEntry.size; + + if (expectedEnd != fileEnd) { + return; + } + + // Remove the last entry from the free list + freeList.removeLast(); + _freeList = freeList; + + // Truncate the file + int newLength = lastEntry.pointer.address; + _file.truncateSync(newLength); + }); + + } + +} + +void main() { + File file = File('example.bin'); + if (file.existsSync()) { + file.deleteSync(); + } + file.createSync(); + BinaryTable table = BinaryTable(file.path); + table.initialise(); + + print("File dump:"); + print(binaryDump(file.readAsBytesSync())); + print("File size: ${file.lengthSync()} bytes"); + print(" "); + + table["int_array"] = [6, 3, 9, 2, 5]; + table["float_array"] = [1.5, 2.5, 3.5]; + table["empty"] = []; + + table["int_array"][0] = 1; + table["float_array"][1] = 4.5; + + print("int_array pointer: ${table["int_array"]._pointer}"); + print("float_array pointer: ${table["float_array"]._pointer}"); + + table["int_array"].add(10); + table["float_array"].add(5.5); + + table["int_array"].addAll([420, 69, 1337, 1738]); + table["float_array"].addAll([6.5, 7.5, 8.5]); + + var readback1 = table["int_array"]; + var readback2 = table["float_array"]; + var readback3 = table["empty"]; + print("Readback1: $readback1"); + print("Readback2: $readback2"); + print("Readback3: $readback3"); + + print(" "); + print("File dump:"); + print(binaryDump(file.readAsBytesSync())); + print("File size: ${file.lengthSync()} bytes"); +} + +extension on RandomAccessFile { + + // Read/Write Int Dynamic - Can specify size in bytes, does not have to align to 1, 2, 4, or 8 bytes. Default is 4 bytes (Int32) + int readIntSync([int size = 4, Endian endianness = Endian.little]) { + if (size < 1 || size > 8) { + throw ArgumentError('Size must be between 1 and 8 bytes'); + } + + List bytes = readSync(size); + + // Build integer from bytes with proper endianness + int result = 0; + if (endianness == Endian.little) { + for (int i = size - 1; i >= 0; i--) { + result = (result << 8) | bytes[i]; + } + } else { + for (int i = 0; i < size; i++) { + result = (result << 8) | bytes[i]; + } + } + + // Sign extend if MSB is set + int signBit = 1 << (size * 8 - 1); + if (result & signBit != 0) { + result -= 1 << (size * 8); + } + + return result; + } + + void writeIntSync(int value, [int size = 4, Endian endianness = Endian.little]) { + if (size < 1 || size > 8) { + throw ArgumentError('Size must be between 1 and 8 bytes'); + } + + List bytes = List.filled(size, 0); + + // Extract bytes with proper endianness + if (endianness == Endian.little) { + for (int i = 0; i < size; i++) { + bytes[i] = (value >> (i * 8)) & 0xFF; + } + } else { + for (int i = 0; i < size; i++) { + bytes[size - 1 - i] = (value >> (i * 8)) & 0xFF; + } + } + + writeFromSync(bytes); + } + + // Read/Write Pointers + BT_Pointer readPointerSync() { + int offset = readIntSync(BT_Type.POINTER.size); + return BT_Pointer(offset); + } + void writePointerSync(BT_Pointer pointer) { + writeIntSync(pointer.address, BT_Type.POINTER.size); + } + + // Read/Write Float32 + double readFloat32Sync([Endian endianness = Endian.little]) { + List bytes = readSync(4); + return ByteData.sublistView(Uint8List.fromList(bytes)).getFloat32(0, endianness); + } + void writeFloat32Sync(double value, [Endian endianness = Endian.little]) { + ByteData byteData = ByteData(4); + byteData.setFloat32(0, value, endianness); + writeFromSync(byteData.buffer.asUint8List()); + } + + // Read/Write Float64 (Double) + double readFloat64Sync([Endian endianness = Endian.little]) { + List bytes = readSync(8); + return ByteData.sublistView(Uint8List.fromList(bytes)).getFloat64(0, endianness); + } + void writeFloat64Sync(double value, [Endian endianness = Endian.little]) { + ByteData byteData = ByteData(8); + byteData.setFloat64(0, value, endianness); + writeFromSync(byteData.buffer.asUint8List()); + } + +} + +String binaryDump(Uint8List data) { + StringBuffer buffer = StringBuffer(); + + for (int i = 0; i < data.length; i += 16) { + // Address + buffer.write('0x${i.toRadixString(16).padLeft(4, '0').toUpperCase()} (${i.toString().padLeft(4)}) | '); + + // Hex bytes + for (int j = 0; j < 16; j++) { + if (i + j < data.length) { + buffer.write('${data[i + j].toRadixString(16).padLeft(2, '0').toUpperCase()} '); + } else { + buffer.write(' '); + } + } + + buffer.write(' | '); + + // Integer representation + for (int j = 0; j < 16; j++) { + if (i + j < data.length) { + buffer.write('${data[i + j].toString().padLeft(3)} '); + } else { + buffer.write(' '); + } + } + + buffer.write(' | '); + + // ASCII representation + for (int j = 0; j < 16; j++) { + if (i + j < data.length) { + int byte = data[i + j]; + if (byte >= 32 && byte <= 126) { + buffer.write(String.fromCharCode(byte)); + } else { + buffer.write('.'); + } + } + } + + buffer.write(' | '); + if (i + 16 < data.length) buffer.writeln(); + } + + return buffer.toString(); +} diff --git a/file_formats.iml b/file_formats.iml new file mode 100644 index 0000000..75734c9 --- /dev/null +++ b/file_formats.iml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/pubspec.lock b/pubspec.lock new file mode 100644 index 0000000..7f40e54 --- /dev/null +++ b/pubspec.lock @@ -0,0 +1,405 @@ +# Generated by pub +# See https://dart.dev/tools/pub/glossary#lockfile +packages: + _fe_analyzer_shared: + dependency: transitive + description: + name: _fe_analyzer_shared + sha256: f0bb5d1648339c8308cc0b9838d8456b3cfe5c91f9dc1a735b4d003269e5da9a + url: "https://pub.dev" + source: hosted + version: "88.0.0" + analyzer: + dependency: transitive + description: + name: analyzer + sha256: "0b7b9c329d2879f8f05d6c05b32ee9ec025f39b077864bdb5ac9a7b63418a98f" + url: "https://pub.dev" + source: hosted + version: "8.1.1" + args: + dependency: "direct main" + description: + name: args + sha256: d0481093c50b1da8910eb0bb301626d4d8eb7284aa739614d2b394ee09e3ea04 + url: "https://pub.dev" + source: hosted + version: "2.7.0" + async: + dependency: transitive + description: + name: async + sha256: "758e6d74e971c3e5aceb4110bfd6698efc7f501675bcfe0c775459a8140750eb" + url: "https://pub.dev" + source: hosted + version: "2.13.0" + boolean_selector: + dependency: transitive + description: + name: boolean_selector + sha256: "8aab1771e1243a5063b8b0ff68042d67334e3feab9e95b9490f9a6ebf73b42ea" + url: "https://pub.dev" + source: hosted + version: "2.1.2" + cli_config: + dependency: transitive + description: + name: cli_config + sha256: ac20a183a07002b700f0c25e61b7ee46b23c309d76ab7b7640a028f18e4d99ec + url: "https://pub.dev" + source: hosted + version: "0.2.0" + collection: + dependency: transitive + description: + name: collection + sha256: "2f5709ae4d3d59dd8f7cd309b4e023046b57d8a6c82130785d2b0e5868084e76" + url: "https://pub.dev" + source: hosted + version: "1.19.1" + color_simp: + dependency: "direct main" + description: + name: color_simp + sha256: "4393798a85ab78a6151e6e1ce2fabaed4b36897fdb5a03c585cad53eeed4b264" + url: "https://pub.dev" + source: hosted + version: "1.0.3" + convert: + dependency: transitive + description: + name: convert + sha256: b30acd5944035672bc15c6b7a8b47d773e41e2f17de064350988c5d02adb1c68 + url: "https://pub.dev" + source: hosted + version: "3.1.2" + coverage: + dependency: transitive + description: + name: coverage + sha256: "5da775aa218eaf2151c721b16c01c7676fbfdd99cebba2bf64e8b807a28ff94d" + url: "https://pub.dev" + source: hosted + version: "1.15.0" + crypto: + dependency: transitive + description: + name: crypto + sha256: "1e445881f28f22d6140f181e07737b22f1e099a5e1ff94b0af2f9e4a463f4855" + url: "https://pub.dev" + source: hosted + version: "3.0.6" + file: + dependency: transitive + description: + name: file + sha256: a3b4f84adafef897088c160faf7dfffb7696046cb13ae90b508c2cbc95d3b8d4 + url: "https://pub.dev" + source: hosted + version: "7.0.1" + frontend_server_client: + dependency: transitive + description: + name: frontend_server_client + sha256: f64a0333a82f30b0cca061bc3d143813a486dc086b574bfb233b7c1372427694 + url: "https://pub.dev" + source: hosted + version: "4.0.0" + glob: + dependency: transitive + description: + name: glob + sha256: c3f1ee72c96f8f78935e18aa8cecced9ab132419e8625dc187e1c2408efc20de + url: "https://pub.dev" + source: hosted + version: "2.1.3" + http_multi_server: + dependency: transitive + description: + name: http_multi_server + sha256: aa6199f908078bb1c5efb8d8638d4ae191aac11b311132c3ef48ce352fb52ef8 + url: "https://pub.dev" + source: hosted + version: "3.2.2" + http_parser: + dependency: transitive + description: + name: http_parser + sha256: "178d74305e7866013777bab2c3d8726205dc5a4dd935297175b19a23a2e66571" + url: "https://pub.dev" + source: hosted + version: "4.1.2" + io: + dependency: transitive + description: + name: io + sha256: dfd5a80599cf0165756e3181807ed3e77daf6dd4137caaad72d0b7931597650b + url: "https://pub.dev" + source: hosted + version: "1.0.5" + js: + dependency: transitive + description: + name: js + sha256: "53385261521cc4a0c4658fd0ad07a7d14591cf8fc33abbceae306ddb974888dc" + url: "https://pub.dev" + source: hosted + version: "0.7.2" + lints: + dependency: "direct dev" + description: + name: lints + sha256: c35bb79562d980e9a453fc715854e1ed39e24e7d0297a880ef54e17f9874a9d7 + url: "https://pub.dev" + source: hosted + version: "5.1.1" + logging: + dependency: transitive + description: + name: logging + sha256: c8245ada5f1717ed44271ed1c26b8ce85ca3228fd2ffdb75468ab01979309d61 + url: "https://pub.dev" + source: hosted + version: "1.3.0" + matcher: + dependency: transitive + description: + name: matcher + sha256: dc58c723c3c24bf8d3e2d3ad3f2f9d7bd9cf43ec6feaa64181775e60190153f2 + url: "https://pub.dev" + source: hosted + version: "0.12.17" + meta: + dependency: transitive + description: + name: meta + sha256: "23f08335362185a5ea2ad3a4e597f1375e78bce8a040df5c600c8d3552ef2394" + url: "https://pub.dev" + source: hosted + version: "1.17.0" + mime: + dependency: transitive + description: + name: mime + sha256: "41a20518f0cb1256669420fdba0cd90d21561e560ac240f26ef8322e45bb7ed6" + url: "https://pub.dev" + source: hosted + version: "2.0.0" + node_preamble: + dependency: transitive + description: + name: node_preamble + sha256: "6e7eac89047ab8a8d26cf16127b5ed26de65209847630400f9aefd7cd5c730db" + url: "https://pub.dev" + source: hosted + version: "2.0.2" + package_config: + dependency: transitive + description: + name: package_config + sha256: f096c55ebb7deb7e384101542bfba8c52696c1b56fca2eb62827989ef2353bbc + url: "https://pub.dev" + source: hosted + version: "2.2.0" + path: + dependency: transitive + description: + name: path + sha256: "75cca69d1490965be98c73ceaea117e8a04dd21217b37b292c9ddbec0d955bc5" + url: "https://pub.dev" + source: hosted + version: "1.9.1" + pool: + dependency: transitive + description: + name: pool + sha256: "20fe868b6314b322ea036ba325e6fc0711a22948856475e2c2b6306e8ab39c2a" + url: "https://pub.dev" + source: hosted + version: "1.5.1" + pub_semver: + dependency: transitive + description: + name: pub_semver + sha256: "5bfcf68ca79ef689f8990d1160781b4bad40a3bd5e5218ad4076ddb7f4081585" + url: "https://pub.dev" + source: hosted + version: "2.2.0" + shelf: + dependency: transitive + description: + name: shelf + sha256: e7dd780a7ffb623c57850b33f43309312fc863fb6aa3d276a754bb299839ef12 + url: "https://pub.dev" + source: hosted + version: "1.4.2" + shelf_packages_handler: + dependency: transitive + description: + name: shelf_packages_handler + sha256: "89f967eca29607c933ba9571d838be31d67f53f6e4ee15147d5dc2934fee1b1e" + url: "https://pub.dev" + source: hosted + version: "3.0.2" + shelf_static: + dependency: transitive + description: + name: shelf_static + sha256: c87c3875f91262785dade62d135760c2c69cb217ac759485334c5857ad89f6e3 + url: "https://pub.dev" + source: hosted + version: "1.1.3" + shelf_web_socket: + dependency: transitive + description: + name: shelf_web_socket + sha256: "3632775c8e90d6c9712f883e633716432a27758216dfb61bd86a8321c0580925" + url: "https://pub.dev" + source: hosted + version: "3.0.0" + source_map_stack_trace: + dependency: transitive + description: + name: source_map_stack_trace + sha256: c0713a43e323c3302c2abe2a1cc89aa057a387101ebd280371d6a6c9fa68516b + url: "https://pub.dev" + source: hosted + version: "2.1.2" + source_maps: + dependency: transitive + description: + name: source_maps + sha256: "190222579a448b03896e0ca6eca5998fa810fda630c1d65e2f78b3f638f54812" + url: "https://pub.dev" + source: hosted + version: "0.10.13" + source_span: + dependency: transitive + description: + name: source_span + sha256: "254ee5351d6cb365c859e20ee823c3bb479bf4a293c22d17a9f1bf144ce86f7c" + url: "https://pub.dev" + source: hosted + version: "1.10.1" + stack_trace: + dependency: transitive + description: + name: stack_trace + sha256: "8b27215b45d22309b5cddda1aa2b19bdfec9df0e765f2de506401c071d38d1b1" + url: "https://pub.dev" + source: hosted + version: "1.12.1" + stream_channel: + dependency: transitive + description: + name: stream_channel + sha256: "969e04c80b8bcdf826f8f16579c7b14d780458bd97f56d107d3950fdbeef059d" + url: "https://pub.dev" + source: hosted + version: "2.1.4" + string_scanner: + dependency: transitive + description: + name: string_scanner + sha256: "921cd31725b72fe181906c6a94d987c78e3b98c2e205b397ea399d4054872b43" + url: "https://pub.dev" + source: hosted + version: "1.4.1" + term_glyph: + dependency: transitive + description: + name: term_glyph + sha256: "7f554798625ea768a7518313e58f83891c7f5024f88e46e7182a4558850a4b8e" + url: "https://pub.dev" + source: hosted + version: "1.2.2" + test: + dependency: "direct dev" + description: + name: test + sha256: "75906bf273541b676716d1ca7627a17e4c4070a3a16272b7a3dc7da3b9f3f6b7" + url: "https://pub.dev" + source: hosted + version: "1.26.3" + test_api: + dependency: transitive + description: + name: test_api + sha256: ab2726c1a94d3176a45960b6234466ec367179b87dd74f1611adb1f3b5fb9d55 + url: "https://pub.dev" + source: hosted + version: "0.7.7" + test_core: + dependency: transitive + description: + name: test_core + sha256: "0cc24b5ff94b38d2ae73e1eb43cc302b77964fbf67abad1e296025b78deb53d0" + url: "https://pub.dev" + source: hosted + version: "0.6.12" + typed_data: + dependency: transitive + description: + name: typed_data + sha256: f9049c039ebfeb4cf7a7104a675823cd72dba8297f264b6637062516699fa006 + url: "https://pub.dev" + source: hosted + version: "1.4.0" + vm_service: + dependency: transitive + description: + name: vm_service + sha256: "45caa6c5917fa127b5dbcfbd1fa60b14e583afdc08bfc96dda38886ca252eb60" + url: "https://pub.dev" + source: hosted + version: "15.0.2" + watcher: + dependency: transitive + description: + name: watcher + sha256: "5bf046f41320ac97a469d506261797f35254fa61c641741ef32dacda98b7d39c" + url: "https://pub.dev" + source: hosted + version: "1.1.3" + web: + dependency: transitive + description: + name: web + sha256: "868d88a33d8a87b18ffc05f9f030ba328ffefba92d6c127917a2ba740f9cfe4a" + url: "https://pub.dev" + source: hosted + version: "1.1.1" + web_socket: + dependency: transitive + description: + name: web_socket + sha256: "34d64019aa8e36bf9842ac014bb5d2f5586ca73df5e4d9bf5c936975cae6982c" + url: "https://pub.dev" + source: hosted + version: "1.0.1" + web_socket_channel: + dependency: transitive + description: + name: web_socket_channel + sha256: d645757fb0f4773d602444000a8131ff5d48c9e47adfe9772652dd1a4f2d45c8 + url: "https://pub.dev" + source: hosted + version: "3.0.3" + webkit_inspection_protocol: + dependency: transitive + description: + name: webkit_inspection_protocol + sha256: "87d3f2333bb240704cd3f1c6b5b7acd8a10e7f0bc28c28dcf14e782014f4a572" + url: "https://pub.dev" + source: hosted + version: "1.2.1" + yaml: + dependency: transitive + description: + name: yaml + sha256: b9da305ac7c39faa3f030eccd175340f968459dae4af175130b3fc47e40d76ce + url: "https://pub.dev" + source: hosted + version: "3.1.3" +sdks: + dart: ">=3.7.0 <4.0.0" diff --git a/pubspec.yaml b/pubspec.yaml new file mode 100644 index 0000000..4f7086a --- /dev/null +++ b/pubspec.yaml @@ -0,0 +1,16 @@ +name: file_formats +description: A sample command-line application with basic argument parsing. +version: 0.0.1 +# repository: https://github.com/my_org/my_repo + +environment: + sdk: ^3.0.0 + +# Add regular dependencies here. +dependencies: + args: ^2.5.0 + color_simp: ^1.0.3 + +dev_dependencies: + lints: ^5.0.0 + test: ^1.24.0