IMBENJI.NET/Duriin-API
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: scrub invalid nul bytes during postgres migration

Browse Source
This commit is contained in:
ImBenji
2026-08-04 22:35:50 +01:00
parent ffc85719ff
commit 0047adcb6d
1 changed files with 11 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/migrate-sqlite-to-postgres.js
+11 -1
View File
@@ -21,6 +21,9 @@ const DERIVED_SQLITE_TABLES = new Set([
'article_embeddings', 'article_embeddings_chunks', 'article_embeddings_info', 'article_embeddings', 'article_embeddings_chunks', 'article_embeddings_info',
'article_embeddings_rowids', 'article_embeddings_vector_chunks00', 'article_embeddings_rowids', 'article_embeddings_vector_chunks00',
]); ]);
const NUL_BYTE = '\u0000';
const NUL_BYTES = /\u0000/g;
let sanitizedTextValues = 0;
function quote(name) { return `"${String(name).replaceAll('"', '""')}"`; } function quote(name) { return `"${String(name).replaceAll('"', '""')}"`; }
function pgType(sqliteType) { function pgType(sqliteType) {
@@ -31,6 +34,12 @@ function pgType(sqliteType) {
return 'TEXT'; return 'TEXT';
} }
function normalizeValue(value) {
if (typeof value !== 'string' || !value.includes(NUL_BYTE)) return value ?? null;
sanitizedTextValues += 1;
return value.replace(NUL_BYTES, '');
}
async function ensureSchema(client, schema) { async function ensureSchema(client, schema) {
await client.query(`CREATE SCHEMA IF NOT EXISTS ${quote(schema)}`); await client.query(`CREATE SCHEMA IF NOT EXISTS ${quote(schema)}`);
await client.query(`CREATE TABLE IF NOT EXISTS ${quote(schema)}.${quote('_migration_progress')} ( await client.query(`CREATE TABLE IF NOT EXISTS ${quote(schema)}.${quote('_migration_progress')} (
@@ -76,7 +85,7 @@ async function copyTable(client, schema, sqlite, table, columns) {
if (!rows.length) break; if (!rows.length) break;
await client.query('BEGIN'); await client.query('BEGIN');
try { try {
for (const row of rows) await client.query(insert, names.map((name) => row[name] ?? null)); for (const row of rows) await client.query(insert, names.map((name) => normalizeValue(row[name])));
cursor = key ? Number(rows[rows.length - 1][key.name]) : cursor + rows.length; cursor = key ? Number(rows[rows.length - 1][key.name]) : cursor + rows.length;
await client.query(`INSERT INTO ${quote(schema)}.${quote('_migration_progress')} (table_name, source_rows, copied_rows, updated_at) await client.query(`INSERT INTO ${quote(schema)}.${quote('_migration_progress')} (table_name, source_rows, copied_rows, updated_at)
VALUES ($1,$2,$3,NOW()) ON CONFLICT (table_name) DO UPDATE SET source_rows=EXCLUDED.source_rows, copied_rows=EXCLUDED.copied_rows, updated_at=NOW()`, [table, sourceRows, cursor]); VALUES ($1,$2,$3,NOW()) ON CONFLICT (table_name) DO UPDATE SET source_rows=EXCLUDED.source_rows, copied_rows=EXCLUDED.copied_rows, updated_at=NOW()`, [table, sourceRows, cursor]);
@@ -108,5 +117,6 @@ async function migrateDatabase(client, schema, file) {
await migrateDatabase(client, 'archive', archivePath); await migrateDatabase(client, 'archive', archivePath);
await migrateDatabase(client, 'intelligence', intelligencePath); await migrateDatabase(client, 'intelligence', intelligencePath);
console.log('[migrate] SQLite logical data verified in PostgreSQL. SQLite remains the live source until application cutover.'); console.log('[migrate] SQLite logical data verified in PostgreSQL. SQLite remains the live source until application cutover.');
if (sanitizedTextValues) console.log(`[migrate] sanitized ${sanitizedTextValues} text values containing NUL bytes rejected by PostgreSQL text columns.`);
} finally { client.release(); await pool.end(); } } finally { client.release(); await pool.end(); }
})().catch((error) => { console.error('[migrate] fatal:', error); process.exit(1); }); })().catch((error) => { console.error('[migrate] fatal:', error); process.exit(1); });