From 0047adcb6df374434cb6024ff178c27b8bf7d287 Mon Sep 17 00:00:00 2001
From: ImBenji <benjamin.watt@imbenji.net>
Date: Tue, 4 Aug 2026 22:35:50 +0100
Subject: [PATCH] fix: scrub invalid nul bytes during postgres migration

---
 scripts/migrate-sqlite-to-postgres.js | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/scripts/migrate-sqlite-to-postgres.js b/scripts/migrate-sqlite-to-postgres.js
index 21a7fd4..1c8de32 100644
--- a/scripts/migrate-sqlite-to-postgres.js
+++ b/scripts/migrate-sqlite-to-postgres.js
@@ -21,6 +21,9 @@ const DERIVED_SQLITE_TABLES = new Set([
   'article_embeddings', 'article_embeddings_chunks', 'article_embeddings_info',
   'article_embeddings_rowids', 'article_embeddings_vector_chunks00',
 ]);
+const NUL_BYTE = '\u0000';
+const NUL_BYTES = /\u0000/g;
+let sanitizedTextValues = 0;
 
 function quote(name) { return `"${String(name).replaceAll('"', '""')}"`; }
 function pgType(sqliteType) {
@@ -31,6 +34,12 @@ function pgType(sqliteType) {
   return 'TEXT';
 }
 
+function normalizeValue(value) {
+  if (typeof value !== 'string' || !value.includes(NUL_BYTE)) return value ?? null;
+  sanitizedTextValues += 1;
+  return value.replace(NUL_BYTES, '');
+}
+
 async function ensureSchema(client, schema) {
   await client.query(`CREATE SCHEMA IF NOT EXISTS ${quote(schema)}`);
   await client.query(`CREATE TABLE IF NOT EXISTS ${quote(schema)}.${quote('_migration_progress')} (
@@ -76,7 +85,7 @@ async function copyTable(client, schema, sqlite, table, columns) {
     if (!rows.length) break;
     await client.query('BEGIN');
     try {
-      for (const row of rows) await client.query(insert, names.map((name) => row[name] ?? null));
+      for (const row of rows) await client.query(insert, names.map((name) => normalizeValue(row[name])));
       cursor = key ? Number(rows[rows.length - 1][key.name]) : cursor + rows.length;
       await client.query(`INSERT INTO ${quote(schema)}.${quote('_migration_progress')} (table_name, source_rows, copied_rows, updated_at)
         VALUES ($1,$2,$3,NOW()) ON CONFLICT (table_name) DO UPDATE SET source_rows=EXCLUDED.source_rows, copied_rows=EXCLUDED.copied_rows, updated_at=NOW()`, [table, sourceRows, cursor]);
@@ -108,5 +117,6 @@ async function migrateDatabase(client, schema, file) {
     await migrateDatabase(client, 'archive', archivePath);
     await migrateDatabase(client, 'intelligence', intelligencePath);
     console.log('[migrate] SQLite logical data verified in PostgreSQL. SQLite remains the live source until application cutover.');
+    if (sanitizedTextValues) console.log(`[migrate] sanitized ${sanitizedTextValues} text values containing NUL bytes rejected by PostgreSQL text columns.`);
   } finally { client.release(); await pool.end(); }
 })().catch((error) => { console.error('[migrate] fatal:', error); process.exit(1); });