perf: compute archive stats off the API thread

src/adminStatsWorker.js

@@ -0,0 +1,33 @@
+const { parentPort, workerData } = require('node:worker_threads');
+const Database = require('better-sqlite3');
+
+try {
+  const db = new Database(workerData.databasePath, { readonly: true, fileMustExist: true });
+  const counts = db.prepare(`
+    SELECT
+      (SELECT COUNT(*) FROM articles) AS total,
+      (SELECT COUNT(*) FROM article_embedding_meta) AS withContent,
+      (SELECT COUNT(*) FROM article_embedding_meta) AS withEmbedding,
+      (SELECT COUNT(*) FROM events) AS eventCount,
+      (SELECT COUNT(*) FROM articles WHERE ingested_at >= datetime('now', '-1 hour')) AS ingestedPerHour,
+      (SELECT COUNT(*) FROM article_embedding_meta WHERE embedded_at >= datetime('now', '-1 hour')) AS contentPerHour
+  `).get();
+  const bySource = db.prepare(`
+    SELECT source, COUNT(*) AS n FROM articles GROUP BY source ORDER BY n DESC
+  `).all();
+  const byStatus = db.prepare(`
+    SELECT COALESCE(content_status, 'null') AS status, COUNT(*) AS n
+    FROM articles GROUP BY content_status ORDER BY n DESC
+  `).all();
+  let embeddingsPerHour = 0;
+  try {
+    embeddingsPerHour = db.prepare(`
+      SELECT COUNT(*) AS n FROM article_embedding_meta
+      WHERE embedded_at >= datetime('now', '-1 hour')
+    `).get().n;
+  } catch (_) {}
+  db.close();
+  parentPort.postMessage({ value: { ...counts, bySource, byStatus, embeddingsPerHour } });
+} catch (error) {
+  parentPort.postMessage({ error: error.message });
+}

src/routes/admin.js

@@ -1,6 +1,7 @@
 const fs = require('fs');
 const path = require('path');
 const fastifyStatic = require('@fastify/static');
+const { Worker } = require('node:worker_threads');
 const db = require('../db');
 const config = require('../config');
 const Database = require('better-sqlite3');
@@ -9,6 +10,33 @@ let idb = null;
 let statsSummaryCache = null;
 let statsDetailCache = null;
 
+function calculateArchiveStats() {
+  const databasePath = path.resolve(__dirname, '..', '..', config.database.path || './archive.sqlite');
+  const workerPath = path.resolve(__dirname, '..', 'adminStatsWorker.js');
+  return new Promise((resolve, reject) => {
+    const worker = new Worker(workerPath, { workerData: { databasePath } });
+    const timer = setTimeout(() => {
+      worker.terminate();
+      reject(new Error('archive statistics timed out'));
+    }, 60_000);
+    worker.once('message', (message) => {
+      clearTimeout(timer);
+      if (message.error) reject(new Error(message.error));
+      else resolve(message.value);
+    });
+    worker.once('error', (error) => {
+      clearTimeout(timer);
+      reject(error);
+    });
+    worker.once('exit', (code) => {
+      if (code !== 0) {
+        clearTimeout(timer);
+        reject(new Error(`archive statistics worker exited with code ${code}`));
+      }
+    });
+  });
+}
+
 function getIntelligenceDb() {
   if (idb) return idb;
 
@@ -863,33 +891,7 @@ async function adminRoutes(fastify) {
       return statsDetailCache.value;
     }
 
-    const counts = db.prepare(`
+    const value = await calculateArchiveStats();
-      SELECT
-        (SELECT COUNT(*) FROM articles) as total,
-        (SELECT COUNT(*) FROM article_embedding_meta) as withContent,
-        (SELECT COUNT(*) FROM article_embedding_meta) as withEmbedding,
-        (SELECT COUNT(*) FROM events) as eventCount,
-        (SELECT COUNT(*) FROM articles WHERE ingested_at >= datetime('now', '-1 hour')) as ingestedPerHour,
-        (SELECT COUNT(*) FROM article_embedding_meta WHERE embedded_at >= datetime('now', '-1 hour')) as contentPerHour
-    `).get();
-
-    const bySource = db.prepare(`
-      SELECT source, COUNT(*) as n FROM articles GROUP BY source ORDER BY n DESC
-    `).all();
-
-    const byStatus = db.prepare(`
-      SELECT COALESCE(content_status, 'null') as status, COUNT(*) as n
-      FROM articles GROUP BY content_status ORDER BY n DESC
-    `).all();
-
-    let embeddingsPerHour = 0;
-    try {
-      embeddingsPerHour = db.prepare(`
-        SELECT COUNT(*) as n FROM article_embedding_meta WHERE embedded_at >= datetime('now', '-1 hour')
-      `).get().n;
-    } catch (_) {}
-
-    const value = { ...counts, bySource, byStatus, embeddingsPerHour };
     statsDetailCache = { at: Date.now(), value };
     statsSummaryCache = {
       at: Date.now(),