diff --git a/src/adminStatsWorker.js b/src/adminStatsWorker.js new file mode 100644 index 0000000..5c6fe7c --- /dev/null +++ b/src/adminStatsWorker.js @@ -0,0 +1,33 @@ +const { parentPort, workerData } = require('node:worker_threads'); +const Database = require('better-sqlite3'); + +try { + const db = new Database(workerData.databasePath, { readonly: true, fileMustExist: true }); + const counts = db.prepare(` + SELECT + (SELECT COUNT(*) FROM articles) AS total, + (SELECT COUNT(*) FROM article_embedding_meta) AS withContent, + (SELECT COUNT(*) FROM article_embedding_meta) AS withEmbedding, + (SELECT COUNT(*) FROM events) AS eventCount, + (SELECT COUNT(*) FROM articles WHERE ingested_at >= datetime('now', '-1 hour')) AS ingestedPerHour, + (SELECT COUNT(*) FROM article_embedding_meta WHERE embedded_at >= datetime('now', '-1 hour')) AS contentPerHour + `).get(); + const bySource = db.prepare(` + SELECT source, COUNT(*) AS n FROM articles GROUP BY source ORDER BY n DESC + `).all(); + const byStatus = db.prepare(` + SELECT COALESCE(content_status, 'null') AS status, COUNT(*) AS n + FROM articles GROUP BY content_status ORDER BY n DESC + `).all(); + let embeddingsPerHour = 0; + try { + embeddingsPerHour = db.prepare(` + SELECT COUNT(*) AS n FROM article_embedding_meta + WHERE embedded_at >= datetime('now', '-1 hour') + `).get().n; + } catch (_) {} + db.close(); + parentPort.postMessage({ value: { ...counts, bySource, byStatus, embeddingsPerHour } }); +} catch (error) { + parentPort.postMessage({ error: error.message }); +} diff --git a/src/routes/admin.js b/src/routes/admin.js index 0894070..f475481 100644 --- a/src/routes/admin.js +++ b/src/routes/admin.js @@ -1,6 +1,7 @@ const fs = require('fs'); const path = require('path'); const fastifyStatic = require('@fastify/static'); +const { Worker } = require('node:worker_threads'); const db = require('../db'); const config = require('../config'); const Database = require('better-sqlite3'); @@ -9,6 +10,33 @@ let idb = null; let statsSummaryCache = null; let statsDetailCache = null; +function calculateArchiveStats() { + const databasePath = path.resolve(__dirname, '..', '..', config.database.path || './archive.sqlite'); + const workerPath = path.resolve(__dirname, '..', 'adminStatsWorker.js'); + return new Promise((resolve, reject) => { + const worker = new Worker(workerPath, { workerData: { databasePath } }); + const timer = setTimeout(() => { + worker.terminate(); + reject(new Error('archive statistics timed out')); + }, 60_000); + worker.once('message', (message) => { + clearTimeout(timer); + if (message.error) reject(new Error(message.error)); + else resolve(message.value); + }); + worker.once('error', (error) => { + clearTimeout(timer); + reject(error); + }); + worker.once('exit', (code) => { + if (code !== 0) { + clearTimeout(timer); + reject(new Error(`archive statistics worker exited with code ${code}`)); + } + }); + }); +} + function getIntelligenceDb() { if (idb) return idb; @@ -863,33 +891,7 @@ async function adminRoutes(fastify) { return statsDetailCache.value; } - const counts = db.prepare(` - SELECT - (SELECT COUNT(*) FROM articles) as total, - (SELECT COUNT(*) FROM article_embedding_meta) as withContent, - (SELECT COUNT(*) FROM article_embedding_meta) as withEmbedding, - (SELECT COUNT(*) FROM events) as eventCount, - (SELECT COUNT(*) FROM articles WHERE ingested_at >= datetime('now', '-1 hour')) as ingestedPerHour, - (SELECT COUNT(*) FROM article_embedding_meta WHERE embedded_at >= datetime('now', '-1 hour')) as contentPerHour - `).get(); - - const bySource = db.prepare(` - SELECT source, COUNT(*) as n FROM articles GROUP BY source ORDER BY n DESC - `).all(); - - const byStatus = db.prepare(` - SELECT COALESCE(content_status, 'null') as status, COUNT(*) as n - FROM articles GROUP BY content_status ORDER BY n DESC - `).all(); - - let embeddingsPerHour = 0; - try { - embeddingsPerHour = db.prepare(` - SELECT COUNT(*) as n FROM article_embedding_meta WHERE embedded_at >= datetime('now', '-1 hour') - `).get().n; - } catch (_) {} - - const value = { ...counts, bySource, byStatus, embeddingsPerHour }; + const value = await calculateArchiveStats(); statsDetailCache = { at: Date.now(), value }; statsSummaryCache = { at: Date.now(),