IMBENJI.NET/Duriin-API
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: compute archive stats off the API thread

Browse Source
This commit is contained in:
ImBenji
2026-08-04 21:45:17 +01:00
parent 002b7b320c
commit f5f092cfb0
2 changed files with 62 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/adminStatsWorker.js
+33
View File
@@ -0,0 +1,33 @@
const { parentPort, workerData } = require('node:worker_threads');
const Database = require('better-sqlite3');
try {
const db = new Database(workerData.databasePath, { readonly: true, fileMustExist: true });
const counts = db.prepare(`
SELECT
(SELECT COUNT(*) FROM articles) AS total,
(SELECT COUNT(*) FROM article_embedding_meta) AS withContent,
(SELECT COUNT(*) FROM article_embedding_meta) AS withEmbedding,
(SELECT COUNT(*) FROM events) AS eventCount,
(SELECT COUNT(*) FROM articles WHERE ingested_at >= datetime('now', '-1 hour')) AS ingestedPerHour,
(SELECT COUNT(*) FROM article_embedding_meta WHERE embedded_at >= datetime('now', '-1 hour')) AS contentPerHour
`).get();
const bySource = db.prepare(`
SELECT source, COUNT(*) AS n FROM articles GROUP BY source ORDER BY n DESC
`).all();
const byStatus = db.prepare(`
SELECT COALESCE(content_status, 'null') AS status, COUNT(*) AS n
FROM articles GROUP BY content_status ORDER BY n DESC
`).all();
let embeddingsPerHour = 0;
try {
embeddingsPerHour = db.prepare(`
SELECT COUNT(*) AS n FROM article_embedding_meta
WHERE embedded_at >= datetime('now', '-1 hour')
`).get().n;
} catch (_) {}
db.close();
parentPort.postMessage({ value: { ...counts, bySource, byStatus, embeddingsPerHour } });
} catch (error) {
parentPort.postMessage({ error: error.message });
}
src/routes/admin.js
+29 -27
View File
@@ -1,6 +1,7 @@
const fs = require('fs');
const path = require('path');
const fastifyStatic = require('@fastify/static');
const { Worker } = require('node:worker_threads');
const db = require('../db');
const config = require('../config');
const Database = require('better-sqlite3');
@@ -9,6 +10,33 @@ let idb = null;
let statsSummaryCache = null;
let statsDetailCache = null;
function calculateArchiveStats() {
const databasePath = path.resolve(__dirname, '..', '..', config.database.path || './archive.sqlite');
const workerPath = path.resolve(__dirname, '..', 'adminStatsWorker.js');
return new Promise((resolve, reject) => {
const worker = new Worker(workerPath, { workerData: { databasePath } });
const timer = setTimeout(() => {
worker.terminate();
reject(new Error('archive statistics timed out'));
}, 60_000);
worker.once('message', (message) => {
clearTimeout(timer);
if (message.error) reject(new Error(message.error));
else resolve(message.value);
});
worker.once('error', (error) => {
clearTimeout(timer);
reject(error);
});
worker.once('exit', (code) => {
if (code !== 0) {
clearTimeout(timer);
reject(new Error(`archive statistics worker exited with code ${code}`));
}
});
});
}
function getIntelligenceDb() {
if (idb) return idb;
@@ -863,33 +891,7 @@ async function adminRoutes(fastify) {
return statsDetailCache.value;
}
const counts = db.prepare(`
SELECT
(SELECT COUNT(*) FROM articles) as total,
(SELECT COUNT(*) FROM article_embedding_meta) as withContent,
(SELECT COUNT(*) FROM article_embedding_meta) as withEmbedding,
(SELECT COUNT(*) FROM events) as eventCount,
(SELECT COUNT(*) FROM articles WHERE ingested_at >= datetime('now', '-1 hour')) as ingestedPerHour,
(SELECT COUNT(*) FROM article_embedding_meta WHERE embedded_at >= datetime('now', '-1 hour')) as contentPerHour
`).get();
const bySource = db.prepare(`
SELECT source, COUNT(*) as n FROM articles GROUP BY source ORDER BY n DESC
`).all();
const byStatus = db.prepare(`
SELECT COALESCE(content_status, 'null') as status, COUNT(*) as n
FROM articles GROUP BY content_status ORDER BY n DESC
`).all();
let embeddingsPerHour = 0;
try {
embeddingsPerHour = db.prepare(`
SELECT COUNT(*) as n FROM article_embedding_meta WHERE embedded_at >= datetime('now', '-1 hour')
`).get().n;
} catch (_) {}
const value = { ...counts, bySource, byStatus, embeddingsPerHour };
const value = await calculateArchiveStats();
statsDetailCache = { at: Date.now(), value };
statsSummaryCache = {
at: Date.now(),