perf: source statistics from the catalog

This commit is contained in:
ImBenji
2026-08-04 21:50:02 +01:00
parent 6e666838ba
commit b3a55fac08
+6 -3
View File
@@ -1,5 +1,7 @@
const { parentPort, workerData } = require('node:worker_threads');
const Database = require('better-sqlite3');
const fs = require('node:fs');
const path = require('node:path');
try {
const db = new Database(workerData.databasePath, { readonly: true, fileMustExist: true });
@@ -12,9 +14,10 @@ try {
(SELECT COUNT(*) FROM articles WHERE ingested_at >= datetime('now', '-1 hour')) AS ingestedPerHour,
(SELECT COUNT(*) FROM article_embedding_meta WHERE embedded_at >= datetime('now', '-1 hour')) AS contentPerHour
`).get();
const bySource = db.prepare(`
SELECT DISTINCT source FROM articles ORDER BY source
`).all();
const sourceCatalog = JSON.parse(fs.readFileSync(path.resolve(__dirname, '..', 'sources.json'), 'utf8'));
const bySource = sourceCatalog
.map((source) => ({ source: source.label || source.id }))
.sort((a, b) => a.source.localeCompare(b.source));
// Avoid scanning the 3.7 GB article table for legacy content-status values.
// Readiness is the useful operational distinction and is available from the
// compact embedding metadata table.