perf: source statistics from the catalog
This commit is contained in:
@@ -1,5 +1,7 @@
|
||||
const { parentPort, workerData } = require('node:worker_threads');
|
||||
const Database = require('better-sqlite3');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
|
||||
try {
|
||||
const db = new Database(workerData.databasePath, { readonly: true, fileMustExist: true });
|
||||
@@ -12,9 +14,10 @@ try {
|
||||
(SELECT COUNT(*) FROM articles WHERE ingested_at >= datetime('now', '-1 hour')) AS ingestedPerHour,
|
||||
(SELECT COUNT(*) FROM article_embedding_meta WHERE embedded_at >= datetime('now', '-1 hour')) AS contentPerHour
|
||||
`).get();
|
||||
const bySource = db.prepare(`
|
||||
SELECT DISTINCT source FROM articles ORDER BY source
|
||||
`).all();
|
||||
const sourceCatalog = JSON.parse(fs.readFileSync(path.resolve(__dirname, '..', 'sources.json'), 'utf8'));
|
||||
const bySource = sourceCatalog
|
||||
.map((source) => ({ source: source.label || source.id }))
|
||||
.sort((a, b) => a.source.localeCompare(b.source));
|
||||
// Avoid scanning the 3.7 GB article table for legacy content-status values.
|
||||
// Readiness is the useful operational distinction and is available from the
|
||||
// compact embedding metadata table.
|
||||
|
||||
Reference in New Issue
Block a user