IMBENJI.NET/Duriin-API
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: avoid full archive scans in statistics

Browse Source
This commit is contained in:
ImBenji
2026-08-04 21:46:56 +01:00
parent f5f092cfb0
commit d020fa3811
3 changed files with 17 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
public/admin/assets/js/stats.js
+1 -1
View File
@@ -11,7 +11,7 @@ async function loadStatsPage() {
.map(r => `<tr><td>${escapeHtml(r.source)}</td><td style="text-align:right; padding-left:24px">${r.n.toLocaleString()}</td></tr>`).join(""); .map(r => `<tr><td>${escapeHtml(r.source)}</td><td style="text-align:right; padding-left:24px">${r.n.toLocaleString()}</td></tr>`).join("");
document.getElementById("statusTable").innerHTML = data.byStatus document.getElementById("statusTable").innerHTML = data.byStatus
.map(r => `<tr><td>${badgeHtml(r.status === "null" ? null : r.status)}</td><td style="text-align:right; padding-left:24px">${r.n.toLocaleString()}</td></tr>`).join(""); .map(r => `<tr><td><span class="badge ${r.status === "ready" ? "ok" : "pending"}">${escapeHtml(r.status)}</span></td><td style="text-align:right; padding-left:24px">${r.n.toLocaleString()}</td></tr>`).join("");
document.getElementById("rate-ingested").textContent = (data.ingestedPerHour || 0).toLocaleString(); document.getElementById("rate-ingested").textContent = (data.ingestedPerHour || 0).toLocaleString();
document.getElementById("rate-content").textContent = (data.contentPerHour || 0).toLocaleString(); document.getElementById("rate-content").textContent = (data.contentPerHour || 0).toLocaleString();
public/admin/pages/stats.html
+7 -7
View File
@@ -4,10 +4,10 @@
<meta charset="UTF-8"> <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Duriin Admin — Stats</title> <title>Duriin Admin — Stats</title>
<link rel="stylesheet" href="/admin/assets/css/base.css?v=20260804-3"> <link rel="stylesheet" href="/admin/assets/css/base.css?v=20260804-4">
<link rel="stylesheet" href="/admin/assets/css/layout.css?v=20260804-3"> <link rel="stylesheet" href="/admin/assets/css/layout.css?v=20260804-4">
<link rel="stylesheet" href="/admin/assets/css/components.css?v=20260804-3"> <link rel="stylesheet" href="/admin/assets/css/components.css?v=20260804-4">
<link rel="stylesheet" href="/admin/assets/css/intel.css?v=20260804-3"> <link rel="stylesheet" href="/admin/assets/css/intel.css?v=20260804-4">
</head> </head>
<body> <body>
@@ -60,7 +60,7 @@
</div> </div>
<div> <div>
<div class="section-heading">By content status</div> <div class="section-heading">By archive readiness</div>
<div class="table-wrap" style="width:auto; min-width:180px"> <div class="table-wrap" style="width:auto; min-width:180px">
<table style="width:auto"> <table style="width:auto">
<thead><tr><th>Status</th><th style="text-align:right">Count</th></tr></thead> <thead><tr><th>Status</th><th style="text-align:right">Count</th></tr></thead>
@@ -74,7 +74,7 @@
<div id="toast"><span class="toast-dot"></span><span id="toast-msg"></span></div> <div id="toast"><span class="toast-dot"></span><span id="toast-msg"></span></div>
<script src="/admin/assets/js/app.js?v=20260804-3"></script> <script src="/admin/assets/js/app.js?v=20260804-4"></script>
<script src="/admin/assets/js/stats.js?v=20260804-3"></script> <script src="/admin/assets/js/stats.js?v=20260804-4"></script>
</body> </body>
</html> </html>
src/adminStatsWorker.js
+9 -6
View File
@@ -5,20 +5,23 @@ try {
const db = new Database(workerData.databasePath, { readonly: true, fileMustExist: true }); const db = new Database(workerData.databasePath, { readonly: true, fileMustExist: true });
const counts = db.prepare(` const counts = db.prepare(`
SELECT SELECT
(SELECT COUNT(*) FROM articles) AS total, COALESCE((SELECT seq FROM sqlite_sequence WHERE name='articles'), 0) AS total,
(SELECT COUNT(*) FROM article_embedding_meta) AS withContent, (SELECT COUNT(*) FROM article_embedding_meta) AS withContent,
(SELECT COUNT(*) FROM article_embedding_meta) AS withEmbedding, (SELECT COUNT(*) FROM article_embedding_meta) AS withEmbedding,
(SELECT COUNT(*) FROM events) AS eventCount, COALESCE((SELECT seq FROM sqlite_sequence WHERE name='events'), 0) AS eventCount,
(SELECT COUNT(*) FROM articles WHERE ingested_at >= datetime('now', '-1 hour')) AS ingestedPerHour, (SELECT COUNT(*) FROM articles WHERE ingested_at >= datetime('now', '-1 hour')) AS ingestedPerHour,
(SELECT COUNT(*) FROM article_embedding_meta WHERE embedded_at >= datetime('now', '-1 hour')) AS contentPerHour (SELECT COUNT(*) FROM article_embedding_meta WHERE embedded_at >= datetime('now', '-1 hour')) AS contentPerHour
`).get(); `).get();
const bySource = db.prepare(` const bySource = db.prepare(`
SELECT source, COUNT(*) AS n FROM articles GROUP BY source ORDER BY n DESC SELECT source, COUNT(*) AS n FROM articles GROUP BY source ORDER BY n DESC
`).all(); `).all();
const byStatus = db.prepare(` // Avoid scanning the 3.7 GB article table for legacy content-status values.
SELECT COALESCE(content_status, 'null') AS status, COUNT(*) AS n // Readiness is the useful operational distinction and is available from the
FROM articles GROUP BY content_status ORDER BY n DESC // compact embedding metadata table.
`).all(); const byStatus = [
{ status: 'ready', n: counts.withEmbedding },
{ status: 'unprocessed', n: Math.max(0, counts.total - counts.withEmbedding) },
];
let embeddingsPerHour = 0; let embeddingsPerHour = 0;
try { try {
embeddingsPerHour = db.prepare(` embeddingsPerHour = db.prepare(`