refine article filtering to ensure only usable articles are returned

src/routes/status.js

@@ -3,30 +3,38 @@ const { getLastIngestionBySource } = require('../state');
 
 async function statusRoutes(fastify) {
   fastify.get('/status', async () => {
-    const total = db.prepare('SELECT COUNT(*) AS count FROM articles').get().count;
-    const bySourceRows = db.prepare('SELECT source, COUNT(*) AS count FROM articles GROUP BY source ORDER BY source').all();
-    const contentCoverage = db.prepare(`
+    const bySourceRows = db.prepare(`
       SELECT
+        source,
         COUNT(*) AS total,
-        SUM(CASE WHEN content IS NOT NULL AND TRIM(content) <> '' THEN 1 ELSE 0 END) AS with_content,
-        SUM(CASE WHEN image IS NOT NULL AND TRIM(image) <> '' THEN 1 ELSE 0 END) AS with_image
+
+        SUM(CASE
+          WHEN content IS NOT NULL AND content != ''
+            AND is_index_page = 0
+            AND EXISTS (SELECT 1 FROM article_embeddings WHERE article_id = articles.id)
+          THEN 1 ELSE 0
+        END) AS usable
       FROM articles
-    `).get();
-    const embeddingCoverage = db.prepare('SELECT COUNT(*) AS count FROM article_embeddings').get();
+      GROUP BY source
+      ORDER BY source
+    `).all();
+
+    const totals = bySourceRows.reduce(
+      (acc, row) => {
+        acc.total += row.total;
+        acc.usable += row.usable;
+        return acc;
+      },
+      { total: 0, usable: 0 }
+    );
 
     return {
-      totalArticles: total,
-      countsBySource: Object.fromEntries(bySourceRows.map((row) => [row.source, row.count])),
+      total: totals.total,
+      usable: totals.usable,
       lastIngestionBySource: getLastIngestionBySource(),
-      contentFetchCoverage: {
-        withContent: contentCoverage.with_content || 0,
-        withImage: contentCoverage.with_image || 0,
-        withEmbedding: embeddingCoverage.count || 0,
-        total: contentCoverage.total || 0,
-        contentRatio: contentCoverage.total ? Number((contentCoverage.with_content / contentCoverage.total).toFixed(4)) : 0,
-        imageRatio: contentCoverage.total ? Number((contentCoverage.with_image / contentCoverage.total).toFixed(4)) : 0,
-        embeddingRatio: contentCoverage.total ? Number((embeddingCoverage.count / contentCoverage.total).toFixed(4)) : 0,
-      },
+      bySource: Object.fromEntries(
+        bySourceRows.map((row) => [row.source, { total: row.total, usable: row.usable }])
+      ),
     };
   });
 }