refactor content fetching and embedding processes for improved concurrency and error handling

src/routes/status.js

@@ -1,8 +1,17 @@
 const db = require('../db');
 const { getLastIngestionBySource } = require('../state');
 
+let statusCache = null;
+let statusCacheAt = 0;
+const STATUS_CACHE_TTL_MS = 30 * 1000;
+
 async function statusRoutes(fastify) {
   fastify.get('/status', async () => {
+    const now = Date.now();
+    if (statusCache && now - statusCacheAt < STATUS_CACHE_TTL_MS) {
+      return statusCache;
+    }
+
     const bySourceRows = db.prepare(`
       SELECT
         source,
@@ -11,7 +20,7 @@ async function statusRoutes(fastify) {
         SUM(CASE
           WHEN content IS NOT NULL AND content != ''
             AND is_index_page = 0
-            AND EXISTS (SELECT 1 FROM article_embedding_meta WHERE article_id = articles.id)
+            AND has_embedding = 1
           THEN 1 ELSE 0
         END) AS usable
       FROM articles
@@ -38,7 +47,7 @@ async function statusRoutes(fastify) {
       ORDER BY article_count DESC
     `).all();
 
-    return {
+    const result = {
       total: totals.total,
       usable: totals.usable,
       lastIngestionBySource: getLastIngestionBySource(),
@@ -51,6 +60,10 @@ async function statusRoutes(fastify) {
         dimensions: row.sample_bytes ? row.sample_bytes / 4 : null,
       })),
     };
+
+    statusCache = result;
+    statusCacheAt = now;
+    return result;
   });
 }