perf: make admin navigation responsive

src/db/index.js

@@ -47,7 +47,12 @@ db.exec(`
   CREATE INDEX IF NOT EXISTS idx_articles_normalized_title ON articles(normalized_title);
   CREATE INDEX IF NOT EXISTS idx_articles_event_id ON articles(event_id);
   CREATE INDEX IF NOT EXISTS idx_articles_has_embedding ON articles(has_embedding);
+  CREATE INDEX IF NOT EXISTS idx_articles_content_status ON articles(content_status);
+  CREATE INDEX IF NOT EXISTS idx_articles_content_attempted_at ON articles(content_attempted_at);
   CREATE INDEX IF NOT EXISTS idx_articles_pub_date_effective ON articles(pub_date_effective DESC);
+  CREATE INDEX IF NOT EXISTS idx_articles_has_content
+    ON articles(id)
+    WHERE content IS NOT NULL AND content != '';
   CREATE INDEX IF NOT EXISTS idx_articles_usable_pub_date
     ON articles(pub_date_effective DESC, id DESC)
     WHERE content IS NOT NULL
@@ -74,6 +79,11 @@ db.exec(`
   );
 `);
 
+db.exec(`
+  CREATE INDEX IF NOT EXISTS idx_article_embedding_meta_embedded_at
+    ON article_embedding_meta(embedded_at);
+`);
+
 db.exec(`
   CREATE VIRTUAL TABLE IF NOT EXISTS article_embeddings USING vec0(
     article_id INTEGER PRIMARY KEY,

src/routes/admin.js

@@ -6,6 +6,8 @@ const config = require('../config');
 const Database = require('better-sqlite3');
 
 let idb = null;
+let statsSummaryCache = null;
+let statsDetailCache = null;
 
 function getIntelligenceDb() {
   if (idb) return idb;
@@ -372,28 +374,22 @@ async function adminRoutes(fastify) {
     const orderBy = sortMap[q.sort] || sortMap.created_desc;
 
     const whereClause  = where.length  ? `WHERE ${where.join(' AND ')}`    : '';
-    const havingClause = having.length ? `HAVING ${having.join(' AND ')}`  : '';
-
-    // total count has to respect the HAVING clause too, so wrap the grouped query
-    const totalRow = db.prepare(`
-      SELECT COUNT(*) as n FROM (
-        SELECT e.id, COUNT(a.id) as article_count
-        FROM events e
-        LEFT JOIN articles a ON a.event_id = e.id
-        ${whereClause}
-        GROUP BY e.id
-        ${havingClause}
-      )
-    `).get(...whereParams, ...havingParams);
+    const countWhereClause = having.length ? `WHERE ${having.join(' AND ')}` : '';
 
+    // Avoid grouping the entire article archive for every page visit. The
+    // correlated count uses idx_articles_event_id and touches only the events
+    // that survive filtering/pagination.
+    const eventProjection = `
+      SELECT e.id, e.title, e.created_at,
+             (SELECT COUNT(*) FROM articles a WHERE a.event_id = e.id) AS article_count
+      FROM events e ${whereClause}
+    `;
+    const filteredEvents = `SELECT * FROM (${eventProjection}) ${countWhereClause}`;
+    const totalRow = db.prepare(`SELECT COUNT(*) AS n FROM (${filteredEvents})`)
+      .get(...whereParams, ...havingParams);
     const rows = db.prepare(`
-      SELECT e.id, e.title, e.created_at, COUNT(a.id) as article_count
-      FROM events e
-      LEFT JOIN articles a ON a.event_id = e.id
-      ${whereClause}
-      GROUP BY e.id
-      ${havingClause}
-      ORDER BY ${orderBy}
+      ${filteredEvents}
+      ORDER BY ${orderBy.replaceAll('e.', '')}
       LIMIT ? OFFSET ?
     `).all(...whereParams, ...havingParams, limit, offset);
 
@@ -838,9 +834,31 @@ async function adminRoutes(fastify) {
     return { results, elapsed: Date.now() - start };
   });
 
-  // stats for dashboard header
+  // Lightweight header summary, cached independently from the detailed stats
+  // page so navigation never waits for source/status aggregation.
+  fastify.get('/admin/api/stats/summary', async (request, reply) => {
+    if (!checkAuth(request, reply)) return;
+    if (statsSummaryCache && Date.now() - statsSummaryCache.at < 60_000) {
+      return statsSummaryCache.value;
+    }
+    const counts = db.prepare(`
+      SELECT
+        (SELECT COUNT(*) FROM articles) AS total,
+        (SELECT COUNT(*) FROM articles WHERE content IS NOT NULL AND content != '') AS withContent,
+        (SELECT COUNT(*) FROM articles WHERE has_embedding = 1) AS withEmbedding,
+        (SELECT COUNT(*) FROM events) AS eventCount
+    `).get();
+    statsSummaryCache = { at: Date.now(), value: counts };
+    return counts;
+  });
+
+  // Detailed statistics are cached because they summarize the full archive and
+  // do not need second-by-second precision.
   fastify.get('/admin/api/stats', async (request, reply) => {
     if (!checkAuth(request, reply)) return;
+    if (statsDetailCache && Date.now() - statsDetailCache.at < 5 * 60_000) {
+      return statsDetailCache.value;
+    }
 
     const counts = db.prepare(`
       SELECT
@@ -868,7 +886,18 @@ async function adminRoutes(fastify) {
       `).get().n;
     } catch (_) {}
 
-    return { ...counts, bySource, byStatus, embeddingsPerHour };
+    const value = { ...counts, bySource, byStatus, embeddingsPerHour };
+    statsDetailCache = { at: Date.now(), value };
+    statsSummaryCache = {
+      at: Date.now(),
+      value: {
+        total: counts.total,
+        withContent: counts.withContent,
+        withEmbedding: counts.withEmbedding,
+        eventCount: counts.eventCount,
+      },
+    };
+    return value;
   });
 }