diff --git a/public/admin/assets/css/components.css b/public/admin/assets/css/components.css index 214e404..f8487c0 100644 --- a/public/admin/assets/css/components.css +++ b/public/admin/assets/css/components.css @@ -68,6 +68,40 @@ tbody tr:hover { background: rgba(255,255,255,.022); } .empty-state { padding: 42px 24px; color: var(--muted-dark); text-align: center; } +.skeleton-row:hover { background: transparent; } +.skeleton-row td { height: 205px; padding: 22px 18px; vertical-align: top; } +.skeleton-row td span { + width: min(680px, 82%); + height: 11px; + margin-bottom: 20px; + display: block; + border-radius: 2px; + background: linear-gradient(90deg, #171b17 20%, #252b24 42%, #171b17 64%); + background-size: 300% 100%; + animation: skeleton-wave 1.6s ease infinite; +} +.skeleton-row td span:nth-child(2) { width: 58%; } +.skeleton-row td span:nth-child(3) { width: 72%; } +.skeleton-row td span:nth-child(4) { width: 45%; } +.content-loading { min-height: 160px; position: relative; } +.content-loading::after { content: "Loading"; position: absolute; inset: 0; display: grid; place-items: center; color: var(--muted-dark); font-family: var(--mono); font-size: 9px; letter-spacing: .1em; text-transform: uppercase; } +@keyframes skeleton-wave { from { background-position: 100% 0; } to { background-position: 0 0; } } + +html::after { + content: ""; + position: fixed; + z-index: 500; + top: 0; + left: var(--rail); + width: 0; + height: 2px; + opacity: 0; + background: var(--accent); + box-shadow: 0 0 12px rgba(223,255,79,.45); + transition: width 220ms ease, opacity 120ms; +} +html.navigating::after { width: calc(100% - var(--rail)); opacity: 1; } + @media (max-width: 640px) { .table-wrap { border-radius: 0; margin-inline: -16px; border-left: 0; border-right: 0; } th, td { padding-inline: 11px; } diff --git a/public/admin/assets/js/app.js b/public/admin/assets/js/app.js index cc61c01..98b3505 100644 --- a/public/admin/assets/js/app.js +++ b/public/admin/assets/js/app.js @@ -148,7 +148,7 @@ async function loadGlobalStats() { if (!bar) return; try { - const data = await api("/admin/api/stats"); + const data = await api("/admin/api/stats/summary"); const t = document.getElementById("s-total"); if (t) t.textContent = data.total.toLocaleString(); const c = document.getElementById("s-content"); @@ -161,6 +161,49 @@ async function loadGlobalStats() { } +function installLoadingStates() { + document.querySelectorAll("tbody:empty").forEach(tbody => { + const columns = Math.max(1, tbody.closest("table")?.querySelectorAll("thead th").length || 5); + tbody.setAttribute("aria-busy", "true"); + tbody.innerHTML = ``; + const observer = new MutationObserver(() => { + if (!tbody.querySelector(".skeleton-row")) { + tbody.removeAttribute("aria-busy"); + observer.disconnect(); + } + }); + observer.observe(tbody, { childList: true }); + }); + ["intel-signals-grid", "sourceTable", "statusTable"].forEach(id => { + const node = document.getElementById(id); + if (node && !node.children.length) node.classList.add("content-loading"); + }); +} + + +function installNavigationPrefetch() { + const seen = new Set(); + const prefetch = link => { + const href = link?.href; + if (!href || seen.has(href) || link.origin !== location.origin) return; + seen.add(href); + const hint = document.createElement("link"); + hint.rel = "prefetch"; + hint.href = href; + hint.as = "document"; + document.head.appendChild(hint); + }; + document.querySelectorAll(".tabs a, .subnav a").forEach(link => { + link.addEventListener("pointerenter", () => prefetch(link), { once: true }); + link.addEventListener("focus", () => prefetch(link), { once: true }); + link.addEventListener("click", () => document.documentElement.classList.add("navigating")); + }); + const warm = () => document.querySelectorAll(".tabs a, .subnav a").forEach(prefetch); + if ("requestIdleCallback" in window) requestIdleCallback(warm, { timeout: 2500 }); + else setTimeout(warm, 1200); +} + + // common overlay close-on-backdrop wiring function wireOverlays() { document.querySelectorAll(".overlay").forEach(ov => { @@ -173,6 +216,10 @@ function wireOverlays() { document.addEventListener("DOMContentLoaded", () => { installChrome(); + installLoadingStates(); + installNavigationPrefetch(); wireOverlays(); - loadGlobalStats(); + const loadStats = () => loadGlobalStats(); + if ("requestIdleCallback" in window) requestIdleCallback(loadStats, { timeout: 1800 }); + else setTimeout(loadStats, 500); }); diff --git a/public/admin/assets/js/intel-signals.js b/public/admin/assets/js/intel-signals.js index ffc7a08..e31f363 100644 --- a/public/admin/assets/js/intel-signals.js +++ b/public/admin/assets/js/intel-signals.js @@ -13,6 +13,7 @@ const signalsById = new Map(); function renderSignals(data) { const grid = document.getElementById("intel-signals-grid"); const empty = document.getElementById("intel-signals-empty"); + grid.classList.remove("content-loading"); if (!data || data.length === 0) { grid.innerHTML = ""; diff --git a/public/admin/assets/js/stats.js b/public/admin/assets/js/stats.js index 64c3640..e9ad3f6 100644 --- a/public/admin/assets/js/stats.js +++ b/public/admin/assets/js/stats.js @@ -4,6 +4,9 @@ async function loadStatsPage() { const data = await api("/admin/api/stats"); + document.getElementById("sourceTable").classList.remove("content-loading"); + document.getElementById("statusTable").classList.remove("content-loading"); + document.getElementById("sourceTable").innerHTML = data.bySource .map(r => `${escapeHtml(r.source)}${r.n.toLocaleString()}`).join(""); diff --git a/public/admin/pages/autonomy.html b/public/admin/pages/autonomy.html index b7dd3dd..0b4b6cd 100644 --- a/public/admin/pages/autonomy.html +++ b/public/admin/pages/autonomy.html @@ -5,10 +5,10 @@ Duriin — Autonomy - - - - + + + + @@ -131,7 +131,7 @@
- - + + diff --git a/public/admin/pages/ingest/articles.html b/public/admin/pages/ingest/articles.html index b01924b..d256397 100644 --- a/public/admin/pages/ingest/articles.html +++ b/public/admin/pages/ingest/articles.html @@ -4,9 +4,9 @@ Duriin Admin — Articles - - - + + + @@ -134,7 +134,7 @@
- - + + diff --git a/public/admin/pages/ingest/events.html b/public/admin/pages/ingest/events.html index 00da17d..5cd1cc4 100644 --- a/public/admin/pages/ingest/events.html +++ b/public/admin/pages/ingest/events.html @@ -4,9 +4,9 @@ Duriin Admin — Events - - - + + + @@ -94,7 +94,7 @@
- - + + diff --git a/public/admin/pages/intelligence/graph.html b/public/admin/pages/intelligence/graph.html index 7dd730d..7219c71 100644 --- a/public/admin/pages/intelligence/graph.html +++ b/public/admin/pages/intelligence/graph.html @@ -4,10 +4,10 @@ Duriin Admin — Intelligence / Graph - - - - + + + + @@ -72,9 +72,9 @@
- - - - + + + + diff --git a/public/admin/pages/intelligence/knowledge.html b/public/admin/pages/intelligence/knowledge.html index fab59d1..7ac6bd7 100644 --- a/public/admin/pages/intelligence/knowledge.html +++ b/public/admin/pages/intelligence/knowledge.html @@ -4,10 +4,10 @@ Duriin Admin — Intelligence / Knowledge - - - - + + + + @@ -90,8 +90,8 @@
- - - + + + diff --git a/public/admin/pages/intelligence/predictions.html b/public/admin/pages/intelligence/predictions.html index 3900f9e..fecc49a 100644 --- a/public/admin/pages/intelligence/predictions.html +++ b/public/admin/pages/intelligence/predictions.html @@ -4,10 +4,10 @@ Duriin Admin — Intelligence / Predictions - - - - + + + + @@ -81,8 +81,8 @@
- - - + + + diff --git a/public/admin/pages/intelligence/signals.html b/public/admin/pages/intelligence/signals.html index f601957..f81d0e7 100644 --- a/public/admin/pages/intelligence/signals.html +++ b/public/admin/pages/intelligence/signals.html @@ -4,10 +4,10 @@ Duriin Admin — Intelligence / Signals - - - - + + + + @@ -137,8 +137,8 @@
- - - + + + diff --git a/public/admin/pages/sql.html b/public/admin/pages/sql.html index 082e854..28ad628 100644 --- a/public/admin/pages/sql.html +++ b/public/admin/pages/sql.html @@ -4,9 +4,9 @@ Duriin Admin — SQL - - - + + + @@ -45,7 +45,7 @@
- - + + diff --git a/public/admin/pages/stats.html b/public/admin/pages/stats.html index 2a4e965..a6f2f5f 100644 --- a/public/admin/pages/stats.html +++ b/public/admin/pages/stats.html @@ -4,10 +4,10 @@ Duriin Admin — Stats - - - - + + + + @@ -74,7 +74,7 @@
- - + + diff --git a/src/db/index.js b/src/db/index.js index 9ea1cb2..f65b854 100644 --- a/src/db/index.js +++ b/src/db/index.js @@ -47,7 +47,12 @@ db.exec(` CREATE INDEX IF NOT EXISTS idx_articles_normalized_title ON articles(normalized_title); CREATE INDEX IF NOT EXISTS idx_articles_event_id ON articles(event_id); CREATE INDEX IF NOT EXISTS idx_articles_has_embedding ON articles(has_embedding); + CREATE INDEX IF NOT EXISTS idx_articles_content_status ON articles(content_status); + CREATE INDEX IF NOT EXISTS idx_articles_content_attempted_at ON articles(content_attempted_at); CREATE INDEX IF NOT EXISTS idx_articles_pub_date_effective ON articles(pub_date_effective DESC); + CREATE INDEX IF NOT EXISTS idx_articles_has_content + ON articles(id) + WHERE content IS NOT NULL AND content != ''; CREATE INDEX IF NOT EXISTS idx_articles_usable_pub_date ON articles(pub_date_effective DESC, id DESC) WHERE content IS NOT NULL @@ -74,6 +79,11 @@ db.exec(` ); `); +db.exec(` + CREATE INDEX IF NOT EXISTS idx_article_embedding_meta_embedded_at + ON article_embedding_meta(embedded_at); +`); + db.exec(` CREATE VIRTUAL TABLE IF NOT EXISTS article_embeddings USING vec0( article_id INTEGER PRIMARY KEY, diff --git a/src/routes/admin.js b/src/routes/admin.js index ec0e655..d58ff21 100644 --- a/src/routes/admin.js +++ b/src/routes/admin.js @@ -6,6 +6,8 @@ const config = require('../config'); const Database = require('better-sqlite3'); let idb = null; +let statsSummaryCache = null; +let statsDetailCache = null; function getIntelligenceDb() { if (idb) return idb; @@ -372,28 +374,22 @@ async function adminRoutes(fastify) { const orderBy = sortMap[q.sort] || sortMap.created_desc; const whereClause = where.length ? `WHERE ${where.join(' AND ')}` : ''; - const havingClause = having.length ? `HAVING ${having.join(' AND ')}` : ''; - - // total count has to respect the HAVING clause too, so wrap the grouped query - const totalRow = db.prepare(` - SELECT COUNT(*) as n FROM ( - SELECT e.id, COUNT(a.id) as article_count - FROM events e - LEFT JOIN articles a ON a.event_id = e.id - ${whereClause} - GROUP BY e.id - ${havingClause} - ) - `).get(...whereParams, ...havingParams); + const countWhereClause = having.length ? `WHERE ${having.join(' AND ')}` : ''; + // Avoid grouping the entire article archive for every page visit. The + // correlated count uses idx_articles_event_id and touches only the events + // that survive filtering/pagination. + const eventProjection = ` + SELECT e.id, e.title, e.created_at, + (SELECT COUNT(*) FROM articles a WHERE a.event_id = e.id) AS article_count + FROM events e ${whereClause} + `; + const filteredEvents = `SELECT * FROM (${eventProjection}) ${countWhereClause}`; + const totalRow = db.prepare(`SELECT COUNT(*) AS n FROM (${filteredEvents})`) + .get(...whereParams, ...havingParams); const rows = db.prepare(` - SELECT e.id, e.title, e.created_at, COUNT(a.id) as article_count - FROM events e - LEFT JOIN articles a ON a.event_id = e.id - ${whereClause} - GROUP BY e.id - ${havingClause} - ORDER BY ${orderBy} + ${filteredEvents} + ORDER BY ${orderBy.replaceAll('e.', '')} LIMIT ? OFFSET ? `).all(...whereParams, ...havingParams, limit, offset); @@ -838,9 +834,31 @@ async function adminRoutes(fastify) { return { results, elapsed: Date.now() - start }; }); - // stats for dashboard header + // Lightweight header summary, cached independently from the detailed stats + // page so navigation never waits for source/status aggregation. + fastify.get('/admin/api/stats/summary', async (request, reply) => { + if (!checkAuth(request, reply)) return; + if (statsSummaryCache && Date.now() - statsSummaryCache.at < 60_000) { + return statsSummaryCache.value; + } + const counts = db.prepare(` + SELECT + (SELECT COUNT(*) FROM articles) AS total, + (SELECT COUNT(*) FROM articles WHERE content IS NOT NULL AND content != '') AS withContent, + (SELECT COUNT(*) FROM articles WHERE has_embedding = 1) AS withEmbedding, + (SELECT COUNT(*) FROM events) AS eventCount + `).get(); + statsSummaryCache = { at: Date.now(), value: counts }; + return counts; + }); + + // Detailed statistics are cached because they summarize the full archive and + // do not need second-by-second precision. fastify.get('/admin/api/stats', async (request, reply) => { if (!checkAuth(request, reply)) return; + if (statsDetailCache && Date.now() - statsDetailCache.at < 5 * 60_000) { + return statsDetailCache.value; + } const counts = db.prepare(` SELECT @@ -868,7 +886,18 @@ async function adminRoutes(fastify) { `).get().n; } catch (_) {} - return { ...counts, bySource, byStatus, embeddingsPerHour }; + const value = { ...counts, bySource, byStatus, embeddingsPerHour }; + statsDetailCache = { at: Date.now(), value }; + statsSummaryCache = { + at: Date.now(), + value: { + total: counts.total, + withContent: counts.withContent, + withEmbedding: counts.withEmbedding, + eventCount: counts.eventCount, + }, + }; + return value; }); }