From e10de8fcecd7bf08eef5c8311399dde568373983 Mon Sep 17 00:00:00 2001
From: ImBenji <benjamin.watt@imbenji.net>
Date: Thu, 23 Apr 2026 17:26:15 +0100
Subject: [PATCH] add worker event tracking; implement worker rates display in
 admin panel

---
 admin.html          | 86 +++++++++++++++------------------------------
 src/routes/admin.js | 17 ++++++++-
 2 files changed, 45 insertions(+), 58 deletions(-)

diff --git a/admin.html b/admin.html
index d1d2e08..9795c9b 100644
--- a/admin.html
+++ b/admin.html
@@ -698,6 +698,7 @@
           <div id="intel-graph-svg-wrap">
             <svg id="intel-graph-svg"></svg>
             <div id="graph-empty" style="display:none; position:absolute; inset:0; color:var(--muted); font-size:13px; text-align:center; padding-top:120px">No relationship data yet</div>
+            <button onclick="if(window._graphFit) window._graphFit()" style="position:absolute; top:10px; right:120px; padding:5px 10px; font-size:12px; background:var(--bg-card); border:1px solid var(--border); color:var(--muted); border-radius:var(--radius); cursor:pointer; z-index:10" title="Fit all nodes into view">⊡ Fit</button>
             <button id="graph-expand-btn" onclick="toggleGraphExpand()" style="position:absolute; top:10px; right:10px; padding:5px 10px; font-size:12px; background:var(--bg-card); border:1px solid var(--border); color:var(--muted); border-radius:var(--radius); cursor:pointer; z-index:10" title="Expand graph">⤢ Expand</button>
           </div>
           <div id="graph-sidebar">
@@ -715,10 +716,6 @@
             <span>Show untracked entities</span>
           </label>
 
-          <label style="display:flex; align-items:center; gap:6px; margin-left:16px; user-select:none">
-            <span>Min confirmations</span>
-            <input type="number" id="graph-min-count" value="2" min="1" style="width:52px; min-width:unset; padding:3px 6px; font-size:12px" />
-          </label>
 
           <span style="margin-left:auto; color:var(--muted-dark)">Scroll to zoom · drag nodes · click for facts</span>
         </div>
@@ -746,9 +743,20 @@
   <div id="tab-stats" style="display:none">
 
     <div style="margin-bottom:28px">
-      <div class="section-heading">Worker throughput</div>
-      <div id="worker-rates-row" style="display:flex; gap:12px; flex-wrap:wrap; margin-top:10px">
-        <span style="color:var(--muted); font-size:13px">Loading...</span>
+      <div class="section-heading">Pipeline throughput — last 1 hour</div>
+      <div style="display:flex; gap:12px; flex-wrap:wrap; margin-top:10px">
+        <div class="intel-stat-card" style="min-width:180px">
+          <span class="label">Articles ingested</span>
+          <span class="value" id="rate-ingested">—</span>
+        </div>
+        <div class="intel-stat-card" style="min-width:180px">
+          <span class="label">Content fetched</span>
+          <span class="value" id="rate-content">—</span>
+        </div>
+        <div class="intel-stat-card" style="min-width:180px">
+          <span class="label">Embeddings generated</span>
+          <span class="value" id="rate-embeddings">—</span>
+        </div>
       </div>
     </div>
 
@@ -901,33 +909,9 @@ async function loadStats() {
   document.getElementById('statusTable').innerHTML = data.byStatus
     .map(r => `<tr><td>${badgeHtml(r.status === 'null' ? null : r.status)}</td><td style="text-align:right; padding-left:24px">${r.n.toLocaleString()}</td></tr>`).join('');
 
-  renderWorkerRates();
-}
-
-function renderWorkerRates() {
-  const rates = window._workerRates || [];
-  const el = document.getElementById('worker-rates-row');
-  if (!el) return;
-
-  const workerLabels = { augor: 'Augor (events)', consolidation: 'Consolidation (companies)', graph: 'Graph (edges)' };
-
-  if (rates.length === 0) {
-    el.innerHTML = '<span style="color:var(--muted); font-size:13px">No data yet — worker_events table may not exist until workers restart.</span>';
-    return;
-  }
-
-  el.innerHTML = rates.map(r => {
-    const label = workerLabels[r.worker] || r.worker;
-    const avg5m = (r.n5m / 5).toFixed(1);
-    const last1m = r.n1m;
-
-    return `
-      <div class="intel-stat-card" style="min-width:180px">
-        <span class="label">${label}</span>
-        <span class="value" style="font-size:18px">${avg5m}<span style="font-size:12px; font-weight:400; color:var(--muted)">/min</span></span>
-        <span style="font-size:11px; color:var(--muted-dark); margin-top:4px">${last1m} in last 1m</span>
-      </div>`;
-  }).join('');
+  document.getElementById('rate-ingested').textContent = (data.ingestedPerHour || 0).toLocaleString();
+  document.getElementById('rate-content').textContent = (data.contentPerHour || 0).toLocaleString();
+  document.getElementById('rate-embeddings').textContent = (data.embeddingsPerHour || 0).toLocaleString();
 }
 
 // ── source dropdown ────────────────────────────────────────────────────────
@@ -1164,8 +1148,6 @@ async function loadIntelligenceStats() {
     </div>
   `).join('');
 
-  // stash worker rates for the stats tab
-  window._workerRates = data.workerRates || [];
 
   return true;
 }
@@ -1363,9 +1345,6 @@ document.getElementById('graph-show-untracked').addEventListener('change', () =>
   if (graphAllNodes.length) renderIntelGraph();
 });
 
-document.getElementById('graph-min-count').addEventListener('change', () => {
-  if (graphAllNodes.length) renderIntelGraph();
-});
 
 
 function toggleGraphExpand() {
@@ -1380,9 +1359,7 @@ function toggleGraphExpand() {
 
 function renderIntelGraph() {
   const showUntracked = document.getElementById('graph-show-untracked').checked;
-  const minCount = parseInt(document.getElementById('graph-min-count').value, 10) || 1;
-
-  const filteredLinks = graphAllLinks.filter(l => (l.count || 1) >= minCount);
+  const filteredLinks = graphAllLinks.slice();
 
   // only show nodes that still have at least one edge after the count filter
   const activeKeys = new Set();
@@ -1500,11 +1477,11 @@ function renderIntelGraph() {
 
   const simulation = d3.forceSimulation(nodesCopy)
     .force('link', d3.forceLink(linksCopy).id(d => d.key).distance(d => {
-      return Math.max(90, d.type.length * 5 + 60);
+      return Math.max(130, d.type.length * 5 + 90);
     }))
-    .force('charge', d3.forceManyBody().strength(-120))
+    .force('charge', d3.forceManyBody().strength(-220))
     .force('center', d3.forceCenter(width / 2, height / 2))
-    .force('collide', d3.forceCollide(d => d.tracked ? 48 : 34).strength(1))
+    .force('collide', d3.forceCollide(d => d.tracked ? 56 : 40).strength(1))
     .force('nodeEdgeRepulsion', forceNodeEdgeRepulsion())
     .alphaDecay(0.015);
 
@@ -1513,8 +1490,7 @@ function renderIntelGraph() {
   const linkG = g.append('g');
   let linkLabels;
 
-  // stroke-width min 10 so text (7px) always fits inside the line
-  const strokeWidth = d => 10 + Math.pow(d.count / maxCount, 0.55) * 8;
+  const strokeWidth = d => 1.5 + Math.pow(d.count / maxCount, 0.55) * 3.5;
 
   const linkPaths = linkG.selectAll('path.ig-edge')
     .data(linksCopy)
@@ -1649,21 +1625,17 @@ function renderIntelGraph() {
     nodeGroups.attr('transform', d => `translate(${d.x},${d.y})`);
   });
 
-  simulation.on('end', () => {
+  // expose fit function for the button
+  window._graphFit = () => {
     const bounds = g.node().getBBox();
     if (!bounds.width || !bounds.height) return;
-
     const pad = 60;
-    const scaleX = width / (bounds.width + pad * 2);
-    const scaleY = height / (bounds.height + pad * 2);
-    const scale = Math.min(scaleX, scaleY, 1);
-
+    const scale = Math.min(width / (bounds.width + pad * 2), height / (bounds.height + pad * 2), 1);
     const tx = width / 2 - scale * (bounds.x + bounds.width / 2);
     const ty = height / 2 - scale * (bounds.y + bounds.height / 2);
-
-    svg.transition().duration(700)
+    svg.transition().duration(500)
       .call(zoomBehavior.transform, d3.zoomIdentity.translate(tx, ty).scale(scale));
-  });
+  };
 }
 
 
@@ -1827,7 +1799,7 @@ function switchTab(tab) {
   location.hash = tab;
 
   if (tab === 'events') loadEvents();
-  if (tab === 'stats') { loadStats(); loadIntelligenceStats(); }
+  if (tab === 'stats') loadStats();
   if (tab === 'intelligence') { intelOffset = 0; loadIntelligenceStats().then(ok => { if (ok) { loadIntelligenceCompanies(); loadIntelligence(); } }); }
 }
 
diff --git a/src/routes/admin.js b/src/routes/admin.js
index fd75323..662812b 100644
--- a/src/routes/admin.js
+++ b/src/routes/admin.js
@@ -432,7 +432,22 @@ async function adminRoutes(fastify) {
       FROM articles GROUP BY content_status ORDER BY n DESC
     `).all();
 
-    return { total, withContent, withEmbedding, eventCount, bySource, byStatus };
+    const ingestedPerHour = db.prepare(`
+      SELECT COUNT(*) as n FROM articles WHERE ingested_at >= datetime('now', '-1 hour')
+    `).get().n;
+
+    const contentPerHour = db.prepare(`
+      SELECT COUNT(*) as n FROM articles WHERE content_attempted_at >= datetime('now', '-1 hour')
+    `).get().n;
+
+    let embeddingsPerHour = 0;
+    try {
+      embeddingsPerHour = db.prepare(`
+        SELECT COUNT(*) as n FROM article_embedding_meta WHERE embedded_at >= datetime('now', '-1 hour')
+      `).get().n;
+    } catch (_) {}
+
+    return { total, withContent, withEmbedding, eventCount, bySource, byStatus, ingestedPerHour, contentPerHour, embeddingsPerHour };
   });
 }