-
Worker throughput
-
-
Loading...
+
Pipeline throughput — last 1 hour
+
+
+ Articles ingested
+ —
+
+
+ Content fetched
+ —
+
+
+ Embeddings generated
+ —
+
@@ -901,33 +909,9 @@ async function loadStats() {
document.getElementById('statusTable').innerHTML = data.byStatus
.map(r => `
| ${badgeHtml(r.status === 'null' ? null : r.status)} | ${r.n.toLocaleString()} |
`).join('');
- renderWorkerRates();
-}
-
-function renderWorkerRates() {
- const rates = window._workerRates || [];
- const el = document.getElementById('worker-rates-row');
- if (!el) return;
-
- const workerLabels = { augor: 'Augor (events)', consolidation: 'Consolidation (companies)', graph: 'Graph (edges)' };
-
- if (rates.length === 0) {
- el.innerHTML = '
No data yet — worker_events table may not exist until workers restart.';
- return;
- }
-
- el.innerHTML = rates.map(r => {
- const label = workerLabels[r.worker] || r.worker;
- const avg5m = (r.n5m / 5).toFixed(1);
- const last1m = r.n1m;
-
- return `
-
- ${label}
- ${avg5m}/min
- ${last1m} in last 1m
-
`;
- }).join('');
+ document.getElementById('rate-ingested').textContent = (data.ingestedPerHour || 0).toLocaleString();
+ document.getElementById('rate-content').textContent = (data.contentPerHour || 0).toLocaleString();
+ document.getElementById('rate-embeddings').textContent = (data.embeddingsPerHour || 0).toLocaleString();
}
// ── source dropdown ────────────────────────────────────────────────────────
@@ -1164,8 +1148,6 @@ async function loadIntelligenceStats() {
`).join('');
- // stash worker rates for the stats tab
- window._workerRates = data.workerRates || [];
return true;
}
@@ -1363,9 +1345,6 @@ document.getElementById('graph-show-untracked').addEventListener('change', () =>
if (graphAllNodes.length) renderIntelGraph();
});
-document.getElementById('graph-min-count').addEventListener('change', () => {
- if (graphAllNodes.length) renderIntelGraph();
-});
function toggleGraphExpand() {
@@ -1380,9 +1359,7 @@ function toggleGraphExpand() {
function renderIntelGraph() {
const showUntracked = document.getElementById('graph-show-untracked').checked;
- const minCount = parseInt(document.getElementById('graph-min-count').value, 10) || 1;
-
- const filteredLinks = graphAllLinks.filter(l => (l.count || 1) >= minCount);
+ const filteredLinks = graphAllLinks.slice();
// only show nodes that still have at least one edge after the count filter
const activeKeys = new Set();
@@ -1500,11 +1477,11 @@ function renderIntelGraph() {
const simulation = d3.forceSimulation(nodesCopy)
.force('link', d3.forceLink(linksCopy).id(d => d.key).distance(d => {
- return Math.max(90, d.type.length * 5 + 60);
+ return Math.max(130, d.type.length * 5 + 90);
}))
- .force('charge', d3.forceManyBody().strength(-120))
+ .force('charge', d3.forceManyBody().strength(-220))
.force('center', d3.forceCenter(width / 2, height / 2))
- .force('collide', d3.forceCollide(d => d.tracked ? 48 : 34).strength(1))
+ .force('collide', d3.forceCollide(d => d.tracked ? 56 : 40).strength(1))
.force('nodeEdgeRepulsion', forceNodeEdgeRepulsion())
.alphaDecay(0.015);
@@ -1513,8 +1490,7 @@ function renderIntelGraph() {
const linkG = g.append('g');
let linkLabels;
- // stroke-width min 10 so text (7px) always fits inside the line
- const strokeWidth = d => 10 + Math.pow(d.count / maxCount, 0.55) * 8;
+ const strokeWidth = d => 1.5 + Math.pow(d.count / maxCount, 0.55) * 3.5;
const linkPaths = linkG.selectAll('path.ig-edge')
.data(linksCopy)
@@ -1649,21 +1625,17 @@ function renderIntelGraph() {
nodeGroups.attr('transform', d => `translate(${d.x},${d.y})`);
});
- simulation.on('end', () => {
+ // expose fit function for the button
+ window._graphFit = () => {
const bounds = g.node().getBBox();
if (!bounds.width || !bounds.height) return;
-
const pad = 60;
- const scaleX = width / (bounds.width + pad * 2);
- const scaleY = height / (bounds.height + pad * 2);
- const scale = Math.min(scaleX, scaleY, 1);
-
+ const scale = Math.min(width / (bounds.width + pad * 2), height / (bounds.height + pad * 2), 1);
const tx = width / 2 - scale * (bounds.x + bounds.width / 2);
const ty = height / 2 - scale * (bounds.y + bounds.height / 2);
-
- svg.transition().duration(700)
+ svg.transition().duration(500)
.call(zoomBehavior.transform, d3.zoomIdentity.translate(tx, ty).scale(scale));
- });
+ };
}
@@ -1827,7 +1799,7 @@ function switchTab(tab) {
location.hash = tab;
if (tab === 'events') loadEvents();
- if (tab === 'stats') { loadStats(); loadIntelligenceStats(); }
+ if (tab === 'stats') loadStats();
if (tab === 'intelligence') { intelOffset = 0; loadIntelligenceStats().then(ok => { if (ok) { loadIntelligenceCompanies(); loadIntelligence(); } }); }
}
diff --git a/src/routes/admin.js b/src/routes/admin.js
index fd75323..662812b 100644
--- a/src/routes/admin.js
+++ b/src/routes/admin.js
@@ -432,7 +432,22 @@ async function adminRoutes(fastify) {
FROM articles GROUP BY content_status ORDER BY n DESC
`).all();
- return { total, withContent, withEmbedding, eventCount, bySource, byStatus };
+ const ingestedPerHour = db.prepare(`
+ SELECT COUNT(*) as n FROM articles WHERE ingested_at >= datetime('now', '-1 hour')
+ `).get().n;
+
+ const contentPerHour = db.prepare(`
+ SELECT COUNT(*) as n FROM articles WHERE content_attempted_at >= datetime('now', '-1 hour')
+ `).get().n;
+
+ let embeddingsPerHour = 0;
+ try {
+ embeddingsPerHour = db.prepare(`
+ SELECT COUNT(*) as n FROM article_embedding_meta WHERE embedded_at >= datetime('now', '-1 hour')
+ `).get().n;
+ } catch (_) {}
+
+ return { total, withContent, withEmbedding, eventCount, bySource, byStatus, ingestedPerHour, contentPerHour, embeddingsPerHour };
});
}