add worker event tracking; implement worker rates display in admin panel
This commit is contained in:
ImBenji
parent
30104c6e66
commit
e10de8fcec
2 changed files with 45 additions and 58 deletions
86
admin.html
86
admin.html
|
|
@ -698,6 +698,7 @@
|
|||
<div id="intel-graph-svg-wrap">
|
||||
<svg id="intel-graph-svg"></svg>
|
||||
<div id="graph-empty" style="display:none; position:absolute; inset:0; color:var(--muted); font-size:13px; text-align:center; padding-top:120px">No relationship data yet</div>
|
||||
<button onclick="if(window._graphFit) window._graphFit()" style="position:absolute; top:10px; right:120px; padding:5px 10px; font-size:12px; background:var(--bg-card); border:1px solid var(--border); color:var(--muted); border-radius:var(--radius); cursor:pointer; z-index:10" title="Fit all nodes into view">⊡ Fit</button>
|
||||
<button id="graph-expand-btn" onclick="toggleGraphExpand()" style="position:absolute; top:10px; right:10px; padding:5px 10px; font-size:12px; background:var(--bg-card); border:1px solid var(--border); color:var(--muted); border-radius:var(--radius); cursor:pointer; z-index:10" title="Expand graph">⤢ Expand</button>
|
||||
</div>
|
||||
<div id="graph-sidebar">
|
||||
|
|
@ -715,10 +716,6 @@
|
|||
<span>Show untracked entities</span>
|
||||
</label>
|
||||
|
||||
<label style="display:flex; align-items:center; gap:6px; margin-left:16px; user-select:none">
|
||||
<span>Min confirmations</span>
|
||||
<input type="number" id="graph-min-count" value="2" min="1" style="width:52px; min-width:unset; padding:3px 6px; font-size:12px" />
|
||||
</label>
|
||||
|
||||
<span style="margin-left:auto; color:var(--muted-dark)">Scroll to zoom · drag nodes · click for facts</span>
|
||||
</div>
|
||||
|
|
@ -746,9 +743,20 @@
|
|||
<div id="tab-stats" style="display:none">
|
||||
|
||||
<div style="margin-bottom:28px">
|
||||
<div class="section-heading">Worker throughput</div>
|
||||
<div id="worker-rates-row" style="display:flex; gap:12px; flex-wrap:wrap; margin-top:10px">
|
||||
<span style="color:var(--muted); font-size:13px">Loading...</span>
|
||||
<div class="section-heading">Pipeline throughput — last 1 hour</div>
|
||||
<div style="display:flex; gap:12px; flex-wrap:wrap; margin-top:10px">
|
||||
<div class="intel-stat-card" style="min-width:180px">
|
||||
<span class="label">Articles ingested</span>
|
||||
<span class="value" id="rate-ingested">—</span>
|
||||
</div>
|
||||
<div class="intel-stat-card" style="min-width:180px">
|
||||
<span class="label">Content fetched</span>
|
||||
<span class="value" id="rate-content">—</span>
|
||||
</div>
|
||||
<div class="intel-stat-card" style="min-width:180px">
|
||||
<span class="label">Embeddings generated</span>
|
||||
<span class="value" id="rate-embeddings">—</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
|
@ -901,33 +909,9 @@ async function loadStats() {
|
|||
document.getElementById('statusTable').innerHTML = data.byStatus
|
||||
.map(r => `<tr><td>${badgeHtml(r.status === 'null' ? null : r.status)}</td><td style="text-align:right; padding-left:24px">${r.n.toLocaleString()}</td></tr>`).join('');
|
||||
|
||||
renderWorkerRates();
|
||||
}
|
||||
|
||||
function renderWorkerRates() {
|
||||
const rates = window._workerRates || [];
|
||||
const el = document.getElementById('worker-rates-row');
|
||||
if (!el) return;
|
||||
|
||||
const workerLabels = { augor: 'Augor (events)', consolidation: 'Consolidation (companies)', graph: 'Graph (edges)' };
|
||||
|
||||
if (rates.length === 0) {
|
||||
el.innerHTML = '<span style="color:var(--muted); font-size:13px">No data yet — worker_events table may not exist until workers restart.</span>';
|
||||
return;
|
||||
}
|
||||
|
||||
el.innerHTML = rates.map(r => {
|
||||
const label = workerLabels[r.worker] || r.worker;
|
||||
const avg5m = (r.n5m / 5).toFixed(1);
|
||||
const last1m = r.n1m;
|
||||
|
||||
return `
|
||||
<div class="intel-stat-card" style="min-width:180px">
|
||||
<span class="label">${label}</span>
|
||||
<span class="value" style="font-size:18px">${avg5m}<span style="font-size:12px; font-weight:400; color:var(--muted)">/min</span></span>
|
||||
<span style="font-size:11px; color:var(--muted-dark); margin-top:4px">${last1m} in last 1m</span>
|
||||
</div>`;
|
||||
}).join('');
|
||||
document.getElementById('rate-ingested').textContent = (data.ingestedPerHour || 0).toLocaleString();
|
||||
document.getElementById('rate-content').textContent = (data.contentPerHour || 0).toLocaleString();
|
||||
document.getElementById('rate-embeddings').textContent = (data.embeddingsPerHour || 0).toLocaleString();
|
||||
}
|
||||
|
||||
// ── source dropdown ────────────────────────────────────────────────────────
|
||||
|
|
@ -1164,8 +1148,6 @@ async function loadIntelligenceStats() {
|
|||
</div>
|
||||
`).join('');
|
||||
|
||||
// stash worker rates for the stats tab
|
||||
window._workerRates = data.workerRates || [];
|
||||
|
||||
return true;
|
||||
}
|
||||
|
|
@ -1363,9 +1345,6 @@ document.getElementById('graph-show-untracked').addEventListener('change', () =>
|
|||
if (graphAllNodes.length) renderIntelGraph();
|
||||
});
|
||||
|
||||
document.getElementById('graph-min-count').addEventListener('change', () => {
|
||||
if (graphAllNodes.length) renderIntelGraph();
|
||||
});
|
||||
|
||||
|
||||
function toggleGraphExpand() {
|
||||
|
|
@ -1380,9 +1359,7 @@ function toggleGraphExpand() {
|
|||
|
||||
function renderIntelGraph() {
|
||||
const showUntracked = document.getElementById('graph-show-untracked').checked;
|
||||
const minCount = parseInt(document.getElementById('graph-min-count').value, 10) || 1;
|
||||
|
||||
const filteredLinks = graphAllLinks.filter(l => (l.count || 1) >= minCount);
|
||||
const filteredLinks = graphAllLinks.slice();
|
||||
|
||||
// only show nodes that still have at least one edge after the count filter
|
||||
const activeKeys = new Set();
|
||||
|
|
@ -1500,11 +1477,11 @@ function renderIntelGraph() {
|
|||
|
||||
const simulation = d3.forceSimulation(nodesCopy)
|
||||
.force('link', d3.forceLink(linksCopy).id(d => d.key).distance(d => {
|
||||
return Math.max(90, d.type.length * 5 + 60);
|
||||
return Math.max(130, d.type.length * 5 + 90);
|
||||
}))
|
||||
.force('charge', d3.forceManyBody().strength(-120))
|
||||
.force('charge', d3.forceManyBody().strength(-220))
|
||||
.force('center', d3.forceCenter(width / 2, height / 2))
|
||||
.force('collide', d3.forceCollide(d => d.tracked ? 48 : 34).strength(1))
|
||||
.force('collide', d3.forceCollide(d => d.tracked ? 56 : 40).strength(1))
|
||||
.force('nodeEdgeRepulsion', forceNodeEdgeRepulsion())
|
||||
.alphaDecay(0.015);
|
||||
|
||||
|
|
@ -1513,8 +1490,7 @@ function renderIntelGraph() {
|
|||
const linkG = g.append('g');
|
||||
let linkLabels;
|
||||
|
||||
// stroke-width min 10 so text (7px) always fits inside the line
|
||||
const strokeWidth = d => 10 + Math.pow(d.count / maxCount, 0.55) * 8;
|
||||
const strokeWidth = d => 1.5 + Math.pow(d.count / maxCount, 0.55) * 3.5;
|
||||
|
||||
const linkPaths = linkG.selectAll('path.ig-edge')
|
||||
.data(linksCopy)
|
||||
|
|
@ -1649,21 +1625,17 @@ function renderIntelGraph() {
|
|||
nodeGroups.attr('transform', d => `translate(${d.x},${d.y})`);
|
||||
});
|
||||
|
||||
simulation.on('end', () => {
|
||||
// expose fit function for the button
|
||||
window._graphFit = () => {
|
||||
const bounds = g.node().getBBox();
|
||||
if (!bounds.width || !bounds.height) return;
|
||||
|
||||
const pad = 60;
|
||||
const scaleX = width / (bounds.width + pad * 2);
|
||||
const scaleY = height / (bounds.height + pad * 2);
|
||||
const scale = Math.min(scaleX, scaleY, 1);
|
||||
|
||||
const scale = Math.min(width / (bounds.width + pad * 2), height / (bounds.height + pad * 2), 1);
|
||||
const tx = width / 2 - scale * (bounds.x + bounds.width / 2);
|
||||
const ty = height / 2 - scale * (bounds.y + bounds.height / 2);
|
||||
|
||||
svg.transition().duration(700)
|
||||
svg.transition().duration(500)
|
||||
.call(zoomBehavior.transform, d3.zoomIdentity.translate(tx, ty).scale(scale));
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -1827,7 +1799,7 @@ function switchTab(tab) {
|
|||
location.hash = tab;
|
||||
|
||||
if (tab === 'events') loadEvents();
|
||||
if (tab === 'stats') { loadStats(); loadIntelligenceStats(); }
|
||||
if (tab === 'stats') loadStats();
|
||||
if (tab === 'intelligence') { intelOffset = 0; loadIntelligenceStats().then(ok => { if (ok) { loadIntelligenceCompanies(); loadIntelligence(); } }); }
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -432,7 +432,22 @@ async function adminRoutes(fastify) {
|
|||
FROM articles GROUP BY content_status ORDER BY n DESC
|
||||
`).all();
|
||||
|
||||
return { total, withContent, withEmbedding, eventCount, bySource, byStatus };
|
||||
const ingestedPerHour = db.prepare(`
|
||||
SELECT COUNT(*) as n FROM articles WHERE ingested_at >= datetime('now', '-1 hour')
|
||||
`).get().n;
|
||||
|
||||
const contentPerHour = db.prepare(`
|
||||
SELECT COUNT(*) as n FROM articles WHERE content_attempted_at >= datetime('now', '-1 hour')
|
||||
`).get().n;
|
||||
|
||||
let embeddingsPerHour = 0;
|
||||
try {
|
||||
embeddingsPerHour = db.prepare(`
|
||||
SELECT COUNT(*) as n FROM article_embedding_meta WHERE embedded_at >= datetime('now', '-1 hour')
|
||||
`).get().n;
|
||||
} catch (_) {}
|
||||
|
||||
return { total, withContent, withEmbedding, eventCount, bySource, byStatus, ingestedPerHour, contentPerHour, embeddingsPerHour };
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue