IMBENJI.NET/Duriin-API
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: remove blocking archive counts from navigation

Browse Source
This commit is contained in:
ImBenji
2026-08-04 21:42:23 +01:00
parent 2020a0fde2
commit 002b7b320c
12 changed files with 87 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
public/admin/assets/js/articles.js
+6 -3
View File
@@ -74,11 +74,14 @@ async function loadArticles() {
`).join(""); `).join("");
const total = data.total; const total = data.total;
document.getElementById("pageInfo").textContent = const start = data.rows.length ? articleOffset + 1 : 0;
`${articleOffset + 1}–${Math.min(articleOffset + PAGE, total)} of ${total.toLocaleString()}`; const end = articleOffset + data.rows.length;
document.getElementById("pageInfo").textContent = total == null
? `${start}–${end} · filtered results`
: `${start}–${end} of approximately ${total.toLocaleString()}`;
document.getElementById("prevBtn").disabled = articleOffset === 0; document.getElementById("prevBtn").disabled = articleOffset === 0;
document.getElementById("nextBtn").disabled = articleOffset + PAGE >= total; document.getElementById("nextBtn").disabled = !data.hasMore;
} }
public/admin/assets/js/events.js
+6 -3
View File
@@ -51,10 +51,13 @@ async function loadEvents() {
`).join(""); `).join("");
const total = data.total; const total = data.total;
document.getElementById("ePageInfo").textContent = const start = data.rows.length ? eventOffset + 1 : 0;
`${eventOffset + 1}–${Math.min(eventOffset + PAGE, total)} of ${total.toLocaleString()}`; const end = eventOffset + data.rows.length;
document.getElementById("ePageInfo").textContent = total == null
? `${start}–${end} · filtered results`
: `${start}–${end} of approximately ${total.toLocaleString()}`;
document.getElementById("ePrevBtn").disabled = eventOffset === 0; document.getElementById("ePrevBtn").disabled = eventOffset === 0;
document.getElementById("eNextBtn").disabled = eventOffset + PAGE >= total; document.getElementById("eNextBtn").disabled = !data.hasMore;
} }
public/admin/pages/autonomy.html
+6 -6
View File
@@ -5,10 +5,10 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta name="theme-color" content="#090b0a"> <meta name="theme-color" content="#090b0a">
<title>Duriin — Autonomy</title> <title>Duriin — Autonomy</title>
<link rel="stylesheet" href="/admin/assets/css/base.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/base.css?v=20260804-3">
<link rel="stylesheet" href="/admin/assets/css/layout.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/layout.css?v=20260804-3">
<link rel="stylesheet" href="/admin/assets/css/components.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/components.css?v=20260804-3">
<link rel="stylesheet" href="/admin/assets/css/autonomy.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/autonomy.css?v=20260804-3">
</head> </head>
<body class="page-autonomy"> <body class="page-autonomy">
@@ -131,7 +131,7 @@
</main> </main>
<div id="toast"><span class="toast-dot"></span><span id="toast-msg"></span></div> <div id="toast"><span class="toast-dot"></span><span id="toast-msg"></span></div>
<script src="/admin/assets/js/app.js?v=20260804-2"></script> <script src="/admin/assets/js/app.js?v=20260804-3"></script>
<script src="/admin/assets/js/autonomy.js?v=20260804-2"></script> <script src="/admin/assets/js/autonomy.js?v=20260804-3"></script>
</body> </body>
</html> </html>
public/admin/pages/ingest/articles.html
+5 -5
View File
@@ -4,9 +4,9 @@
<meta charset="UTF-8"> <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Duriin Admin — Articles</title> <title>Duriin Admin — Articles</title>
<link rel="stylesheet" href="/admin/assets/css/base.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/base.css?v=20260804-3">
<link rel="stylesheet" href="/admin/assets/css/layout.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/layout.css?v=20260804-3">
<link rel="stylesheet" href="/admin/assets/css/components.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/components.css?v=20260804-3">
</head> </head>
<body> <body>
@@ -134,7 +134,7 @@
<div id="toast"><span class="toast-dot"></span><span id="toast-msg"></span></div> <div id="toast"><span class="toast-dot"></span><span id="toast-msg"></span></div>
<script src="/admin/assets/js/app.js?v=20260804-2"></script> <script src="/admin/assets/js/app.js?v=20260804-3"></script>
<script src="/admin/assets/js/articles.js?v=20260804-2"></script> <script src="/admin/assets/js/articles.js?v=20260804-3"></script>
</body> </body>
</html> </html>
public/admin/pages/ingest/events.html
+5 -5
View File
@@ -4,9 +4,9 @@
<meta charset="UTF-8"> <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Duriin Admin — Events</title> <title>Duriin Admin — Events</title>
<link rel="stylesheet" href="/admin/assets/css/base.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/base.css?v=20260804-3">
<link rel="stylesheet" href="/admin/assets/css/layout.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/layout.css?v=20260804-3">
<link rel="stylesheet" href="/admin/assets/css/components.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/components.css?v=20260804-3">
</head> </head>
<body> <body>
@@ -94,7 +94,7 @@
<div id="toast"><span class="toast-dot"></span><span id="toast-msg"></span></div> <div id="toast"><span class="toast-dot"></span><span id="toast-msg"></span></div>
<script src="/admin/assets/js/app.js?v=20260804-2"></script> <script src="/admin/assets/js/app.js?v=20260804-3"></script>
<script src="/admin/assets/js/events.js?v=20260804-2"></script> <script src="/admin/assets/js/events.js?v=20260804-3"></script>
</body> </body>
</html> </html>
public/admin/pages/intelligence/graph.html
+8 -8
View File
@@ -4,10 +4,10 @@
<meta charset="UTF-8"> <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Duriin Admin — Intelligence / Graph</title> <title>Duriin Admin — Intelligence / Graph</title>
<link rel="stylesheet" href="/admin/assets/css/base.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/base.css?v=20260804-3">
<link rel="stylesheet" href="/admin/assets/css/layout.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/layout.css?v=20260804-3">
<link rel="stylesheet" href="/admin/assets/css/components.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/components.css?v=20260804-3">
<link rel="stylesheet" href="/admin/assets/css/intel.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/intel.css?v=20260804-3">
</head> </head>
<body class="page-graph"> <body class="page-graph">
@@ -72,9 +72,9 @@
<div id="toast"><span class="toast-dot"></span><span id="toast-msg"></span></div> <div id="toast"><span class="toast-dot"></span><span id="toast-msg"></span></div>
<script src="/admin/assets/js/d3.min.js?v=20260804-2"></script> <script src="/admin/assets/js/d3.min.js?v=20260804-3"></script>
<script src="/admin/assets/js/app.js?v=20260804-2"></script> <script src="/admin/assets/js/app.js?v=20260804-3"></script>
<script src="/admin/assets/js/intel-shared.js?v=20260804-2"></script> <script src="/admin/assets/js/intel-shared.js?v=20260804-3"></script>
<script src="/admin/assets/js/intel-graph.js?v=20260804-2"></script> <script src="/admin/assets/js/intel-graph.js?v=20260804-3"></script>
</body> </body>
</html> </html>
public/admin/pages/intelligence/knowledge.html
+7 -7
View File
@@ -4,10 +4,10 @@
<meta charset="UTF-8"> <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Duriin Admin — Intelligence / Knowledge</title> <title>Duriin Admin — Intelligence / Knowledge</title>
<link rel="stylesheet" href="/admin/assets/css/base.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/base.css?v=20260804-3">
<link rel="stylesheet" href="/admin/assets/css/layout.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/layout.css?v=20260804-3">
<link rel="stylesheet" href="/admin/assets/css/components.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/components.css?v=20260804-3">
<link rel="stylesheet" href="/admin/assets/css/intel.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/intel.css?v=20260804-3">
</head> </head>
<body> <body>
@@ -90,8 +90,8 @@
<div id="toast"><span class="toast-dot"></span><span id="toast-msg"></span></div> <div id="toast"><span class="toast-dot"></span><span id="toast-msg"></span></div>
<script src="/admin/assets/js/app.js?v=20260804-2"></script> <script src="/admin/assets/js/app.js?v=20260804-3"></script>
<script src="/admin/assets/js/intel-shared.js?v=20260804-2"></script> <script src="/admin/assets/js/intel-shared.js?v=20260804-3"></script>
<script src="/admin/assets/js/intel-knowledge.js?v=20260804-2"></script> <script src="/admin/assets/js/intel-knowledge.js?v=20260804-3"></script>
</body> </body>
</html> </html>
public/admin/pages/intelligence/predictions.html
+7 -7
View File
@@ -4,10 +4,10 @@
<meta charset="UTF-8"> <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Duriin Admin — Intelligence / Predictions</title> <title>Duriin Admin — Intelligence / Predictions</title>
<link rel="stylesheet" href="/admin/assets/css/base.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/base.css?v=20260804-3">
<link rel="stylesheet" href="/admin/assets/css/layout.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/layout.css?v=20260804-3">
<link rel="stylesheet" href="/admin/assets/css/components.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/components.css?v=20260804-3">
<link rel="stylesheet" href="/admin/assets/css/intel.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/intel.css?v=20260804-3">
</head> </head>
<body> <body>
@@ -81,8 +81,8 @@
<div id="toast"><span class="toast-dot"></span><span id="toast-msg"></span></div> <div id="toast"><span class="toast-dot"></span><span id="toast-msg"></span></div>
<script src="/admin/assets/js/app.js?v=20260804-2"></script> <script src="/admin/assets/js/app.js?v=20260804-3"></script>
<script src="/admin/assets/js/intel-shared.js?v=20260804-2"></script> <script src="/admin/assets/js/intel-shared.js?v=20260804-3"></script>
<script src="/admin/assets/js/intel-predictions.js?v=20260804-2"></script> <script src="/admin/assets/js/intel-predictions.js?v=20260804-3"></script>
</body> </body>
</html> </html>
public/admin/pages/intelligence/signals.html
+7 -7
View File
@@ -4,10 +4,10 @@
<meta charset="UTF-8"> <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Duriin Admin — Intelligence / Signals</title> <title>Duriin Admin — Intelligence / Signals</title>
<link rel="stylesheet" href="/admin/assets/css/base.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/base.css?v=20260804-3">
<link rel="stylesheet" href="/admin/assets/css/layout.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/layout.css?v=20260804-3">
<link rel="stylesheet" href="/admin/assets/css/components.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/components.css?v=20260804-3">
<link rel="stylesheet" href="/admin/assets/css/intel.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/intel.css?v=20260804-3">
</head> </head>
<body> <body>
@@ -137,8 +137,8 @@
<div id="toast"><span class="toast-dot"></span><span id="toast-msg"></span></div> <div id="toast"><span class="toast-dot"></span><span id="toast-msg"></span></div>
<script src="/admin/assets/js/app.js?v=20260804-2"></script> <script src="/admin/assets/js/app.js?v=20260804-3"></script>
<script src="/admin/assets/js/intel-shared.js?v=20260804-2"></script> <script src="/admin/assets/js/intel-shared.js?v=20260804-3"></script>
<script src="/admin/assets/js/intel-signals.js?v=20260804-2"></script> <script src="/admin/assets/js/intel-signals.js?v=20260804-3"></script>
</body> </body>
</html> </html>
public/admin/pages/sql.html
+5 -5
View File
@@ -4,9 +4,9 @@
<meta charset="UTF-8"> <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Duriin Admin — SQL</title> <title>Duriin Admin — SQL</title>
<link rel="stylesheet" href="/admin/assets/css/base.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/base.css?v=20260804-3">
<link rel="stylesheet" href="/admin/assets/css/layout.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/layout.css?v=20260804-3">
<link rel="stylesheet" href="/admin/assets/css/components.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/components.css?v=20260804-3">
</head> </head>
<body> <body>
@@ -45,7 +45,7 @@
<div id="toast"><span class="toast-dot"></span><span id="toast-msg"></span></div> <div id="toast"><span class="toast-dot"></span><span id="toast-msg"></span></div>
<script src="/admin/assets/js/app.js?v=20260804-2"></script> <script src="/admin/assets/js/app.js?v=20260804-3"></script>
<script src="/admin/assets/js/sql.js?v=20260804-2"></script> <script src="/admin/assets/js/sql.js?v=20260804-3"></script>
</body> </body>
</html> </html>
public/admin/pages/stats.html
+6 -6
View File
@@ -4,10 +4,10 @@
<meta charset="UTF-8"> <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Duriin Admin — Stats</title> <title>Duriin Admin — Stats</title>
<link rel="stylesheet" href="/admin/assets/css/base.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/base.css?v=20260804-3">
<link rel="stylesheet" href="/admin/assets/css/layout.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/layout.css?v=20260804-3">
<link rel="stylesheet" href="/admin/assets/css/components.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/components.css?v=20260804-3">
<link rel="stylesheet" href="/admin/assets/css/intel.css?v=20260804-2"> <link rel="stylesheet" href="/admin/assets/css/intel.css?v=20260804-3">
</head> </head>
<body> <body>
@@ -74,7 +74,7 @@
<div id="toast"><span class="toast-dot"></span><span id="toast-msg"></span></div> <div id="toast"><span class="toast-dot"></span><span id="toast-msg"></span></div>
<script src="/admin/assets/js/app.js?v=20260804-2"></script> <script src="/admin/assets/js/app.js?v=20260804-3"></script>
<script src="/admin/assets/js/stats.js?v=20260804-2"></script> <script src="/admin/assets/js/stats.js?v=20260804-3"></script>
</body> </body>
</html> </html>
src/routes/admin.js
+19 -20
View File
@@ -251,18 +251,20 @@ async function adminRoutes(fastify) {
const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : ''; const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : '';
const total = db.prepare(`SELECT COUNT(*) as n FROM articles ${where}`).get(...params).n; const total = conditions.length
? null
: (db.prepare("SELECT seq FROM sqlite_sequence WHERE name='articles'").get()?.seq || 0);
params.push(limit, offset); params.push(limit + 1, offset);
const rows = db.prepare(` const fetchedRows = db.prepare(`
SELECT id, title, url, source, pub_date, ingested_at, content_status, is_index_page, has_embedding, language SELECT id, title, url, source, pub_date, ingested_at, content_status, is_index_page, has_embedding, language
FROM articles FROM articles
${where} ${where}
ORDER BY ingested_at DESC, id DESC ORDER BY id DESC
LIMIT ? OFFSET ? LIMIT ? OFFSET ?
`).all(...params); `).all(...params);
return { total, rows }; return { total, rows: fetchedRows.slice(0, limit), hasMore: fetchedRows.length > limit };
}); });
fastify.get('/admin/api/articles/:id', async (request, reply) => { fastify.get('/admin/api/articles/:id', async (request, reply) => {
@@ -366,7 +368,7 @@ async function adminRoutes(fastify) {
// whitelist sort columns + direction so user input cant break the query // whitelist sort columns + direction so user input cant break the query
const sortMap = { const sortMap = {
created_desc: 'e.created_at DESC', created_desc: 'e.id DESC',
created_asc: 'e.created_at ASC', created_asc: 'e.created_at ASC',
articles_desc: 'article_count DESC', articles_desc: 'article_count DESC',
articles_asc: 'article_count ASC', articles_asc: 'article_count ASC',
@@ -385,15 +387,16 @@ async function adminRoutes(fastify) {
FROM events e ${whereClause} FROM events e ${whereClause}
`; `;
const filteredEvents = `SELECT * FROM (${eventProjection}) ${countWhereClause}`; const filteredEvents = `SELECT * FROM (${eventProjection}) ${countWhereClause}`;
const totalRow = db.prepare(`SELECT COUNT(*) AS n FROM (${filteredEvents})`) const total = where.length || having.length
.get(...whereParams, ...havingParams); ? null
const rows = db.prepare(` : (db.prepare("SELECT seq FROM sqlite_sequence WHERE name='events'").get()?.seq || 0);
const fetchedRows = db.prepare(`
${filteredEvents} ${filteredEvents}
ORDER BY ${orderBy.replaceAll('e.', '')} ORDER BY ${orderBy.replaceAll('e.', '')}
LIMIT ? OFFSET ? LIMIT ? OFFSET ?
`).all(...whereParams, ...havingParams, limit, offset); `).all(...whereParams, ...havingParams, limit + 1, offset);
return { total: totalRow.n, rows }; return { total, rows: fetchedRows.slice(0, limit), hasMore: fetchedRows.length > limit };
}); });
fastify.delete('/admin/api/events/:id', async (request, reply) => { fastify.delete('/admin/api/events/:id', async (request, reply) => {
@@ -844,10 +847,8 @@ async function adminRoutes(fastify) {
const counts = db.prepare(` const counts = db.prepare(`
SELECT SELECT
COALESCE((SELECT seq FROM sqlite_sequence WHERE name='articles'), 0) AS total, COALESCE((SELECT seq FROM sqlite_sequence WHERE name='articles'), 0) AS total,
(SELECT COUNT(*) FROM articles (SELECT COUNT(*) FROM article_embedding_meta) AS withContent,
WHERE content IS NOT NULL AND content != '' (SELECT COUNT(*) FROM article_embedding_meta) AS withEmbedding,
AND is_index_page = 0 AND has_embedding = 1) AS withContent,
(SELECT COUNT(*) FROM articles WHERE has_embedding = 1) AS withEmbedding,
COALESCE((SELECT seq FROM sqlite_sequence WHERE name='events'), 0) AS eventCount COALESCE((SELECT seq FROM sqlite_sequence WHERE name='events'), 0) AS eventCount
`).get(); `).get();
statsSummaryCache = { at: Date.now(), value: counts }; statsSummaryCache = { at: Date.now(), value: counts };
@@ -865,13 +866,11 @@ async function adminRoutes(fastify) {
const counts = db.prepare(` const counts = db.prepare(`
SELECT SELECT
(SELECT COUNT(*) FROM articles) as total, (SELECT COUNT(*) FROM articles) as total,
(SELECT COUNT(*) FROM articles (SELECT COUNT(*) FROM article_embedding_meta) as withContent,
WHERE content IS NOT NULL AND content != '' (SELECT COUNT(*) FROM article_embedding_meta) as withEmbedding,
AND is_index_page = 0 AND has_embedding = 1) as withContent,
(SELECT COUNT(*) FROM articles WHERE has_embedding = 1) as withEmbedding,
(SELECT COUNT(*) FROM events) as eventCount, (SELECT COUNT(*) FROM events) as eventCount,
(SELECT COUNT(*) FROM articles WHERE ingested_at >= datetime('now', '-1 hour')) as ingestedPerHour, (SELECT COUNT(*) FROM articles WHERE ingested_at >= datetime('now', '-1 hour')) as ingestedPerHour,
(SELECT COUNT(*) FROM articles WHERE content_attempted_at >= datetime('now', '-1 hour')) as contentPerHour (SELECT COUNT(*) FROM article_embedding_meta WHERE embedded_at >= datetime('now', '-1 hour')) as contentPerHour
`).get(); `).get();
const bySource = db.prepare(` const bySource = db.prepare(`