diff --git a/admin.html b/admin.html new file mode 100644 index 0000000..983664b --- /dev/null +++ b/admin.html @@ -0,0 +1,692 @@ + + + + + +Duriin Admin + + + + +
+

Duriin Admin

+
+ + + +
+
+ +
+
Total articles—
+
With content—
+
With embedding—
+
Events—
+
+ +
+ + +
+
+ + + + + + +
+ + + + + + + + + + + + + +
IDTitleSourceStatusIngested
+ + +
+ + + + + + + +
+ + +
+
+

Article

+
+ +
+ + +
+
+ + +
+
+ + +
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ +
+ + + +
+
+
+ + +
+
+

Edit Event

+
+ + +
+
+ + + +
+
+
+ +
+ + + + diff --git a/config.json b/config.json index 025dd19..559ae0d 100644 --- a/config.json +++ b/config.json @@ -3,6 +3,10 @@ "port": 3001, "host": "0.0.0.0" }, + "admin": { + "username": "admin", + "password": "changeme" + }, "database": { "path": "./archive.sqlite" }, @@ -19,7 +23,7 @@ "tickers": [] }, "openRouter": { - "enabled": false, + "enabled": true, "apiKey": "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", "embeddingModel": "qwen/qwen3-embedding-8b" }, diff --git a/server.js b/server.js index c8de679..ce1a82e 100644 --- a/server.js +++ b/server.js @@ -4,6 +4,7 @@ const articleRoutes = require('./src/routes/articles'); const statusRoutes = require('./src/routes/status'); const sourcesRoutes = require('./src/routes/sources'); const eventRoutes = require('./src/routes/events'); +const adminRoutes = require('./src/routes/admin'); const config = require('./src/config'); const { startScheduler } = require('./src/scheduler'); @@ -14,6 +15,7 @@ app.register(articleRoutes); app.register(statusRoutes); app.register(sourcesRoutes); app.register(eventRoutes); +app.register(adminRoutes); app.get('/', async () => ({ ok: true })); diff --git a/src/routes/admin.js b/src/routes/admin.js new file mode 100644 index 0000000..9b94879 --- /dev/null +++ b/src/routes/admin.js @@ -0,0 +1,229 @@ +const fs = require('fs'); +const path = require('path'); +const db = require('../db'); +const config = require('../config'); + +const adminUser = (config.admin && config.admin.username) || 'admin'; +const adminPass = (config.admin && config.admin.password) || 'changeme'; + +function checkAuth(request, reply) { + const header = request.headers['authorization'] || ''; + if (!header.startsWith('Basic ')) { + reply.header('WWW-Authenticate', 'Basic realm="Duriin Admin"'); + reply.code(401).send('Unauthorized'); + return false; + } + + const decoded = Buffer.from(header.slice(6), 'base64').toString('utf8'); + const colon = decoded.indexOf(':'); + const user = decoded.slice(0, colon); + const pass = decoded.slice(colon + 1); + + if (user !== adminUser || pass !== adminPass) { + reply.header('WWW-Authenticate', 'Basic realm="Duriin Admin"'); + reply.code(401).send('Unauthorized'); + return false; + } + + return true; +} + +const htmlPath = path.join(__dirname, '..', '..', 'admin.html'); + +async function adminRoutes(fastify) { + + fastify.get('/admin', async (request, reply) => { + if (!checkAuth(request, reply)) return; + reply.type('text/html'); + return fs.createReadStream(htmlPath); + }); + + // list articles — all of them, not just the ones with embeddings + fastify.get('/admin/api/articles', async (request, reply) => { + if (!checkAuth(request, reply)) return; + + const q = request.query || {}; + const limit = Math.min(parseInt(q.limit, 10) || 50, 200); + const offset = parseInt(q.offset, 10) || 0; + + const conditions = []; + const params = []; + + if (q.keyword) { + conditions.push('(title LIKE ? OR description LIKE ? OR content LIKE ?)'); + const like = `%${q.keyword}%`; + params.push(like, like, like); + } + + if (q.source) { + conditions.push('source = ?'); + params.push(q.source); + } + + if (q.content_status) { + if (q.content_status === 'null') { + conditions.push('content_status IS NULL'); + } else { + conditions.push('content_status = ?'); + params.push(q.content_status); + } + } + + if (q.from) { + conditions.push('ingested_at >= ?'); + params.push(q.from); + } + + if (q.to) { + conditions.push('ingested_at <= ?'); + params.push(q.to); + } + + const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : ''; + + const total = db.prepare(`SELECT COUNT(*) as n FROM articles ${where}`).get(...params).n; + + params.push(limit, offset); + const rows = db.prepare(` + SELECT id, title, url, source, pub_date, ingested_at, content_status, is_index_page, has_embedding, language + FROM articles + ${where} + ORDER BY ingested_at DESC, id DESC + LIMIT ? OFFSET ? + `).all(...params); + + return { total, rows }; + }); + + fastify.get('/admin/api/articles/:id', async (request, reply) => { + if (!checkAuth(request, reply)) return; + + const row = db.prepare(` + SELECT id, title, description, content, url, normalized_title, source, + pub_date, pub_date_effective, ingested_at, content_status, content_error, + content_attempted_at, content_attempt_count, is_index_page, has_embedding, language, event_id + FROM articles WHERE id = ? + `).get(request.params.id); + + if (!row) { reply.code(404); return { error: 'not found' }; } + return row; + }); + + fastify.patch('/admin/api/articles/:id', async (request, reply) => { + if (!checkAuth(request, reply)) return; + + const id = parseInt(request.params.id, 10); + const body = request.body || {}; + + // only allow editing these fields + const allowed = ['title', 'description', 'content', 'content_status', 'is_index_page', 'language', 'pub_date']; + const updates = []; + const params = []; + + for (const key of allowed) { + if (Object.prototype.hasOwnProperty.call(body, key)) { + updates.push(`${key} = ?`); + params.push(body[key]); + } + } + + if (updates.length === 0) { + reply.code(400); + return { error: 'no valid fields provided' }; + } + + params.push(id); + db.prepare(`UPDATE articles SET ${updates.join(', ')} WHERE id = ?`).run(...params); + return { ok: true }; + }); + + fastify.delete('/admin/api/articles/:id', async (request, reply) => { + if (!checkAuth(request, reply)) return; + + const id = parseInt(request.params.id, 10); + + // remove embeddings first so foreign key stuff doesnt bite us + db.prepare(`DELETE FROM article_embedding_store WHERE article_id = ?`).run(id); + db.prepare(`DELETE FROM article_embedding_meta WHERE article_id = ?`).run(id); + + try { + db.prepare(`DELETE FROM article_embeddings WHERE article_id = ?`).run(id); + } catch (_) {} + + db.prepare(`DELETE FROM articles WHERE id = ?`).run(id); + return { ok: true }; + }); + + // sources list for filter dropdown + fastify.get('/admin/api/sources', async (request, reply) => { + if (!checkAuth(request, reply)) return; + + const rows = db.prepare(`SELECT DISTINCT source FROM articles ORDER BY source`).all(); + return rows.map(r => r.source); + }); + + // events + fastify.get('/admin/api/events', async (request, reply) => { + if (!checkAuth(request, reply)) return; + + const q = request.query || {}; + const limit = Math.min(parseInt(q.limit, 10) || 50, 200); + const offset = parseInt(q.offset, 10) || 0; + + const total = db.prepare(`SELECT COUNT(*) as n FROM events`).get().n; + const rows = db.prepare(` + SELECT e.id, e.title, e.created_at, COUNT(a.id) as article_count + FROM events e + LEFT JOIN articles a ON a.event_id = e.id + GROUP BY e.id + ORDER BY e.created_at DESC + LIMIT ? OFFSET ? + `).all(limit, offset); + + return { total, rows }; + }); + + fastify.delete('/admin/api/events/:id', async (request, reply) => { + if (!checkAuth(request, reply)) return; + + const id = parseInt(request.params.id, 10); + // detach articles from this event but dont delete the articles themselves + db.prepare(`UPDATE articles SET event_id = NULL WHERE event_id = ?`).run(id); + db.prepare(`DELETE FROM events WHERE id = ?`).run(id); + return { ok: true }; + }); + + fastify.patch('/admin/api/events/:id', async (request, reply) => { + if (!checkAuth(request, reply)) return; + + const id = parseInt(request.params.id, 10); + const body = request.body || {}; + + if (!body.title) { reply.code(400); return { error: 'title is required' }; } + db.prepare(`UPDATE events SET title = ? WHERE id = ?`).run(body.title, id); + return { ok: true }; + }); + + // stats for dashboard header + fastify.get('/admin/api/stats', async (request, reply) => { + if (!checkAuth(request, reply)) return; + + const total = db.prepare(`SELECT COUNT(*) as n FROM articles`).get().n; + const withContent = db.prepare(`SELECT COUNT(*) as n FROM articles WHERE content IS NOT NULL AND content != ''`).get().n; + const withEmbedding = db.prepare(`SELECT COUNT(*) as n FROM articles WHERE has_embedding = 1`).get().n; + const eventCount = db.prepare(`SELECT COUNT(*) as n FROM events`).get().n; + + const bySource = db.prepare(` + SELECT source, COUNT(*) as n FROM articles GROUP BY source ORDER BY n DESC + `).all(); + + const byStatus = db.prepare(` + SELECT COALESCE(content_status, 'null') as status, COUNT(*) as n + FROM articles GROUP BY content_status ORDER BY n DESC + `).all(); + + return { total, withContent, withEmbedding, eventCount, bySource, byStatus }; + }); +} + +module.exports = adminRoutes; \ No newline at end of file diff --git a/src/routes/events.js b/src/routes/events.js index 574281e..b7e0037 100644 --- a/src/routes/events.js +++ b/src/routes/events.js @@ -14,54 +14,51 @@ async function eventRoutes(fastify) { fastify.get('/events', async (request, reply) => { const query = request.query || {}; + const conditions = []; + const params = []; + if (query.id) { const id = Number.parseInt(query.id, 10); if (!Number.isFinite(id)) { reply.code(400); return { error: 'id must be a number' }; } - - const event = db.prepare(`SELECT id, title, created_at FROM events WHERE id = ?`).get(id); - if (!event) { - reply.code(404); - return { error: 'Event not found' }; - } - - const articles = db.prepare(` - SELECT id, title, description, content, url, normalized_title, source, pub_date, ingested_at - FROM articles - WHERE event_id = ? - AND content IS NOT NULL AND content != '' - AND is_index_page = 0 - ORDER BY pub_date_effective DESC, id DESC - `).all(id); - - return { ...event, articles }; + conditions.push('e.id = ?'); + params.push(id); } const limit = parseLimit(query.limit); const offset = parseOffset(query.offset); const SORT_COLUMNS = { - created_at: 'e.created_at', + pub_date: '(SELECT MIN(a.pub_date_effective) FROM articles a WHERE a.event_id = e.id AND a.content IS NOT NULL AND a.content != \'\' AND a.is_index_page = 0)', id: 'e.id', - article_count: 'article_count', }; - const sortBy = SORT_COLUMNS[query.sort_by] || SORT_COLUMNS.created_at; + const sortBy = SORT_COLUMNS[query.sort_by] || SORT_COLUMNS.pub_date; const order = String(query.order || '').toLowerCase() === 'asc' ? 'ASC' : 'DESC'; - return db.prepare(` - SELECT e.id, e.title, e.created_at, - COUNT(a.id) AS article_count + const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : ''; + + const events = db.prepare(` + SELECT e.id, e.title, + (SELECT MIN(a.pub_date_effective) FROM articles a WHERE a.event_id = e.id AND a.content IS NOT NULL AND a.content != '' AND a.is_index_page = 0) AS pub_date FROM events e - LEFT JOIN articles a ON a.event_id = e.id - AND a.content IS NOT NULL AND a.content != '' - AND a.is_index_page = 0 - GROUP BY e.id + ${where} ORDER BY ${sortBy} ${order}, e.id ${order} LIMIT ? OFFSET ? - `).all(limit, offset); + `).all(...params, limit, offset); + + const getArticles = db.prepare(` + SELECT id, title, description, content, url, normalized_title, source, pub_date, ingested_at + FROM articles + WHERE event_id = ? + AND content IS NOT NULL AND content != '' + AND is_index_page = 0 + ORDER BY pub_date_effective DESC, id DESC + `); + + return events.map(e => ({ ...e, articles: getArticles.all(e.id) })); }); } diff --git a/summary-prompt.md b/summary-prompt.md new file mode 100644 index 0000000..4911a9d --- /dev/null +++ b/summary-prompt.md @@ -0,0 +1,11 @@ +You are a news summarizer. Extract only the facts. Write in plain declarative sentences. No hedging, no attribution ("the article says", "according to", "per the article"), no filler, no transitions. + +Each bullet: one fact, one sentence, under 20 words. Lead with the subject. Use past tense. + +Preserve: +- Named entities (people, companies, tickers) +- Figures (prices, percentages, dates, quantities) +- Cause-effect relationships +- Direct quotes only if they carry unique signal — strip the quote wrapper, state it as fact + +Maximum 10 bullets. No intro. No conclusion. No meta-commentary. \ No newline at end of file