@@ -653,12 +712,191 @@ document.getElementById('eDeleteBtn').onclick = async () => {
} catch (e) { toast('Delete failed', true); }
};
+// ── intelligence ───────────────────────────────────────────────────────────
+
+let intelOffset = 0;
+
+async function loadIntelligenceStats() {
+ const data = await api('/admin/api/intelligence/stats');
+ if (!data.available) {
+ document.getElementById('intel-unavailable').style.display = '';
+ document.getElementById('intel-content').style.display = 'none';
+ return false;
+ }
+
+ document.getElementById('intel-unavailable').style.display = 'none';
+ document.getElementById('intel-content').style.display = '';
+
+ const queueMap = {};
+ (data.queue || []).forEach(r => queueMap[r.status] = r.n);
+
+ document.getElementById('intel-stats-row').innerHTML = [
+ ['Queue pending', (queueMap.pending || 0).toLocaleString()],
+ ['Processed', (queueMap.processed || 0).toLocaleString()],
+ ['Skipped', (queueMap.skipped || 0).toLocaleString()],
+ ['Knowledge rows', data.knowledge.toLocaleString()],
+ ['Predictions', data.predictions.toLocaleString()],
+ ['Companies', `${data.embeddings}/${data.companies} embedded`],
+ ].map(([label, value]) => `
+
+ ${label}
+ ${value}
+
+ `).join('');
+
+ return true;
+}
+
+async function loadIntelligenceCompanies() {
+ const companies = await api('/admin/api/intelligence/companies');
+ const sel = document.getElementById('i-company');
+ sel.innerHTML = '
';
+ companies.forEach(c => {
+ const opt = document.createElement('option');
+ opt.value = c.id;
+ opt.textContent = `${c.name} (${c.ticker})`;
+ sel.appendChild(opt);
+ });
+}
+
+async function loadIntelligence() {
+ const view = document.getElementById('i-view').value;
+ const companyId = document.getElementById('i-company').value;
+ const type = document.getElementById('i-type').value;
+
+ const params = new URLSearchParams({ limit: PAGE, offset: intelOffset });
+ if (companyId) params.set('company_id', companyId);
+
+ if (view === 'knowledge') {
+ document.getElementById('i-type').parentElement.style.display = '';
+ if (type) params.set('type', type);
+
+ const data = await api(`/admin/api/intelligence/knowledge?${params}`);
+
+ document.getElementById('intel-thead').innerHTML = `
+
| ID | Company | Event | Type | Data | Created |
|---|
`;
+
+ document.getElementById('intel-tbody').innerHTML = data.rows.map(r => {
+ let parsed = {};
+ try { parsed = JSON.parse(r.data); } catch (_) {}
+ const summary = Object.values(parsed).filter(v => typeof v === 'string').join(' · ').slice(0, 120);
+ return `
+ | ${r.id} |
+ ${r.company_name} |
+ ${r.event_id} |
+ ${r.type} |
+ ${summary} |
+ ${r.created_at ? r.created_at.slice(0,16) : '—'} |
+
`;
+ }).join('');
+
+ const total = data.total;
+ document.getElementById('iPageInfo').textContent =
+ `${intelOffset + 1}–${Math.min(intelOffset + PAGE, total)} of ${total.toLocaleString()}`;
+ document.getElementById('iPrevBtn').disabled = intelOffset === 0;
+ document.getElementById('iNextBtn').disabled = intelOffset + PAGE >= total;
+
+ } else {
+ document.getElementById('i-type').parentElement.style.display = 'none';
+
+ const data = await api(`/admin/api/intelligence/predictions?${params}`);
+
+ document.getElementById('intel-thead').innerHTML = `
+
| ID | Company | Event | Type | Direction | Magnitude | Timeframe | Rationale | Created |
|---|
`;
+
+ document.getElementById('intel-tbody').innerHTML = data.rows.map(r => `
+
+ | ${r.id} |
+ ${r.company_name} |
+ ${r.event_id} |
+ ${r.type} |
+ ${r.direction || '—'} |
+ ${r.magnitude || '—'} |
+ ${r.timeframe || '—'} |
+ ${r.rationale || '—'} |
+ ${r.created_at ? r.created_at.slice(0,16) : '—'} |
+
+ `).join('');
+
+ const total = data.total;
+ document.getElementById('iPageInfo').textContent =
+ `${intelOffset + 1}–${Math.min(intelOffset + PAGE, total)} of ${total.toLocaleString()}`;
+ document.getElementById('iPrevBtn').disabled = intelOffset === 0;
+ document.getElementById('iNextBtn').disabled = intelOffset + PAGE >= total;
+ }
+}
+
+document.getElementById('iPrevBtn').onclick = () => { intelOffset = Math.max(0, intelOffset - PAGE); loadIntelligence(); };
+document.getElementById('iNextBtn').onclick = () => { intelOffset += PAGE; loadIntelligence(); };
+
+document.getElementById('i-view').onchange = () => { intelOffset = 0; loadIntelligence(); };
+
+// ── sql console ────────────────────────────────────────────────────────────
+
+async function runSql() {
+ const sql = document.getElementById('sql-input').value.trim();
+ if (!sql) return;
+
+ const database = document.getElementById('sql-db').value;
+ const errEl = document.getElementById('sql-error');
+ const resultsEl = document.getElementById('sql-results');
+ const elapsedEl = document.getElementById('sql-elapsed');
+
+ errEl.style.display = 'none';
+ resultsEl.innerHTML = '';
+ elapsedEl.textContent = '';
+
+ try {
+ const data = await fetch('/admin/api/sql', {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ sql, database }),
+ }).then(r => r.json());
+
+ if (data.error) {
+ errEl.textContent = data.error;
+ errEl.style.display = '';
+ return;
+ }
+
+ elapsedEl.textContent = `${data.elapsed}ms`;
+
+ if (data.rows && data.rows.length > 0) {
+ const cols = Object.keys(data.rows[0]);
+ resultsEl.innerHTML = `
+
+ ${cols.map(c => `| ${c} | `).join('')}
+ ${data.rows.map(r =>
+ `${cols.map(c => `| ${r[c] ?? 'NULL'} | `).join('')}
`
+ ).join('')}
+
+
${data.rows.length} row${data.rows.length !== 1 ? 's' : ''}
+ `;
+ } else if (data.rows) {
+ resultsEl.innerHTML = '
No rows returned.
';
+ } else {
+ resultsEl.innerHTML = `
${data.changes} row${data.changes !== 1 ? 's' : ''} affected.
`;
+ }
+ } catch (e) {
+ errEl.textContent = e.message;
+ errEl.style.display = '';
+ }
+}
+
+document.getElementById('sql-run-btn').onclick = runSql;
+
+document.getElementById('sql-input').addEventListener('keydown', e => {
+ if (e.key === 'Enter' && (e.metaKey || e.ctrlKey)) runSql();
+});
+
// ── tabs ───────────────────────────────────────────────────────────────────
const tabContents = {
articles: document.getElementById('tab-articles'),
events: document.getElementById('tab-events'),
stats: document.getElementById('tab-stats'),
+ intelligence: document.getElementById('tab-intelligence'),
+ sql: document.getElementById('tab-sql'),
};
document.querySelectorAll('.tab-btn').forEach(btn => {
@@ -671,6 +909,7 @@ document.querySelectorAll('.tab-btn').forEach(btn => {
});
if (tab === 'events') loadEvents();
if (tab === 'stats') loadStats();
+ if (tab === 'intelligence') { intelOffset = 0; loadIntelligenceStats().then(ok => { if (ok) { loadIntelligenceCompanies(); loadIntelligence(); } }); }
};
});
diff --git a/config.json b/config.json
index 2737cec..d143027 100644
--- a/config.json
+++ b/config.json
@@ -1,6 +1,6 @@
{
- "duriin_db": "/data/archive.sqlite",
- "intelligence_db": "/data/intelligence.sqlite",
+ "duriin_db": "./archive.sqlite",
+ "intelligence_db": "./intelligence.sqlite",
"llm": {
"baseUrl": "https://openrouter.ai/api/v1",
"model": "qwen/qwen3-235b-a22b-2507",
diff --git a/docker-compose.yml b/docker-compose.yml
index fdb4add..07169c6 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -22,6 +22,8 @@ services:
- ./data:/data
environment:
NODE_ENV: production
+ DURIIN_DB: /data/archive.sqlite
+ INTELLIGENCE_DB: /data/intelligence.sqlite
restart: unless-stopped
networks:
- nginx_proxy_manager_default
diff --git a/intelligence/db.js b/intelligence/db.js
index 2bdfaa8..6dc901b 100644
--- a/intelligence/db.js
+++ b/intelligence/db.js
@@ -1,4 +1,5 @@
const Database = require("better-sqlite3");
+const sqliteVec = require("sqlite-vec");
let archiveDb = null;
let intelligenceDb = null;
@@ -6,6 +7,7 @@ let intelligenceDb = null;
function getArchiveDb(dbPath) {
if (!archiveDb) {
archiveDb = new Database(dbPath, { readonly: true });
+ sqliteVec.load(archiveDb);
archiveDb.pragma("journal_mode = WAL");
}
return archiveDb;
diff --git a/intelligence/index.js b/intelligence/index.js
index 7ede601..d53055c 100644
--- a/intelligence/index.js
+++ b/intelligence/index.js
@@ -17,8 +17,8 @@ function resolvePath(p, fallback) {
}
const config = {
- duriin_db: resolvePath(rawConfig.duriin_db, path.resolve(configDir, "archive.sqlite")),
- intelligence_db: resolvePath(rawConfig.intelligence_db, path.resolve(configDir, "intelligence.sqlite")),
+ duriin_db: process.env.DURIIN_DB || resolvePath(rawConfig.duriin_db, path.resolve(configDir, "archive.sqlite")),
+ intelligence_db: process.env.INTELLIGENCE_DB || resolvePath(rawConfig.intelligence_db, path.resolve(configDir, "intelligence.sqlite")),
llm: rawConfig.llm || {},
workers: rawConfig.workers || {},
openRouter: rawConfig.openRouter || {},
diff --git a/package.json b/package.json
index 9bf7332..e2dce5c 100644
--- a/package.json
+++ b/package.json
@@ -4,7 +4,8 @@
"description": "News ingestion archive server",
"main": "server.js",
"scripts": {
- "start": "node server.js"
+ "start": "node server.js",
+ "intelligence": "node intelligence/index.js"
},
"keywords": [],
"author": "",
diff --git a/src/db.js b/src/db.js
index e0e95e0..e04889a 100644
--- a/src/db.js
+++ b/src/db.js
@@ -9,8 +9,6 @@ sqliteVec.load(db);
db.pragma('journal_mode = WAL');
-// the image column is retained as a no-op for backwards compat with old rows.
-// new code never writes to it; drop in a future migration if you really want
db.exec(`
CREATE TABLE IF NOT EXISTS articles (
id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -21,97 +19,29 @@ db.exec(`
content_status TEXT,
content_error TEXT,
content_attempted_at TEXT,
+ content_attempt_count INTEGER NOT NULL DEFAULT 0,
+ content_retry_after TEXT,
is_index_page INTEGER NOT NULL DEFAULT 0,
+ has_embedding INTEGER NOT NULL DEFAULT 0,
url TEXT NOT NULL UNIQUE,
normalized_title TEXT NOT NULL,
source TEXT NOT NULL,
pub_date TEXT,
+ pub_date_effective TEXT,
+ language TEXT,
+ event_id INTEGER REFERENCES events(id),
ingested_at TEXT NOT NULL DEFAULT (datetime('now'))
);
`);
-function rebuildArticlesTableIfNeeded() {
- const indexes = db.prepare(`PRAGMA index_list('articles')`).all();
- const hasUniqueNormalizedTitleIndex = indexes.some((index) => {
- if (index.origin !== 'u' || !index.name) {
- return false;
- }
-
- const columns = db.prepare(`PRAGMA index_info('${index.name.replace(/'/g, "''")}')`).all();
- return columns.length === 1 && columns[0].name === 'normalized_title';
- });
-
- if (!hasUniqueNormalizedTitleIndex) {
- return;
- }
-
- db.exec(`
- BEGIN;
-
- CREATE TABLE articles_rebuild (
- id INTEGER PRIMARY KEY AUTOINCREMENT,
- title TEXT NOT NULL,
- description TEXT,
- content TEXT,
- image TEXT,
- content_status TEXT,
- content_error TEXT,
- content_attempted_at TEXT,
- is_index_page INTEGER NOT NULL DEFAULT 0,
- url TEXT NOT NULL UNIQUE,
- normalized_title TEXT NOT NULL,
- source TEXT NOT NULL,
- pub_date TEXT,
- ingested_at TEXT NOT NULL DEFAULT (datetime('now'))
- );
-
- INSERT INTO articles_rebuild (
- id,
- title,
- description,
- content,
- image,
- content_status,
- content_error,
- content_attempted_at,
- is_index_page,
- url,
- normalized_title,
- source,
- pub_date,
- ingested_at
- )
- SELECT
- id,
- title,
- description,
- content,
- image,
- content_status,
- content_error,
- content_attempted_at,
- 0,
- url,
- normalized_title,
- source,
- pub_date,
- ingested_at
- FROM articles;
-
- DROP TABLE articles;
- ALTER TABLE articles_rebuild RENAME TO articles;
-
- COMMIT;
- `);
-}
-
-rebuildArticlesTableIfNeeded();
-
db.exec(`
CREATE INDEX IF NOT EXISTS idx_articles_source ON articles(source);
CREATE INDEX IF NOT EXISTS idx_articles_pub_date ON articles(pub_date);
CREATE INDEX IF NOT EXISTS idx_articles_ingested_at ON articles(ingested_at);
CREATE INDEX IF NOT EXISTS idx_articles_normalized_title ON articles(normalized_title);
+ CREATE INDEX IF NOT EXISTS idx_articles_event_id ON articles(event_id);
+ CREATE INDEX IF NOT EXISTS idx_articles_has_embedding ON articles(has_embedding);
+ CREATE INDEX IF NOT EXISTS idx_articles_pub_date_effective ON articles(pub_date_effective DESC);
`);
db.exec(`
@@ -132,93 +62,22 @@ db.exec(`
);
`);
-// vec0 table — fixed at 8192 dims to cover any model on openrouter, shorter embeddings get zero-padded
-{
- const existing = db.prepare(`SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'article_embeddings'`).get();
- const currentDim = existing && existing.sql && existing.sql.match(/FLOAT\[(\d+)\]/);
- const needsMigration = existing && (!currentDim || parseInt(currentDim[1], 10) !== 8192);
+db.exec(`
+ CREATE VIRTUAL TABLE IF NOT EXISTS article_embeddings USING vec0(
+ article_id INTEGER PRIMARY KEY,
+ embedding FLOAT[8192]
+ );
+`);
- if (needsMigration) {
- // save everything in vec0 to the store before dropping it, keyed by whatever model is in meta
- try {
- const BATCH = 500;
- let offset = 0;
-
- const fetchBatch = db.prepare(`
- SELECT e.article_id, m.model, e.embedding
- FROM article_embeddings e
- JOIN article_embedding_meta m ON m.article_id = e.article_id
- LIMIT ? OFFSET ?
- `);
-
- const insert = db.prepare(`
- INSERT OR IGNORE INTO article_embedding_store (article_id, model, embedding)
- VALUES (?, ?, ?)
- `);
-
- const insertMany = db.transaction((rows) => {
- for (const row of rows) insert.run(row.article_id, row.model, row.embedding);
- });
-
- while (true) {
- const rows = fetchBatch.all(BATCH, offset);
- if (rows.length === 0) break;
- insertMany(rows);
- offset += rows.length;
- if (rows.length < BATCH) break;
- }
- } catch (err) {
- console.error('failed to rescue embeddings from vec0 before migration:', err);
- }
-
- db.exec(`DROP TABLE article_embeddings`);
- db.exec(`DELETE FROM article_embedding_meta`);
- }
-
- if (!existing || needsMigration) {
- db.exec(`
- CREATE VIRTUAL TABLE article_embeddings USING vec0(
- article_id INTEGER PRIMARY KEY,
- embedding FLOAT[8192]
- );
- `);
- }
-}
-
-// migrate query_embeddings to include model in primary key
-{
- const cols = db.prepare(`PRAGMA table_info(query_embeddings)`).all();
- const hasModel = cols.some(c => c.name === 'model');
-
- if (!hasModel) {
- db.exec(`
- BEGIN;
-
- CREATE TABLE query_embeddings_new (
- query TEXT NOT NULL,
- model TEXT NOT NULL,
- embedding BLOB NOT NULL,
- created_at TEXT NOT NULL DEFAULT (datetime('now')),
- PRIMARY KEY (query, model)
- );
-
- DROP TABLE IF EXISTS query_embeddings;
- ALTER TABLE query_embeddings_new RENAME TO query_embeddings;
-
- COMMIT;
- `);
- } else {
- db.exec(`
- CREATE TABLE IF NOT EXISTS query_embeddings (
- query TEXT NOT NULL,
- model TEXT NOT NULL,
- embedding BLOB NOT NULL,
- created_at TEXT NOT NULL DEFAULT (datetime('now')),
- PRIMARY KEY (query, model)
- );
- `);
- }
-}
+db.exec(`
+ CREATE TABLE IF NOT EXISTS query_embeddings (
+ query TEXT NOT NULL,
+ model TEXT NOT NULL,
+ embedding BLOB NOT NULL,
+ created_at TEXT NOT NULL DEFAULT (datetime('now')),
+ PRIMARY KEY (query, model)
+ );
+`);
db.exec(`
CREATE TABLE IF NOT EXISTS events (
@@ -228,22 +87,6 @@ db.exec(`
);
`);
-for (const statement of [
- 'ALTER TABLE articles ADD COLUMN event_id INTEGER REFERENCES events(id)',
-]) {
- try {
- db.exec(statement);
- } catch (error) {
- if (!String(error.message).includes('duplicate column name')) {
- throw error;
- }
- }
-}
-
-db.exec(`
- CREATE INDEX IF NOT EXISTS idx_articles_event_id ON articles(event_id);
-`);
-
db.exec(`
CREATE TABLE IF NOT EXISTS gdelt_backfill_windows (
source_id TEXT NOT NULL,
@@ -287,9 +130,6 @@ db.exec(`
);
`);
-// per-domain fetch policy — caches whether plain http or browser is needed
-// so we dont waste a round trip on every article from a known js-only site.
-// expires_at lets us re-probe domains that may have recovered
db.exec(`
CREATE TABLE IF NOT EXISTS domain_fetch_policy (
domain TEXT PRIMARY KEY,
@@ -303,30 +143,4 @@ db.exec(`
);
`);
-for (const statement of [
- 'ALTER TABLE articles ADD COLUMN image TEXT',
- 'ALTER TABLE articles ADD COLUMN content_status TEXT',
- 'ALTER TABLE articles ADD COLUMN content_error TEXT',
- 'ALTER TABLE articles ADD COLUMN content_attempted_at TEXT',
- 'ALTER TABLE articles ADD COLUMN content_attempt_count INTEGER NOT NULL DEFAULT 0',
- 'ALTER TABLE articles ADD COLUMN content_retry_after TEXT',
- 'ALTER TABLE articles ADD COLUMN is_index_page INTEGER NOT NULL DEFAULT 0',
- 'ALTER TABLE articles ADD COLUMN has_embedding INTEGER NOT NULL DEFAULT 0',
- 'ALTER TABLE articles ADD COLUMN pub_date_effective TEXT',
- 'ALTER TABLE articles ADD COLUMN language TEXT'
-]) {
- try {
- db.exec(statement);
- } catch (error) {
- if (!String(error.message).includes('duplicate column name')) {
- throw error;
- }
- }
-}
-
-db.exec(`
- CREATE INDEX IF NOT EXISTS idx_articles_has_embedding ON articles(has_embedding);
- CREATE INDEX IF NOT EXISTS idx_articles_pub_date_effective ON articles(pub_date_effective DESC);
-`);
-
-module.exports = db;
+module.exports = db;
\ No newline at end of file
diff --git a/src/routes/admin.js b/src/routes/admin.js
index 9b94879..dc230b3 100644
--- a/src/routes/admin.js
+++ b/src/routes/admin.js
@@ -2,6 +2,24 @@ const fs = require('fs');
const path = require('path');
const db = require('../db');
const config = require('../config');
+const Database = require('better-sqlite3');
+
+let idb = null;
+
+function getIntelligenceDb() {
+ if (idb) return idb;
+
+ const configDir = path.resolve(__dirname, '..', '..');
+ const rawPath = process.env.INTELLIGENCE_DB
+ || (config.intelligence_db
+ ? (path.isAbsolute(config.intelligence_db) ? config.intelligence_db : path.resolve(configDir, config.intelligence_db))
+ : path.resolve(configDir, 'intelligence.sqlite'));
+
+ if (!fs.existsSync(rawPath)) return null;
+
+ idb = new Database(rawPath, { readonly: true });
+ return idb;
+}
const adminUser = (config.admin && config.admin.username) || 'admin';
const adminPass = (config.admin && config.admin.password) || 'changeme';
@@ -204,6 +222,122 @@ async function adminRoutes(fastify) {
return { ok: true };
});
+ // intelligence endpoints
+ fastify.get('/admin/api/intelligence/stats', async (request, reply) => {
+ if (!checkAuth(request, reply)) return;
+ const db = getIntelligenceDb();
+ if (!db) return { available: false };
+
+ const queue = db.prepare(`SELECT status, COUNT(*) as n FROM article_queue GROUP BY status`).all();
+ const knowledge = db.prepare(`SELECT COUNT(*) as n FROM event_knowledge`).get().n;
+ const predictions = db.prepare(`SELECT COUNT(*) as n FROM event_predictions`).get().n;
+ const companies = db.prepare(`SELECT COUNT(*) as n FROM tracked_companies`).get().n;
+ const embeddings = db.prepare(`SELECT COUNT(*) as n FROM company_embeddings`).get().n;
+
+ return { available: true, queue, knowledge, predictions, companies, embeddings };
+ });
+
+ fastify.get('/admin/api/intelligence/companies', async (request, reply) => {
+ if (!checkAuth(request, reply)) return;
+ const db = getIntelligenceDb();
+ if (!db) return [];
+ return db.prepare(`SELECT * FROM tracked_companies ORDER BY name`).all();
+ });
+
+ fastify.get('/admin/api/intelligence/knowledge', async (request, reply) => {
+ if (!checkAuth(request, reply)) return;
+ const db = getIntelligenceDb();
+ if (!db) return { total: 0, rows: [] };
+
+ const q = request.query || {};
+ const limit = Math.min(parseInt(q.limit, 10) || 50, 200);
+ const offset = parseInt(q.offset, 10) || 0;
+ const companyId = q.company_id ? parseInt(q.company_id, 10) : null;
+ const type = q.type || null;
+
+ const conditions = [];
+ const params = [];
+ if (companyId) { conditions.push('ek.company_id = ?'); params.push(companyId); }
+ if (type) { conditions.push('ek.type = ?'); params.push(type); }
+ const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : '';
+
+ const total = db.prepare(`SELECT COUNT(*) as n FROM event_knowledge ek ${where}`).get(...params).n;
+ const rows = db.prepare(`
+ SELECT ek.id, ek.event_id, ek.type, ek.data, ek.created_at,
+ tc.name as company_name
+ FROM event_knowledge ek
+ JOIN tracked_companies tc ON tc.id = ek.company_id
+ ${where}
+ ORDER BY ek.id DESC
+ LIMIT ? OFFSET ?
+ `).all(...params, limit, offset);
+
+ return { total, rows };
+ });
+
+ fastify.get('/admin/api/intelligence/predictions', async (request, reply) => {
+ if (!checkAuth(request, reply)) return;
+ const db = getIntelligenceDb();
+ if (!db) return { total: 0, rows: [] };
+
+ const q = request.query || {};
+ const limit = Math.min(parseInt(q.limit, 10) || 50, 200);
+ const offset = parseInt(q.offset, 10) || 0;
+ const companyId = q.company_id ? parseInt(q.company_id, 10) : null;
+
+ const conditions = [];
+ const params = [];
+ if (companyId) { conditions.push('ep.company_id = ?'); params.push(companyId); }
+ const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : '';
+
+ const total = db.prepare(`SELECT COUNT(*) as n FROM event_predictions ep ${where}`).get(...params).n;
+ const rows = db.prepare(`
+ SELECT ep.*, tc.name as company_name
+ FROM event_predictions ep
+ JOIN tracked_companies tc ON tc.id = ep.company_id
+ ${where}
+ ORDER BY ep.id DESC
+ LIMIT ? OFFSET ?
+ `).all(...params, limit, offset);
+
+ return { total, rows };
+ });
+
+ // raw sql console
+ fastify.post('/admin/api/sql', async (request, reply) => {
+ if (!checkAuth(request, reply)) return;
+
+ const { sql, database } = request.body || {};
+ if (!sql || !sql.trim()) { reply.code(400); return { error: 'no sql provided' }; }
+
+ const target = database === 'intelligence' ? getIntelligenceDb() : db;
+ if (!target) { reply.code(400); return { error: 'database not available' }; }
+
+ try {
+ const stmt = target.prepare(sql);
+ const start = Date.now();
+
+ let rows, changes, lastInsertRowid;
+ if (stmt.reader) {
+ rows = stmt.all();
+ } else {
+ const info = stmt.run();
+ changes = info.changes;
+ lastInsertRowid = info.lastInsertRowid;
+ }
+
+ return {
+ rows: rows || null,
+ changes: changes ?? null,
+ lastInsertRowid: lastInsertRowid ?? null,
+ elapsed: Date.now() - start,
+ };
+ } catch (err) {
+ reply.code(400);
+ return { error: err.message };
+ }
+ });
+
// stats for dashboard header
fastify.get('/admin/api/stats', async (request, reply) => {
if (!checkAuth(request, reply)) return;
diff --git a/src/routes/dev.js b/src/routes/dev.js
index f9fc54f..4076bef 100644
--- a/src/routes/dev.js
+++ b/src/routes/dev.js
@@ -1,26 +1,34 @@
const fs = require('fs');
const path = require('path');
+const os = require('os');
const config = require('../config');
+const db = require('../db');
async function devRoutes(fastify) {
if (!config.dev || !config.dev.enabled) return;
fastify.get('/dev/db/download', async (req, reply) => {
- const dbPath = path.resolve(config.duriin_db || './archive.sqlite');
+ const tmpPath = path.join(os.tmpdir(), `duriin_snapshot_${Date.now()}.sqlite`);
- if (!fs.existsSync(dbPath)) {
- return reply.code(404).send({ error: 'database file not found' });
+ try {
+ // VACUUM INTO gives us a consistent, defragmented copy with no mid-write corruption
+ db.prepare(`VACUUM INTO ?`).run(tmpPath);
+
+ const stat = fs.statSync(tmpPath);
+
+ reply.header('Content-Type', 'application/octet-stream');
+ reply.header('Content-Disposition', 'attachment; filename="archive.sqlite"');
+ reply.header('Content-Length', stat.size);
+
+ const stream = fs.createReadStream(tmpPath);
+ stream.on('close', () => fs.unlink(tmpPath, () => {}));
+
+ return reply.send(stream);
+ } catch (err) {
+ fs.unlink(tmpPath, () => {});
+ throw err;
}
-
- const stat = fs.statSync(dbPath);
- const filename = path.basename(dbPath);
-
- reply.header('Content-Type', 'application/octet-stream');
- reply.header('Content-Disposition', `attachment; filename="${filename}"`);
- reply.header('Content-Length', stat.size);
-
- return reply.send(fs.createReadStream(dbPath));
});
}