925 lines
33 KiB
JavaScript
925 lines
33 KiB
JavaScript
const fs = require('fs');
|
|
const path = require('path');
|
|
const fastifyStatic = require('@fastify/static');
|
|
const { Worker } = require('node:worker_threads');
|
|
const db = require('../db');
|
|
const config = require('../config');
|
|
const Database = require('better-sqlite3');
|
|
|
|
let idb = null;
|
|
let statsSummaryCache = null;
|
|
let statsDetailCache = null;
|
|
|
|
function calculateArchiveStats() {
|
|
const databasePath = path.resolve(__dirname, '..', '..', config.database.path || './archive.sqlite');
|
|
const workerPath = path.resolve(__dirname, '..', 'adminStatsWorker.js');
|
|
return new Promise((resolve, reject) => {
|
|
const worker = new Worker(workerPath, { workerData: { databasePath } });
|
|
const timer = setTimeout(() => {
|
|
worker.terminate();
|
|
reject(new Error('archive statistics timed out'));
|
|
}, 60_000);
|
|
worker.once('message', (message) => {
|
|
clearTimeout(timer);
|
|
if (message.error) reject(new Error(message.error));
|
|
else resolve(message.value);
|
|
});
|
|
worker.once('error', (error) => {
|
|
clearTimeout(timer);
|
|
reject(error);
|
|
});
|
|
worker.once('exit', (code) => {
|
|
if (code !== 0) {
|
|
clearTimeout(timer);
|
|
reject(new Error(`archive statistics worker exited with code ${code}`));
|
|
}
|
|
});
|
|
});
|
|
}
|
|
|
|
function getIntelligenceDb() {
|
|
if (idb) return idb;
|
|
|
|
const configDir = path.resolve(__dirname, '..', '..');
|
|
const rawPath = process.env.INTELLIGENCE_DB
|
|
|| (config.intelligence_db
|
|
? (path.isAbsolute(config.intelligence_db) ? config.intelligence_db : path.resolve(configDir, config.intelligence_db))
|
|
: path.resolve(configDir, 'intelligence.sqlite'));
|
|
|
|
if (!fs.existsSync(rawPath)) return null;
|
|
|
|
idb = new Database(rawPath);
|
|
return idb;
|
|
}
|
|
|
|
const adminUser = (config.admin && config.admin.username) || 'admin';
|
|
const adminPass = (config.admin && config.admin.password) || 'changeme';
|
|
|
|
function checkAuth(request, reply) {
|
|
const header = request.headers['authorization'] || '';
|
|
if (!header.startsWith('Basic ')) {
|
|
reply.header('WWW-Authenticate', 'Basic realm="Duriin Admin"');
|
|
reply.code(401).send('Unauthorized');
|
|
return false;
|
|
}
|
|
|
|
const decoded = Buffer.from(header.slice(6), 'base64').toString('utf8');
|
|
const colon = decoded.indexOf(':');
|
|
const user = decoded.slice(0, colon);
|
|
const pass = decoded.slice(colon + 1);
|
|
|
|
if (user !== adminUser || pass !== adminPass) {
|
|
reply.header('WWW-Authenticate', 'Basic realm="Duriin Admin"');
|
|
reply.code(401).send('Unauthorized');
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
const publicDir = path.join(__dirname, '..', '..', 'public', 'admin');
|
|
const assetsDir = path.join(publicDir, 'assets');
|
|
const pagesDir = path.join(publicDir, 'pages');
|
|
|
|
// map pretty url → page html file. keep these close to the routes so its
|
|
// obvious when a page gets added or renamed.
|
|
const pageMap = {
|
|
'/admin/autonomy': path.join(pagesDir, 'autonomy.html'),
|
|
'/admin/ingest/articles': path.join(pagesDir, 'ingest', 'articles.html'),
|
|
'/admin/ingest/events': path.join(pagesDir, 'ingest', 'events.html'),
|
|
'/admin/stats': path.join(pagesDir, 'stats.html'),
|
|
'/admin/sql': path.join(pagesDir, 'sql.html'),
|
|
'/admin/intelligence/knowledge': path.join(pagesDir, 'intelligence', 'knowledge.html'),
|
|
'/admin/intelligence/predictions': path.join(pagesDir, 'intelligence', 'predictions.html'),
|
|
'/admin/intelligence/signals': path.join(pagesDir, 'intelligence', 'signals.html'),
|
|
'/admin/intelligence/graph': path.join(pagesDir, 'intelligence', 'graph.html'),
|
|
};
|
|
|
|
function sendPage(reply, filePath) {
|
|
reply.type('text/html');
|
|
reply.header('Cache-Control', 'no-cache');
|
|
return fs.createReadStream(filePath);
|
|
}
|
|
|
|
async function adminRoutes(fastify) {
|
|
|
|
// gate every request under /admin/* behind basic auth (covers pages, api, and assets)
|
|
fastify.addHook('onRequest', async (request, reply) => {
|
|
if (!checkAuth(request, reply)) return reply;
|
|
});
|
|
|
|
// Static assets are revalidated so an operator never runs stale UI code
|
|
// against a newly deployed autonomy API.
|
|
fastify.register(fastifyStatic, {
|
|
root: assetsDir,
|
|
prefix: '/admin/assets/',
|
|
decorateReply: false,
|
|
cacheControl: false,
|
|
etag: false,
|
|
lastModified: false,
|
|
setHeaders(res) {
|
|
res.setHeader('Cache-Control', 'no-store, no-cache, must-revalidate');
|
|
},
|
|
});
|
|
|
|
// top-level entry — the autonomy control room is the primary product surface
|
|
fastify.get('/admin', async (request, reply) => {
|
|
reply.redirect('/admin/autonomy');
|
|
});
|
|
|
|
// ingest root — redirect to the articles subsection
|
|
fastify.get('/admin/ingest', async (request, reply) => {
|
|
reply.redirect('/admin/ingest/articles');
|
|
});
|
|
|
|
// intelligence root — redirect to the knowledge subsection
|
|
fastify.get('/admin/intelligence', async (request, reply) => {
|
|
reply.redirect('/admin/intelligence/knowledge');
|
|
});
|
|
|
|
// backward-compat redirects from the pre-merge urls
|
|
fastify.get('/admin/articles', async (request, reply) => reply.redirect('/admin/ingest/articles'));
|
|
fastify.get('/admin/events', async (request, reply) => reply.redirect('/admin/ingest/events'));
|
|
|
|
// wire up each pretty page path
|
|
for (const [route, filePath] of Object.entries(pageMap)) {
|
|
fastify.get(route, async (request, reply) => sendPage(reply, filePath));
|
|
}
|
|
|
|
// Autonomy control-room data. Keep this behind admin auth: it exposes model
|
|
// output, broker state and operational queue details that do not belong on the
|
|
// public status endpoint.
|
|
fastify.get('/admin/api/autonomy/overview', async (request, reply) => {
|
|
if (!checkAuth(request, reply)) return;
|
|
const intelligenceDb = getIntelligenceDb();
|
|
if (!intelligenceDb) return { enabled: false, reason: 'intelligence database unavailable' };
|
|
const hasSchema = intelligenceDb.prepare(
|
|
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='autonomy_jobs'"
|
|
).get();
|
|
if (!hasSchema) return { enabled: false, reason: 'autonomy schema is not initialized' };
|
|
|
|
const jobs = intelligenceDb.prepare(`
|
|
SELECT lane, status, COUNT(*) AS count
|
|
FROM autonomy_jobs GROUP BY lane, status ORDER BY lane, status
|
|
`).all();
|
|
const predictionCounts = intelligenceDb.prepare(`
|
|
SELECT status, COUNT(*) AS count FROM autonomy_predictions GROUP BY status
|
|
`).all();
|
|
const decisionCounts = intelligenceDb.prepare(`
|
|
SELECT action, COUNT(*) AS count FROM autonomy_decisions GROUP BY action
|
|
`).all();
|
|
const proposalCounts = intelligenceDb.prepare(`
|
|
SELECT status, COUNT(*) AS count FROM autonomy_proposals GROUP BY status
|
|
`).all();
|
|
const outcomeSummary = intelligenceDb.prepare(`
|
|
SELECT COUNT(*) AS total, SUM(direction_correct) AS correct,
|
|
AVG(excess_return) AS average_excess_return
|
|
FROM autonomy_outcomes o
|
|
JOIN autonomy_predictions p ON p.id = o.prediction_id
|
|
WHERE p.origin = 'live'
|
|
`).get();
|
|
const instruments = intelligenceDb.prepare(`
|
|
SELECT COUNT(*) AS count FROM autonomy_instruments WHERE active=1 AND tradable=1
|
|
`).get();
|
|
const latestPredictions = intelligenceDb.prepare(`
|
|
SELECT p.id, p.instrument, p.direction, p.event_type, p.causal_channel,
|
|
p.horizon_days, p.information_cutoff, p.evidence_article_ids,
|
|
p.invalidation_condition, p.learning_eligible, p.status, p.created_at,
|
|
d.action, d.calibrated_probability, d.expected_excess_return, d.rationale,
|
|
o.excess_return, o.direction_correct
|
|
FROM autonomy_predictions p
|
|
LEFT JOIN autonomy_decisions d ON d.id = (
|
|
SELECT MAX(d2.id) FROM autonomy_decisions d2 WHERE d2.prediction_id = p.id
|
|
)
|
|
LEFT JOIN autonomy_outcomes o ON o.prediction_id = p.id
|
|
ORDER BY p.id DESC LIMIT 12
|
|
`).all().map((row) => {
|
|
let evidenceCount = 0;
|
|
try { evidenceCount = JSON.parse(row.evidence_article_ids || '[]').length; } catch (_) {}
|
|
const { evidence_article_ids: ignored, ...safeRow } = row;
|
|
return { ...safeRow, evidence_count: evidenceCount };
|
|
});
|
|
const latestOrders = intelligenceDb.prepare(`
|
|
SELECT oi.id, oi.client_order_id, oi.instrument, oi.side, oi.notional,
|
|
oi.status, oi.broker_order_id, oi.attempts, oi.last_error,
|
|
oi.created_at, oi.updated_at, d.action
|
|
FROM autonomy_order_intents oi
|
|
JOIN autonomy_decisions d ON d.id = oi.decision_id
|
|
ORDER BY oi.id DESC LIMIT 12
|
|
`).all();
|
|
const account = intelligenceDb.prepare(`
|
|
SELECT broker, equity, cash, buying_power, captured_at
|
|
FROM autonomy_account_snapshots ORDER BY id DESC LIMIT 1
|
|
`).get() || null;
|
|
const calibration = intelligenceDb.prepare(`
|
|
SELECT cohort_key, sample_size, effective_sample_size, directional_probability,
|
|
expected_excess_return, lower_return, upper_return, created_at
|
|
FROM autonomy_calibration_snapshots ORDER BY id DESC LIMIT 8
|
|
`).all();
|
|
const replay = intelligenceDb.prepare(`
|
|
SELECT r.id, r.status, r.watermark_at, r.cursor_article_id, r.cursor_effective_at,
|
|
r.processed_articles, r.updated_at,
|
|
SUM(CASE WHEN p.status = 'resolved' THEN 1 ELSE 0 END) AS resolved_predictions,
|
|
COUNT(p.id) AS predictions,
|
|
SUM(o.direction_correct) AS correct_predictions,
|
|
(SELECT COUNT(*) FROM autonomy_replay_evaluations e WHERE e.replay_run_id = r.id) AS evaluations
|
|
FROM autonomy_replay_runs r
|
|
LEFT JOIN autonomy_predictions p ON p.replay_run_id = r.id
|
|
LEFT JOIN autonomy_outcomes o ON o.prediction_id = p.id
|
|
GROUP BY r.id ORDER BY r.id DESC LIMIT 1
|
|
`).get() || null;
|
|
|
|
return {
|
|
enabled: true,
|
|
mode: process.env.AUTONOMY_EXECUTION_MODE || 'shadow',
|
|
broker: {
|
|
name: 'Alpaca Paper',
|
|
configured: Boolean(process.env.ALPACA_PAPER_KEY_ID && process.env.ALPACA_PAPER_SECRET_KEY),
|
|
},
|
|
jobs,
|
|
predictionCounts,
|
|
decisionCounts,
|
|
proposalCounts,
|
|
outcomes: outcomeSummary,
|
|
allowlistedInstruments: instruments.count,
|
|
latestPredictions,
|
|
latestOrders,
|
|
account,
|
|
calibration,
|
|
replay,
|
|
generatedAt: new Date().toISOString(),
|
|
};
|
|
});
|
|
|
|
// list articles — all of them, not just the ones with embeddings
|
|
fastify.get('/admin/api/articles', async (request, reply) => {
|
|
if (!checkAuth(request, reply)) return;
|
|
|
|
const q = request.query || {};
|
|
const limit = Math.min(parseInt(q.limit, 10) || 50, 200);
|
|
const offset = parseInt(q.offset, 10) || 0;
|
|
|
|
const conditions = [];
|
|
const params = [];
|
|
|
|
if (q.keyword) {
|
|
conditions.push('(title LIKE ? OR description LIKE ? OR content LIKE ?)');
|
|
const like = `%${q.keyword}%`;
|
|
params.push(like, like, like);
|
|
}
|
|
|
|
if (q.source) {
|
|
conditions.push('source = ?');
|
|
params.push(q.source);
|
|
}
|
|
|
|
if (q.content_status) {
|
|
if (q.content_status === 'null') {
|
|
conditions.push('content_status IS NULL');
|
|
} else {
|
|
conditions.push('content_status = ?');
|
|
params.push(q.content_status);
|
|
}
|
|
}
|
|
|
|
if (q.from) {
|
|
conditions.push('ingested_at >= ?');
|
|
params.push(q.from);
|
|
}
|
|
|
|
if (q.to) {
|
|
conditions.push('ingested_at <= ?');
|
|
params.push(q.to);
|
|
}
|
|
|
|
const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : '';
|
|
|
|
const total = conditions.length
|
|
? null
|
|
: (db.prepare("SELECT seq FROM sqlite_sequence WHERE name='articles'").get()?.seq || 0);
|
|
|
|
params.push(limit + 1, offset);
|
|
const fetchedRows = db.prepare(`
|
|
SELECT id, title, url, source, pub_date, ingested_at, content_status, is_index_page, has_embedding, language
|
|
FROM articles
|
|
${where}
|
|
ORDER BY id DESC
|
|
LIMIT ? OFFSET ?
|
|
`).all(...params);
|
|
|
|
return { total, rows: fetchedRows.slice(0, limit), hasMore: fetchedRows.length > limit };
|
|
});
|
|
|
|
fastify.get('/admin/api/articles/:id', async (request, reply) => {
|
|
if (!checkAuth(request, reply)) return;
|
|
|
|
const row = db.prepare(`
|
|
SELECT id, title, description, content, url, normalized_title, source,
|
|
pub_date, pub_date_effective, ingested_at, content_status, content_error,
|
|
content_attempted_at, content_attempt_count, is_index_page, has_embedding, language, event_id
|
|
FROM articles WHERE id = ?
|
|
`).get(request.params.id);
|
|
|
|
if (!row) { reply.code(404); return { error: 'not found' }; }
|
|
return row;
|
|
});
|
|
|
|
fastify.patch('/admin/api/articles/:id', async (request, reply) => {
|
|
if (!checkAuth(request, reply)) return;
|
|
|
|
const id = parseInt(request.params.id, 10);
|
|
const body = request.body || {};
|
|
|
|
// only allow editing these fields
|
|
const allowed = ['title', 'description', 'content', 'content_status', 'is_index_page', 'language', 'pub_date'];
|
|
const updates = [];
|
|
const params = [];
|
|
|
|
for (const key of allowed) {
|
|
if (Object.prototype.hasOwnProperty.call(body, key)) {
|
|
updates.push(`${key} = ?`);
|
|
params.push(body[key]);
|
|
}
|
|
}
|
|
|
|
if (updates.length === 0) {
|
|
reply.code(400);
|
|
return { error: 'no valid fields provided' };
|
|
}
|
|
|
|
params.push(id);
|
|
db.prepare(`UPDATE articles SET ${updates.join(', ')} WHERE id = ?`).run(...params);
|
|
return { ok: true };
|
|
});
|
|
|
|
fastify.delete('/admin/api/articles/:id', async (request, reply) => {
|
|
if (!checkAuth(request, reply)) return;
|
|
|
|
const id = parseInt(request.params.id, 10);
|
|
|
|
// remove embeddings first so foreign key stuff doesnt bite us
|
|
db.prepare(`DELETE FROM article_embedding_store WHERE article_id = ?`).run(id);
|
|
db.prepare(`DELETE FROM article_embedding_meta WHERE article_id = ?`).run(id);
|
|
|
|
try {
|
|
db.prepare(`DELETE FROM article_embeddings WHERE article_id = ?`).run(id);
|
|
} catch (_) {}
|
|
|
|
db.prepare(`DELETE FROM articles WHERE id = ?`).run(id);
|
|
return { ok: true };
|
|
});
|
|
|
|
// sources list for filter dropdown
|
|
fastify.get('/admin/api/sources', async (request, reply) => {
|
|
if (!checkAuth(request, reply)) return;
|
|
|
|
const rows = db.prepare(`SELECT DISTINCT source FROM articles ORDER BY source`).all();
|
|
return rows.map(r => r.source);
|
|
});
|
|
|
|
// events — supports keyword / date range / min-article-count / sort
|
|
fastify.get('/admin/api/events', async (request, reply) => {
|
|
if (!checkAuth(request, reply)) return;
|
|
|
|
const q = request.query || {};
|
|
const limit = Math.min(parseInt(q.limit, 10) || 50, 200);
|
|
const offset = parseInt(q.offset, 10) || 0;
|
|
|
|
const where = [];
|
|
const whereParams = [];
|
|
|
|
if (q.keyword) {
|
|
where.push('e.title LIKE ?');
|
|
whereParams.push(`%${q.keyword}%`);
|
|
}
|
|
if (q.from) {
|
|
where.push('e.created_at >= ?');
|
|
whereParams.push(q.from);
|
|
}
|
|
if (q.to) {
|
|
where.push('e.created_at <= ?');
|
|
whereParams.push(q.to);
|
|
}
|
|
|
|
// having filters operate on the aggregate count
|
|
const having = [];
|
|
const havingParams = [];
|
|
if (q.min_articles) {
|
|
const n = parseInt(q.min_articles, 10);
|
|
if (!isNaN(n)) { having.push('article_count >= ?'); havingParams.push(n); }
|
|
}
|
|
|
|
// whitelist sort columns + direction so user input cant break the query
|
|
const sortMap = {
|
|
created_desc: 'e.id DESC',
|
|
created_asc: 'e.created_at ASC',
|
|
articles_desc: 'article_count DESC',
|
|
articles_asc: 'article_count ASC',
|
|
};
|
|
const orderBy = sortMap[q.sort] || sortMap.created_desc;
|
|
|
|
const whereClause = where.length ? `WHERE ${where.join(' AND ')}` : '';
|
|
const countWhereClause = having.length ? `WHERE ${having.join(' AND ')}` : '';
|
|
|
|
// Avoid grouping the entire article archive for every page visit. The
|
|
// correlated count uses idx_articles_event_id and touches only the events
|
|
// that survive filtering/pagination.
|
|
const eventProjection = `
|
|
SELECT e.id, e.title, e.created_at,
|
|
(SELECT COUNT(*) FROM articles a WHERE a.event_id = e.id) AS article_count
|
|
FROM events e ${whereClause}
|
|
`;
|
|
const filteredEvents = `SELECT * FROM (${eventProjection}) ${countWhereClause}`;
|
|
const total = where.length || having.length
|
|
? null
|
|
: (db.prepare("SELECT seq FROM sqlite_sequence WHERE name='events'").get()?.seq || 0);
|
|
const fetchedRows = db.prepare(`
|
|
${filteredEvents}
|
|
ORDER BY ${orderBy.replaceAll('e.', '')}
|
|
LIMIT ? OFFSET ?
|
|
`).all(...whereParams, ...havingParams, limit + 1, offset);
|
|
|
|
return { total, rows: fetchedRows.slice(0, limit), hasMore: fetchedRows.length > limit };
|
|
});
|
|
|
|
fastify.delete('/admin/api/events/:id', async (request, reply) => {
|
|
if (!checkAuth(request, reply)) return;
|
|
|
|
const id = parseInt(request.params.id, 10);
|
|
// detach articles from this event but dont delete the articles themselves
|
|
db.prepare(`UPDATE articles SET event_id = NULL WHERE event_id = ?`).run(id);
|
|
db.prepare(`DELETE FROM events WHERE id = ?`).run(id);
|
|
return { ok: true };
|
|
});
|
|
|
|
fastify.patch('/admin/api/events/:id', async (request, reply) => {
|
|
if (!checkAuth(request, reply)) return;
|
|
|
|
const id = parseInt(request.params.id, 10);
|
|
const body = request.body || {};
|
|
|
|
if (!body.title) { reply.code(400); return { error: 'title is required' }; }
|
|
db.prepare(`UPDATE events SET title = ? WHERE id = ?`).run(body.title, id);
|
|
return { ok: true };
|
|
});
|
|
|
|
// intelligence endpoints
|
|
fastify.get('/admin/api/intelligence/stats', async (request, reply) => {
|
|
if (!checkAuth(request, reply)) return;
|
|
const db = getIntelligenceDb();
|
|
if (!db) return { available: false };
|
|
|
|
const queue = db.prepare(`SELECT status, COUNT(*) as n FROM article_queue GROUP BY status`).all();
|
|
const icounts = db.prepare(`
|
|
SELECT
|
|
(SELECT COUNT(*) FROM event_knowledge) as knowledge,
|
|
(SELECT COUNT(*) FROM event_predictions) as predictions,
|
|
(SELECT COUNT(*) FROM tracked_companies) as companies,
|
|
(SELECT COUNT(*) FROM company_embeddings) as embeddings
|
|
`).get();
|
|
const { knowledge, predictions, companies, embeddings } = icounts;
|
|
|
|
let workerRates = [];
|
|
try {
|
|
workerRates = db.prepare(`
|
|
SELECT
|
|
worker,
|
|
SUM(CASE WHEN completed_at >= datetime('now', '-5 minutes') THEN 1 ELSE 0 END) as n5m,
|
|
SUM(CASE WHEN completed_at >= datetime('now', '-1 minute') THEN 1 ELSE 0 END) as n1m
|
|
FROM worker_events
|
|
GROUP BY worker
|
|
`).all();
|
|
} catch (_) {}
|
|
|
|
return { available: true, queue, knowledge, predictions, companies, embeddings, workerRates };
|
|
});
|
|
|
|
fastify.get('/admin/api/intelligence/companies', async (request, reply) => {
|
|
if (!checkAuth(request, reply)) return;
|
|
const db = getIntelligenceDb();
|
|
if (!db) return [];
|
|
return db.prepare(`SELECT * FROM tracked_companies ORDER BY name`).all();
|
|
});
|
|
|
|
fastify.get('/admin/api/intelligence/knowledge', async (request, reply) => {
|
|
if (!checkAuth(request, reply)) return;
|
|
const db = getIntelligenceDb();
|
|
if (!db) return { total: 0, rows: [] };
|
|
|
|
const q = request.query || {};
|
|
const limit = Math.min(parseInt(q.limit, 10) || 50, 200);
|
|
const offset = parseInt(q.offset, 10) || 0;
|
|
const companyId = q.company_id ? parseInt(q.company_id, 10) : null;
|
|
const type = q.type || null;
|
|
|
|
const conditions = [];
|
|
const params = [];
|
|
if (companyId) { conditions.push('ek.company_id = ?'); params.push(companyId); }
|
|
if (type) { conditions.push('ek.type = ?'); params.push(type); }
|
|
const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : '';
|
|
|
|
const sortCol = q.sort === 'event_date' ? 'ek.event_date' : 'ek.id';
|
|
|
|
const total = db.prepare(`SELECT COUNT(*) as n FROM event_knowledge ek ${where}`).get(...params).n;
|
|
const rows = db.prepare(`
|
|
SELECT ek.id, ek.event_id, ek.type, ek.data, ek.event_date, ek.created_at,
|
|
tc.name as company_name
|
|
FROM event_knowledge ek
|
|
JOIN tracked_companies tc ON tc.id = ek.company_id
|
|
${where}
|
|
ORDER BY ${sortCol} DESC
|
|
LIMIT ? OFFSET ?
|
|
`).all(...params, limit, offset);
|
|
|
|
return { total, rows };
|
|
});
|
|
|
|
fastify.get('/admin/api/intelligence/predictions', async (request, reply) => {
|
|
if (!checkAuth(request, reply)) return;
|
|
const db = getIntelligenceDb();
|
|
if (!db) return { total: 0, rows: [] };
|
|
|
|
const q = request.query || {};
|
|
const limit = Math.min(parseInt(q.limit, 10) || 50, 200);
|
|
const offset = parseInt(q.offset, 10) || 0;
|
|
const companyId = q.company_id ? parseInt(q.company_id, 10) : null;
|
|
|
|
const conditions = [];
|
|
const params = [];
|
|
if (companyId) { conditions.push('ep.company_id = ?'); params.push(companyId); }
|
|
const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : '';
|
|
|
|
const sortCol = q.sort === 'event_date' ? 'ep.event_date' : 'ep.id';
|
|
|
|
const total = db.prepare(`SELECT COUNT(*) as n FROM event_predictions ep ${where}`).get(...params).n;
|
|
const rows = db.prepare(`
|
|
SELECT ep.*, tc.name as company_name
|
|
FROM event_predictions ep
|
|
JOIN tracked_companies tc ON tc.id = ep.company_id
|
|
${where}
|
|
ORDER BY ${sortCol} DESC
|
|
LIMIT ? OFFSET ?
|
|
`).all(...params, limit, offset);
|
|
|
|
return { total, rows };
|
|
});
|
|
|
|
// intelligence graph — nodes + edges from company_relationships
|
|
fastify.get('/admin/api/intelligence/graph', async (request, reply) => {
|
|
if (!checkAuth(request, reply)) return;
|
|
const idb = getIntelligenceDb();
|
|
if (!idb) return { nodes: [], edges: [] };
|
|
|
|
const edges = idb.prepare(`SELECT * FROM company_relationships`).all();
|
|
|
|
const trackedIds = new Set();
|
|
for (const e of edges) {
|
|
trackedIds.add(e.from_company_id);
|
|
if (e.to_company_id) trackedIds.add(e.to_company_id);
|
|
}
|
|
|
|
const allTracked = idb.prepare(`SELECT id, name, ticker FROM tracked_companies`).all();
|
|
|
|
// build a lowercase name+alias set so we can exclude untracked entities
|
|
// that are just unresolved references to tracked companies
|
|
const trackedNameSet = new Set();
|
|
for (const c of allTracked) {
|
|
trackedNameSet.add(c.name.toLowerCase());
|
|
trackedNameSet.add(c.ticker.toLowerCase());
|
|
}
|
|
|
|
const untrackedSeen = new Set();
|
|
for (const e of edges) {
|
|
if (!e.to_company_id && e.to_entity) {
|
|
if (!trackedNameSet.has(e.to_entity.toLowerCase())) {
|
|
untrackedSeen.add(e.to_entity);
|
|
}
|
|
}
|
|
}
|
|
|
|
const nodes = [
|
|
...allTracked
|
|
.filter(c => trackedIds.has(c.id))
|
|
.map(c => ({ id: c.id, name: c.name, ticker: c.ticker, tracked: true })),
|
|
|
|
...[...untrackedSeen].map(name => ({ id: null, name, ticker: null, tracked: false })),
|
|
];
|
|
|
|
return { nodes, edges };
|
|
});
|
|
|
|
|
|
// trade signals
|
|
fastify.get('/admin/api/intelligence/signals', async (request, reply) => {
|
|
if (!checkAuth(request, reply)) return;
|
|
const db = getIntelligenceDb();
|
|
if (!db) return [];
|
|
|
|
let rows;
|
|
try {
|
|
rows = db.prepare(`
|
|
SELECT ts.*, tc.name as company_name, tc.ticker
|
|
FROM trade_signals ts
|
|
JOIN tracked_companies tc ON tc.id = ts.company_id
|
|
ORDER BY ts.generated_at DESC
|
|
`).all();
|
|
} catch (_) {
|
|
return [];
|
|
}
|
|
|
|
// collect all prediction ids across all signals in one pass, then
|
|
// fetch event_dates in a single query instead of one per signal
|
|
const allPredIds = new Set();
|
|
for (const row of rows) {
|
|
let ids = [];
|
|
try { ids = JSON.parse(row.supporting_prediction_ids || '[]'); } catch (_) {}
|
|
for (const id of ids) allPredIds.add(id);
|
|
}
|
|
|
|
const predEventDates = new Map();
|
|
if (allPredIds.size > 0) {
|
|
const ids = [...allPredIds];
|
|
const ph = ids.map(() => '?').join(',');
|
|
const predRows = db.prepare(`
|
|
SELECT id, event_date FROM event_predictions
|
|
WHERE id IN (${ph}) AND event_date IS NOT NULL
|
|
`).all(...ids);
|
|
for (const p of predRows) predEventDates.set(p.id, p.event_date);
|
|
}
|
|
|
|
for (const row of rows) {
|
|
let ids = [];
|
|
try { ids = JSON.parse(row.supporting_prediction_ids || '[]'); } catch (_) {}
|
|
let latest = null;
|
|
for (const id of ids) {
|
|
const d = predEventDates.get(id);
|
|
if (d && (!latest || d > latest)) latest = d;
|
|
}
|
|
row.latest_event_date = latest;
|
|
}
|
|
|
|
return rows;
|
|
});
|
|
|
|
fastify.delete('/admin/api/intelligence/signals/:company_id', async (request, reply) => {
|
|
if (!checkAuth(request, reply)) return;
|
|
const db = getIntelligenceDb();
|
|
if (!db) { reply.code(503); return { error: 'intelligence db unavailable' }; }
|
|
|
|
const companyId = parseInt(request.params.company_id, 10);
|
|
db.prepare('DELETE FROM trade_signals WHERE company_id = ?').run(companyId);
|
|
return { ok: true };
|
|
});
|
|
|
|
|
|
// references behind a signal — walks predictions → events → articles
|
|
// so the frontend can show the actual sources that fed the signal
|
|
fastify.get('/admin/api/intelligence/signals/:company_id/references', async (request, reply) => {
|
|
if (!checkAuth(request, reply)) return;
|
|
const idb = getIntelligenceDb();
|
|
if (!idb) return { events: [] };
|
|
|
|
const companyId = parseInt(request.params.company_id, 10);
|
|
|
|
const signal = idb.prepare(`
|
|
SELECT supporting_prediction_ids FROM trade_signals WHERE company_id = ?
|
|
`).get(companyId);
|
|
|
|
if (!signal) return { events: [] };
|
|
|
|
let predIds = [];
|
|
try { predIds = JSON.parse(signal.supporting_prediction_ids || "[]"); } catch (_) {}
|
|
if (!predIds.length) return { events: [] };
|
|
|
|
const placeholders = predIds.map(() => '?').join(',');
|
|
|
|
// pull the event_ids (and keep the newest event_date per event) from the
|
|
// predictions that fed the signal
|
|
const predRows = idb.prepare(`
|
|
SELECT event_id, MAX(event_date) as event_date
|
|
FROM event_predictions
|
|
WHERE id IN (${placeholders})
|
|
GROUP BY event_id
|
|
`).all(...predIds);
|
|
|
|
const eventMeta = new Map();
|
|
for (const p of predRows) {
|
|
if (p.event_id != null) eventMeta.set(p.event_id, p.event_date || null);
|
|
}
|
|
|
|
if (eventMeta.size === 0) return { events: [] };
|
|
|
|
const eventIds = [...eventMeta.keys()];
|
|
const eventPh = eventIds.map(() => '?').join(',');
|
|
|
|
const eventRows = db.prepare(`
|
|
SELECT id, title, created_at FROM events
|
|
WHERE id IN (${eventPh})
|
|
ORDER BY created_at DESC
|
|
`).all(...eventIds);
|
|
|
|
const artRows = db.prepare(`
|
|
SELECT id, title, source, pub_date, url, event_id
|
|
FROM articles
|
|
WHERE event_id IN (${eventPh})
|
|
ORDER BY COALESCE(pub_date, ingested_at) DESC
|
|
`).all(...eventIds);
|
|
|
|
const artsByEvent = new Map();
|
|
for (const a of artRows) {
|
|
if (!artsByEvent.has(a.event_id)) artsByEvent.set(a.event_id, []);
|
|
const list = artsByEvent.get(a.event_id);
|
|
if (list.length < 5) list.push(a);
|
|
}
|
|
|
|
const events = eventRows.map(ev => ({
|
|
...ev,
|
|
event_date: eventMeta.get(ev.id) || null,
|
|
articles: artsByEvent.get(ev.id) || [],
|
|
}));
|
|
|
|
return { events };
|
|
});
|
|
|
|
// per-company facts for the graph sidebar
|
|
fastify.get('/admin/api/intelligence/facts/:company_id', async (request, reply) => {
|
|
if (!checkAuth(request, reply)) return;
|
|
const idb = getIntelligenceDb();
|
|
if (!idb) return [];
|
|
|
|
const companyId = parseInt(request.params.company_id, 10);
|
|
return idb.prepare(`
|
|
SELECT type, claim, confidence, confirmation_count
|
|
FROM company_facts
|
|
WHERE company_id = ?
|
|
ORDER BY confirmation_count DESC
|
|
LIMIT 10
|
|
`).all(companyId);
|
|
});
|
|
|
|
|
|
// evidence backing a single graph edge — the relationship row itself,
|
|
// the consolidated facts that produced it, and the source events+articles.
|
|
fastify.get('/admin/api/intelligence/edge-evidence', async (request, reply) => {
|
|
if (!checkAuth(request, reply)) return;
|
|
const idb = getIntelligenceDb();
|
|
if (!idb) return { edge: null, facts: [], events: [] };
|
|
|
|
const fromId = parseInt(request.query.from_id, 10);
|
|
const toId = parseInt(request.query.to_id, 10);
|
|
const type = (request.query.type || '').toLowerCase();
|
|
|
|
if (!fromId || !toId || !type) {
|
|
reply.code(400);
|
|
return { error: 'missing from_id / to_id / type' };
|
|
}
|
|
|
|
// direct edge — try the exact match first
|
|
let edge = idb.prepare(`
|
|
SELECT * FROM company_relationships
|
|
WHERE from_company_id = ? AND to_company_id = ? AND relationship_type = ?
|
|
`).get(fromId, toId, type);
|
|
|
|
// investor edges are stored as 'dependency' in the reverse direction too —
|
|
// the frontend normalizes dependency→investor, so lookup the flipped row
|
|
if (!edge && type === 'investor') {
|
|
edge = idb.prepare(`
|
|
SELECT * FROM company_relationships
|
|
WHERE from_company_id = ? AND to_company_id = ? AND relationship_type = 'dependency'
|
|
`).get(toId, fromId);
|
|
}
|
|
|
|
const toCompany = idb.prepare(`SELECT name FROM tracked_companies WHERE id = ?`).get(toId);
|
|
const toName = toCompany ? toCompany.name : null;
|
|
|
|
// backing facts — relationship-type facts on the source company that mention the target
|
|
let facts = [];
|
|
if (toName) {
|
|
facts = idb.prepare(`
|
|
SELECT id, claim, confidence, confirmation_count, supporting_event_ids, first_seen_at, last_seen_at
|
|
FROM company_facts
|
|
WHERE company_id = ? AND type = 'relationship' AND claim LIKE ?
|
|
ORDER BY confirmation_count DESC
|
|
LIMIT 20
|
|
`).all(fromId, `%${toName}%`);
|
|
}
|
|
|
|
// merge event ids from the edge row + all backing facts
|
|
const eventIds = new Set();
|
|
if (edge && edge.supporting_event_ids) {
|
|
try { JSON.parse(edge.supporting_event_ids).forEach(id => eventIds.add(id)); } catch (_) {}
|
|
}
|
|
for (const f of facts) {
|
|
try { JSON.parse(f.supporting_event_ids || '[]').forEach(id => eventIds.add(id)); } catch (_) {}
|
|
}
|
|
|
|
// resolve events + articles from archive.sqlite
|
|
const events = [];
|
|
if (eventIds.size > 0) {
|
|
const ids = [...eventIds];
|
|
const placeholders = ids.map(() => '?').join(',');
|
|
const eventRows = db.prepare(`
|
|
SELECT id, title, created_at FROM events
|
|
WHERE id IN (${placeholders})
|
|
ORDER BY created_at DESC
|
|
`).all(...ids);
|
|
|
|
const edgeArtRows = db.prepare(`
|
|
SELECT id, title, source, pub_date, url, event_id
|
|
FROM articles
|
|
WHERE event_id IN (${placeholders})
|
|
ORDER BY COALESCE(pub_date, ingested_at) DESC
|
|
`).all(...ids);
|
|
|
|
const edgeArtsByEvent = new Map();
|
|
for (const a of edgeArtRows) {
|
|
if (!edgeArtsByEvent.has(a.event_id)) edgeArtsByEvent.set(a.event_id, []);
|
|
const list = edgeArtsByEvent.get(a.event_id);
|
|
if (list.length < 8) list.push(a);
|
|
}
|
|
|
|
for (const ev of eventRows) {
|
|
events.push({ ...ev, articles: edgeArtsByEvent.get(ev.id) || [] });
|
|
}
|
|
}
|
|
|
|
return { edge, facts, events };
|
|
});
|
|
|
|
|
|
// raw sql console — supports multiple statements separated by ;
|
|
fastify.post('/admin/api/sql', async (request, reply) => {
|
|
if (!checkAuth(request, reply)) return;
|
|
|
|
const { sql, database } = request.body || {};
|
|
if (!sql || !sql.trim()) { reply.code(400); return { error: 'no sql provided' }; }
|
|
|
|
const target = database === 'intelligence' ? getIntelligenceDb() : db;
|
|
if (!target) { reply.code(400); return { error: 'database not available' }; }
|
|
|
|
// split on semicolons, drop empty statements
|
|
const statements = sql.split(';').map(s => s.trim()).filter(s => s.length > 0);
|
|
|
|
const results = [];
|
|
const start = Date.now();
|
|
|
|
for (const s of statements) {
|
|
try {
|
|
const stmt = target.prepare(s);
|
|
if (stmt.reader) {
|
|
results.push({ sql: s, rows: stmt.all() });
|
|
} else {
|
|
const info = stmt.run();
|
|
results.push({ sql: s, changes: info.changes, lastInsertRowid: info.lastInsertRowid });
|
|
}
|
|
} catch (err) {
|
|
results.push({ sql: s, error: err.message });
|
|
}
|
|
}
|
|
|
|
return { results, elapsed: Date.now() - start };
|
|
});
|
|
|
|
// Lightweight header summary, cached independently from the detailed stats
|
|
// page so navigation never waits for source/status aggregation.
|
|
fastify.get('/admin/api/stats/summary', async (request, reply) => {
|
|
if (!checkAuth(request, reply)) return;
|
|
if (statsSummaryCache && Date.now() - statsSummaryCache.at < 60_000) {
|
|
return statsSummaryCache.value;
|
|
}
|
|
const counts = db.prepare(`
|
|
SELECT
|
|
COALESCE((SELECT seq FROM sqlite_sequence WHERE name='articles'), 0) AS total,
|
|
(SELECT COUNT(*) FROM article_embedding_meta) AS withContent,
|
|
(SELECT COUNT(*) FROM article_embedding_meta) AS withEmbedding,
|
|
COALESCE((SELECT seq FROM sqlite_sequence WHERE name='events'), 0) AS eventCount
|
|
`).get();
|
|
statsSummaryCache = { at: Date.now(), value: counts };
|
|
return counts;
|
|
});
|
|
|
|
// Detailed statistics are cached because they summarize the full archive and
|
|
// do not need second-by-second precision.
|
|
fastify.get('/admin/api/stats', async (request, reply) => {
|
|
if (!checkAuth(request, reply)) return;
|
|
if (statsDetailCache && Date.now() - statsDetailCache.at < 5 * 60_000) {
|
|
return statsDetailCache.value;
|
|
}
|
|
|
|
const value = await calculateArchiveStats();
|
|
statsDetailCache = { at: Date.now(), value };
|
|
statsSummaryCache = {
|
|
at: Date.now(),
|
|
value: {
|
|
total: value.total,
|
|
withContent: value.withContent,
|
|
withEmbedding: value.withEmbedding,
|
|
eventCount: value.eventCount,
|
|
},
|
|
};
|
|
return value;
|
|
});
|
|
}
|
|
|
|
module.exports = adminRoutes;
|