fix: restart the stalled autonomy pipeline and make calibration honest
Archive ingestion had been dead since 2026-08-02 because nothing in the compose stack actually ran it. Everything downstream starved from there. - add ingest + enrichment services. server.js only starts the scheduler when DURIIN_RUN_SCHEDULER is not "false", and workers/index.js was not running at all, so articles never got event_id/content/has_embedding and the coordinator had nothing to lease. - pass an explicit origin from coordinatorWorker. it was never passed, so acceptProposal defaulted to 'live' and 464 historical backfill predictions were recorded as live. that also meant verifyEvidence got a null cutoff and skipped its date check entirely. - coarsen cohortKey to event families + horizon buckets. 201 free text event types produced 221 cohorts averaging 2.76 samples, so the n>=30 gate could never be reached and everything abstained for the wrong reason. - gate on cohort diversity, not just sample count. one ticker was roughly half of all resolved outcomes, so a pure count gate was measuring one company. unknown diversity abstains rather than passing. - resolve the admin archive db explicitly and probe it. it relied on a Dockerfile symlink, and without it better-sqlite3 quietly creates an empty file and serves a phantom archive. - clamp implausible future publication dates at ingest. - pin the db backend to sqlite by default. compose hardcoded postgres "true", which would have overridden the operator's own .env on the next redeploy and pointed everything at a stale snapshot. scripts/repair-autonomy-labels.js relabels the affected rows. it is dry run by default and has not been applied. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WnNxwxfXSbeNtjvtz5gayb
This commit is contained in:
ImBenji
+243
-28
@@ -9,11 +9,74 @@ const { openRuntimeDb, isPostgresEnabled } = require('../db/runtime');
|
||||
const pg = require('../db/pgAsync');
|
||||
|
||||
let idb = null;
|
||||
let adb = null;
|
||||
let statsSummaryCache = null;
|
||||
let statsDetailCache = null;
|
||||
|
||||
const configDir = path.resolve(__dirname, '..', '..');
|
||||
|
||||
// The archive is resolved exactly like the workers do it (workers/index.js:31):
|
||||
// DURIIN_DB wins, then config, and only then the repo relative default. The old
|
||||
// code here went straight to config.database.path — a repo relative
|
||||
// "./archive.sqlite" — which inside the container only ever pointed at the real
|
||||
// data because of a build time symlink, and which quietly opens a brand new
|
||||
// empty database when that symlink is not there.
|
||||
function resolveArchivePath() {
|
||||
const raw = process.env.DURIIN_DB
|
||||
|| config.duriin_db
|
||||
|| (config.database && config.database.path)
|
||||
|| './archive.sqlite';
|
||||
|
||||
return path.isAbsolute(raw) ? raw : path.resolve(configDir, raw);
|
||||
}
|
||||
|
||||
function resolveIntelligencePath() {
|
||||
return process.env.INTELLIGENCE_DB
|
||||
|| (config.intelligence_db
|
||||
? (path.isAbsolute(config.intelligence_db) ? config.intelligence_db : path.resolve(configDir, config.intelligence_db))
|
||||
: path.resolve(configDir, 'intelligence.sqlite'));
|
||||
}
|
||||
|
||||
// Opens the archive and *proves* it is the archive before handing it back. Any
|
||||
// failure throws with the resolved target in the message — serving the wrong
|
||||
// database silently is far worse than an error on the sql console.
|
||||
function getArchiveDb() {
|
||||
if (adb) return adb;
|
||||
|
||||
const target = isPostgresEnabled() ? 'postgres schema "archive"' : resolveArchivePath();
|
||||
|
||||
try {
|
||||
if (isPostgresEnabled()) {
|
||||
const handle = openRuntimeDb(resolveArchivePath(), { schema: 'archive' });
|
||||
const probe = handle.prepare("SELECT to_regclass('archive.articles') AS relation").get();
|
||||
if (!probe || !probe.relation) throw new Error('the archive schema has no articles table');
|
||||
adb = handle;
|
||||
return adb;
|
||||
}
|
||||
|
||||
const filePath = resolveArchivePath();
|
||||
if (!fs.existsSync(filePath)) throw new Error('no such file');
|
||||
|
||||
const handle = new Database(filePath, { fileMustExist: true });
|
||||
try {
|
||||
const probe = handle.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='articles'").get();
|
||||
if (!probe) throw new Error('this file has no articles table, so it is not the archive');
|
||||
} catch (probeError) {
|
||||
handle.close();
|
||||
throw probeError;
|
||||
}
|
||||
|
||||
adb = handle;
|
||||
return adb;
|
||||
} catch (error) {
|
||||
// never cached — if the volume shows up later the next request recovers
|
||||
console.error(`[admin] archive database unavailable (${target}):`, error);
|
||||
throw new Error(`archive database unavailable (${target}): ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
function calculateArchiveStats() {
|
||||
const databasePath = path.resolve(__dirname, '..', '..', config.database.path || './archive.sqlite');
|
||||
const databasePath = resolveArchivePath();
|
||||
const workerPath = path.resolve(__dirname, '..', 'adminStatsWorker.js');
|
||||
return new Promise((resolve, reject) => {
|
||||
const worker = new Worker(workerPath, { workerData: { databasePath } });
|
||||
@@ -42,18 +105,134 @@ function calculateArchiveStats() {
|
||||
function getIntelligenceDb() {
|
||||
if (idb) return idb;
|
||||
|
||||
const configDir = path.resolve(__dirname, '..', '..');
|
||||
const rawPath = process.env.INTELLIGENCE_DB
|
||||
|| (config.intelligence_db
|
||||
? (path.isAbsolute(config.intelligence_db) ? config.intelligence_db : path.resolve(configDir, config.intelligence_db))
|
||||
: path.resolve(configDir, 'intelligence.sqlite'));
|
||||
const rawPath = resolveIntelligencePath();
|
||||
|
||||
if (!isPostgresEnabled() && !fs.existsSync(rawPath)) return null;
|
||||
if (!isPostgresEnabled() && !fs.existsSync(rawPath)) {
|
||||
console.error(`[admin] intelligence database unavailable: no such file (${rawPath})`);
|
||||
return null;
|
||||
}
|
||||
|
||||
idb = isPostgresEnabled() ? openRuntimeDb(rawPath, { schema: 'intelligence' }) : new Database(rawPath);
|
||||
return idb;
|
||||
}
|
||||
|
||||
// Prediction origins are not interchangeable. 'live' is genuine real time work,
|
||||
// 'historical' is coordinator backfill over the archive and 'replay' is
|
||||
// walk-forward replay. Averaging them into a single accuracy number reads like
|
||||
// live edge when it is nothing of the sort, so the overview reports them side by
|
||||
// side and lets the page say "nothing live yet" out loud.
|
||||
const OUTCOME_ORIGINS = ['live', 'historical', 'replay'];
|
||||
|
||||
const OUTCOMES_BY_ORIGIN_SQL = `
|
||||
SELECT p.origin AS origin,
|
||||
COUNT(*) AS total,
|
||||
SUM(o.direction_correct) AS correct,
|
||||
AVG(o.excess_return) AS average_excess_return
|
||||
FROM autonomy_outcomes o
|
||||
JOIN autonomy_predictions p ON p.id = o.prediction_id
|
||||
GROUP BY p.origin
|
||||
`;
|
||||
|
||||
const PREDICTIONS_BY_ORIGIN_SQL = `
|
||||
SELECT origin, status, COUNT(*) AS count
|
||||
FROM autonomy_predictions
|
||||
GROUP BY origin, status
|
||||
`;
|
||||
|
||||
function summarizeOutcomeOrigins(rows) {
|
||||
const buckets = new Map();
|
||||
for (const name of OUTCOME_ORIGINS) {
|
||||
buckets.set(name, { origin: name, total: 0, correct: 0, average_excess_return: null });
|
||||
}
|
||||
|
||||
for (const row of rows || []) {
|
||||
const origin = String(row.origin || 'unknown').toLowerCase();
|
||||
if (!buckets.has(origin)) buckets.set(origin, { origin, total: 0, correct: 0, average_excess_return: null });
|
||||
|
||||
const bucket = buckets.get(origin);
|
||||
bucket.total = Number(row.total || 0);
|
||||
bucket.correct = Number(row.correct || 0);
|
||||
bucket.average_excess_return = row.average_excess_return == null ? null : Number(row.average_excess_return);
|
||||
}
|
||||
|
||||
const byOrigin = [...buckets.values()];
|
||||
const live = byOrigin.find((bucket) => bucket.origin === 'live');
|
||||
return { byOrigin, live };
|
||||
}
|
||||
|
||||
// Diversification gate, mirrored from the policy layer. A cohort only earns the
|
||||
// right to authorise a trade when it is big enough, spread over enough tickers
|
||||
// and not dominated by a single one. Missing diversity data counts as a fail —
|
||||
// the policy treats unknown as disqualifying and the admin view has to agree,
|
||||
// otherwise the screen says "qualified" while the trader abstains.
|
||||
const CALIBRATION_MIN_SAMPLES = 30;
|
||||
const CALIBRATION_MIN_INSTRUMENTS = 5;
|
||||
const CALIBRATION_MAX_CONCENTRATION = 0.5;
|
||||
|
||||
const CALIBRATION_BASE_COLUMNS = [
|
||||
'cohort_key', 'sample_size', 'effective_sample_size', 'directional_probability',
|
||||
'expected_excess_return', 'lower_return', 'upper_return', 'created_at',
|
||||
];
|
||||
// added by the diversification work; pre-existing rows/deployments may not have
|
||||
// them yet so they are selected only when they really exist
|
||||
const CALIBRATION_OPTIONAL_COLUMNS = ['distinct_instruments', 'top_instrument_share', 'source'];
|
||||
|
||||
function calibrationSnapshotSql(available) {
|
||||
const columns = CALIBRATION_BASE_COLUMNS.slice();
|
||||
for (const name of CALIBRATION_OPTIONAL_COLUMNS) {
|
||||
columns.push(available.has(name) ? name : `NULL AS ${name}`);
|
||||
}
|
||||
return `SELECT ${columns.join(', ')} FROM autonomy_calibration_snapshots ORDER BY id DESC LIMIT 8`;
|
||||
}
|
||||
|
||||
function gate(value, ok, threshold) {
|
||||
return { value: value == null ? null : Number(value), threshold, ok, known: value != null };
|
||||
}
|
||||
|
||||
function decorateCalibration(rows) {
|
||||
return (rows || []).map((row) => {
|
||||
const samples = row.sample_size == null ? null : Number(row.sample_size);
|
||||
const instruments = row.distinct_instruments == null ? null : Number(row.distinct_instruments);
|
||||
const share = row.top_instrument_share == null ? null : Number(row.top_instrument_share);
|
||||
|
||||
const checks = {
|
||||
sample_size: gate(samples, samples != null && samples >= CALIBRATION_MIN_SAMPLES, CALIBRATION_MIN_SAMPLES),
|
||||
distinct_instruments: gate(instruments, instruments != null && instruments >= CALIBRATION_MIN_INSTRUMENTS, CALIBRATION_MIN_INSTRUMENTS),
|
||||
top_instrument_share: gate(share, share != null && share <= CALIBRATION_MAX_CONCENTRATION, CALIBRATION_MAX_CONCENTRATION),
|
||||
};
|
||||
|
||||
const reasons = [];
|
||||
if (!checks.sample_size.ok) {
|
||||
reasons.push(samples == null ? 'sample size unknown' : `only ${samples} samples, needs ${CALIBRATION_MIN_SAMPLES}`);
|
||||
}
|
||||
if (!checks.distinct_instruments.ok) {
|
||||
reasons.push(instruments == null ? 'instrument spread unknown' : `only ${instruments} distinct ticker${instruments === 1 ? '' : 's'}, needs ${CALIBRATION_MIN_INSTRUMENTS}`);
|
||||
}
|
||||
if (!checks.top_instrument_share.ok) {
|
||||
reasons.push(share == null ? 'concentration unknown' : `${Math.round(share * 100)}% sits in one ticker, cap is ${Math.round(CALIBRATION_MAX_CONCENTRATION * 100)}%`);
|
||||
}
|
||||
|
||||
// cohort keys are versioned now. legacy rows use the old key shape and will
|
||||
// never match a current lookup, so they must not read as live calibration.
|
||||
const legacy = !String(row.cohort_key || '').startsWith('v2|');
|
||||
|
||||
return {
|
||||
...row,
|
||||
source: row.source || null,
|
||||
legacy_cohort_key: legacy,
|
||||
qualification: { qualified: reasons.length === 0, checks, reasons },
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeOriginCounts(rows) {
|
||||
return (rows || []).map((row) => ({
|
||||
origin: String(row.origin || 'unknown').toLowerCase(),
|
||||
status: row.status,
|
||||
count: Number(row.count || 0),
|
||||
}));
|
||||
}
|
||||
|
||||
const adminUser = (config.admin && config.admin.username) || 'admin';
|
||||
const adminPass = (config.admin && config.admin.password) || 'changeme';
|
||||
|
||||
@@ -156,12 +335,18 @@ async function adminRoutes(fastify) {
|
||||
if (isPostgresEnabled()) {
|
||||
const hasSchema = await pg.get('intelligence', "SELECT 1 FROM information_schema.tables WHERE table_schema = $1 AND table_name = $2", ['intelligence', 'autonomy_jobs']);
|
||||
if (!hasSchema) return { enabled: false, reason: 'autonomy schema is not initialized' };
|
||||
const [jobs, predictionCounts, decisionCounts, proposalCounts, outcomeSummary, instruments, latestRows, latestOrders, account, calibration, replay] = await Promise.all([
|
||||
|
||||
const calibrationColumns = new Set((await pg.all('intelligence',
|
||||
'SELECT column_name FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2',
|
||||
['intelligence', 'autonomy_calibration_snapshots'])).map((row) => row.column_name));
|
||||
|
||||
const [jobs, predictionCounts, predictionOriginRows, decisionCounts, proposalCounts, outcomeRows, instruments, latestRows, latestOrders, account, calibration, replay] = await Promise.all([
|
||||
pg.all('intelligence', 'SELECT lane, status, COUNT(*) AS count FROM autonomy_jobs GROUP BY lane, status ORDER BY lane, status'),
|
||||
pg.all('intelligence', 'SELECT status, COUNT(*) AS count FROM autonomy_predictions GROUP BY status'),
|
||||
pg.all('intelligence', PREDICTIONS_BY_ORIGIN_SQL),
|
||||
pg.all('intelligence', 'SELECT action, COUNT(*) AS count FROM autonomy_decisions GROUP BY action'),
|
||||
pg.all('intelligence', 'SELECT status, COUNT(*) AS count FROM autonomy_proposals GROUP BY status'),
|
||||
pg.get('intelligence', "SELECT COUNT(*) AS total, SUM(direction_correct) AS correct, AVG(excess_return) AS average_excess_return FROM autonomy_outcomes o JOIN autonomy_predictions p ON p.id = o.prediction_id WHERE p.origin = 'live'"),
|
||||
pg.all('intelligence', OUTCOMES_BY_ORIGIN_SQL),
|
||||
pg.get('intelligence', 'SELECT COUNT(*) AS count FROM autonomy_instruments WHERE active=1 AND tradable=1'),
|
||||
pg.all('intelligence', `
|
||||
SELECT p.id, p.instrument, p.direction, p.event_type, p.causal_channel,
|
||||
@@ -185,7 +370,7 @@ async function adminRoutes(fastify) {
|
||||
ORDER BY oi.id DESC LIMIT 12
|
||||
`),
|
||||
pg.get('intelligence', 'SELECT broker, equity, cash, buying_power, captured_at FROM autonomy_account_snapshots ORDER BY id DESC LIMIT 1'),
|
||||
pg.all('intelligence', 'SELECT cohort_key, sample_size, effective_sample_size, directional_probability, expected_excess_return, lower_return, upper_return, created_at FROM autonomy_calibration_snapshots ORDER BY id DESC LIMIT 8'),
|
||||
pg.all('intelligence', calibrationSnapshotSql(calibrationColumns)),
|
||||
pg.get('intelligence', `
|
||||
SELECT r.id, r.status, r.watermark_at, r.cursor_article_id, r.cursor_effective_at,
|
||||
r.processed_articles, r.updated_at,
|
||||
@@ -205,13 +390,18 @@ async function adminRoutes(fastify) {
|
||||
const { evidence_article_ids: ignored, ...safeRow } = row;
|
||||
return { ...safeRow, evidence_count: evidenceCount };
|
||||
});
|
||||
const origins = summarizeOutcomeOrigins(outcomeRows);
|
||||
return {
|
||||
enabled: true,
|
||||
mode: process.env.AUTONOMY_EXECUTION_MODE || 'shadow',
|
||||
broker: { name: 'Alpaca Paper', configured: Boolean(process.env.ALPACA_PAPER_KEY_ID && process.env.ALPACA_PAPER_SECRET_KEY) },
|
||||
jobs, predictionCounts, decisionCounts, proposalCounts, outcomes: outcomeSummary,
|
||||
jobs, predictionCounts, decisionCounts, proposalCounts,
|
||||
predictionsByOrigin: normalizeOriginCounts(predictionOriginRows),
|
||||
outcomes: origins.live,
|
||||
outcomesByOrigin: origins.byOrigin,
|
||||
hasLiveOutcomes: origins.live.total > 0,
|
||||
allowlistedInstruments: instruments.count, latestPredictions, latestOrders,
|
||||
account: account || null, calibration, replay: replay || null,
|
||||
account: account || null, calibration: decorateCalibration(calibration), replay: replay || null,
|
||||
generatedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
@@ -235,13 +425,8 @@ async function adminRoutes(fastify) {
|
||||
const proposalCounts = intelligenceDb.prepare(`
|
||||
SELECT status, COUNT(*) AS count FROM autonomy_proposals GROUP BY status
|
||||
`).all();
|
||||
const outcomeSummary = intelligenceDb.prepare(`
|
||||
SELECT COUNT(*) AS total, SUM(direction_correct) AS correct,
|
||||
AVG(excess_return) AS average_excess_return
|
||||
FROM autonomy_outcomes o
|
||||
JOIN autonomy_predictions p ON p.id = o.prediction_id
|
||||
WHERE p.origin = 'live'
|
||||
`).get();
|
||||
const predictionOriginRows = intelligenceDb.prepare(PREDICTIONS_BY_ORIGIN_SQL).all();
|
||||
const origins = summarizeOutcomeOrigins(intelligenceDb.prepare(OUTCOMES_BY_ORIGIN_SQL).all());
|
||||
const instruments = intelligenceDb.prepare(`
|
||||
SELECT COUNT(*) AS count FROM autonomy_instruments WHERE active=1 AND tradable=1
|
||||
`).get();
|
||||
@@ -275,11 +460,12 @@ async function adminRoutes(fastify) {
|
||||
SELECT broker, equity, cash, buying_power, captured_at
|
||||
FROM autonomy_account_snapshots ORDER BY id DESC LIMIT 1
|
||||
`).get() || null;
|
||||
const calibration = intelligenceDb.prepare(`
|
||||
SELECT cohort_key, sample_size, effective_sample_size, directional_probability,
|
||||
expected_excess_return, lower_return, upper_return, created_at
|
||||
FROM autonomy_calibration_snapshots ORDER BY id DESC LIMIT 8
|
||||
`).all();
|
||||
const calibrationColumns = new Set(
|
||||
intelligenceDb.prepare('PRAGMA table_info(autonomy_calibration_snapshots)').all().map((row) => row.name)
|
||||
);
|
||||
const calibration = decorateCalibration(
|
||||
intelligenceDb.prepare(calibrationSnapshotSql(calibrationColumns)).all()
|
||||
);
|
||||
const replay = intelligenceDb.prepare(`
|
||||
SELECT r.id, r.status, r.watermark_at, r.cursor_article_id, r.cursor_effective_at,
|
||||
r.processed_articles, r.updated_at,
|
||||
@@ -302,9 +488,12 @@ async function adminRoutes(fastify) {
|
||||
},
|
||||
jobs,
|
||||
predictionCounts,
|
||||
predictionsByOrigin: normalizeOriginCounts(predictionOriginRows),
|
||||
decisionCounts,
|
||||
proposalCounts,
|
||||
outcomes: outcomeSummary,
|
||||
outcomes: origins.live,
|
||||
outcomesByOrigin: origins.byOrigin,
|
||||
hasLiveOutcomes: origins.live.total > 0,
|
||||
allowlistedInstruments: instruments.count,
|
||||
latestPredictions,
|
||||
latestOrders,
|
||||
@@ -918,8 +1107,29 @@ async function adminRoutes(fastify) {
|
||||
const { sql, database } = request.body || {};
|
||||
if (!sql || !sql.trim()) { reply.code(400); return { error: 'no sql provided' }; }
|
||||
|
||||
const target = database === 'intelligence' ? getIntelligenceDb() : db;
|
||||
if (!target) { reply.code(400); return { error: 'database not available' }; }
|
||||
// empty/omitted means archive, the historic default. anything else has to be
|
||||
// spelled correctly — a typo used to silently run against the archive.
|
||||
const requested = String(database || 'archive').trim().toLowerCase() || 'archive';
|
||||
if (requested !== 'archive' && requested !== 'intelligence') {
|
||||
reply.code(400);
|
||||
return { error: `unknown database "${requested}" — expected "archive" or "intelligence"` };
|
||||
}
|
||||
|
||||
let target = null;
|
||||
try {
|
||||
target = requested === 'intelligence' ? getIntelligenceDb() : getArchiveDb();
|
||||
} catch (error) {
|
||||
console.error(`[admin] sql console cannot reach the ${requested} database:`, error);
|
||||
reply.code(503);
|
||||
return { error: error.message };
|
||||
}
|
||||
|
||||
if (!target) {
|
||||
const where = isPostgresEnabled() ? `postgres schema "${requested}"` : resolveIntelligencePath();
|
||||
console.error(`[admin] sql console cannot reach the ${requested} database (${where})`);
|
||||
reply.code(503);
|
||||
return { error: `${requested} database unavailable (${where})` };
|
||||
}
|
||||
|
||||
// split on semicolons, drop empty statements
|
||||
const statements = sql.split(';').map(s => s.trim()).filter(s => s.length > 0);
|
||||
@@ -930,13 +1140,18 @@ async function adminRoutes(fastify) {
|
||||
for (const s of statements) {
|
||||
try {
|
||||
const stmt = target.prepare(s);
|
||||
if (stmt.reader) {
|
||||
// the postgres adapters dont expose better-sqlite3's `reader` flag, so
|
||||
// without this fallback every SELECT went down the run() path and came
|
||||
// back as a change count with no rows at all
|
||||
const reads = typeof stmt.reader === 'boolean' ? stmt.reader : /^\s*(SELECT|WITH|PRAGMA|EXPLAIN|SHOW)\b/i.test(s);
|
||||
if (reads) {
|
||||
results.push({ sql: s, rows: stmt.all() });
|
||||
} else {
|
||||
const info = stmt.run();
|
||||
results.push({ sql: s, changes: info.changes, lastInsertRowid: info.lastInsertRowid });
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`[admin] sql console statement failed on ${requested}:`, s, err);
|
||||
results.push({ sql: s, error: err.message });
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user