diff --git a/scripts/repair-unresolvable-outcomes.js b/scripts/repair-unresolvable-outcomes.js new file mode 100644 index 0000000..49ae724 --- /dev/null +++ b/scripts/repair-unresolvable-outcomes.js @@ -0,0 +1,207 @@ +#!/usr/bin/env node +/* + * Re-score predictions that were labelled unresolvable when they were nothing of + * the kind. + * + * The outcome worker held one price series per symbol in a map that never + * expired, and on the null path it only ever dropped the instrument, never the + * benchmark. So a single degraded-but-200 SPY fetch poisoned the benchmark for + * the whole life of the worker, calculateOutcome returned null for every symbol + * after that, and each matured prediction got retired as "no market data" once + * its grace ran out. Outcomes stopped on 2026-09-20 and ~150 rows were mislabeled + * while the prices needed to score them were sitting on yahoo the entire time. + * + * Nothing was lost, only mislabeled, so this just asks each one again with fresh + * data. A row that genuinely cannot be scored is left exactly as it is. + * + * Writes through openRuntimeDb so it honours DURIIN_DB_BACKEND rather than + * assuming sqlite, and it reports row counts either side of the change. + * + * node scripts/repair-unresolvable-outcomes.js --dry-run + * node scripts/repair-unresolvable-outcomes.js --origin live --dry-run + * node scripts/repair-unresolvable-outcomes.js --commit + */ +const https = require("https"); +const { openRuntimeDb } = require("../src/db/runtime"); +const { initAutonomySchema } = require("../src/autonomy/schema"); +const { calculateOutcome, addTradingDays, yahooSymbol } = require("../src/autonomy/outcomes"); + +const INTELLIGENCE = process.env.INTELLIGENCE_DB || "/data/intelligence.sqlite"; +const MIN_HISTORY_BARS = 50; +const BENCHMARK = "SPY"; + +function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } + +function options() { + const argv = process.argv.slice(2); + const out = {}; + for (let i = 0; i < argv.length; i += 1) { + if (!argv[i].startsWith("--")) continue; + const key = argv[i].slice(2); + const next = argv[i + 1]; + out[key] = (next && !next.startsWith("--")) ? (i += 1, next) : true; + } + return out; +} + +function httpGet(url) { + return new Promise((resolve, reject) => { + const request = https.get(url, { headers: { "User-Agent": "duriin-autonomy/1.0" } }, (response) => { + let body = ""; + response.setEncoding("utf8"); + response.on("data", (chunk) => { body += chunk; }); + response.on("end", () => response.statusCode >= 200 && response.statusCode < 300 + ? resolve(body) : reject(new Error(`market data returned ${response.statusCode}`))); + }); + request.setTimeout(15000, () => request.destroy(new Error("market data timeout"))); + request.on("error", reject); + }); +} + +async function history(symbol) { + const url = `https://query1.finance.yahoo.com/v8/finance/chart/${encodeURIComponent(yahooSymbol(symbol))}` + + `?period1=946684800&period2=${Math.floor(Date.now() / 1000)}&interval=1d`; + const body = JSON.parse(await httpGet(url)); + const result = body?.chart?.result?.[0]; + if (!result) return []; + return (result.timestamp || []).map((timestamp, index) => ({ + date: new Date(timestamp * 1000).toISOString().slice(0, 10), + close: result.indicators?.quote?.[0]?.close?.[index], + })).filter((row) => Number.isFinite(row.close)); +} + +// same rule the worker now uses. a short series is a failed fetch, not data +async function seriesFor(cache, symbol) { + if (cache.has(symbol)) return cache.get(symbol); + const bars = await history(symbol); + if (!Array.isArray(bars) || bars.length < MIN_HISTORY_BARS) { + throw new Error(`market data for ${symbol} came back with only ${bars ? bars.length : 0} usable bars`); + } + cache.set(symbol, bars); + await sleep(400); + return bars; +} + +function snapshot(db) { + const byStatus = {}; + for (const row of db.prepare("SELECT status, COUNT(*) AS c FROM autonomy_predictions GROUP BY status").all()) { + byStatus[row.status] = row.c; + } + return { outcomes: db.prepare("SELECT COUNT(*) AS c FROM autonomy_outcomes").get().c, byStatus }; +} + +function report(label, snap) { + console.log(` ${label.padEnd(8)} outcomes=${snap.outcomes} ` + + Object.entries(snap.byStatus).map(([k, v]) => `${k}=${v}`).join(" ")); +} + +async function main() { + const opts = options(); + const commit = opts.commit === true; + const limit = Number(opts.limit) || 2000; + + const db = openRuntimeDb(INTELLIGENCE, { schema: "intelligence" }); + db.pragma("busy_timeout = 20000"); + initAutonomySchema(db); + + const where = ["p.status = 'unresolvable'", "o.prediction_id IS NULL"]; + const params = []; + if (opts.origin && opts.origin !== true) { where.push("p.origin = ?"); params.push(String(opts.origin)); } + + const stranded = db.prepare(` + SELECT p.* FROM autonomy_predictions p + LEFT JOIN autonomy_outcomes o ON o.prediction_id = p.id + WHERE ${where.join(" AND ")} + ORDER BY p.information_cutoff ASC LIMIT ${limit} + `).all(...params); + + const before = snapshot(db); + console.log(`unresolvable predictions with no outcome row: ${stranded.length}`); + report("before", before); + console.log(""); + + const cache = new Map(); + const repaired = []; + const stillStuck = []; + + for (const prediction of stranded) { + const horizonDate = addTradingDays(String(prediction.information_cutoff).slice(0, 10), prediction.horizon_days); + try { + const series = await seriesFor(cache, prediction.instrument); + const benchmark = await seriesFor(cache, BENCHMARK); + const result = calculateOutcome(prediction, series, benchmark); + if (!result) { + const lastBar = series.length ? series[series.length - 1].date : "none"; + stillStuck.push({ prediction, reason: `no exit bar for horizon ${horizonDate}, series ends ${lastBar}` }); + continue; + } + repaired.push({ prediction, result }); + } catch (error) { + console.error(`[repair] ${prediction.id} (${prediction.instrument}) fetch failed:`, error.message); + stillStuck.push({ prediction, reason: `fetch failed: ${error.message}` }); + } + } + + console.log(`scorable now : ${repaired.length}`); + console.log(`still stuck : ${stillStuck.length}`); + const reasons = {}; + for (const row of stillStuck) { + const key = row.reason.startsWith("fetch failed") ? "fetch failed" : "no exit bar yet"; + reasons[key] = (reasons[key] || 0) + 1; + } + for (const [reason, count] of Object.entries(reasons)) console.log(` ${reason}: ${count}`); + + if (repaired.length) { + const byOrigin = {}; + for (const { prediction } of repaired) byOrigin[prediction.origin] = (byOrigin[prediction.origin] || 0) + 1; + console.log(` repairable by origin: ${Object.entries(byOrigin).map(([k, v]) => `${k}=${v}`).join(" ")}`); + const hits = repaired.filter(({ result }) => result.directionCorrect).length; + console.log(` and for what its worth they score ${(100 * hits / repaired.length).toFixed(2)}%` + + ` (${hits}/${repaired.length}) — not a result, just a sanity check that they look like outcomes`); + } + + if (!commit) { + console.log("\ndry run, nothing written. pass --commit to apply."); + return; + } + + // one transaction, so a dropped connection cannot leave half of them resolved + const apply = db.transaction(() => { + const insert = db.prepare(` + INSERT INTO autonomy_outcomes + (prediction_id, price_0, price_horizon, benchmark_0, benchmark_horizon, excess_return, direction_correct, error_type) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(prediction_id) DO UPDATE SET + price_0=excluded.price_0, + price_horizon=excluded.price_horizon, + benchmark_0=excluded.benchmark_0, + benchmark_horizon=excluded.benchmark_horizon, + excess_return=excluded.excess_return, + direction_correct=excluded.direction_correct, + error_type=excluded.error_type, + evaluated_at=datetime('now') + `); + const reopen = db.prepare("UPDATE autonomy_predictions SET status = 'resolved' WHERE id = ?"); + for (const { prediction, result } of repaired) { + insert.run(prediction.id, result.price0, result.priceHorizon, result.benchmark0, result.benchmarkHorizon, + result.excessReturn, result.directionCorrect, result.directionCorrect ? null : "direction_error"); + reopen.run(prediction.id); + } + }); + apply(); + + const after = snapshot(db); + console.log(""); + report("before", before); + report("after", after); + const gained = after.outcomes - before.outcomes; + console.log(` outcomes +${gained}, unresolvable ${before.byStatus.unresolvable || 0} -> ${after.byStatus.unresolvable || 0}`); + if (gained !== repaired.length) { + console.error(`[repair] expected +${repaired.length} outcomes but got +${gained}, check for pre-existing rows`); + } +} + +main().catch((error) => { + console.error("[repair] failed:", error.message, error.stack); + process.exit(1); +}); diff --git a/test/autonomy.test.js b/test/autonomy.test.js index 11ef3af..7a033ac 100644 --- a/test/autonomy.test.js +++ b/test/autonomy.test.js @@ -9,7 +9,7 @@ const { calibrateOutcomes, cohortKey } = require('../src/autonomy/calibration'); const { decide } = require('../src/autonomy/policy'); const { validatePaperIntent, createSimulator } = require('../src/autonomy/execution'); const { calculateOutcome, addTradingDays } = require('../src/autonomy/outcomes'); -const { yahooSymbol } = require('../workers/outcomeAutonomyWorker'); +const { yahooSymbol, coversHorizon } = require('../workers/outcomeAutonomyWorker'); const { createOrderIntent } = require('../src/autonomy/orderIntents'); const { enqueueCoordinatorEvent, reconcileArchiveBatch, reconcileLiveBatch, isTransientCoordinatorFailure } = require('../workers/autonomyWorker'); const { buildGraphContext } = require('../src/autonomy/graphContext'); @@ -210,6 +210,23 @@ test('a leftover job from an older run cannot hijack the active run', () => { assert.equal(runForJob(db, { id: 12, idempotency_key: 'replay:999:article:1' }, active).id, second); }); +test('a price series that stops short of the horizon is a data gap, not an unresolvable prediction', () => { + const bars = [ + { date: '2026-09-01', close: 10 }, + { date: '2026-09-15', close: 11 }, + ]; + // the exit bar simply has not published yet. the worker used to retire these, + // which is how ~400 scorable rows got labelled unresolvable + assert.equal(coversHorizon(bars, '2026-09-29'), false); + assert.equal(coversHorizon(bars, '2026-09-15'), true, 'the horizon bar itself counts'); + assert.equal(coversHorizon(bars, '2026-09-10'), true); + + // an empty or missing series can never cover anything, and must not read as true + assert.equal(coversHorizon([], '2026-09-01'), false); + assert.equal(coversHorizon(undefined, '2026-09-01'), false); + assert.equal(coversHorizon(null, '2026-09-01'), false); +}); + test('calibration and policy abstain on insufficient evidence', () => { const calibration = calibrateOutcomes([ { excess_return: 0.02, direction_correct: 1 }, diff --git a/workers/outcomeAutonomyWorker.js b/workers/outcomeAutonomyWorker.js index 88c2e0b..9c86ef0 100644 --- a/workers/outcomeAutonomyWorker.js +++ b/workers/outcomeAutonomyWorker.js @@ -38,6 +38,35 @@ function withTimeout(promise, ms, label) { // how far past the horizon we keep trying before accepting there is no data const UNRESOLVABLE_GRACE_DAYS = 3; +// A 200 from yahoo is not the same thing as usable data. An empty or stub series +// got cached as if it were real, and because nothing in here ever expired, one +// bad benchmark fetch quietly poisoned every score this worker made for the rest +// of its life. Thats what stopped outcomes dead on the 20th while calculateOutcome +// was perfectly capable of scoring the rows it was busy retiring. +const MIN_HISTORY_BARS = 50; +const HISTORY_TTL_MS = 6 * 60 * 60 * 1000; + +async function seriesFor(cache, symbol) { + const hit = cache.get(symbol); + if (hit && (Date.now() - hit.fetchedAt) < HISTORY_TTL_MS) return hit.bars; + + const bars = await withTimeout(history(symbol), HISTORY_HARD_TIMEOUT, `market data for ${symbol}`); + if (!Array.isArray(bars) || bars.length < MIN_HISTORY_BARS) { + // never cache a dud. throwing puts it on the retry path instead + cache.delete(symbol); + throw new Error(`market data for ${symbol} came back with only ${bars ? bars.length : 0} usable bars`); + } + + cache.set(symbol, { bars, fetchedAt: Date.now() }); + return bars; +} + +// the exit bar cannot exist yet if the series stops short of the horizon, and +// thats a hole in what we fetched rather than a prediction nobody can ever score +function coversHorizon(bars, horizonDate) { + return !!(bars && bars.length) && bars[bars.length - 1].date >= horizonDate; +} + async function history(symbol) { // GDELT backfills predate the normal rolling quote window. Use an explicit // point-in-time range so replay outcomes do not silently become unresolvable. @@ -84,14 +113,9 @@ async function resolveAutonomyOutcomes({ intelligencePath, workerId = `outcome-$ } for (const prediction of predictions) { try { - if (!cache.has(prediction.instrument)) { - cache.set(prediction.instrument, await withTimeout(history(prediction.instrument), - HISTORY_HARD_TIMEOUT, `market data for ${prediction.instrument}`)); - } - if (!cache.has('SPY')) { - cache.set('SPY', await withTimeout(history('SPY'), HISTORY_HARD_TIMEOUT, 'market data for SPY')); - } - const result = calculateOutcome(prediction, cache.get(prediction.instrument), cache.get('SPY')); + const series = await seriesFor(cache, prediction.instrument); + const benchmark = await seriesFor(cache, "SPY"); + const result = calculateOutcome(prediction, series, benchmark); if (!result) { // A null here almost always means the exit bar has not published yet, not // that the prediction can never be scored. The sql due-check counts @@ -101,6 +125,18 @@ async function resolveAutonomyOutcomes({ intelligencePath, workerId = `outcome-$ // predictions we are waiting on. Wait until the horizon is properly past // before giving up on it. const horizonDate = addTradingDays(String(prediction.information_cutoff).slice(0, 10), prediction.horizon_days); + + // Before blaming the prediction, check the prices we hold actually reach + // the horizon. Retiring everything that came back null is how roughly + // 400 scorable rows ended up labelled unresolvable, live ones included. + if (!coversHorizon(series, horizonDate) || !coversHorizon(benchmark, horizonDate)) { + cache.delete(prediction.instrument); + cache.delete("SPY"); + console.error(`[autonomy-outcome] ${workerId} prediction ${prediction.id} (${prediction.instrument})` + + ` left open, price data stops before horizon ${horizonDate}`); + continue; + } + const graceExpired = addTradingDays(horizonDate, UNRESOLVABLE_GRACE_DAYS) < new Date().toISOString().slice(0, 10); if (graceExpired) { db.prepare("UPDATE autonomy_predictions SET status = 'unresolvable' WHERE id = ?").run(prediction.id); @@ -149,4 +185,4 @@ async function resolveAutonomyOutcomes({ intelligencePath, workerId = `outcome-$ } } -module.exports = { calculateOutcome, resolveAutonomyOutcomes, yahooSymbol }; +module.exports = { calculateOutcome, resolveAutonomyOutcomes, yahooSymbol, coversHorizon };