The outcome worker kept one price series per symbol in a map that never expired, and on the null path it only ever dropped the instrument, never the benchmark. So a single degraded-but-200 SPY response poisoned the benchmark for the whole life of the worker. calculateOutcome then returned null for every symbol, and each matured prediction got retired as "no market data" the moment its grace ran out. Outcomes stopped dead on 2026-09-20. Three unrelated tickers failing in the same pass was the tell. I ran calculateOutcome against the exact predictions the worker was retiring and all three scored fine, with GSBC sitting there on yahoo with 6,732 bars through yesterday. Nothing was ever missing. Fourth time now that unmanaged state in a long lived loop has quietly broken this pipeline, so the cache gets a ttl, a dud series is treated as a failed fetch rather than cached as though it were real, and the benchmark is dropped alongside the instrument when a score comes back empty. The guard that actually matters is coversHorizon. A prediction is only retired when the prices we hold genuinely reach its horizon. If the series stops short then the exit bar has not published yet, which is a hole in what we fetched and not a prediction nobody can score. That distinction is the whole bug. scripts/repair-unresolvable-outcomes.js re-asks every mislabeled row with fresh data and leaves anything genuinely unscorable alone. Dry run by default, writes through openRuntimeDb so it honours the configured backend, and prints row counts either side. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
208 lines
8.5 KiB
JavaScript
208 lines
8.5 KiB
JavaScript
#!/usr/bin/env node
|
|
/*
|
|
* Re-score predictions that were labelled unresolvable when they were nothing of
|
|
* the kind.
|
|
*
|
|
* The outcome worker held one price series per symbol in a map that never
|
|
* expired, and on the null path it only ever dropped the instrument, never the
|
|
* benchmark. So a single degraded-but-200 SPY fetch poisoned the benchmark for
|
|
* the whole life of the worker, calculateOutcome returned null for every symbol
|
|
* after that, and each matured prediction got retired as "no market data" once
|
|
* its grace ran out. Outcomes stopped on 2026-09-20 and ~150 rows were mislabeled
|
|
* while the prices needed to score them were sitting on yahoo the entire time.
|
|
*
|
|
* Nothing was lost, only mislabeled, so this just asks each one again with fresh
|
|
* data. A row that genuinely cannot be scored is left exactly as it is.
|
|
*
|
|
* Writes through openRuntimeDb so it honours DURIIN_DB_BACKEND rather than
|
|
* assuming sqlite, and it reports row counts either side of the change.
|
|
*
|
|
* node scripts/repair-unresolvable-outcomes.js --dry-run
|
|
* node scripts/repair-unresolvable-outcomes.js --origin live --dry-run
|
|
* node scripts/repair-unresolvable-outcomes.js --commit
|
|
*/
|
|
const https = require("https");
|
|
const { openRuntimeDb } = require("../src/db/runtime");
|
|
const { initAutonomySchema } = require("../src/autonomy/schema");
|
|
const { calculateOutcome, addTradingDays, yahooSymbol } = require("../src/autonomy/outcomes");
|
|
|
|
const INTELLIGENCE = process.env.INTELLIGENCE_DB || "/data/intelligence.sqlite";
|
|
const MIN_HISTORY_BARS = 50;
|
|
const BENCHMARK = "SPY";
|
|
|
|
function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); }
|
|
|
|
function options() {
|
|
const argv = process.argv.slice(2);
|
|
const out = {};
|
|
for (let i = 0; i < argv.length; i += 1) {
|
|
if (!argv[i].startsWith("--")) continue;
|
|
const key = argv[i].slice(2);
|
|
const next = argv[i + 1];
|
|
out[key] = (next && !next.startsWith("--")) ? (i += 1, next) : true;
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function httpGet(url) {
|
|
return new Promise((resolve, reject) => {
|
|
const request = https.get(url, { headers: { "User-Agent": "duriin-autonomy/1.0" } }, (response) => {
|
|
let body = "";
|
|
response.setEncoding("utf8");
|
|
response.on("data", (chunk) => { body += chunk; });
|
|
response.on("end", () => response.statusCode >= 200 && response.statusCode < 300
|
|
? resolve(body) : reject(new Error(`market data returned ${response.statusCode}`)));
|
|
});
|
|
request.setTimeout(15000, () => request.destroy(new Error("market data timeout")));
|
|
request.on("error", reject);
|
|
});
|
|
}
|
|
|
|
async function history(symbol) {
|
|
const url = `https://query1.finance.yahoo.com/v8/finance/chart/${encodeURIComponent(yahooSymbol(symbol))}`
|
|
+ `?period1=946684800&period2=${Math.floor(Date.now() / 1000)}&interval=1d`;
|
|
const body = JSON.parse(await httpGet(url));
|
|
const result = body?.chart?.result?.[0];
|
|
if (!result) return [];
|
|
return (result.timestamp || []).map((timestamp, index) => ({
|
|
date: new Date(timestamp * 1000).toISOString().slice(0, 10),
|
|
close: result.indicators?.quote?.[0]?.close?.[index],
|
|
})).filter((row) => Number.isFinite(row.close));
|
|
}
|
|
|
|
// same rule the worker now uses. a short series is a failed fetch, not data
|
|
async function seriesFor(cache, symbol) {
|
|
if (cache.has(symbol)) return cache.get(symbol);
|
|
const bars = await history(symbol);
|
|
if (!Array.isArray(bars) || bars.length < MIN_HISTORY_BARS) {
|
|
throw new Error(`market data for ${symbol} came back with only ${bars ? bars.length : 0} usable bars`);
|
|
}
|
|
cache.set(symbol, bars);
|
|
await sleep(400);
|
|
return bars;
|
|
}
|
|
|
|
function snapshot(db) {
|
|
const byStatus = {};
|
|
for (const row of db.prepare("SELECT status, COUNT(*) AS c FROM autonomy_predictions GROUP BY status").all()) {
|
|
byStatus[row.status] = row.c;
|
|
}
|
|
return { outcomes: db.prepare("SELECT COUNT(*) AS c FROM autonomy_outcomes").get().c, byStatus };
|
|
}
|
|
|
|
function report(label, snap) {
|
|
console.log(` ${label.padEnd(8)} outcomes=${snap.outcomes} `
|
|
+ Object.entries(snap.byStatus).map(([k, v]) => `${k}=${v}`).join(" "));
|
|
}
|
|
|
|
async function main() {
|
|
const opts = options();
|
|
const commit = opts.commit === true;
|
|
const limit = Number(opts.limit) || 2000;
|
|
|
|
const db = openRuntimeDb(INTELLIGENCE, { schema: "intelligence" });
|
|
db.pragma("busy_timeout = 20000");
|
|
initAutonomySchema(db);
|
|
|
|
const where = ["p.status = 'unresolvable'", "o.prediction_id IS NULL"];
|
|
const params = [];
|
|
if (opts.origin && opts.origin !== true) { where.push("p.origin = ?"); params.push(String(opts.origin)); }
|
|
|
|
const stranded = db.prepare(`
|
|
SELECT p.* FROM autonomy_predictions p
|
|
LEFT JOIN autonomy_outcomes o ON o.prediction_id = p.id
|
|
WHERE ${where.join(" AND ")}
|
|
ORDER BY p.information_cutoff ASC LIMIT ${limit}
|
|
`).all(...params);
|
|
|
|
const before = snapshot(db);
|
|
console.log(`unresolvable predictions with no outcome row: ${stranded.length}`);
|
|
report("before", before);
|
|
console.log("");
|
|
|
|
const cache = new Map();
|
|
const repaired = [];
|
|
const stillStuck = [];
|
|
|
|
for (const prediction of stranded) {
|
|
const horizonDate = addTradingDays(String(prediction.information_cutoff).slice(0, 10), prediction.horizon_days);
|
|
try {
|
|
const series = await seriesFor(cache, prediction.instrument);
|
|
const benchmark = await seriesFor(cache, BENCHMARK);
|
|
const result = calculateOutcome(prediction, series, benchmark);
|
|
if (!result) {
|
|
const lastBar = series.length ? series[series.length - 1].date : "none";
|
|
stillStuck.push({ prediction, reason: `no exit bar for horizon ${horizonDate}, series ends ${lastBar}` });
|
|
continue;
|
|
}
|
|
repaired.push({ prediction, result });
|
|
} catch (error) {
|
|
console.error(`[repair] ${prediction.id} (${prediction.instrument}) fetch failed:`, error.message);
|
|
stillStuck.push({ prediction, reason: `fetch failed: ${error.message}` });
|
|
}
|
|
}
|
|
|
|
console.log(`scorable now : ${repaired.length}`);
|
|
console.log(`still stuck : ${stillStuck.length}`);
|
|
const reasons = {};
|
|
for (const row of stillStuck) {
|
|
const key = row.reason.startsWith("fetch failed") ? "fetch failed" : "no exit bar yet";
|
|
reasons[key] = (reasons[key] || 0) + 1;
|
|
}
|
|
for (const [reason, count] of Object.entries(reasons)) console.log(` ${reason}: ${count}`);
|
|
|
|
if (repaired.length) {
|
|
const byOrigin = {};
|
|
for (const { prediction } of repaired) byOrigin[prediction.origin] = (byOrigin[prediction.origin] || 0) + 1;
|
|
console.log(` repairable by origin: ${Object.entries(byOrigin).map(([k, v]) => `${k}=${v}`).join(" ")}`);
|
|
const hits = repaired.filter(({ result }) => result.directionCorrect).length;
|
|
console.log(` and for what its worth they score ${(100 * hits / repaired.length).toFixed(2)}%`
|
|
+ ` (${hits}/${repaired.length}) — not a result, just a sanity check that they look like outcomes`);
|
|
}
|
|
|
|
if (!commit) {
|
|
console.log("\ndry run, nothing written. pass --commit to apply.");
|
|
return;
|
|
}
|
|
|
|
// one transaction, so a dropped connection cannot leave half of them resolved
|
|
const apply = db.transaction(() => {
|
|
const insert = db.prepare(`
|
|
INSERT INTO autonomy_outcomes
|
|
(prediction_id, price_0, price_horizon, benchmark_0, benchmark_horizon, excess_return, direction_correct, error_type)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(prediction_id) DO UPDATE SET
|
|
price_0=excluded.price_0,
|
|
price_horizon=excluded.price_horizon,
|
|
benchmark_0=excluded.benchmark_0,
|
|
benchmark_horizon=excluded.benchmark_horizon,
|
|
excess_return=excluded.excess_return,
|
|
direction_correct=excluded.direction_correct,
|
|
error_type=excluded.error_type,
|
|
evaluated_at=datetime('now')
|
|
`);
|
|
const reopen = db.prepare("UPDATE autonomy_predictions SET status = 'resolved' WHERE id = ?");
|
|
for (const { prediction, result } of repaired) {
|
|
insert.run(prediction.id, result.price0, result.priceHorizon, result.benchmark0, result.benchmarkHorizon,
|
|
result.excessReturn, result.directionCorrect, result.directionCorrect ? null : "direction_error");
|
|
reopen.run(prediction.id);
|
|
}
|
|
});
|
|
apply();
|
|
|
|
const after = snapshot(db);
|
|
console.log("");
|
|
report("before", before);
|
|
report("after", after);
|
|
const gained = after.outcomes - before.outcomes;
|
|
console.log(` outcomes +${gained}, unresolvable ${before.byStatus.unresolvable || 0} -> ${after.byStatus.unresolvable || 0}`);
|
|
if (gained !== repaired.length) {
|
|
console.error(`[repair] expected +${repaired.length} outcomes but got +${gained}, check for pre-existing rows`);
|
|
}
|
|
}
|
|
|
|
main().catch((error) => {
|
|
console.error("[repair] failed:", error.message, error.stack);
|
|
process.exit(1);
|
|
});
|