fix: one bad SPY fetch stopped outcome scoring for nineteen days
The outcome worker kept one price series per symbol in a map that never expired, and on the null path it only ever dropped the instrument, never the benchmark. So a single degraded-but-200 SPY response poisoned the benchmark for the whole life of the worker. calculateOutcome then returned null for every symbol, and each matured prediction got retired as "no market data" the moment its grace ran out. Outcomes stopped dead on 2026-09-20. Three unrelated tickers failing in the same pass was the tell. I ran calculateOutcome against the exact predictions the worker was retiring and all three scored fine, with GSBC sitting there on yahoo with 6,732 bars through yesterday. Nothing was ever missing. Fourth time now that unmanaged state in a long lived loop has quietly broken this pipeline, so the cache gets a ttl, a dud series is treated as a failed fetch rather than cached as though it were real, and the benchmark is dropped alongside the instrument when a score comes back empty. The guard that actually matters is coversHorizon. A prediction is only retired when the prices we hold genuinely reach its horizon. If the series stops short then the exit bar has not published yet, which is a hole in what we fetched and not a prediction nobody can score. That distinction is the whole bug. scripts/repair-unresolvable-outcomes.js re-asks every mislabeled row with fresh data and leaves anything genuinely unscorable alone. Dry run by default, writes through openRuntimeDb so it honours the configured backend, and prints row counts either side. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -38,6 +38,35 @@ function withTimeout(promise, ms, label) {
|
||||
// how far past the horizon we keep trying before accepting there is no data
|
||||
const UNRESOLVABLE_GRACE_DAYS = 3;
|
||||
|
||||
// A 200 from yahoo is not the same thing as usable data. An empty or stub series
|
||||
// got cached as if it were real, and because nothing in here ever expired, one
|
||||
// bad benchmark fetch quietly poisoned every score this worker made for the rest
|
||||
// of its life. Thats what stopped outcomes dead on the 20th while calculateOutcome
|
||||
// was perfectly capable of scoring the rows it was busy retiring.
|
||||
const MIN_HISTORY_BARS = 50;
|
||||
const HISTORY_TTL_MS = 6 * 60 * 60 * 1000;
|
||||
|
||||
async function seriesFor(cache, symbol) {
|
||||
const hit = cache.get(symbol);
|
||||
if (hit && (Date.now() - hit.fetchedAt) < HISTORY_TTL_MS) return hit.bars;
|
||||
|
||||
const bars = await withTimeout(history(symbol), HISTORY_HARD_TIMEOUT, `market data for ${symbol}`);
|
||||
if (!Array.isArray(bars) || bars.length < MIN_HISTORY_BARS) {
|
||||
// never cache a dud. throwing puts it on the retry path instead
|
||||
cache.delete(symbol);
|
||||
throw new Error(`market data for ${symbol} came back with only ${bars ? bars.length : 0} usable bars`);
|
||||
}
|
||||
|
||||
cache.set(symbol, { bars, fetchedAt: Date.now() });
|
||||
return bars;
|
||||
}
|
||||
|
||||
// the exit bar cannot exist yet if the series stops short of the horizon, and
|
||||
// thats a hole in what we fetched rather than a prediction nobody can ever score
|
||||
function coversHorizon(bars, horizonDate) {
|
||||
return !!(bars && bars.length) && bars[bars.length - 1].date >= horizonDate;
|
||||
}
|
||||
|
||||
async function history(symbol) {
|
||||
// GDELT backfills predate the normal rolling quote window. Use an explicit
|
||||
// point-in-time range so replay outcomes do not silently become unresolvable.
|
||||
@@ -84,14 +113,9 @@ async function resolveAutonomyOutcomes({ intelligencePath, workerId = `outcome-$
|
||||
}
|
||||
for (const prediction of predictions) {
|
||||
try {
|
||||
if (!cache.has(prediction.instrument)) {
|
||||
cache.set(prediction.instrument, await withTimeout(history(prediction.instrument),
|
||||
HISTORY_HARD_TIMEOUT, `market data for ${prediction.instrument}`));
|
||||
}
|
||||
if (!cache.has('SPY')) {
|
||||
cache.set('SPY', await withTimeout(history('SPY'), HISTORY_HARD_TIMEOUT, 'market data for SPY'));
|
||||
}
|
||||
const result = calculateOutcome(prediction, cache.get(prediction.instrument), cache.get('SPY'));
|
||||
const series = await seriesFor(cache, prediction.instrument);
|
||||
const benchmark = await seriesFor(cache, "SPY");
|
||||
const result = calculateOutcome(prediction, series, benchmark);
|
||||
if (!result) {
|
||||
// A null here almost always means the exit bar has not published yet, not
|
||||
// that the prediction can never be scored. The sql due-check counts
|
||||
@@ -101,6 +125,18 @@ async function resolveAutonomyOutcomes({ intelligencePath, workerId = `outcome-$
|
||||
// predictions we are waiting on. Wait until the horizon is properly past
|
||||
// before giving up on it.
|
||||
const horizonDate = addTradingDays(String(prediction.information_cutoff).slice(0, 10), prediction.horizon_days);
|
||||
|
||||
// Before blaming the prediction, check the prices we hold actually reach
|
||||
// the horizon. Retiring everything that came back null is how roughly
|
||||
// 400 scorable rows ended up labelled unresolvable, live ones included.
|
||||
if (!coversHorizon(series, horizonDate) || !coversHorizon(benchmark, horizonDate)) {
|
||||
cache.delete(prediction.instrument);
|
||||
cache.delete("SPY");
|
||||
console.error(`[autonomy-outcome] ${workerId} prediction ${prediction.id} (${prediction.instrument})`
|
||||
+ ` left open, price data stops before horizon ${horizonDate}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const graceExpired = addTradingDays(horizonDate, UNRESOLVABLE_GRACE_DAYS) < new Date().toISOString().slice(0, 10);
|
||||
if (graceExpired) {
|
||||
db.prepare("UPDATE autonomy_predictions SET status = 'unresolvable' WHERE id = ?").run(prediction.id);
|
||||
@@ -149,4 +185,4 @@ async function resolveAutonomyOutcomes({ intelligencePath, workerId = `outcome-$
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { calculateOutcome, resolveAutonomyOutcomes, yahooSymbol };
|
||||
module.exports = { calculateOutcome, resolveAutonomyOutcomes, yahooSymbol, coversHorizon };
|
||||
|
||||
Reference in New Issue
Block a user