diff --git a/src/autonomy/outcomes.js b/src/autonomy/outcomes.js index e0a01e4..4a34c6a 100644 --- a/src/autonomy/outcomes.js +++ b/src/autonomy/outcomes.js @@ -9,18 +9,31 @@ function addTradingDays(date, days) { return value.toISOString().slice(0, 10); } +function barOnOrAfter(history, date) { + return history.find((row) => row.date >= date) || null; +} + function nearestOnOrAfter(history, date) { - return history.find((row) => row.date >= date)?.close ?? null; + return barOnOrAfter(history, date)?.close ?? null; } function calculateOutcome(prediction, instrumentHistory, benchmarkHistory) { const eventDate = String(prediction.information_cutoff).slice(0, 10); const horizonDate = addTradingDays(eventDate, prediction.horizon_days); - const price0 = nearestOnOrAfter(instrumentHistory, eventDate); - const priceHorizon = nearestOnOrAfter(instrumentHistory, horizonDate); - const benchmark0 = nearestOnOrAfter(benchmarkHistory, eventDate); - const benchmarkHorizon = nearestOnOrAfter(benchmarkHistory, horizonDate); + const entryBar = barOnOrAfter(instrumentHistory, eventDate); + const exitBar = barOnOrAfter(instrumentHistory, horizonDate); + const benchEntryBar = barOnOrAfter(benchmarkHistory, eventDate); + const benchExitBar = barOnOrAfter(benchmarkHistory, horizonDate); + const price0 = entryBar?.close ?? null; + const priceHorizon = exitBar?.close ?? null; + const benchmark0 = benchEntryBar?.close ?? null; + const benchmarkHorizon = benchExitBar?.close ?? null; if (![price0, priceHorizon, benchmark0, benchmarkHorizon].every(Number.isFinite)) return null; + + // On a short horizon the entry and exit lookups can land on the same bar, which + // yields an excess return of exactly zero and gets scored as a directional miss. + // That is not a result, it means the horizon has not actually elapsed yet. + if (entryBar.date === exitBar.date || benchEntryBar.date === benchExitBar.date) return null; const instrumentReturn = (priceHorizon - price0) / price0; const benchmarkReturn = (benchmarkHorizon - benchmark0) / benchmark0; const excessReturn = instrumentReturn - benchmarkReturn; diff --git a/test/autonomy.test.js b/test/autonomy.test.js index 09447ec..a0cf0bb 100644 --- a/test/autonomy.test.js +++ b/test/autonomy.test.js @@ -8,7 +8,7 @@ const { normalizeProposal, acceptProposal } = require('../src/autonomy/coordinat const { calibrateOutcomes, cohortKey } = require('../src/autonomy/calibration'); const { decide } = require('../src/autonomy/policy'); const { validatePaperIntent, createSimulator } = require('../src/autonomy/execution'); -const { calculateOutcome } = require('../src/autonomy/outcomes'); +const { calculateOutcome, addTradingDays } = require('../src/autonomy/outcomes'); const { yahooSymbol } = require('../workers/outcomeAutonomyWorker'); const { createOrderIntent } = require('../src/autonomy/orderIntents'); const { enqueueCoordinatorEvent, reconcileArchiveBatch, reconcileLiveBatch } = require('../workers/autonomyWorker'); @@ -306,3 +306,33 @@ test('dotted tickers are translated to the format the price feed expects', () => assert.equal(yahooSymbol('NVDA'), 'NVDA'); assert.equal(yahooSymbol(' spy '), 'SPY'); }); + +test('an outcome whose horizon has not actually elapsed is not a result', () => { + // ITW and WMT both scored exactly 0.00% excess in production because at + // horizon 1 the entry and exit lookups landed on the same bar. Zero is not a + // measurement, and it was being recorded as a directional miss. + const sameBar = calculateOutcome( + { information_cutoff: '2026-01-02T00:00:00Z', horizon_days: 1, direction: 'positive' }, + [{ date: '2026-01-05', close: 100 }], + [{ date: '2026-01-05', close: 100 }] + ); + assert.equal(sameBar, null); + + // the exit bar genuinely existing still scores normally + const real = calculateOutcome( + { information_cutoff: '2026-01-02T00:00:00Z', horizon_days: 1, direction: 'positive' }, + [{ date: '2026-01-02', close: 100 }, { date: '2026-01-05', close: 104 }], + [{ date: '2026-01-02', close: 100 }, { date: '2026-01-05', close: 102 }] + ); + assert.equal(real.directionCorrect, 1); + assert.ok(Math.abs(real.excessReturn - 0.02) < 1e-9); +}); + +test('trading day arithmetic steps over weekends', () => { + // friday + 1 trading day is monday, not saturday. the old due-check counted + // calendar days and so called a friday horizon-1 prediction due on saturday, + // when monday's close cannot exist yet. + assert.equal(addTradingDays('2026-01-02', 1), '2026-01-05'); + assert.equal(addTradingDays('2026-01-02', 5), '2026-01-09'); + assert.equal(addTradingDays('2026-01-02', 0), '2026-01-02'); +}); diff --git a/workers/outcomeAutonomyWorker.js b/workers/outcomeAutonomyWorker.js index 1cca1ac..4d81ebf 100644 --- a/workers/outcomeAutonomyWorker.js +++ b/workers/outcomeAutonomyWorker.js @@ -2,7 +2,7 @@ const os = require('os'); const https = require('https'); const { openRuntimeDb } = require('../src/db/runtime'); const { initAutonomySchema } = require('../src/autonomy/schema'); -const { calculateOutcome } = require('../src/autonomy/outcomes'); +const { calculateOutcome, addTradingDays } = require('../src/autonomy/outcomes'); function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } function httpGet(url) { @@ -20,6 +20,8 @@ function httpGet(url) { } const MAX_OUTCOME_ATTEMPTS = 5; +// how far past the horizon we keep trying before accepting there is no data +const UNRESOLVABLE_GRACE_DAYS = 3; // Yahoo writes class shares with a dash, BRK.B is BRK-B there. Our allowlist is // full of dotted symbols, and every one of them 404s forever otherwise. @@ -48,20 +50,46 @@ async function resolveAutonomyOutcomes({ intelligencePath, workerId = `outcome-$ const cache = new Map(); const failures = new Map(); while (true) { - const predictions = db.prepare(` + // The sql filter is deliberately loose, it only counts calendar days and cannot + // know about weekends or when a close actually publishes. Trading day + // arithmetic, the same arithmetic calculateOutcome uses to find the exit bar, + // then decides what is genuinely ready. + const candidates = db.prepare(` SELECT p.* FROM autonomy_predictions p LEFT JOIN autonomy_outcomes o ON o.prediction_id = p.id WHERE p.status = 'open' AND o.prediction_id IS NULL AND datetime(p.information_cutoff, '+' || p.horizon_days || ' days') <= datetime('now') - ORDER BY p.information_cutoff ASC LIMIT 25 + ORDER BY p.information_cutoff ASC LIMIT 100 `).all(); + + const today = new Date().toISOString().slice(0, 10); + const predictions = candidates.filter((p) => { + const horizonDate = addTradingDays(String(p.information_cutoff).slice(0, 10), p.horizon_days); + // strictly before today, so the exit session has closed and published + return horizonDate < today; + }).slice(0, 25); for (const prediction of predictions) { try { if (!cache.has(prediction.instrument)) cache.set(prediction.instrument, await history(prediction.instrument)); if (!cache.has('SPY')) cache.set('SPY', await history('SPY')); const result = calculateOutcome(prediction, cache.get(prediction.instrument), cache.get('SPY')); if (!result) { - db.prepare("UPDATE autonomy_predictions SET status = 'unresolvable' WHERE id = ?").run(prediction.id); + // A null here almost always means the exit bar has not published yet, not + // that the prediction can never be scored. The sql due-check counts + // calendar days while the price lookup counts trading days, so a friday + // horizon-1 call looks due on saturday when monday's close cannot exist. + // Retiring it there permanently destroyed exactly the short-horizon live + // predictions we are waiting on. Wait until the horizon is properly past + // before giving up on it. + const horizonDate = addTradingDays(String(prediction.information_cutoff).slice(0, 10), prediction.horizon_days); + const graceExpired = addTradingDays(horizonDate, UNRESOLVABLE_GRACE_DAYS) < new Date().toISOString().slice(0, 10); + if (graceExpired) { + db.prepare("UPDATE autonomy_predictions SET status = 'unresolvable' WHERE id = ?").run(prediction.id); + console.error(`[autonomy-outcome] ${workerId} prediction ${prediction.id} (${prediction.instrument})` + + ` unresolvable: no market data ${UNRESOLVABLE_GRACE_DAYS} trading days past horizon ${horizonDate}`); + } else { + cache.delete(prediction.instrument); + } continue; } db.prepare(`