From 028c07688d51f8fe27023fbcb37e8ce33d0c8780 Mon Sep 17 00:00:00 2001 From: ImBenji Date: Tue, 8 Sep 2026 00:25:52 +0100 Subject: [PATCH] fix: bound the market data fetch so the outcome worker cannot wedge Live outcomes have been stuck at 4 while 35 matured live predictions sat unscored, the oldest four days past its horizon. Running those three by hand resolves them in 200-400ms each, so the data was there and the maths was fine. The worker itself was wedged: up three days, last log 6 Sept, processing nothing. req.setTimeout only covers socket inactivity. A response that opens and then stalls leaves the promise pending forever, and with it the entire loop, because the fetch is awaited inline. There is now a hard bound around it. This is the third time an unbounded await inside a long lived loop has silently stopped a worker: the browser session in content, the content round itself, and now market data. In every case the container stayed up, nothing threw, and nothing was logged, which is the worst possible failure shape. So the worker also announces what it is about to score, because an idle worker and a dead one should not look identical from outside. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WnNxwxfXSbeNtjvtz5gayb --- workers/outcomeAutonomyWorker.js | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/workers/outcomeAutonomyWorker.js b/workers/outcomeAutonomyWorker.js index c01c2de..88c2e0b 100644 --- a/workers/outcomeAutonomyWorker.js +++ b/workers/outcomeAutonomyWorker.js @@ -20,6 +20,21 @@ function httpGet(url) { } const MAX_OUTCOME_ATTEMPTS = 5; +// req.setTimeout only covers socket inactivity. A response that opens and then +// stalls, or a socket that never emits anything at all, leaves the promise +// pending forever and the whole loop with it. This worker sat "Up 3 days" and +// silent while predictions it could resolve in 400ms went unscored, which is the +// third time an unbounded await in a long lived loop has quietly stopped a +// worker. This is the bound that cannot be skipped. +const HISTORY_HARD_TIMEOUT = 30000; + +function withTimeout(promise, ms, label) { + let timer; + const expired = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`${label} exceeded ${ms}ms`)), ms); + }); + return Promise.race([promise, expired]).finally(() => clearTimeout(timer)); +} // how far past the horizon we keep trying before accepting there is no data const UNRESOLVABLE_GRACE_DAYS = 3; @@ -62,10 +77,20 @@ async function resolveAutonomyOutcomes({ intelligencePath, workerId = `outcome-$ // strictly before today, so the exit session has closed and published return horizonDate < today; }).slice(0, 25); + + if (predictions.length) { + console.log(`[autonomy-outcome] ${workerId} scoring ${predictions.length} matured predictions` + + ` (${candidates.length} candidates)`); + } for (const prediction of predictions) { try { - if (!cache.has(prediction.instrument)) cache.set(prediction.instrument, await history(prediction.instrument)); - if (!cache.has('SPY')) cache.set('SPY', await history('SPY')); + if (!cache.has(prediction.instrument)) { + cache.set(prediction.instrument, await withTimeout(history(prediction.instrument), + HISTORY_HARD_TIMEOUT, `market data for ${prediction.instrument}`)); + } + if (!cache.has('SPY')) { + cache.set('SPY', await withTimeout(history('SPY'), HISTORY_HARD_TIMEOUT, 'market data for SPY')); + } const result = calculateOutcome(prediction, cache.get(prediction.instrument), cache.get('SPY')); if (!result) { // A null here almost always means the exit bar has not published yet, not