diff --git a/workers/outcomeAutonomyWorker.js b/workers/outcomeAutonomyWorker.js index c01c2de..88c2e0b 100644 --- a/workers/outcomeAutonomyWorker.js +++ b/workers/outcomeAutonomyWorker.js @@ -20,6 +20,21 @@ function httpGet(url) { } const MAX_OUTCOME_ATTEMPTS = 5; +// req.setTimeout only covers socket inactivity. A response that opens and then +// stalls, or a socket that never emits anything at all, leaves the promise +// pending forever and the whole loop with it. This worker sat "Up 3 days" and +// silent while predictions it could resolve in 400ms went unscored, which is the +// third time an unbounded await in a long lived loop has quietly stopped a +// worker. This is the bound that cannot be skipped. +const HISTORY_HARD_TIMEOUT = 30000; + +function withTimeout(promise, ms, label) { + let timer; + const expired = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`${label} exceeded ${ms}ms`)), ms); + }); + return Promise.race([promise, expired]).finally(() => clearTimeout(timer)); +} // how far past the horizon we keep trying before accepting there is no data const UNRESOLVABLE_GRACE_DAYS = 3; @@ -62,10 +77,20 @@ async function resolveAutonomyOutcomes({ intelligencePath, workerId = `outcome-$ // strictly before today, so the exit session has closed and published return horizonDate < today; }).slice(0, 25); + + if (predictions.length) { + console.log(`[autonomy-outcome] ${workerId} scoring ${predictions.length} matured predictions` + + ` (${candidates.length} candidates)`); + } for (const prediction of predictions) { try { - if (!cache.has(prediction.instrument)) cache.set(prediction.instrument, await history(prediction.instrument)); - if (!cache.has('SPY')) cache.set('SPY', await history('SPY')); + if (!cache.has(prediction.instrument)) { + cache.set(prediction.instrument, await withTimeout(history(prediction.instrument), + HISTORY_HARD_TIMEOUT, `market data for ${prediction.instrument}`)); + } + if (!cache.has('SPY')) { + cache.set('SPY', await withTimeout(history('SPY'), HISTORY_HARD_TIMEOUT, 'market data for SPY')); + } const result = calculateOutcome(prediction, cache.get(prediction.instrument), cache.get('SPY')); if (!result) { // A null here almost always means the exit bar has not published yet, not