fix: bound the market data fetch so the outcome worker cannot wedge
Live outcomes have been stuck at 4 while 35 matured live predictions sat unscored, the oldest four days past its horizon. Running those three by hand resolves them in 200-400ms each, so the data was there and the maths was fine. The worker itself was wedged: up three days, last log 6 Sept, processing nothing. req.setTimeout only covers socket inactivity. A response that opens and then stalls leaves the promise pending forever, and with it the entire loop, because the fetch is awaited inline. There is now a hard bound around it. This is the third time an unbounded await inside a long lived loop has silently stopped a worker: the browser session in content, the content round itself, and now market data. In every case the container stayed up, nothing threw, and nothing was logged, which is the worst possible failure shape. So the worker also announces what it is about to score, because an idle worker and a dead one should not look identical from outside. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WnNxwxfXSbeNtjvtz5gayb
This commit is contained in:
@@ -20,6 +20,21 @@ function httpGet(url) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const MAX_OUTCOME_ATTEMPTS = 5;
|
const MAX_OUTCOME_ATTEMPTS = 5;
|
||||||
|
// req.setTimeout only covers socket inactivity. A response that opens and then
|
||||||
|
// stalls, or a socket that never emits anything at all, leaves the promise
|
||||||
|
// pending forever and the whole loop with it. This worker sat "Up 3 days" and
|
||||||
|
// silent while predictions it could resolve in 400ms went unscored, which is the
|
||||||
|
// third time an unbounded await in a long lived loop has quietly stopped a
|
||||||
|
// worker. This is the bound that cannot be skipped.
|
||||||
|
const HISTORY_HARD_TIMEOUT = 30000;
|
||||||
|
|
||||||
|
function withTimeout(promise, ms, label) {
|
||||||
|
let timer;
|
||||||
|
const expired = new Promise((_, reject) => {
|
||||||
|
timer = setTimeout(() => reject(new Error(`${label} exceeded ${ms}ms`)), ms);
|
||||||
|
});
|
||||||
|
return Promise.race([promise, expired]).finally(() => clearTimeout(timer));
|
||||||
|
}
|
||||||
// how far past the horizon we keep trying before accepting there is no data
|
// how far past the horizon we keep trying before accepting there is no data
|
||||||
const UNRESOLVABLE_GRACE_DAYS = 3;
|
const UNRESOLVABLE_GRACE_DAYS = 3;
|
||||||
|
|
||||||
@@ -62,10 +77,20 @@ async function resolveAutonomyOutcomes({ intelligencePath, workerId = `outcome-$
|
|||||||
// strictly before today, so the exit session has closed and published
|
// strictly before today, so the exit session has closed and published
|
||||||
return horizonDate < today;
|
return horizonDate < today;
|
||||||
}).slice(0, 25);
|
}).slice(0, 25);
|
||||||
|
|
||||||
|
if (predictions.length) {
|
||||||
|
console.log(`[autonomy-outcome] ${workerId} scoring ${predictions.length} matured predictions`
|
||||||
|
+ ` (${candidates.length} candidates)`);
|
||||||
|
}
|
||||||
for (const prediction of predictions) {
|
for (const prediction of predictions) {
|
||||||
try {
|
try {
|
||||||
if (!cache.has(prediction.instrument)) cache.set(prediction.instrument, await history(prediction.instrument));
|
if (!cache.has(prediction.instrument)) {
|
||||||
if (!cache.has('SPY')) cache.set('SPY', await history('SPY'));
|
cache.set(prediction.instrument, await withTimeout(history(prediction.instrument),
|
||||||
|
HISTORY_HARD_TIMEOUT, `market data for ${prediction.instrument}`));
|
||||||
|
}
|
||||||
|
if (!cache.has('SPY')) {
|
||||||
|
cache.set('SPY', await withTimeout(history('SPY'), HISTORY_HARD_TIMEOUT, 'market data for SPY'));
|
||||||
|
}
|
||||||
const result = calculateOutcome(prediction, cache.get(prediction.instrument), cache.get('SPY'));
|
const result = calculateOutcome(prediction, cache.get(prediction.instrument), cache.get('SPY'));
|
||||||
if (!result) {
|
if (!result) {
|
||||||
// A null here almost always means the exit bar has not published yet, not
|
// A null here almost always means the exit bar has not published yet, not
|
||||||
|
|||||||
Reference in New Issue
Block a user