fix: stop three loops that retry forever, and let budget dead letters recover
The event outcome worker re-requested PSTG and GROQ on every poll for as long as the process lived, because a fetch failure only logged and continued while the prediction stayed pending. Ten requests every five minutes, indefinitely. Per ticker backoff now doubles to an hour, so a symbol with no market data costs one request an hour instead of one a minute. It also translates dotted tickers the same way the autonomy worker does, which is why that helper moved into the shared price module rather than being copied. The gdelt loop had no pause on its error path at all, so once the api started refusing connections it spun through failures continuously, burning cpu and filling the log with the same stack. It has been doing that for days. Backs off to half an hour now and resets on success. isTransientCoordinatorFailure matched 408, 429 and 5xx but not a budget 402/403, so the 380 jobs that dead-lettered during the exhausted quota window could never come back on their own, including 55 live events. Budget failures are transient in a way an ordinary auth failure is not, and a wrong key still dies permanently because it says invalid or unauthorized rather than naming credits. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WnNxwxfXSbeNtjvtz5gayb
This commit is contained in:
+19
-1
@@ -11,7 +11,7 @@ const { validatePaperIntent, createSimulator } = require('../src/autonomy/execut
|
||||
const { calculateOutcome, addTradingDays } = require('../src/autonomy/outcomes');
|
||||
const { yahooSymbol } = require('../workers/outcomeAutonomyWorker');
|
||||
const { createOrderIntent } = require('../src/autonomy/orderIntents');
|
||||
const { enqueueCoordinatorEvent, reconcileArchiveBatch, reconcileLiveBatch } = require('../workers/autonomyWorker');
|
||||
const { enqueueCoordinatorEvent, reconcileArchiveBatch, reconcileLiveBatch, isTransientCoordinatorFailure } = require('../workers/autonomyWorker');
|
||||
const { scheduleNext } = require('../workers/replayWorker');
|
||||
const { refreshHistoricalCalibration, createDecisions, ensureCalibrationColumns } = require('../workers/calibrationWorker');
|
||||
|
||||
@@ -336,3 +336,21 @@ test('trading day arithmetic steps over weekends', () => {
|
||||
assert.equal(addTradingDays('2026-01-02', 5), '2026-01-09');
|
||||
assert.equal(addTradingDays('2026-01-02', 0), '2026-01-02');
|
||||
});
|
||||
|
||||
test('a budget failure is transient but a bad key is not', () => {
|
||||
const quota = 'Error: coordinator request failed with 403: {"error":{"message":"Key limit exceeded (monthly limit)."}}';
|
||||
const credits = 'LLM 402: {"error":{"message":"Insufficient credits. Add more using ..."}}';
|
||||
const afford = 'coordinator request failed with 402: can only afford 3921 tokens';
|
||||
assert.equal(isTransientCoordinatorFailure(quota), true);
|
||||
assert.equal(isTransientCoordinatorFailure(credits), true);
|
||||
assert.equal(isTransientCoordinatorFailure(afford), true);
|
||||
|
||||
// these must stay dead, retrying them forever helps nobody
|
||||
assert.equal(isTransientCoordinatorFailure('request failed with 403: invalid api key'), false);
|
||||
assert.equal(isTransientCoordinatorFailure('request failed with 401: unauthorized'), false);
|
||||
assert.equal(isTransientCoordinatorFailure('proposal references missing evidence'), false);
|
||||
|
||||
// and the pre-existing transient cases still are
|
||||
assert.equal(isTransientCoordinatorFailure('TypeError: fetch failed'), true);
|
||||
assert.equal(isTransientCoordinatorFailure('request failed with 503'), true);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user