Separate bug from the cache, same symptom. The candidate query approximated maturity with '+horizon_days days', which is looser than the truth because 20 trading days is 28 calendar days. So the oldest hundred open rows were almost all horizon-30s that were not due yet, 98 of them, and those slots do not free up for weeks. Measured just now: 54 live predictions genuinely ready to score, zero of them inside the window. The worker sat there logging nothing at all. It only looked like it was working before because the three rows it could see were being retired as unresolvable every poll, which freed the slots. Fixing the cache stopped the retiring and revealed the stall underneath. 1.5x + 2 calendar days overshoots a trading day span at every horizon in use, so the limit now gets spent on rows that can plausibly be scored. The js trading day filter is still what decides. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
196 lines
10 KiB
JavaScript
196 lines
10 KiB
JavaScript
const os = require('os');
|
|
const https = require('https');
|
|
const { openRuntimeDb } = require('../src/db/runtime');
|
|
const { initAutonomySchema } = require('../src/autonomy/schema');
|
|
const { calculateOutcome, addTradingDays, yahooSymbol } = require('../src/autonomy/outcomes');
|
|
|
|
function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); }
|
|
function httpGet(url) {
|
|
return new Promise((resolve, reject) => {
|
|
const request = https.get(url, { headers: { 'User-Agent': 'duriin-autonomy/1.0' } }, (response) => {
|
|
let body = '';
|
|
response.setEncoding('utf8');
|
|
response.on('data', (chunk) => { body += chunk; });
|
|
response.on('end', () => response.statusCode >= 200 && response.statusCode < 300
|
|
? resolve(body) : reject(new Error(`market data returned ${response.statusCode}`)));
|
|
});
|
|
request.setTimeout(15000, () => request.destroy(new Error('market data timeout')));
|
|
request.on('error', reject);
|
|
});
|
|
}
|
|
|
|
const MAX_OUTCOME_ATTEMPTS = 5;
|
|
// req.setTimeout only covers socket inactivity. A response that opens and then
|
|
// stalls, or a socket that never emits anything at all, leaves the promise
|
|
// pending forever and the whole loop with it. This worker sat "Up 3 days" and
|
|
// silent while predictions it could resolve in 400ms went unscored, which is the
|
|
// third time an unbounded await in a long lived loop has quietly stopped a
|
|
// worker. This is the bound that cannot be skipped.
|
|
const HISTORY_HARD_TIMEOUT = 30000;
|
|
|
|
function withTimeout(promise, ms, label) {
|
|
let timer;
|
|
const expired = new Promise((_, reject) => {
|
|
timer = setTimeout(() => reject(new Error(`${label} exceeded ${ms}ms`)), ms);
|
|
});
|
|
return Promise.race([promise, expired]).finally(() => clearTimeout(timer));
|
|
}
|
|
// how far past the horizon we keep trying before accepting there is no data
|
|
const UNRESOLVABLE_GRACE_DAYS = 3;
|
|
|
|
// A 200 from yahoo is not the same thing as usable data. An empty or stub series
|
|
// got cached as if it were real, and because nothing in here ever expired, one
|
|
// bad benchmark fetch quietly poisoned every score this worker made for the rest
|
|
// of its life. Thats what stopped outcomes dead on the 20th while calculateOutcome
|
|
// was perfectly capable of scoring the rows it was busy retiring.
|
|
const MIN_HISTORY_BARS = 50;
|
|
const HISTORY_TTL_MS = 6 * 60 * 60 * 1000;
|
|
|
|
async function seriesFor(cache, symbol) {
|
|
const hit = cache.get(symbol);
|
|
if (hit && (Date.now() - hit.fetchedAt) < HISTORY_TTL_MS) return hit.bars;
|
|
|
|
const bars = await withTimeout(history(symbol), HISTORY_HARD_TIMEOUT, `market data for ${symbol}`);
|
|
if (!Array.isArray(bars) || bars.length < MIN_HISTORY_BARS) {
|
|
// never cache a dud. throwing puts it on the retry path instead
|
|
cache.delete(symbol);
|
|
throw new Error(`market data for ${symbol} came back with only ${bars ? bars.length : 0} usable bars`);
|
|
}
|
|
|
|
cache.set(symbol, { bars, fetchedAt: Date.now() });
|
|
return bars;
|
|
}
|
|
|
|
// the exit bar cannot exist yet if the series stops short of the horizon, and
|
|
// thats a hole in what we fetched rather than a prediction nobody can ever score
|
|
function coversHorizon(bars, horizonDate) {
|
|
return !!(bars && bars.length) && bars[bars.length - 1].date >= horizonDate;
|
|
}
|
|
|
|
async function history(symbol) {
|
|
// GDELT backfills predate the normal rolling quote window. Use an explicit
|
|
// point-in-time range so replay outcomes do not silently become unresolvable.
|
|
const url = `https://query1.finance.yahoo.com/v8/finance/chart/${encodeURIComponent(yahooSymbol(symbol))}?period1=946684800&period2=${Math.floor(Date.now() / 1000)}&interval=1d`;
|
|
const body = JSON.parse(await httpGet(url));
|
|
const result = body?.chart?.result?.[0];
|
|
if (!result) return [];
|
|
return (result.timestamp || []).map((timestamp, index) => ({
|
|
date: new Date(timestamp * 1000).toISOString().slice(0, 10),
|
|
close: result.indicators?.quote?.[0]?.close?.[index],
|
|
})).filter((row) => Number.isFinite(row.close));
|
|
}
|
|
|
|
async function resolveAutonomyOutcomes({ intelligencePath, workerId = `outcome-${os.hostname()}-${process.pid}`, pollMs = 60000 } = {}) {
|
|
const db = openRuntimeDb(intelligencePath, { schema: 'intelligence' });
|
|
db.pragma('journal_mode = WAL');
|
|
db.pragma('busy_timeout = 5000');
|
|
initAutonomySchema(db);
|
|
const cache = new Map();
|
|
const failures = new Map();
|
|
while (true) {
|
|
// The sql filter cannot do trading day arithmetic, so it approximates and the
|
|
// js filter below is still the authority on what is genuinely ready.
|
|
//
|
|
// It used to approximate the other way, '+horizon_days days', which is LOOSER
|
|
// than the truth since 20 trading days is 28 calendar days. The oldest 100 open
|
|
// rows were then almost all not-yet-due horizon-30s, 98 of them, and they sit
|
|
// there for weeks. 54 live predictions that were genuinely ready never got into
|
|
// the window at all and the worker quietly settled at zero scored per poll.
|
|
// 1.5x + 2 overshoots a trading day span at every horizon we use, so the limit
|
|
// gets spent on rows that can plausibly be scored. Costs a day or two of lag on
|
|
// the long horizons, which is a fair trade for not starving.
|
|
const candidates = db.prepare(`
|
|
SELECT p.* FROM autonomy_predictions p
|
|
LEFT JOIN autonomy_outcomes o ON o.prediction_id = p.id
|
|
WHERE p.status = 'open' AND o.prediction_id IS NULL
|
|
AND datetime(p.information_cutoff, '+' || CAST(p.horizon_days * 1.5 + 2 AS INTEGER) || ' days') <= datetime('now')
|
|
ORDER BY p.information_cutoff ASC LIMIT 100
|
|
`).all();
|
|
|
|
const today = new Date().toISOString().slice(0, 10);
|
|
const predictions = candidates.filter((p) => {
|
|
const horizonDate = addTradingDays(String(p.information_cutoff).slice(0, 10), p.horizon_days);
|
|
// strictly before today, so the exit session has closed and published
|
|
return horizonDate < today;
|
|
}).slice(0, 25);
|
|
|
|
if (predictions.length) {
|
|
console.log(`[autonomy-outcome] ${workerId} scoring ${predictions.length} matured predictions`
|
|
+ ` (${candidates.length} candidates)`);
|
|
}
|
|
for (const prediction of predictions) {
|
|
try {
|
|
const series = await seriesFor(cache, prediction.instrument);
|
|
const benchmark = await seriesFor(cache, "SPY");
|
|
const result = calculateOutcome(prediction, series, benchmark);
|
|
if (!result) {
|
|
// A null here almost always means the exit bar has not published yet, not
|
|
// that the prediction can never be scored. The sql due-check counts
|
|
// calendar days while the price lookup counts trading days, so a friday
|
|
// horizon-1 call looks due on saturday when monday's close cannot exist.
|
|
// Retiring it there permanently destroyed exactly the short-horizon live
|
|
// predictions we are waiting on. Wait until the horizon is properly past
|
|
// before giving up on it.
|
|
const horizonDate = addTradingDays(String(prediction.information_cutoff).slice(0, 10), prediction.horizon_days);
|
|
|
|
// Before blaming the prediction, check the prices we hold actually reach
|
|
// the horizon. Retiring everything that came back null is how roughly
|
|
// 400 scorable rows ended up labelled unresolvable, live ones included.
|
|
if (!coversHorizon(series, horizonDate) || !coversHorizon(benchmark, horizonDate)) {
|
|
cache.delete(prediction.instrument);
|
|
cache.delete("SPY");
|
|
console.error(`[autonomy-outcome] ${workerId} prediction ${prediction.id} (${prediction.instrument})`
|
|
+ ` left open, price data stops before horizon ${horizonDate}`);
|
|
continue;
|
|
}
|
|
|
|
const graceExpired = addTradingDays(horizonDate, UNRESOLVABLE_GRACE_DAYS) < new Date().toISOString().slice(0, 10);
|
|
if (graceExpired) {
|
|
db.prepare("UPDATE autonomy_predictions SET status = 'unresolvable' WHERE id = ?").run(prediction.id);
|
|
console.error(`[autonomy-outcome] ${workerId} prediction ${prediction.id} (${prediction.instrument})`
|
|
+ ` unresolvable: no market data ${UNRESOLVABLE_GRACE_DAYS} trading days past horizon ${horizonDate}`);
|
|
} else {
|
|
cache.delete(prediction.instrument);
|
|
}
|
|
continue;
|
|
}
|
|
db.prepare(`
|
|
INSERT INTO autonomy_outcomes
|
|
(prediction_id, price_0, price_horizon, benchmark_0, benchmark_horizon, excess_return, direction_correct, error_type)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(prediction_id) DO UPDATE SET
|
|
price_0=excluded.price_0,
|
|
price_horizon=excluded.price_horizon,
|
|
benchmark_0=excluded.benchmark_0,
|
|
benchmark_horizon=excluded.benchmark_horizon,
|
|
excess_return=excluded.excess_return,
|
|
direction_correct=excluded.direction_correct,
|
|
error_type=excluded.error_type,
|
|
evaluated_at=datetime('now')
|
|
`).run(prediction.id, result.price0, result.priceHorizon, result.benchmark0, result.benchmarkHorizon,
|
|
result.excessReturn, result.directionCorrect, result.directionCorrect ? null : 'direction_error');
|
|
db.prepare("UPDATE autonomy_predictions SET status = 'resolved' WHERE id = ?").run(prediction.id);
|
|
} catch (error) {
|
|
// A prediction that keeps failing stays 'open' and comes straight back on the
|
|
// next poll, so a symbol market data will never have just spins forever. Give
|
|
// it a few goes for genuinely transient failures, then retire it.
|
|
const attempts = (failures.get(prediction.id) || 0) + 1;
|
|
failures.set(prediction.id, attempts);
|
|
console.error(`[autonomy-outcome] ${workerId} prediction ${prediction.id} (${prediction.instrument})`
|
|
+ ` attempt ${attempts}/${MAX_OUTCOME_ATTEMPTS}:`, error.message);
|
|
if (attempts >= MAX_OUTCOME_ATTEMPTS) {
|
|
db.prepare("UPDATE autonomy_predictions SET status = 'unresolvable' WHERE id = ?").run(prediction.id);
|
|
failures.delete(prediction.id);
|
|
console.error(`[autonomy-outcome] ${workerId} prediction ${prediction.id} marked unresolvable`
|
|
+ ` after ${attempts} failed attempts on ${prediction.instrument}`);
|
|
}
|
|
cache.delete(prediction.instrument);
|
|
}
|
|
await sleep(800);
|
|
}
|
|
await sleep(pollMs);
|
|
}
|
|
}
|
|
|
|
module.exports = { calculateOutcome, resolveAutonomyOutcomes, yahooSymbol, coversHorizon };
|