IMBENJI.NET/Duriin-API
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add isolated historical replay calibration

Browse Source
This commit is contained in:
ImBenji
2026-08-04 22:00:11 +01:00
parent be18eb77f0
commit 5877783862
13 changed files with 308 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
workers/calibrationWorker.js
+45 -6
View File
@@ -11,7 +11,7 @@ function refreshCalibration(db, version = `cal-${Date.now()}`) {
SELECT p.direction, p.event_type, p.horizon_days, o.*
FROM autonomy_predictions p
JOIN autonomy_outcomes o ON o.prediction_id = p.id
WHERE p.status = 'resolved' AND p.learning_eligible = 1
WHERE p.status = 'resolved' AND p.learning_eligible = 1 AND p.origin = 'live'
`).all().reduce((map, row) => {
const key = cohortKey({ direction: row.direction, eventType: row.event_type, horizonDays: row.horizon_days });
if (!map.has(key)) map.set(key, []);
@@ -21,8 +21,8 @@ function refreshCalibration(db, version = `cal-${Date.now()}`) {
const insert = db.prepare(`
INSERT INTO autonomy_calibration_snapshots
(cohort_key, sample_size, effective_sample_size, directional_probability,
expected_excess_return, lower_return, upper_return, parent_cohort_key, version)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
expected_excess_return, lower_return, upper_return, parent_cohort_key, version, source, replay_run_id)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'live', NULL)
`);
const tx = db.transaction(() => {
for (const [key, rows] of groups) {
@@ -40,7 +40,7 @@ function createDecisions(db, strategyVersion = 'autonomy-1') {
const predictions = db.prepare(`
SELECT p.* FROM autonomy_predictions p
LEFT JOIN autonomy_decisions d ON d.prediction_id = p.id
WHERE d.prediction_id IS NULL AND p.status = 'open'
WHERE d.prediction_id IS NULL AND p.status = 'open' AND p.origin = 'live'
`).all();
const latest = db.prepare(`
SELECT * FROM autonomy_calibration_snapshots
@@ -68,6 +68,44 @@ function createDecisions(db, strategyVersion = 'autonomy-1') {
return created;
}
// Replay evaluations are walk-forward: each historical prediction is scored
// against calibration data that had matured strictly before its cutoff. They
// are stored in their own ledger, never in autonomy_decisions.
function refreshReplayEvaluations(db) {
const predictions = db.prepare(`
SELECT p.*, o.excess_return, o.direction_correct
FROM autonomy_predictions p JOIN autonomy_outcomes o ON o.prediction_id = p.id
LEFT JOIN autonomy_replay_evaluations e ON e.prediction_id = p.id
WHERE p.origin = 'replay' AND p.status = 'resolved' AND e.prediction_id IS NULL
ORDER BY datetime(p.information_cutoff), p.id LIMIT 200
`).all();
const prior = db.prepare(`
SELECT p.direction, p.event_type, p.horizon_days, o.*
FROM autonomy_predictions p JOIN autonomy_outcomes o ON o.prediction_id = p.id
WHERE p.origin = 'replay' AND p.status = 'resolved'
AND datetime(p.information_cutoff, '+' || p.horizon_days || ' days') < datetime(?)
`);
const insert = db.prepare(`
INSERT INTO autonomy_replay_evaluations
(prediction_id, replay_run_id, snapshot_cutoff, sample_size, action, calibrated_probability, expected_excess_return, rationale)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
`);
const tx = db.transaction(() => {
for (const prediction of predictions) {
const key = cohortKey({ direction: prediction.direction, eventType: prediction.event_type, horizonDays: prediction.horizon_days });
const rows = prior.all(prediction.information_cutoff).filter((row) =>
cohortKey({ direction: row.direction, eventType: row.event_type, horizonDays: row.horizon_days }) === key);
const calibration = rows.length ? calibrateOutcomes(rows) : null;
const decision = calibration ? decide({ ...calibration, direction: prediction.direction }, { minSampleSize: 30 })
: { action: 'ABSTAIN', rationale: 'walk-forward calibration unavailable' };
insert.run(prediction.id, prediction.replay_run_id, prediction.information_cutoff, rows.length, decision.action,
calibration?.directionalProbability || null, calibration?.expectedExcessReturn || null, decision.rationale);
}
});
tx();
return predictions.length;
}
async function runCalibrationWorker({ intelligencePath, pollMs = 60000, workerId = `calibration-${os.hostname()}-${process.pid}` } = {}) {
const db = new Database(intelligencePath);
db.pragma('journal_mode = WAL');
@@ -78,7 +116,8 @@ async function runCalibrationWorker({ intelligencePath, pollMs = 60000, workerId
const state = db.prepare('SELECT COUNT(*) AS count, COALESCE(MAX(prediction_id), 0) AS max_id FROM autonomy_outcomes').get();
const groups = refreshCalibration(db, `cal-${state.count}-${state.max_id}`);
const decisions = createDecisions(db);
if (groups || decisions) console.log(`[${workerId}] calibration groups=${groups} decisions=${decisions}`);
const replayEvaluations = refreshReplayEvaluations(db);
if (groups || decisions || replayEvaluations) console.log(`[${workerId}] calibration groups=${groups} decisions=${decisions} replay_evaluations=${replayEvaluations}`);
} catch (error) {
console.error(`[${workerId}] calibration error:`, error.message);
}
@@ -86,4 +125,4 @@ async function runCalibrationWorker({ intelligencePath, pollMs = 60000, workerId
}
}
module.exports = { refreshCalibration, createDecisions, runCalibrationWorker };
module.exports = { refreshCalibration, createDecisions, refreshReplayEvaluations, runCalibrationWorker };