feat: add isolated historical replay calibration

2026-08-04 22:00:11 +01:00
parent be18eb77f0
commit 5877783862
13 changed files with 308 additions and 17 deletions
@@ -34,9 +34,22 @@ function normalizeProposal(raw, { informationCutoff, model = 'unknown', promptVe
  };
 }

-function verifyEvidence(archiveDb, articleIds) {
+function verifyEvidence(archiveDb, articleIds, informationCutoff = null) {
  const placeholders = articleIds.map(() => '?').join(',');
-  const rows = archiveDb.prepare(`SELECT id FROM articles WHERE id IN (${placeholders})`).all(...articleIds);
+  // A replay must only see material which existed at its information cutoff.
+  // Live proposals retain the simpler existence check.
+  const cutoffClause = informationCutoff ? ' AND datetime(COALESCE(pub_date_effective, pub_date, ingested_at)) <= datetime(?)' : '';
+  let rows;
+  try {
+    rows = archiveDb.prepare(`SELECT id FROM articles WHERE id IN (${placeholders})${cutoffClause}`)
+      .all(...articleIds, ...(informationCutoff ? [informationCutoff] : []));
+  } catch (error) {
+    // Minimal/test archives may not retain publication metadata.  A production
+    // replay archive is required to have it, so this fallback is only for the
+    // existing live evidence contract.
+    if (informationCutoff) throw error;
+    rows = archiveDb.prepare(`SELECT id FROM articles WHERE id IN (${placeholders})`).all(...articleIds);
+  }
  const found = new Set(rows.map((row) => row.id));
  return articleIds.every((id) => found.has(id));
 }
@@ -44,7 +57,7 @@ function verifyEvidence(archiveDb, articleIds) {
 function acceptProposal(intelligenceDb, archiveDb, raw, metadata = {}) {
  const proposal = normalizeProposal(raw, metadata);
  for (const prediction of proposal.predictions) {
-    if (!verifyEvidence(archiveDb, prediction.evidenceArticleIds)) {
+    if (!verifyEvidence(archiveDb, prediction.evidenceArticleIds, metadata.origin === 'replay' ? proposal.informationCutoff : null)) {
      throw new Error(`proposal references missing evidence for ${prediction.instrument}`);
    }
    const instrument = intelligenceDb.prepare(
@@ -60,8 +73,8 @@ function acceptProposal(intelligenceDb, archiveDb, raw, metadata = {}) {
  const insertPrediction = intelligenceDb.prepare(`
      INSERT INTO autonomy_predictions
      (proposal_id, event_id, instrument, direction, event_type, causal_channel,
-       horizon_days, information_cutoff, evidence_article_ids, invalidation_condition, learning_eligible, strategy_version)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+       horizon_days, information_cutoff, evidence_article_ids, invalidation_condition, learning_eligible, strategy_version, origin, replay_run_id)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
  `);
  const tx = intelligenceDb.transaction(() => {
    const proposalResult = insert.run(metadata.eventId || null, JSON.stringify(proposal), proposal.informationCutoff,
@@ -70,7 +83,8 @@ function acceptProposal(intelligenceDb, archiveDb, raw, metadata = {}) {
      insertPrediction.run(proposalResult.lastInsertRowid, metadata.eventId || null, prediction.instrument,
        prediction.direction, prediction.eventType, prediction.causalChannel, prediction.horizonDays,
        proposal.informationCutoff, JSON.stringify(prediction.evidenceArticleIds), prediction.invalidationCondition,
-        metadata.learningEligible ? 1 : 0, metadata.strategyVersion || 'autonomy-1');
+        metadata.learningEligible ? 1 : 0, metadata.strategyVersion || 'autonomy-1',
+        metadata.origin || 'live', metadata.replayRunId || null);
    }
    return Number(proposalResult.lastInsertRowid);
  });
@@ -1,4 +1,4 @@
-const AUTONOMY_SCHEMA_VERSION = 1;
+const AUTONOMY_SCHEMA_VERSION = 2;

 function initAutonomySchema(db) {
  db.exec(`
@@ -178,12 +178,52 @@ function initAutonomySchema(db) {
      captured_at TEXT NOT NULL DEFAULT (datetime('now'))
    );

+    -- Historical replay is a separate evidence path.  It deliberately never
+    -- writes to autonomy_decisions or order intents.
+    CREATE TABLE IF NOT EXISTS autonomy_replay_runs (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      status TEXT NOT NULL DEFAULT 'running'
+        CHECK (status IN ('running', 'paused', 'complete', 'superseded', 'failed')),
+      start_at TEXT,
+      watermark_at TEXT,
+      cursor_article_id INTEGER NOT NULL DEFAULT 0,
+      cursor_effective_at TEXT,
+      processed_articles INTEGER NOT NULL DEFAULT 0,
+      strategy_version TEXT NOT NULL,
+      prompt_version TEXT NOT NULL,
+      coordinator_model TEXT,
+      price_provider TEXT NOT NULL DEFAULT 'yahoo',
+      created_at TEXT NOT NULL DEFAULT (datetime('now')),
+      updated_at TEXT NOT NULL DEFAULT (datetime('now')),
+      completed_at TEXT,
+      last_error TEXT
+    );
+    CREATE INDEX IF NOT EXISTS idx_autonomy_replay_runs_active
+      ON autonomy_replay_runs(status, cursor_article_id);
+
+    CREATE TABLE IF NOT EXISTS autonomy_replay_evaluations (
+      prediction_id INTEGER PRIMARY KEY REFERENCES autonomy_predictions(id),
+      replay_run_id INTEGER NOT NULL REFERENCES autonomy_replay_runs(id),
+      snapshot_cutoff TEXT NOT NULL,
+      sample_size INTEGER NOT NULL,
+      action TEXT NOT NULL CHECK (action IN ('BUY', 'SELL', 'HOLD', 'ABSTAIN')),
+      calibrated_probability REAL,
+      expected_excess_return REAL,
+      rationale TEXT NOT NULL,
+      created_at TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+
    INSERT OR IGNORE INTO autonomy_schema(version) VALUES (${AUTONOMY_SCHEMA_VERSION});
  `);
  for (const statement of [
    'ALTER TABLE autonomy_order_intents ADD COLUMN attempts INTEGER NOT NULL DEFAULT 0',
    'ALTER TABLE autonomy_order_intents ADD COLUMN last_error TEXT',
    'ALTER TABLE autonomy_predictions ADD COLUMN learning_eligible INTEGER NOT NULL DEFAULT 0',
+    "ALTER TABLE autonomy_predictions ADD COLUMN origin TEXT NOT NULL DEFAULT 'live'",
+    'ALTER TABLE autonomy_predictions ADD COLUMN replay_run_id INTEGER',
+    'ALTER TABLE autonomy_replay_runs ADD COLUMN cursor_effective_at TEXT',
+    "ALTER TABLE autonomy_calibration_snapshots ADD COLUMN source TEXT NOT NULL DEFAULT 'live'",
+    'ALTER TABLE autonomy_calibration_snapshots ADD COLUMN replay_run_id INTEGER',
  ]) {
    try { db.exec(statement); } catch (_) {}
  }
@@ -174,7 +174,9 @@ async function adminRoutes(fastify) {
    const outcomeSummary = intelligenceDb.prepare(`
      SELECT COUNT(*) AS total, SUM(direction_correct) AS correct,
             AVG(excess_return) AS average_excess_return
-      FROM autonomy_outcomes
+      FROM autonomy_outcomes o
+      JOIN autonomy_predictions p ON p.id = o.prediction_id
+      WHERE p.origin = 'live'
    `).get();
    const instruments = intelligenceDb.prepare(`
      SELECT COUNT(*) AS count FROM autonomy_instruments WHERE active=1 AND tradable=1
@@ -214,6 +216,18 @@ async function adminRoutes(fastify) {
             expected_excess_return, lower_return, upper_return, created_at
      FROM autonomy_calibration_snapshots ORDER BY id DESC LIMIT 8
    `).all();
+    const replay = intelligenceDb.prepare(`
+      SELECT r.id, r.status, r.watermark_at, r.cursor_article_id, r.cursor_effective_at,
+             r.processed_articles, r.updated_at,
+             SUM(CASE WHEN p.status = 'resolved' THEN 1 ELSE 0 END) AS resolved_predictions,
+             COUNT(p.id) AS predictions,
+             SUM(o.direction_correct) AS correct_predictions,
+             (SELECT COUNT(*) FROM autonomy_replay_evaluations e WHERE e.replay_run_id = r.id) AS evaluations
+      FROM autonomy_replay_runs r
+      LEFT JOIN autonomy_predictions p ON p.replay_run_id = r.id
+      LEFT JOIN autonomy_outcomes o ON o.prediction_id = p.id
+      GROUP BY r.id ORDER BY r.id DESC LIMIT 1
+    `).get() || null;

    return {
      enabled: true,
@@ -232,6 +246,7 @@ async function adminRoutes(fastify) {
      latestOrders,
      account,
      calibration,
+      replay,
      generatedAt: new Date().toISOString(),
    };
  });