fix: restart the stalled autonomy pipeline and make calibration honest

Archive ingestion had been dead since 2026-08-02 because nothing in the compose stack actually ran it. Everything downstream starved from there. - add ingest + enrichment services. server.js only starts the scheduler when DURIIN_RUN_SCHEDULER is not "false", and workers/index.js was not running at all, so articles never got event_id/content/has_embedding and the coordinator had nothing to lease. - pass an explicit origin from coordinatorWorker. it was never passed, so acceptProposal defaulted to 'live' and 464 historical backfill predictions were recorded as live. that also meant verifyEvidence got a null cutoff and skipped its date check entirely. - coarsen cohortKey to event families + horizon buckets. 201 free text event types produced 221 cohorts averaging 2.76 samples, so the n>=30 gate could never be reached and everything abstained for the wrong reason. - gate on cohort diversity, not just sample count. one ticker was roughly half of all resolved outcomes, so a pure count gate was measuring one company. unknown diversity abstains rather than passing. - resolve the admin archive db explicitly and probe it. it relied on a Dockerfile symlink, and without it better-sqlite3 quietly creates an empty file and serves a phantom archive. - clamp implausible future publication dates at ingest. - pin the db backend to sqlite by default. compose hardcoded postgres "true", which would have overridden the operator's own .env on the next redeploy and pointed everything at a stale snapshot. scripts/repair-autonomy-labels.js relabels the affected rows. it is dry run by default and has not been applied. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WnNxwxfXSbeNtjvtz5gayb
2026-08-29 21:43:24 +01:00
parent d778a02bfb
commit 6f1d1eee2d
19 changed files with 1497 additions and 100 deletions
@@ -12,7 +12,7 @@ const { calculateOutcome } = require('../src/autonomy/outcomes');
 const { createOrderIntent } = require('../src/autonomy/orderIntents');
 const { enqueueCoordinatorEvent, reconcileArchiveBatch, reconcileLiveBatch } = require('../workers/autonomyWorker');
 const { scheduleNext } = require('../workers/replayWorker');
-const { refreshHistoricalCalibration, createDecisions } = require('../workers/calibrationWorker');
+const { refreshHistoricalCalibration, createDecisions, ensureCalibrationColumns } = require('../workers/calibrationWorker');

 test('autonomy schema and leased jobs are restart-safe', () => {
  const db = new Database(':memory:');
@@ -117,8 +117,11 @@ test('calibration and policy abstain on insufficient evidence', () => {
  assert.equal(calibration.sampleSize, 2);
  const result = decide({ ...calibration }, { minSampleSize: 30 });
  assert.equal(result.action, 'ABSTAIN');
-  assert.equal(cohortKey({ sector: 'tech', eventType: 'earnings', horizonDays: 10, direction: 'positive' }), 'tech|earnings|10|positive');
-  assert.equal(decide({ direction: 'negative', probability: 0.8, expectedExcessReturn: -0.02, lowerReturn: -0.04, sampleSize: 40 }).action, 'SELL');
+  assert.equal(cohortKey({ sector: 'tech', eventType: 'earnings', horizonDays: 10, direction: 'positive' }), 'v2|tech|earnings|medium|positive');
+  assert.equal(decide({
+    direction: 'negative', probability: 0.8, expectedExcessReturn: -0.02, lowerReturn: -0.04,
+    sampleSize: 40, distinctInstruments: 9, topInstrumentShare: 0.25,
+  }).action, 'SELL');
 });

 test('historical replay outcomes create replay calibration snapshots', () => {
@@ -136,8 +139,9 @@ test('historical replay outcomes create replay calibration snapshots', () => {
    VALUES (?, 0.04, 1)
  `).run(prediction.lastInsertRowid);

-  assert.equal(refreshHistoricalCalibration(db, 'test-cal'), 1);
-  const snapshot = db.prepare("SELECT source, replay_run_id, sample_size, directional_probability FROM autonomy_calibration_snapshots").get();
+  // one per-run replay snapshot plus the pooled historical/replay snapshot
+  assert.equal(refreshHistoricalCalibration(db, 'test-cal'), 2);
+  const snapshot = db.prepare("SELECT source, replay_run_id, sample_size, directional_probability FROM autonomy_calibration_snapshots WHERE source='replay'").get();
  assert.equal(snapshot.source, 'replay');
  assert.equal(snapshot.replay_run_id, 7);
  assert.equal(snapshot.sample_size, 1);
@@ -154,12 +158,14 @@ test('live decisions map calibration snapshot fields into policy inputs', () =>
       learning_eligible, strategy_version, origin, status)
    VALUES (?, 'NVDA', 'positive', 'earnings', 10, datetime('now'), '[1]', 1, 'test', 'live', 'open')
  `).run(proposal.lastInsertRowid);
+  ensureCalibrationColumns(db);
  db.prepare(`
    INSERT INTO autonomy_calibration_snapshots
      (cohort_key, sample_size, effective_sample_size, directional_probability, expected_excess_return,
       lower_return, upper_return, parent_cohort_key, version, source)
-    VALUES ('unknown|earnings|10|positive', 40, 42, 0.7, 0.02, -0.01, 0.06, NULL, 'test-cal', 'replay')
+    VALUES ('v2|unknown|earnings|medium|positive', 40, 42, 0.7, 0.02, -0.01, 0.06, NULL, 'test-cal', 'historical')
  `).run();
+  db.prepare("UPDATE autonomy_calibration_snapshots SET distinct_instruments = 11, top_instrument_share = 0.2").run();

  assert.equal(createDecisions(db), 1);
  const decision = db.prepare('SELECT * FROM autonomy_decisions WHERE prediction_id=?').get(prediction.lastInsertRowid);
@@ -0,0 +1,168 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+
+const {
+  normalizeEventType,
+  horizonBucket,
+  cohortKey,
+  legacyCohortKey,
+  calibrateOutcomes,
+  EVENT_FAMILY_NAMES,
+} = require('../src/autonomy/calibration');
+const { decide, DEFAULT_POLICY_RULES } = require('../src/autonomy/policy');
+
+test('event types collapse onto a small closed set of families', () => {
+  assert.equal(normalizeEventType('earnings_beat'), 'earnings');
+  assert.equal(normalizeEventType('Q3 Earnings Report'), 'earnings');
+  assert.equal(normalizeEventType('guidance_raise'), 'guidance');
+  assert.equal(normalizeEventType('supply_constraint'), 'supply_chain');
+  assert.equal(normalizeEventType('supplyConstraint'), 'supply_chain');
+  assert.equal(normalizeEventType('antitrust probe'), 'regulatory');
+  assert.equal(normalizeEventType('ceo_resignation'), 'leadership');
+  assert.equal(normalizeEventType('analyst-downgrade'), 'analyst_action');
+  assert.equal(normalizeEventType('share buyback'), 'capital');
+  assert.equal(normalizeEventType('data breach'), 'security_incident');
+  assert.equal(normalizeEventType('interest rate decision'), 'macro');
+  assert.equal(normalizeEventType('product_launch'), 'product');
+  assert.equal(normalizeEventType('acquisition_rumor'), 'm_and_a');
+  assert.equal(normalizeEventType('patent lawsuit'), 'legal');
+});
+
+test('unrecognised or empty event types fall back to other, never to their own cohort', () => {
+  assert.equal(normalizeEventType('zebra_convention'), 'other');
+  assert.equal(normalizeEventType(''), 'other');
+  assert.equal(normalizeEventType('   '), 'other');
+  assert.equal(normalizeEventType(null), 'other');
+  assert.equal(normalizeEventType(undefined), 'other');
+  assert.equal(normalizeEventType(42), 'other');
+  assert.ok(EVENT_FAMILY_NAMES.includes('other'));
+  assert.ok(EVENT_FAMILY_NAMES.length <= 15, `taxonomy grew to ${EVENT_FAMILY_NAMES.length} families`);
+});
+
+test('every allowed horizon lands in one of three buckets', () => {
+  assert.equal(horizonBucket(1), 'short');
+  assert.equal(horizonBucket(5), 'short');
+  assert.equal(horizonBucket(10), 'medium');
+  assert.equal(horizonBucket(20), 'medium');
+  assert.equal(horizonBucket(30), 'long');
+  assert.equal(horizonBucket(60), 'long');
+  assert.equal(horizonBucket(90), 'long');
+  assert.equal(horizonBucket(null), 'unknown');
+  assert.equal(horizonBucket('nope'), 'unknown');
+});
+
+test('cohort key is versioned, coarse and stable, and the legacy key is still available', () => {
+  assert.equal(
+    cohortKey({ direction: 'positive', eventType: 'earnings_beat', horizonDays: 10 }),
+    'v2|unknown|earnings|medium|positive'
+  );
+  // different raw event text, same family and horizon bucket -> same cohort
+  assert.equal(
+    cohortKey({ direction: 'positive', eventType: 'quarterly results miss', horizonDays: 20 }),
+    cohortKey({ direction: 'positive', eventType: 'earnings_beat', horizonDays: 10 })
+  );
+  assert.notEqual(
+    cohortKey({ direction: 'negative', eventType: 'earnings_beat', horizonDays: 10 }),
+    cohortKey({ direction: 'positive', eventType: 'earnings_beat', horizonDays: 10 })
+  );
+  assert.equal(
+    legacyCohortKey({ direction: 'positive', eventType: 'earnings_beat', horizonDays: 10 }),
+    'unknown|earnings_beat|10|positive'
+  );
+});
+
+test('the coarse taxonomy actually collapses a realistic spread of free text', () => {
+  const raw = [
+    'earnings_beat', 'earnings_miss', 'q2_earnings', 'revenue_growth', 'margin_expansion',
+    'guidance_raise', 'guidance_cut', 'outlook_downgrade', 'profit_warning',
+    'supply_constraint', 'chip_shortage', 'production_halt', 'capacity_expansion',
+    'analyst_upgrade', 'price_target_raise', 'ceo_departure', 'board_shakeup',
+    'antitrust_probe', 'export_controls', 'tariff_announcement',
+  ];
+  const families = new Set(raw.map(normalizeEventType));
+  assert.ok(families.size <= 8, `expected heavy collapse, got ${families.size} families`);
+});
+
+test('calibrateOutcomes reports instrument diversity and concentration', () => {
+  const rows = [
+    { excess_return: 0.02, direction_correct: 1, instrument: 'NVDA' },
+    { excess_return: 0.01, direction_correct: 1, instrument: 'nvda' },
+    { excess_return: -0.01, direction_correct: 0, instrument: 'NVDA' },
+    { excess_return: 0.03, direction_correct: 1, instrument: 'AMD' },
+  ];
+  const result = calibrateOutcomes(rows);
+  assert.equal(result.sampleSize, 4);
+  assert.equal(result.distinctInstruments, 2);
+  assert.equal(result.topInstrumentShare, 0.75);
+
+  const empty = calibrateOutcomes([]);
+  assert.equal(empty.distinctInstruments, 0);
+  assert.equal(empty.topInstrumentShare, null);
+
+  const unlabelled = calibrateOutcomes([{ excess_return: 0.01, direction_correct: 1 }]);
+  assert.equal(unlabelled.distinctInstruments, 0);
+});
+
+test('the diversification gate blocks single ticker cohorts however large they are', () => {
+  const base = { direction: 'positive', probability: 0.8, expectedExcessReturn: 0.02, lowerReturn: -0.01 };
+
+  // 300 samples, one name: this is the NVDA case, and it must not qualify
+  const concentrated = decide({ ...base, sampleSize: 300, distinctInstruments: 1, topInstrumentShare: 1 });
+  assert.equal(concentrated.action, 'ABSTAIN');
+  assert.match(concentrated.rationale, /diversity/);
+
+  // enough names but still dominated by one of them
+  const dominated = decide({ ...base, sampleSize: 300, distinctInstruments: 9, topInstrumentShare: 0.82 });
+  assert.equal(dominated.action, 'ABSTAIN');
+  assert.match(dominated.rationale, /dominated/);
+
+  // pre-diversification snapshots carry no count, unknown is not adequate
+  const unknown = decide({ ...base, sampleSize: 300, distinctInstruments: null, topInstrumentShare: null });
+  assert.equal(unknown.action, 'ABSTAIN');
+  assert.match(unknown.rationale, /unknown/);
+
+  const qualified = decide({ ...base, sampleSize: 40, distinctInstruments: 9, topInstrumentShare: 0.3 });
+  assert.equal(qualified.action, 'BUY');
+});
+
+test('both evidence thresholds are overridable and default conservatively', () => {
+  assert.equal(DEFAULT_POLICY_RULES.minSampleSize, 30);
+  assert.equal(DEFAULT_POLICY_RULES.minDistinctInstruments, 5);
+  assert.equal(DEFAULT_POLICY_RULES.maxInstrumentConcentration, 0.5);
+
+  const input = {
+    direction: 'positive', probability: 0.8, expectedExcessReturn: 0.02, lowerReturn: -0.01,
+    sampleSize: 12, distinctInstruments: 3, topInstrumentShare: 0.4,
+  };
+  assert.equal(decide(input).action, 'ABSTAIN');
+  assert.equal(decide(input, { minSampleSize: 10, minDistinctInstruments: 2 }).action, 'BUY');
+  assert.equal(decide(input, { minSampleSize: 10, minDistinctInstruments: 2, maxInstrumentConcentration: 0.3 }).action, 'ABSTAIN');
+});
+
+test('sample size gate still runs before the diversity gate', () => {
+  const result = decide({
+    direction: 'positive', probability: 0.9, expectedExcessReturn: 0.05,
+    sampleSize: 2, distinctInstruments: 40, topInstrumentShare: 0.1,
+  });
+  assert.equal(result.action, 'ABSTAIN');
+  assert.match(result.rationale, /insufficient calibration sample/);
+});
+
+test('a missing concentration share cannot sneak past the cap as a zero', () => {
+  const base = {
+    direction: 'positive', probability: 0.8, expectedExcessReturn: 0.02,
+    lowerReturn: -0.01, sampleSize: 300, distinctInstruments: 40,
+  };
+
+  // Number(null) is 0, which used to slide straight under the cap even though we
+  // had no idea what the real concentration was.
+  for (const share of [null, undefined]) {
+    const verdict = decide({ ...base, topInstrumentShare: share });
+    assert.equal(verdict.action, 'ABSTAIN');
+    assert.match(verdict.rationale, /concentration unknown/);
+  }
+
+  // a genuinely broad cohort still gets through, we havent just bolted it shut
+  const broad = decide({ ...base, topInstrumentShare: 0.12 });
+  assert.equal(broad.action, 'BUY');
+});
@@ -0,0 +1,112 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const Database = require('better-sqlite3');
+
+const { initAutonomySchema } = require('../src/autonomy/schema');
+const { cohortKey } = require('../src/autonomy/calibration');
+const {
+  refreshCalibration,
+  refreshHistoricalCalibration,
+  createDecisions,
+  calibrationHealth,
+} = require('../workers/calibrationWorker');
+
+function seedDb() {
+  const db = new Database(':memory:');
+  initAutonomySchema(db);
+  db.prepare("INSERT INTO autonomy_proposals(payload, information_cutoff, status) VALUES ('{}', '2026-01-01T00:00:00Z', 'accepted')").run();
+  return db;
+}
+
+function addPrediction(db, { instrument, direction = 'positive', eventType = 'earnings_beat', horizonDays = 10,
+  origin = 'live', status = 'resolved', learningEligible = 0, replayRunId = null, excessReturn = null, correct = null }) {
+  const prediction = db.prepare(`
+    INSERT INTO autonomy_predictions
+      (proposal_id, instrument, direction, event_type, horizon_days, information_cutoff, evidence_article_ids,
+       learning_eligible, strategy_version, origin, replay_run_id, status)
+    VALUES (1, ?, ?, ?, ?, '2026-01-01T00:00:00Z', '[1]', ?, 'test', ?, ?, ?)
+  `).run(instrument, direction, eventType, horizonDays, learningEligible, origin, replayRunId, status);
+  if (excessReturn !== null) {
+    db.prepare('INSERT INTO autonomy_outcomes(prediction_id, excess_return, direction_correct) VALUES (?, ?, ?)')
+      .run(prediction.lastInsertRowid, excessReturn, correct);
+  }
+  return prediction.lastInsertRowid;
+}
+
+test('live calibration no longer starves on the never-set learning_eligible flag', () => {
+  const db = seedDb();
+  addPrediction(db, { instrument: 'NVDA', excessReturn: 0.03, correct: 1 });
+  addPrediction(db, { instrument: 'AMD', excessReturn: -0.01, correct: 0 });
+
+  assert.equal(refreshCalibration(db, 'live-cal'), 1);
+  const snapshot = db.prepare("SELECT * FROM autonomy_calibration_snapshots WHERE source='live'").get();
+  assert.equal(snapshot.sample_size, 2);
+  assert.equal(snapshot.distinct_instruments, 2);
+
+  // the old behaviour is still reachable on purpose, for once the flag is populated
+  assert.equal(refreshCalibration(db, 'strict-cal', { requireLearningEligible: true }), 0);
+
+  // counters report snapshots written, so a steady state poll is genuinely quiet
+  assert.equal(refreshCalibration(db, 'live-cal'), 0);
+  assert.equal(db.prepare("SELECT COUNT(*) c FROM autonomy_calibration_snapshots WHERE source='live'").get().c, 1);
+});
+
+test('historical calibration pools origin historical and replay together', () => {
+  const db = seedDb();
+  addPrediction(db, { instrument: 'NVDA', origin: 'historical', excessReturn: 0.02, correct: 1 });
+  addPrediction(db, { instrument: 'AMD', origin: 'historical', excessReturn: 0.01, correct: 1 });
+  addPrediction(db, { instrument: 'INTC', origin: 'replay', replayRunId: 3, excessReturn: -0.02, correct: 0 });
+
+  refreshHistoricalCalibration(db, 'hist-cal');
+  const pooled = db.prepare("SELECT * FROM autonomy_calibration_snapshots WHERE source='historical'").get();
+  assert.equal(pooled.sample_size, 3, 'the historical lane must not drop the relabelled rows');
+  assert.equal(pooled.distinct_instruments, 3);
+  const perRun = db.prepare("SELECT * FROM autonomy_calibration_snapshots WHERE source='replay'").get();
+  assert.equal(perRun.replay_run_id, 3);
+  assert.equal(perRun.sample_size, 1);
+});
+
+test('decisions are only written for open live predictions', () => {
+  const db = seedDb();
+  const open = addPrediction(db, { instrument: 'NVDA', status: 'open' });
+  addPrediction(db, { instrument: 'AMD', status: 'resolved', excessReturn: 0.01, correct: 1 });
+  addPrediction(db, { instrument: 'INTC', status: 'open', origin: 'historical' });
+  addPrediction(db, { instrument: 'MU', status: 'open', origin: 'replay', replayRunId: 3 });
+
+  assert.equal(createDecisions(db), 1);
+  const rows = db.prepare('SELECT prediction_id, action FROM autonomy_decisions').all();
+  assert.equal(rows.length, 1);
+  assert.equal(rows[0].prediction_id, open);
+  assert.equal(rows[0].action, 'ABSTAIN');
+  // second pass must not duplicate
+  assert.equal(createDecisions(db), 0);
+});
+
+test('a big single ticker historical cohort still cannot authorise a live buy', () => {
+  const db = seedDb();
+  for (let index = 0; index < 60; index++) {
+    addPrediction(db, { instrument: 'NVDA', origin: 'historical', excessReturn: 0.04, correct: 1 });
+  }
+  refreshHistoricalCalibration(db, 'hist-cal');
+  const prediction = addPrediction(db, { instrument: 'NVDA', status: 'open' });
+
+  assert.equal(createDecisions(db), 1);
+  const decision = db.prepare('SELECT * FROM autonomy_decisions WHERE prediction_id=?').get(prediction);
+  assert.equal(decision.action, 'ABSTAIN');
+  assert.match(decision.rationale, /diversity|dominated/);
+  assert.match(decision.rationale, new RegExp(cohortKey({ direction: 'positive', eventType: 'earnings_beat', horizonDays: 10 }).replace(/\|/g, '\\|')));
+});
+
+test('calibration health reports the stall instead of staying silent', () => {
+  const db = seedDb();
+  addPrediction(db, { instrument: 'NVDA', origin: 'historical', excessReturn: 0.02, correct: 1 });
+  addPrediction(db, { instrument: 'AMD', status: 'open' });
+  refreshHistoricalCalibration(db, 'hist-cal');
+
+  const health = calibrationHealth(db);
+  assert.equal(health.liveOpen, 1);
+  assert.equal(health.offlineResolved, 1);
+  assert.equal(health.learningEligible, 0);
+  assert.ok(health.cohorts >= 1);
+  assert.equal(health.qualifyingCohorts, 0);
+});
@@ -0,0 +1,74 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+
+const { checkPubDate, guardEffectivePubDate, DEFAULT_TOLERANCE_MS } = require('../src/pubDateGuard');
+
+const NOW = Date.parse('2026-08-29T12:00:00.000Z');
+const HOUR = 60 * 60 * 1000;
+
+test('ordinary past publication dates pass straight through', () => {
+  const verdict = checkPubDate('2026-08-27T09:30:00.000Z', NOW);
+  assert.equal(verdict.ok, true);
+  assert.equal(verdict.value, '2026-08-27T09:30:00.000Z');
+});
+
+test('a date-only feed value from an eastern timezone is still accepted', () => {
+  // "2026-08-30" stored as midnight UTC is 12 hours ahead of now — legitimate
+  const verdict = checkPubDate('2026-08-30T00:00:00.000Z', NOW);
+  assert.equal(verdict.ok, true);
+});
+
+test('mild clock skew inside the tolerance is accepted', () => {
+  const verdict = checkPubDate(new Date(NOW + 47 * HOUR).toISOString(), NOW);
+  assert.equal(verdict.ok, true);
+});
+
+test('anything past the tolerance is rejected', () => {
+  const verdict = checkPubDate(new Date(NOW + 49 * HOUR).toISOString(), NOW);
+  assert.equal(verdict.ok, false);
+  assert.equal(verdict.value, null);
+  assert.ok(verdict.skewMs > DEFAULT_TOLERANCE_MS);
+});
+
+test('the real production offender is caught', () => {
+  const verdict = checkPubDate('2026-12-22T00:00:00.000Z', NOW);
+  assert.equal(verdict.ok, false);
+});
+
+test('missing and unparseable dates are not treated as future dates', () => {
+  assert.equal(checkPubDate(null, NOW).ok, true);
+  assert.equal(checkPubDate('', NOW).ok, true);
+  assert.equal(checkPubDate('not a date at all', NOW).ok, true);
+  assert.equal(checkPubDate('not a date at all', NOW).value, null);
+});
+
+test('the tolerance boundary itself is inclusive', () => {
+  assert.equal(checkPubDate(new Date(NOW + DEFAULT_TOLERANCE_MS).toISOString(), NOW).ok, true);
+  assert.equal(checkPubDate(new Date(NOW + DEFAULT_TOLERANCE_MS + 1).toISOString(), NOW).ok, false);
+});
+
+test('a rejected date falls back to ingestion time and never drops the article', () => {
+  const ingestedAt = new Date().toISOString();
+  const future = new Date(Date.now() + 120 * 24 * HOUR).toISOString();
+
+  const warnings = [];
+  const original = console.warn;
+  console.warn = (message) => warnings.push(message);
+  try {
+    const effective = guardEffectivePubDate(future, ingestedAt, { source: 'gdelt', url: 'https://example.com/a' });
+    assert.equal(effective, ingestedAt);
+  } finally {
+    console.warn = original;
+  }
+
+  assert.equal(warnings.length, 1);
+  assert.match(warnings[0], /gdelt/);
+  assert.match(warnings[0], /https:\/\/example\.com\/a/);
+  assert.ok(warnings[0].includes(future));
+});
+
+test('a good date is kept, and a missing one falls back quietly', () => {
+  const ingestedAt = '2026-08-29T12:00:00.000Z';
+  assert.equal(guardEffectivePubDate('2026-08-01T00:00:00.000Z', ingestedAt, {}), '2026-08-01T00:00:00.000Z');
+  assert.equal(guardEffectivePubDate(null, ingestedAt, {}), ingestedAt);
+});