fix: restart the stalled autonomy pipeline and make calibration honest

Archive ingestion had been dead since 2026-08-02 because nothing in the
compose stack actually ran it. Everything downstream starved from there.

- add ingest + enrichment services. server.js only starts the scheduler when
  DURIIN_RUN_SCHEDULER is not "false", and workers/index.js was not running at
  all, so articles never got event_id/content/has_embedding and the coordinator
  had nothing to lease.
- pass an explicit origin from coordinatorWorker. it was never passed, so
  acceptProposal defaulted to 'live' and 464 historical backfill predictions
  were recorded as live. that also meant verifyEvidence got a null cutoff and
  skipped its date check entirely.
- coarsen cohortKey to event families + horizon buckets. 201 free text event
  types produced 221 cohorts averaging 2.76 samples, so the n>=30 gate could
  never be reached and everything abstained for the wrong reason.
- gate on cohort diversity, not just sample count. one ticker was roughly half
  of all resolved outcomes, so a pure count gate was measuring one company.
  unknown diversity abstains rather than passing.
- resolve the admin archive db explicitly and probe it. it relied on a
  Dockerfile symlink, and without it better-sqlite3 quietly creates an empty
  file and serves a phantom archive.
- clamp implausible future publication dates at ingest.
- pin the db backend to sqlite by default. compose hardcoded postgres "true",
  which would have overridden the operator's own .env on the next redeploy and
  pointed everything at a stale snapshot.

scripts/repair-autonomy-labels.js relabels the affected rows. it is dry run by
default and has not been applied.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WnNxwxfXSbeNtjvtz5gayb

test/calibrationCohorts.test.js

@@ -0,0 +1,168 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+
+const {
+  normalizeEventType,
+  horizonBucket,
+  cohortKey,
+  legacyCohortKey,
+  calibrateOutcomes,
+  EVENT_FAMILY_NAMES,
+} = require('../src/autonomy/calibration');
+const { decide, DEFAULT_POLICY_RULES } = require('../src/autonomy/policy');
+
+test('event types collapse onto a small closed set of families', () => {
+  assert.equal(normalizeEventType('earnings_beat'), 'earnings');
+  assert.equal(normalizeEventType('Q3 Earnings Report'), 'earnings');
+  assert.equal(normalizeEventType('guidance_raise'), 'guidance');
+  assert.equal(normalizeEventType('supply_constraint'), 'supply_chain');
+  assert.equal(normalizeEventType('supplyConstraint'), 'supply_chain');
+  assert.equal(normalizeEventType('antitrust probe'), 'regulatory');
+  assert.equal(normalizeEventType('ceo_resignation'), 'leadership');
+  assert.equal(normalizeEventType('analyst-downgrade'), 'analyst_action');
+  assert.equal(normalizeEventType('share buyback'), 'capital');
+  assert.equal(normalizeEventType('data breach'), 'security_incident');
+  assert.equal(normalizeEventType('interest rate decision'), 'macro');
+  assert.equal(normalizeEventType('product_launch'), 'product');
+  assert.equal(normalizeEventType('acquisition_rumor'), 'm_and_a');
+  assert.equal(normalizeEventType('patent lawsuit'), 'legal');
+});
+
+test('unrecognised or empty event types fall back to other, never to their own cohort', () => {
+  assert.equal(normalizeEventType('zebra_convention'), 'other');
+  assert.equal(normalizeEventType(''), 'other');
+  assert.equal(normalizeEventType('   '), 'other');
+  assert.equal(normalizeEventType(null), 'other');
+  assert.equal(normalizeEventType(undefined), 'other');
+  assert.equal(normalizeEventType(42), 'other');
+  assert.ok(EVENT_FAMILY_NAMES.includes('other'));
+  assert.ok(EVENT_FAMILY_NAMES.length <= 15, `taxonomy grew to ${EVENT_FAMILY_NAMES.length} families`);
+});
+
+test('every allowed horizon lands in one of three buckets', () => {
+  assert.equal(horizonBucket(1), 'short');
+  assert.equal(horizonBucket(5), 'short');
+  assert.equal(horizonBucket(10), 'medium');
+  assert.equal(horizonBucket(20), 'medium');
+  assert.equal(horizonBucket(30), 'long');
+  assert.equal(horizonBucket(60), 'long');
+  assert.equal(horizonBucket(90), 'long');
+  assert.equal(horizonBucket(null), 'unknown');
+  assert.equal(horizonBucket('nope'), 'unknown');
+});
+
+test('cohort key is versioned, coarse and stable, and the legacy key is still available', () => {
+  assert.equal(
+    cohortKey({ direction: 'positive', eventType: 'earnings_beat', horizonDays: 10 }),
+    'v2|unknown|earnings|medium|positive'
+  );
+  // different raw event text, same family and horizon bucket -> same cohort
+  assert.equal(
+    cohortKey({ direction: 'positive', eventType: 'quarterly results miss', horizonDays: 20 }),
+    cohortKey({ direction: 'positive', eventType: 'earnings_beat', horizonDays: 10 })
+  );
+  assert.notEqual(
+    cohortKey({ direction: 'negative', eventType: 'earnings_beat', horizonDays: 10 }),
+    cohortKey({ direction: 'positive', eventType: 'earnings_beat', horizonDays: 10 })
+  );
+  assert.equal(
+    legacyCohortKey({ direction: 'positive', eventType: 'earnings_beat', horizonDays: 10 }),
+    'unknown|earnings_beat|10|positive'
+  );
+});
+
+test('the coarse taxonomy actually collapses a realistic spread of free text', () => {
+  const raw = [
+    'earnings_beat', 'earnings_miss', 'q2_earnings', 'revenue_growth', 'margin_expansion',
+    'guidance_raise', 'guidance_cut', 'outlook_downgrade', 'profit_warning',
+    'supply_constraint', 'chip_shortage', 'production_halt', 'capacity_expansion',
+    'analyst_upgrade', 'price_target_raise', 'ceo_departure', 'board_shakeup',
+    'antitrust_probe', 'export_controls', 'tariff_announcement',
+  ];
+  const families = new Set(raw.map(normalizeEventType));
+  assert.ok(families.size <= 8, `expected heavy collapse, got ${families.size} families`);
+});
+
+test('calibrateOutcomes reports instrument diversity and concentration', () => {
+  const rows = [
+    { excess_return: 0.02, direction_correct: 1, instrument: 'NVDA' },
+    { excess_return: 0.01, direction_correct: 1, instrument: 'nvda' },
+    { excess_return: -0.01, direction_correct: 0, instrument: 'NVDA' },
+    { excess_return: 0.03, direction_correct: 1, instrument: 'AMD' },
+  ];
+  const result = calibrateOutcomes(rows);
+  assert.equal(result.sampleSize, 4);
+  assert.equal(result.distinctInstruments, 2);
+  assert.equal(result.topInstrumentShare, 0.75);
+
+  const empty = calibrateOutcomes([]);
+  assert.equal(empty.distinctInstruments, 0);
+  assert.equal(empty.topInstrumentShare, null);
+
+  const unlabelled = calibrateOutcomes([{ excess_return: 0.01, direction_correct: 1 }]);
+  assert.equal(unlabelled.distinctInstruments, 0);
+});
+
+test('the diversification gate blocks single ticker cohorts however large they are', () => {
+  const base = { direction: 'positive', probability: 0.8, expectedExcessReturn: 0.02, lowerReturn: -0.01 };
+
+  // 300 samples, one name: this is the NVDA case, and it must not qualify
+  const concentrated = decide({ ...base, sampleSize: 300, distinctInstruments: 1, topInstrumentShare: 1 });
+  assert.equal(concentrated.action, 'ABSTAIN');
+  assert.match(concentrated.rationale, /diversity/);
+
+  // enough names but still dominated by one of them
+  const dominated = decide({ ...base, sampleSize: 300, distinctInstruments: 9, topInstrumentShare: 0.82 });
+  assert.equal(dominated.action, 'ABSTAIN');
+  assert.match(dominated.rationale, /dominated/);
+
+  // pre-diversification snapshots carry no count, unknown is not adequate
+  const unknown = decide({ ...base, sampleSize: 300, distinctInstruments: null, topInstrumentShare: null });
+  assert.equal(unknown.action, 'ABSTAIN');
+  assert.match(unknown.rationale, /unknown/);
+
+  const qualified = decide({ ...base, sampleSize: 40, distinctInstruments: 9, topInstrumentShare: 0.3 });
+  assert.equal(qualified.action, 'BUY');
+});
+
+test('both evidence thresholds are overridable and default conservatively', () => {
+  assert.equal(DEFAULT_POLICY_RULES.minSampleSize, 30);
+  assert.equal(DEFAULT_POLICY_RULES.minDistinctInstruments, 5);
+  assert.equal(DEFAULT_POLICY_RULES.maxInstrumentConcentration, 0.5);
+
+  const input = {
+    direction: 'positive', probability: 0.8, expectedExcessReturn: 0.02, lowerReturn: -0.01,
+    sampleSize: 12, distinctInstruments: 3, topInstrumentShare: 0.4,
+  };
+  assert.equal(decide(input).action, 'ABSTAIN');
+  assert.equal(decide(input, { minSampleSize: 10, minDistinctInstruments: 2 }).action, 'BUY');
+  assert.equal(decide(input, { minSampleSize: 10, minDistinctInstruments: 2, maxInstrumentConcentration: 0.3 }).action, 'ABSTAIN');
+});
+
+test('sample size gate still runs before the diversity gate', () => {
+  const result = decide({
+    direction: 'positive', probability: 0.9, expectedExcessReturn: 0.05,
+    sampleSize: 2, distinctInstruments: 40, topInstrumentShare: 0.1,
+  });
+  assert.equal(result.action, 'ABSTAIN');
+  assert.match(result.rationale, /insufficient calibration sample/);
+});
+
+test('a missing concentration share cannot sneak past the cap as a zero', () => {
+  const base = {
+    direction: 'positive', probability: 0.8, expectedExcessReturn: 0.02,
+    lowerReturn: -0.01, sampleSize: 300, distinctInstruments: 40,
+  };
+
+  // Number(null) is 0, which used to slide straight under the cap even though we
+  // had no idea what the real concentration was.
+  for (const share of [null, undefined]) {
+    const verdict = decide({ ...base, topInstrumentShare: share });
+    assert.equal(verdict.action, 'ABSTAIN');
+    assert.match(verdict.rationale, /concentration unknown/);
+  }
+
+  // a genuinely broad cohort still gets through, we havent just bolted it shut
+  const broad = decide({ ...base, topInstrumentShare: 0.12 });
+  assert.equal(broad.action, 'BUY');
+});