fix: restart the stalled autonomy pipeline and make calibration honest

Archive ingestion had been dead since 2026-08-02 because nothing in the compose stack actually ran it. Everything downstream starved from there. - add ingest + enrichment services. server.js only starts the scheduler when DURIIN_RUN_SCHEDULER is not "false", and workers/index.js was not running at all, so articles never got event_id/content/has_embedding and the coordinator had nothing to lease. - pass an explicit origin from coordinatorWorker. it was never passed, so acceptProposal defaulted to 'live' and 464 historical backfill predictions were recorded as live. that also meant verifyEvidence got a null cutoff and skipped its date check entirely. - coarsen cohortKey to event families + horizon buckets. 201 free text event types produced 221 cohorts averaging 2.76 samples, so the n>=30 gate could never be reached and everything abstained for the wrong reason. - gate on cohort diversity, not just sample count. one ticker was roughly half of all resolved outcomes, so a pure count gate was measuring one company. unknown diversity abstains rather than passing. - resolve the admin archive db explicitly and probe it. it relied on a Dockerfile symlink, and without it better-sqlite3 quietly creates an empty file and serves a phantom archive. - clamp implausible future publication dates at ingest. - pin the db backend to sqlite by default. compose hardcoded postgres "true", which would have overridden the operator's own .env on the next redeploy and pointed everything at a stale snapshot. scripts/repair-autonomy-labels.js relabels the affected rows. it is dry run by default and has not been applied. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WnNxwxfXSbeNtjvtz5gayb
2026-08-29 21:43:24 +01:00
parent d778a02bfb
commit 6f1d1eee2d
19 changed files with 1497 additions and 100 deletions
@@ -50,6 +50,11 @@ services:
    networks:
      - nginx_proxy_manager_default

+  # DB backend is one switch for the whole stack and it defaults to sqlite on purpose.
+  # the postgres copy is a stale snapshot (predictions stop around 2026-08-17) and the
+  # migrate script only appends, it never replays UPDATEs, so a `compose up` must not
+  # quietly flip us over. set DURIIN_DB_BACKEND=postgres + DURIIN_USE_POSTGRES=true in
+  # .env only after a fresh intelligence migration has been run and verifyed.
  api:
    build:
      context: .
@@ -61,8 +66,8 @@ services:
    environment:
      NODE_ENV: production
      INTELLIGENCE_DB: /data/intelligence.sqlite
-      DURIIN_DB_BACKEND: postgres
-      DURIIN_USE_POSTGRES: "true"
+      DURIIN_DB_BACKEND: "${DURIIN_DB_BACKEND:-sqlite}"
+      DURIIN_USE_POSTGRES: "${DURIIN_USE_POSTGRES:-false}"
      DURIIN_POSTGRES_URL: "postgresql://${POSTGRES_USER:-duriin}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-duriin}"
      DURIIN_RUN_SCHEDULER: "false"
      AUTONOMY_EXECUTION_MODE: "${AUTONOMY_EXECUTION_MODE:-shadow}"
@@ -73,7 +78,60 @@ services:
    networks:
      - nginx_proxy_manager_default

+  # same image as api, but this one actually runs the cron scheduler
+  # (rss / gdelt / edgar / alphavantage / finnhub). it also boots fastify on
+  # 3001 but nothing proxies to it, so its effectivly ingestion only.
+  ingest:
+    build:
+      context: .
+      provenance: false
+    env_file: .env
+    volumes:
+      - ./config.json:/app/config.json:ro
+      - ./data:/data
+    environment:
+      NODE_ENV: production
+      INTELLIGENCE_DB: /data/intelligence.sqlite
+      DURIIN_DB_BACKEND: "${DURIIN_DB_BACKEND:-sqlite}"
+      DURIIN_USE_POSTGRES: "${DURIIN_USE_POSTGRES:-false}"
+      DURIIN_POSTGRES_URL: "postgresql://${POSTGRES_USER:-duriin}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-duriin}"
+      DURIIN_RUN_SCHEDULER: "true"
+      AUTONOMY_EXECUTION_MODE: "${AUTONOMY_EXECUTION_MODE:-shadow}"
+    depends_on:
+      postgres:
+        condition: service_healthy
+    restart: unless-stopped
+    networks:
+      - nginx_proxy_manager_default
+
+  # enrichment chain: queue feeder -> augor -> consolidation -> graph -> signal -> outcome.
+  # this is what fills event_id / content / has_embedding, which the autonomy
+  # reconcilers require before they enqueue anything.
+  enrichment:
+    build:
+      context: .
+      provenance: false
+    command: node workers/index.js
+    env_file: .env
+    volumes:
+      - ./config.json:/app/config.json:ro
+      - ./data:/data
+    environment:
+      NODE_ENV: production
+      DURIIN_DB: /data/archive.sqlite
+      INTELLIGENCE_DB: /data/intelligence.sqlite
+      DURIIN_DB_BACKEND: "${DURIIN_DB_BACKEND:-sqlite}"
+      DURIIN_USE_POSTGRES: "${DURIIN_USE_POSTGRES:-false}"
+      DURIIN_POSTGRES_URL: "postgresql://${POSTGRES_USER:-duriin}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-duriin}"
+    depends_on:
+      postgres:
+        condition: service_healthy
+    restart: unless-stopped
+    networks:
+      - nginx_proxy_manager_default
+
  intelligence:
+    # superseded by the "enrichment" service above, kept behind a profile
    profiles: [legacy]
    build:
      context: .
@@ -104,8 +162,8 @@ services:
      NODE_ENV: production
      DURIIN_DB: /data/archive.sqlite
      INTELLIGENCE_DB: /data/intelligence.sqlite
-      DURIIN_DB_BACKEND: postgres
-      DURIIN_USE_POSTGRES: "true"
+      DURIIN_DB_BACKEND: "${DURIIN_DB_BACKEND:-sqlite}"
+      DURIIN_USE_POSTGRES: "${DURIIN_USE_POSTGRES:-false}"
      DURIIN_POSTGRES_URL: "postgresql://${POSTGRES_USER:-duriin}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-duriin}"
      AUTONOMY_POLL_MS: "${AUTONOMY_POLL_MS:-5000}"
    depends_on:
@@ -130,8 +188,8 @@ services:
      NODE_ENV: production
      DURIIN_DB: /data/archive.sqlite
      INTELLIGENCE_DB: /data/intelligence.sqlite
-      DURIIN_DB_BACKEND: postgres
-      DURIIN_USE_POSTGRES: "true"
+      DURIIN_DB_BACKEND: "${DURIIN_DB_BACKEND:-sqlite}"
+      DURIIN_USE_POSTGRES: "${DURIIN_USE_POSTGRES:-false}"
      DURIIN_POSTGRES_URL: "postgresql://${POSTGRES_USER:-duriin}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-duriin}"
      AUTONOMY_POLL_MS: "${AUTONOMY_COORDINATOR_POLL_MS:-5000}"
    depends_on:
@@ -154,8 +212,8 @@ services:
    environment:
      NODE_ENV: production
      INTELLIGENCE_DB: /data/intelligence.sqlite
-      DURIIN_DB_BACKEND: postgres
-      DURIIN_USE_POSTGRES: "true"
+      DURIIN_DB_BACKEND: "${DURIIN_DB_BACKEND:-sqlite}"
+      DURIIN_USE_POSTGRES: "${DURIIN_USE_POSTGRES:-false}"
      DURIIN_POSTGRES_URL: "postgresql://${POSTGRES_USER:-duriin}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-duriin}"
      AUTONOMY_OUTCOME_POLL_MS: "${AUTONOMY_OUTCOME_POLL_MS:-60000}"
    depends_on:
@@ -180,8 +238,8 @@ services:
      NODE_ENV: production
      DURIIN_DB: /data/archive.sqlite
      INTELLIGENCE_DB: /data/intelligence.sqlite
-      DURIIN_DB_BACKEND: postgres
-      DURIIN_USE_POSTGRES: "true"
+      DURIIN_DB_BACKEND: "${DURIIN_DB_BACKEND:-sqlite}"
+      DURIIN_USE_POSTGRES: "${DURIIN_USE_POSTGRES:-false}"
      DURIIN_POSTGRES_URL: "postgresql://${POSTGRES_USER:-duriin}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-duriin}"
      AUTONOMY_REPLAY_POLL_MS: "${AUTONOMY_REPLAY_POLL_MS:-15000}"
      AUTONOMY_REPLAY_DAILY_BUDGET: "${AUTONOMY_REPLAY_DAILY_BUDGET:-100}"
@@ -206,8 +264,8 @@ services:
    environment:
      NODE_ENV: production
      INTELLIGENCE_DB: /data/intelligence.sqlite
-      DURIIN_DB_BACKEND: postgres
-      DURIIN_USE_POSTGRES: "true"
+      DURIIN_DB_BACKEND: "${DURIIN_DB_BACKEND:-sqlite}"
+      DURIIN_USE_POSTGRES: "${DURIIN_USE_POSTGRES:-false}"
      DURIIN_POSTGRES_URL: "postgresql://${POSTGRES_USER:-duriin}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-duriin}"
      AUTONOMY_CALIBRATION_POLL_MS: "${AUTONOMY_CALIBRATION_POLL_MS:-60000}"
    depends_on:
@@ -230,8 +288,8 @@ services:
    environment:
      NODE_ENV: production
      INTELLIGENCE_DB: /data/intelligence.sqlite
-      DURIIN_DB_BACKEND: postgres
-      DURIIN_USE_POSTGRES: "true"
+      DURIIN_DB_BACKEND: "${DURIIN_DB_BACKEND:-sqlite}"
+      DURIIN_USE_POSTGRES: "${DURIIN_USE_POSTGRES:-false}"
      DURIIN_POSTGRES_URL: "postgresql://${POSTGRES_USER:-duriin}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-duriin}"
      AUTONOMY_EXECUTION_MODE: "${AUTONOMY_EXECUTION_MODE:-shadow}"
      AUTONOMY_DEFAULT_NOTIONAL: "${AUTONOMY_DEFAULT_NOTIONAL:-100}"