feat: let the generator read its own results, and measure it honestly

Nothing in the pipeline has ever fed an outcome back to the thing that makes
predictions. Calibration reads autonomy_outcomes, but calibration only gates
whether to act on a prediction, never what the prediction is. So the only thing
that has ever changed this system's output is a human editing the prompt.

score-replay-runs.js asks "compared to what". The answer is not flattering:
over 2,114 scored replay predictions the system is right 50.05% of the time
while answering "negative" to every one of the same bars scores 54.45%. It is
4.4 points below a constant, z=-4.06. The whole deficit is the prior. It says
positive on 65% of calls when 45.5% of bars beat SPY, a 19 point skew. Its
discrimination, P(up|positive) minus P(up|negative), is +3.2 points with
p=0.15, so the direction it picks is weakly informative and completely buried
by how often it defaults to positive.

The first version of that script compared each direction group's accuracy to
"always that direction" on the same rows, which is an identity and tests
nothing. T3 replaces it with the two proportion test that actually asks whether
the choice of direction carries information.

build-feedback-brief.js turns a run's scored outcomes into a memo the next run
reads before predicting. Generated from the data, not written by hand, or it is
just me editing the prompt again with extra steps.

Replay can now be pinned to an explicit article set, which is what makes two
runs comparable at all. Comparing two calendar windows of one run compares two
market regimes: the epochs in run 1 line up exactly with article vintage, E0 is
late 2024 and E2 is 2026, so nothing could be attributed. A new run also
inherits its parent's watermark instead of recomputing it from today, which
silently guaranteed a different archive slice every time.

prompt_version never moved across four material prompt changes, so every
proposal on record claims to come from the first prompt. coordinator-2 and
replay-coordinator-2.

docs/replay-run-2-preregistration.md fixes the bar before the run exists,
including which result counts as learning and which is only calibration.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WnNxwxfXSbeNtjvtz5gayb

scripts/start-replay-run.js

@@ -0,0 +1,144 @@
+#!/usr/bin/env node
+/*
+ * Start the next replay run over the SAME articles as the previous one, with a
+ * feedback brief built from the previous run's own scored outcomes.
+ *
+ * The point is a single variable. The evaluation slice is the articles the
+ * parent answered under the current prompt and the current model, so run N+1
+ * differs from run N by the brief and nothing else. Articles the parent
+ * answered under an older prompt are the TRAINING half and are never replayed,
+ * because deriving the lesson and grading it on the same rows measures nothing.
+ *
+ * What it writes, all additive:
+ *   - parent run status running -> paused. Its cursor is untouched, so it can
+ *     be resumed later exactly where it stopped.
+ *   - one new row in autonomy_replay_runs carrying the brief.
+ *   - one row per evaluation article in autonomy_replay_run_articles.
+ * Nothing is deleted and no existing prediction, outcome or snapshot is touched.
+ *
+ *   node scripts/start-replay-run.js --parent 1 --split "2026-09-04 19:30" --dry-run
+ *   node scripts/start-replay-run.js --parent 1 --split "2026-09-04 19:30" --commit
+ */
+const Database = require("better-sqlite3");
+const { buildFeedbackBrief } = require("./build-feedback-brief");
+const { STRATEGY_VERSION, PROMPT_VERSION } = require("../workers/replayWorker");
+
+const INTELLIGENCE = process.env.INTELLIGENCE_DB || "/data/intelligence.sqlite";
+
+function options() {
+  const argv = process.argv.slice(2);
+  const out = {};
+  for (let i = 0; i < argv.length; i += 1) {
+    if (!argv[i].startsWith("--")) continue;
+    const key = argv[i].slice(2);
+    const next = argv[i + 1];
+    out[key] = (next && !next.startsWith("--")) ? (i += 1, next) : true;
+  }
+  return out;
+}
+
+// the pinned table arrives with the schema migration, so a dry run from a
+// container that has not been redeployed yet should still be able to report
+function countOf(db, table) {
+  try {
+    return db.prepare(`SELECT COUNT(*) AS c FROM ${table}`).get().c;
+  } catch (error) {
+    console.error(`[replay-run] cannot count ${table}:`, error.message);
+    return 0;
+  }
+}
+
+function articleIdOf(raw, predictionId) {
+  try {
+    const parsed = JSON.parse(raw || "[]");
+    return Array.isArray(parsed) && parsed.length ? Number(parsed[0]) : null;
+  } catch (error) {
+    console.error(`[replay-run] unparseable evidence on prediction ${predictionId}:`, error.message);
+    return null;
+  }
+}
+
+function main() {
+  const opts = options();
+  const parentId = Number(opts.parent || 1);
+  const split = String(opts.split || "2026-09-04 19:30");
+  const commit = opts.commit === true;
+  const model = String(opts.model || process.env.OPEN_ROUTER_LLM_MODEL || "");
+
+  const db = new Database(INTELLIGENCE, { readonly: !commit });
+  db.pragma("busy_timeout = 20000");
+
+  const parent = db.prepare("SELECT * FROM autonomy_replay_runs WHERE id = ?").get(parentId);
+  if (!parent) throw new Error(`replay run ${parentId} does not exist`);
+
+  // row counts before, so the report can show nothing went missing
+  const before = {
+    runs: countOf(db, "autonomy_replay_runs"),
+    predictions: countOf(db, "autonomy_predictions"),
+    outcomes: countOf(db, "autonomy_outcomes"),
+    pinned: countOf(db, "autonomy_replay_run_articles"),
+  };
+
+  const evaluation = db.prepare(`
+    SELECT p.id, p.evidence_article_ids, p.information_cutoff
+    FROM autonomy_predictions p
+    WHERE p.origin = 'replay' AND p.replay_run_id = ? AND p.created_at >= ?
+  `).all(parentId, split);
+
+  const articles = new Map();
+  for (const row of evaluation) {
+    const id = articleIdOf(row.evidence_article_ids, row.id);
+    if (id && !articles.has(id)) articles.set(id, String(row.information_cutoff));
+  }
+
+  const { text: brief, stats } = buildFeedbackBrief(db, { runId: parentId, createdBefore: split });
+
+  console.log(`parent run #${parentId} ${parent.status}, watermark ${parent.watermark_at}`);
+  console.log(`split at ${split}`);
+  console.log(`  training predictions (before split, feed the brief): ${stats.n}`);
+  console.log(`  evaluation predictions (at or after split):          ${evaluation.length}`);
+  console.log(`  evaluation articles to replay:                       ${articles.size}`);
+  console.log(`  brief: ${brief.split("\n").length} lines, ${brief.length} chars`);
+  console.log(`  new run would be ${STRATEGY_VERSION}/${PROMPT_VERSION} on ${model || "(model from config at run time)"}`);
+
+  if (!articles.size) throw new Error("no evaluation articles, refusing to create an empty run");
+
+  if (!commit) {
+    console.log("\ndry run, nothing written. pass --commit to apply.");
+    db.close();
+    return;
+  }
+
+  const apply = db.transaction(() => {
+    db.prepare("UPDATE autonomy_replay_runs SET status = 'paused', updated_at = datetime('now') WHERE id = ? AND status = 'running'").run(parentId);
+    const created = db.prepare(`
+      INSERT INTO autonomy_replay_runs (watermark_at, strategy_version, prompt_version, coordinator_model, parent_run_id, feedback_brief)
+      VALUES (?, ?, ?, ?, ?, ?)
+    `).run(parent.watermark_at, STRATEGY_VERSION, PROMPT_VERSION, model || parent.coordinator_model, parentId, brief);
+    const runId = created.lastInsertRowid;
+    const insert = db.prepare("INSERT OR IGNORE INTO autonomy_replay_run_articles (run_id, article_id, effective_at) VALUES (?, ?, ?)");
+    for (const [articleId, effectiveAt] of articles) insert.run(runId, articleId, effectiveAt);
+    return runId;
+  });
+  const runId = apply();
+
+  const after = {
+    runs: countOf(db, "autonomy_replay_runs"),
+    predictions: countOf(db, "autonomy_predictions"),
+    outcomes: countOf(db, "autonomy_outcomes"),
+    pinned: countOf(db, "autonomy_replay_run_articles"),
+  };
+  console.log(`\ncreated replay run #${runId}, parent #${parentId} is now`
+    + ` ${db.prepare("SELECT status FROM autonomy_replay_runs WHERE id = ?").get(parentId).status}`);
+  console.log("row counts before -> after");
+  for (const key of Object.keys(before)) {
+    const moved = after[key] - before[key];
+    console.log(`  ${key.padEnd(12)} ${before[key]} -> ${after[key]}  (${moved >= 0 ? "+" : ""}${moved})`);
+  }
+  if (after.predictions !== before.predictions || after.outcomes !== before.outcomes) {
+    console.error("predictions or outcomes changed, that should not happen here");
+  }
+  db.close();
+}
+
+main();