Duriin-API/scripts/start-replay-run.js

#!/usr/bin/env node
/*
 * Start the next replay run over the SAME articles as the previous one, with a
 * feedback brief built from the previous run's own scored outcomes.
 *
 * The point is a single variable. The evaluation slice is the articles the
 * parent answered under the current prompt and the current model, so run N+1
 * differs from run N by the brief and nothing else. Articles the parent
 * answered under an older prompt are the TRAINING half and are never replayed,
 * because deriving the lesson and grading it on the same rows measures nothing.
 *
 * What it writes, all additive:
 *   - parent run status running -> paused. Its cursor is untouched, so it can
 *     be resumed later exactly where it stopped.
 *   - one new row in autonomy_replay_runs carrying the brief.
 *   - one row per evaluation article in autonomy_replay_run_articles.
 * Nothing is deleted and no existing prediction, outcome or snapshot is touched.
 *
 *   node scripts/start-replay-run.js --parent 1 --split "2026-09-04 19:30" --dry-run
 *   node scripts/start-replay-run.js --parent 1 --split "2026-09-04 19:30" --commit
 */
const Database = require("better-sqlite3");
const { buildFeedbackBrief } = require("./build-feedback-brief");
const { STRATEGY_VERSION, PROMPT_VERSION } = require("../workers/replayWorker");

const INTELLIGENCE = process.env.INTELLIGENCE_DB || "/data/intelligence.sqlite";

// When duriin-api-replay-1 restarted onto the prompt it runs today. Not the
// commit timestamp, which is five minutes later and would have been wrong.
// Replay was between daily budgets across the restart, so there is an eight
// hour hole in predictions around it and every candidate split inside that
// hole partitions the data identically.
const SPLIT = "2026-09-04 19:17:43";

function options() {
  const argv = process.argv.slice(2);
  const out = {};
  for (let i = 0; i < argv.length; i += 1) {
    if (!argv[i].startsWith("--")) continue;
    const key = argv[i].slice(2);
    const next = argv[i + 1];
    out[key] = (next && !next.startsWith("--")) ? (i += 1, next) : true;
  }
  return out;
}

// the pinned table arrives with the schema migration, so a dry run from a
// container that has not been redeployed yet should still be able to report
function countOf(db, table) {
  try {
    return db.prepare(`SELECT COUNT(*) AS c FROM ${table}`).get().c;
  } catch (error) {
    console.error(`[replay-run] cannot count ${table}:`, error.message);
    return 0;
  }
}

function articleIdOf(raw, predictionId) {
  try {
    const parsed = JSON.parse(raw || "[]");
    return Array.isArray(parsed) && parsed.length ? Number(parsed[0]) : null;
  } catch (error) {
    console.error(`[replay-run] unparseable evidence on prediction ${predictionId}:`, error.message);
    return null;
  }
}

function main() {
  const opts = options();
  const parentId = Number(opts.parent || 1);
  const split = String(opts.split || SPLIT);
  const commit = opts.commit === true;
  const model = String(opts.model || process.env.OPEN_ROUTER_LLM_MODEL || "");

  const db = new Database(INTELLIGENCE, { readonly: !commit });
  db.pragma("busy_timeout = 20000");

  const parent = db.prepare("SELECT * FROM autonomy_replay_runs WHERE id = ?").get(parentId);
  if (!parent) throw new Error(`replay run ${parentId} does not exist`);

  // row counts before, so the report can show nothing went missing
  const before = {
    runs: countOf(db, "autonomy_replay_runs"),
    predictions: countOf(db, "autonomy_predictions"),
    outcomes: countOf(db, "autonomy_outcomes"),
    pinned: countOf(db, "autonomy_replay_run_articles"),
  };

  const evaluation = db.prepare(`
    SELECT p.id, p.evidence_article_ids, p.information_cutoff
    FROM autonomy_predictions p
    WHERE p.origin = 'replay' AND p.replay_run_id = ? AND p.created_at >= ?
  `).all(parentId, split);

  const articles = new Map();
  for (const row of evaluation) {
    const id = articleIdOf(row.evidence_article_ids, row.id);
    if (id && !articles.has(id)) articles.set(id, String(row.information_cutoff));
  }

  const { text: brief, stats } = buildFeedbackBrief(db, { runId: parentId, createdBefore: split });

  console.log(`parent run #${parentId} ${parent.status}, watermark ${parent.watermark_at}`);
  console.log(`split at ${split}`);
  console.log(`  training predictions (before split, feed the brief): ${stats.n}`);
  console.log(`  evaluation predictions (at or after split):          ${evaluation.length}`);
  console.log(`  evaluation articles to replay:                       ${articles.size}`);
  console.log(`  brief: ${brief.split("\n").length} lines, ${brief.length} chars`);
  console.log(`  new run would be ${STRATEGY_VERSION}/${PROMPT_VERSION} on ${model || "(model from config at run time)"}`);

  if (!articles.size) throw new Error("no evaluation articles, refusing to create an empty run");
  // inheriting the parent's model here is how run 1 ended up labelled qwen for
  // predictions deepseek made. a label nobody set is worse than a failure.
  if (commit && !model) {
    throw new Error("pass --model, or set OPEN_ROUTER_LLM_MODEL. refusing to guess what will run this");
  }

  if (!commit) {
    console.log("\ndry run, nothing written. pass --commit to apply.");
    db.close();
    return;
  }

  const apply = db.transaction(() => {
    db.prepare("UPDATE autonomy_replay_runs SET status = 'paused', updated_at = datetime('now') WHERE id = ? AND status = 'running'").run(parentId);
    const created = db.prepare(`
      INSERT INTO autonomy_replay_runs (watermark_at, strategy_version, prompt_version, coordinator_model, parent_run_id, feedback_brief)
      VALUES (?, ?, ?, ?, ?, ?)
    `).run(parent.watermark_at, STRATEGY_VERSION, PROMPT_VERSION, model, parentId, brief);
    const runId = created.lastInsertRowid;
    const insert = db.prepare("INSERT OR IGNORE INTO autonomy_replay_run_articles (run_id, article_id, effective_at) VALUES (?, ?, ?)");
    for (const [articleId, effectiveAt] of articles) insert.run(runId, articleId, effectiveAt);
    return runId;
  });
  const runId = apply();

  const after = {
    runs: countOf(db, "autonomy_replay_runs"),
    predictions: countOf(db, "autonomy_predictions"),
    outcomes: countOf(db, "autonomy_outcomes"),
    pinned: countOf(db, "autonomy_replay_run_articles"),
  };
  console.log(`\ncreated replay run #${runId}, parent #${parentId} is now`
    + ` ${db.prepare("SELECT status FROM autonomy_replay_runs WHERE id = ?").get(parentId).status}`);
  console.log("row counts before -> after");
  for (const key of Object.keys(before)) {
    const moved = after[key] - before[key];
    console.log(`  ${key.padEnd(12)} ${before[key]} -> ${after[key]}  (${moved >= 0 ? "+" : ""}${moved})`);
  }
  if (after.predictions !== before.predictions || after.outcomes !== before.outcomes) {
    console.error("predictions or outcomes changed, that should not happen here");
  }
  db.close();
}

main();