feat: let the generator read its own results, and measure it honestly
Nothing in the pipeline has ever fed an outcome back to the thing that makes predictions. Calibration reads autonomy_outcomes, but calibration only gates whether to act on a prediction, never what the prediction is. So the only thing that has ever changed this system's output is a human editing the prompt. score-replay-runs.js asks "compared to what". The answer is not flattering: over 2,114 scored replay predictions the system is right 50.05% of the time while answering "negative" to every one of the same bars scores 54.45%. It is 4.4 points below a constant, z=-4.06. The whole deficit is the prior. It says positive on 65% of calls when 45.5% of bars beat SPY, a 19 point skew. Its discrimination, P(up|positive) minus P(up|negative), is +3.2 points with p=0.15, so the direction it picks is weakly informative and completely buried by how often it defaults to positive. The first version of that script compared each direction group's accuracy to "always that direction" on the same rows, which is an identity and tests nothing. T3 replaces it with the two proportion test that actually asks whether the choice of direction carries information. build-feedback-brief.js turns a run's scored outcomes into a memo the next run reads before predicting. Generated from the data, not written by hand, or it is just me editing the prompt again with extra steps. Replay can now be pinned to an explicit article set, which is what makes two runs comparable at all. Comparing two calendar windows of one run compares two market regimes: the epochs in run 1 line up exactly with article vintage, E0 is late 2024 and E2 is 2026, so nothing could be attributed. A new run also inherits its parent's watermark instead of recomputing it from today, which silently guaranteed a different archive slice every time. prompt_version never moved across four material prompt changes, so every proposal on record claims to come from the first prompt. coordinator-2 and replay-coordinator-2. docs/replay-run-2-preregistration.md fixes the bar before the run exists, including which result counts as learning and which is only calibration. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WnNxwxfXSbeNtjvtz5gayb
This commit is contained in:
ImBenji
@@ -0,0 +1,163 @@
|
||||
#!/usr/bin/env node
|
||||
/*
|
||||
* Turn a replay run's own scored outcomes into a memo the next run is told
|
||||
* before it predicts anything.
|
||||
*
|
||||
* This is the piece that was missing. The generator has never once seen its own
|
||||
* results: nothing in coordinatorWorker, replayWorker, llm.js or graphContext
|
||||
* reads autonomy_outcomes. Calibration reads them, but calibration only decides
|
||||
* whether to ACT on a prediction, it never changes what gets predicted. So the
|
||||
* only thing that has ever altered this system's output is a human editing the
|
||||
* prompt. A memo generated from the data is not a human editing the prompt.
|
||||
*
|
||||
* Everything here is computed from a TRAIN slice bounded by --until. Nothing
|
||||
* from the evaluation window may appear in the text or the comparison is just
|
||||
* fitting to the answer sheet.
|
||||
*
|
||||
* node scripts/build-feedback-brief.js --run 1 --until "2026-09-04 19:30"
|
||||
*/
|
||||
const Database = require("better-sqlite3");
|
||||
|
||||
const INTELLIGENCE = process.env.INTELLIGENCE_DB || "/data/intelligence.sqlite";
|
||||
|
||||
function pct(x, digits = 1) { return `${(x * 100).toFixed(digits)}%`; }
|
||||
|
||||
function erf(x) {
|
||||
const sign = x < 0 ? -1 : 1;
|
||||
const z = Math.abs(x);
|
||||
const t = 1 / (1 + 0.3275911 * z);
|
||||
const y = 1 - ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t * Math.exp(-z * z);
|
||||
return sign * y;
|
||||
}
|
||||
function twoSided(z) { return 2 * (1 - 0.5 * (1 + erf(Math.abs(z) / Math.SQRT2))); }
|
||||
|
||||
function loadTrain(db, { runId, createdBefore }) {
|
||||
return db.prepare(`
|
||||
SELECT p.direction, p.event_type, p.horizon_days, p.instrument,
|
||||
o.direction_correct, o.excess_return
|
||||
FROM autonomy_predictions p
|
||||
JOIN autonomy_outcomes o ON o.prediction_id = p.id
|
||||
WHERE p.origin = 'replay' AND p.replay_run_id = ? AND p.created_at < ?
|
||||
`).all(runId, createdBefore);
|
||||
}
|
||||
|
||||
// families and horizons that sit far enough below the constant to be worth
|
||||
// naming. n floor keeps a handful of unlucky calls out of the memo.
|
||||
function weakSlices(rows, key, { minimum = 40, factor }) {
|
||||
const groups = new Map();
|
||||
for (const row of rows) {
|
||||
const k = String(row[key]);
|
||||
if (!groups.has(k)) groups.set(k, []);
|
||||
groups.get(k).push(row);
|
||||
}
|
||||
const kept = [...groups.entries()].filter(([, v]) => v.length >= minimum);
|
||||
const adjust = factor || kept.length || 1;
|
||||
return kept.map(([k, v]) => {
|
||||
const hits = v.filter((r) => r.direction_correct).length;
|
||||
const acc = hits / v.length;
|
||||
const down = v.filter((r) => r.excess_return <= 0).length / v.length;
|
||||
const bar = Math.max(down, 1 - down);
|
||||
const se = Math.sqrt(bar * (1 - bar) / v.length);
|
||||
const z = se > 0 ? (acc - bar) / se : 0;
|
||||
return { key: k, n: v.length, acc, bar, p: Math.min(1, twoSided(z) * adjust) };
|
||||
}).sort((a, b) => a.acc - b.acc);
|
||||
}
|
||||
|
||||
function buildFeedbackBrief(db, { runId, createdBefore }) {
|
||||
const rows = loadTrain(db, { runId, createdBefore });
|
||||
if (rows.length < 200) {
|
||||
throw new Error(`only ${rows.length} scored training predictions for run ${runId}, refusing to write a brief off that`);
|
||||
}
|
||||
const n = rows.length;
|
||||
const positives = rows.filter((r) => r.direction === "positive");
|
||||
const positiveShare = positives.length / n;
|
||||
const actuallyUp = rows.filter((r) => r.excess_return > 0).length / n;
|
||||
const acc = rows.filter((r) => r.direction_correct).length / n;
|
||||
const alwaysNegative = 1 - actuallyUp;
|
||||
|
||||
const upGivenPositive = positives.filter((r) => r.excess_return > 0).length / (positives.length || 1);
|
||||
const negatives = rows.filter((r) => r.direction === "negative");
|
||||
const upGivenNegative = negatives.filter((r) => r.excess_return > 0).length / (negatives.length || 1);
|
||||
|
||||
const families = weakSlices(rows, "event_type", { minimum: 40 });
|
||||
const horizons = weakSlices(rows, "horizon_days", { minimum: 40 });
|
||||
const weakFamilies = families.filter((f) => f.acc < f.bar - 0.08).slice(0, 5);
|
||||
const strongFamilies = [...families].reverse().filter((f) => f.acc > f.bar + 0.05).slice(0, 4);
|
||||
const weakHorizons = horizons.filter((h) => h.acc < h.bar - 0.08).slice(0, 3);
|
||||
|
||||
const lines = [];
|
||||
lines.push(`CALIBRATION FEEDBACK. The following comes from ${n} of your own earlier predictions on this`);
|
||||
lines.push(`archive, every one of them scored on realised excess return against SPY. It describes how you`);
|
||||
lines.push(`have actually performed, not how you think you perform. Use it.`);
|
||||
lines.push("");
|
||||
lines.push(`1. Your directional prior is wrong. You said "positive" on ${pct(positiveShare)} of those predictions.`);
|
||||
lines.push(` Only ${pct(actuallyUp)} of the same bars actually beat SPY. Most individual names underperform a`);
|
||||
lines.push(` cap weighted index over any horizon, so "positive" is the minority answer, not the default.`);
|
||||
lines.push(` You were right ${pct(acc)} of the time. Answering "negative" to every single one of those bars`);
|
||||
lines.push(` would have scored ${pct(alwaysNegative)}. You are currently below a constant.`);
|
||||
lines.push("");
|
||||
lines.push(`2. Beating SPY is the bar, not the company doing well. Good news that the index already had`);
|
||||
lines.push(` priced, or that lifts the whole sector, is not a positive excess return. Ask whether this name`);
|
||||
lines.push(` outperforms the market, never whether the story is upbeat.`);
|
||||
lines.push("");
|
||||
// a spread under two points is not worth telling it to keep, it would just be
|
||||
// flattering noise back at itself
|
||||
if (upGivenPositive - upGivenNegative >= 0.02) {
|
||||
lines.push(`3. Your instinct on WHICH way is weakly right: bars you called positive beat SPY ${pct(upGivenPositive)}`);
|
||||
lines.push(` of the time versus ${pct(upGivenNegative)} for the ones you called negative. That separation is small`);
|
||||
lines.push(` enough that it could still be luck, and it is buried by how often you default to positive.`);
|
||||
} else {
|
||||
lines.push(`3. Your choice of direction carries no information yet: bars you called positive beat SPY`);
|
||||
lines.push(` ${pct(upGivenPositive)} of the time versus ${pct(upGivenNegative)} for the ones you called negative. Only predict when`);
|
||||
lines.push(` the evidence gives you a genuine mechanism, and return nothing otherwise.`);
|
||||
}
|
||||
lines.push("");
|
||||
if (weakFamilies.length) {
|
||||
lines.push(`4. Event families you read worst, accuracy against the constant on the same bars:`);
|
||||
for (const f of weakFamilies) {
|
||||
lines.push(` ${f.key}: you ${pct(f.acc)}, constant ${pct(f.bar)}, n=${f.n}`);
|
||||
}
|
||||
lines.push(` On these, prefer returning an empty predictions array over a weak call.`);
|
||||
lines.push("");
|
||||
}
|
||||
if (strongFamilies.length) {
|
||||
lines.push(`5. Event families you read best, where a confident call is warranted:`);
|
||||
for (const f of strongFamilies) {
|
||||
lines.push(` ${f.key}: you ${pct(f.acc)}, constant ${pct(f.bar)}, n=${f.n}`);
|
||||
}
|
||||
lines.push("");
|
||||
}
|
||||
if (weakHorizons.length) {
|
||||
lines.push(`6. Horizons that went worst for you: ${weakHorizons.map((h) => `${h.key}d (${pct(h.acc)}, n=${h.n})`).join(", ")}.`);
|
||||
lines.push(` The longer the horizon the more of the move is market and sector rather than the event.`);
|
||||
lines.push("");
|
||||
}
|
||||
lines.push(`None of this tells you what to answer for the evidence below. It tells you which of your habits`);
|
||||
lines.push(`have already cost you. An empty predictions array is always available and costs nothing.`);
|
||||
|
||||
return { text: lines.join("\n"), stats: { n, positiveShare, actuallyUp, acc, alwaysNegative,
|
||||
upGivenPositive, upGivenNegative, weakFamilies, strongFamilies, weakHorizons } };
|
||||
}
|
||||
|
||||
function main() {
|
||||
const argv = process.argv.slice(2);
|
||||
const opts = {};
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
if (!argv[i].startsWith("--")) continue;
|
||||
const key = argv[i].slice(2);
|
||||
const next = argv[i + 1];
|
||||
opts[key] = (next && !next.startsWith("--")) ? (i += 1, next) : true;
|
||||
}
|
||||
const db = new Database(INTELLIGENCE, { readonly: true });
|
||||
db.pragma("busy_timeout = 20000");
|
||||
const { text, stats } = buildFeedbackBrief(db, {
|
||||
runId: Number(opts.run || 1),
|
||||
createdBefore: String(opts.until || "2026-09-04 19:30"),
|
||||
});
|
||||
console.log(text);
|
||||
console.log(`\n--- derived from ${stats.n} scored training predictions ---`);
|
||||
db.close();
|
||||
}
|
||||
|
||||
if (require.main === module) main();
|
||||
module.exports = { buildFeedbackBrief };
|
||||
@@ -0,0 +1,321 @@
|
||||
#!/usr/bin/env node
|
||||
/*
|
||||
* Scoreboard and paired comparison for replay runs.
|
||||
*
|
||||
* Two jobs:
|
||||
* 1. score one run against constant null models, so "50% accuracy" has to
|
||||
* answer the question "compared to what". A model that always says
|
||||
* negative scores whatever share of the sample actually went down, and if
|
||||
* the system cannot beat that it has no directional skill at all.
|
||||
* 2. compare two runs over the SAME articles. Replay walks the archive in
|
||||
* cursor order, so different runs are the only way to hold article
|
||||
* vintage fixed. Comparing two calendar periods of one run compares two
|
||||
* market regimes, not two prompts.
|
||||
*
|
||||
* Read only. Opens intelligence read only and writes nothing.
|
||||
*
|
||||
* PRE REGISTERED TESTS (declared here so the buckets cannot be tuned later):
|
||||
* T1 is run accuracy above the BEST constant baseline on the same sample?
|
||||
* one sided binomial z. the best constant is used as the bar because it
|
||||
* is the hardest of the two, which is conservative for us.
|
||||
* T2 is the direction signed excess return above the BEST constant on the
|
||||
* same bars? paired two sided t. testing it against zero was the first
|
||||
* version and it flattered us: a unit short in everything also earns a
|
||||
* positive number on this sample, so zero is not the bar.
|
||||
* T3 DISCRIMINATION. is P(up | it said positive) above P(up | it said
|
||||
* negative)? two proportion z. this is the only one of the four that a
|
||||
* change of prior cannot fake: it asks whether the choice of direction
|
||||
* carries information, separately from how often it picks each one.
|
||||
* comparing a direction group's accuracy to "always that direction" on
|
||||
* the same rows is an identity and tests nothing, which is what the
|
||||
* first version of this file printed.
|
||||
* T4 paired: on articles both runs answered, is the candidate's per article
|
||||
* accuracy above the baseline's? two sided paired t on the differences.
|
||||
* Everything under "descriptive" is NOT a test. Slice p values carry a
|
||||
* bonferroni factor and are there to generate hypotheses, not confirm them.
|
||||
*
|
||||
* node scripts/score-replay-runs.js
|
||||
* node scripts/score-replay-runs.js --run 1
|
||||
* node scripts/score-replay-runs.js --baseline 1 --candidate 2
|
||||
* node scripts/score-replay-runs.js --baseline 1 --candidate 2 --since 2026-02-01
|
||||
*/
|
||||
const Database = require("better-sqlite3");
|
||||
|
||||
const INTELLIGENCE = process.env.INTELLIGENCE_DB || "/data/intelligence.sqlite";
|
||||
|
||||
function args() {
|
||||
const out = {};
|
||||
const argv = process.argv.slice(2);
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
if (!argv[i].startsWith("--")) continue;
|
||||
const key = argv[i].slice(2);
|
||||
const next = argv[i + 1];
|
||||
out[key] = (next && !next.startsWith("--")) ? (i += 1, next) : true;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Abramowitz and Stegun 7.1.26. The last analysis used a logistic shortcut and
|
||||
// it returned p > 1 for negative z, which is nonsense that survived because
|
||||
// nobody looks at a p value and asks whether it is even in range.
|
||||
function erf(x) {
|
||||
const sign = x < 0 ? -1 : 1;
|
||||
const z = Math.abs(x);
|
||||
const t = 1 / (1 + 0.3275911 * z);
|
||||
const y = 1 - ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t * Math.exp(-z * z);
|
||||
return sign * y;
|
||||
}
|
||||
function normalCdf(z) { return 0.5 * (1 + erf(z / Math.SQRT2)); }
|
||||
function twoSided(z) { return 2 * (1 - normalCdf(Math.abs(z))); }
|
||||
function oneSidedUpper(z) { return 1 - normalCdf(z); }
|
||||
|
||||
function pct(x, digits = 2) { return Number.isFinite(x) ? `${(x * 100).toFixed(digits)}%` : "n/a"; }
|
||||
|
||||
// accuracy of `hits` out of `n` against a fixed reference rate
|
||||
function binomialZ(hits, n, p0) {
|
||||
if (!n || p0 <= 0 || p0 >= 1) return { z: NaN, p: NaN };
|
||||
const phat = hits / n;
|
||||
const z = (phat - p0) / Math.sqrt(p0 * (1 - p0) / n);
|
||||
return { z, p: oneSidedUpper(z) };
|
||||
}
|
||||
|
||||
function tStat(values) {
|
||||
const n = values.length;
|
||||
if (n < 2) return { n, mean: NaN, z: NaN, p: NaN };
|
||||
const mean = values.reduce((a, b) => a + b, 0) / n;
|
||||
const variance = values.reduce((a, b) => a + (b - mean) ** 2, 0) / (n - 1);
|
||||
const se = Math.sqrt(variance / n);
|
||||
const z = se > 0 ? mean / se : 0;
|
||||
return { n, mean, se, z, p: twoSided(z) };
|
||||
}
|
||||
|
||||
|
||||
// does the choice of direction carry information at all. invariant to how
|
||||
// often it picks each side, unlike raw accuracy.
|
||||
function discrimination(rows) {
|
||||
const pos = rows.filter((r) => r.direction === "positive");
|
||||
const neg = rows.filter((r) => r.direction === "negative");
|
||||
const upPos = pos.filter((r) => r.excess_return > 0).length;
|
||||
const upNeg = neg.filter((r) => r.excess_return > 0).length;
|
||||
const p1 = pos.length ? upPos / pos.length : NaN;
|
||||
const p2 = neg.length ? upNeg / neg.length : NaN;
|
||||
const pooled = (upPos + upNeg) / (pos.length + neg.length);
|
||||
const se = Math.sqrt(pooled * (1 - pooled) * (1 / pos.length + 1 / neg.length));
|
||||
const z = se > 0 ? (p1 - p2) / se : 0;
|
||||
return { pUpGivenPositive: p1, pUpGivenNegative: p2, nPositive: pos.length, nNegative: neg.length,
|
||||
spread: p1 - p2, z, p: twoSided(z), positiveShare: pos.length / rows.length };
|
||||
}
|
||||
|
||||
function signed(row) {
|
||||
// what a unit position in the predicted direction actually earned
|
||||
return row.direction === "negative" ? -row.excess_return : row.excess_return;
|
||||
}
|
||||
|
||||
function articleOf(row) {
|
||||
try {
|
||||
const parsed = JSON.parse(row.evidence_article_ids || "[]");
|
||||
return Array.isArray(parsed) && parsed.length ? String(parsed[0]) : null;
|
||||
} catch (error) {
|
||||
console.error(`[score] unparseable evidence on prediction ${row.id}:`, error.message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function load(db, { runId, since, until, createdSince }) {
|
||||
const where = ["p.origin = 'replay'", "o.prediction_id IS NOT NULL"];
|
||||
const params = [];
|
||||
if (runId) { where.push("p.replay_run_id = ?"); params.push(runId); }
|
||||
if (since) { where.push("date(p.information_cutoff) >= date(?)"); params.push(since); }
|
||||
if (until) { where.push("date(p.information_cutoff) <= date(?)"); params.push(until); }
|
||||
// the train/test split is on when the prediction was MADE, because that is
|
||||
// what fixes which prompt produced it. cutoff dates only correlate with it.
|
||||
if (createdSince) { where.push("p.created_at >= ?"); params.push(createdSince); }
|
||||
return db.prepare(`
|
||||
SELECT p.id, p.instrument, p.direction, p.event_type, p.horizon_days, p.information_cutoff,
|
||||
p.evidence_article_ids, p.replay_run_id, p.strategy_version,
|
||||
pr.prompt_version, pr.coordinator_model,
|
||||
o.direction_correct, o.excess_return
|
||||
FROM autonomy_predictions p
|
||||
JOIN autonomy_proposals pr ON pr.id = p.proposal_id
|
||||
JOIN autonomy_outcomes o ON o.prediction_id = p.id
|
||||
WHERE ${where.join(" AND ")}
|
||||
ORDER BY p.information_cutoff ASC, p.id ASC
|
||||
`).all(...params);
|
||||
}
|
||||
|
||||
function baselines(rows) {
|
||||
const n = rows.length;
|
||||
const up = rows.filter((r) => r.excess_return > 0).length;
|
||||
return {
|
||||
n,
|
||||
alwaysPositive: up / n,
|
||||
alwaysNegative: (n - up) / n,
|
||||
// mean of a unit long in every name, which is what always_positive earns
|
||||
alwaysPositiveExcess: rows.reduce((a, r) => a + r.excess_return, 0) / n,
|
||||
};
|
||||
}
|
||||
|
||||
function scoreRun(rows, label) {
|
||||
const n = rows.length;
|
||||
if (!n) { console.log(`\n${label}: no scored predictions\n`); return null; }
|
||||
const hits = rows.filter((r) => r.direction_correct).length;
|
||||
const acc = hits / n;
|
||||
const base = baselines(rows);
|
||||
const best = Math.max(base.alwaysPositive, base.alwaysNegative);
|
||||
const bestName = base.alwaysNegative >= base.alwaysPositive ? "always_negative" : "always_positive";
|
||||
const t1 = binomialZ(hits, n, best);
|
||||
// pair against the constant on the identical bars. where the system already
|
||||
// agrees with the constant the difference is zero and contributes nothing,
|
||||
// which is exactly right.
|
||||
const constantSign = bestName === "always_negative" ? -1 : 1;
|
||||
const t2 = tStat(rows.map((r) => signed(r) - constantSign * r.excess_return));
|
||||
const rawSigned = tStat(rows.map(signed));
|
||||
const constantExcess = rows.reduce((a, r) => a + constantSign * r.excess_return, 0) / n;
|
||||
|
||||
console.log(`\n=== ${label} ===`);
|
||||
const models = [...new Set(rows.map((r) => r.coordinator_model))];
|
||||
const prompts = [...new Set(rows.map((r) => r.prompt_version))];
|
||||
console.log(` span ${rows[0].information_cutoff.slice(0, 10)} .. ${rows[n - 1].information_cutoff.slice(0, 10)}`);
|
||||
console.log(` models ${models.join(", ")}`);
|
||||
console.log(` prompts ${prompts.join(", ")}`);
|
||||
console.log(` scored ${n} predictions over ${new Set(rows.map(articleOf)).size} articles`);
|
||||
console.log("");
|
||||
console.log(` system accuracy ${pct(acc)} (${hits}/${n})`);
|
||||
console.log(` always_negative ${pct(base.alwaysNegative)} <- share of bars that underperformed SPY`);
|
||||
console.log(` always_positive ${pct(base.alwaysPositive)}`);
|
||||
console.log(` coin flip 50.00%`);
|
||||
console.log("");
|
||||
console.log(` T1 vs ${bestName}: z=${t1.z.toFixed(3)} p=${t1.p.toFixed(4)} (one sided, does the system beat the bar)`);
|
||||
if (t1.z < 0) console.log(` the system is BELOW the constant. p=${twoSided(t1.z).toFixed(4)} two sided on being different from it.`);
|
||||
console.log(` edge over the bar ${((acc - best) * 100).toFixed(2)} points`);
|
||||
console.log(` T2 signed excess vs ${bestName}: ${pct(t2.mean, 3)} per prediction,`
|
||||
+ ` t=${t2.z.toFixed(3)} p=${t2.p.toFixed(4)}`);
|
||||
console.log(` system ${pct(rawSigned.mean, 3)} ${bestName} ${pct(constantExcess, 3)}`
|
||||
+ ` always_positive ${pct(base.alwaysPositiveExcess, 3)}`);
|
||||
const t3 = discrimination(rows);
|
||||
console.log("");
|
||||
console.log(` T3 discrimination: P(up | said positive) ${pct(t3.pUpGivenPositive)} (n=${t3.nPositive})`);
|
||||
console.log(` P(up | said negative) ${pct(t3.pUpGivenNegative)} (n=${t3.nNegative})`);
|
||||
console.log(` spread ${(t3.spread * 100).toFixed(2)} points, z=${t3.z.toFixed(3)} p=${t3.p.toFixed(4)}`);
|
||||
console.log(` it says positive on ${pct(t3.positiveShare)} of calls while ${pct(base.alwaysPositive)}`
|
||||
+ ` of the bars went up, so the prior is off by ${((t3.positiveShare - base.alwaysPositive) * 100).toFixed(1)} points`);
|
||||
return { n, acc, hits, best, bestName, t1, t2, t3, base };
|
||||
}
|
||||
|
||||
function slice(rows, key, label, minimum = 40) {
|
||||
const groups = new Map();
|
||||
for (const row of rows) {
|
||||
const k = String(row[key]);
|
||||
if (!groups.has(k)) groups.set(k, []);
|
||||
groups.get(k).push(row);
|
||||
}
|
||||
const kept = [...groups.entries()].filter(([, v]) => v.length >= minimum);
|
||||
if (!kept.length) return;
|
||||
const factor = kept.length;
|
||||
console.log(`\n descriptive by ${label} (n>=${minimum}, bonferroni x${factor}, NOT a test)`);
|
||||
const scored = kept.map(([k, v]) => {
|
||||
const hits = v.filter((r) => r.direction_correct).length;
|
||||
const base = baselines(v);
|
||||
const bar = Math.max(base.alwaysPositive, base.alwaysNegative);
|
||||
const { z } = binomialZ(hits, v.length, bar);
|
||||
return { k, n: v.length, acc: hits / v.length, bar, z, p: Math.min(1, twoSided(z) * factor),
|
||||
excess: tStat(v.map(signed)).mean };
|
||||
}).sort((a, b) => b.acc - a.acc);
|
||||
for (const s of scored) {
|
||||
// a small p here can mean significantly WORSE than the constant, which read
|
||||
// like good news the first time this printed. say which side it fell on.
|
||||
const side = s.acc >= s.bar ? "above" : "below";
|
||||
const flag = s.p < 0.05 ? ` * ${side} bar` : "";
|
||||
console.log(` ${s.k.padEnd(24)} n=${String(s.n).padEnd(5)} acc=${pct(s.acc).padEnd(8)}`
|
||||
+ ` bar=${pct(s.bar).padEnd(8)} signed_excess=${pct(s.excess, 3).padEnd(9)} p_adj=${s.p.toFixed(3)}${flag}`);
|
||||
}
|
||||
}
|
||||
|
||||
// per article accuracy, so an article that produced 11 predictions does not
|
||||
// count eleven times against one that produced a single call
|
||||
function byArticle(rows) {
|
||||
const map = new Map();
|
||||
for (const row of rows) {
|
||||
const id = articleOf(row);
|
||||
if (!id) continue;
|
||||
if (!map.has(id)) map.set(id, []);
|
||||
map.get(id).push(row);
|
||||
}
|
||||
const out = new Map();
|
||||
for (const [id, list] of map) {
|
||||
out.set(id, {
|
||||
accuracy: list.filter((r) => r.direction_correct).length / list.length,
|
||||
signed: list.reduce((a, r) => a + signed(r), 0) / list.length,
|
||||
count: list.length,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function paired(baseRows, candRows) {
|
||||
const a = byArticle(baseRows);
|
||||
const b = byArticle(candRows);
|
||||
const shared = [...a.keys()].filter((id) => b.has(id));
|
||||
console.log(`\n=== T4 paired comparison ===`);
|
||||
console.log(` baseline articles ${a.size}, candidate articles ${b.size}, shared ${shared.length}`);
|
||||
if (shared.length < 30) {
|
||||
console.log(" not enough shared articles to say anything. run the candidate over the baseline's articles first.");
|
||||
return;
|
||||
}
|
||||
const accDiff = shared.map((id) => b.get(id).accuracy - a.get(id).accuracy);
|
||||
const excessDiff = shared.map((id) => b.get(id).signed - a.get(id).signed);
|
||||
const baseAcc = shared.reduce((s, id) => s + a.get(id).accuracy, 0) / shared.length;
|
||||
const candAcc = shared.reduce((s, id) => s + b.get(id).accuracy, 0) / shared.length;
|
||||
const tAcc = tStat(accDiff);
|
||||
const tExc = tStat(excessDiff);
|
||||
const better = shared.filter((id) => b.get(id).accuracy > a.get(id).accuracy).length;
|
||||
const worse = shared.filter((id) => b.get(id).accuracy < a.get(id).accuracy).length;
|
||||
|
||||
console.log(` baseline per article accuracy ${pct(baseAcc)}`);
|
||||
console.log(` candidate per article accuracy ${pct(candAcc)}`);
|
||||
console.log(` articles improved ${better}, degraded ${worse}, unchanged ${shared.length - better - worse}`);
|
||||
console.log(` T4 accuracy delta ${pct(tAcc.mean)} t=${tAcc.z.toFixed(3)} p=${tAcc.p.toFixed(4)}`);
|
||||
console.log(` signed excess delta ${pct(tExc.mean, 3)} t=${tExc.z.toFixed(3)} p=${tExc.p.toFixed(4)}`);
|
||||
console.log(tAcc.p < 0.05
|
||||
? (tAcc.mean > 0 ? " VERDICT: the candidate is better on the same articles." : " VERDICT: the candidate is WORSE on the same articles.")
|
||||
: " VERDICT: no detectable difference on the same articles.");
|
||||
}
|
||||
|
||||
function main() {
|
||||
const opts = args();
|
||||
const db = new Database(INTELLIGENCE, { readonly: true });
|
||||
db.pragma("busy_timeout = 20000");
|
||||
|
||||
const runs = db.prepare("SELECT * FROM autonomy_replay_runs ORDER BY id").all();
|
||||
console.log("replay runs on record:");
|
||||
for (const run of runs) {
|
||||
console.log(` #${run.id} ${run.status.padEnd(9)} watermark=${String(run.watermark_at).slice(0, 10)}`
|
||||
+ ` ${run.strategy_version}/${run.prompt_version} ${run.coordinator_model}`
|
||||
+ ` processed=${run.processed_articles}`);
|
||||
}
|
||||
|
||||
if (opts.baseline && opts.candidate) {
|
||||
const window = { since: opts.since, until: opts.until, createdSince: opts["created-since"] };
|
||||
const baseRows = load(db, { runId: Number(opts.baseline), ...window });
|
||||
const candRows = load(db, { runId: Number(opts.candidate), ...window });
|
||||
scoreRun(baseRows, `run ${opts.baseline} (baseline)`);
|
||||
scoreRun(candRows, `run ${opts.candidate} (candidate)`);
|
||||
paired(baseRows, candRows);
|
||||
db.close();
|
||||
return;
|
||||
}
|
||||
|
||||
const runId = opts.run && opts.run !== true ? Number(opts.run) : null;
|
||||
const rows = load(db, { runId, since: opts.since, until: opts.until, createdSince: opts["created-since"] });
|
||||
const summary = scoreRun(rows, runId ? `run ${runId}` : "all replay runs");
|
||||
if (summary) {
|
||||
slice(rows, "direction", "direction");
|
||||
slice(rows, "horizon_days", "horizon");
|
||||
slice(rows, "event_type", "event family");
|
||||
slice(rows, "instrument", "instrument");
|
||||
console.log("");
|
||||
}
|
||||
db.close();
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,144 @@
|
||||
#!/usr/bin/env node
|
||||
/*
|
||||
* Start the next replay run over the SAME articles as the previous one, with a
|
||||
* feedback brief built from the previous run's own scored outcomes.
|
||||
*
|
||||
* The point is a single variable. The evaluation slice is the articles the
|
||||
* parent answered under the current prompt and the current model, so run N+1
|
||||
* differs from run N by the brief and nothing else. Articles the parent
|
||||
* answered under an older prompt are the TRAINING half and are never replayed,
|
||||
* because deriving the lesson and grading it on the same rows measures nothing.
|
||||
*
|
||||
* What it writes, all additive:
|
||||
* - parent run status running -> paused. Its cursor is untouched, so it can
|
||||
* be resumed later exactly where it stopped.
|
||||
* - one new row in autonomy_replay_runs carrying the brief.
|
||||
* - one row per evaluation article in autonomy_replay_run_articles.
|
||||
* Nothing is deleted and no existing prediction, outcome or snapshot is touched.
|
||||
*
|
||||
* node scripts/start-replay-run.js --parent 1 --split "2026-09-04 19:30" --dry-run
|
||||
* node scripts/start-replay-run.js --parent 1 --split "2026-09-04 19:30" --commit
|
||||
*/
|
||||
const Database = require("better-sqlite3");
|
||||
const { buildFeedbackBrief } = require("./build-feedback-brief");
|
||||
const { STRATEGY_VERSION, PROMPT_VERSION } = require("../workers/replayWorker");
|
||||
|
||||
const INTELLIGENCE = process.env.INTELLIGENCE_DB || "/data/intelligence.sqlite";
|
||||
|
||||
function options() {
|
||||
const argv = process.argv.slice(2);
|
||||
const out = {};
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
if (!argv[i].startsWith("--")) continue;
|
||||
const key = argv[i].slice(2);
|
||||
const next = argv[i + 1];
|
||||
out[key] = (next && !next.startsWith("--")) ? (i += 1, next) : true;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// the pinned table arrives with the schema migration, so a dry run from a
|
||||
// container that has not been redeployed yet should still be able to report
|
||||
function countOf(db, table) {
|
||||
try {
|
||||
return db.prepare(`SELECT COUNT(*) AS c FROM ${table}`).get().c;
|
||||
} catch (error) {
|
||||
console.error(`[replay-run] cannot count ${table}:`, error.message);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
function articleIdOf(raw, predictionId) {
|
||||
try {
|
||||
const parsed = JSON.parse(raw || "[]");
|
||||
return Array.isArray(parsed) && parsed.length ? Number(parsed[0]) : null;
|
||||
} catch (error) {
|
||||
console.error(`[replay-run] unparseable evidence on prediction ${predictionId}:`, error.message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function main() {
|
||||
const opts = options();
|
||||
const parentId = Number(opts.parent || 1);
|
||||
const split = String(opts.split || "2026-09-04 19:30");
|
||||
const commit = opts.commit === true;
|
||||
const model = String(opts.model || process.env.OPEN_ROUTER_LLM_MODEL || "");
|
||||
|
||||
const db = new Database(INTELLIGENCE, { readonly: !commit });
|
||||
db.pragma("busy_timeout = 20000");
|
||||
|
||||
const parent = db.prepare("SELECT * FROM autonomy_replay_runs WHERE id = ?").get(parentId);
|
||||
if (!parent) throw new Error(`replay run ${parentId} does not exist`);
|
||||
|
||||
// row counts before, so the report can show nothing went missing
|
||||
const before = {
|
||||
runs: countOf(db, "autonomy_replay_runs"),
|
||||
predictions: countOf(db, "autonomy_predictions"),
|
||||
outcomes: countOf(db, "autonomy_outcomes"),
|
||||
pinned: countOf(db, "autonomy_replay_run_articles"),
|
||||
};
|
||||
|
||||
const evaluation = db.prepare(`
|
||||
SELECT p.id, p.evidence_article_ids, p.information_cutoff
|
||||
FROM autonomy_predictions p
|
||||
WHERE p.origin = 'replay' AND p.replay_run_id = ? AND p.created_at >= ?
|
||||
`).all(parentId, split);
|
||||
|
||||
const articles = new Map();
|
||||
for (const row of evaluation) {
|
||||
const id = articleIdOf(row.evidence_article_ids, row.id);
|
||||
if (id && !articles.has(id)) articles.set(id, String(row.information_cutoff));
|
||||
}
|
||||
|
||||
const { text: brief, stats } = buildFeedbackBrief(db, { runId: parentId, createdBefore: split });
|
||||
|
||||
console.log(`parent run #${parentId} ${parent.status}, watermark ${parent.watermark_at}`);
|
||||
console.log(`split at ${split}`);
|
||||
console.log(` training predictions (before split, feed the brief): ${stats.n}`);
|
||||
console.log(` evaluation predictions (at or after split): ${evaluation.length}`);
|
||||
console.log(` evaluation articles to replay: ${articles.size}`);
|
||||
console.log(` brief: ${brief.split("\n").length} lines, ${brief.length} chars`);
|
||||
console.log(` new run would be ${STRATEGY_VERSION}/${PROMPT_VERSION} on ${model || "(model from config at run time)"}`);
|
||||
|
||||
if (!articles.size) throw new Error("no evaluation articles, refusing to create an empty run");
|
||||
|
||||
if (!commit) {
|
||||
console.log("\ndry run, nothing written. pass --commit to apply.");
|
||||
db.close();
|
||||
return;
|
||||
}
|
||||
|
||||
const apply = db.transaction(() => {
|
||||
db.prepare("UPDATE autonomy_replay_runs SET status = 'paused', updated_at = datetime('now') WHERE id = ? AND status = 'running'").run(parentId);
|
||||
const created = db.prepare(`
|
||||
INSERT INTO autonomy_replay_runs (watermark_at, strategy_version, prompt_version, coordinator_model, parent_run_id, feedback_brief)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
`).run(parent.watermark_at, STRATEGY_VERSION, PROMPT_VERSION, model || parent.coordinator_model, parentId, brief);
|
||||
const runId = created.lastInsertRowid;
|
||||
const insert = db.prepare("INSERT OR IGNORE INTO autonomy_replay_run_articles (run_id, article_id, effective_at) VALUES (?, ?, ?)");
|
||||
for (const [articleId, effectiveAt] of articles) insert.run(runId, articleId, effectiveAt);
|
||||
return runId;
|
||||
});
|
||||
const runId = apply();
|
||||
|
||||
const after = {
|
||||
runs: countOf(db, "autonomy_replay_runs"),
|
||||
predictions: countOf(db, "autonomy_predictions"),
|
||||
outcomes: countOf(db, "autonomy_outcomes"),
|
||||
pinned: countOf(db, "autonomy_replay_run_articles"),
|
||||
};
|
||||
console.log(`\ncreated replay run #${runId}, parent #${parentId} is now`
|
||||
+ ` ${db.prepare("SELECT status FROM autonomy_replay_runs WHERE id = ?").get(parentId).status}`);
|
||||
console.log("row counts before -> after");
|
||||
for (const key of Object.keys(before)) {
|
||||
const moved = after[key] - before[key];
|
||||
console.log(` ${key.padEnd(12)} ${before[key]} -> ${after[key]} (${moved >= 0 ? "+" : ""}${moved})`);
|
||||
}
|
||||
if (after.predictions !== before.predictions || after.outcomes !== before.outcomes) {
|
||||
console.error("predictions or outcomes changed, that should not happen here");
|
||||
}
|
||||
db.close();
|
||||
}
|
||||
|
||||
main();
|
||||
Reference in New Issue
Block a user