feat: measure whether the initial market reaction conditions anything
The coordinator is scored on excess return vs SPY starting at the information cutoff, so the announcement move sits outside the scored window. That move is the best documented conditioner for post event drift and we were discarding it. This measures whether keeping it would buy us anything, before any of it gets wired into the cohort key or the prompt. Reaction is measured from the last close before the event's first article up to the outcome's own entry price, so the reaction and forward windows touch but never overlap. Read only, and it caches price history so it can be re-run cheaply as more outcomes mature. T1 and T2 are pre-registered in the header because sweeping buckets over 611 outcomes that are half one ticker will always turn up something. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WnNxwxfXSbeNtjvtz5gayb
This commit is contained in:
ImBenji
@@ -0,0 +1,184 @@
|
||||
#!/usr/bin/env node
|
||||
/*
|
||||
* Lever 1: does conditioning on the initial market reaction reveal any
|
||||
* discrimination in this pipeline?
|
||||
*
|
||||
* The coordinator is scored on excess return vs SPY starting at the information
|
||||
* cutoff, so the announcement move itself sits OUTSIDE the scored window. That
|
||||
* move is the best documented conditioner for post event drift, and we throw it
|
||||
* away. This measures whether it is worth keeping.
|
||||
*
|
||||
* reaction = instrument excess vs SPY from the last close BEFORE the event's
|
||||
* first article, up to the outcome's own entry price at the cutoff.
|
||||
* forward = the already stored excess_return over the horizon.
|
||||
* The two windows touch but never overlap, so there is no lookahead.
|
||||
*
|
||||
* Read only. Opens both databases read only and writes nothing but a price cache.
|
||||
*
|
||||
* PRE REGISTERED TESTS (declared before looking, so the buckets cannot be tuned):
|
||||
* T1 does the reaction bucket predict the SIGN of forward excess return?
|
||||
* (is there drift/reversal in this sample at all, model aside)
|
||||
* T2 within a bucket, does the coordinator's direction discriminate?
|
||||
* (does the model add anything on top of T1)
|
||||
* Everything else printed is descriptive, not a test.
|
||||
*/
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const Database = require('better-sqlite3');
|
||||
|
||||
const ARCHIVE = process.env.DURIIN_DB || '/data/archive.sqlite';
|
||||
const INTELLIGENCE = process.env.INTELLIGENCE_DB || '/data/intelligence.sqlite';
|
||||
const CACHE = process.env.PRICE_CACHE || '/data/.price-cache';
|
||||
const BENCHMARK = 'SPY';
|
||||
|
||||
// fixed bands on the reaction, declared up front. not quantiles, so the cut
|
||||
// points cannot drift with the data.
|
||||
const BANDS = [
|
||||
{ key: 'strong_down', min: -Infinity, max: -0.05 },
|
||||
{ key: 'down', min: -0.05, max: -0.015 },
|
||||
{ key: 'flat', min: -0.015, max: 0.015 },
|
||||
{ key: 'up', min: 0.015, max: 0.05 },
|
||||
{ key: 'strong_up', min: 0.05, max: Infinity },
|
||||
];
|
||||
function band(reaction) {
|
||||
return (BANDS.find((b) => reaction >= b.min && reaction < b.max) || { key: 'unknown' }).key;
|
||||
}
|
||||
|
||||
function sleep(ms) { return new Promise((r) => setTimeout(r, ms)); }
|
||||
|
||||
async function fetchHistory(symbol) {
|
||||
fs.mkdirSync(CACHE, { recursive: true });
|
||||
const file = path.join(CACHE, `${symbol.replace(/[^A-Za-z0-9._-]/g, '_')}.json`);
|
||||
if (fs.existsSync(file)) {
|
||||
try { return JSON.parse(fs.readFileSync(file, 'utf8')); }
|
||||
catch (error) { console.error(`[reaction] bad cache for ${symbol}, refetching:`, error.message); }
|
||||
}
|
||||
const url = `https://query1.finance.yahoo.com/v8/finance/chart/${encodeURIComponent(symbol)}`
|
||||
+ `?period1=946684800&period2=${Math.floor(Date.now() / 1000)}&interval=1d`;
|
||||
let rows = [];
|
||||
try {
|
||||
const response = await fetch(url, { headers: { 'User-Agent': 'Mozilla/5.0' } });
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||
const result = (await response.json())?.chart?.result?.[0];
|
||||
rows = (result?.timestamp || []).map((ts, i) => ({
|
||||
date: new Date(ts * 1000).toISOString().slice(0, 10),
|
||||
close: result.indicators?.quote?.[0]?.close?.[i],
|
||||
})).filter((r) => Number.isFinite(r.close));
|
||||
} catch (error) {
|
||||
console.error(`[reaction] history fetch failed for ${symbol}:`, error.message);
|
||||
}
|
||||
fs.writeFileSync(file, JSON.stringify(rows));
|
||||
await sleep(250);
|
||||
return rows;
|
||||
}
|
||||
|
||||
const onOrAfter = (h, d) => h.find((r) => r.date >= d)?.close ?? null;
|
||||
const strictlyBefore = (h, d) => { for (let i = h.length - 1; i >= 0; i--) if (h[i].date < d) return h[i].close; return null; };
|
||||
|
||||
// two proportion z test. returns z and a normal approx two sided p.
|
||||
function ztest(x1, n1, x2, n2) {
|
||||
if (!n1 || !n2) return null;
|
||||
const p1 = x1 / n1, p2 = x2 / n2, p = (x1 + x2) / (n1 + n2);
|
||||
const se = Math.sqrt(p * (1 - p) * (1 / n1 + 1 / n2));
|
||||
if (!se) return null;
|
||||
const z = (p1 - p2) / se;
|
||||
// logistic approx to the normal cdf. has to be fed |z|, otherwise the two
|
||||
// sided p comes back above 1 for negative z, which is nonsense.
|
||||
const cdf = (v) => 1 / (1 + Math.exp(-0.07056 * v ** 3 - 1.5976 * v));
|
||||
const pValue = Math.min(1, 2 * (1 - cdf(Math.abs(z))));
|
||||
return { z, p: pValue, p1, p2 };
|
||||
}
|
||||
const pct = (x) => (x === null || x === undefined || Number.isNaN(x) ? ' n/a ' : `${(100 * x).toFixed(1)}%`);
|
||||
|
||||
async function main() {
|
||||
const intel = new Database(INTELLIGENCE, { readonly: true });
|
||||
const archive = new Database(ARCHIVE, { readonly: true });
|
||||
|
||||
const rows = intel.prepare(`
|
||||
SELECT p.id, p.instrument, p.direction, p.event_id, p.information_cutoff, p.horizon_days, p.origin,
|
||||
o.excess_return, o.direction_correct, o.price_0, o.benchmark_0
|
||||
FROM autonomy_predictions p
|
||||
JOIN autonomy_outcomes o ON o.prediction_id = p.id
|
||||
WHERE o.excess_return IS NOT NULL
|
||||
`).all();
|
||||
console.log(`[reaction] matured outcomes: ${rows.length}`);
|
||||
|
||||
const eventStart = archive.prepare('SELECT MIN(pub_date_effective) AS first FROM articles WHERE event_id = ?');
|
||||
const symbols = [...new Set(rows.map((r) => r.instrument))];
|
||||
console.log(`[reaction] distinct instruments: ${symbols.length}, fetching history (cached)...`);
|
||||
const hist = new Map();
|
||||
for (const s of [BENCHMARK, ...symbols]) hist.set(s, await fetchHistory(s));
|
||||
|
||||
const bench = hist.get(BENCHMARK);
|
||||
const scored = [];
|
||||
let skipped = 0;
|
||||
for (const r of rows) {
|
||||
const first = r.event_id ? eventStart.get(r.event_id)?.first : null;
|
||||
if (!first) { skipped++; continue; }
|
||||
const startDay = String(first).slice(0, 10);
|
||||
const cutoffDay = String(r.information_cutoff).slice(0, 10);
|
||||
const h = hist.get(r.instrument) || [];
|
||||
// baseline is the last close strictly before the first article, so the whole
|
||||
// announcement move is inside the reaction window
|
||||
const pPre = strictlyBefore(h, startDay);
|
||||
const bPre = strictlyBefore(bench, startDay);
|
||||
const pAt = Number.isFinite(r.price_0) ? r.price_0 : onOrAfter(h, cutoffDay);
|
||||
const bAt = Number.isFinite(r.benchmark_0) ? r.benchmark_0 : onOrAfter(bench, cutoffDay);
|
||||
if (![pPre, bPre, pAt, bAt].every(Number.isFinite) || !pPre || !bPre) { skipped++; continue; }
|
||||
const reaction = (pAt - pPre) / pPre - (bAt - bPre) / bPre;
|
||||
if (!Number.isFinite(reaction)) { skipped++; continue; }
|
||||
scored.push({ ...r, reaction, bucket: band(reaction), fwdUp: r.excess_return > 0 ? 1 : 0 });
|
||||
}
|
||||
console.log(`[reaction] scored: ${scored.length}, skipped for missing prices/dates: ${skipped}\n`);
|
||||
if (!scored.length) return;
|
||||
|
||||
const report = (label, set) => {
|
||||
if (!set.length) return;
|
||||
console.log(`\n================ ${label} (n=${set.length}) ================`);
|
||||
console.log('bucket n fwd_up% model_acc% mean_fwd% P(up|pred+) P(up|pred-) z p');
|
||||
for (const b of BANDS) {
|
||||
const g = set.filter((r) => r.bucket === b.key);
|
||||
if (!g.length) { console.log(`${b.key.padEnd(12)} 0 - - - - - - -`); continue; }
|
||||
const pos = g.filter((r) => r.direction === 'positive');
|
||||
const neg = g.filter((r) => r.direction === 'negative');
|
||||
const t = ztest(pos.filter((r) => r.fwdUp).length, pos.length, neg.filter((r) => r.fwdUp).length, neg.length);
|
||||
const acc = g.filter((r) => r.direction_correct).length / g.length;
|
||||
const meanFwd = g.reduce((s, r) => s + r.excess_return, 0) / g.length;
|
||||
console.log(
|
||||
`${b.key.padEnd(12)} ${String(g.length).padStart(4)} ${pct(g.filter((r) => r.fwdUp).length / g.length)} ${pct(acc)} `
|
||||
+ `${(100 * meanFwd).toFixed(2).padStart(6)}% ${pct(t ? t.p1 : null)} ${pct(t ? t.p2 : null)} `
|
||||
+ `${t ? t.z.toFixed(2).padStart(6) : ' n/a'} ${t ? t.p.toFixed(3) : ' n/a'}`
|
||||
);
|
||||
}
|
||||
|
||||
// T1: does the reaction itself predict the forward sign?
|
||||
const upSide = set.filter((r) => r.bucket === 'up' || r.bucket === 'strong_up');
|
||||
const downSide = set.filter((r) => r.bucket === 'down' || r.bucket === 'strong_down');
|
||||
const t1 = ztest(upSide.filter((r) => r.fwdUp).length, upSide.length, downSide.filter((r) => r.fwdUp).length, downSide.length);
|
||||
console.log(`\n[T1] forward-up rate after a POSITIVE reaction vs after a NEGATIVE reaction`);
|
||||
if (t1) {
|
||||
console.log(` ${pct(t1.p1)} (n=${upSide.length}) vs ${pct(t1.p2)} (n=${downSide.length}) z=${t1.z.toFixed(2)} p=${t1.p.toFixed(3)}`);
|
||||
console.log(` ${Math.abs(t1.z) >= 1.96 ? (t1.z > 0 ? '=> CONTINUATION (drift) is present' : '=> REVERSAL is present') : '=> no drift or reversal detectable'}`);
|
||||
} else console.log(' insufficient data');
|
||||
|
||||
// T2: pooled model discrimination within buckets, vs unconditional
|
||||
const allPos = set.filter((r) => r.direction === 'positive');
|
||||
const allNeg = set.filter((r) => r.direction === 'negative');
|
||||
const t2 = ztest(allPos.filter((r) => r.fwdUp).length, allPos.length, allNeg.filter((r) => r.fwdUp).length, allNeg.length);
|
||||
console.log(`\n[T2] unconditional model discrimination P(up|pred+) - P(up|pred-)`);
|
||||
if (t2) console.log(` ${pct(t2.p1)} vs ${pct(t2.p2)} z=${t2.z.toFixed(2)} p=${t2.p.toFixed(3)}`
|
||||
+ ` ${Math.abs(t2.z) >= 1.96 ? '=> SIGNIFICANT' : '=> not distinguishable from zero'}`);
|
||||
else console.log(' insufficient data');
|
||||
};
|
||||
|
||||
report('ALL', scored);
|
||||
const topTicker = [...scored.reduce((m, r) => m.set(r.instrument, (m.get(r.instrument) || 0) + 1), new Map())]
|
||||
.sort((a, b) => b[1] - a[1])[0];
|
||||
console.log(`\n\n(most common instrument: ${topTicker[0]} with ${topTicker[1]} of ${scored.length})`);
|
||||
report(`EXCLUDING ${topTicker[0]}`, scored.filter((r) => r.instrument !== topTicker[0]));
|
||||
|
||||
console.log('\n[reaction] reminder: T1 and T2 were pre-registered. Per-bucket rows are');
|
||||
console.log('[reaction] descriptive only, do not read a single bucket as a finding.');
|
||||
}
|
||||
|
||||
main().catch((error) => { console.error('[reaction] fatal:', error.message, error.stack); process.exit(1); });
|
||||
Reference in New Issue
Block a user