diff --git a/scripts/analyze-reaction-conditioning.js b/scripts/analyze-reaction-conditioning.js new file mode 100644 index 0000000..7a4c47a --- /dev/null +++ b/scripts/analyze-reaction-conditioning.js @@ -0,0 +1,184 @@ +#!/usr/bin/env node +/* + * Lever 1: does conditioning on the initial market reaction reveal any + * discrimination in this pipeline? + * + * The coordinator is scored on excess return vs SPY starting at the information + * cutoff, so the announcement move itself sits OUTSIDE the scored window. That + * move is the best documented conditioner for post event drift, and we throw it + * away. This measures whether it is worth keeping. + * + * reaction = instrument excess vs SPY from the last close BEFORE the event's + * first article, up to the outcome's own entry price at the cutoff. + * forward = the already stored excess_return over the horizon. + * The two windows touch but never overlap, so there is no lookahead. + * + * Read only. Opens both databases read only and writes nothing but a price cache. + * + * PRE REGISTERED TESTS (declared before looking, so the buckets cannot be tuned): + * T1 does the reaction bucket predict the SIGN of forward excess return? + * (is there drift/reversal in this sample at all, model aside) + * T2 within a bucket, does the coordinator's direction discriminate? + * (does the model add anything on top of T1) + * Everything else printed is descriptive, not a test. + */ +const fs = require('fs'); +const path = require('path'); +const Database = require('better-sqlite3'); + +const ARCHIVE = process.env.DURIIN_DB || '/data/archive.sqlite'; +const INTELLIGENCE = process.env.INTELLIGENCE_DB || '/data/intelligence.sqlite'; +const CACHE = process.env.PRICE_CACHE || '/data/.price-cache'; +const BENCHMARK = 'SPY'; + +// fixed bands on the reaction, declared up front. not quantiles, so the cut +// points cannot drift with the data. +const BANDS = [ + { key: 'strong_down', min: -Infinity, max: -0.05 }, + { key: 'down', min: -0.05, max: -0.015 }, + { key: 'flat', min: -0.015, max: 0.015 }, + { key: 'up', min: 0.015, max: 0.05 }, + { key: 'strong_up', min: 0.05, max: Infinity }, +]; +function band(reaction) { + return (BANDS.find((b) => reaction >= b.min && reaction < b.max) || { key: 'unknown' }).key; +} + +function sleep(ms) { return new Promise((r) => setTimeout(r, ms)); } + +async function fetchHistory(symbol) { + fs.mkdirSync(CACHE, { recursive: true }); + const file = path.join(CACHE, `${symbol.replace(/[^A-Za-z0-9._-]/g, '_')}.json`); + if (fs.existsSync(file)) { + try { return JSON.parse(fs.readFileSync(file, 'utf8')); } + catch (error) { console.error(`[reaction] bad cache for ${symbol}, refetching:`, error.message); } + } + const url = `https://query1.finance.yahoo.com/v8/finance/chart/${encodeURIComponent(symbol)}` + + `?period1=946684800&period2=${Math.floor(Date.now() / 1000)}&interval=1d`; + let rows = []; + try { + const response = await fetch(url, { headers: { 'User-Agent': 'Mozilla/5.0' } }); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const result = (await response.json())?.chart?.result?.[0]; + rows = (result?.timestamp || []).map((ts, i) => ({ + date: new Date(ts * 1000).toISOString().slice(0, 10), + close: result.indicators?.quote?.[0]?.close?.[i], + })).filter((r) => Number.isFinite(r.close)); + } catch (error) { + console.error(`[reaction] history fetch failed for ${symbol}:`, error.message); + } + fs.writeFileSync(file, JSON.stringify(rows)); + await sleep(250); + return rows; +} + +const onOrAfter = (h, d) => h.find((r) => r.date >= d)?.close ?? null; +const strictlyBefore = (h, d) => { for (let i = h.length - 1; i >= 0; i--) if (h[i].date < d) return h[i].close; return null; }; + +// two proportion z test. returns z and a normal approx two sided p. +function ztest(x1, n1, x2, n2) { + if (!n1 || !n2) return null; + const p1 = x1 / n1, p2 = x2 / n2, p = (x1 + x2) / (n1 + n2); + const se = Math.sqrt(p * (1 - p) * (1 / n1 + 1 / n2)); + if (!se) return null; + const z = (p1 - p2) / se; + // logistic approx to the normal cdf. has to be fed |z|, otherwise the two + // sided p comes back above 1 for negative z, which is nonsense. + const cdf = (v) => 1 / (1 + Math.exp(-0.07056 * v ** 3 - 1.5976 * v)); + const pValue = Math.min(1, 2 * (1 - cdf(Math.abs(z)))); + return { z, p: pValue, p1, p2 }; +} +const pct = (x) => (x === null || x === undefined || Number.isNaN(x) ? ' n/a ' : `${(100 * x).toFixed(1)}%`); + +async function main() { + const intel = new Database(INTELLIGENCE, { readonly: true }); + const archive = new Database(ARCHIVE, { readonly: true }); + + const rows = intel.prepare(` + SELECT p.id, p.instrument, p.direction, p.event_id, p.information_cutoff, p.horizon_days, p.origin, + o.excess_return, o.direction_correct, o.price_0, o.benchmark_0 + FROM autonomy_predictions p + JOIN autonomy_outcomes o ON o.prediction_id = p.id + WHERE o.excess_return IS NOT NULL + `).all(); + console.log(`[reaction] matured outcomes: ${rows.length}`); + + const eventStart = archive.prepare('SELECT MIN(pub_date_effective) AS first FROM articles WHERE event_id = ?'); + const symbols = [...new Set(rows.map((r) => r.instrument))]; + console.log(`[reaction] distinct instruments: ${symbols.length}, fetching history (cached)...`); + const hist = new Map(); + for (const s of [BENCHMARK, ...symbols]) hist.set(s, await fetchHistory(s)); + + const bench = hist.get(BENCHMARK); + const scored = []; + let skipped = 0; + for (const r of rows) { + const first = r.event_id ? eventStart.get(r.event_id)?.first : null; + if (!first) { skipped++; continue; } + const startDay = String(first).slice(0, 10); + const cutoffDay = String(r.information_cutoff).slice(0, 10); + const h = hist.get(r.instrument) || []; + // baseline is the last close strictly before the first article, so the whole + // announcement move is inside the reaction window + const pPre = strictlyBefore(h, startDay); + const bPre = strictlyBefore(bench, startDay); + const pAt = Number.isFinite(r.price_0) ? r.price_0 : onOrAfter(h, cutoffDay); + const bAt = Number.isFinite(r.benchmark_0) ? r.benchmark_0 : onOrAfter(bench, cutoffDay); + if (![pPre, bPre, pAt, bAt].every(Number.isFinite) || !pPre || !bPre) { skipped++; continue; } + const reaction = (pAt - pPre) / pPre - (bAt - bPre) / bPre; + if (!Number.isFinite(reaction)) { skipped++; continue; } + scored.push({ ...r, reaction, bucket: band(reaction), fwdUp: r.excess_return > 0 ? 1 : 0 }); + } + console.log(`[reaction] scored: ${scored.length}, skipped for missing prices/dates: ${skipped}\n`); + if (!scored.length) return; + + const report = (label, set) => { + if (!set.length) return; + console.log(`\n================ ${label} (n=${set.length}) ================`); + console.log('bucket n fwd_up% model_acc% mean_fwd% P(up|pred+) P(up|pred-) z p'); + for (const b of BANDS) { + const g = set.filter((r) => r.bucket === b.key); + if (!g.length) { console.log(`${b.key.padEnd(12)} 0 - - - - - - -`); continue; } + const pos = g.filter((r) => r.direction === 'positive'); + const neg = g.filter((r) => r.direction === 'negative'); + const t = ztest(pos.filter((r) => r.fwdUp).length, pos.length, neg.filter((r) => r.fwdUp).length, neg.length); + const acc = g.filter((r) => r.direction_correct).length / g.length; + const meanFwd = g.reduce((s, r) => s + r.excess_return, 0) / g.length; + console.log( + `${b.key.padEnd(12)} ${String(g.length).padStart(4)} ${pct(g.filter((r) => r.fwdUp).length / g.length)} ${pct(acc)} ` + + `${(100 * meanFwd).toFixed(2).padStart(6)}% ${pct(t ? t.p1 : null)} ${pct(t ? t.p2 : null)} ` + + `${t ? t.z.toFixed(2).padStart(6) : ' n/a'} ${t ? t.p.toFixed(3) : ' n/a'}` + ); + } + + // T1: does the reaction itself predict the forward sign? + const upSide = set.filter((r) => r.bucket === 'up' || r.bucket === 'strong_up'); + const downSide = set.filter((r) => r.bucket === 'down' || r.bucket === 'strong_down'); + const t1 = ztest(upSide.filter((r) => r.fwdUp).length, upSide.length, downSide.filter((r) => r.fwdUp).length, downSide.length); + console.log(`\n[T1] forward-up rate after a POSITIVE reaction vs after a NEGATIVE reaction`); + if (t1) { + console.log(` ${pct(t1.p1)} (n=${upSide.length}) vs ${pct(t1.p2)} (n=${downSide.length}) z=${t1.z.toFixed(2)} p=${t1.p.toFixed(3)}`); + console.log(` ${Math.abs(t1.z) >= 1.96 ? (t1.z > 0 ? '=> CONTINUATION (drift) is present' : '=> REVERSAL is present') : '=> no drift or reversal detectable'}`); + } else console.log(' insufficient data'); + + // T2: pooled model discrimination within buckets, vs unconditional + const allPos = set.filter((r) => r.direction === 'positive'); + const allNeg = set.filter((r) => r.direction === 'negative'); + const t2 = ztest(allPos.filter((r) => r.fwdUp).length, allPos.length, allNeg.filter((r) => r.fwdUp).length, allNeg.length); + console.log(`\n[T2] unconditional model discrimination P(up|pred+) - P(up|pred-)`); + if (t2) console.log(` ${pct(t2.p1)} vs ${pct(t2.p2)} z=${t2.z.toFixed(2)} p=${t2.p.toFixed(3)}` + + ` ${Math.abs(t2.z) >= 1.96 ? '=> SIGNIFICANT' : '=> not distinguishable from zero'}`); + else console.log(' insufficient data'); + }; + + report('ALL', scored); + const topTicker = [...scored.reduce((m, r) => m.set(r.instrument, (m.get(r.instrument) || 0) + 1), new Map())] + .sort((a, b) => b[1] - a[1])[0]; + console.log(`\n\n(most common instrument: ${topTicker[0]} with ${topTicker[1]} of ${scored.length})`); + report(`EXCLUDING ${topTicker[0]}`, scored.filter((r) => r.instrument !== topTicker[0])); + + console.log('\n[reaction] reminder: T1 and T2 were pre-registered. Per-bucket rows are'); + console.log('[reaction] descriptive only, do not read a single bucket as a finding.'); +} + +main().catch((error) => { console.error('[reaction] fatal:', error.message, error.stack); process.exit(1); });