#!/usr/bin/env node
/*
 * Lever 1: does conditioning on the initial market reaction reveal any
 * discrimination in this pipeline?
 *
 * The coordinator is scored on excess return vs SPY starting at the information
 * cutoff, so the announcement move itself sits OUTSIDE the scored window. That
 * move is the best documented conditioner for post event drift, and we throw it
 * away. This measures whether it is worth keeping.
 *
 * reaction  = instrument excess vs SPY from the last close BEFORE the event's
 *             first article, up to the outcome's own entry price at the cutoff.
 * forward   = the already stored excess_return over the horizon.
 * The two windows touch but never overlap, so there is no lookahead.
 *
 * Read only. Opens both databases read only and writes nothing but a price cache.
 *
 * PRE REGISTERED TESTS (declared before looking, so the buckets cannot be tuned):
 *   T1 does the reaction bucket predict the SIGN of forward excess return?
 *      (is there drift/reversal in this sample at all, model aside)
 *   T2 within a bucket, does the coordinator's direction discriminate?
 *      (does the model add anything on top of T1)
 * Everything else printed is descriptive, not a test.
 */
const fs = require('fs');
const path = require('path');
const Database = require('better-sqlite3');

const ARCHIVE = process.env.DURIIN_DB || '/data/archive.sqlite';
const INTELLIGENCE = process.env.INTELLIGENCE_DB || '/data/intelligence.sqlite';
const CACHE = process.env.PRICE_CACHE || '/data/.price-cache';
const BENCHMARK = 'SPY';

// fixed bands on the reaction, declared up front. not quantiles, so the cut
// points cannot drift with the data.
const BANDS = [
  { key: 'strong_down', min: -Infinity, max: -0.05 },
  { key: 'down', min: -0.05, max: -0.015 },
  { key: 'flat', min: -0.015, max: 0.015 },
  { key: 'up', min: 0.015, max: 0.05 },
  { key: 'strong_up', min: 0.05, max: Infinity },
];
function band(reaction) {
  return (BANDS.find((b) => reaction >= b.min && reaction < b.max) || { key: 'unknown' }).key;
}

function sleep(ms) { return new Promise((r) => setTimeout(r, ms)); }

async function fetchHistory(symbol) {
  fs.mkdirSync(CACHE, { recursive: true });
  const file = path.join(CACHE, `${symbol.replace(/[^A-Za-z0-9._-]/g, '_')}.json`);
  if (fs.existsSync(file)) {
    try { return JSON.parse(fs.readFileSync(file, 'utf8')); }
    catch (error) { console.error(`[reaction] bad cache for ${symbol}, refetching:`, error.message); }
  }
  const url = `https://query1.finance.yahoo.com/v8/finance/chart/${encodeURIComponent(symbol)}`
    + `?period1=946684800&period2=${Math.floor(Date.now() / 1000)}&interval=1d`;
  let rows = [];
  try {
    const response = await fetch(url, { headers: { 'User-Agent': 'Mozilla/5.0' } });
    if (!response.ok) throw new Error(`HTTP ${response.status}`);
    const result = (await response.json())?.chart?.result?.[0];
    rows = (result?.timestamp || []).map((ts, i) => ({
      date: new Date(ts * 1000).toISOString().slice(0, 10),
      close: result.indicators?.quote?.[0]?.close?.[i],
    })).filter((r) => Number.isFinite(r.close));
  } catch (error) {
    console.error(`[reaction] history fetch failed for ${symbol}:`, error.message);
  }
  fs.writeFileSync(file, JSON.stringify(rows));
  await sleep(250);
  return rows;
}

const onOrAfter = (h, d) => h.find((r) => r.date >= d)?.close ?? null;
const strictlyBefore = (h, d) => { for (let i = h.length - 1; i >= 0; i--) if (h[i].date < d) return h[i].close; return null; };

// two proportion z test. returns z and a normal approx two sided p.
function ztest(x1, n1, x2, n2) {
  if (!n1 || !n2) return null;
  const p1 = x1 / n1, p2 = x2 / n2, p = (x1 + x2) / (n1 + n2);
  const se = Math.sqrt(p * (1 - p) * (1 / n1 + 1 / n2));
  if (!se) return null;
  const z = (p1 - p2) / se;
  // logistic approx to the normal cdf. has to be fed |z|, otherwise the two
  // sided p comes back above 1 for negative z, which is nonsense.
  const cdf = (v) => 1 / (1 + Math.exp(-0.07056 * v ** 3 - 1.5976 * v));
  const pValue = Math.min(1, 2 * (1 - cdf(Math.abs(z))));
  return { z, p: pValue, p1, p2 };
}
const pct = (x) => (x === null || x === undefined || Number.isNaN(x) ? '  n/a ' : `${(100 * x).toFixed(1)}%`);

async function main() {
  const intel = new Database(INTELLIGENCE, { readonly: true });
  const archive = new Database(ARCHIVE, { readonly: true });

  const rows = intel.prepare(`
    SELECT p.id, p.instrument, p.direction, p.event_id, p.information_cutoff, p.horizon_days, p.origin,
           o.excess_return, o.direction_correct, o.price_0, o.benchmark_0
    FROM autonomy_predictions p
    JOIN autonomy_outcomes o ON o.prediction_id = p.id
    WHERE o.excess_return IS NOT NULL
  `).all();
  console.log(`[reaction] matured outcomes: ${rows.length}`);

  const eventStart = archive.prepare('SELECT MIN(pub_date_effective) AS first FROM articles WHERE event_id = ?');
  const symbols = [...new Set(rows.map((r) => r.instrument))];
  console.log(`[reaction] distinct instruments: ${symbols.length}, fetching history (cached)...`);
  const hist = new Map();
  for (const s of [BENCHMARK, ...symbols]) hist.set(s, await fetchHistory(s));

  const bench = hist.get(BENCHMARK);
  const scored = [];
  let skipped = 0;
  for (const r of rows) {
    const first = r.event_id ? eventStart.get(r.event_id)?.first : null;
    if (!first) { skipped++; continue; }
    const startDay = String(first).slice(0, 10);
    const cutoffDay = String(r.information_cutoff).slice(0, 10);
    const h = hist.get(r.instrument) || [];
    // baseline is the last close strictly before the first article, so the whole
    // announcement move is inside the reaction window
    const pPre = strictlyBefore(h, startDay);
    const bPre = strictlyBefore(bench, startDay);
    const pAt = Number.isFinite(r.price_0) ? r.price_0 : onOrAfter(h, cutoffDay);
    const bAt = Number.isFinite(r.benchmark_0) ? r.benchmark_0 : onOrAfter(bench, cutoffDay);
    if (![pPre, bPre, pAt, bAt].every(Number.isFinite) || !pPre || !bPre) { skipped++; continue; }
    const reaction = (pAt - pPre) / pPre - (bAt - bPre) / bPre;
    if (!Number.isFinite(reaction)) { skipped++; continue; }
    scored.push({ ...r, reaction, bucket: band(reaction), fwdUp: r.excess_return > 0 ? 1 : 0 });
  }
  console.log(`[reaction] scored: ${scored.length}, skipped for missing prices/dates: ${skipped}\n`);
  if (!scored.length) return;

  const report = (label, set) => {
    if (!set.length) return;
    console.log(`\n================ ${label}  (n=${set.length}) ================`);
    console.log('bucket        n    fwd_up%   model_acc%   mean_fwd%   P(up|pred+)  P(up|pred-)   z      p');
    for (const b of BANDS) {
      const g = set.filter((r) => r.bucket === b.key);
      if (!g.length) { console.log(`${b.key.padEnd(12)}  0        -            -           -            -            -        -      -`); continue; }
      const pos = g.filter((r) => r.direction === 'positive');
      const neg = g.filter((r) => r.direction === 'negative');
      const t = ztest(pos.filter((r) => r.fwdUp).length, pos.length, neg.filter((r) => r.fwdUp).length, neg.length);
      const acc = g.filter((r) => r.direction_correct).length / g.length;
      const meanFwd = g.reduce((s, r) => s + r.excess_return, 0) / g.length;
      console.log(
        `${b.key.padEnd(12)} ${String(g.length).padStart(4)}   ${pct(g.filter((r) => r.fwdUp).length / g.length)}    ${pct(acc)}      `
        + `${(100 * meanFwd).toFixed(2).padStart(6)}%   ${pct(t ? t.p1 : null)}       ${pct(t ? t.p2 : null)}    `
        + `${t ? t.z.toFixed(2).padStart(6) : '   n/a'}  ${t ? t.p.toFixed(3) : ' n/a'}`
      );
    }

    // T1: does the reaction itself predict the forward sign?
    const upSide = set.filter((r) => r.bucket === 'up' || r.bucket === 'strong_up');
    const downSide = set.filter((r) => r.bucket === 'down' || r.bucket === 'strong_down');
    const t1 = ztest(upSide.filter((r) => r.fwdUp).length, upSide.length, downSide.filter((r) => r.fwdUp).length, downSide.length);
    console.log(`\n[T1] forward-up rate after a POSITIVE reaction vs after a NEGATIVE reaction`);
    if (t1) {
      console.log(`     ${pct(t1.p1)} (n=${upSide.length})  vs  ${pct(t1.p2)} (n=${downSide.length})   z=${t1.z.toFixed(2)}  p=${t1.p.toFixed(3)}`);
      console.log(`     ${Math.abs(t1.z) >= 1.96 ? (t1.z > 0 ? '=> CONTINUATION (drift) is present' : '=> REVERSAL is present') : '=> no drift or reversal detectable'}`);
    } else console.log('     insufficient data');

    // T2: pooled model discrimination within buckets, vs unconditional
    const allPos = set.filter((r) => r.direction === 'positive');
    const allNeg = set.filter((r) => r.direction === 'negative');
    const t2 = ztest(allPos.filter((r) => r.fwdUp).length, allPos.length, allNeg.filter((r) => r.fwdUp).length, allNeg.length);
    console.log(`\n[T2] unconditional model discrimination P(up|pred+) - P(up|pred-)`);
    if (t2) console.log(`     ${pct(t2.p1)} vs ${pct(t2.p2)}   z=${t2.z.toFixed(2)}  p=${t2.p.toFixed(3)}`
      + `   ${Math.abs(t2.z) >= 1.96 ? '=> SIGNIFICANT' : '=> not distinguishable from zero'}`);
    else console.log('     insufficient data');
  };

  report('ALL', scored);
  const topTicker = [...scored.reduce((m, r) => m.set(r.instrument, (m.get(r.instrument) || 0) + 1), new Map())]
    .sort((a, b) => b[1] - a[1])[0];
  console.log(`\n\n(most common instrument: ${topTicker[0]} with ${topTicker[1]} of ${scored.length})`);
  report(`EXCLUDING ${topTicker[0]}`, scored.filter((r) => r.instrument !== topTicker[0]));

  console.log('\n[reaction] reminder: T1 and T2 were pre-registered. Per-bucket rows are');
  console.log('[reaction] descriptive only, do not read a single bucket as a finding.');
}

main().catch((error) => { console.error('[reaction] fatal:', error.message, error.stack); process.exit(1); });