Duriin-API/src/autonomy/coordinator.js

const { EVENT_FAMILY_NAMES, normalizeEventType } = require('./calibration');

const ALLOWED_DIRECTIONS = new Set(['positive', 'negative']);
const ALLOWED_HORIZONS = new Set([1, 5, 10, 20, 30, 60, 90]);
// The prompt used to say event_type was a 'stable_enum' and then never listed the
// enum, so the model invented one label per event and production ended up with 201
// distinct values. Same closed set the cohort key uses, so a label can never mean
// one thing in the prompt and another in calibration.
const ALLOWED_EVENT_TYPES = new Set(EVENT_FAMILY_NAMES);

// 96% of rejected proposals name an instrument we cannot trade: indices (SPX,
// DXY, ^TNX), fx (EURUSD, XAU/USD), futures (CL=F, BZ=F) and foreign listings
// (VOW3.DE, RHM.DE, 1211.HK). The analysis behind those is usually fine, it is
// the ticker that is unusable, so tell the model what the allowlist actually
// holds instead of paying for the call and discarding it at validation.
// Every symbol named below was checked against the live allowlist.
const INSTRUMENT_RULES = `Instrument rules. instrument must be a symbol tradable on a US exchange, and only these resolve:
- US listed common stock or ETF, by its US ticker.
- A foreign company only via its US listing or ADR, never its home listing. Volkswagen is VWAGY not VOW3.DE, Alibaba is BABA, Toyota is TM, Sony is SONY. If you do not know the company has a US listing, omit the prediction.
- An index, currency, rate or commodity only via a US listed ETF that tracks it: S&P 500 -> SPY, Nasdaq 100 -> QQQ, gold -> GLD, crude oil -> USO, US dollar -> UUP, treasuries -> TLT. Never emit SPX, DXY, ^TNX, EURUSD, XAU/USD, CL=F or any futures or fx symbol.
- Never SPY itself as the prediction, it is the benchmark and its excess return is always zero.
If the only instrument the evidence supports is untradable under these rules, leave it out rather than substituting something loosely related.`;

// An exact family is what we want. If the model ignores the list we try to salvage
// the label through the same mapper calibration uses, and only give up when it is
// unplaceable -- an explicit 'other' is a legitimate answer, unplaceable free text
// is not, and the difference is what stops 'other' quietly becoming the bin again.
function normalizeProposedEventType(raw, instrument) {
  const value = String(raw || '').trim().toLowerCase().replace(/[\s-]+/g, '_');
  if (ALLOWED_EVENT_TYPES.has(value)) return value;

  const salvaged = normalizeEventType(raw);
  if (salvaged !== 'other') {
    console.warn(`[coordinator] ${instrument} event_type "${raw}" is not in the enum, mapped to "${salvaged}"`);
    return salvaged;
  }
  throw new Error(`event_type must be one of ${EVENT_FAMILY_NAMES.join(', ')} (got "${raw}")`);
}

function normalizeProposal(raw, { informationCutoff, model = 'unknown', promptVersion = 'unknown' } = {}) {
  if (!raw || typeof raw !== 'object') throw new Error('coordinator output must be an object');
  const predictions = Array.isArray(raw.predictions) ? raw.predictions : [];
  const normalized = predictions.map((item) => {
    const instrument = String(item.instrument || item.ticker || '').trim().toUpperCase();
    const direction = String(item.direction || '').trim().toLowerCase();
    const horizonDays = Number(item.horizon_days || item.horizonDays);
    if (!instrument) throw new Error('prediction instrument is required');
    if (!ALLOWED_DIRECTIONS.has(direction)) throw new Error(`invalid direction: ${direction}`);
    if (!ALLOWED_HORIZONS.has(horizonDays)) throw new Error(`invalid horizon_days: ${horizonDays}`);
    const articleIds = Array.isArray(item.evidence_article_ids)
      ? item.evidence_article_ids.map(Number).filter(Number.isInteger)
      : [];
    if (articleIds.length === 0) throw new Error(`prediction for ${instrument} has no evidence`);
    return {
      instrument,
      direction,
      eventType: normalizeProposedEventType(item.event_type, instrument),
      causalChannel: item.causal_channel ? String(item.causal_channel).trim() : null,
      horizonDays,
      evidenceArticleIds: [...new Set(articleIds)],
      invalidationCondition: item.invalidation_condition ? String(item.invalidation_condition).trim() : null,
    };
  });
  return {
    schemaVersion: 1,
    informationCutoff: informationCutoff || new Date().toISOString(),
    coordinatorModel: model,
    promptVersion,
    predictions: normalized,
  };
}

function verifyEvidence(archiveDb, articleIds, informationCutoff = null) {
  const placeholders = articleIds.map(() => '?').join(',');
  // No proposal, whatever lane produced it, may cite material which did not yet
  // exist at its own information cutoff.  This used to be a replay-only rule and
  // that was a lookahead hole for every other origin.
  const cutoffClause = informationCutoff ? ' AND datetime(COALESCE(pub_date_effective, pub_date, ingested_at)) <= datetime(?)' : '';
  let rows;
  try {
    rows = archiveDb.prepare(`SELECT id FROM articles WHERE id IN (${placeholders})${cutoffClause}`)
      .all(...articleIds, ...(informationCutoff ? [informationCutoff] : []));
  } catch (error) {
    // Minimal/test archives may not retain publication metadata at all, in which
    // case the cutoff clause cannot even be prepared.  We degrade to a plain
    // existence check rather than blocking the pipeline, but the degredation is
    // never silent - a production archive missing these columns is a real bug.
    console.warn('[coordinator] evidence cutoff check unavailable, falling back to existence only.',
      `cutoff=${informationCutoff} articles=${JSON.stringify(articleIds)} reason=${error && error.message}`);
    if (error && error.stack) console.warn(error.stack);
    rows = archiveDb.prepare(`SELECT id FROM articles WHERE id IN (${placeholders})`).all(...articleIds);
  }
  const found = new Set(rows.map((row) => row.id));
  return articleIds.every((id) => found.has(id));
}

function acceptProposal(intelligenceDb, archiveDb, raw, metadata = {}) {
  const proposal = normalizeProposal(raw, metadata);

  // Tradability is a filter, not an integrity failure, so it is applied per
  // prediction. One untradable ticker used to reject the entire proposal and take
  // its valid siblings down with it: in a single day 93 proposals were rejected
  // this way, discarding 171 predictions of which 76 named something we could
  // trade perfectly well.
  const allowlisted = intelligenceDb.prepare(
    "SELECT tradable FROM autonomy_instruments WHERE symbol = ? AND active = 1 AND tradable = 1"
  );
  const dropped = [];
  const tradable = proposal.predictions.filter((prediction) => {
    if (allowlisted.get(prediction.instrument)) return true;
    dropped.push(prediction.instrument);
    return false;
  });
  if (dropped.length) {
    console.warn(`[coordinator] dropped ${dropped.length} untradable instrument(s): ${dropped.join(', ')}`
      + ` (kept ${tradable.length})`);
  }

  // Lookahead stays all or nothing. A proposal citing evidence that did not exist
  // at its own cutoff is corrupt rather than merely untradable, and quietly keeping
  // the rest of it would hide exactly the thing we most need to see.
  for (const prediction of tradable) {
    if (!verifyEvidence(archiveDb, prediction.evidenceArticleIds, proposal.informationCutoff)) {
      throw new Error(`proposal references missing evidence for ${prediction.instrument}`);
    }
  }

  // what actually got stored, plus a record of what was filtered and why
  const stored = { ...proposal, predictions: tradable, droppedInstruments: dropped };
  const insert = intelligenceDb.prepare(`
    INSERT INTO autonomy_proposals
      (event_id, payload, information_cutoff, coordinator_model, prompt_version, status)
    VALUES (?, ?, ?, ?, ?, 'accepted')
  `);
  const insertPrediction = intelligenceDb.prepare(`
      INSERT INTO autonomy_predictions
      (proposal_id, event_id, instrument, direction, event_type, causal_channel,
       horizon_days, information_cutoff, evidence_article_ids, invalidation_condition, learning_eligible, strategy_version, origin, replay_run_id)
    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
  `);
  const tx = intelligenceDb.transaction(() => {
    const proposalResult = insert.run(metadata.eventId || null, JSON.stringify(stored), stored.informationCutoff,
      stored.coordinatorModel, stored.promptVersion);
    for (const prediction of tradable) {
      insertPrediction.run(proposalResult.lastInsertRowid, metadata.eventId || null, prediction.instrument,
        prediction.direction, prediction.eventType, prediction.causalChannel, prediction.horizonDays,
        stored.informationCutoff, JSON.stringify(prediction.evidenceArticleIds), prediction.invalidationCondition,
        metadata.learningEligible ? 1 : 0, metadata.strategyVersion || 'autonomy-1',
        metadata.origin || 'live', metadata.replayRunId || null);
    }
    return Number(proposalResult.lastInsertRowid);
  });
  return { proposalId: tx(), predictionCount: tradable.length, droppedInstruments: dropped };
}

function recordRejectedProposal(intelligenceDb, raw, metadata = {}, reason = 'validation failed') {
  const payload = raw && typeof raw === 'object' ? raw : { raw: String(raw) };
  return intelligenceDb.prepare(`
    INSERT INTO autonomy_proposals
      (event_id, payload, information_cutoff, coordinator_model, prompt_version, status, rejection_reason, reviewed_at)
    VALUES (?, ?, ?, ?, ?, 'rejected', ?, datetime('now'))
  `).run(metadata.eventId || null, JSON.stringify(payload), metadata.informationCutoff || new Date().toISOString(),
    metadata.model || 'unknown', metadata.promptVersion || 'unknown', String(reason).slice(0, 1000)).lastInsertRowid;
}

module.exports = { normalizeProposal, verifyEvidence, acceptProposal, recordRejectedProposal, INSTRUMENT_RULES };