Nothing in the pipeline has ever fed an outcome back to the thing that makes predictions. Calibration reads autonomy_outcomes, but calibration only gates whether to act on a prediction, never what the prediction is. So the only thing that has ever changed this system's output is a human editing the prompt. score-replay-runs.js asks "compared to what". The answer is not flattering: over 2,114 scored replay predictions the system is right 50.05% of the time while answering "negative" to every one of the same bars scores 54.45%. It is 4.4 points below a constant, z=-4.06. The whole deficit is the prior. It says positive on 65% of calls when 45.5% of bars beat SPY, a 19 point skew. Its discrimination, P(up|positive) minus P(up|negative), is +3.2 points with p=0.15, so the direction it picks is weakly informative and completely buried by how often it defaults to positive. The first version of that script compared each direction group's accuracy to "always that direction" on the same rows, which is an identity and tests nothing. T3 replaces it with the two proportion test that actually asks whether the choice of direction carries information. build-feedback-brief.js turns a run's scored outcomes into a memo the next run reads before predicting. Generated from the data, not written by hand, or it is just me editing the prompt again with extra steps. Replay can now be pinned to an explicit article set, which is what makes two runs comparable at all. Comparing two calendar windows of one run compares two market regimes: the epochs in run 1 line up exactly with article vintage, E0 is late 2024 and E2 is 2026, so nothing could be attributed. A new run also inherits its parent's watermark instead of recomputing it from today, which silently guaranteed a different archive slice every time. prompt_version never moved across four material prompt changes, so every proposal on record claims to come from the first prompt. coordinator-2 and replay-coordinator-2. docs/replay-run-2-preregistration.md fixes the bar before the run exists, including which result counts as learning and which is only calibration. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WnNxwxfXSbeNtjvtz5gayb
512 lines
28 KiB
JavaScript
512 lines
28 KiB
JavaScript
const test = require('node:test');
|
|
const assert = require('node:assert/strict');
|
|
const Database = require('better-sqlite3');
|
|
|
|
const { initAutonomySchema } = require('../src/autonomy/schema');
|
|
const { enqueueJob, leaseNextJob, completeJob } = require('../src/autonomy/jobs');
|
|
const { normalizeProposal, acceptProposal } = require('../src/autonomy/coordinator');
|
|
const { calibrateOutcomes, cohortKey } = require('../src/autonomy/calibration');
|
|
const { decide } = require('../src/autonomy/policy');
|
|
const { validatePaperIntent, createSimulator } = require('../src/autonomy/execution');
|
|
const { calculateOutcome, addTradingDays } = require('../src/autonomy/outcomes');
|
|
const { yahooSymbol } = require('../workers/outcomeAutonomyWorker');
|
|
const { createOrderIntent } = require('../src/autonomy/orderIntents');
|
|
const { enqueueCoordinatorEvent, reconcileArchiveBatch, reconcileLiveBatch, isTransientCoordinatorFailure } = require('../workers/autonomyWorker');
|
|
const { buildGraphContext } = require('../src/autonomy/graphContext');
|
|
const { buildPrompt } = require('../workers/coordinatorWorker');
|
|
const { scheduleNext, replayPrompt } = require('../workers/replayWorker');
|
|
const { refreshHistoricalCalibration, createDecisions, ensureCalibrationColumns } = require('../workers/calibrationWorker');
|
|
|
|
test('autonomy schema and leased jobs are restart-safe', () => {
|
|
const db = new Database(':memory:');
|
|
initAutonomySchema(db);
|
|
assert.equal(enqueueJob(db, {
|
|
jobType: 'enrich', lane: 'live', priority: 10, entityType: 'article', entityId: 42,
|
|
idempotencyKey: 'enrich:42',
|
|
}).inserted, true);
|
|
assert.equal(enqueueJob(db, {
|
|
jobType: 'enrich', lane: 'live', priority: 10, entityType: 'article', entityId: 42,
|
|
idempotencyKey: 'enrich:42',
|
|
}).inserted, false);
|
|
const job = leaseNextJob(db, 'test-worker');
|
|
assert.equal(job.lane, 'live');
|
|
assert.equal(completeJob(db, job.id, 'test-worker'), true);
|
|
assert.equal(db.prepare("SELECT status FROM autonomy_jobs WHERE id = ?").get(job.id).status, 'complete');
|
|
});
|
|
|
|
test('coordinator proposals require evidence and contain no arbitrary numeric confidence', () => {
|
|
const normalized = normalizeProposal({ predictions: [{
|
|
instrument: 'nvda', direction: 'positive', event_type: 'supply_constraint',
|
|
horizon_days: 10, evidence_article_ids: [7],
|
|
}] }, { informationCutoff: '2026-01-01T00:00:00Z', model: 'test-model' });
|
|
assert.equal(normalized.predictions[0].instrument, 'NVDA');
|
|
assert.equal('probability' in normalized.predictions[0], false);
|
|
assert.throws(() => normalizeProposal({ predictions: [{ instrument: 'NVDA', direction: 'positive', horizon_days: 10 }] }));
|
|
});
|
|
|
|
test('accepted proposal preserves evidence and creates immutable prediction', () => {
|
|
const archive = new Database(':memory:');
|
|
archive.exec('CREATE TABLE articles (id INTEGER PRIMARY KEY)');
|
|
archive.prepare('INSERT INTO articles (id) VALUES (?)').run(7);
|
|
const intelligence = new Database(':memory:');
|
|
initAutonomySchema(intelligence);
|
|
intelligence.prepare("INSERT INTO autonomy_instruments(symbol, broker, active, tradable) VALUES ('NVDA', 'test', 1, 1)").run();
|
|
const result = acceptProposal(intelligence, archive, {
|
|
predictions: [{ instrument: 'NVDA', direction: 'positive', event_type: 'earnings', horizon_days: 10, evidence_article_ids: [7] }],
|
|
}, { informationCutoff: '2026-01-01T00:00:00Z', strategyVersion: 'test' });
|
|
assert.equal(result.predictionCount, 1);
|
|
assert.deepEqual(JSON.parse(intelligence.prepare('SELECT evidence_article_ids FROM autonomy_predictions').get().evidence_article_ids), [7]);
|
|
});
|
|
|
|
test('replay evidence cannot look beyond its information cutoff and remains non-executable', () => {
|
|
const archive = new Database(':memory:');
|
|
archive.exec('CREATE TABLE articles (id INTEGER PRIMARY KEY, pub_date_effective TEXT, pub_date TEXT, ingested_at TEXT)');
|
|
archive.prepare("INSERT INTO articles VALUES (7, '2020-01-01T00:00:00Z', NULL, '2020-01-01T00:00:00Z')").run();
|
|
const intelligence = new Database(':memory:');
|
|
initAutonomySchema(intelligence);
|
|
intelligence.prepare("INSERT INTO autonomy_instruments(symbol, broker, active, tradable) VALUES ('NVDA', 'test', 1, 1)").run();
|
|
assert.throws(() => acceptProposal(intelligence, archive, {
|
|
predictions: [{ instrument: 'NVDA', direction: 'positive', event_type: 'earnings', horizon_days: 10, evidence_article_ids: [7] }],
|
|
}, { informationCutoff: '2019-12-31T00:00:00Z', origin: 'replay', replayRunId: 1 }), /missing evidence/);
|
|
const proposal = intelligence.prepare("INSERT INTO autonomy_proposals(payload, information_cutoff, status) VALUES ('{}', datetime('now'), 'accepted')").run();
|
|
const prediction = intelligence.prepare(`INSERT INTO autonomy_predictions
|
|
(proposal_id, instrument, direction, event_type, horizon_days, information_cutoff, evidence_article_ids, strategy_version, origin)
|
|
VALUES (?, 'NVDA', 'positive', 'test', 10, datetime('now'), '[7]', 'test', 'replay')`).run(proposal.lastInsertRowid);
|
|
intelligence.prepare("INSERT INTO autonomy_decisions(prediction_id, action, rationale, strategy_version) VALUES (?, 'BUY', 'test', 'test')").run(prediction.lastInsertRowid);
|
|
const executable = intelligence.prepare(`SELECT d.id FROM autonomy_decisions d JOIN autonomy_predictions p ON p.id=d.prediction_id
|
|
WHERE d.action IN ('BUY','SELL') AND p.origin='live'`).all();
|
|
assert.equal(executable.length, 0);
|
|
});
|
|
|
|
test('replay scheduler skips terminal replay jobs instead of pinning the cursor', () => {
|
|
const archive = new Database(':memory:');
|
|
archive.exec(`
|
|
CREATE TABLE articles (
|
|
id INTEGER PRIMARY KEY,
|
|
title TEXT,
|
|
description TEXT,
|
|
content TEXT,
|
|
pub_date_effective TEXT,
|
|
is_index_page INTEGER
|
|
)
|
|
`);
|
|
archive.prepare("INSERT INTO articles VALUES (1, 'bad', '', 'content', '2020-01-01T00:00:00Z', 0)").run();
|
|
archive.prepare("INSERT INTO articles VALUES (2, 'next', '', 'content', '2020-01-02T00:00:00Z', 0)").run();
|
|
|
|
const intelligence = new Database(':memory:');
|
|
initAutonomySchema(intelligence);
|
|
const runId = intelligence.prepare(`
|
|
INSERT INTO autonomy_replay_runs (watermark_at, strategy_version, prompt_version, coordinator_model)
|
|
VALUES ('2020-01-03T00:00:00Z', 'test', 'test', 'test')
|
|
`).run().lastInsertRowid;
|
|
enqueueJob(intelligence, {
|
|
jobType: 'replay_article', lane: 'historical', priority: 1, entityType: 'article', entityId: 1,
|
|
idempotencyKey: `replay:${runId}:article:1`,
|
|
});
|
|
intelligence.prepare("UPDATE autonomy_jobs SET status='dead_letter', attempts=5, last_error='fetch failed'").run();
|
|
|
|
const run = intelligence.prepare('SELECT * FROM autonomy_replay_runs WHERE id=?').get(runId);
|
|
const next = scheduleNext(intelligence, archive, run);
|
|
assert.equal(next.id, 2);
|
|
assert.equal(intelligence.prepare('SELECT cursor_article_id FROM autonomy_replay_runs WHERE id=?').get(runId).cursor_article_id, 1);
|
|
assert.equal(intelligence.prepare("SELECT COUNT(*) count FROM autonomy_jobs WHERE status='pending' AND entity_id='2'").get().count, 1);
|
|
});
|
|
|
|
test('a run with a pinned article set walks only that set, in date order', () => {
|
|
const archive = new Database(':memory:');
|
|
archive.exec(`
|
|
CREATE TABLE articles (
|
|
id INTEGER PRIMARY KEY, title TEXT, description TEXT, content TEXT,
|
|
pub_date_effective TEXT, is_index_page INTEGER
|
|
)
|
|
`);
|
|
for (const id of [1, 2, 3, 4]) {
|
|
archive.prepare('INSERT INTO articles VALUES (?, ?, ?, ?, ?, 0)')
|
|
.run(id, `a${id}`, '', 'content', `2020-01-0${id}T00:00:00Z`);
|
|
}
|
|
|
|
const intelligence = new Database(':memory:');
|
|
initAutonomySchema(intelligence);
|
|
const runId = intelligence.prepare(`
|
|
INSERT INTO autonomy_replay_runs (watermark_at, strategy_version, prompt_version, coordinator_model)
|
|
VALUES ('2020-01-09T00:00:00Z', 'test', 'test', 'test')
|
|
`).run().lastInsertRowid;
|
|
// deliberately out of order and deliberately not article 1, the whole point
|
|
// is that the run ignores the archive walk and answers these
|
|
for (const [articleId, at] of [[4, '2020-01-04T00:00:00Z'], [2, '2020-01-02T00:00:00Z']]) {
|
|
intelligence.prepare('INSERT INTO autonomy_replay_run_articles (run_id, article_id, effective_at) VALUES (?, ?, ?)')
|
|
.run(runId, articleId, at);
|
|
}
|
|
|
|
const run = intelligence.prepare('SELECT * FROM autonomy_replay_runs WHERE id=?').get(runId);
|
|
const first = scheduleNext(intelligence, archive, run);
|
|
assert.equal(first.id, 2, 'earliest pinned article first, not article 1');
|
|
|
|
intelligence.prepare("UPDATE autonomy_jobs SET status='complete' WHERE entity_id='2'").run();
|
|
intelligence.prepare('UPDATE autonomy_replay_runs SET cursor_article_id=2, cursor_effective_at=? WHERE id=?')
|
|
.run('2020-01-02T00:00:00Z', runId);
|
|
const second = scheduleNext(intelligence, archive, intelligence.prepare('SELECT * FROM autonomy_replay_runs WHERE id=?').get(runId));
|
|
assert.equal(second.id, 4);
|
|
|
|
intelligence.prepare("UPDATE autonomy_jobs SET status='complete' WHERE entity_id='4'").run();
|
|
intelligence.prepare('UPDATE autonomy_replay_runs SET cursor_article_id=4, cursor_effective_at=? WHERE id=?')
|
|
.run('2020-01-04T00:00:00Z', runId);
|
|
const exhausted = scheduleNext(intelligence, archive, intelligence.prepare('SELECT * FROM autonomy_replay_runs WHERE id=?').get(runId));
|
|
assert.equal(exhausted, null, 'a pinned run stops when its set is done, it does not fall back to the archive');
|
|
});
|
|
|
|
test('a run with no pinned set still walks the archive exactly as before', () => {
|
|
const archive = new Database(':memory:');
|
|
archive.exec(`
|
|
CREATE TABLE articles (
|
|
id INTEGER PRIMARY KEY, title TEXT, description TEXT, content TEXT,
|
|
pub_date_effective TEXT, is_index_page INTEGER
|
|
)
|
|
`);
|
|
archive.prepare("INSERT INTO articles VALUES (7, 'only', '', 'content', '2020-01-01T00:00:00Z', 0)").run();
|
|
const intelligence = new Database(':memory:');
|
|
initAutonomySchema(intelligence);
|
|
const runId = intelligence.prepare(`
|
|
INSERT INTO autonomy_replay_runs (watermark_at, strategy_version, prompt_version, coordinator_model)
|
|
VALUES ('2020-01-09T00:00:00Z', 'test', 'test', 'test')
|
|
`).run().lastInsertRowid;
|
|
const run = intelligence.prepare('SELECT * FROM autonomy_replay_runs WHERE id=?').get(runId);
|
|
assert.equal(scheduleNext(intelligence, archive, run).id, 7);
|
|
});
|
|
|
|
test('the feedback brief reaches the prompt and stays out of it when empty', () => {
|
|
const article = { id: 1, title: 't', content: 'body', effective_at: '2020-01-01T00:00:00Z' };
|
|
const withBrief = replayPrompt(article, 'CALIBRATION FEEDBACK. you over-call positive.');
|
|
assert.ok(withBrief.includes('you over-call positive'));
|
|
assert.ok(withBrief.indexOf('CALIBRATION FEEDBACK') < withBrief.indexOf('[Evidence 1]'),
|
|
'the brief has to land before the evidence, not after it');
|
|
assert.ok(!replayPrompt(article).includes('CALIBRATION FEEDBACK'));
|
|
});
|
|
|
|
test('calibration and policy abstain on insufficient evidence', () => {
|
|
const calibration = calibrateOutcomes([
|
|
{ excess_return: 0.02, direction_correct: 1 },
|
|
{ excess_return: -0.01, direction_correct: 0 },
|
|
]);
|
|
assert.equal(calibration.sampleSize, 2);
|
|
const result = decide({ ...calibration }, { minSampleSize: 30 });
|
|
assert.equal(result.action, 'ABSTAIN');
|
|
assert.equal(cohortKey({ sector: 'tech', eventType: 'earnings', horizonDays: 10, direction: 'positive' }), 'v2|tech|earnings|medium|positive');
|
|
assert.equal(decide({
|
|
direction: 'negative', probability: 0.8, expectedExcessReturn: -0.02, lowerReturn: -0.04,
|
|
sampleSize: 40, distinctInstruments: 9, topInstrumentShare: 0.25,
|
|
}).action, 'SELL');
|
|
});
|
|
|
|
test('historical replay outcomes create replay calibration snapshots', () => {
|
|
const db = new Database(':memory:');
|
|
initAutonomySchema(db);
|
|
const proposal = db.prepare("INSERT INTO autonomy_proposals(payload, information_cutoff, status) VALUES ('{}', '2020-01-01T00:00:00Z', 'accepted')").run();
|
|
const prediction = db.prepare(`
|
|
INSERT INTO autonomy_predictions
|
|
(proposal_id, instrument, direction, event_type, horizon_days, information_cutoff, evidence_article_ids,
|
|
learning_eligible, strategy_version, origin, replay_run_id, status)
|
|
VALUES (?, 'NVDA', 'positive', 'earnings', 10, '2020-01-01T00:00:00Z', '[1]', 0, 'test', 'replay', 7, 'resolved')
|
|
`).run(proposal.lastInsertRowid);
|
|
db.prepare(`
|
|
INSERT INTO autonomy_outcomes(prediction_id, excess_return, direction_correct)
|
|
VALUES (?, 0.04, 1)
|
|
`).run(prediction.lastInsertRowid);
|
|
|
|
// one per-run replay snapshot plus the pooled historical/replay snapshot
|
|
assert.equal(refreshHistoricalCalibration(db, 'test-cal'), 2);
|
|
const snapshot = db.prepare("SELECT source, replay_run_id, sample_size, directional_probability FROM autonomy_calibration_snapshots WHERE source='replay'").get();
|
|
assert.equal(snapshot.source, 'replay');
|
|
assert.equal(snapshot.replay_run_id, 7);
|
|
assert.equal(snapshot.sample_size, 1);
|
|
assert(snapshot.directional_probability > 0.5);
|
|
});
|
|
|
|
test('live decisions map calibration snapshot fields into policy inputs', () => {
|
|
const db = new Database(':memory:');
|
|
initAutonomySchema(db);
|
|
const proposal = db.prepare("INSERT INTO autonomy_proposals(payload, information_cutoff, status) VALUES ('{}', datetime('now'), 'accepted')").run();
|
|
const prediction = db.prepare(`
|
|
INSERT INTO autonomy_predictions
|
|
(proposal_id, instrument, direction, event_type, horizon_days, information_cutoff, evidence_article_ids,
|
|
learning_eligible, strategy_version, origin, status)
|
|
VALUES (?, 'NVDA', 'positive', 'earnings', 10, datetime('now'), '[1]', 1, 'test', 'live', 'open')
|
|
`).run(proposal.lastInsertRowid);
|
|
ensureCalibrationColumns(db);
|
|
db.prepare(`
|
|
INSERT INTO autonomy_calibration_snapshots
|
|
(cohort_key, sample_size, effective_sample_size, directional_probability, expected_excess_return,
|
|
lower_return, upper_return, parent_cohort_key, version, source)
|
|
VALUES ('v2|unknown|earnings|medium|positive', 40, 42, 0.7, 0.02, -0.01, 0.06, NULL, 'test-cal', 'live')
|
|
`).run();
|
|
db.prepare("UPDATE autonomy_calibration_snapshots SET distinct_instruments = 11, top_instrument_share = 0.2").run();
|
|
|
|
assert.equal(createDecisions(db), 1);
|
|
const decision = db.prepare('SELECT * FROM autonomy_decisions WHERE prediction_id=?').get(prediction.lastInsertRowid);
|
|
assert.equal(decision.action, 'BUY');
|
|
assert.equal(decision.calibrated_probability, 0.7);
|
|
assert.equal(decision.expected_excess_return, 0.02);
|
|
});
|
|
|
|
test('paper execution is allowlisted, bounded and idempotent', () => {
|
|
const intent = validatePaperIntent({ decisionId: 12, instrument: 'NVDA', action: 'BUY', notional: 100 }, { tradable: true, maxNotional: 500 });
|
|
const broker = createSimulator();
|
|
assert.deepEqual(broker.submit(intent), broker.submit(intent));
|
|
assert.throws(() => validatePaperIntent({ decisionId: 13, instrument: 'PRIVATE', action: 'BUY', notional: 100 }, { tradable: false }));
|
|
assert.throws(() => validatePaperIntent({ decisionId: 14, instrument: 'NVDA', action: 'BUY', notional: 501 }, { tradable: true, maxNotional: 500 }));
|
|
});
|
|
|
|
test('archive reconciliation prioritizes recent usable events and is bounded', () => {
|
|
const archive = new Database(':memory:');
|
|
archive.exec('CREATE TABLE articles (id INTEGER PRIMARY KEY, event_id INTEGER, ingested_at TEXT, content TEXT, has_embedding INTEGER)');
|
|
archive.prepare('INSERT INTO articles VALUES (1, 99, ?, ?, 1)').run(new Date().toISOString(), 'content');
|
|
const intelligence = new Database(':memory:');
|
|
initAutonomySchema(intelligence);
|
|
const result = reconcileArchiveBatch(archive, intelligence, 1);
|
|
assert.equal(result.scanned, 1);
|
|
const job = intelligence.prepare("SELECT lane, priority FROM autonomy_jobs WHERE job_type='coordinator_event'").get();
|
|
assert.equal(job.lane, 'live');
|
|
assert.equal(job.priority, 100);
|
|
const live = reconcileLiveBatch(archive, intelligence, 1);
|
|
assert.equal(live.scanned, 1);
|
|
});
|
|
|
|
test('archive reconciliation recovers transient dead-lettered coordinator jobs', () => {
|
|
const intelligence = new Database(':memory:');
|
|
initAutonomySchema(intelligence);
|
|
enqueueJob(intelligence, {
|
|
jobType: 'coordinator_event', lane: 'historical', priority: 10, entityType: 'event', entityId: 99,
|
|
idempotencyKey: 'coordinator_event:99',
|
|
});
|
|
intelligence.prepare(`
|
|
UPDATE autonomy_jobs
|
|
SET status='dead_letter', attempts=5, last_error='TypeError: fetch failed'
|
|
WHERE idempotency_key='coordinator_event:99'
|
|
`).run();
|
|
|
|
const result = enqueueCoordinatorEvent(intelligence, {
|
|
event_id: 99,
|
|
ingested_at: new Date().toISOString(),
|
|
content: 'content',
|
|
has_embedding: 1,
|
|
});
|
|
assert.equal(result.recovered, true);
|
|
const job = intelligence.prepare("SELECT status, lane, priority, attempts, last_error FROM autonomy_jobs WHERE idempotency_key='coordinator_event:99'").get();
|
|
assert.equal(job.status, 'pending');
|
|
assert.equal(job.lane, 'live');
|
|
assert.equal(job.priority, 100);
|
|
assert.equal(job.attempts, 0);
|
|
assert.match(job.last_error, /Recovered transient coordinator failure/);
|
|
});
|
|
|
|
test('archive reconciliation leaves non-transient coordinator dead letters alone', () => {
|
|
const intelligence = new Database(':memory:');
|
|
initAutonomySchema(intelligence);
|
|
enqueueJob(intelligence, {
|
|
jobType: 'coordinator_event', lane: 'historical', priority: 10, entityType: 'event', entityId: 100,
|
|
idempotencyKey: 'coordinator_event:100',
|
|
});
|
|
intelligence.prepare(`
|
|
UPDATE autonomy_jobs
|
|
SET status='dead_letter', attempts=5, last_error='no tradable instruments are allowlisted'
|
|
WHERE idempotency_key='coordinator_event:100'
|
|
`).run();
|
|
|
|
const result = enqueueCoordinatorEvent(intelligence, {
|
|
event_id: 100,
|
|
ingested_at: new Date().toISOString(),
|
|
content: 'content',
|
|
has_embedding: 1,
|
|
});
|
|
assert.equal(result.recovered, false);
|
|
const job = intelligence.prepare("SELECT status, attempts, last_error FROM autonomy_jobs WHERE idempotency_key='coordinator_event:100'").get();
|
|
assert.equal(job.status, 'dead_letter');
|
|
assert.equal(job.attempts, 5);
|
|
assert.equal(job.last_error, 'no tradable instruments are allowlisted');
|
|
});
|
|
|
|
test('outcome calculation uses benchmark-relative return', () => {
|
|
const outcome = calculateOutcome(
|
|
{ information_cutoff: '2026-01-02T00:00:00Z', horizon_days: 5, direction: 'positive' },
|
|
[{ date: '2026-01-02', close: 100 }, { date: '2026-01-09', close: 110 }],
|
|
[{ date: '2026-01-02', close: 100 }, { date: '2026-01-09', close: 105 }]
|
|
);
|
|
assert.equal(outcome.directionCorrect, 1);
|
|
assert.equal(outcome.excessReturn, 0.05);
|
|
});
|
|
|
|
test('order intents require the explicit instrument allowlist', () => {
|
|
const db = new Database(':memory:');
|
|
initAutonomySchema(db);
|
|
db.prepare("INSERT INTO autonomy_instruments(symbol, broker, active, tradable) VALUES ('NVDA', 'sim', 1, 1)").run();
|
|
const proposal = db.prepare(`INSERT INTO autonomy_proposals(payload, information_cutoff, status) VALUES ('{}', datetime('now'), 'accepted')`).run();
|
|
const prediction = db.prepare(`INSERT INTO autonomy_predictions(proposal_id, instrument, direction, event_type, horizon_days, information_cutoff, evidence_article_ids, learning_eligible, strategy_version) VALUES (?, 'NVDA', 'positive', 'test', 10, datetime('now'), '[1]', 1, 'test')`).run(proposal.lastInsertRowid);
|
|
const decision = db.prepare(`INSERT INTO autonomy_decisions(prediction_id, action, rationale, strategy_version) VALUES (?, 'BUY', 'test', 'test')`).run(prediction.lastInsertRowid);
|
|
const intent = createOrderIntent(db, decision.lastInsertRowid, 100, { maxNotional: 100 });
|
|
assert.equal(intent.side, 'buy');
|
|
assert.equal(db.prepare('SELECT status FROM autonomy_order_intents').get().status, 'shadow');
|
|
});
|
|
|
|
test('event_type is held to the closed family enum', () => {
|
|
const base = { instrument: 'NVDA', direction: 'positive', horizon_days: 10, evidence_article_ids: [1] };
|
|
const typeOf = (eventType) => normalizeProposal({ predictions: [{ ...base, event_type: eventType }] }).predictions[0].eventType;
|
|
|
|
// the enum itself, in the shapes a model actually emits
|
|
assert.equal(typeOf('earnings'), 'earnings');
|
|
assert.equal(typeOf('Supply Chain'), 'supply_chain');
|
|
assert.equal(typeOf('M_AND_A'), 'm_and_a');
|
|
|
|
// "none of these fit" is a legitimate answer and has to survive
|
|
assert.equal(typeOf('other'), 'other');
|
|
|
|
// off-enum but placeable: salvaged onto the family calibration would have
|
|
// picked anyway, so we dont throw away a usable prediction over a label
|
|
assert.equal(typeOf('earnings_beat_q3'), 'earnings');
|
|
assert.equal(typeOf('ceo resignation'), 'leadership');
|
|
|
|
// unplaceable free text is the 201-distinct-values failure, and is rejected
|
|
assert.throws(() => typeOf('vibes_shifted'), /event_type must be one of/);
|
|
assert.throws(() => typeOf(''), /event_type must be one of/);
|
|
});
|
|
|
|
test('dotted tickers are translated to the format the price feed expects', () => {
|
|
// BRK.B 404d forever and the outcome worker retried it in a hot loop
|
|
assert.equal(yahooSymbol('BRK.B'), 'BRK-B');
|
|
assert.equal(yahooSymbol('ABR.PRD'), 'ABR-PRD');
|
|
assert.equal(yahooSymbol('aac.u'), 'AAC-U');
|
|
|
|
// ordinary symbols must pass through untouched
|
|
assert.equal(yahooSymbol('NVDA'), 'NVDA');
|
|
assert.equal(yahooSymbol(' spy '), 'SPY');
|
|
});
|
|
|
|
test('an outcome whose horizon has not actually elapsed is not a result', () => {
|
|
// ITW and WMT both scored exactly 0.00% excess in production because at
|
|
// horizon 1 the entry and exit lookups landed on the same bar. Zero is not a
|
|
// measurement, and it was being recorded as a directional miss.
|
|
const sameBar = calculateOutcome(
|
|
{ information_cutoff: '2026-01-02T00:00:00Z', horizon_days: 1, direction: 'positive' },
|
|
[{ date: '2026-01-05', close: 100 }],
|
|
[{ date: '2026-01-05', close: 100 }]
|
|
);
|
|
assert.equal(sameBar, null);
|
|
|
|
// the exit bar genuinely existing still scores normally
|
|
const real = calculateOutcome(
|
|
{ information_cutoff: '2026-01-02T00:00:00Z', horizon_days: 1, direction: 'positive' },
|
|
[{ date: '2026-01-02', close: 100 }, { date: '2026-01-05', close: 104 }],
|
|
[{ date: '2026-01-02', close: 100 }, { date: '2026-01-05', close: 102 }]
|
|
);
|
|
assert.equal(real.directionCorrect, 1);
|
|
assert.ok(Math.abs(real.excessReturn - 0.02) < 1e-9);
|
|
});
|
|
|
|
test('trading day arithmetic steps over weekends', () => {
|
|
// friday + 1 trading day is monday, not saturday. the old due-check counted
|
|
// calendar days and so called a friday horizon-1 prediction due on saturday,
|
|
// when monday's close cannot exist yet.
|
|
assert.equal(addTradingDays('2026-01-02', 1), '2026-01-05');
|
|
assert.equal(addTradingDays('2026-01-02', 5), '2026-01-09');
|
|
assert.equal(addTradingDays('2026-01-02', 0), '2026-01-02');
|
|
});
|
|
|
|
test('a budget failure is transient but a bad key is not', () => {
|
|
const quota = 'Error: coordinator request failed with 403: {"error":{"message":"Key limit exceeded (monthly limit)."}}';
|
|
const credits = 'LLM 402: {"error":{"message":"Insufficient credits. Add more using ..."}}';
|
|
const afford = 'coordinator request failed with 402: can only afford 3921 tokens';
|
|
assert.equal(isTransientCoordinatorFailure(quota), true);
|
|
assert.equal(isTransientCoordinatorFailure(credits), true);
|
|
assert.equal(isTransientCoordinatorFailure(afford), true);
|
|
|
|
// these must stay dead, retrying them forever helps nobody
|
|
assert.equal(isTransientCoordinatorFailure('request failed with 403: invalid api key'), false);
|
|
assert.equal(isTransientCoordinatorFailure('request failed with 401: unauthorized'), false);
|
|
assert.equal(isTransientCoordinatorFailure('proposal references missing evidence'), false);
|
|
|
|
// and the pre-existing transient cases still are
|
|
assert.equal(isTransientCoordinatorFailure('TypeError: fetch failed'), true);
|
|
assert.equal(isTransientCoordinatorFailure('request failed with 503'), true);
|
|
});
|
|
|
|
test('an untradable instrument drops itself, not its valid siblings', () => {
|
|
const archive = new Database(':memory:');
|
|
archive.exec('CREATE TABLE articles (id INTEGER PRIMARY KEY, pub_date_effective TEXT, pub_date TEXT, ingested_at TEXT)');
|
|
archive.prepare("INSERT INTO articles VALUES (7, '2020-01-01T00:00:00Z', NULL, '2020-01-01T00:00:00Z')").run();
|
|
const intelligence = new Database(':memory:');
|
|
initAutonomySchema(intelligence);
|
|
intelligence.prepare("INSERT INTO autonomy_instruments(symbol, broker, active, tradable) VALUES ('NVDA','test',1,1)").run();
|
|
|
|
const pred = (instrument) => ({
|
|
instrument, direction: 'positive', event_type: 'earnings',
|
|
horizon_days: 10, evidence_article_ids: [7],
|
|
});
|
|
|
|
// EURUSD used to take NVDA down with it and lose the whole proposal
|
|
const result = acceptProposal(intelligence, archive,
|
|
{ predictions: [pred('NVDA'), pred('EURUSD')] },
|
|
{ informationCutoff: '2026-01-01T00:00:00Z', strategyVersion: 'test' });
|
|
|
|
assert.equal(result.predictionCount, 1);
|
|
assert.deepEqual(result.droppedInstruments, ['EURUSD']);
|
|
const stored = intelligence.prepare('SELECT instrument FROM autonomy_predictions').all();
|
|
assert.deepEqual(stored.map((r) => r.instrument), ['NVDA']);
|
|
});
|
|
|
|
test('lookahead still rejects the whole proposal, not just one prediction', () => {
|
|
const archive = new Database(':memory:');
|
|
archive.exec('CREATE TABLE articles (id INTEGER PRIMARY KEY, pub_date_effective TEXT, pub_date TEXT, ingested_at TEXT)');
|
|
archive.prepare("INSERT INTO articles VALUES (7, '2020-06-01T00:00:00Z', NULL, '2020-06-01T00:00:00Z')").run();
|
|
const intelligence = new Database(':memory:');
|
|
initAutonomySchema(intelligence);
|
|
intelligence.prepare("INSERT INTO autonomy_instruments(symbol, broker, active, tradable) VALUES ('NVDA','test',1,1)").run();
|
|
|
|
// evidence postdates the cutoff: corrupt, not merely untradable
|
|
assert.throws(() => acceptProposal(intelligence, archive, {
|
|
predictions: [{ instrument: 'NVDA', direction: 'positive', event_type: 'earnings', horizon_days: 10, evidence_article_ids: [7] }],
|
|
}, { informationCutoff: '2020-01-01T00:00:00Z' }), /missing evidence/);
|
|
assert.equal(intelligence.prepare('SELECT COUNT(*) AS n FROM autonomy_predictions').get().n, 0);
|
|
});
|
|
|
|
test('graph context never shows a relationship the world had not revealed yet', () => {
|
|
const db = new Database(':memory:');
|
|
initAutonomySchema(db);
|
|
db.exec(`
|
|
CREATE TABLE tracked_companies (id INTEGER PRIMARY KEY, name TEXT, ticker TEXT, aliases TEXT);
|
|
CREATE TABLE event_knowledge (id INTEGER PRIMARY KEY, event_id INTEGER, company_id INTEGER, type TEXT, data TEXT, event_date TEXT);
|
|
CREATE TABLE company_relationships (id INTEGER PRIMARY KEY, from_company_id INTEGER, relationship_type TEXT,
|
|
to_entity TEXT, to_company_id INTEGER, confidence TEXT, confirmation_count INTEGER, first_seen_at DATETIME,
|
|
last_seen_at DATETIME, supporting_event_ids TEXT);
|
|
`);
|
|
db.prepare("INSERT INTO tracked_companies VALUES (1,'Nvidia','NVDA','[]')").run();
|
|
db.prepare("INSERT INTO event_knowledge (event_id, company_id, type) VALUES (55, 1, 'x')").run();
|
|
const edge = db.prepare(`INSERT INTO company_relationships
|
|
(from_company_id, relationship_type, to_entity, confidence, confirmation_count, first_seen_at)
|
|
VALUES (1, ?, ?, 'high', ?, ?)`);
|
|
edge.run('supplier', 'Taiwan Semiconductor', 9, '2025-01-01T00:00:00Z');
|
|
edge.run('customer', 'Future Corp', 4, '2026-01-01T00:00:00Z');
|
|
|
|
// a proposal dated mid-2025 may only see what existed by then
|
|
const past = buildGraphContext(db, 55, '2025-06-01T00:00:00Z');
|
|
assert.match(past, /Taiwan Semiconductor/);
|
|
assert.equal(/Future Corp/.test(past), false);
|
|
|
|
// later, both are legitimately visible
|
|
const now = buildGraphContext(db, 55, '2026-06-01T00:00:00Z');
|
|
assert.match(now, /Taiwan Semiconductor/);
|
|
assert.match(now, /Future Corp/);
|
|
|
|
// an event we know nothing about contributes nothing rather than a stray header
|
|
assert.equal(buildGraphContext(db, 999, '2026-06-01T00:00:00Z'), '');
|
|
assert.equal(buildGraphContext(db, 55, null), '');
|
|
});
|
|
|
|
test('the coordinator prompt carries graph context only when there is some', () => {
|
|
const withCtx = buildPrompt({ id: 1, title: 't' }, [{ id: 9, title: 'a', content: 'c' }],
|
|
'Known company relationships, as they stood at the information cutoff:\n\nNvidia (NVDA):\n - supplier: TSMC');
|
|
assert.match(withCtx, /Known company relationships/);
|
|
assert.match(withCtx, /supplier: TSMC/);
|
|
|
|
const without = buildPrompt({ id: 1, title: 't' }, [{ id: 9, title: 'a', content: 'c' }]);
|
|
assert.equal(/Known company relationships/.test(without), false);
|
|
});
|