feat: add isolated historical replay calibration
This commit is contained in:
@@ -92,6 +92,28 @@ services:
|
|||||||
networks:
|
networks:
|
||||||
- nginx_proxy_manager_default
|
- nginx_proxy_manager_default
|
||||||
|
|
||||||
|
replay:
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
provenance: false
|
||||||
|
command: node workers/replay-entrypoint.js
|
||||||
|
env_file: .env
|
||||||
|
volumes:
|
||||||
|
- ./config.json:/app/config.json:ro
|
||||||
|
- ./data:/data
|
||||||
|
environment:
|
||||||
|
NODE_ENV: production
|
||||||
|
DURIIN_DB: /data/archive.sqlite
|
||||||
|
INTELLIGENCE_DB: /data/intelligence.sqlite
|
||||||
|
AUTONOMY_REPLAY_POLL_MS: "${AUTONOMY_REPLAY_POLL_MS:-15000}"
|
||||||
|
AUTONOMY_REPLAY_DAILY_BUDGET: "${AUTONOMY_REPLAY_DAILY_BUDGET:-100}"
|
||||||
|
AUTONOMY_REPLAY_WATERMARK_DAYS: "${AUTONOMY_REPLAY_WATERMARK_DAYS:-7}"
|
||||||
|
cpus: "0.50"
|
||||||
|
mem_limit: 768m
|
||||||
|
restart: unless-stopped
|
||||||
|
networks:
|
||||||
|
- nginx_proxy_manager_default
|
||||||
|
|
||||||
calibration:
|
calibration:
|
||||||
build:
|
build:
|
||||||
context: .
|
context: .
|
||||||
|
|||||||
+2
-1
@@ -10,7 +10,8 @@
|
|||||||
"autonomy:init": "node scripts/initialize-autonomy.js",
|
"autonomy:init": "node scripts/initialize-autonomy.js",
|
||||||
"autonomy:allowlist": "node scripts/set-autonomy-instrument.js",
|
"autonomy:allowlist": "node scripts/set-autonomy-instrument.js",
|
||||||
"autonomy:sync-assets": "node scripts/sync-paper-assets.js",
|
"autonomy:sync-assets": "node scripts/sync-paper-assets.js",
|
||||||
"autonomy:import-legacy": "node scripts/import-legacy-intelligence.js"
|
"autonomy:import-legacy": "node scripts/import-legacy-intelligence.js",
|
||||||
|
"autonomy:replay": "node workers/replay-entrypoint.js"
|
||||||
},
|
},
|
||||||
"keywords": [],
|
"keywords": [],
|
||||||
"author": "",
|
"author": "",
|
||||||
|
|||||||
@@ -79,6 +79,12 @@
|
|||||||
byId("perf-cohorts").textContent = formatNumber(data.calibration?.length || 0);
|
byId("perf-cohorts").textContent = formatNumber(data.calibration?.length || 0);
|
||||||
if (outcomes) byId("performance-note").textContent = `Measured on ${outcomes} matured predictions. Results remain descriptive until the sample is large enough for stable calibration.`;
|
if (outcomes) byId("performance-note").textContent = `Measured on ${outcomes} matured predictions. Results remain descriptive until the sample is large enough for stable calibration.`;
|
||||||
|
|
||||||
|
const replay = data.replay;
|
||||||
|
byId("replay-status").textContent = replay ? replay.status : "Not started";
|
||||||
|
byId("replay-articles").textContent = replay ? formatNumber(replay.processed_articles || 0) : "—";
|
||||||
|
byId("replay-evaluations").textContent = replay ? formatNumber(replay.evaluations || 0) : "—";
|
||||||
|
byId("replay-watermark").textContent = replay?.watermark_at ? formatRelative(replay.watermark_at) : "—";
|
||||||
|
|
||||||
byId("runtime-queue").textContent = `${formatNumber(pendingHistorical, true)} pending`;
|
byId("runtime-queue").textContent = `${formatNumber(pendingHistorical, true)} pending`;
|
||||||
byId("runtime-coordinator").textContent = pendingHistorical ? "Processing" : "Watching";
|
byId("runtime-coordinator").textContent = pendingHistorical ? "Processing" : "Watching";
|
||||||
if (data.account) {
|
if (data.account) {
|
||||||
|
|||||||
@@ -128,6 +128,16 @@
|
|||||||
</div>
|
</div>
|
||||||
</aside>
|
</aside>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<section class="panel" aria-label="Historical replay">
|
||||||
|
<div class="section-head"><div><span class="section-index">06</span><h3>Historical replay</h3></div><span class="mode-pill">Isolated from execution</span></div>
|
||||||
|
<div class="performance-facts">
|
||||||
|
<div><span>Run state</span><strong id="replay-status">Starting…</strong></div>
|
||||||
|
<div><span>Articles replayed</span><strong id="replay-articles">—</strong></div>
|
||||||
|
<div><span>Walk-forward evaluations</span><strong id="replay-evaluations">—</strong></div>
|
||||||
|
<div><span>Watermark</span><strong id="replay-watermark">—</strong></div>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
</main>
|
</main>
|
||||||
|
|
||||||
<div id="toast"><span class="toast-dot"></span><span id="toast-msg"></span></div>
|
<div id="toast"><span class="toast-dot"></span><span id="toast-msg"></span></div>
|
||||||
|
|||||||
@@ -34,9 +34,22 @@ function normalizeProposal(raw, { informationCutoff, model = 'unknown', promptVe
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function verifyEvidence(archiveDb, articleIds) {
|
function verifyEvidence(archiveDb, articleIds, informationCutoff = null) {
|
||||||
const placeholders = articleIds.map(() => '?').join(',');
|
const placeholders = articleIds.map(() => '?').join(',');
|
||||||
const rows = archiveDb.prepare(`SELECT id FROM articles WHERE id IN (${placeholders})`).all(...articleIds);
|
// A replay must only see material which existed at its information cutoff.
|
||||||
|
// Live proposals retain the simpler existence check.
|
||||||
|
const cutoffClause = informationCutoff ? ' AND datetime(COALESCE(pub_date_effective, pub_date, ingested_at)) <= datetime(?)' : '';
|
||||||
|
let rows;
|
||||||
|
try {
|
||||||
|
rows = archiveDb.prepare(`SELECT id FROM articles WHERE id IN (${placeholders})${cutoffClause}`)
|
||||||
|
.all(...articleIds, ...(informationCutoff ? [informationCutoff] : []));
|
||||||
|
} catch (error) {
|
||||||
|
// Minimal/test archives may not retain publication metadata. A production
|
||||||
|
// replay archive is required to have it, so this fallback is only for the
|
||||||
|
// existing live evidence contract.
|
||||||
|
if (informationCutoff) throw error;
|
||||||
|
rows = archiveDb.prepare(`SELECT id FROM articles WHERE id IN (${placeholders})`).all(...articleIds);
|
||||||
|
}
|
||||||
const found = new Set(rows.map((row) => row.id));
|
const found = new Set(rows.map((row) => row.id));
|
||||||
return articleIds.every((id) => found.has(id));
|
return articleIds.every((id) => found.has(id));
|
||||||
}
|
}
|
||||||
@@ -44,7 +57,7 @@ function verifyEvidence(archiveDb, articleIds) {
|
|||||||
function acceptProposal(intelligenceDb, archiveDb, raw, metadata = {}) {
|
function acceptProposal(intelligenceDb, archiveDb, raw, metadata = {}) {
|
||||||
const proposal = normalizeProposal(raw, metadata);
|
const proposal = normalizeProposal(raw, metadata);
|
||||||
for (const prediction of proposal.predictions) {
|
for (const prediction of proposal.predictions) {
|
||||||
if (!verifyEvidence(archiveDb, prediction.evidenceArticleIds)) {
|
if (!verifyEvidence(archiveDb, prediction.evidenceArticleIds, metadata.origin === 'replay' ? proposal.informationCutoff : null)) {
|
||||||
throw new Error(`proposal references missing evidence for ${prediction.instrument}`);
|
throw new Error(`proposal references missing evidence for ${prediction.instrument}`);
|
||||||
}
|
}
|
||||||
const instrument = intelligenceDb.prepare(
|
const instrument = intelligenceDb.prepare(
|
||||||
@@ -60,8 +73,8 @@ function acceptProposal(intelligenceDb, archiveDb, raw, metadata = {}) {
|
|||||||
const insertPrediction = intelligenceDb.prepare(`
|
const insertPrediction = intelligenceDb.prepare(`
|
||||||
INSERT INTO autonomy_predictions
|
INSERT INTO autonomy_predictions
|
||||||
(proposal_id, event_id, instrument, direction, event_type, causal_channel,
|
(proposal_id, event_id, instrument, direction, event_type, causal_channel,
|
||||||
horizon_days, information_cutoff, evidence_article_ids, invalidation_condition, learning_eligible, strategy_version)
|
horizon_days, information_cutoff, evidence_article_ids, invalidation_condition, learning_eligible, strategy_version, origin, replay_run_id)
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
`);
|
`);
|
||||||
const tx = intelligenceDb.transaction(() => {
|
const tx = intelligenceDb.transaction(() => {
|
||||||
const proposalResult = insert.run(metadata.eventId || null, JSON.stringify(proposal), proposal.informationCutoff,
|
const proposalResult = insert.run(metadata.eventId || null, JSON.stringify(proposal), proposal.informationCutoff,
|
||||||
@@ -70,7 +83,8 @@ function acceptProposal(intelligenceDb, archiveDb, raw, metadata = {}) {
|
|||||||
insertPrediction.run(proposalResult.lastInsertRowid, metadata.eventId || null, prediction.instrument,
|
insertPrediction.run(proposalResult.lastInsertRowid, metadata.eventId || null, prediction.instrument,
|
||||||
prediction.direction, prediction.eventType, prediction.causalChannel, prediction.horizonDays,
|
prediction.direction, prediction.eventType, prediction.causalChannel, prediction.horizonDays,
|
||||||
proposal.informationCutoff, JSON.stringify(prediction.evidenceArticleIds), prediction.invalidationCondition,
|
proposal.informationCutoff, JSON.stringify(prediction.evidenceArticleIds), prediction.invalidationCondition,
|
||||||
metadata.learningEligible ? 1 : 0, metadata.strategyVersion || 'autonomy-1');
|
metadata.learningEligible ? 1 : 0, metadata.strategyVersion || 'autonomy-1',
|
||||||
|
metadata.origin || 'live', metadata.replayRunId || null);
|
||||||
}
|
}
|
||||||
return Number(proposalResult.lastInsertRowid);
|
return Number(proposalResult.lastInsertRowid);
|
||||||
});
|
});
|
||||||
|
|||||||
+41
-1
@@ -1,4 +1,4 @@
|
|||||||
const AUTONOMY_SCHEMA_VERSION = 1;
|
const AUTONOMY_SCHEMA_VERSION = 2;
|
||||||
|
|
||||||
function initAutonomySchema(db) {
|
function initAutonomySchema(db) {
|
||||||
db.exec(`
|
db.exec(`
|
||||||
@@ -178,12 +178,52 @@ function initAutonomySchema(db) {
|
|||||||
captured_at TEXT NOT NULL DEFAULT (datetime('now'))
|
captured_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||||
);
|
);
|
||||||
|
|
||||||
|
-- Historical replay is a separate evidence path. It deliberately never
|
||||||
|
-- writes to autonomy_decisions or order intents.
|
||||||
|
CREATE TABLE IF NOT EXISTS autonomy_replay_runs (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
status TEXT NOT NULL DEFAULT 'running'
|
||||||
|
CHECK (status IN ('running', 'paused', 'complete', 'superseded', 'failed')),
|
||||||
|
start_at TEXT,
|
||||||
|
watermark_at TEXT,
|
||||||
|
cursor_article_id INTEGER NOT NULL DEFAULT 0,
|
||||||
|
cursor_effective_at TEXT,
|
||||||
|
processed_articles INTEGER NOT NULL DEFAULT 0,
|
||||||
|
strategy_version TEXT NOT NULL,
|
||||||
|
prompt_version TEXT NOT NULL,
|
||||||
|
coordinator_model TEXT,
|
||||||
|
price_provider TEXT NOT NULL DEFAULT 'yahoo',
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
completed_at TEXT,
|
||||||
|
last_error TEXT
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_autonomy_replay_runs_active
|
||||||
|
ON autonomy_replay_runs(status, cursor_article_id);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS autonomy_replay_evaluations (
|
||||||
|
prediction_id INTEGER PRIMARY KEY REFERENCES autonomy_predictions(id),
|
||||||
|
replay_run_id INTEGER NOT NULL REFERENCES autonomy_replay_runs(id),
|
||||||
|
snapshot_cutoff TEXT NOT NULL,
|
||||||
|
sample_size INTEGER NOT NULL,
|
||||||
|
action TEXT NOT NULL CHECK (action IN ('BUY', 'SELL', 'HOLD', 'ABSTAIN')),
|
||||||
|
calibrated_probability REAL,
|
||||||
|
expected_excess_return REAL,
|
||||||
|
rationale TEXT NOT NULL,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||||
|
);
|
||||||
|
|
||||||
INSERT OR IGNORE INTO autonomy_schema(version) VALUES (${AUTONOMY_SCHEMA_VERSION});
|
INSERT OR IGNORE INTO autonomy_schema(version) VALUES (${AUTONOMY_SCHEMA_VERSION});
|
||||||
`);
|
`);
|
||||||
for (const statement of [
|
for (const statement of [
|
||||||
'ALTER TABLE autonomy_order_intents ADD COLUMN attempts INTEGER NOT NULL DEFAULT 0',
|
'ALTER TABLE autonomy_order_intents ADD COLUMN attempts INTEGER NOT NULL DEFAULT 0',
|
||||||
'ALTER TABLE autonomy_order_intents ADD COLUMN last_error TEXT',
|
'ALTER TABLE autonomy_order_intents ADD COLUMN last_error TEXT',
|
||||||
'ALTER TABLE autonomy_predictions ADD COLUMN learning_eligible INTEGER NOT NULL DEFAULT 0',
|
'ALTER TABLE autonomy_predictions ADD COLUMN learning_eligible INTEGER NOT NULL DEFAULT 0',
|
||||||
|
"ALTER TABLE autonomy_predictions ADD COLUMN origin TEXT NOT NULL DEFAULT 'live'",
|
||||||
|
'ALTER TABLE autonomy_predictions ADD COLUMN replay_run_id INTEGER',
|
||||||
|
'ALTER TABLE autonomy_replay_runs ADD COLUMN cursor_effective_at TEXT',
|
||||||
|
"ALTER TABLE autonomy_calibration_snapshots ADD COLUMN source TEXT NOT NULL DEFAULT 'live'",
|
||||||
|
'ALTER TABLE autonomy_calibration_snapshots ADD COLUMN replay_run_id INTEGER',
|
||||||
]) {
|
]) {
|
||||||
try { db.exec(statement); } catch (_) {}
|
try { db.exec(statement); } catch (_) {}
|
||||||
}
|
}
|
||||||
|
|||||||
+16
-1
@@ -174,7 +174,9 @@ async function adminRoutes(fastify) {
|
|||||||
const outcomeSummary = intelligenceDb.prepare(`
|
const outcomeSummary = intelligenceDb.prepare(`
|
||||||
SELECT COUNT(*) AS total, SUM(direction_correct) AS correct,
|
SELECT COUNT(*) AS total, SUM(direction_correct) AS correct,
|
||||||
AVG(excess_return) AS average_excess_return
|
AVG(excess_return) AS average_excess_return
|
||||||
FROM autonomy_outcomes
|
FROM autonomy_outcomes o
|
||||||
|
JOIN autonomy_predictions p ON p.id = o.prediction_id
|
||||||
|
WHERE p.origin = 'live'
|
||||||
`).get();
|
`).get();
|
||||||
const instruments = intelligenceDb.prepare(`
|
const instruments = intelligenceDb.prepare(`
|
||||||
SELECT COUNT(*) AS count FROM autonomy_instruments WHERE active=1 AND tradable=1
|
SELECT COUNT(*) AS count FROM autonomy_instruments WHERE active=1 AND tradable=1
|
||||||
@@ -214,6 +216,18 @@ async function adminRoutes(fastify) {
|
|||||||
expected_excess_return, lower_return, upper_return, created_at
|
expected_excess_return, lower_return, upper_return, created_at
|
||||||
FROM autonomy_calibration_snapshots ORDER BY id DESC LIMIT 8
|
FROM autonomy_calibration_snapshots ORDER BY id DESC LIMIT 8
|
||||||
`).all();
|
`).all();
|
||||||
|
const replay = intelligenceDb.prepare(`
|
||||||
|
SELECT r.id, r.status, r.watermark_at, r.cursor_article_id, r.cursor_effective_at,
|
||||||
|
r.processed_articles, r.updated_at,
|
||||||
|
SUM(CASE WHEN p.status = 'resolved' THEN 1 ELSE 0 END) AS resolved_predictions,
|
||||||
|
COUNT(p.id) AS predictions,
|
||||||
|
SUM(o.direction_correct) AS correct_predictions,
|
||||||
|
(SELECT COUNT(*) FROM autonomy_replay_evaluations e WHERE e.replay_run_id = r.id) AS evaluations
|
||||||
|
FROM autonomy_replay_runs r
|
||||||
|
LEFT JOIN autonomy_predictions p ON p.replay_run_id = r.id
|
||||||
|
LEFT JOIN autonomy_outcomes o ON o.prediction_id = p.id
|
||||||
|
GROUP BY r.id ORDER BY r.id DESC LIMIT 1
|
||||||
|
`).get() || null;
|
||||||
|
|
||||||
return {
|
return {
|
||||||
enabled: true,
|
enabled: true,
|
||||||
@@ -232,6 +246,7 @@ async function adminRoutes(fastify) {
|
|||||||
latestOrders,
|
latestOrders,
|
||||||
account,
|
account,
|
||||||
calibration,
|
calibration,
|
||||||
|
replay,
|
||||||
generatedAt: new Date().toISOString(),
|
generatedAt: new Date().toISOString(),
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -53,6 +53,26 @@ test('accepted proposal preserves evidence and creates immutable prediction', ()
|
|||||||
assert.deepEqual(JSON.parse(intelligence.prepare('SELECT evidence_article_ids FROM autonomy_predictions').get().evidence_article_ids), [7]);
|
assert.deepEqual(JSON.parse(intelligence.prepare('SELECT evidence_article_ids FROM autonomy_predictions').get().evidence_article_ids), [7]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('replay evidence cannot look beyond its information cutoff and remains non-executable', () => {
|
||||||
|
const archive = new Database(':memory:');
|
||||||
|
archive.exec('CREATE TABLE articles (id INTEGER PRIMARY KEY, pub_date_effective TEXT, pub_date TEXT, ingested_at TEXT)');
|
||||||
|
archive.prepare("INSERT INTO articles VALUES (7, '2020-01-01T00:00:00Z', NULL, '2020-01-01T00:00:00Z')").run();
|
||||||
|
const intelligence = new Database(':memory:');
|
||||||
|
initAutonomySchema(intelligence);
|
||||||
|
intelligence.prepare("INSERT INTO autonomy_instruments(symbol, broker, active, tradable) VALUES ('NVDA', 'test', 1, 1)").run();
|
||||||
|
assert.throws(() => acceptProposal(intelligence, archive, {
|
||||||
|
predictions: [{ instrument: 'NVDA', direction: 'positive', event_type: 'earnings', horizon_days: 10, evidence_article_ids: [7] }],
|
||||||
|
}, { informationCutoff: '2019-12-31T00:00:00Z', origin: 'replay', replayRunId: 1 }), /missing evidence/);
|
||||||
|
const proposal = intelligence.prepare("INSERT INTO autonomy_proposals(payload, information_cutoff, status) VALUES ('{}', datetime('now'), 'accepted')").run();
|
||||||
|
const prediction = intelligence.prepare(`INSERT INTO autonomy_predictions
|
||||||
|
(proposal_id, instrument, direction, event_type, horizon_days, information_cutoff, evidence_article_ids, strategy_version, origin)
|
||||||
|
VALUES (?, 'NVDA', 'positive', 'test', 10, datetime('now'), '[7]', 'test', 'replay')`).run(proposal.lastInsertRowid);
|
||||||
|
intelligence.prepare("INSERT INTO autonomy_decisions(prediction_id, action, rationale, strategy_version) VALUES (?, 'BUY', 'test', 'test')").run(prediction.lastInsertRowid);
|
||||||
|
const executable = intelligence.prepare(`SELECT d.id FROM autonomy_decisions d JOIN autonomy_predictions p ON p.id=d.prediction_id
|
||||||
|
WHERE d.action IN ('BUY','SELL') AND p.origin='live'`).all();
|
||||||
|
assert.equal(executable.length, 0);
|
||||||
|
});
|
||||||
|
|
||||||
test('calibration and policy abstain on insufficient evidence', () => {
|
test('calibration and policy abstain on insufficient evidence', () => {
|
||||||
const calibration = calibrateOutcomes([
|
const calibration = calibrateOutcomes([
|
||||||
{ excess_return: 0.02, direction_correct: 1 },
|
{ excess_return: 0.02, direction_correct: 1 },
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ function refreshCalibration(db, version = `cal-${Date.now()}`) {
|
|||||||
SELECT p.direction, p.event_type, p.horizon_days, o.*
|
SELECT p.direction, p.event_type, p.horizon_days, o.*
|
||||||
FROM autonomy_predictions p
|
FROM autonomy_predictions p
|
||||||
JOIN autonomy_outcomes o ON o.prediction_id = p.id
|
JOIN autonomy_outcomes o ON o.prediction_id = p.id
|
||||||
WHERE p.status = 'resolved' AND p.learning_eligible = 1
|
WHERE p.status = 'resolved' AND p.learning_eligible = 1 AND p.origin = 'live'
|
||||||
`).all().reduce((map, row) => {
|
`).all().reduce((map, row) => {
|
||||||
const key = cohortKey({ direction: row.direction, eventType: row.event_type, horizonDays: row.horizon_days });
|
const key = cohortKey({ direction: row.direction, eventType: row.event_type, horizonDays: row.horizon_days });
|
||||||
if (!map.has(key)) map.set(key, []);
|
if (!map.has(key)) map.set(key, []);
|
||||||
@@ -21,8 +21,8 @@ function refreshCalibration(db, version = `cal-${Date.now()}`) {
|
|||||||
const insert = db.prepare(`
|
const insert = db.prepare(`
|
||||||
INSERT INTO autonomy_calibration_snapshots
|
INSERT INTO autonomy_calibration_snapshots
|
||||||
(cohort_key, sample_size, effective_sample_size, directional_probability,
|
(cohort_key, sample_size, effective_sample_size, directional_probability,
|
||||||
expected_excess_return, lower_return, upper_return, parent_cohort_key, version)
|
expected_excess_return, lower_return, upper_return, parent_cohort_key, version, source, replay_run_id)
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'live', NULL)
|
||||||
`);
|
`);
|
||||||
const tx = db.transaction(() => {
|
const tx = db.transaction(() => {
|
||||||
for (const [key, rows] of groups) {
|
for (const [key, rows] of groups) {
|
||||||
@@ -40,7 +40,7 @@ function createDecisions(db, strategyVersion = 'autonomy-1') {
|
|||||||
const predictions = db.prepare(`
|
const predictions = db.prepare(`
|
||||||
SELECT p.* FROM autonomy_predictions p
|
SELECT p.* FROM autonomy_predictions p
|
||||||
LEFT JOIN autonomy_decisions d ON d.prediction_id = p.id
|
LEFT JOIN autonomy_decisions d ON d.prediction_id = p.id
|
||||||
WHERE d.prediction_id IS NULL AND p.status = 'open'
|
WHERE d.prediction_id IS NULL AND p.status = 'open' AND p.origin = 'live'
|
||||||
`).all();
|
`).all();
|
||||||
const latest = db.prepare(`
|
const latest = db.prepare(`
|
||||||
SELECT * FROM autonomy_calibration_snapshots
|
SELECT * FROM autonomy_calibration_snapshots
|
||||||
@@ -68,6 +68,44 @@ function createDecisions(db, strategyVersion = 'autonomy-1') {
|
|||||||
return created;
|
return created;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Replay evaluations are walk-forward: each historical prediction is scored
|
||||||
|
// against calibration data that had matured strictly before its cutoff. They
|
||||||
|
// are stored in their own ledger, never in autonomy_decisions.
|
||||||
|
function refreshReplayEvaluations(db) {
|
||||||
|
const predictions = db.prepare(`
|
||||||
|
SELECT p.*, o.excess_return, o.direction_correct
|
||||||
|
FROM autonomy_predictions p JOIN autonomy_outcomes o ON o.prediction_id = p.id
|
||||||
|
LEFT JOIN autonomy_replay_evaluations e ON e.prediction_id = p.id
|
||||||
|
WHERE p.origin = 'replay' AND p.status = 'resolved' AND e.prediction_id IS NULL
|
||||||
|
ORDER BY datetime(p.information_cutoff), p.id LIMIT 200
|
||||||
|
`).all();
|
||||||
|
const prior = db.prepare(`
|
||||||
|
SELECT p.direction, p.event_type, p.horizon_days, o.*
|
||||||
|
FROM autonomy_predictions p JOIN autonomy_outcomes o ON o.prediction_id = p.id
|
||||||
|
WHERE p.origin = 'replay' AND p.status = 'resolved'
|
||||||
|
AND datetime(p.information_cutoff, '+' || p.horizon_days || ' days') < datetime(?)
|
||||||
|
`);
|
||||||
|
const insert = db.prepare(`
|
||||||
|
INSERT INTO autonomy_replay_evaluations
|
||||||
|
(prediction_id, replay_run_id, snapshot_cutoff, sample_size, action, calibrated_probability, expected_excess_return, rationale)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
`);
|
||||||
|
const tx = db.transaction(() => {
|
||||||
|
for (const prediction of predictions) {
|
||||||
|
const key = cohortKey({ direction: prediction.direction, eventType: prediction.event_type, horizonDays: prediction.horizon_days });
|
||||||
|
const rows = prior.all(prediction.information_cutoff).filter((row) =>
|
||||||
|
cohortKey({ direction: row.direction, eventType: row.event_type, horizonDays: row.horizon_days }) === key);
|
||||||
|
const calibration = rows.length ? calibrateOutcomes(rows) : null;
|
||||||
|
const decision = calibration ? decide({ ...calibration, direction: prediction.direction }, { minSampleSize: 30 })
|
||||||
|
: { action: 'ABSTAIN', rationale: 'walk-forward calibration unavailable' };
|
||||||
|
insert.run(prediction.id, prediction.replay_run_id, prediction.information_cutoff, rows.length, decision.action,
|
||||||
|
calibration?.directionalProbability || null, calibration?.expectedExcessReturn || null, decision.rationale);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
tx();
|
||||||
|
return predictions.length;
|
||||||
|
}
|
||||||
|
|
||||||
async function runCalibrationWorker({ intelligencePath, pollMs = 60000, workerId = `calibration-${os.hostname()}-${process.pid}` } = {}) {
|
async function runCalibrationWorker({ intelligencePath, pollMs = 60000, workerId = `calibration-${os.hostname()}-${process.pid}` } = {}) {
|
||||||
const db = new Database(intelligencePath);
|
const db = new Database(intelligencePath);
|
||||||
db.pragma('journal_mode = WAL');
|
db.pragma('journal_mode = WAL');
|
||||||
@@ -78,7 +116,8 @@ async function runCalibrationWorker({ intelligencePath, pollMs = 60000, workerId
|
|||||||
const state = db.prepare('SELECT COUNT(*) AS count, COALESCE(MAX(prediction_id), 0) AS max_id FROM autonomy_outcomes').get();
|
const state = db.prepare('SELECT COUNT(*) AS count, COALESCE(MAX(prediction_id), 0) AS max_id FROM autonomy_outcomes').get();
|
||||||
const groups = refreshCalibration(db, `cal-${state.count}-${state.max_id}`);
|
const groups = refreshCalibration(db, `cal-${state.count}-${state.max_id}`);
|
||||||
const decisions = createDecisions(db);
|
const decisions = createDecisions(db);
|
||||||
if (groups || decisions) console.log(`[${workerId}] calibration groups=${groups} decisions=${decisions}`);
|
const replayEvaluations = refreshReplayEvaluations(db);
|
||||||
|
if (groups || decisions || replayEvaluations) console.log(`[${workerId}] calibration groups=${groups} decisions=${decisions} replay_evaluations=${replayEvaluations}`);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`[${workerId}] calibration error:`, error.message);
|
console.error(`[${workerId}] calibration error:`, error.message);
|
||||||
}
|
}
|
||||||
@@ -86,4 +125,4 @@ async function runCalibrationWorker({ intelligencePath, pollMs = 60000, workerId
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = { refreshCalibration, createDecisions, runCalibrationWorker };
|
module.exports = { refreshCalibration, createDecisions, refreshReplayEvaluations, runCalibrationWorker };
|
||||||
|
|||||||
@@ -51,8 +51,9 @@ async function runExecutionWorker({ intelligencePath, pollMs = 10000, mode = 'sh
|
|||||||
}
|
}
|
||||||
const decisions = db.prepare(`
|
const decisions = db.prepare(`
|
||||||
SELECT d.id FROM autonomy_decisions d
|
SELECT d.id FROM autonomy_decisions d
|
||||||
|
JOIN autonomy_predictions p ON p.id = d.prediction_id
|
||||||
LEFT JOIN autonomy_order_intents oi ON oi.decision_id = d.id
|
LEFT JOIN autonomy_order_intents oi ON oi.decision_id = d.id
|
||||||
WHERE oi.id IS NULL AND d.action IN ('BUY', 'SELL')
|
WHERE oi.id IS NULL AND d.action IN ('BUY', 'SELL') AND p.origin = 'live'
|
||||||
ORDER BY d.created_at ASC LIMIT 25
|
ORDER BY d.created_at ASC LIMIT 25
|
||||||
`).all();
|
`).all();
|
||||||
for (const decision of decisions) {
|
for (const decision of decisions) {
|
||||||
|
|||||||
@@ -20,7 +20,9 @@ function httpGet(url) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function history(symbol) {
|
async function history(symbol) {
|
||||||
const url = `https://query1.finance.yahoo.com/v8/finance/chart/${encodeURIComponent(symbol)}?range=10y&interval=1d`;
|
// GDELT backfills predate the normal rolling quote window. Use an explicit
|
||||||
|
// point-in-time range so replay outcomes do not silently become unresolvable.
|
||||||
|
const url = `https://query1.finance.yahoo.com/v8/finance/chart/${encodeURIComponent(symbol)}?period1=946684800&period2=${Math.floor(Date.now() / 1000)}&interval=1d`;
|
||||||
const body = JSON.parse(await httpGet(url));
|
const body = JSON.parse(await httpGet(url));
|
||||||
const result = body?.chart?.result?.[0];
|
const result = body?.chart?.result?.[0];
|
||||||
if (!result) return [];
|
if (!result) return [];
|
||||||
|
|||||||
@@ -0,0 +1,8 @@
|
|||||||
|
const path = require('path');
|
||||||
|
const { runReplayWorker } = require('./replayWorker');
|
||||||
|
|
||||||
|
runReplayWorker({
|
||||||
|
archivePath: process.env.DURIIN_DB || path.resolve('/data/archive.sqlite'),
|
||||||
|
intelligencePath: process.env.INTELLIGENCE_DB || path.resolve('/data/intelligence.sqlite'),
|
||||||
|
pollMs: Number(process.env.AUTONOMY_REPLAY_POLL_MS) || 15000,
|
||||||
|
}).catch((error) => { console.error('[replay] fatal:', error); process.exit(1); });
|
||||||
@@ -0,0 +1,113 @@
|
|||||||
|
const os = require('os');
|
||||||
|
const fs = require('fs');
|
||||||
|
const path = require('path');
|
||||||
|
const Database = require('better-sqlite3');
|
||||||
|
const { initAutonomySchema } = require('../src/autonomy/schema');
|
||||||
|
const { enqueueJob, leaseNextJob, completeJob, failJob } = require('../src/autonomy/jobs');
|
||||||
|
const { callCoordinator } = require('../src/autonomy/llm');
|
||||||
|
const { acceptProposal, recordRejectedProposal } = require('../src/autonomy/coordinator');
|
||||||
|
|
||||||
|
function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); }
|
||||||
|
|
||||||
|
function loadConfig() {
|
||||||
|
const configPath = path.resolve(process.env.DURIIN_CONFIG || path.join(__dirname, '..', 'config.json'));
|
||||||
|
const raw = JSON.parse(fs.readFileSync(configPath, 'utf8'));
|
||||||
|
require('dotenv').config({ path: path.resolve(path.dirname(configPath), '.env') });
|
||||||
|
raw.openRouter = { ...(raw.openRouter || {}) };
|
||||||
|
if (process.env.OPEN_ROUTER_API_KEY) raw.openRouter.apiKey = process.env.OPEN_ROUTER_API_KEY;
|
||||||
|
if (process.env.OPEN_ROUTER_LLM_MODEL) raw.openRouter.llmModel = process.env.OPEN_ROUTER_LLM_MODEL;
|
||||||
|
return raw;
|
||||||
|
}
|
||||||
|
|
||||||
|
function articleTimeColumns(archiveDb) {
|
||||||
|
const columns = new Set(archiveDb.prepare('PRAGMA table_info(articles)').all().map((row) => row.name));
|
||||||
|
const candidates = ['pub_date_effective', 'pub_date', 'ingested_at'].filter((name) => columns.has(name));
|
||||||
|
if (!candidates.length) throw new Error('archive articles need publication or ingestion timestamps for replay');
|
||||||
|
return { columns, effective: `COALESCE(${candidates.join(', ')})` };
|
||||||
|
}
|
||||||
|
|
||||||
|
function replayPrompt(article) {
|
||||||
|
return `Historical evidence cutoff: ${article.effective_at}\n\n` +
|
||||||
|
`[Evidence 1] article_id=${article.id}\nTitle: ${article.title || ''}\n${String(article.content || article.description || '').slice(0, 6000)}\n\n` +
|
||||||
|
`Return JSON only in this shape:\n${JSON.stringify({ predictions: [{
|
||||||
|
instrument: 'NVDA', direction: 'positive|negative', event_type: 'stable_enum',
|
||||||
|
causal_channel: 'short description', horizon_days: 10, evidence_article_ids: [123],
|
||||||
|
invalidation_condition: 'condition',
|
||||||
|
}] }, null, 2)}\n\n` +
|
||||||
|
'Use only this dated evidence. Return an empty predictions array when there is no clear, tradable hypothesis. Never include probabilities, returns, confidence, position sizes, or actions.';
|
||||||
|
}
|
||||||
|
|
||||||
|
function activeRun(db, config) {
|
||||||
|
let run = db.prepare("SELECT * FROM autonomy_replay_runs WHERE status = 'running' ORDER BY id DESC LIMIT 1").get();
|
||||||
|
if (run) return run;
|
||||||
|
const watermarkDays = Math.max(1, Number(process.env.AUTONOMY_REPLAY_WATERMARK_DAYS) || 7);
|
||||||
|
const result = db.prepare(`
|
||||||
|
INSERT INTO autonomy_replay_runs (watermark_at, strategy_version, prompt_version, coordinator_model)
|
||||||
|
VALUES (datetime('now', ?), 'autonomy-1', 'replay-coordinator-1', ?)
|
||||||
|
`).run(`-${watermarkDays} days`, config.openRouter.llmModel || 'unknown');
|
||||||
|
return db.prepare('SELECT * FROM autonomy_replay_runs WHERE id = ?').get(result.lastInsertRowid);
|
||||||
|
}
|
||||||
|
|
||||||
|
function scheduleNext(db, archiveDb, run) {
|
||||||
|
const { columns, effective } = articleTimeColumns(archiveDb);
|
||||||
|
const content = columns.has('content') ? "content IS NOT NULL AND content != ''" : '1=1';
|
||||||
|
const indexFilter = columns.has('is_index_page') ? 'AND (is_index_page = 0 OR is_index_page IS NULL)' : '';
|
||||||
|
const cursorFilter = run.cursor_effective_at
|
||||||
|
? `AND (datetime(${effective}) > datetime(?) OR (datetime(${effective}) = datetime(?) AND id > ?))`
|
||||||
|
: '';
|
||||||
|
const params = [run.watermark_at];
|
||||||
|
if (run.cursor_effective_at) params.push(run.cursor_effective_at, run.cursor_effective_at, run.cursor_article_id);
|
||||||
|
const article = archiveDb.prepare(`
|
||||||
|
SELECT id, title, description, content, ${effective} AS effective_at
|
||||||
|
FROM articles
|
||||||
|
WHERE ${content} ${indexFilter} AND datetime(${effective}) <= datetime(?) ${cursorFilter}
|
||||||
|
ORDER BY datetime(${effective}) ASC, id ASC LIMIT 1
|
||||||
|
`).get(...params);
|
||||||
|
if (!article) return null;
|
||||||
|
enqueueJob(db, {
|
||||||
|
jobType: 'replay_article', lane: 'historical', priority: 1, entityType: 'article', entityId: article.id,
|
||||||
|
idempotencyKey: `replay:${run.id}:article:${article.id}`,
|
||||||
|
});
|
||||||
|
return article;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runReplayWorker({ archivePath, intelligencePath, workerId = `replay-${os.hostname()}-${process.pid}`, pollMs = 15000 } = {}) {
|
||||||
|
const archiveDb = new Database(archivePath, { readonly: true });
|
||||||
|
const db = new Database(intelligencePath);
|
||||||
|
db.pragma('journal_mode = WAL');
|
||||||
|
db.pragma('busy_timeout = 5000');
|
||||||
|
initAutonomySchema(db);
|
||||||
|
const config = loadConfig();
|
||||||
|
const dailyBudget = Math.max(1, Number(process.env.AUTONOMY_REPLAY_DAILY_BUDGET) || 100);
|
||||||
|
while (true) {
|
||||||
|
try {
|
||||||
|
const completedToday = db.prepare("SELECT COUNT(*) AS count FROM autonomy_jobs WHERE job_type='replay_article' AND status='complete' AND date(completed_at) = date('now')").get().count;
|
||||||
|
if (completedToday >= dailyBudget) { await sleep(Math.max(pollMs, 60000)); continue; }
|
||||||
|
const run = activeRun(db, config);
|
||||||
|
scheduleNext(db, archiveDb, run);
|
||||||
|
const job = leaseNextJob(db, workerId, 300, ['replay_article']);
|
||||||
|
if (!job) { await sleep(pollMs); continue; }
|
||||||
|
try {
|
||||||
|
const { effective } = articleTimeColumns(archiveDb);
|
||||||
|
const article = archiveDb.prepare(`SELECT id, title, description, content, ${effective} AS effective_at FROM articles WHERE id=?`).get(job.entity_id);
|
||||||
|
if (!article || !article.effective_at) throw new Error(`replay article ${job.entity_id} is unavailable`);
|
||||||
|
const raw = await callCoordinator(config, replayPrompt(article));
|
||||||
|
try {
|
||||||
|
acceptProposal(db, archiveDb, raw, {
|
||||||
|
informationCutoff: article.effective_at, model: config.openRouter.llmModel || 'unknown',
|
||||||
|
promptVersion: 'replay-coordinator-1', strategyVersion: 'autonomy-1', learningEligible: false,
|
||||||
|
origin: 'replay', replayRunId: run.id,
|
||||||
|
});
|
||||||
|
} catch (validationError) {
|
||||||
|
recordRejectedProposal(db, raw, { informationCutoff: article.effective_at, model: config.openRouter.llmModel || 'unknown', promptVersion: 'replay-coordinator-1' }, validationError.message);
|
||||||
|
}
|
||||||
|
db.prepare(`UPDATE autonomy_replay_runs SET cursor_article_id=?, cursor_effective_at=?, processed_articles=processed_articles+1, updated_at=datetime('now') WHERE id=?`)
|
||||||
|
.run(article.id, article.effective_at, run.id);
|
||||||
|
completeJob(db, job.id, workerId);
|
||||||
|
} catch (error) { failJob(db, job.id, workerId, error); }
|
||||||
|
} catch (error) { console.error(`[${workerId}] replay error:`, error.message); }
|
||||||
|
await sleep(pollMs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { articleTimeColumns, replayPrompt, scheduleNext, runReplayWorker };
|
||||||
Reference in New Issue
Block a user