diff --git a/public/admin/app.html b/public/admin/app.html
new file mode 100644
index 0000000..532c726
--- /dev/null
+++ b/public/admin/app.html
@@ -0,0 +1,27 @@
+
+
+
+Duriin Ops
+
+
+
DURIIN
+
loading console…
+
+
<${Skeleton} rows=${4} /> `;
+
+ const controls = (data && data.controls) || {};
+ const dead = (data && data.deadLetters) || [];
+ const deadTotal = dead.reduce((sum, row) => sum + Number(row.n || 0), 0);
+ const killed = Boolean(controls.killSwitch);
+
+ return html`
+
+ ${error && html`
<${Dot} tone="warn" />${error}
`}
+
+ <${Card} title="Execution"
+ right=${html`<${Pill} tone=${killed ? 'bad' : (controls.mode === 'paper' ? 'warn' : 'ok')}>
+ ${killed ? 'halted' : (controls.mode || 'unknown')}
+ `}>
+
+
+
+
+ <${Dot} tone=${killed ? 'bad' : 'ok'} />
+ Kill switch ${killed ? 'engaged' : 'released'}
+
+
+ While engaged no order intents are created. Broker reconciliation keeps
+ running so the account stays visible. Takes effect within one poll,
+ no redeploy.
+
+
+ <${Confirm}
+ label=${killed ? 'release' : 'engage kill switch'}
+ confirmLabel=${killed ? 'yes, release' : 'yes, halt trading'}
+ danger=${!killed}
+ busy=${busy === 'kill'}
+ onConfirm=${() => act('kill',
+ () => api('/admin/api/ops/settings', { method: 'POST', body: { killSwitch: !killed } }),
+ (r) => `Kill switch ${r.controls.killSwitch ? 'engaged' : 'released'}`)} />
+
+
+
+
+
Mode
+
+ shadow records intents without touching the broker.
+ paper submits them to Alpaca paper.
+ Currently from ${controls.modeSource === 'settings' ? 'this dashboard' : 'the environment'}.
+
+
+ <${Confirm}
+ label=${controls.mode === 'paper' ? 'switch to shadow' : 'switch to paper'}
+ confirmLabel=${controls.mode === 'paper' ? 'yes, shadow' : 'yes, submit to broker'}
+ danger=${controls.mode !== 'paper'}
+ busy=${busy === 'mode'}
+ onConfirm=${() => act('mode',
+ () => api('/admin/api/ops/settings', {
+ method: 'POST',
+ body: { mode: controls.mode === 'paper' ? 'shadow' : 'paper' },
+ }),
+ (r) => `Execution mode is now ${r.controls.mode}`)} />
+
+
+
+
+ <${Card} title="Dead letters"
+ right=${deadTotal > 0 && html`<${Pill} tone="bad">${num(deadTotal)} stuck`}>
+ ${!deadTotal
+ ? html`<${Empty}>Nothing dead-lettered. Good.`
+ : html`
+
+
+ Requeue moves rows from dead_letter back to
+ pending . It never deletes and never edits a payload,
+ so the worst case is repeated work.
+
+
+
`}
+
+
+ <${Card} title="Analyses">
+
+
+
+
Reaction conditioning
+
+ Tests whether the initial market reaction predicts anything, on every
+ matured outcome. Read only, takes a few minutes, caches price history.
+
+
+
{
+ setAnalysis(null);
+ const result = await act('analysis',
+ () => api('/admin/api/ops/analysis/reaction', { method: 'POST' }),
+ (r) => (r.ok ? 'Analysis finished' : 'Analysis exited with an error'));
+ if (result) setAnalysis(result);
+ }}>
+ ${busy === 'analysis' ? 'running…' : 'run'}
+
+
+ ${analysis && html`
+
+ ${analysis.error && html`
${analysis.error}
`}
+
${analysis.output || '(no output)'}
+
`}
+
+
+
+ <${Card} title=${title}
+ right=${html`
${total === undefined ? '' : `${num(total)} total`} `}>
+ ${searchable && html`
+
`}
+
+ ${error && html`
${error}
`}
+
+ ${loading && !data
+ ? html`<${Skeleton} rows=${6} />`
+ : !rows.length
+ ? html`<${Empty}>Nothing here.`
+ : html`
+
`}
+
+
+
+ ${rows.length ? `${num(offset + 1)}–${num(offset + rows.length)}` : '—'}
+
+
+ setOffset(Math.max(0, offset - PAGE))}>prev
+ setOffset(offset + PAGE)}>next
+
+
+
+
${r.title || '(untitled)'}
+ ${shortUrl(r.url)}
` },
+ { key: 'source', label: 'source', render: (r) => html`${r.source} ` },
+ { key: 'status', label: 'content', render: (r) => html`
+ <${Pill} tone=${statusTone(r.content_status)}>${r.content_status || 'unfetched'}` },
+ { key: 'pub', label: 'published', render: (r) => html`
+ ${String(r.pub_date_effective || r.pub_date || '').slice(0, 16)} ` },
+ ]} />`;
+
+export const Events = () => html`<${ListView}
+ title="Events" path="/admin/api/events"
+ columns=${[
+ { key: 'title', label: 'title', render: (r) => r.title || '(untitled)' },
+ { key: 'articles', label: 'articles', num: true, render: (r) => num(r.article_count) },
+ { key: 'created', label: 'created', render: (r) => html`
+ ${String(r.created_at || '').slice(0, 16)} ` },
+ ]} />`;
+
+export function Intelligence() {
+ const summary = usePoll('/admin/api/stats/summary', 15000);
+ const intel = usePoll('/admin/api/intelligence/stats', 15000);
+
+ const s = summary.data || {};
+ const i = intel.data || {};
+
+ return html`
+
+
+ <${Stat} label="Articles" value=${compact(s.total)}
+ sub=${`${compact(s.withContent)} with content · ${compact(s.withEmbedding)} embedded`} />
+ <${Stat} label="Events" value=${compact(s.eventCount)} sub="clustered" />
+ <${Stat} label="Knowledge rows" value=${compact(i.knowledge)}
+ sub=${`${compact(i.companies)} companies tracked`} />
+
+
+
+ <${Card} title="Worker rates" right=${html`
rows written `}>
+ ${!(i.workerRates || []).length
+ ? html`<${Empty}>No worker activity recorded.`
+ : html`
+
+ worker last 1m last 5m
+ ${i.workerRates.map((w) => html`
+
+ ${w.worker}
+ ${num(w.n1m)}
+ ${num(w.n5m)}
+ `)}
+
+
`}
+
+
+ <${Card} title="Article queue">
+ ${!(i.queue || []).length
+ ? html`<${Empty}>Queue is empty.`
+ : html`
+
+ status n
+ ${i.queue.map((q) => html`
+
+ <${Pill} tone=${statusTone(q.status)}>${q.status}
+ ${num(q.n)}
+ `)}
+
+
`}
+
+
+
+ DURIIN
+ ${ROUTES.map((r) => html`
+
+ ${r.label}
+ ${r.id === 'controls' && health.deadTotal > 0
+ && html`${num(health.deadTotal)} `}
+ ${r.id === 'overview' && health.halted && html`<${Dot} tone="bad" />`}
+ `)}
+
+
+
+ <${Dot} tone=${health.tone} live />
+ ${health.label}
+
+
ingest ${ago(health.ingestMinutes)}
+
`;
+}
+
+function App() {
+ const route = useHashRoute('overview');
+ const [toast, setToast] = useState(null);
+
+ // A single cheap poll drives the sidebar so every view does not need its own.
+ const { data } = usePoll('/admin/api/ops/overview', 10000);
+ const controls = (data && data.controls) || {};
+ const deadTotal = ((data && data.deadLetters) || [])
+ .reduce((sum, row) => sum + Number(row.n || 0), 0);
+ const ingestMinutes = data && data.freshness ? data.freshness.ingestMinutes : null;
+
+ const halted = Boolean(controls.killSwitch);
+ const health = {
+ deadTotal,
+ halted,
+ ingestMinutes,
+ tone: halted ? 'bad' : (deadTotal > 0 ? 'warn' : 'ok'),
+ label: halted ? 'trading halted' : (controls.mode ? `${controls.mode} mode` : 'connecting…'),
+ };
+
+ const active = ROUTES.find((r) => r.id === route) || ROUTES[0];
+ const View = active.view;
+
+ return html`
+
+ <${Nav} route=${active.id} health=${health} />
+
+
+ ${active.label}
+ kill switch engaged `}
+
+ <${View} notify=${setToast} />
+
+ <${Toast} toast=${toast} />
+
`;
+}
+
+ReactDOM.createRoot(document.getElementById('root')).render(html`<${App} />`);
diff --git a/public/admin/assets/js/ops/overview.js b/public/admin/assets/js/ops/overview.js
new file mode 100644
index 0000000..b86a710
--- /dev/null
+++ b/public/admin/assets/js/ops/overview.js
@@ -0,0 +1,216 @@
+import { html, usePoll, num, compact, pct, ago, whenDate, health } from './core.js';
+import { Card, Stat, Dot, Pill, Bar, Skeleton, Empty } from './ui.js';
+
+const GATE = { minSample: 30, minInstruments: 5, maxConcentration: 0.5 };
+
+const ORIGIN_NOTE = {
+ live: 'genuine real-time work, the only thing that can authorise an order',
+ historical: 'coordinator backfill, training evidence only',
+ replay: 'walk-forward replay, training evidence only',
+};
+
+function PipelineRow({ label, tone, detail, note }) {
+ return html`
+
+
+ <${Dot} tone=${tone} live />
+ ${label}
+
+
+ ${note && html`${note} `}
+ ${detail}
+
+
`;
+}
+
+function LiveEvidence({ maturity, liveOpen, liveResolved }) {
+ const total = (maturity || []).reduce((sum, row) => sum + Number(row.n || 0), 0);
+ const next = (maturity || [])
+ .map((row) => row.first_matures).filter(Boolean).sort()[0];
+
+ if (!total && !liveResolved) {
+ return html`<${Empty}>No live predictions yet.`;
+ }
+
+ return html`
+
+
+ ${num(liveResolved)}
+ matured of ${num(liveOpen + liveResolved)} live
+
+ <${Bar} value=${liveResolved} max=${liveOpen + liveResolved}
+ tone=${liveResolved > 0 ? 'ok' : 'warn'} />
+
+ ${liveResolved > 0
+ ? 'Live evidence is accumulating.'
+ : html`Nothing has matured yet. First outcome ${html`${whenDate(next)} `}.`}
+
+
+
+
+ ${qualifying.length}
+ live cohorts clear the gate
+
+ ${!live.length
+ ? html`<${Empty}>No live calibration exists yet, so every decision abstains. Offline cohorts cannot authorise orders.`
+ : html`
+
+ ${rows.map((row) => {
+ const passed = row.invert ? row.value <= row.need : row.value >= row.need;
+ return html`
+
+
${row.label}
+
+ ${row.fmt(row.value)} ${passed ? '✓' : `/ ${row.fmt(row.need)}`}
+
+
+ <${Bar} value=${row.invert ? Math.max(0, 1 - row.value) : row.value}
+ max=${row.invert ? 1 : row.need} tone=${passed ? 'ok' : 'warn'} />
+
+
`;
+ })}
+
+
Best-populated live cohort shown.
`}
+
+ ${[0, 1, 2].map((i) => html`<${Skeleton} rows=${2} /> `)}
+
`;
+ }
+ if (error && !data) return html`<${Card} title="Overview">${error}
`;
+
+ const p = data.predictions || {};
+ const f = data.freshness || {};
+ const thresholds = f.thresholds || {};
+ const deadTotal = (data.deadLetters || []).reduce((s, r) => s + Number(r.n || 0), 0);
+ const abstain = (data.decisions || []).find((d) => d.action === 'ABSTAIN');
+ const acting = (data.decisions || []).filter((d) => d.action === 'BUY' || d.action === 'SELL')
+ .reduce((s, d) => s + Number(d.n || 0), 0);
+
+ return html`
+
+ ${error && html`
+ <${Dot} tone="warn" />Live updates interrupted: ${error}. Showing the last good snapshot.
+
`}
+
+ ${deadTotal > 0 && html`
+
+ <${Dot} tone="bad" />
+
${num(deadTotal)} dead-lettered jobs are not being retried.
+
+
Go to controls →
+
`}
+
+
+ <${Stat} label="Live predictions open" value=${num(p.live_open)}
+ sub=${`${num(p.live_resolved)} matured · ${num(p.total)} total`} />
+ <${Stat} label="Order intents" value=${num(data.orderIntents)}
+ tone=${Number(data.orderIntents) > 0 ? 'warn' : null}
+ sub=${acting ? `${num(acting)} actionable decisions` : 'every decision has abstained'} />
+ <${Stat} label="Archive" value=${compact(data.archive && data.archive.max_id)}
+ sub=${`last ingest ${ago(f.ingestMinutes)}`} />
+
+
+
+ <${Card} title="Pipeline">
+
+ <${PipelineRow} label="Ingest" note="articles"
+ tone=${health(f.ingestMinutes, thresholds.ingest)}
+ detail=${ago(f.ingestMinutes)} />
+ <${PipelineRow} label="Coordinator" note="predictions"
+ tone=${health(f.predictionMinutes, thresholds.prediction)}
+ detail=${ago(f.predictionMinutes)} />
+ <${PipelineRow} label="Outcomes" note="scored"
+ tone=${health(f.outcomeMinutes, thresholds.outcome)}
+ detail=${ago(f.outcomeMinutes)} />
+ <${PipelineRow} label="Execution"
+ note=${data.controls ? (data.controls.killSwitch ? 'kill switch on' : data.controls.mode) : ''}
+ tone=${data.controls && data.controls.killSwitch ? 'bad' : 'ok'}
+ detail=${`${num(data.orderIntents)} intents`} />
+
+
+
+ <${Card} title="Can it trade yet?">
+ <${GateCard} cohorts=${data.cohorts} />
+
+
+ <${Card} title="Live evidence">
+ <${LiveEvidence} maturity=${data.maturity}
+ liveOpen=${Number(p.live_open || 0)} liveResolved=${Number(p.live_resolved || 0)} />
+
+
+ <${Card} title="Accuracy by origin"
+ right=${html`
only live counts as a track record `}>
+ ${!(data.byOrigin || []).length
+ ? html`<${Empty}>No scored outcomes yet.`
+ : html`
+
`}
+
+
+
+
+ ${abstain ? `${num(abstain.n)} decisions, all ABSTAIN · ` : ''}
+ updated ${at ? new Date(at).toLocaleTimeString() : '—'}
+
+
+ <${Card} title="Query"
+ right=${html`
+
+ setDatabase(e.target.value)}>
+ intelligence
+ archive
+
+
+ ${busy ? 'running…' : 'run'}
+
+
`}>
+
+
+ samples:
+ ${SAMPLES.map((s) => html`
+ { setSql(s.sql); setDatabase(s.db); }}>${s.label} `)}
+ ⌘/ctrl + enter
+
+
+
+ ${result && html`
+ <${Card} title="Result"
+ right=${html`
+ <${Pill}>${num(rows.length)} rows
+ <${Pill}>${num(result.elapsed)}ms server
+ <${Pill}>${num(result.clientMs)}ms total
+ `}>
+ ${first && first.error
+ ? html`
${first.error}
`
+ : !rows.length
+ ? html`<${Empty}>No rows.`
+ : html`
+
+ ${rows.length > 500 && html`
+
+ Showing the first 500 of ${num(rows.length)} rows.
+
`}`}
+ `}
+
+ ${title && html`
+
+
${title}
+ ${right}
+ `}
+ ${children}
+ `;
+
+export const Stat = ({ label, value, sub, tone }) => html`
+
+ ${label}
+ ${value}
+ ${sub && html`${sub}
`}
+ `;
+
+export const Dot = ({ tone = 'idle', live }) => html`
+ ${children} `;
+
+export const Bar = ({ value, max, tone }) => {
+ const width = max > 0 ? Math.min(100, (Number(value) / Number(max)) * 100) : 0;
+ return html`
`;
+};
+
+export const Skeleton = ({ rows = 3 }) => html`
+
+ ${Array.from({ length: rows }, (_, i) => html`
+
`)}
+
${children}
`;
+
+export function Toast({ toast }) {
+ const [shown, setShown] = useState(toast);
+ useEffect(() => {
+ setShown(toast);
+ if (!toast) return undefined;
+ const timer = setTimeout(() => setShown(null), toast.sticky ? 12000 : 4500);
+ return () => clearTimeout(timer);
+ }, [toast]);
+ if (!shown) return null;
+ return html`${shown.message}
`;
+}
+
+// Anything that changes production asks twice. The second click is a different
+// button label so muscle memory cannot carry you through both.
+export function Confirm({ label, confirmLabel, onConfirm, danger, disabled, busy }) {
+ const [armed, setArmed] = useState(false);
+ useEffect(() => {
+ if (!armed) return undefined;
+ const timer = setTimeout(() => setArmed(false), 5000);
+ return () => clearTimeout(timer);
+ }, [armed]);
+
+ if (busy) return html`working… `;
+ if (!armed) {
+ return html` setArmed(true)}>${label} `;
+ }
+ return html`
+
+ { setArmed(false); onConfirm(); }}>${confirmLabel || 'confirm'}
+ setArmed(false)}>cancel
+ `;
+}
diff --git a/src/autonomy/schema.js b/src/autonomy/schema.js
index 69704c1..cf912c5 100644
--- a/src/autonomy/schema.js
+++ b/src/autonomy/schema.js
@@ -221,6 +221,16 @@ function initAutonomySchema(db) {
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);
+ -- Runtime knobs the operator can change without a redeploy. Execution mode used
+ -- to live only in AUTONOMY_EXECUTION_MODE, which means flipping it needed a
+ -- container restart, which is the last thing you want during an incident.
+ CREATE TABLE IF NOT EXISTS autonomy_settings (
+ key TEXT PRIMARY KEY,
+ value TEXT NOT NULL,
+ updated_at TEXT NOT NULL DEFAULT (datetime('now')),
+ updated_by TEXT
+ );
+
INSERT OR IGNORE INTO autonomy_schema(version) VALUES (${AUTONOMY_SCHEMA_VERSION});
`);
for (const statement of [
diff --git a/src/autonomy/settings.js b/src/autonomy/settings.js
new file mode 100644
index 0000000..2ca37be
--- /dev/null
+++ b/src/autonomy/settings.js
@@ -0,0 +1,67 @@
+// Runtime settings for the autonomy stack. The env vars stay the default, the
+// table is the override, so nothing changes behaviour until somebody deliberately
+// writes a row. Kept deliberately tiny -- this is a control plane, not a config
+// system, and every key here can move real money or stop the pipeline.
+
+const EXECUTION_MODES = ['shadow', 'paper'];
+
+const KEYS = {
+ executionMode: 'execution_mode',
+ killSwitch: 'execution_kill_switch',
+};
+
+function readSetting(db, key) {
+ try {
+ const row = db.prepare('SELECT value FROM autonomy_settings WHERE key = ?').get(key);
+ return row ? row.value : null;
+ } catch (error) {
+ // A missing table means an older schema, which should behave like "no override"
+ // rather than taking the worker down with it.
+ console.error(`[settings] could not read ${key}:`, error.message);
+ return null;
+ }
+}
+
+function writeSetting(db, key, value, updatedBy = 'admin') {
+ db.prepare(`
+ INSERT INTO autonomy_settings(key, value, updated_by) VALUES (?, ?, ?)
+ ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_by = excluded.updated_by,
+ updated_at = datetime('now')
+ `).run(key, String(value), updatedBy);
+}
+
+// Truthy strings people actually type, rather than only accepting 'true'
+function isOn(value) {
+ return ['1', 'true', 'on', 'yes', 'engaged'].includes(String(value || '').trim().toLowerCase());
+}
+
+function getExecutionControls(db, env = process.env) {
+ const stored = readSetting(db, KEYS.executionMode);
+ const fallback = env.AUTONOMY_EXECUTION_MODE || 'shadow';
+ const mode = EXECUTION_MODES.includes(String(stored)) ? String(stored) : fallback;
+
+ return {
+ mode: EXECUTION_MODES.includes(mode) ? mode : 'shadow',
+ modeSource: EXECUTION_MODES.includes(String(stored)) ? 'settings' : 'env',
+ killSwitch: isOn(readSetting(db, KEYS.killSwitch)),
+ };
+}
+
+function setExecutionMode(db, mode, updatedBy) {
+ if (!EXECUTION_MODES.includes(mode)) {
+ throw new Error(`unsupported execution mode: ${mode} (expected ${EXECUTION_MODES.join(' or ')})`);
+ }
+ writeSetting(db, KEYS.executionMode, mode, updatedBy);
+ return getExecutionControls(db);
+}
+
+function setKillSwitch(db, engaged, updatedBy) {
+ writeSetting(db, KEYS.killSwitch, engaged ? 'true' : 'false', updatedBy);
+ return getExecutionControls(db);
+}
+
+module.exports = {
+ EXECUTION_MODES, KEYS, isOn,
+ readSetting, writeSetting,
+ getExecutionControls, setExecutionMode, setKillSwitch,
+};
diff --git a/src/routes/admin.js b/src/routes/admin.js
index db385bd..e0c253b 100644
--- a/src/routes/admin.js
+++ b/src/routes/admin.js
@@ -7,6 +7,7 @@ const config = require('../config');
const Database = require('better-sqlite3');
const { openRuntimeDb, isPostgresEnabled } = require('../db/runtime');
const pg = require('../db/pgAsync');
+const opsRoutes = require('./ops');
let idb = null;
let adb = null;
@@ -265,6 +266,7 @@ const pagesDir = path.join(publicDir, 'pages');
// map pretty url → page html file. keep these close to the routes so its
// obvious when a page gets added or renamed.
const pageMap = {
+ '/admin/console': path.join(publicDir, 'app.html'),
'/admin/autonomy': path.join(pagesDir, 'autonomy.html'),
'/admin/ingest/articles': path.join(pagesDir, 'ingest', 'articles.html'),
'/admin/ingest/events': path.join(pagesDir, 'ingest', 'events.html'),
@@ -283,6 +285,10 @@ function sendPage(reply, filePath) {
}
async function adminRoutes(fastify) {
+ // Control plane for the ops dashboard. Lives in its own file, shares this one's
+ // auth and db handles so there is exactly one of each.
+ fastify.register(opsRoutes, { checkAuth, getIntelligenceDb, getArchiveDb });
+
// gate every request under /admin/* behind basic auth (covers pages, api, and assets)
fastify.addHook('onRequest', async (request, reply) => {
@@ -303,9 +309,11 @@ async function adminRoutes(fastify) {
},
});
- // top-level entry — the autonomy control room is the primary product surface
+ // top-level entry — the ops console is the primary surface now. The older
+ // per-page admin is still served underneath, the console frames the d3 graph
+ // from it rather than duplicating that visualisation.
fastify.get('/admin', async (request, reply) => {
- reply.redirect('/admin/autonomy');
+ reply.redirect('/admin/console');
});
// ingest root — redirect to the articles subsection
diff --git a/src/routes/ops.js b/src/routes/ops.js
new file mode 100644
index 0000000..8a60d39
--- /dev/null
+++ b/src/routes/ops.js
@@ -0,0 +1,254 @@
+// Control plane for the operations dashboard.
+//
+// Kept apart from admin.js on purpose: everything in here either changes what the
+// autonomy stack does or is read by an operator while something is on fire, so it
+// wants to stay small enough to audit in one sitting.
+const { execFile } = require('child_process');
+const path = require('path');
+const { getExecutionControls, setExecutionMode, setKillSwitch, EXECUTION_MODES } = require('../autonomy/settings');
+
+// Which of the pipeline stages we consider "recent enough to be alive". These are
+// generous, they exist to catch a stall not to police a few seconds of jitter.
+const STALE_AFTER_MINUTES = { ingest: 90, prediction: 180, outcome: 24 * 60 };
+
+const ANALYSES = {
+ reaction: {
+ label: 'Reaction conditioning',
+ script: 'scripts/analyze-reaction-conditioning.js',
+ detail: 'Does the initial market reaction predict anything. Read only, a few minutes.',
+ },
+};
+
+function minutesSince(value) {
+ if (!value) return null;
+ const stamp = String(value).includes('T') ? String(value) : `${String(value).replace(' ', 'T')}Z`;
+ const then = Date.parse(stamp);
+ if (!Number.isFinite(then)) return null;
+ return Math.max(0, (Date.now() - then) / 60000);
+}
+
+function one(db, sql, params = []) {
+ try {
+ return db.prepare(sql).get(...params) || {};
+ } catch (error) {
+ console.error('[ops] query failed:', sql.trim().slice(0, 80), error.message);
+ return {};
+ }
+}
+
+function many(db, sql, params = []) {
+ try {
+ return db.prepare(sql).all(...params) || [];
+ } catch (error) {
+ console.error('[ops] query failed:', sql.trim().slice(0, 80), error.message);
+ return [];
+ }
+}
+
+// One request for the whole dashboard. The old admin made the browser fire a
+// handful of sequential calls and stitch them together, which is most of why it
+// felt sluggish even though every individual endpoint was fast.
+function buildOverview(intel) {
+ const predictions = one(intel, `
+ SELECT COUNT(*) AS total,
+ SUM(CASE WHEN origin='live' AND status='open' THEN 1 ELSE 0 END) AS live_open,
+ SUM(CASE WHEN origin='live' AND status='resolved' THEN 1 ELSE 0 END) AS live_resolved,
+ SUM(CASE WHEN origin IN ('historical','replay') AND status='resolved' THEN 1 ELSE 0 END) AS offline_resolved,
+ SUM(CASE WHEN status='unresolvable' THEN 1 ELSE 0 END) AS unresolvable,
+ MAX(created_at) AS latest
+ FROM autonomy_predictions
+ `);
+
+ const byOrigin = many(intel, `
+ SELECT p.origin, COUNT(*) AS total,
+ SUM(o.direction_correct) AS correct,
+ AVG(o.excess_return) AS mean_excess
+ FROM autonomy_predictions p JOIN autonomy_outcomes o ON o.prediction_id = p.id
+ GROUP BY p.origin
+ `);
+
+ // When live evidence actually arrives. This is the number that decides whether
+ // anything can ever qualify, and nothing in the old UI showed it.
+ const maturity = many(intel, `
+ SELECT horizon_days, COUNT(*) AS n,
+ MIN(date(information_cutoff, '+' || horizon_days || ' days')) AS first_matures
+ FROM autonomy_predictions
+ WHERE origin='live' AND status='open'
+ GROUP BY horizon_days ORDER BY horizon_days
+ `);
+
+ const decisions = many(intel, "SELECT action, COUNT(*) AS n FROM autonomy_decisions GROUP BY action");
+ const intents = one(intel, 'SELECT COUNT(*) AS n FROM autonomy_order_intents');
+
+ const jobs = many(intel, `
+ SELECT job_type, lane, status, COUNT(*) AS n, MAX(created_at) AS newest
+ FROM autonomy_jobs GROUP BY job_type, lane, status
+ `);
+ const deadLetters = many(intel, `
+ SELECT job_type, lane, COUNT(*) AS n, substr(MAX(last_error), 1, 160) AS sample_error
+ FROM autonomy_jobs WHERE status='dead_letter' GROUP BY job_type, lane
+ `);
+
+ let cohorts = [];
+ try {
+ cohorts = many(intel, `
+ SELECT cohort_key, source, sample_size, distinct_instruments, top_instrument_share,
+ directional_probability, expected_excess_return
+ FROM autonomy_calibration_snapshots
+ WHERE cohort_key LIKE 'v2|%'
+ GROUP BY cohort_key, source
+ HAVING MAX(created_at) = created_at
+ ORDER BY sample_size DESC LIMIT 40
+ `);
+ } catch (error) {
+ console.error('[ops] cohort snapshot read failed:', error.message);
+ }
+
+ const outcomes = one(intel, 'SELECT COUNT(*) AS n, MAX(evaluated_at) AS latest FROM autonomy_outcomes');
+
+ return {
+ generatedAt: new Date().toISOString(),
+ predictions,
+ byOrigin,
+ maturity,
+ decisions,
+ orderIntents: intents.n || 0,
+ outcomes,
+ jobs,
+ deadLetters,
+ cohorts,
+ freshness: {
+ predictionMinutes: minutesSince(predictions.latest),
+ outcomeMinutes: minutesSince(outcomes.latest),
+ thresholds: STALE_AFTER_MINUTES,
+ },
+ };
+}
+
+function opsRoutes(fastify, options) {
+ const { checkAuth, getIntelligenceDb, getArchiveDb } = options;
+
+ const withIntel = (reply) => {
+ const intel = getIntelligenceDb();
+ if (!intel) {
+ reply.code(503).send({ error: 'intelligence database unavailable' });
+ return null;
+ }
+ return intel;
+ };
+
+ fastify.get('/admin/api/ops/overview', async (request, reply) => {
+ if (!checkAuth(request, reply)) return;
+ const intel = withIntel(reply);
+ if (!intel) return;
+
+ const payload = buildOverview(intel);
+ try {
+ payload.controls = getExecutionControls(intel);
+ } catch (error) {
+ console.error('[ops] could not read execution controls:', error.message);
+ payload.controls = null;
+ }
+
+ // Archive counts are the one genuinely expensive thing here, so they are
+ // cheap approximations rather than COUNT(*) over 2.2M rows on every poll.
+ try {
+ const archive = getArchiveDb ? getArchiveDb() : null;
+ if (archive) {
+ payload.archive = one(archive, `
+ SELECT MAX(id) AS max_id, MAX(ingested_at) AS latest_ingest FROM articles
+ `);
+ payload.freshness.ingestMinutes = minutesSince(payload.archive.latest_ingest);
+ }
+ } catch (error) {
+ console.error('[ops] archive probe failed:', error.message);
+ payload.archive = null;
+ }
+ return payload;
+ });
+
+ fastify.get('/admin/api/ops/settings', async (request, reply) => {
+ if (!checkAuth(request, reply)) return;
+ const intel = withIntel(reply);
+ if (!intel) return;
+ return { controls: getExecutionControls(intel), modes: EXECUTION_MODES, analyses: ANALYSES };
+ });
+
+ fastify.post('/admin/api/ops/settings', async (request, reply) => {
+ if (!checkAuth(request, reply)) return;
+ const intel = withIntel(reply);
+ if (!intel) return;
+ const { mode, killSwitch } = request.body || {};
+ try {
+ if (mode !== undefined) setExecutionMode(intel, String(mode), 'admin-ui');
+ if (killSwitch !== undefined) setKillSwitch(intel, Boolean(killSwitch), 'admin-ui');
+ } catch (error) {
+ console.error('[ops] settings write rejected:', error.message);
+ reply.code(400).send({ error: error.message });
+ return;
+ }
+ const controls = getExecutionControls(intel);
+ console.log(`[ops] execution controls now mode=${controls.mode} kill=${controls.killSwitch}`);
+ return { controls };
+ });
+
+ // Requeue is deliberately narrow: it only ever moves dead_letter back to pending,
+ // it never deletes and never edits payloads, so the worst case is repeated work.
+ fastify.post('/admin/api/ops/dead-letters/requeue', async (request, reply) => {
+ if (!checkAuth(request, reply)) return;
+ const intel = withIntel(reply);
+ if (!intel) return;
+ const { jobType, lane } = request.body || {};
+
+ const filters = ["status = 'dead_letter'"];
+ const params = [];
+ if (jobType) { filters.push('job_type = ?'); params.push(String(jobType)); }
+ if (lane) { filters.push('lane = ?'); params.push(String(lane)); }
+ const where = filters.join(' AND ');
+
+ try {
+ const before = one(intel, `SELECT COUNT(*) AS n FROM autonomy_jobs WHERE ${where}`, params).n || 0;
+ if (!before) return { requeued: 0, remaining: 0 };
+ const result = intel.prepare(`
+ UPDATE autonomy_jobs
+ SET status='pending', attempts=0, available_at=datetime('now'),
+ leased_by=NULL, lease_expires_at=NULL,
+ last_error='requeued from the ops dashboard'
+ WHERE ${where}
+ `).run(...params);
+ const remaining = one(intel, `SELECT COUNT(*) AS n FROM autonomy_jobs WHERE ${where}`, params).n || 0;
+ console.log(`[ops] requeued ${result.changes} dead letters (jobType=${jobType || 'any'} lane=${lane || 'any'})`);
+ return { requeued: result.changes, before, remaining };
+ } catch (error) {
+ console.error('[ops] requeue failed:', error.message, error.stack);
+ reply.code(500).send({ error: error.message });
+ }
+ });
+
+ fastify.post('/admin/api/ops/analysis/:name', async (request, reply) => {
+ if (!checkAuth(request, reply)) return;
+ const spec = ANALYSES[request.params.name];
+ if (!spec) {
+ reply.code(404).send({ error: `unknown analysis: ${request.params.name}` });
+ return;
+ }
+ const script = path.resolve(__dirname, '..', '..', spec.script);
+ return new Promise((resolve) => {
+ execFile('node', [script], { timeout: 15 * 60 * 1000, maxBuffer: 8 * 1024 * 1024 },
+ (error, stdout, stderr) => {
+ if (error) console.error(`[ops] analysis ${request.params.name} failed:`, error.message);
+ resolve({
+ analysis: request.params.name,
+ label: spec.label,
+ ok: !error,
+ output: String(stdout || '').slice(-20000),
+ error: error ? String(stderr || error.message).slice(-4000) : null,
+ });
+ });
+ });
+ });
+}
+
+module.exports = opsRoutes;
+module.exports.buildOverview = buildOverview;
+module.exports.ANALYSES = ANALYSES;
diff --git a/workers/executionWorker.js b/workers/executionWorker.js
index da3740b..8a13e17 100644
--- a/workers/executionWorker.js
+++ b/workers/executionWorker.js
@@ -3,6 +3,7 @@ const { openRuntimeDb } = require('../src/db/runtime');
const { initAutonomySchema } = require('../src/autonomy/schema');
const { createOrderIntent } = require('../src/autonomy/orderIntents');
const { createAlpacaPaperClient } = require('../src/brokers/alpacaPaper');
+const { getExecutionControls } = require('../src/autonomy/settings');
function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); }
@@ -12,10 +13,27 @@ async function runExecutionWorker({ intelligencePath, pollMs = 10000, mode = 'sh
db.pragma('journal_mode = WAL');
db.pragma('busy_timeout = 5000');
initAutonomySchema(db);
- const paperClient = mode === 'paper'
- ? createAlpacaPaperClient({ keyId: process.env.ALPACA_PAPER_KEY_ID, secretKey: process.env.ALPACA_PAPER_SECRET_KEY })
- : null;
+
+ // mode is re-read every poll now. the env var is only the default, so the kill
+ // switch actually works during an incident instead of needing a redeploy first.
+ let paperClient = null;
+ let lastMode = null;
+ let lastKill = null;
+
while (true) {
+ const controls = getExecutionControls(db, { AUTONOMY_EXECUTION_MODE: mode });
+ if (controls.mode !== lastMode) {
+ console.log(`[${workerId}] execution mode = ${controls.mode} (from ${controls.modeSource})`);
+ lastMode = controls.mode;
+ }
+ if (controls.killSwitch !== lastKill) {
+ console.log(`[${workerId}] kill switch ${controls.killSwitch ? 'ENGAGED, no orders will be placed' : 'released'}`);
+ lastKill = controls.killSwitch;
+ }
+ if (controls.mode === 'paper' && !paperClient) {
+ paperClient = createAlpacaPaperClient({ keyId: process.env.ALPACA_PAPER_KEY_ID, secretKey: process.env.ALPACA_PAPER_SECRET_KEY });
+ }
+ if (controls.mode !== 'paper') paperClient = null;
if (paperClient) {
try {
const [account, positions, orders] = await Promise.all([
@@ -49,7 +67,9 @@ async function runExecutionWorker({ intelligencePath, pollMs = 10000, mode = 'sh
console.error(`[${workerId}] broker reconciliation:`, error.message);
}
}
- const decisions = db.prepare(`
+ // Broker reconciliation above still runs while the switch is engaged, we want
+ // to keep seeing the account. It is order creation specifically that stops.
+ const decisions = controls.killSwitch ? [] : db.prepare(`
SELECT d.id FROM autonomy_decisions d
JOIN autonomy_predictions p ON p.id = d.prediction_id
LEFT JOIN autonomy_order_intents oi ON oi.decision_id = d.id
@@ -59,8 +79,8 @@ async function runExecutionWorker({ intelligencePath, pollMs = 10000, mode = 'sh
for (const decision of decisions) {
try {
const intent = createOrderIntent(db, decision.id, notional, { tradable: true, maxNotional: notional });
- if (mode === 'paper') db.prepare("UPDATE autonomy_order_intents SET status='pending', updated_at=datetime('now') WHERE client_order_id=?").run(intent.clientOrderId);
- console.log(`[${workerId}] ${mode} intent ${intent.clientOrderId}`);
+ if (controls.mode === 'paper') db.prepare("UPDATE autonomy_order_intents SET status='pending', updated_at=datetime('now') WHERE client_order_id=?").run(intent.clientOrderId);
+ console.log(`[${workerId}] ${controls.mode} intent ${intent.clientOrderId}`);
} catch (error) {
console.error(`[${workerId}] decision ${decision.id}:`, error.message);
}