From 18d062fd2d66cf9c52cf4512f596a83ecc71d734 Mon Sep 17 00:00:00 2001
From: ImBenji <benjamin.watt@imbenji.net>
Date: Wed, 22 Apr 2026 20:50:08 +0100
Subject: [PATCH] add intelligence and SQL tabs to admin interface with
 corresponding API endpoints

---
 admin.html            | 239 ++++++++++++++++++++++++++++++++++++++++++
 config.json           |   4 +-
 docker-compose.yml    |   2 +
 intelligence/db.js    |   2 +
 intelligence/index.js |   4 +-
 package.json          |   3 +-
 src/db.js             | 236 +++++------------------------------------
 src/routes/admin.js   | 134 +++++++++++++++++++++++
 src/routes/dev.js     |  32 +++---
 9 files changed, 428 insertions(+), 228 deletions(-)

diff --git a/admin.html b/admin.html
index 983664b..f715009 100644
--- a/admin.html
+++ b/admin.html
@@ -247,6 +247,8 @@
     <button class="tab-btn active" data-tab="articles">Articles</button>
     <button class="tab-btn" data-tab="events">Events</button>
     <button class="tab-btn" data-tab="stats">Stats</button>
+    <button class="tab-btn" data-tab="intelligence">Intelligence</button>
+    <button class="tab-btn" data-tab="sql">SQL</button>
   </div>
 </header>
 
@@ -320,6 +322,63 @@
     </div>
   </div>
 
+  <!-- Intelligence tab -->
+  <div id="tab-intelligence" style="display:none">
+
+    <div id="intel-unavailable" style="display:none; color:#94a3b8; padding: 24px 0">intelligence.sqlite not found — is the intelligence worker running?</div>
+
+    <div id="intel-content">
+      <div style="display:flex; gap:24px; flex-wrap:wrap; margin-bottom:24px" id="intel-stats-row"></div>
+
+      <div style="display:flex; gap:16px; margin-bottom:14px; flex-wrap:wrap; align-items:flex-end">
+        <label style="display:flex; flex-direction:column; gap:4px; font-size:12px; color:#94a3b8">Company
+          <select id="i-company"><option value="">All companies</option></select>
+        </label>
+        <label style="display:flex; flex-direction:column; gap:4px; font-size:12px; color:#94a3b8">View
+          <select id="i-view">
+            <option value="knowledge">Knowledge</option>
+            <option value="predictions">Predictions</option>
+          </select>
+        </label>
+        <label style="display:flex; flex-direction:column; gap:4px; font-size:12px; color:#94a3b8">Type
+          <select id="i-type">
+            <option value="">All types</option>
+            <option value="relationship">Relationship</option>
+            <option value="theme">Theme</option>
+            <option value="factor">Factor</option>
+          </select>
+        </label>
+        <button class="primary" onclick="loadIntelligence()">Filter</button>
+      </div>
+
+      <table>
+        <thead id="intel-thead"></thead>
+        <tbody id="intel-tbody"></tbody>
+      </table>
+
+      <div class="pagination">
+        <button id="iPrevBtn">← Prev</button>
+        <span id="iPageInfo"></span>
+        <button id="iNextBtn">Next →</button>
+      </div>
+    </div>
+  </div>
+
+  <!-- SQL tab -->
+  <div id="tab-sql" style="display:none">
+    <div style="display:flex; gap:10px; margin-bottom:10px; align-items:center">
+      <select id="sql-db" style="min-width:160px">
+        <option value="archive">archive.sqlite</option>
+        <option value="intelligence">intelligence.sqlite</option>
+      </select>
+      <button class="primary" id="sql-run-btn">Run</button>
+      <span id="sql-elapsed" style="color:#64748b; font-size:12px"></span>
+    </div>
+    <textarea id="sql-input" style="width:100%; min-height:120px; font-family:monospace; font-size:13px; margin-bottom:12px" placeholder="SELECT ..."></textarea>
+    <div id="sql-error" style="color:#fca5a5; font-size:13px; margin-bottom:10px; display:none"></div>
+    <div id="sql-results" style="overflow-x:auto"></div>
+  </div>
+
   <!-- Stats tab -->
   <div id="tab-stats" style="display:none">
     <div style="display:flex; gap:32px; flex-wrap:wrap; padding-top:8px">
@@ -653,12 +712,191 @@ document.getElementById('eDeleteBtn').onclick = async () => {
   } catch (e) { toast('Delete failed', true); }
 };
 
+// ── intelligence ───────────────────────────────────────────────────────────
+
+let intelOffset = 0;
+
+async function loadIntelligenceStats() {
+  const data = await api('/admin/api/intelligence/stats');
+  if (!data.available) {
+    document.getElementById('intel-unavailable').style.display = '';
+    document.getElementById('intel-content').style.display = 'none';
+    return false;
+  }
+
+  document.getElementById('intel-unavailable').style.display = 'none';
+  document.getElementById('intel-content').style.display = '';
+
+  const queueMap = {};
+  (data.queue || []).forEach(r => queueMap[r.status] = r.n);
+
+  document.getElementById('intel-stats-row').innerHTML = [
+    ['Queue pending', (queueMap.pending || 0).toLocaleString()],
+    ['Processed', (queueMap.processed || 0).toLocaleString()],
+    ['Skipped', (queueMap.skipped || 0).toLocaleString()],
+    ['Knowledge rows', data.knowledge.toLocaleString()],
+    ['Predictions', data.predictions.toLocaleString()],
+    ['Companies', `${data.embeddings}/${data.companies} embedded`],
+  ].map(([label, value]) => `
+    <div class="stat">
+      <span class="label">${label}</span>
+      <span class="value" style="font-size:16px">${value}</span>
+    </div>
+  `).join('');
+
+  return true;
+}
+
+async function loadIntelligenceCompanies() {
+  const companies = await api('/admin/api/intelligence/companies');
+  const sel = document.getElementById('i-company');
+  sel.innerHTML = '<option value="">All companies</option>';
+  companies.forEach(c => {
+    const opt = document.createElement('option');
+    opt.value = c.id;
+    opt.textContent = `${c.name} (${c.ticker})`;
+    sel.appendChild(opt);
+  });
+}
+
+async function loadIntelligence() {
+  const view = document.getElementById('i-view').value;
+  const companyId = document.getElementById('i-company').value;
+  const type = document.getElementById('i-type').value;
+
+  const params = new URLSearchParams({ limit: PAGE, offset: intelOffset });
+  if (companyId) params.set('company_id', companyId);
+
+  if (view === 'knowledge') {
+    document.getElementById('i-type').parentElement.style.display = '';
+    if (type) params.set('type', type);
+
+    const data = await api(`/admin/api/intelligence/knowledge?${params}`);
+
+    document.getElementById('intel-thead').innerHTML = `
+      <tr><th>ID</th><th>Company</th><th>Event</th><th>Type</th><th>Data</th><th>Created</th></tr>`;
+
+    document.getElementById('intel-tbody').innerHTML = data.rows.map(r => {
+      let parsed = {};
+      try { parsed = JSON.parse(r.data); } catch (_) {}
+      const summary = Object.values(parsed).filter(v => typeof v === 'string').join(' · ').slice(0, 120);
+      return `<tr>
+        <td style="color:#64748b">${r.id}</td>
+        <td style="white-space:nowrap">${r.company_name}</td>
+        <td style="color:#64748b">${r.event_id}</td>
+        <td><span class="badge null">${r.type}</span></td>
+        <td><span class="truncate" style="max-width:360px" title="${r.data.replace(/"/g,'&quot;')}">${summary}</span></td>
+        <td style="color:#64748b; white-space:nowrap">${r.created_at ? r.created_at.slice(0,16) : '—'}</td>
+      </tr>`;
+    }).join('');
+
+    const total = data.total;
+    document.getElementById('iPageInfo').textContent =
+      `${intelOffset + 1}–${Math.min(intelOffset + PAGE, total)} of ${total.toLocaleString()}`;
+    document.getElementById('iPrevBtn').disabled = intelOffset === 0;
+    document.getElementById('iNextBtn').disabled = intelOffset + PAGE >= total;
+
+  } else {
+    document.getElementById('i-type').parentElement.style.display = 'none';
+
+    const data = await api(`/admin/api/intelligence/predictions?${params}`);
+
+    document.getElementById('intel-thead').innerHTML = `
+      <tr><th>ID</th><th>Company</th><th>Event</th><th>Type</th><th>Direction</th><th>Magnitude</th><th>Timeframe</th><th>Rationale</th><th>Created</th></tr>`;
+
+    document.getElementById('intel-tbody').innerHTML = data.rows.map(r => `
+      <tr>
+        <td style="color:#64748b">${r.id}</td>
+        <td style="white-space:nowrap">${r.company_name}</td>
+        <td style="color:#64748b">${r.event_id}</td>
+        <td><span class="badge null">${r.type}</span></td>
+        <td>${r.direction || '—'}</td>
+        <td>${r.magnitude || '—'}</td>
+        <td>${r.timeframe || '—'}</td>
+        <td><span class="truncate" style="max-width:300px" title="${(r.rationale||'').replace(/"/g,'&quot;')}">${r.rationale || '—'}</span></td>
+        <td style="color:#64748b; white-space:nowrap">${r.created_at ? r.created_at.slice(0,16) : '—'}</td>
+      </tr>
+    `).join('');
+
+    const total = data.total;
+    document.getElementById('iPageInfo').textContent =
+      `${intelOffset + 1}–${Math.min(intelOffset + PAGE, total)} of ${total.toLocaleString()}`;
+    document.getElementById('iPrevBtn').disabled = intelOffset === 0;
+    document.getElementById('iNextBtn').disabled = intelOffset + PAGE >= total;
+  }
+}
+
+document.getElementById('iPrevBtn').onclick = () => { intelOffset = Math.max(0, intelOffset - PAGE); loadIntelligence(); };
+document.getElementById('iNextBtn').onclick = () => { intelOffset += PAGE; loadIntelligence(); };
+
+document.getElementById('i-view').onchange = () => { intelOffset = 0; loadIntelligence(); };
+
+// ── sql console ────────────────────────────────────────────────────────────
+
+async function runSql() {
+  const sql = document.getElementById('sql-input').value.trim();
+  if (!sql) return;
+
+  const database = document.getElementById('sql-db').value;
+  const errEl = document.getElementById('sql-error');
+  const resultsEl = document.getElementById('sql-results');
+  const elapsedEl = document.getElementById('sql-elapsed');
+
+  errEl.style.display = 'none';
+  resultsEl.innerHTML = '';
+  elapsedEl.textContent = '';
+
+  try {
+    const data = await fetch('/admin/api/sql', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ sql, database }),
+    }).then(r => r.json());
+
+    if (data.error) {
+      errEl.textContent = data.error;
+      errEl.style.display = '';
+      return;
+    }
+
+    elapsedEl.textContent = `${data.elapsed}ms`;
+
+    if (data.rows && data.rows.length > 0) {
+      const cols = Object.keys(data.rows[0]);
+      resultsEl.innerHTML = `
+        <table>
+          <thead><tr>${cols.map(c => `<th>${c}</th>`).join('')}</tr></thead>
+          <tbody>${data.rows.map(r =>
+            `<tr>${cols.map(c => `<td><span class="truncate" style="max-width:300px" title="${String(r[c] ?? '').replace(/"/g,'&quot;')}">${r[c] ?? '<span style="color:#64748b">NULL</span>'}</span></td>`).join('')}</tr>`
+          ).join('')}</tbody>
+        </table>
+        <div style="color:#64748b; font-size:12px; margin-top:8px">${data.rows.length} row${data.rows.length !== 1 ? 's' : ''}</div>
+      `;
+    } else if (data.rows) {
+      resultsEl.innerHTML = '<div style="color:#64748b; font-size:13px">No rows returned.</div>';
+    } else {
+      resultsEl.innerHTML = `<div style="color:#86efac; font-size:13px">${data.changes} row${data.changes !== 1 ? 's' : ''} affected.</div>`;
+    }
+  } catch (e) {
+    errEl.textContent = e.message;
+    errEl.style.display = '';
+  }
+}
+
+document.getElementById('sql-run-btn').onclick = runSql;
+
+document.getElementById('sql-input').addEventListener('keydown', e => {
+  if (e.key === 'Enter' && (e.metaKey || e.ctrlKey)) runSql();
+});
+
 // ── tabs ───────────────────────────────────────────────────────────────────
 
 const tabContents = {
   articles: document.getElementById('tab-articles'),
   events: document.getElementById('tab-events'),
   stats: document.getElementById('tab-stats'),
+  intelligence: document.getElementById('tab-intelligence'),
+  sql: document.getElementById('tab-sql'),
 };
 
 document.querySelectorAll('.tab-btn').forEach(btn => {
@@ -671,6 +909,7 @@ document.querySelectorAll('.tab-btn').forEach(btn => {
     });
     if (tab === 'events') loadEvents();
     if (tab === 'stats') loadStats();
+    if (tab === 'intelligence') { intelOffset = 0; loadIntelligenceStats().then(ok => { if (ok) { loadIntelligenceCompanies(); loadIntelligence(); } }); }
   };
 });
 
diff --git a/config.json b/config.json
index 2737cec..d143027 100644
--- a/config.json
+++ b/config.json
@@ -1,6 +1,6 @@
 {
-  "duriin_db": "/data/archive.sqlite",
-  "intelligence_db": "/data/intelligence.sqlite",
+  "duriin_db": "./archive.sqlite",
+  "intelligence_db": "./intelligence.sqlite",
   "llm": {
     "baseUrl": "https://openrouter.ai/api/v1",
     "model": "qwen/qwen3-235b-a22b-2507",
diff --git a/docker-compose.yml b/docker-compose.yml
index fdb4add..07169c6 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -22,6 +22,8 @@ services:
       - ./data:/data
     environment:
       NODE_ENV: production
+      DURIIN_DB: /data/archive.sqlite
+      INTELLIGENCE_DB: /data/intelligence.sqlite
     restart: unless-stopped
     networks:
       - nginx_proxy_manager_default
diff --git a/intelligence/db.js b/intelligence/db.js
index 2bdfaa8..6dc901b 100644
--- a/intelligence/db.js
+++ b/intelligence/db.js
@@ -1,4 +1,5 @@
 const Database = require("better-sqlite3");
+const sqliteVec = require("sqlite-vec");
 
 let archiveDb = null;
 let intelligenceDb = null;
@@ -6,6 +7,7 @@ let intelligenceDb = null;
 function getArchiveDb(dbPath) {
   if (!archiveDb) {
     archiveDb = new Database(dbPath, { readonly: true });
+    sqliteVec.load(archiveDb);
     archiveDb.pragma("journal_mode = WAL");
   }
   return archiveDb;
diff --git a/intelligence/index.js b/intelligence/index.js
index 7ede601..d53055c 100644
--- a/intelligence/index.js
+++ b/intelligence/index.js
@@ -17,8 +17,8 @@ function resolvePath(p, fallback) {
 }
 
 const config = {
-  duriin_db: resolvePath(rawConfig.duriin_db, path.resolve(configDir, "archive.sqlite")),
-  intelligence_db: resolvePath(rawConfig.intelligence_db, path.resolve(configDir, "intelligence.sqlite")),
+  duriin_db: process.env.DURIIN_DB || resolvePath(rawConfig.duriin_db, path.resolve(configDir, "archive.sqlite")),
+  intelligence_db: process.env.INTELLIGENCE_DB || resolvePath(rawConfig.intelligence_db, path.resolve(configDir, "intelligence.sqlite")),
   llm: rawConfig.llm || {},
   workers: rawConfig.workers || {},
   openRouter: rawConfig.openRouter || {},
diff --git a/package.json b/package.json
index 9bf7332..e2dce5c 100644
--- a/package.json
+++ b/package.json
@@ -4,7 +4,8 @@
   "description": "News ingestion archive server",
   "main": "server.js",
   "scripts": {
-    "start": "node server.js"
+    "start": "node server.js",
+    "intelligence": "node intelligence/index.js"
   },
   "keywords": [],
   "author": "",
diff --git a/src/db.js b/src/db.js
index e0e95e0..e04889a 100644
--- a/src/db.js
+++ b/src/db.js
@@ -9,8 +9,6 @@ sqliteVec.load(db);
 
 db.pragma('journal_mode = WAL');
 
-// the image column is retained as a no-op for backwards compat with old rows.
-// new code never writes to it; drop in a future migration if you really want
 db.exec(`
   CREATE TABLE IF NOT EXISTS articles (
     id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -21,97 +19,29 @@ db.exec(`
     content_status TEXT,
     content_error TEXT,
     content_attempted_at TEXT,
+    content_attempt_count INTEGER NOT NULL DEFAULT 0,
+    content_retry_after TEXT,
     is_index_page INTEGER NOT NULL DEFAULT 0,
+    has_embedding INTEGER NOT NULL DEFAULT 0,
     url TEXT NOT NULL UNIQUE,
     normalized_title TEXT NOT NULL,
     source TEXT NOT NULL,
     pub_date TEXT,
+    pub_date_effective TEXT,
+    language TEXT,
+    event_id INTEGER REFERENCES events(id),
     ingested_at TEXT NOT NULL DEFAULT (datetime('now'))
   );
 `);
 
-function rebuildArticlesTableIfNeeded() {
-  const indexes = db.prepare(`PRAGMA index_list('articles')`).all();
-  const hasUniqueNormalizedTitleIndex = indexes.some((index) => {
-    if (index.origin !== 'u' || !index.name) {
-      return false;
-    }
-
-    const columns = db.prepare(`PRAGMA index_info('${index.name.replace(/'/g, "''")}')`).all();
-    return columns.length === 1 && columns[0].name === 'normalized_title';
-  });
-
-  if (!hasUniqueNormalizedTitleIndex) {
-    return;
-  }
-
-  db.exec(`
-    BEGIN;
-
-    CREATE TABLE articles_rebuild (
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      title TEXT NOT NULL,
-      description TEXT,
-      content TEXT,
-      image TEXT,
-      content_status TEXT,
-      content_error TEXT,
-      content_attempted_at TEXT,
-      is_index_page INTEGER NOT NULL DEFAULT 0,
-      url TEXT NOT NULL UNIQUE,
-      normalized_title TEXT NOT NULL,
-      source TEXT NOT NULL,
-      pub_date TEXT,
-      ingested_at TEXT NOT NULL DEFAULT (datetime('now'))
-    );
-
-    INSERT INTO articles_rebuild (
-      id,
-      title,
-      description,
-      content,
-      image,
-      content_status,
-      content_error,
-      content_attempted_at,
-      is_index_page,
-      url,
-      normalized_title,
-      source,
-      pub_date,
-      ingested_at
-    )
-    SELECT
-      id,
-      title,
-      description,
-      content,
-      image,
-      content_status,
-      content_error,
-      content_attempted_at,
-      0,
-      url,
-      normalized_title,
-      source,
-      pub_date,
-      ingested_at
-    FROM articles;
-
-    DROP TABLE articles;
-    ALTER TABLE articles_rebuild RENAME TO articles;
-
-    COMMIT;
-  `);
-}
-
-rebuildArticlesTableIfNeeded();
-
 db.exec(`
   CREATE INDEX IF NOT EXISTS idx_articles_source ON articles(source);
   CREATE INDEX IF NOT EXISTS idx_articles_pub_date ON articles(pub_date);
   CREATE INDEX IF NOT EXISTS idx_articles_ingested_at ON articles(ingested_at);
   CREATE INDEX IF NOT EXISTS idx_articles_normalized_title ON articles(normalized_title);
+  CREATE INDEX IF NOT EXISTS idx_articles_event_id ON articles(event_id);
+  CREATE INDEX IF NOT EXISTS idx_articles_has_embedding ON articles(has_embedding);
+  CREATE INDEX IF NOT EXISTS idx_articles_pub_date_effective ON articles(pub_date_effective DESC);
 `);
 
 db.exec(`
@@ -132,93 +62,22 @@ db.exec(`
   );
 `);
 
-// vec0 table — fixed at 8192 dims to cover any model on openrouter, shorter embeddings get zero-padded
-{
-  const existing = db.prepare(`SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'article_embeddings'`).get();
-  const currentDim = existing && existing.sql && existing.sql.match(/FLOAT\[(\d+)\]/);
-  const needsMigration = existing && (!currentDim || parseInt(currentDim[1], 10) !== 8192);
+db.exec(`
+  CREATE VIRTUAL TABLE IF NOT EXISTS article_embeddings USING vec0(
+    article_id INTEGER PRIMARY KEY,
+    embedding FLOAT[8192]
+  );
+`);
 
-  if (needsMigration) {
-    // save everything in vec0 to the store before dropping it, keyed by whatever model is in meta
-    try {
-      const BATCH = 500;
-      let offset = 0;
-
-      const fetchBatch = db.prepare(`
-        SELECT e.article_id, m.model, e.embedding
-        FROM article_embeddings e
-        JOIN article_embedding_meta m ON m.article_id = e.article_id
-        LIMIT ? OFFSET ?
-      `);
-
-      const insert = db.prepare(`
-        INSERT OR IGNORE INTO article_embedding_store (article_id, model, embedding)
-        VALUES (?, ?, ?)
-      `);
-
-      const insertMany = db.transaction((rows) => {
-        for (const row of rows) insert.run(row.article_id, row.model, row.embedding);
-      });
-
-      while (true) {
-        const rows = fetchBatch.all(BATCH, offset);
-        if (rows.length === 0) break;
-        insertMany(rows);
-        offset += rows.length;
-        if (rows.length < BATCH) break;
-      }
-    } catch (err) {
-      console.error('failed to rescue embeddings from vec0 before migration:', err);
-    }
-
-    db.exec(`DROP TABLE article_embeddings`);
-    db.exec(`DELETE FROM article_embedding_meta`);
-  }
-
-  if (!existing || needsMigration) {
-    db.exec(`
-      CREATE VIRTUAL TABLE article_embeddings USING vec0(
-        article_id INTEGER PRIMARY KEY,
-        embedding FLOAT[8192]
-      );
-    `);
-  }
-}
-
-// migrate query_embeddings to include model in primary key
-{
-  const cols = db.prepare(`PRAGMA table_info(query_embeddings)`).all();
-  const hasModel = cols.some(c => c.name === 'model');
-
-  if (!hasModel) {
-    db.exec(`
-      BEGIN;
-
-      CREATE TABLE query_embeddings_new (
-        query TEXT NOT NULL,
-        model TEXT NOT NULL,
-        embedding BLOB NOT NULL,
-        created_at TEXT NOT NULL DEFAULT (datetime('now')),
-        PRIMARY KEY (query, model)
-      );
-
-      DROP TABLE IF EXISTS query_embeddings;
-      ALTER TABLE query_embeddings_new RENAME TO query_embeddings;
-
-      COMMIT;
-    `);
-  } else {
-    db.exec(`
-      CREATE TABLE IF NOT EXISTS query_embeddings (
-        query TEXT NOT NULL,
-        model TEXT NOT NULL,
-        embedding BLOB NOT NULL,
-        created_at TEXT NOT NULL DEFAULT (datetime('now')),
-        PRIMARY KEY (query, model)
-      );
-    `);
-  }
-}
+db.exec(`
+  CREATE TABLE IF NOT EXISTS query_embeddings (
+    query TEXT NOT NULL,
+    model TEXT NOT NULL,
+    embedding BLOB NOT NULL,
+    created_at TEXT NOT NULL DEFAULT (datetime('now')),
+    PRIMARY KEY (query, model)
+  );
+`);
 
 db.exec(`
   CREATE TABLE IF NOT EXISTS events (
@@ -228,22 +87,6 @@ db.exec(`
   );
 `);
 
-for (const statement of [
-  'ALTER TABLE articles ADD COLUMN event_id INTEGER REFERENCES events(id)',
-]) {
-  try {
-    db.exec(statement);
-  } catch (error) {
-    if (!String(error.message).includes('duplicate column name')) {
-      throw error;
-    }
-  }
-}
-
-db.exec(`
-  CREATE INDEX IF NOT EXISTS idx_articles_event_id ON articles(event_id);
-`);
-
 db.exec(`
   CREATE TABLE IF NOT EXISTS gdelt_backfill_windows (
     source_id TEXT NOT NULL,
@@ -287,9 +130,6 @@ db.exec(`
   );
 `);
 
-// per-domain fetch policy — caches whether plain http or browser is needed
-// so we dont waste a round trip on every article from a known js-only site.
-// expires_at lets us re-probe domains that may have recovered
 db.exec(`
   CREATE TABLE IF NOT EXISTS domain_fetch_policy (
     domain TEXT PRIMARY KEY,
@@ -303,30 +143,4 @@ db.exec(`
   );
 `);
 
-for (const statement of [
-  'ALTER TABLE articles ADD COLUMN image TEXT',
-  'ALTER TABLE articles ADD COLUMN content_status TEXT',
-  'ALTER TABLE articles ADD COLUMN content_error TEXT',
-  'ALTER TABLE articles ADD COLUMN content_attempted_at TEXT',
-  'ALTER TABLE articles ADD COLUMN content_attempt_count INTEGER NOT NULL DEFAULT 0',
-  'ALTER TABLE articles ADD COLUMN content_retry_after TEXT',
-  'ALTER TABLE articles ADD COLUMN is_index_page INTEGER NOT NULL DEFAULT 0',
-  'ALTER TABLE articles ADD COLUMN has_embedding INTEGER NOT NULL DEFAULT 0',
-  'ALTER TABLE articles ADD COLUMN pub_date_effective TEXT',
-  'ALTER TABLE articles ADD COLUMN language TEXT'
-]) {
-  try {
-    db.exec(statement);
-  } catch (error) {
-    if (!String(error.message).includes('duplicate column name')) {
-      throw error;
-    }
-  }
-}
-
-db.exec(`
-  CREATE INDEX IF NOT EXISTS idx_articles_has_embedding ON articles(has_embedding);
-  CREATE INDEX IF NOT EXISTS idx_articles_pub_date_effective ON articles(pub_date_effective DESC);
-`);
-
-module.exports = db;
+module.exports = db;
\ No newline at end of file
diff --git a/src/routes/admin.js b/src/routes/admin.js
index 9b94879..dc230b3 100644
--- a/src/routes/admin.js
+++ b/src/routes/admin.js
@@ -2,6 +2,24 @@ const fs = require('fs');
 const path = require('path');
 const db = require('../db');
 const config = require('../config');
+const Database = require('better-sqlite3');
+
+let idb = null;
+
+function getIntelligenceDb() {
+  if (idb) return idb;
+
+  const configDir = path.resolve(__dirname, '..', '..');
+  const rawPath = process.env.INTELLIGENCE_DB
+    || (config.intelligence_db
+      ? (path.isAbsolute(config.intelligence_db) ? config.intelligence_db : path.resolve(configDir, config.intelligence_db))
+      : path.resolve(configDir, 'intelligence.sqlite'));
+
+  if (!fs.existsSync(rawPath)) return null;
+
+  idb = new Database(rawPath, { readonly: true });
+  return idb;
+}
 
 const adminUser = (config.admin && config.admin.username) || 'admin';
 const adminPass = (config.admin && config.admin.password) || 'changeme';
@@ -204,6 +222,122 @@ async function adminRoutes(fastify) {
     return { ok: true };
   });
 
+  // intelligence endpoints
+  fastify.get('/admin/api/intelligence/stats', async (request, reply) => {
+    if (!checkAuth(request, reply)) return;
+    const db = getIntelligenceDb();
+    if (!db) return { available: false };
+
+    const queue = db.prepare(`SELECT status, COUNT(*) as n FROM article_queue GROUP BY status`).all();
+    const knowledge = db.prepare(`SELECT COUNT(*) as n FROM event_knowledge`).get().n;
+    const predictions = db.prepare(`SELECT COUNT(*) as n FROM event_predictions`).get().n;
+    const companies = db.prepare(`SELECT COUNT(*) as n FROM tracked_companies`).get().n;
+    const embeddings = db.prepare(`SELECT COUNT(*) as n FROM company_embeddings`).get().n;
+
+    return { available: true, queue, knowledge, predictions, companies, embeddings };
+  });
+
+  fastify.get('/admin/api/intelligence/companies', async (request, reply) => {
+    if (!checkAuth(request, reply)) return;
+    const db = getIntelligenceDb();
+    if (!db) return [];
+    return db.prepare(`SELECT * FROM tracked_companies ORDER BY name`).all();
+  });
+
+  fastify.get('/admin/api/intelligence/knowledge', async (request, reply) => {
+    if (!checkAuth(request, reply)) return;
+    const db = getIntelligenceDb();
+    if (!db) return { total: 0, rows: [] };
+
+    const q = request.query || {};
+    const limit = Math.min(parseInt(q.limit, 10) || 50, 200);
+    const offset = parseInt(q.offset, 10) || 0;
+    const companyId = q.company_id ? parseInt(q.company_id, 10) : null;
+    const type = q.type || null;
+
+    const conditions = [];
+    const params = [];
+    if (companyId) { conditions.push('ek.company_id = ?'); params.push(companyId); }
+    if (type) { conditions.push('ek.type = ?'); params.push(type); }
+    const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : '';
+
+    const total = db.prepare(`SELECT COUNT(*) as n FROM event_knowledge ek ${where}`).get(...params).n;
+    const rows = db.prepare(`
+      SELECT ek.id, ek.event_id, ek.type, ek.data, ek.created_at,
+             tc.name as company_name
+      FROM event_knowledge ek
+      JOIN tracked_companies tc ON tc.id = ek.company_id
+      ${where}
+      ORDER BY ek.id DESC
+      LIMIT ? OFFSET ?
+    `).all(...params, limit, offset);
+
+    return { total, rows };
+  });
+
+  fastify.get('/admin/api/intelligence/predictions', async (request, reply) => {
+    if (!checkAuth(request, reply)) return;
+    const db = getIntelligenceDb();
+    if (!db) return { total: 0, rows: [] };
+
+    const q = request.query || {};
+    const limit = Math.min(parseInt(q.limit, 10) || 50, 200);
+    const offset = parseInt(q.offset, 10) || 0;
+    const companyId = q.company_id ? parseInt(q.company_id, 10) : null;
+
+    const conditions = [];
+    const params = [];
+    if (companyId) { conditions.push('ep.company_id = ?'); params.push(companyId); }
+    const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : '';
+
+    const total = db.prepare(`SELECT COUNT(*) as n FROM event_predictions ep ${where}`).get(...params).n;
+    const rows = db.prepare(`
+      SELECT ep.*, tc.name as company_name
+      FROM event_predictions ep
+      JOIN tracked_companies tc ON tc.id = ep.company_id
+      ${where}
+      ORDER BY ep.id DESC
+      LIMIT ? OFFSET ?
+    `).all(...params, limit, offset);
+
+    return { total, rows };
+  });
+
+  // raw sql console
+  fastify.post('/admin/api/sql', async (request, reply) => {
+    if (!checkAuth(request, reply)) return;
+
+    const { sql, database } = request.body || {};
+    if (!sql || !sql.trim()) { reply.code(400); return { error: 'no sql provided' }; }
+
+    const target = database === 'intelligence' ? getIntelligenceDb() : db;
+    if (!target) { reply.code(400); return { error: 'database not available' }; }
+
+    try {
+      const stmt = target.prepare(sql);
+      const start = Date.now();
+
+      let rows, changes, lastInsertRowid;
+      if (stmt.reader) {
+        rows = stmt.all();
+      } else {
+        const info = stmt.run();
+        changes = info.changes;
+        lastInsertRowid = info.lastInsertRowid;
+      }
+
+      return {
+        rows: rows || null,
+        changes: changes ?? null,
+        lastInsertRowid: lastInsertRowid ?? null,
+        elapsed: Date.now() - start,
+      };
+    } catch (err) {
+      reply.code(400);
+      return { error: err.message };
+    }
+  });
+
   // stats for dashboard header
   fastify.get('/admin/api/stats', async (request, reply) => {
     if (!checkAuth(request, reply)) return;
diff --git a/src/routes/dev.js b/src/routes/dev.js
index f9fc54f..4076bef 100644
--- a/src/routes/dev.js
+++ b/src/routes/dev.js
@@ -1,26 +1,34 @@
 const fs = require('fs');
 const path = require('path');
+const os = require('os');
 const config = require('../config');
+const db = require('../db');
 
 
 async function devRoutes(fastify) {
   if (!config.dev || !config.dev.enabled) return;
 
   fastify.get('/dev/db/download', async (req, reply) => {
-    const dbPath = path.resolve(config.duriin_db || './archive.sqlite');
+    const tmpPath = path.join(os.tmpdir(), `duriin_snapshot_${Date.now()}.sqlite`);
 
-    if (!fs.existsSync(dbPath)) {
-      return reply.code(404).send({ error: 'database file not found' });
+    try {
+      // VACUUM INTO gives us a consistent, defragmented copy with no mid-write corruption
+      db.prepare(`VACUUM INTO ?`).run(tmpPath);
+
+      const stat = fs.statSync(tmpPath);
+
+      reply.header('Content-Type', 'application/octet-stream');
+      reply.header('Content-Disposition', 'attachment; filename="archive.sqlite"');
+      reply.header('Content-Length', stat.size);
+
+      const stream = fs.createReadStream(tmpPath);
+      stream.on('close', () => fs.unlink(tmpPath, () => {}));
+
+      return reply.send(stream);
+    } catch (err) {
+      fs.unlink(tmpPath, () => {});
+      throw err;
     }
-
-    const stat = fs.statSync(dbPath);
-    const filename = path.basename(dbPath);
-
-    reply.header('Content-Type', 'application/octet-stream');
-    reply.header('Content-Disposition', `attachment; filename="${filename}"`);
-    reply.header('Content-Length', stat.size);
-
-    return reply.send(fs.createReadStream(dbPath));
   });
 }