refactor: update worker commands and add new scripts for API rebuilding and queue feeding
This commit is contained in:
ImBenji
@@ -0,0 +1,263 @@
|
||||
const https = require("https");
|
||||
const http = require("http");
|
||||
|
||||
const { findMatchedCompaniesByEmbedding } = require("./embeddings");
|
||||
|
||||
async function runAugorWorker(archiveDb, intelligenceDb, config) {
|
||||
const loopDelay = config.workers?.augorLoopDelayMs ?? 1500;
|
||||
const llmConfig = config.openRouter || {};
|
||||
|
||||
const getPending = intelligenceDb.prepare(`
|
||||
SELECT * FROM article_queue WHERE status = 'pending' LIMIT 1
|
||||
`);
|
||||
|
||||
const recordEvent = intelligenceDb.prepare(
|
||||
`INSERT INTO worker_events (worker) VALUES ('augor')`
|
||||
);
|
||||
|
||||
const pruneEvents = intelligenceDb.prepare(
|
||||
`DELETE FROM worker_events WHERE worker = 'augor' AND completed_at < datetime('now', '-1 hour')`
|
||||
);
|
||||
let pruneCounter = 0;
|
||||
|
||||
const setStatus = intelligenceDb.prepare(`
|
||||
UPDATE article_queue SET status = ?, updated_at = CURRENT_TIMESTAMP WHERE article_id = ?
|
||||
`);
|
||||
|
||||
const getEventArticleIds = archiveDb.prepare(
|
||||
"SELECT id FROM articles WHERE event_id = ?"
|
||||
);
|
||||
|
||||
const setStatusByArticleId = intelligenceDb.prepare(`
|
||||
UPDATE article_queue SET status = 'processed', updated_at = CURRENT_TIMESTAMP
|
||||
WHERE article_id = ? AND status = 'pending'
|
||||
`);
|
||||
|
||||
const deleteKnowledge = intelligenceDb.prepare(
|
||||
"DELETE FROM event_knowledge WHERE event_id = ?"
|
||||
);
|
||||
const deletePredictions = intelligenceDb.prepare(
|
||||
"DELETE FROM event_predictions WHERE event_id = ?"
|
||||
);
|
||||
|
||||
const insertKnowledge = intelligenceDb.prepare(`
|
||||
INSERT INTO event_knowledge (event_id, company_id, type, data, event_date)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
`);
|
||||
const insertPrediction = intelligenceDb.prepare(`
|
||||
INSERT INTO event_predictions (event_id, company_id, type, direction, magnitude, timeframe, rationale, event_date)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
|
||||
const getEventDate = archiveDb.prepare(`
|
||||
SELECT pub_date_effective FROM articles
|
||||
WHERE event_id = ? AND pub_date_effective IS NOT NULL
|
||||
ORDER BY pub_date_effective ASC LIMIT 1
|
||||
`);
|
||||
|
||||
const getCompanyFacts = intelligenceDb.prepare(`
|
||||
SELECT claim, confidence, confirmation_count FROM company_facts
|
||||
WHERE company_id = ? ORDER BY confirmation_count DESC LIMIT 30
|
||||
`);
|
||||
|
||||
|
||||
while (true) {
|
||||
try {
|
||||
const queueRow = getPending.get();
|
||||
|
||||
if (!queueRow) {
|
||||
await sleep(loopDelay);
|
||||
continue;
|
||||
}
|
||||
|
||||
const article = archiveDb.prepare(`
|
||||
SELECT id, event_id, content, has_embedding
|
||||
FROM articles WHERE id = ?
|
||||
`).get(queueRow.article_id);
|
||||
|
||||
if (!article || !article.content || !article.has_embedding || !article.event_id) {
|
||||
setStatus.run("skipped", queueRow.article_id);
|
||||
continue;
|
||||
}
|
||||
|
||||
const eventId = article.event_id;
|
||||
|
||||
const event = archiveDb.prepare("SELECT * FROM events WHERE id = ?").get(eventId);
|
||||
if (!event) {
|
||||
setStatus.run("skipped", queueRow.article_id);
|
||||
continue;
|
||||
}
|
||||
|
||||
const eventArticles = archiveDb.prepare(`
|
||||
SELECT id, title, description, content
|
||||
FROM articles
|
||||
WHERE event_id = ? AND content IS NOT NULL AND content != ''
|
||||
ORDER BY id ASC
|
||||
LIMIT 25
|
||||
`).all(eventId);
|
||||
|
||||
const eventArticleIds = eventArticles.map(a => a.id);
|
||||
|
||||
const matchedCompanies = findMatchedCompaniesByEmbedding(
|
||||
eventArticleIds, archiveDb, intelligenceDb, config
|
||||
);
|
||||
|
||||
if (matchedCompanies.length === 0) {
|
||||
for (const r of getEventArticleIds.all(eventId)) setStatusByArticleId.run(r.id);
|
||||
console.log(`[augor] event ${eventId} — no company match, skipped`);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
deleteKnowledge.run(eventId);
|
||||
deletePredictions.run(eventId);
|
||||
|
||||
const eventDateRow = getEventDate.get(eventId);
|
||||
const eventDate = eventDateRow ? eventDateRow.pub_date_effective : null;
|
||||
|
||||
const articleText = eventArticles.map((a, i) => {
|
||||
const body = (a.content || a.description || "").slice(0, 2000);
|
||||
return `[Article ${i + 1}] ${a.title}\n${body}`;
|
||||
}).join("\n\n---\n\n");
|
||||
|
||||
for (const company of matchedCompanies) {
|
||||
try {
|
||||
const facts = getCompanyFacts.all(company.id);
|
||||
|
||||
let factsBlock = null;
|
||||
if (facts.length > 0) {
|
||||
const lines = facts.map(f => `- ${f.claim} (confirmed ${f.confirmation_count} times)`).join("\n");
|
||||
factsBlock = `Known facts about ${company.name}:\n${lines}`;
|
||||
}
|
||||
|
||||
const result = await callLlm(llmConfig, buildPrompt(company.name, event.title, articleText, factsBlock));
|
||||
|
||||
if (result) {
|
||||
const writeAll = intelligenceDb.transaction(() => {
|
||||
for (const r of (result.knowledge?.relationships || [])) {
|
||||
insertKnowledge.run(eventId, company.id, "relationship", JSON.stringify(r), eventDate);
|
||||
}
|
||||
for (const t of (result.knowledge?.themes || [])) {
|
||||
insertKnowledge.run(eventId, company.id, "theme", JSON.stringify(t), eventDate);
|
||||
}
|
||||
for (const f of (result.knowledge?.factors || [])) {
|
||||
insertKnowledge.run(eventId, company.id, "factor", JSON.stringify(f), eventDate);
|
||||
}
|
||||
|
||||
for (const p of (result.predictions || [])) {
|
||||
insertPrediction.run(eventId, company.id, p.type, p.direction, p.magnitude, p.timeframe, p.rationale, eventDate);
|
||||
}
|
||||
});
|
||||
|
||||
writeAll();
|
||||
}
|
||||
|
||||
} catch (llmErr) {
|
||||
console.error(`[augor] LLM error for ${company.name} on event ${eventId}:`, llmErr.message);
|
||||
}
|
||||
}
|
||||
|
||||
for (const r of getEventArticleIds.all(eventId)) setStatusByArticleId.run(r.id);
|
||||
recordEvent.run();
|
||||
pruneCounter++;
|
||||
if (pruneCounter >= 100) { pruneEvents.run(); pruneCounter = 0; }
|
||||
console.log(`[augor] processed event ${eventId} (${matchedCompanies.length} companies, ${eventArticles.length} articles)`);
|
||||
|
||||
} catch (err) {
|
||||
console.error("[augor] error:", err.message);
|
||||
await sleep(loopDelay);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function buildPrompt(companyName, eventTitle, articleText, factsBlock) {
|
||||
const factsPart = factsBlock ? `${factsBlock}\n\n` : "";
|
||||
|
||||
return `You are a financial intelligence analyst focused on ${companyName}. Always respond in English regardless of the language of the input articles.
|
||||
|
||||
${factsPart}Assess the impact of the following news event on ${companyName} given what you already know about the company.
|
||||
|
||||
Event: ${eventTitle}
|
||||
|
||||
${articleText}
|
||||
|
||||
Return JSON only — no explanation. Shape:
|
||||
{
|
||||
"knowledge": {
|
||||
"relationships": [
|
||||
{ "type": "supplier|customer|competitor", "entity": "string", "confidence": "high|medium|low", "evidence": "string" }
|
||||
],
|
||||
"themes": [
|
||||
{ "theme": "string", "direction": "increasing|stable|decreasing", "evidence": "string" }
|
||||
],
|
||||
"factors": [
|
||||
{ "factor": "string", "relationship": "string", "evidence": "string" }
|
||||
]
|
||||
},
|
||||
"predictions": [
|
||||
{ "type": "market_share|stock_price|competitive_position|other", "direction": "positive|negative|neutral", "magnitude": "high|medium|low", "timeframe": "short|medium|long", "rationale": "string" }
|
||||
]
|
||||
}
|
||||
|
||||
Only include claims directly supported by the articles. Use empty arrays if nothing applies.`;
|
||||
}
|
||||
|
||||
async function callLlm(llmConfig, prompt) {
|
||||
const body = JSON.stringify({
|
||||
model: llmConfig.llmModel || llmConfig.model,
|
||||
messages: [{ role: "user", content: prompt }],
|
||||
temperature: 0.1,
|
||||
});
|
||||
|
||||
const url = new URL("https://openrouter.ai/api/v1/chat/completions");
|
||||
|
||||
const responseText = await httpPost(url, body, {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": `Bearer ${llmConfig.apiKey || ""}`,
|
||||
});
|
||||
|
||||
let parsed;
|
||||
try {
|
||||
parsed = JSON.parse(responseText);
|
||||
} catch (e) {
|
||||
throw new Error(`LLM response not JSON: ${responseText.slice(0, 300)}`);
|
||||
}
|
||||
const content = parsed.choices?.[0]?.message?.content;
|
||||
if (!content) return null;
|
||||
|
||||
const stripped = content.replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/, '').trim();
|
||||
return JSON.parse(stripped);
|
||||
}
|
||||
|
||||
function httpPost(url, body, headers) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const lib = url.protocol === "https:" ? https : http;
|
||||
const req = lib.request({
|
||||
hostname: url.hostname,
|
||||
port: url.port || (url.protocol === "https:" ? 443 : 80),
|
||||
path: url.pathname + url.search,
|
||||
method: "POST",
|
||||
headers: { ...headers, "Content-Length": Buffer.byteLength(body) },
|
||||
}, (res) => {
|
||||
let data = "";
|
||||
res.on("data", chunk => data += chunk);
|
||||
res.on("end", () => {
|
||||
if (res.statusCode >= 200 && res.statusCode < 300) {
|
||||
resolve(data);
|
||||
} else {
|
||||
reject(new Error(`LLM ${res.statusCode}: ${data.slice(0, 300)}`));
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
req.on("error", reject);
|
||||
req.write(body);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise(r => setTimeout(r, ms));
|
||||
}
|
||||
|
||||
module.exports = { runAugorWorker };
|
||||
@@ -0,0 +1,284 @@
|
||||
// consolidationWorker.js
|
||||
// reads from event_knowledge (never writes to it) and maintains company_facts
|
||||
// runs on a slow loop — one full pass over all tracked companies per cycle
|
||||
|
||||
const https = require("https");
|
||||
const http = require("http");
|
||||
|
||||
|
||||
async function runConsolidationWorker(archiveDb, intelligenceDb, config) {
|
||||
const loopDelay = config.workers?.consolidationLoopDelayMs ?? 60000;
|
||||
const llmConfig = config.openRouter || {};
|
||||
|
||||
const getCompanies = intelligenceDb.prepare("SELECT * FROM tracked_companies");
|
||||
|
||||
const recordEvent = intelligenceDb.prepare(
|
||||
`INSERT INTO worker_events (worker) VALUES ('consolidation')`
|
||||
);
|
||||
const pruneEvents = intelligenceDb.prepare(
|
||||
`DELETE FROM worker_events WHERE worker = 'consolidation' AND completed_at < datetime('now', '-1 hour')`
|
||||
);
|
||||
|
||||
const getKnowledge = intelligenceDb.prepare(`
|
||||
SELECT * FROM event_knowledge WHERE company_id = ? ORDER BY event_date ASC
|
||||
`);
|
||||
|
||||
const getExistingFirstSeen = intelligenceDb.prepare(`
|
||||
SELECT claim, first_seen_at FROM company_facts WHERE company_id = ?
|
||||
`);
|
||||
|
||||
const deleteCompanyFacts = intelligenceDb.prepare(
|
||||
"DELETE FROM company_facts WHERE company_id = ?"
|
||||
);
|
||||
|
||||
const insertFact = intelligenceDb.prepare(`
|
||||
INSERT INTO company_facts
|
||||
(company_id, type, claim, confidence, confirmation_count, first_seen_at, last_seen_at, last_event_id, supporting_event_ids)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
|
||||
// stale = seen only once and not reinforced in 90 days
|
||||
const deleteStaleFacts = intelligenceDb.prepare(`
|
||||
DELETE FROM company_facts
|
||||
WHERE confirmation_count = 1
|
||||
AND last_seen_at < datetime('now', '-90 days')
|
||||
`);
|
||||
|
||||
while (true) {
|
||||
try {
|
||||
const companies = getCompanies.all();
|
||||
|
||||
let pruneCounter = 0;
|
||||
for (const company of companies) {
|
||||
try {
|
||||
await processCompany(
|
||||
company, intelligenceDb, llmConfig,
|
||||
getKnowledge, getExistingFirstSeen, deleteCompanyFacts, insertFact
|
||||
);
|
||||
recordEvent.run();
|
||||
pruneCounter++;
|
||||
if (pruneCounter >= 50) { pruneEvents.run(); pruneCounter = 0; }
|
||||
} catch (err) {
|
||||
console.error(`[consolidation] error on ${company.name}:`, err.message);
|
||||
}
|
||||
}
|
||||
|
||||
const staleResult = deleteStaleFacts.run();
|
||||
if (staleResult.changes > 0) {
|
||||
console.log(`[consolidation] pruned ${staleResult.changes} stale facts`);
|
||||
}
|
||||
|
||||
console.log("[consolidation] cycle complete");
|
||||
|
||||
} catch (err) {
|
||||
console.error("[consolidation] cycle error:", err.message);
|
||||
}
|
||||
|
||||
await sleep(loopDelay);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
async function processCompany(company, intelligenceDb, llmConfig, getKnowledge, getExistingFirstSeen, deleteCompanyFacts, insertFact) {
|
||||
const rows = getKnowledge.all(company.id);
|
||||
if (rows.length === 0) return;
|
||||
|
||||
const rawClaims = [];
|
||||
|
||||
for (const row of rows) {
|
||||
let data;
|
||||
try { data = JSON.parse(row.data); } catch (_) { continue; }
|
||||
|
||||
const text = extractClaimText(row.type, data);
|
||||
if (!text) continue;
|
||||
|
||||
rawClaims.push({
|
||||
text,
|
||||
type: row.type,
|
||||
eventId: row.event_id,
|
||||
eventDate: row.event_date || row.created_at,
|
||||
});
|
||||
}
|
||||
|
||||
if (rawClaims.length === 0) return;
|
||||
|
||||
let groups;
|
||||
try {
|
||||
groups = await normalizeClaims(company.name, rawClaims, llmConfig);
|
||||
} catch (err) {
|
||||
console.error(`[consolidation] LLM failed for ${company.name}:`, err.message);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!groups || groups.length === 0) return;
|
||||
|
||||
// save first_seen_at before we wipe the rows
|
||||
const priorFirstSeen = {};
|
||||
for (const f of getExistingFirstSeen.all(company.id)) {
|
||||
priorFirstSeen[f.claim] = f.first_seen_at;
|
||||
}
|
||||
|
||||
const writeAll = intelligenceDb.transaction(() => {
|
||||
deleteCompanyFacts.run(company.id);
|
||||
|
||||
for (const group of groups) {
|
||||
const { canonical, type, eventIds, dates } = group;
|
||||
|
||||
const distinctEventIds = [...new Set(eventIds)];
|
||||
const count = distinctEventIds.size || distinctEventIds.length;
|
||||
|
||||
const sortedDates = dates.filter(Boolean).sort();
|
||||
const firstSeen = priorFirstSeen[canonical] || sortedDates[0] || new Date().toISOString();
|
||||
const lastSeen = sortedDates[sortedDates.length - 1] || firstSeen;
|
||||
|
||||
const lastEventId = eventIds[eventIds.length - 1];
|
||||
const confidence = countToConfidence(count);
|
||||
|
||||
insertFact.run(
|
||||
company.id,
|
||||
type,
|
||||
canonical,
|
||||
confidence,
|
||||
count,
|
||||
firstSeen,
|
||||
lastSeen,
|
||||
lastEventId,
|
||||
JSON.stringify(distinctEventIds)
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
writeAll();
|
||||
console.log(`[consolidation] ${company.name}: ${groups.length} distinct facts`);
|
||||
}
|
||||
|
||||
|
||||
function extractClaimText(type, data) {
|
||||
if (type === "relationship") {
|
||||
if (!data.entity) return null;
|
||||
return `${data.entity} is a ${data.type || "partner"}`;
|
||||
}
|
||||
|
||||
if (type === "theme") {
|
||||
if (!data.theme) return null;
|
||||
return `${data.theme} is ${data.direction || "relevant"}`;
|
||||
}
|
||||
|
||||
if (type === "factor") {
|
||||
if (!data.factor) return null;
|
||||
return `${data.factor}: ${data.relationship || "relevant"}`;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
function countToConfidence(count) {
|
||||
if (count >= 8) return "very_high";
|
||||
if (count >= 4) return "high";
|
||||
if (count >= 2) return "medium";
|
||||
return "low";
|
||||
}
|
||||
|
||||
|
||||
async function normalizeClaims(companyName, rawClaims, llmConfig) {
|
||||
// cap to keep the prompt manageable
|
||||
const capped = rawClaims.slice(0, 120);
|
||||
|
||||
const lines = capped.map((c, i) => `${i + 1}. [${c.type}] ${c.text}`).join("\n");
|
||||
|
||||
const prompt = `You are normalizing knowledge claims about ${companyName}. Always respond in English regardless of the language of the input articles.
|
||||
|
||||
Group semantically equivalent claims and provide a canonical short form for each group (lowercase, no punctuation).
|
||||
|
||||
Claims:
|
||||
${lines}
|
||||
|
||||
Return JSON only — no explanation. Shape:
|
||||
{
|
||||
"groups": [
|
||||
{ "canonical": "canonical claim text", "type": "relationship|theme|factor", "indices": [1, 2] }
|
||||
]
|
||||
}
|
||||
|
||||
Rules:
|
||||
- canonical must be lowercase with no punctuation marks
|
||||
- merge claims that mean the same thing even if worded differently
|
||||
- each index appears in exactly one group
|
||||
- pick the type that best fits the group`;
|
||||
|
||||
const body = JSON.stringify({
|
||||
model: llmConfig.llmModel || llmConfig.model,
|
||||
messages: [{ role: "user", content: prompt }],
|
||||
temperature: 0.1,
|
||||
});
|
||||
|
||||
const url = new URL("https://openrouter.ai/api/v1/chat/completions");
|
||||
const responseText = await httpPost(url, body, {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": `Bearer ${llmConfig.apiKey || ""}`,
|
||||
});
|
||||
|
||||
let parsed;
|
||||
try {
|
||||
parsed = JSON.parse(responseText);
|
||||
} catch (_) {
|
||||
throw new Error(`LLM response not JSON: ${responseText.slice(0, 200)}`);
|
||||
}
|
||||
|
||||
const content = parsed.choices?.[0]?.message?.content;
|
||||
if (!content) return [];
|
||||
|
||||
const stripped = content.replace(/^```(?:json)?\s*/i, "").replace(/\s*```$/, "").trim();
|
||||
const result = JSON.parse(stripped);
|
||||
|
||||
const groups = [];
|
||||
|
||||
for (const g of (result.groups || [])) {
|
||||
const indices = (g.indices || [])
|
||||
.map(i => i - 1)
|
||||
.filter(i => i >= 0 && i < capped.length);
|
||||
|
||||
if (indices.length === 0) continue;
|
||||
|
||||
const eventIds = indices.map(i => capped[i].eventId);
|
||||
const dates = indices.map(i => capped[i].eventDate);
|
||||
const type = g.type || capped[indices[0]].type;
|
||||
|
||||
groups.push({ canonical: g.canonical, type, eventIds, dates });
|
||||
}
|
||||
|
||||
return groups;
|
||||
}
|
||||
|
||||
|
||||
function httpPost(url, body, headers) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const lib = url.protocol === "https:" ? https : http;
|
||||
|
||||
const req = lib.request({
|
||||
hostname: url.hostname,
|
||||
port: url.port || (url.protocol === "https:" ? 443 : 80),
|
||||
path: url.pathname + url.search,
|
||||
method: "POST",
|
||||
headers: { ...headers, "Content-Length": Buffer.byteLength(body) },
|
||||
}, (res) => {
|
||||
let data = "";
|
||||
res.on("data", chunk => data += chunk);
|
||||
res.on("end", () => {
|
||||
if (res.statusCode >= 200 && res.statusCode < 300) resolve(data);
|
||||
else reject(new Error(`LLM ${res.statusCode}: ${data.slice(0, 300)}`));
|
||||
});
|
||||
});
|
||||
|
||||
req.on("error", reject);
|
||||
req.write(body);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise(r => setTimeout(r, ms));
|
||||
}
|
||||
|
||||
module.exports = { runConsolidationWorker };
|
||||
+327
@@ -0,0 +1,327 @@
|
||||
const Database = require("better-sqlite3");
|
||||
const sqliteVec = require("sqlite-vec");
|
||||
|
||||
let archiveDb = null;
|
||||
let intelligenceDb = null;
|
||||
|
||||
function getArchiveDb(dbPath) {
|
||||
if (!archiveDb) {
|
||||
archiveDb = new Database(dbPath, { readonly: true });
|
||||
sqliteVec.load(archiveDb);
|
||||
archiveDb.pragma("journal_mode = WAL");
|
||||
}
|
||||
return archiveDb;
|
||||
}
|
||||
|
||||
function getIntelligenceDb(dbPath) {
|
||||
if (!intelligenceDb) {
|
||||
intelligenceDb = new Database(dbPath);
|
||||
intelligenceDb.pragma("journal_mode = WAL");
|
||||
}
|
||||
return intelligenceDb;
|
||||
}
|
||||
|
||||
function runMigrations(db) {
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS cursors (
|
||||
key TEXT PRIMARY KEY,
|
||||
value INTEGER
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tracked_companies (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name TEXT,
|
||||
ticker TEXT,
|
||||
aliases TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS article_queue (
|
||||
id INTEGER PRIMARY KEY,
|
||||
article_id INTEGER UNIQUE,
|
||||
status TEXT DEFAULT 'pending',
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS event_knowledge (
|
||||
id INTEGER PRIMARY KEY,
|
||||
event_id INTEGER,
|
||||
company_id INTEGER,
|
||||
type TEXT,
|
||||
data TEXT,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS company_embeddings (
|
||||
company_id INTEGER PRIMARY KEY,
|
||||
embedding BLOB NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
generated_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS event_predictions (
|
||||
id INTEGER PRIMARY KEY,
|
||||
event_id INTEGER,
|
||||
company_id INTEGER,
|
||||
type TEXT,
|
||||
direction TEXT,
|
||||
magnitude TEXT,
|
||||
timeframe TEXT,
|
||||
rationale TEXT,
|
||||
event_date TEXT,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS company_facts (
|
||||
id INTEGER PRIMARY KEY,
|
||||
company_id INTEGER,
|
||||
type TEXT,
|
||||
claim TEXT,
|
||||
confidence TEXT,
|
||||
confirmation_count INTEGER DEFAULT 1,
|
||||
first_seen_at DATETIME,
|
||||
last_seen_at DATETIME,
|
||||
last_event_id INTEGER,
|
||||
supporting_event_ids TEXT
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_company_facts_unique ON company_facts (company_id, claim);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS company_relationships (
|
||||
id INTEGER PRIMARY KEY,
|
||||
from_company_id INTEGER,
|
||||
relationship_type TEXT,
|
||||
to_entity TEXT,
|
||||
to_company_id INTEGER,
|
||||
confidence TEXT,
|
||||
confirmation_count INTEGER DEFAULT 1,
|
||||
first_seen_at DATETIME,
|
||||
last_seen_at DATETIME,
|
||||
supporting_event_ids TEXT
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_company_relationships_unique
|
||||
ON company_relationships (from_company_id, relationship_type, to_entity);
|
||||
|
||||
`);
|
||||
}
|
||||
|
||||
function runColumnMigrations(db) {
|
||||
try { db.exec("ALTER TABLE event_predictions ADD COLUMN event_date TEXT"); } catch (_) {}
|
||||
try { db.exec("ALTER TABLE event_knowledge ADD COLUMN event_date TEXT"); } catch (_) {}
|
||||
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS worker_events (
|
||||
id INTEGER PRIMARY KEY,
|
||||
worker TEXT NOT NULL,
|
||||
completed_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_worker_events_lookup ON worker_events (worker, completed_at);
|
||||
`);
|
||||
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS trade_signals (
|
||||
id INTEGER PRIMARY KEY,
|
||||
company_id INTEGER,
|
||||
signal TEXT,
|
||||
confidence TEXT,
|
||||
timeframe TEXT,
|
||||
risk_level TEXT,
|
||||
risk_factors TEXT,
|
||||
summary TEXT,
|
||||
key_drivers TEXT,
|
||||
supporting_prediction_ids TEXT,
|
||||
generated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
window_days INTEGER
|
||||
);
|
||||
`);
|
||||
|
||||
// prune rows older than 1 hour so the table doesnt grow unbounded
|
||||
db.exec(`DELETE FROM worker_events WHERE completed_at < datetime('now', '-1 hour')`);
|
||||
}
|
||||
|
||||
function seedCompanies(db) {
|
||||
const exists = db.prepare("SELECT id FROM tracked_companies WHERE name = ?");
|
||||
const insert = db.prepare(
|
||||
"INSERT INTO tracked_companies (name, ticker, aliases) VALUES (?, ?, ?)"
|
||||
);
|
||||
|
||||
const companies = [
|
||||
// semiconductors
|
||||
{ name: "NVIDIA", ticker: "NVDA", aliases: ["Nvidia Corporation"] },
|
||||
{ name: "TSMC", ticker: "TSM", aliases: ["Taiwan Semiconductor", "Taiwan Semiconductor Manufacturing Company"] },
|
||||
{ name: "ASML", ticker: "ASML", aliases: ["ASML Holding"] },
|
||||
{ name: "Intel", ticker: "INTC", aliases: ["Intel Corporation"] },
|
||||
{ name: "AMD", ticker: "AMD", aliases: ["Advanced Micro Devices"] },
|
||||
{ name: "Qualcomm", ticker: "QCOM", aliases: ["Qualcomm Incorporated"] },
|
||||
{ name: "Broadcom", ticker: "AVGO", aliases: ["Broadcom Inc"] },
|
||||
{ name: "Micron", ticker: "MU", aliases: ["Micron Technology"] },
|
||||
{ name: "Texas Instruments", ticker: "TXN", aliases: ["TI"] },
|
||||
{ name: "Applied Materials", ticker: "AMAT", aliases: ["Applied Materials Inc"] },
|
||||
{ name: "Lam Research", ticker: "LRCX", aliases: ["Lam Research Corporation"] },
|
||||
{ name: "KLA Corporation", ticker: "KLAC", aliases: ["KLA"] },
|
||||
{ name: "Samsung", ticker: "005930.KS", aliases: ["Samsung Electronics", "Samsung Group"] },
|
||||
{ name: "SK Hynix", ticker: "000660.KS", aliases: ["Hynix"] },
|
||||
|
||||
// big tech / cloud
|
||||
{ name: "Microsoft", ticker: "MSFT", aliases: ["Microsoft Corporation"] },
|
||||
{ name: "Apple", ticker: "AAPL", aliases: ["Apple Inc"] },
|
||||
{ name: "Alphabet", ticker: "GOOGL", aliases: ["Google", "Google LLC", "DeepMind"] },
|
||||
{ name: "Amazon", ticker: "AMZN", aliases: ["Amazon Web Services", "AWS"] },
|
||||
{ name: "Meta", ticker: "META", aliases: ["Meta Platforms", "Facebook"] },
|
||||
{ name: "Tesla", ticker: "TSLA", aliases: ["Tesla Inc", "Tesla Motors"] },
|
||||
|
||||
// AI / infrastructure
|
||||
{ name: "OpenAI", ticker: "OPENAI", aliases: ["Open AI"] },
|
||||
{ name: "Anthropic", ticker: "ANTHROPIC", aliases: [] },
|
||||
{ name: "xAI", ticker: "XAI", aliases: ["x.AI"] },
|
||||
{ name: "Palantir", ticker: "PLTR", aliases: ["Palantir Technologies"] },
|
||||
{ name: "Super Micro Computer", ticker: "SMCI", aliases: ["Supermicro", "SMCI"] },
|
||||
{ name: "Arista Networks", ticker: "ANET", aliases: ["Arista"] },
|
||||
|
||||
// networking / hardware
|
||||
{ name: "Cisco", ticker: "CSCO", aliases: ["Cisco Systems"] },
|
||||
{ name: "Marvell Technology", ticker: "MRVL", aliases: ["Marvell"] },
|
||||
{ name: "Arm Holdings", ticker: "ARM", aliases: ["Arm", "ARM Ltd"] },
|
||||
|
||||
// enterprise software
|
||||
{ name: "Oracle", ticker: "ORCL", aliases: ["Oracle Corporation"] },
|
||||
{ name: "Salesforce", ticker: "CRM", aliases: ["Salesforce Inc"] },
|
||||
{ name: "SAP", ticker: "SAP", aliases: ["SAP SE"] },
|
||||
{ name: "ServiceNow", ticker: "NOW", aliases: [] },
|
||||
|
||||
// storage / infra
|
||||
{ name: "Western Digital", ticker: "WDC", aliases: ["WD", "Western Digital Corporation"] },
|
||||
{ name: "Seagate", ticker: "STX", aliases: ["Seagate Technology"] },
|
||||
{ name: "Pure Storage", ticker: "PSTG", aliases: [] },
|
||||
|
||||
// more AI labs
|
||||
{ name: "Mistral AI", ticker: "MISTRAL", aliases: ["Mistral"] },
|
||||
{ name: "Cohere", ticker: "COHERE", aliases: [] },
|
||||
{ name: "DeepSeek", ticker: "DEEPSEEK", aliases: ["DeepSeek AI"] },
|
||||
{ name: "Stability AI", ticker: "STABILITY", aliases: [] },
|
||||
{ name: "Inflection AI", ticker: "INFLECTION", aliases: [] },
|
||||
{ name: "Scale AI", ticker: "SCALEAI", aliases: ["Scale"] },
|
||||
{ name: "Hugging Face", ticker: "HF", aliases: [] },
|
||||
{ name: "Cerebras Systems", ticker: "CBRS", aliases: ["Cerebras"] },
|
||||
{ name: "Groq", ticker: "GROQ", aliases: [] },
|
||||
|
||||
// chinese tech
|
||||
{ name: "Alibaba", ticker: "BABA", aliases: ["Alibaba Group", "Taobao", "Alipay"] },
|
||||
{ name: "Tencent", ticker: "0700.HK", aliases: ["Tencent Holdings", "WeChat"] },
|
||||
{ name: "Baidu", ticker: "BIDU", aliases: ["Baidu Inc"] },
|
||||
{ name: "Huawei", ticker: "HUAWEI", aliases: ["Huawei Technologies"] },
|
||||
{ name: "ByteDance", ticker: "BYTEDANCE", aliases: ["TikTok", "Douyin"] },
|
||||
{ name: "Xiaomi", ticker: "1810.HK", aliases: [] },
|
||||
{ name: "SMIC", ticker: "688981.SS", aliases: ["Semiconductor Manufacturing International Corporation"] },
|
||||
{ name: "DJI", ticker: "DJI", aliases: ["Da-Jiang Innovations"] },
|
||||
|
||||
// defense / aerospace
|
||||
{ name: "Lockheed Martin", ticker: "LMT", aliases: ["Lockheed"] },
|
||||
{ name: "Raytheon Technologies", ticker: "RTX", aliases: ["RTX", "Raytheon"] },
|
||||
{ name: "Northrop Grumman", ticker: "NOC", aliases: ["Northrop"] },
|
||||
{ name: "Boeing", ticker: "BA", aliases: ["Boeing Company"] },
|
||||
{ name: "General Dynamics", ticker: "GD", aliases: [] },
|
||||
{ name: "BAE Systems", ticker: "BA.L", aliases: ["BAE"] },
|
||||
{ name: "L3Harris Technologies", ticker: "LHX", aliases: ["L3Harris"] },
|
||||
{ name: "Leidos", ticker: "LDOS", aliases: [] },
|
||||
{ name: "SAIC", ticker: "SAIC", aliases: ["Science Applications International Corporation"] },
|
||||
{ name: "Booz Allen Hamilton", ticker: "BAH", aliases: ["Booz Allen"] },
|
||||
{ name: "Thales Group", ticker: "HO.PA", aliases: ["Thales"] },
|
||||
{ name: "Airbus", ticker: "AIR.PA", aliases: ["Airbus SE"] },
|
||||
{ name: "Leonardo", ticker: "LDO.MI", aliases: ["Leonardo SpA"] },
|
||||
{ name: "Rheinmetall", ticker: "RHM.DE", aliases: [] },
|
||||
|
||||
// telecom
|
||||
{ name: "Ericsson", ticker: "ERIC", aliases: ["Telefonaktiebolaget LM Ericsson"] },
|
||||
{ name: "Nokia", ticker: "NOK", aliases: ["Nokia Corporation"] },
|
||||
{ name: "AT&T", ticker: "T", aliases: [] },
|
||||
{ name: "Verizon", ticker: "VZ", aliases: ["Verizon Communications"] },
|
||||
{ name: "T-Mobile", ticker: "TMUS", aliases: ["T-Mobile US"] },
|
||||
{ name: "Deutsche Telekom", ticker: "DTE.DE", aliases: [] },
|
||||
{ name: "SoftBank", ticker: "9984.T", aliases: ["SoftBank Group"] },
|
||||
|
||||
// finance / banking
|
||||
{ name: "JPMorgan Chase", ticker: "JPM", aliases: ["JPMorgan", "JP Morgan"] },
|
||||
{ name: "Goldman Sachs", ticker: "GS", aliases: ["Goldman"] },
|
||||
{ name: "BlackRock", ticker: "BLK", aliases: [] },
|
||||
{ name: "Visa", ticker: "V", aliases: [] },
|
||||
{ name: "Mastercard", ticker: "MA", aliases: [] },
|
||||
{ name: "Morgan Stanley", ticker: "MS", aliases: [] },
|
||||
{ name: "Citigroup", ticker: "C", aliases: ["Citi"] },
|
||||
{ name: "Bank of America", ticker: "BAC", aliases: ["BofA"] },
|
||||
{ name: "HSBC", ticker: "HSBA.L", aliases: ["HSBC Holdings"] },
|
||||
{ name: "UBS", ticker: "UBS", aliases: [] },
|
||||
|
||||
// energy / resources
|
||||
{ name: "ExxonMobil", ticker: "XOM", aliases: ["Exxon"] },
|
||||
{ name: "Chevron", ticker: "CVX", aliases: [] },
|
||||
{ name: "Shell", ticker: "SHEL", aliases: ["Shell plc", "Royal Dutch Shell"] },
|
||||
{ name: "BP", ticker: "BP", aliases: ["British Petroleum"] },
|
||||
{ name: "TotalEnergies", ticker: "TTE.PA", aliases: ["Total"] },
|
||||
{ name: "Saudi Aramco", ticker: "2222.SR", aliases: ["Aramco"] },
|
||||
{ name: "NextEra Energy", ticker: "NEE", aliases: [] },
|
||||
|
||||
// pharma / biotech
|
||||
{ name: "Pfizer", ticker: "PFE", aliases: [] },
|
||||
{ name: "Johnson & Johnson", ticker: "JNJ", aliases: ["J&J"] },
|
||||
{ name: "Moderna", ticker: "MRNA", aliases: [] },
|
||||
{ name: "AstraZeneca", ticker: "AZN", aliases: [] },
|
||||
{ name: "Novartis", ticker: "NVS", aliases: [] },
|
||||
{ name: "Roche", ticker: "ROG.SW", aliases: ["Roche Holding"] },
|
||||
{ name: "Eli Lilly", ticker: "LLY", aliases: [] },
|
||||
{ name: "CRISPR Therapeutics", ticker: "CRSP", aliases: [] },
|
||||
|
||||
// automotive
|
||||
{ name: "Toyota", ticker: "TM", aliases: ["Toyota Motor"] },
|
||||
{ name: "Volkswagen", ticker: "VOW.DE", aliases: ["VW"] },
|
||||
{ name: "Ford", ticker: "F", aliases: ["Ford Motor"] },
|
||||
{ name: "General Motors", ticker: "GM", aliases: ["GM"] },
|
||||
{ name: "BYD", ticker: "002594.SZ", aliases: ["BYD Company"] },
|
||||
{ name: "Rivian", ticker: "RIVN", aliases: [] },
|
||||
|
||||
// media / entertainment
|
||||
{ name: "Netflix", ticker: "NFLX", aliases: [] },
|
||||
{ name: "Disney", ticker: "DIS", aliases: ["The Walt Disney Company"] },
|
||||
{ name: "Comcast", ticker: "CMCSA", aliases: ["NBCUniversal"] },
|
||||
{ name: "Spotify", ticker: "SPOT", aliases: [] },
|
||||
{ name: "Warner Bros Discovery", ticker: "WBD", aliases: ["Warner Bros", "HBO"] },
|
||||
{ name: "News Corp", ticker: "NWSA", aliases: ["Rupert Murdoch"] },
|
||||
|
||||
// retail / e-commerce
|
||||
{ name: "Walmart", ticker: "WMT", aliases: [] },
|
||||
{ name: "Shopify", ticker: "SHOP", aliases: [] },
|
||||
{ name: "eBay", ticker: "EBAY", aliases: [] },
|
||||
|
||||
// cybersecurity
|
||||
{ name: "CrowdStrike", ticker: "CRWD", aliases: [] },
|
||||
{ name: "Palo Alto Networks", ticker: "PANW", aliases: [] },
|
||||
{ name: "SentinelOne", ticker: "S", aliases: [] },
|
||||
{ name: "Fortinet", ticker: "FTNT", aliases: [] },
|
||||
{ name: "Cloudflare", ticker: "NET", aliases: [] },
|
||||
{ name: "Recorded Future", ticker: "RF", aliases: [] },
|
||||
|
||||
// space
|
||||
{ name: "SpaceX", ticker: "SPACEX", aliases: ["Space Exploration Technologies"] },
|
||||
{ name: "Blue Origin", ticker: "BLUEORIGIN", aliases: [] },
|
||||
{ name: "Planet Labs", ticker: "PL", aliases: [] },
|
||||
|
||||
// consulting / professional services
|
||||
{ name: "McKinsey", ticker: "MCKINSEY", aliases: ["McKinsey & Company"] },
|
||||
{ name: "Accenture", ticker: "ACN", aliases: [] },
|
||||
{ name: "Deloitte", ticker: "DELOITTE", aliases: [] },
|
||||
];
|
||||
|
||||
let added = 0;
|
||||
for (const c of companies) {
|
||||
if (!exists.get(c.name)) {
|
||||
insert.run(c.name, c.ticker, JSON.stringify(c.aliases));
|
||||
added++;
|
||||
}
|
||||
}
|
||||
|
||||
if (added > 0) console.log(`[db] seeded ${added} new tracked companies`);
|
||||
}
|
||||
|
||||
module.exports = { getArchiveDb, getIntelligenceDb, runMigrations, runColumnMigrations, seedCompanies };
|
||||
@@ -0,0 +1,134 @@
|
||||
// embedding generation and cosine similarity for the intelligence layer
|
||||
|
||||
async function generateEmbedding(text, openRouterConfig) {
|
||||
const response = await fetch("https://openrouter.ai/api/v1/embeddings", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Authorization": `Bearer ${openRouterConfig.apiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: openRouterConfig.embeddingModel,
|
||||
input: text,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
let msg = `embedding request failed with ${response.status}`;
|
||||
try {
|
||||
const payload = await response.json();
|
||||
if (payload?.error?.message) msg = payload.error.message;
|
||||
} catch (_) {}
|
||||
throw new Error(msg);
|
||||
}
|
||||
|
||||
const payload = await response.json();
|
||||
const embedding = payload?.data?.[0]?.embedding;
|
||||
if (!Array.isArray(embedding) || embedding.length === 0) {
|
||||
throw new Error("invalid embedding response");
|
||||
}
|
||||
|
||||
return embedding;
|
||||
}
|
||||
|
||||
// Float32 BLOB -> Float32Array
|
||||
function blobToFloat32(buf) {
|
||||
return new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
|
||||
}
|
||||
|
||||
function cosineSimilarity(a, b) {
|
||||
if (a.length !== b.length) {
|
||||
// if dims differ just use the shorter length — handles edge cases gracefully
|
||||
const len = Math.min(a.length, b.length);
|
||||
a = a.subarray(0, len);
|
||||
b = b.subarray(0, len);
|
||||
}
|
||||
|
||||
let dot = 0, normA = 0, normB = 0;
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
dot += a[i] * b[i];
|
||||
normA += a[i] * a[i];
|
||||
normB += b[i] * b[i];
|
||||
}
|
||||
|
||||
const denom = Math.sqrt(normA) * Math.sqrt(normB);
|
||||
return denom === 0 ? 0 : dot / denom;
|
||||
}
|
||||
|
||||
// generates company embeddings for any tracked company that doesnt have one yet
|
||||
async function ensureCompanyEmbeddings(intelligenceDb, openRouterConfig) {
|
||||
const companies = intelligenceDb.prepare("SELECT * FROM tracked_companies").all();
|
||||
|
||||
const getEmbed = intelligenceDb.prepare(
|
||||
"SELECT embedding FROM company_embeddings WHERE company_id = ?"
|
||||
);
|
||||
const upsertEmbed = intelligenceDb.prepare(`
|
||||
INSERT INTO company_embeddings (company_id, embedding, model, generated_at)
|
||||
VALUES (?, ?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(company_id) DO UPDATE SET
|
||||
embedding = excluded.embedding,
|
||||
model = excluded.model,
|
||||
generated_at = excluded.generated_at
|
||||
`);
|
||||
|
||||
for (const company of companies) {
|
||||
const existing = getEmbed.get(company.id);
|
||||
if (existing) continue;
|
||||
|
||||
const text = `${company.name} is a company with ticker ${company.ticker}`;
|
||||
try {
|
||||
const embedding = await generateEmbedding(text, openRouterConfig);
|
||||
const buf = Buffer.from(new Float32Array(embedding).buffer);
|
||||
upsertEmbed.run(company.id, buf, openRouterConfig.embeddingModel);
|
||||
console.log(`[embeddings] generated embedding for ${company.name}`);
|
||||
} catch (err) {
|
||||
console.error(`[embeddings] failed for ${company.name}:`, err.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// returns matched company objects from tracked_companies
|
||||
// checks cosine similarity between each company embedding and
|
||||
// the raw embeddings of all articles in the event
|
||||
function findMatchedCompaniesByEmbedding(eventArticleIds, archiveDb, intelligenceDb, config) {
|
||||
const threshold = config.intelligence?.similarityThreshold ?? 0.35;
|
||||
const model = config.openRouter?.embeddingModel;
|
||||
|
||||
const companies = intelligenceDb.prepare(
|
||||
"SELECT id, name, ticker FROM company_embeddings ce JOIN tracked_companies tc ON tc.id = ce.company_id"
|
||||
).all();
|
||||
|
||||
if (companies.length === 0) return [];
|
||||
|
||||
// load article embeddings from archive — only articles that have one
|
||||
const articleEmbeddings = [];
|
||||
for (const articleId of eventArticleIds) {
|
||||
const row = archiveDb.prepare(
|
||||
"SELECT embedding FROM article_embedding_store WHERE article_id = ? AND model = ?"
|
||||
).get(articleId, model);
|
||||
if (row) articleEmbeddings.push(blobToFloat32(row.embedding));
|
||||
}
|
||||
|
||||
if (articleEmbeddings.length === 0) return [];
|
||||
|
||||
const matched = [];
|
||||
for (const company of companies) {
|
||||
const companyRow = intelligenceDb.prepare(
|
||||
"SELECT embedding FROM company_embeddings WHERE company_id = ?"
|
||||
).get(company.id);
|
||||
if (!companyRow) continue;
|
||||
|
||||
const companyVec = blobToFloat32(companyRow.embedding);
|
||||
const hit = articleEmbeddings.some(articleVec => {
|
||||
const sim = cosineSimilarity(companyVec, articleVec);
|
||||
return sim >= threshold;
|
||||
});
|
||||
|
||||
if (hit) matched.push(company);
|
||||
}
|
||||
|
||||
return matched;
|
||||
}
|
||||
|
||||
module.exports = { generateEmbedding, ensureCompanyEmbeddings, findMatchedCompaniesByEmbedding };
|
||||
@@ -0,0 +1,274 @@
|
||||
// builds cross-company relationship graph from company_facts rows where type = 'relationship'
|
||||
|
||||
const VALID_TYPES = ["supplier", "customer", "competitor", "partner", "investor", "dependency"];
|
||||
|
||||
const KEYWORD_MAP = [
|
||||
["manufactur", "supplier"],
|
||||
["suppli", "supplier"],
|
||||
["distribut", "supplier"],
|
||||
["compet", "competitor"],
|
||||
["rival", "competitor"],
|
||||
["invest", "investor"],
|
||||
["fund", "investor"],
|
||||
["backer", "investor"],
|
||||
["customer", "customer"],
|
||||
["client", "customer"],
|
||||
["depend", "dependency"],
|
||||
["partner", "partner"],
|
||||
["collaborat", "partner"],
|
||||
["joint", "partner"],
|
||||
];
|
||||
|
||||
const RECIPROCAL = {
|
||||
supplier: "customer",
|
||||
customer: "supplier",
|
||||
competitor: "competitor",
|
||||
partner: "partner",
|
||||
investor: "dependency",
|
||||
dependency: "investor",
|
||||
};
|
||||
|
||||
|
||||
function parseRelationshipType(rawType) {
|
||||
const t = (rawType || "").toLowerCase().trim();
|
||||
|
||||
if (VALID_TYPES.includes(t)) return t;
|
||||
|
||||
for (const [kw, mapped] of KEYWORD_MAP) {
|
||||
if (t.includes(kw)) return mapped;
|
||||
}
|
||||
|
||||
return "partner";
|
||||
}
|
||||
|
||||
|
||||
function parseClaim(claim) {
|
||||
// format from consolidationWorker: "entity is a type"
|
||||
const parts = claim.split(" is a ");
|
||||
if (parts.length < 2) return null;
|
||||
|
||||
const toEntity = parts[0].trim();
|
||||
const rawType = parts.slice(1).join(" is a ").trim();
|
||||
|
||||
if (!toEntity || !rawType) return null;
|
||||
|
||||
return {
|
||||
toEntity,
|
||||
relationshipType: parseRelationshipType(rawType),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
function resolveCompany(toEntity, companies) {
|
||||
const needle = toEntity.toLowerCase();
|
||||
|
||||
for (const co of companies) {
|
||||
if (co.name.toLowerCase() === needle) return co;
|
||||
if (co.ticker && co.ticker.toLowerCase() === needle) return co;
|
||||
|
||||
let aliases = [];
|
||||
try { aliases = JSON.parse(co.aliases || "[]"); } catch (_) {}
|
||||
|
||||
for (const alias of aliases) {
|
||||
if (alias.toLowerCase() === needle) return co;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
function confidenceFromCount(count) {
|
||||
if (count >= 8) return "very_high";
|
||||
if (count >= 4) return "high";
|
||||
if (count >= 2) return "medium";
|
||||
return "low";
|
||||
}
|
||||
|
||||
|
||||
function mergeEventIds(existingJson, newJson) {
|
||||
let existing = [];
|
||||
let incoming = [];
|
||||
|
||||
try { existing = JSON.parse(existingJson || "[]"); } catch (_) {}
|
||||
try { incoming = JSON.parse(newJson || "[]"); } catch (_) {}
|
||||
|
||||
const merged = [...new Set([...existing, ...incoming])];
|
||||
return JSON.stringify(merged);
|
||||
}
|
||||
|
||||
|
||||
async function runGraphWorker(archiveDb, intelligenceDb, config) {
|
||||
const loopDelay = config.workers?.graphWorkerLoopDelayMs ?? 90000;
|
||||
|
||||
const getRelationshipFacts = intelligenceDb.prepare(`
|
||||
SELECT * FROM company_facts WHERE type = 'relationship'
|
||||
`);
|
||||
|
||||
const recordEvents = intelligenceDb.prepare(
|
||||
`INSERT INTO worker_events (worker) VALUES ('graph')`
|
||||
);
|
||||
const pruneEvents = intelligenceDb.prepare(
|
||||
`DELETE FROM worker_events WHERE worker = 'graph' AND completed_at < datetime('now', '-1 hour')`
|
||||
);
|
||||
|
||||
const getCompanies = intelligenceDb.prepare("SELECT * FROM tracked_companies");
|
||||
|
||||
const getCompanyById = intelligenceDb.prepare("SELECT * FROM tracked_companies WHERE id = ?");
|
||||
|
||||
const getExistingRel = intelligenceDb.prepare(`
|
||||
SELECT * FROM company_relationships
|
||||
WHERE from_company_id = ? AND relationship_type = ? AND to_entity = ?
|
||||
`);
|
||||
|
||||
const insertRel = intelligenceDb.prepare(`
|
||||
INSERT INTO company_relationships
|
||||
(from_company_id, relationship_type, to_entity, to_company_id, confidence, confirmation_count, first_seen_at, last_seen_at, supporting_event_ids)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
|
||||
const updateRel = intelligenceDb.prepare(`
|
||||
UPDATE company_relationships
|
||||
SET confirmation_count = ?,
|
||||
confidence = ?,
|
||||
last_seen_at = ?,
|
||||
supporting_event_ids = ?
|
||||
WHERE from_company_id = ? AND relationship_type = ? AND to_entity = ?
|
||||
`);
|
||||
|
||||
const getExistingReciprocal = intelligenceDb.prepare(`
|
||||
SELECT id FROM company_relationships
|
||||
WHERE from_company_id = ? AND relationship_type = ? AND to_entity = ?
|
||||
`);
|
||||
|
||||
const insertReciprocal = intelligenceDb.prepare(`
|
||||
INSERT OR IGNORE INTO company_relationships
|
||||
(from_company_id, relationship_type, to_entity, to_company_id, confidence, confirmation_count, first_seen_at, last_seen_at, supporting_event_ids)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
|
||||
|
||||
while (true) {
|
||||
try {
|
||||
const facts = getRelationshipFacts.all();
|
||||
const companies = getCompanies.all();
|
||||
|
||||
let upserted = 0;
|
||||
let reciprocals = 0;
|
||||
|
||||
const processAll = intelligenceDb.transaction(() => {
|
||||
for (const fact of facts) {
|
||||
const parsed = parseClaim(fact.claim);
|
||||
if (!parsed) continue;
|
||||
|
||||
const { toEntity, relationshipType } = parsed;
|
||||
|
||||
const resolved = resolveCompany(toEntity, companies);
|
||||
const toCompanyId = resolved ? resolved.id : null;
|
||||
|
||||
const existing = getExistingRel.get(fact.company_id, relationshipType, toEntity);
|
||||
const now = new Date().toISOString();
|
||||
|
||||
let finalCount;
|
||||
let finalFirst;
|
||||
let finalLast;
|
||||
let finalEventIds;
|
||||
|
||||
if (existing) {
|
||||
// derive count from the fact itself, not accumulate — avoids compounding on re-runs
|
||||
finalCount = fact.confirmation_count;
|
||||
finalFirst = existing.first_seen_at;
|
||||
finalLast = fact.last_seen_at || now;
|
||||
finalEventIds = mergeEventIds(existing.supporting_event_ids, fact.supporting_event_ids);
|
||||
|
||||
updateRel.run(
|
||||
finalCount,
|
||||
confidenceFromCount(finalCount),
|
||||
finalLast,
|
||||
finalEventIds,
|
||||
fact.company_id,
|
||||
relationshipType,
|
||||
toEntity
|
||||
);
|
||||
} else {
|
||||
finalCount = fact.confirmation_count || 1;
|
||||
finalFirst = fact.first_seen_at || now;
|
||||
finalLast = fact.last_seen_at || now;
|
||||
finalEventIds = fact.supporting_event_ids || "[]";
|
||||
|
||||
insertRel.run(
|
||||
fact.company_id,
|
||||
relationshipType,
|
||||
toEntity,
|
||||
toCompanyId,
|
||||
confidenceFromCount(finalCount),
|
||||
finalCount,
|
||||
finalFirst,
|
||||
finalLast,
|
||||
finalEventIds
|
||||
);
|
||||
}
|
||||
|
||||
upserted++;
|
||||
|
||||
|
||||
// reciprocal edge — only where both companies are tracked
|
||||
if (toCompanyId) {
|
||||
const fromCompany = getCompanyById.get(fact.company_id);
|
||||
if (!fromCompany) continue;
|
||||
|
||||
const recipType = RECIPROCAL[relationshipType] || "partner";
|
||||
const recipToEntity = fromCompany.name;
|
||||
|
||||
const recipExists = getExistingReciprocal.get(toCompanyId, recipType, recipToEntity);
|
||||
|
||||
if (!recipExists) {
|
||||
insertReciprocal.run(
|
||||
toCompanyId,
|
||||
recipType,
|
||||
recipToEntity,
|
||||
fact.company_id,
|
||||
confidenceFromCount(finalCount),
|
||||
finalCount,
|
||||
finalFirst,
|
||||
finalLast,
|
||||
finalEventIds
|
||||
);
|
||||
|
||||
reciprocals++;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
processAll();
|
||||
|
||||
// record one event per edge upserted so the rate tracks actual work
|
||||
if (upserted > 0) {
|
||||
const insertMany = intelligenceDb.transaction((n) => {
|
||||
for (let i = 0; i < n; i++) recordEvents.run();
|
||||
});
|
||||
insertMany(upserted);
|
||||
pruneEvents.run();
|
||||
}
|
||||
|
||||
if (upserted > 0 || reciprocals > 0) {
|
||||
console.log(`[graph] cycle complete — ${upserted} edges upserted, ${reciprocals} reciprocals added`);
|
||||
} else {
|
||||
console.log("[graph] cycle complete — no new data");
|
||||
}
|
||||
|
||||
} catch (err) {
|
||||
console.error("[graph] cycle error:", err.message);
|
||||
}
|
||||
|
||||
await sleep(loopDelay);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise(r => setTimeout(r, ms));
|
||||
}
|
||||
|
||||
module.exports = { runGraphWorker };
|
||||
@@ -0,0 +1,81 @@
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
|
||||
const { getArchiveDb, getIntelligenceDb, runMigrations, runColumnMigrations, seedCompanies } = require("./db");
|
||||
const { runQueueFeeder } = require("./queueFeeder");
|
||||
const { runAugorWorker } = require("./augorWorker");
|
||||
const { ensureCompanyEmbeddings } = require("./embeddings");
|
||||
const { runConsolidationWorker } = require("./consolidationWorker");
|
||||
const { runGraphWorker } = require("./graphWorker");
|
||||
const { runSignalWorker } = require("./signalWorker");
|
||||
|
||||
|
||||
const configPath = path.resolve(__dirname, "../config.json");
|
||||
const configDir = path.dirname(configPath);
|
||||
const rawConfig = JSON.parse(fs.readFileSync(configPath, "utf8"));
|
||||
|
||||
function resolvePath(p, fallback) {
|
||||
if (!p) return fallback;
|
||||
return path.isAbsolute(p) ? p : path.resolve(configDir, p);
|
||||
}
|
||||
|
||||
const config = {
|
||||
duriin_db: process.env.DURIIN_DB || resolvePath(rawConfig.duriin_db, path.resolve(configDir, "archive.sqlite")),
|
||||
intelligence_db: process.env.INTELLIGENCE_DB || resolvePath(rawConfig.intelligence_db, path.resolve(configDir, "intelligence.sqlite")),
|
||||
llm: rawConfig.llm || {},
|
||||
workers: rawConfig.workers || {},
|
||||
openRouter: rawConfig.openRouter || {},
|
||||
intelligence: rawConfig.intelligence || {},
|
||||
};
|
||||
|
||||
console.log("[intelligence] starting up");
|
||||
console.log(`[intelligence] archive: ${config.duriin_db}`);
|
||||
console.log(`[intelligence] intelligence: ${config.intelligence_db}`);
|
||||
|
||||
const archiveDb = getArchiveDb(config.duriin_db);
|
||||
const intelligenceDb = getIntelligenceDb(config.intelligence_db);
|
||||
|
||||
runMigrations(intelligenceDb);
|
||||
runColumnMigrations(intelligenceDb);
|
||||
seedCompanies(intelligenceDb);
|
||||
|
||||
ensureCompanyEmbeddings(intelligenceDb, config.openRouter).then(() => {
|
||||
console.log("[intelligence] company embeddings ready");
|
||||
}).catch(err => {
|
||||
console.error("[intelligence] company embedding generation failed:", err.message);
|
||||
});
|
||||
|
||||
runQueueFeeder(archiveDb, intelligenceDb, config).catch(err => {
|
||||
console.error("[feeder] fatal:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
runAugorWorker(archiveDb, intelligenceDb, config).catch(err => {
|
||||
console.error("[augor] fatal:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
runConsolidationWorker(archiveDb, intelligenceDb, config).catch(err => {
|
||||
console.error("[consolidation] fatal:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
runGraphWorker(archiveDb, intelligenceDb, config).catch(err => {
|
||||
console.error("[graph] fatal:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
runSignalWorker(archiveDb, intelligenceDb, config).catch(err => {
|
||||
console.error("[signal] fatal:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
process.on("SIGINT", () => {
|
||||
console.log("[intelligence] shutting down");
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
process.on("SIGTERM", () => {
|
||||
console.log("[intelligence] shutting down");
|
||||
process.exit(0);
|
||||
});
|
||||
@@ -0,0 +1,67 @@
|
||||
// Pulls usable articles from duriin into article_queue continuously
|
||||
// Usable = has content, has embedding, has event assignment
|
||||
|
||||
async function runQueueFeeder(archiveDb, intelligenceDb, config) {
|
||||
const batchSize = config.workers?.queueFeederBatchSize ?? 100;
|
||||
const loopDelay = config.workers?.queueFeederLoopDelayMs ?? 3000;
|
||||
|
||||
const getCursor = intelligenceDb.prepare(
|
||||
"SELECT value FROM cursors WHERE key = 'queue_feeder'"
|
||||
);
|
||||
const setCursor = intelligenceDb.prepare(
|
||||
"INSERT OR REPLACE INTO cursors (key, value) VALUES ('queue_feeder', ?)"
|
||||
);
|
||||
|
||||
const insertQueued = intelligenceDb.prepare(`
|
||||
INSERT OR IGNORE INTO article_queue (article_id, status, created_at)
|
||||
VALUES (?, 'pending', CURRENT_TIMESTAMP)
|
||||
`);
|
||||
|
||||
while (true) {
|
||||
try {
|
||||
const cursorRow = getCursor.get();
|
||||
const cursor = cursorRow ? cursorRow.value : 0;
|
||||
|
||||
const articles = archiveDb.prepare(`
|
||||
SELECT id FROM articles
|
||||
WHERE id > ?
|
||||
AND content IS NOT NULL
|
||||
AND content != ''
|
||||
AND has_embedding = 1
|
||||
AND event_id IS NOT NULL
|
||||
ORDER BY id ASC
|
||||
LIMIT ?
|
||||
`).all(cursor, batchSize);
|
||||
|
||||
if (articles.length === 0) {
|
||||
await sleep(loopDelay);
|
||||
continue;
|
||||
}
|
||||
|
||||
let newCursor = cursor;
|
||||
let inserted = 0;
|
||||
|
||||
for (const a of articles) {
|
||||
const info = insertQueued.run(a.id);
|
||||
if (info.changes > 0) inserted++;
|
||||
if (a.id > newCursor) newCursor = a.id;
|
||||
}
|
||||
|
||||
setCursor.run(newCursor);
|
||||
|
||||
if (inserted > 0) {
|
||||
console.log(`[feeder] queued ${inserted} articles, cursor now ${newCursor}`);
|
||||
}
|
||||
|
||||
} catch (err) {
|
||||
console.error("[feeder] error:", err.message);
|
||||
await sleep(loopDelay);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise(r => setTimeout(r, ms));
|
||||
}
|
||||
|
||||
module.exports = { runQueueFeeder };
|
||||
@@ -0,0 +1,256 @@
|
||||
const https = require("https");
|
||||
const http = require("http");
|
||||
|
||||
|
||||
async function runSignalWorker(archiveDb, intelligenceDb, config) {
|
||||
const loopDelay = config.workers?.signalLoopDelayMs ?? 120000;
|
||||
const llmConfig = config.openRouter || {};
|
||||
|
||||
const getCompanies = intelligenceDb.prepare("SELECT * FROM tracked_companies ORDER BY id");
|
||||
|
||||
const getLastSignal = intelligenceDb.prepare(`
|
||||
SELECT generated_at FROM trade_signals WHERE company_id = ? ORDER BY generated_at DESC LIMIT 1
|
||||
`);
|
||||
|
||||
const getFacts = intelligenceDb.prepare(`
|
||||
SELECT claim, type, confidence, confirmation_count
|
||||
FROM company_facts
|
||||
WHERE company_id = ?
|
||||
ORDER BY confirmation_count DESC
|
||||
LIMIT 40
|
||||
`);
|
||||
|
||||
const getPredictions = intelligenceDb.prepare(`
|
||||
SELECT type, direction, magnitude, timeframe, rationale, event_date, id
|
||||
FROM event_predictions
|
||||
WHERE company_id = ?
|
||||
AND created_at >= datetime('now', '-90 days')
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 50
|
||||
`);
|
||||
|
||||
const getRelationships = intelligenceDb.prepare(`
|
||||
SELECT relationship_type, to_entity, confidence, confirmation_count
|
||||
FROM company_relationships
|
||||
WHERE from_company_id = ?
|
||||
ORDER BY confirmation_count DESC
|
||||
LIMIT 20
|
||||
`);
|
||||
|
||||
const deleteSignal = intelligenceDb.prepare(
|
||||
"DELETE FROM trade_signals WHERE company_id = ?"
|
||||
);
|
||||
|
||||
const insertSignal = intelligenceDb.prepare(`
|
||||
INSERT INTO trade_signals
|
||||
(company_id, signal, confidence, timeframe, risk_level, risk_factors, summary, key_drivers, supporting_prediction_ids, window_days)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 90)
|
||||
`);
|
||||
|
||||
const recordEvent = intelligenceDb.prepare(
|
||||
`INSERT INTO worker_events (worker) VALUES ('signal')`
|
||||
);
|
||||
|
||||
const pruneEvents = intelligenceDb.prepare(
|
||||
`DELETE FROM worker_events WHERE worker = 'signal' AND completed_at < datetime('now', '-1 hour')`
|
||||
);
|
||||
|
||||
let pruneCounter = 0;
|
||||
|
||||
while (true) {
|
||||
try {
|
||||
const companies = getCompanies.all();
|
||||
|
||||
// pick the company with the oldest (or missing) signal that has enough predictions
|
||||
let target = null;
|
||||
let oldestTs = null;
|
||||
|
||||
const companiesByAge = [];
|
||||
for (const company of companies) {
|
||||
const row = getLastSignal.get(company.id);
|
||||
companiesByAge.push({ company, ts: row ? row.generated_at : null });
|
||||
}
|
||||
|
||||
// null ts (never generated) first, then oldest
|
||||
companiesByAge.sort((a, b) => {
|
||||
if (a.ts === null && b.ts === null) return 0;
|
||||
if (a.ts === null) return -1;
|
||||
if (b.ts === null) return 1;
|
||||
return a.ts < b.ts ? -1 : 1;
|
||||
});
|
||||
|
||||
let predictions = null;
|
||||
for (const entry of companiesByAge) {
|
||||
const preds = getPredictions.all(entry.company.id);
|
||||
if (preds.length >= 3) {
|
||||
target = entry.company;
|
||||
predictions = preds;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!target) {
|
||||
await sleep(loopDelay);
|
||||
continue;
|
||||
}
|
||||
|
||||
const facts = getFacts.all(target.id);
|
||||
const relationships = getRelationships.all(target.id);
|
||||
|
||||
const prompt = buildPrompt(target.name, facts, relationships, predictions);
|
||||
|
||||
let result;
|
||||
try {
|
||||
result = await callLlm(llmConfig, prompt);
|
||||
} catch (err) {
|
||||
console.error(`[signal] LLM error for ${target.name}:`, err.message);
|
||||
await sleep(loopDelay);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!result) {
|
||||
console.log(`[signal] ${target.name} — LLM returned null, skipping`);
|
||||
await sleep(loopDelay);
|
||||
continue;
|
||||
}
|
||||
|
||||
const predictionIds = predictions.map(p => p.id);
|
||||
|
||||
const write = intelligenceDb.transaction(() => {
|
||||
deleteSignal.run(target.id);
|
||||
insertSignal.run(
|
||||
target.id,
|
||||
result.signal,
|
||||
result.confidence,
|
||||
result.timeframe,
|
||||
result.risk_level,
|
||||
JSON.stringify(result.risk_factors || []),
|
||||
result.summary,
|
||||
JSON.stringify(result.key_drivers || []),
|
||||
JSON.stringify(predictionIds)
|
||||
);
|
||||
});
|
||||
|
||||
write();
|
||||
|
||||
recordEvent.run();
|
||||
pruneCounter++;
|
||||
if (pruneCounter >= 20) { pruneEvents.run(); pruneCounter = 0; }
|
||||
|
||||
console.log(`[signal] ${target.name} — ${result.signal} (${result.confidence} confidence, ${result.risk_level} risk)`);
|
||||
|
||||
} catch (err) {
|
||||
console.error("[signal] cycle error:", err.message);
|
||||
}
|
||||
|
||||
await sleep(loopDelay);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function buildPrompt(companyName, facts, relationships, predictions) {
|
||||
const factsBlock = facts.length > 0
|
||||
? facts.map(f => `- ${f.claim} (confirmed ${f.confirmation_count}x)`).join("\n")
|
||||
: "No known facts yet.";
|
||||
|
||||
const relBlock = relationships.length > 0
|
||||
? relationships.map(r => `- ${r.relationship_type}: ${r.to_entity} (${r.confidence})`).join("\n")
|
||||
: "No known relationships.";
|
||||
|
||||
const predBlock = predictions.map((p, i) =>
|
||||
`${i + 1}. [${p.type}] ${p.direction} / ${p.magnitude} / ${p.timeframe} — ${p.rationale || "no rationale"}`
|
||||
).join("\n");
|
||||
|
||||
return `You are a financial intelligence analyst generating a trade signal for ${companyName}.
|
||||
|
||||
Known facts about ${companyName} (most confirmed first):
|
||||
${factsBlock}
|
||||
|
||||
Known relationships:
|
||||
${relBlock}
|
||||
|
||||
Recent event predictions (last 90 days):
|
||||
${predBlock}
|
||||
|
||||
Generate a trade signal as JSON with this exact shape:
|
||||
{
|
||||
"signal": "BUY | HOLD | SELL",
|
||||
"confidence": "low | medium | high | very_high",
|
||||
"timeframe": "short | medium | long",
|
||||
"risk_level": "low | medium | high | very_high",
|
||||
"risk_factors": ["string", ...],
|
||||
"key_drivers": ["string", ...],
|
||||
"summary": "2-3 sentence plain English summary"
|
||||
}
|
||||
|
||||
Risk factors should be derived from:
|
||||
- Supply chain concentration (heavy dependence on single suppliers)
|
||||
- Geopolitical exposure (relationships with entities in sensitive regions)
|
||||
- Competitive threats (strong competitors gaining ground)
|
||||
- Regulatory exposure (themes mentioning regulation or export controls)
|
||||
- Negative prediction patterns in recent events
|
||||
|
||||
Only output valid JSON. Always respond in English.`;
|
||||
}
|
||||
|
||||
|
||||
async function callLlm(llmConfig, prompt) {
|
||||
const body = JSON.stringify({
|
||||
model: llmConfig.llmModel || llmConfig.model,
|
||||
messages: [{ role: "user", content: prompt }],
|
||||
temperature: 0.1,
|
||||
});
|
||||
|
||||
const url = new URL("https://openrouter.ai/api/v1/chat/completions");
|
||||
|
||||
const responseText = await httpPost(url, body, {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": `Bearer ${llmConfig.apiKey || ""}`,
|
||||
});
|
||||
|
||||
let parsed;
|
||||
try {
|
||||
parsed = JSON.parse(responseText);
|
||||
} catch (_) {
|
||||
throw new Error(`LLM response not JSON: ${responseText.slice(0, 300)}`);
|
||||
}
|
||||
|
||||
const content = parsed.choices?.[0]?.message?.content;
|
||||
if (!content) return null;
|
||||
|
||||
const stripped = content.replace(/^```(?:json)?\s*/i, "").replace(/\s*```$/, "").trim();
|
||||
return JSON.parse(stripped);
|
||||
}
|
||||
|
||||
|
||||
function httpPost(url, body, headers) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const lib = url.protocol === "https:" ? https : http;
|
||||
|
||||
const req = lib.request({
|
||||
hostname: url.hostname,
|
||||
port: url.port || (url.protocol === "https:" ? 443 : 80),
|
||||
path: url.pathname + url.search,
|
||||
method: "POST",
|
||||
headers: { ...headers, "Content-Length": Buffer.byteLength(body) },
|
||||
}, (res) => {
|
||||
let data = "";
|
||||
res.on("data", chunk => data += chunk);
|
||||
res.on("end", () => {
|
||||
if (res.statusCode >= 200 && res.statusCode < 300) resolve(data);
|
||||
else reject(new Error(`LLM ${res.statusCode}: ${data.slice(0, 300)}`));
|
||||
});
|
||||
});
|
||||
|
||||
req.on("error", reject);
|
||||
req.write(body);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise(r => setTimeout(r, ms));
|
||||
}
|
||||
|
||||
module.exports = { runSignalWorker };
|
||||
Reference in New Issue
Block a user