IMBENJI.NET/Duriin-API
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor: update worker commands and add new scripts for API rebuilding and queue feeding

Browse source
This commit is contained in:
ImBenji 2026-04-27 17:30:52 +01:00
parent 04966fac55
commit f00b1a9640
1 changed files with 66 additions and 73 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

139
workers/signalWorker.js
View file

@ -6,45 +6,60 @@ async function runSignalWorker(archiveDb, intelligenceDb, config) {
const loopDelay = config.workers?.signalLoopDelayMs ?? 120000; const loopDelay = config.workers?.signalLoopDelayMs ?? 120000;
const llmConfig = config.openRouter || {}; const llmConfig = config.openRouter || {};
const getCompanies = intelligenceDb.prepare("SELECT * FROM tracked_companies ORDER BY id"); // add as_of column if it doesnt exist yet
try {
intelligenceDb.prepare("ALTER TABLE trade_signals ADD COLUMN as_of TEXT").run();
console.log("[signal] added as_of column to trade_signals");
} catch (_) {
// already exists
}
const getLastSignal = intelligenceDb.prepare(` // all distinct event dates (by day) per company, newest first, that dont already have a signal
SELECT generated_at FROM trade_signals WHERE company_id = ? ORDER BY generated_at DESC LIMIT 1 const getNextCheckpoint = intelligenceDb.prepare(`
`); SELECT company_id, substr(event_date, 1, 10) as checkpoint_date
FROM event_predictions
const getFacts = intelligenceDb.prepare(` WHERE substr(event_date, 1, 10) NOT IN (
SELECT claim, type, confidence, confirmation_count SELECT as_of FROM trade_signals WHERE as_of IS NOT NULL AND company_id = event_predictions.company_id
FROM company_facts )
WHERE company_id = ? GROUP BY company_id, substr(event_date, 1, 10)
ORDER BY confirmation_count DESC HAVING COUNT(*) >= 3
LIMIT 40 ORDER BY checkpoint_date DESC
LIMIT 1
`); `);
const getPredictions = intelligenceDb.prepare(` const getPredictions = intelligenceDb.prepare(`
SELECT type, direction, magnitude, timeframe, rationale, event_date, id SELECT type, direction, magnitude, timeframe, rationale, event_date, id
FROM event_predictions FROM event_predictions
WHERE company_id = ? WHERE company_id = ?
AND created_at >= datetime('now', '-90 days') AND substr(event_date, 1, 10) <= ?
ORDER BY created_at DESC ORDER BY event_date DESC
LIMIT 50 LIMIT 50
`); `);
const getFacts = intelligenceDb.prepare(`
SELECT claim, type, confidence, confirmation_count
FROM company_facts
WHERE company_id = ?
AND first_seen_at <= ?
ORDER BY confirmation_count DESC
LIMIT 40
`);
const getRelationships = intelligenceDb.prepare(` const getRelationships = intelligenceDb.prepare(`
SELECT relationship_type, to_entity, confidence, confirmation_count SELECT relationship_type, to_entity, confidence, confirmation_count
FROM company_relationships FROM company_relationships
WHERE from_company_id = ? WHERE from_company_id = ?
AND first_seen_at <= ?
ORDER BY confirmation_count DESC ORDER BY confirmation_count DESC
LIMIT 20 LIMIT 20
`); `);
const deleteSignal = intelligenceDb.prepare( const getCompanyById = intelligenceDb.prepare("SELECT * FROM tracked_companies WHERE id = ?");
"DELETE FROM trade_signals WHERE company_id = ?"
);
const insertSignal = intelligenceDb.prepare(` const insertSignal = intelligenceDb.prepare(`
INSERT INTO trade_signals INSERT INTO trade_signals
(company_id, signal, confidence, timeframe, risk_level, risk_factors, summary, key_drivers, supporting_prediction_ids, window_days) (company_id, signal, confidence, timeframe, risk_level, risk_factors, summary, key_drivers, supporting_prediction_ids, window_days, as_of)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 90) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`); `);
const recordEvent = intelligenceDb.prepare( const recordEvent = intelligenceDb.prepare(
@ -59,85 +74,63 @@ async function runSignalWorker(archiveDb, intelligenceDb, config) {
while (true) { while (true) {
try { try {
const companies = getCompanies.all(); const next = getNextCheckpoint.get();
// pick the company with the oldest (or missing) signal that has enough predictions if (!next) {
let target = null;
let oldestTs = null;
const companiesByAge = [];
for (const company of companies) {
const row = getLastSignal.get(company.id);
companiesByAge.push({ company, ts: row ? row.generated_at : null });
}
// null ts (never generated) first, then oldest
companiesByAge.sort((a, b) => {
if (a.ts === null && b.ts === null) return 0;
if (a.ts === null) return -1;
if (b.ts === null) return 1;
return a.ts < b.ts ? -1 : 1;
});
let predictions = null;
for (const entry of companiesByAge) {
const preds = getPredictions.all(entry.company.id);
if (preds.length >= 3) {
target = entry.company;
predictions = preds;
break;
}
}
if (!target) {
await sleep(loopDelay); await sleep(loopDelay);
continue; continue;
} }
const facts = getFacts.all(target.id); const { company_id, checkpoint_date } = next;
const relationships = getRelationships.all(target.id); const company = getCompanyById.get(company_id);
const prompt = buildPrompt(target.name, facts, relationships, predictions); if (!company) {
await sleep(loopDelay);
continue;
}
const predictions = getPredictions.all(company_id, checkpoint_date);
const facts = getFacts.all(company_id, checkpoint_date);
const relationships = getRelationships.all(company_id, checkpoint_date);
const prompt = buildPrompt(company.name, facts, relationships, predictions, checkpoint_date);
let result; let result;
try { try {
result = await callLlm(llmConfig, prompt); result = await callLlm(llmConfig, prompt);
} catch (err) { } catch (err) {
console.error(`[signal] LLM error for ${target.name}:`, err.message); console.error(`[signal] LLM error for ${company.name} @ ${checkpoint_date}:`, err.message);
await sleep(loopDelay); await sleep(loopDelay);
continue; continue;
} }
if (!result) { if (!result) {
console.log(`[signal] ${target.name} — LLM returned null, skipping`); console.log(`[signal] ${company.name} @ ${checkpoint_date} — LLM returned null, skipping`);
await sleep(loopDelay); await sleep(loopDelay);
continue; continue;
} }
const predictionIds = predictions.map(p => p.id); const predictionIds = predictions.map(p => p.id);
const write = intelligenceDb.transaction(() => { insertSignal.run(
deleteSignal.run(target.id); company_id,
insertSignal.run( result.signal,
target.id, result.confidence,
result.signal, result.timeframe,
result.confidence, result.risk_level,
result.timeframe, JSON.stringify(result.risk_factors || []),
result.risk_level, result.summary,
JSON.stringify(result.risk_factors || []), JSON.stringify(result.key_drivers || []),
result.summary, JSON.stringify(predictionIds),
JSON.stringify(result.key_drivers || []), null,
JSON.stringify(predictionIds) checkpoint_date
); );
});
write();
recordEvent.run(); recordEvent.run();
pruneCounter++; pruneCounter++;
if (pruneCounter >= 20) { pruneEvents.run(); pruneCounter = 0; } if (pruneCounter >= 20) { pruneEvents.run(); pruneCounter = 0; }
console.log(`[signal] ${target.name} — ${result.signal} (${result.confidence} confidence, ${result.risk_level} risk)`); console.log(`[signal] ${company.name} @ ${checkpoint_date} — ${result.signal} (${result.confidence} confidence, ${result.risk_level} risk)`);
} catch (err) { } catch (err) {
console.error("[signal] cycle error:", err.message); console.error("[signal] cycle error:", err.message);
@ -148,7 +141,7 @@ async function runSignalWorker(archiveDb, intelligenceDb, config) {
} }
function buildPrompt(companyName, facts, relationships, predictions) { function buildPrompt(companyName, facts, relationships, predictions, asOf) {
const factsBlock = facts.length > 0 const factsBlock = facts.length > 0
? facts.map(f => `- ${f.claim} (confirmed ${f.confirmation_count}x)`).join("\n") ? facts.map(f => `- ${f.claim} (confirmed ${f.confirmation_count}x)`).join("\n")
: "No known facts yet."; : "No known facts yet.";
@ -161,7 +154,7 @@ function buildPrompt(companyName, facts, relationships, predictions) {
`${i + 1}. [${p.type}] ${p.direction} / ${p.magnitude} / ${p.timeframe} — ${p.rationale || "no rationale"}` `${i + 1}. [${p.type}] ${p.direction} / ${p.magnitude} / ${p.timeframe} — ${p.rationale || "no rationale"}`
).join("\n"); ).join("\n");
return `You are a financial intelligence analyst generating a trade signal for ${companyName}. return `You are a financial intelligence analyst generating a trade signal for ${companyName} as of ${asOf}.
Known facts about ${companyName} (most confirmed first): Known facts about ${companyName} (most confirmed first):
${factsBlock} ${factsBlock}
@ -169,7 +162,7 @@ ${factsBlock}
Known relationships: Known relationships:
${relBlock} ${relBlock}
Recent event predictions (last 90 days): Event predictions up to ${asOf}:
${predBlock} ${predBlock}
Generate a trade signal as JSON with this exact shape: Generate a trade signal as JSON with this exact shape: