refactor: load environment variables from .env file and update openRouter configuration

workers/signalWorker.js

@@ -1,9 +1,10 @@
 const https = require("https");
 const http = require("http");
 
+const CONCURRENCY = 4;
+
 
 async function runSignalWorker(archiveDb, intelligenceDb, config) {
-  const loopDelay = config.workers?.signalLoopDelayMs ?? 120000;
   const llmConfig = config.openRouter || {};
 
   // add as_of column if it doesnt exist yet
@@ -70,11 +71,37 @@ async function runSignalWorker(archiveDb, intelligenceDb, config) {
     `DELETE FROM worker_events WHERE worker = 'signal' AND completed_at < datetime('now', '-1 hour')`
   );
 
+  // in-process claim set — prevents concurrent workers from grabbing same checkpoint
+  const inFlight = new Set();
   let pruneCounter = 0;
 
+  async function workerLoop(id) {
     while (true) {
       try {
-      const next = getNextCheckpoint.get();
+        // find next checkpoint not already claimed or done
+        let next = null;
+
+        // keep scanning until we find one not in-flight
+        const candidates = intelligenceDb.prepare(`
+          SELECT company_id, substr(event_date, 1, 10) as checkpoint_date
+          FROM event_predictions
+          WHERE substr(event_date, 1, 10) NOT IN (
+            SELECT as_of FROM trade_signals WHERE as_of IS NOT NULL AND company_id = event_predictions.company_id
+          )
+          GROUP BY company_id, substr(event_date, 1, 10)
+          HAVING COUNT(*) >= 3
+          ORDER BY checkpoint_date DESC
+          LIMIT 20
+        `).all();
+
+        for (const c of candidates) {
+          const key = `${c.company_id}|${c.checkpoint_date}`;
+          if (!inFlight.has(key)) {
+            next = c;
+            inFlight.add(key);
+            break;
+          }
+        }
 
         if (!next) {
           await sleep(5000);
@@ -82,9 +109,12 @@ async function runSignalWorker(archiveDb, intelligenceDb, config) {
         }
 
         const { company_id, checkpoint_date } = next;
+        const key = `${company_id}|${checkpoint_date}`;
+
         const company = getCompanyById.get(company_id);
 
         if (!company) {
+          inFlight.delete(key);
           continue;
         }
 
@@ -98,12 +128,14 @@ async function runSignalWorker(archiveDb, intelligenceDb, config) {
         try {
           result = await callLlm(llmConfig, prompt);
         } catch (err) {
-        console.error(`[signal] LLM error for ${company.name} @ ${checkpoint_date}:`, err.message);
+          console.error(`[signal:${id}] LLM error for ${company.name} @ ${checkpoint_date}:`, err.message);
+          inFlight.delete(key);
           continue;
         }
 
         if (!result) {
-        console.log(`[signal] ${company.name} @ ${checkpoint_date} — LLM returned null, skipping`);
+          console.log(`[signal:${id}] ${company.name} @ ${checkpoint_date} — LLM returned null, skipping`);
+          inFlight.delete(key);
           continue;
         }
 
@@ -123,16 +155,27 @@ async function runSignalWorker(archiveDb, intelligenceDb, config) {
           checkpoint_date
         );
 
+        inFlight.delete(key);
+
         recordEvent.run();
         pruneCounter++;
         if (pruneCounter >= 20) { pruneEvents.run(); pruneCounter = 0; }
 
-      console.log(`[signal] ${company.name} @ ${checkpoint_date} — ${result.signal} (${result.confidence} confidence, ${result.risk_level} risk)`);
+        console.log(`[signal:${id}] ${company.name} @ ${checkpoint_date} — ${result.signal} (${result.confidence} confidence, ${result.risk_level} risk)`);
 
       } catch (err) {
-      console.error("[signal] cycle error:", err.message);
+        console.error(`[signal:${id}] cycle error:`, err.message);
       }
     }
+  }
+
+  // spin up CONCURRENCY workers
+  const workers = [];
+  for (let i = 0; i < CONCURRENCY; i++) {
+    workers.push(workerLoop(i + 1));
+  }
+
+  await Promise.all(workers);
 }