refactor: update worker commands and add new scripts for API rebuilding and queue feeding

config.json

@@ -13,30 +13,30 @@
   },
   "admin": {
     "username": "admin",
-    "password": "changeme"
+    "password": ""
   },
   "database": {
     "type": "sqlite",
     "path": "./archive.sqlite",
     "postgres": {
-      "connectionString": "postgresql://user:password@localhost:5432/duriin"
+      "connectionString": ""
     }
   },
   "sec": {
-    "userAgent": "Augor benjamin.watt@imbenji.net",
+    "userAgent": "",
     "tickers": []
   },
   "alphaVantage": {
-    "apiKey": "KJ68ZQEW0PF524UA",
+    "apiKey": "",
     "tickers": []
   },
   "finnhub": {
-    "apiKey": "d7gg0h1r01qmqj4573sgd7gg0h1r01qmqj4573t0",
+    "apiKey": "",
     "tickers": []
   },
   "openRouter": {
     "enabled": true,
-    "apiKey": "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",
+    "apiKey": "",
     "embeddingModel": "qwen/qwen3-embedding-8b",
     "llmModel": "qwen/qwen3-235b-a22b-2507"
   },
@@ -49,7 +49,7 @@
     "lookbackWeeks": 312,
     "requestDelayMs": 12000,
     "maxWindowsPerRun": 4,
-    "bigQueryProject": "duriin",
+    "bigQueryProject": "",
     "bigQueryKeyFile": "./gdelt-credentials.json"
   },
   "scheduler": {

docker-compose.yml

@@ -17,7 +17,7 @@ services:
     build:
       context: .
       provenance: false
-    command: node intelligence/index.js
+    command: node workers/index.js
     volumes:
       - ./config.json:/app/config.json:ro
       - ./data:/data

package-lock.json

@@ -15,6 +15,7 @@
         "@google-cloud/bigquery": "^8.1.1",
         "better-sqlite3": "^12.4.1",
         "deasync": "^0.1.31",
+        "dotenv": "^17.4.2",
         "fastify": "^5.6.1",
         "node-cron": "^4.2.1",
         "pg": "^8.20.0",
@@ -813,6 +814,18 @@
         "url": "https://github.com/fb55/domutils?sponsor=1"
       }
     },
+    "node_modules/dotenv": {
+      "version": "17.4.2",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.2.tgz",
+      "integrity": "sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==",
+      "license": "BSD-2-Clause",
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://dotenvx.com"
+      }
+    },
     "node_modules/duplexify": {
       "version": "4.1.3",
       "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz",

package.json

@@ -5,7 +5,7 @@
   "main": "server.js",
   "scripts": {
     "start": "node server.js",
-    "intelligence": "node intelligence/index.js"
+    "workers": "node workers/index.js"
   },
   "keywords": [],
   "author": "",
@@ -18,6 +18,7 @@
     "@google-cloud/bigquery": "^8.1.1",
     "better-sqlite3": "^12.4.1",
     "deasync": "^0.1.31",
+    "dotenv": "^17.4.2",
     "fastify": "^5.6.1",
     "node-cron": "^4.2.1",
     "pg": "^8.20.0",

src/config.js

@@ -1,7 +1,38 @@
 const fs = require('fs');
 const path = require('path');
 
+require('dotenv').config({ path: path.join(__dirname, '..', '.env') });
+
 const configPath = path.join(__dirname, '..', 'config.json');
 const config = JSON.parse(fs.readFileSync(configPath, 'utf8'));
 
+// overlay env vars over the nested config object
+// env vars take precedence over config.json values
+
+if (process.env.ADMIN_USERNAME)  config.admin.username = process.env.ADMIN_USERNAME;
+if (process.env.ADMIN_PASSWORD)  config.admin.password = process.env.ADMIN_PASSWORD;
+
+if (process.env.DATABASE_PATH) config.database.path = process.env.DATABASE_PATH;
+if (process.env.DATABASE_URL) {
+  config.database.postgres = { ...config.database.postgres, connectionString: process.env.DATABASE_URL };
+  config.database.type = 'postgres';
+} else if (process.env.DATABASE_TYPE) {
+  config.database.type = process.env.DATABASE_TYPE;
+}
+
+if (process.env.SEC_USER_AGENT) config.sec.userAgent = process.env.SEC_USER_AGENT;
+
+if (process.env.ALPHA_VANTAGE_API_KEY) config.alphaVantage.apiKey = process.env.ALPHA_VANTAGE_API_KEY;
+if (process.env.FINNHUB_API_KEY)       config.finnhub.apiKey = process.env.FINNHUB_API_KEY;
+
+if (process.env.OPEN_ROUTER_API_KEY)      config.openRouter.apiKey = process.env.OPEN_ROUTER_API_KEY;
+if (process.env.OPEN_ROUTER_LLM_MODEL)    config.openRouter.llmModel = process.env.OPEN_ROUTER_LLM_MODEL;
+if (process.env.OPEN_ROUTER_EMBED_MODEL)  config.openRouter.embeddingModel = process.env.OPEN_ROUTER_EMBED_MODEL;
+
+if (process.env.GDELT_BQ_PROJECT)   config.gdelt.bigQueryProject = process.env.GDELT_BQ_PROJECT;
+if (process.env.GDELT_BQ_KEY_FILE)  config.gdelt.bigQueryKeyFile = process.env.GDELT_BQ_KEY_FILE;
+
+if (process.env.SERVER_PORT) config.server.port = Number(process.env.SERVER_PORT);
+if (process.env.SERVER_HOST) config.server.host = process.env.SERVER_HOST;
+
 module.exports = config;

src/db/index.js

@@ -1,7 +1,7 @@
-const config = require('./config');
+const config = require('../config');
 
 if (config.database && config.database.type === 'postgres') {
-  module.exports = require('./db-pg');
+  module.exports = require('./postgres');
   return;
 }
 
@@ -9,7 +9,7 @@ const path = require('path');
 const Database = require('better-sqlite3');
 const sqliteVec = require('sqlite-vec');
 
-const dbPath = path.resolve(__dirname, '..', config.database.path || './archive.sqlite');
+const dbPath = path.resolve(__dirname, '../..', config.database.path || './archive.sqlite');
 const db = new Database(dbPath);
 sqliteVec.load(db);
 

src/db/postgres.js

@@ -4,7 +4,7 @@
 
 const { Pool } = require('pg');
 const deasync = require('deasync');
-const config = require('./config');
+const config = require('../config');
 
 const pool = new Pool(config.database.postgres);