From 04966fac55550c3bcee4b7ac2fe0585fecd55bef Mon Sep 17 00:00:00 2001 From: ImBenji Date: Mon, 27 Apr 2026 15:03:15 +0100 Subject: [PATCH] refactor: update worker commands and add new scripts for API rebuilding and queue feeding --- backtest.py => backtest/backtest.py | 0 backtest_full.py => backtest/backtest_full.py | 0 .../backtest_results.csv | 0 .../backtest_results_full.csv | 0 config.json | 14 ++++----- docker-compose.yml | 2 +- summary-prompt.md => docs/summary-prompt.md | 0 package-lock.json | 13 ++++++++ package.json | 3 +- rebuild-api.sh => scripts/rebuild-api.sh | 0 src/config.js | 31 +++++++++++++++++++ src/{db.js => db/index.js} | 6 ++-- src/{db-pg.js => db/postgres.js} | 2 +- {intelligence => workers}/augorWorker.js | 0 .../consolidationWorker.js | 0 {intelligence => workers}/db.js | 0 {intelligence => workers}/embeddings.js | 0 {intelligence => workers}/graphWorker.js | 0 {intelligence => workers}/index.js | 0 {intelligence => workers}/queueFeeder.js | 0 {intelligence => workers}/signalWorker.js | 0 21 files changed, 58 insertions(+), 13 deletions(-) rename backtest.py => backtest/backtest.py (100%) rename backtest_full.py => backtest/backtest_full.py (100%) rename backtest_results.csv => backtest/backtest_results.csv (100%) rename backtest_results_full.csv => backtest/backtest_results_full.csv (100%) rename summary-prompt.md => docs/summary-prompt.md (100%) rename rebuild-api.sh => scripts/rebuild-api.sh (100%) rename src/{db.js => db/index.js} (96%) rename src/{db-pg.js => db/postgres.js} (99%) rename {intelligence => workers}/augorWorker.js (100%) rename {intelligence => workers}/consolidationWorker.js (100%) rename {intelligence => workers}/db.js (100%) rename {intelligence => workers}/embeddings.js (100%) rename {intelligence => workers}/graphWorker.js (100%) rename {intelligence => workers}/index.js (100%) rename {intelligence => workers}/queueFeeder.js (100%) rename {intelligence => workers}/signalWorker.js (100%) diff --git a/backtest.py b/backtest/backtest.py similarity index 100% rename from backtest.py rename to backtest/backtest.py diff --git a/backtest_full.py b/backtest/backtest_full.py similarity index 100% rename from backtest_full.py rename to backtest/backtest_full.py diff --git a/backtest_results.csv b/backtest/backtest_results.csv similarity index 100% rename from backtest_results.csv rename to backtest/backtest_results.csv diff --git a/backtest_results_full.csv b/backtest/backtest_results_full.csv similarity index 100% rename from backtest_results_full.csv rename to backtest/backtest_results_full.csv diff --git a/config.json b/config.json index 06263de..34c0b0f 100644 --- a/config.json +++ b/config.json @@ -13,30 +13,30 @@ }, "admin": { "username": "admin", - "password": "changeme" + "password": "" }, "database": { "type": "sqlite", "path": "./archive.sqlite", "postgres": { - "connectionString": "postgresql://user:password@localhost:5432/duriin" + "connectionString": "" } }, "sec": { - "userAgent": "Augor benjamin.watt@imbenji.net", + "userAgent": "", "tickers": [] }, "alphaVantage": { - "apiKey": "KJ68ZQEW0PF524UA", + "apiKey": "", "tickers": [] }, "finnhub": { - "apiKey": "d7gg0h1r01qmqj4573sgd7gg0h1r01qmqj4573t0", + "apiKey": "", "tickers": [] }, "openRouter": { "enabled": true, - "apiKey": "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", + "apiKey": "", "embeddingModel": "qwen/qwen3-embedding-8b", "llmModel": "qwen/qwen3-235b-a22b-2507" }, @@ -49,7 +49,7 @@ "lookbackWeeks": 312, "requestDelayMs": 12000, "maxWindowsPerRun": 4, - "bigQueryProject": "duriin", + "bigQueryProject": "", "bigQueryKeyFile": "./gdelt-credentials.json" }, "scheduler": { diff --git a/docker-compose.yml b/docker-compose.yml index a2d3b29..0adda9a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,7 +17,7 @@ services: build: context: . provenance: false - command: node intelligence/index.js + command: node workers/index.js volumes: - ./config.json:/app/config.json:ro - ./data:/data diff --git a/summary-prompt.md b/docs/summary-prompt.md similarity index 100% rename from summary-prompt.md rename to docs/summary-prompt.md diff --git a/package-lock.json b/package-lock.json index 117fe1d..0b27720 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "@google-cloud/bigquery": "^8.1.1", "better-sqlite3": "^12.4.1", "deasync": "^0.1.31", + "dotenv": "^17.4.2", "fastify": "^5.6.1", "node-cron": "^4.2.1", "pg": "^8.20.0", @@ -813,6 +814,18 @@ "url": "https://github.com/fb55/domutils?sponsor=1" } }, + "node_modules/dotenv": { + "version": "17.4.2", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.2.tgz", + "integrity": "sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, "node_modules/duplexify": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz", diff --git a/package.json b/package.json index 83de665..df68bdb 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "server.js", "scripts": { "start": "node server.js", - "intelligence": "node intelligence/index.js" + "workers": "node workers/index.js" }, "keywords": [], "author": "", @@ -18,6 +18,7 @@ "@google-cloud/bigquery": "^8.1.1", "better-sqlite3": "^12.4.1", "deasync": "^0.1.31", + "dotenv": "^17.4.2", "fastify": "^5.6.1", "node-cron": "^4.2.1", "pg": "^8.20.0", diff --git a/rebuild-api.sh b/scripts/rebuild-api.sh similarity index 100% rename from rebuild-api.sh rename to scripts/rebuild-api.sh diff --git a/src/config.js b/src/config.js index a7c9689..d7daee4 100644 --- a/src/config.js +++ b/src/config.js @@ -1,7 +1,38 @@ const fs = require('fs'); const path = require('path'); +require('dotenv').config({ path: path.join(__dirname, '..', '.env') }); + const configPath = path.join(__dirname, '..', 'config.json'); const config = JSON.parse(fs.readFileSync(configPath, 'utf8')); +// overlay env vars over the nested config object +// env vars take precedence over config.json values + +if (process.env.ADMIN_USERNAME) config.admin.username = process.env.ADMIN_USERNAME; +if (process.env.ADMIN_PASSWORD) config.admin.password = process.env.ADMIN_PASSWORD; + +if (process.env.DATABASE_PATH) config.database.path = process.env.DATABASE_PATH; +if (process.env.DATABASE_URL) { + config.database.postgres = { ...config.database.postgres, connectionString: process.env.DATABASE_URL }; + config.database.type = 'postgres'; +} else if (process.env.DATABASE_TYPE) { + config.database.type = process.env.DATABASE_TYPE; +} + +if (process.env.SEC_USER_AGENT) config.sec.userAgent = process.env.SEC_USER_AGENT; + +if (process.env.ALPHA_VANTAGE_API_KEY) config.alphaVantage.apiKey = process.env.ALPHA_VANTAGE_API_KEY; +if (process.env.FINNHUB_API_KEY) config.finnhub.apiKey = process.env.FINNHUB_API_KEY; + +if (process.env.OPEN_ROUTER_API_KEY) config.openRouter.apiKey = process.env.OPEN_ROUTER_API_KEY; +if (process.env.OPEN_ROUTER_LLM_MODEL) config.openRouter.llmModel = process.env.OPEN_ROUTER_LLM_MODEL; +if (process.env.OPEN_ROUTER_EMBED_MODEL) config.openRouter.embeddingModel = process.env.OPEN_ROUTER_EMBED_MODEL; + +if (process.env.GDELT_BQ_PROJECT) config.gdelt.bigQueryProject = process.env.GDELT_BQ_PROJECT; +if (process.env.GDELT_BQ_KEY_FILE) config.gdelt.bigQueryKeyFile = process.env.GDELT_BQ_KEY_FILE; + +if (process.env.SERVER_PORT) config.server.port = Number(process.env.SERVER_PORT); +if (process.env.SERVER_HOST) config.server.host = process.env.SERVER_HOST; + module.exports = config; diff --git a/src/db.js b/src/db/index.js similarity index 96% rename from src/db.js rename to src/db/index.js index 9400479..7df2834 100644 --- a/src/db.js +++ b/src/db/index.js @@ -1,7 +1,7 @@ -const config = require('./config'); +const config = require('../config'); if (config.database && config.database.type === 'postgres') { - module.exports = require('./db-pg'); + module.exports = require('./postgres'); return; } @@ -9,7 +9,7 @@ const path = require('path'); const Database = require('better-sqlite3'); const sqliteVec = require('sqlite-vec'); -const dbPath = path.resolve(__dirname, '..', config.database.path || './archive.sqlite'); +const dbPath = path.resolve(__dirname, '../..', config.database.path || './archive.sqlite'); const db = new Database(dbPath); sqliteVec.load(db); diff --git a/src/db-pg.js b/src/db/postgres.js similarity index 99% rename from src/db-pg.js rename to src/db/postgres.js index 98bfb1c..cf70a69 100644 --- a/src/db-pg.js +++ b/src/db/postgres.js @@ -4,7 +4,7 @@ const { Pool } = require('pg'); const deasync = require('deasync'); -const config = require('./config'); +const config = require('../config'); const pool = new Pool(config.database.postgres); diff --git a/intelligence/augorWorker.js b/workers/augorWorker.js similarity index 100% rename from intelligence/augorWorker.js rename to workers/augorWorker.js diff --git a/intelligence/consolidationWorker.js b/workers/consolidationWorker.js similarity index 100% rename from intelligence/consolidationWorker.js rename to workers/consolidationWorker.js diff --git a/intelligence/db.js b/workers/db.js similarity index 100% rename from intelligence/db.js rename to workers/db.js diff --git a/intelligence/embeddings.js b/workers/embeddings.js similarity index 100% rename from intelligence/embeddings.js rename to workers/embeddings.js diff --git a/intelligence/graphWorker.js b/workers/graphWorker.js similarity index 100% rename from intelligence/graphWorker.js rename to workers/graphWorker.js diff --git a/intelligence/index.js b/workers/index.js similarity index 100% rename from intelligence/index.js rename to workers/index.js diff --git a/intelligence/queueFeeder.js b/workers/queueFeeder.js similarity index 100% rename from intelligence/queueFeeder.js rename to workers/queueFeeder.js diff --git a/intelligence/signalWorker.js b/workers/signalWorker.js similarity index 100% rename from intelligence/signalWorker.js rename to workers/signalWorker.js