diff --git a/server.js b/server.js index a930480..04862c7 100644 --- a/server.js +++ b/server.js @@ -2,7 +2,7 @@ const Fastify = require('fastify'); const articleRoutes = require('./src/routes/articles'); const statusRoutes = require('./src/routes/status'); const config = require('./src/config'); -const { startScheduler, runAllIngestions } = require('./src/scheduler'); +const { startScheduler } = require('./src/scheduler'); const app = Fastify({ logger: true }); @@ -14,10 +14,6 @@ app.get('/', async () => ({ ok: true })); async function start() { await app.listen({ port: config.server.port, host: config.server.host }); - runAllIngestions().catch((error) => { - app.log.error(error); - }); - startScheduler(); } diff --git a/src/scheduler.js b/src/scheduler.js index 2dfc6e6..2c39c9a 100644 --- a/src/scheduler.js +++ b/src/scheduler.js @@ -56,33 +56,27 @@ async function runAllIngestions() { } function startScheduler() { - cron.schedule(config.scheduler.rss, async () => { + const runRss = async () => { await runSource('rss', fetchRssArticles); - }); + }; - cron.schedule(config.scheduler.gdelt, async () => { + const runGdelt = async () => { await runSource('gdelt', fetchGdeltArticles); - }); + }; - cron.schedule(config.scheduler.edgar, async () => { + const runEdgar = async () => { await runSource('edgar', fetchEdgarArticles); - }); + }; - cron.schedule(config.scheduler.alphaVantage, async () => { + const runAlphaVantage = async () => { await runSource('alphavantage', fetchAlphaVantageArticles); - }); + }; - cron.schedule(config.scheduler.finnhub, async () => { + const runFinnhub = async () => { await runSource('finnhub', fetchFinnhubArticles); - }); + }; - if (config.scheduler.newsCrawler) { - cron.schedule(config.scheduler.newsCrawler, async () => { - await runCrawlerSources(); - }); - } - - cron.schedule(config.contentBackfill.cron, async () => { + const runContentMaintenance = async () => { try { await backfillMissingContent(); } catch (error) { @@ -94,7 +88,27 @@ function startScheduler() { } catch (error) { console.error('embedding backfill failed:', error); } - }); + }; + + runRss(); + runGdelt(); + runEdgar(); + runAlphaVantage(); + runFinnhub(); + runContentMaintenance(); + + cron.schedule(config.scheduler.rss, runRss); + cron.schedule(config.scheduler.gdelt, runGdelt); + cron.schedule(config.scheduler.edgar, runEdgar); + cron.schedule(config.scheduler.alphaVantage, runAlphaVantage); + cron.schedule(config.scheduler.finnhub, runFinnhub); + + if (config.scheduler.newsCrawler) { + runCrawlerSources(); + cron.schedule(config.scheduler.newsCrawler, runCrawlerSources); + } + + cron.schedule(config.contentBackfill.cron, runContentMaintenance); } module.exports = {