refactor scheduler to streamline source execution and improve maintenance

server.js

@@ -2,7 +2,7 @@ const Fastify = require('fastify');
 const articleRoutes = require('./src/routes/articles');
 const statusRoutes = require('./src/routes/status');
 const config = require('./src/config');
-const { startScheduler, runAllIngestions } = require('./src/scheduler');
+const { startScheduler } = require('./src/scheduler');
 
 const app = Fastify({ logger: true });
 
@@ -14,10 +14,6 @@ app.get('/', async () => ({ ok: true }));
 async function start() {
   await app.listen({ port: config.server.port, host: config.server.host });
 
-  runAllIngestions().catch((error) => {
-    app.log.error(error);
-  });
-
   startScheduler();
 }
 

src/scheduler.js

@@ -56,33 +56,27 @@ async function runAllIngestions() {
 }
 
 function startScheduler() {
-  cron.schedule(config.scheduler.rss, async () => {
+  const runRss = async () => {
     await runSource('rss', fetchRssArticles);
-  });
+  };
 
-  cron.schedule(config.scheduler.gdelt, async () => {
+  const runGdelt = async () => {
     await runSource('gdelt', fetchGdeltArticles);
-  });
+  };
 
-  cron.schedule(config.scheduler.edgar, async () => {
+  const runEdgar = async () => {
     await runSource('edgar', fetchEdgarArticles);
-  });
+  };
 
-  cron.schedule(config.scheduler.alphaVantage, async () => {
+  const runAlphaVantage = async () => {
     await runSource('alphavantage', fetchAlphaVantageArticles);
-  });
+  };
 
-  cron.schedule(config.scheduler.finnhub, async () => {
+  const runFinnhub = async () => {
     await runSource('finnhub', fetchFinnhubArticles);
-  });
+  };
 
-  if (config.scheduler.newsCrawler) {
+  const runContentMaintenance = async () => {
-    cron.schedule(config.scheduler.newsCrawler, async () => {
-      await runCrawlerSources();
-    });
-  }
-
-  cron.schedule(config.contentBackfill.cron, async () => {
     try {
       await backfillMissingContent();
     } catch (error) {
@@ -94,7 +88,27 @@ function startScheduler() {
     } catch (error) {
       console.error('embedding backfill failed:', error);
     }
-  });
+  };
+
+  runRss();
+  runGdelt();
+  runEdgar();
+  runAlphaVantage();
+  runFinnhub();
+  runContentMaintenance();
+
+  cron.schedule(config.scheduler.rss, runRss);
+  cron.schedule(config.scheduler.gdelt, runGdelt);
+  cron.schedule(config.scheduler.edgar, runEdgar);
+  cron.schedule(config.scheduler.alphaVantage, runAlphaVantage);
+  cron.schedule(config.scheduler.finnhub, runFinnhub);
+
+  if (config.scheduler.newsCrawler) {
+    runCrawlerSources();
+    cron.schedule(config.scheduler.newsCrawler, runCrawlerSources);
+  }
+
+  cron.schedule(config.contentBackfill.cron, runContentMaintenance);
 }
 
 module.exports = {