migrate article embeddings to support multi-model architecture and enhance data integrity

src/sources/gdelt.js

@@ -203,8 +203,12 @@ async function fetchGdeltArticlesBigQuery(onWindow) {
   return allArticles;
 }
 
-async function fetchGdeltArticles() {
-  const articles = [];
+async function fetchGdeltArticles(onWindow) {
+  // when onWindow is provided we stream per-window and never hold the full set in memory.
+  // the legacy non-streaming caller still gets an array back, but only for tiny one-shot
+  // runs — anything that loops over many sources should be using the callback path
+  const articles = onWindow ? null : [];
+
   const windows = buildWeeklyWindows();
   const requestDelayMs = Math.max(0, Number(config.gdelt?.requestDelayMs) || 5500);
   const maxWindowsPerRun = Number(config.gdelt?.maxWindowsPerRun) || 0;
@@ -223,9 +227,16 @@ async function fetchGdeltArticles() {
 
       try {
         const windowArticles = await fetchWindow(source, window);
-        articles.push(...windowArticles);
         markWindowCompleted(source.id, window);
         windowsFetched += 1;
+
+        if (onWindow) {
+          if (windowArticles.length > 0) {
+            await onWindow(windowArticles);
+          }
+        } else {
+          articles.push(...windowArticles);
+        }
       } catch (error) {
         if (error && error.status === 429) {
           console.warn(`GDELT window rate-limited for ${source.id} ${window.startKey}-${window.endKey}`);
@@ -246,7 +257,7 @@ async function fetchGdeltArticles() {
     }
   }
 
-  return articles;
+  return articles || [];
 }
 
 function hasPendingWindows() {
@@ -266,7 +277,7 @@ function fetchGdeltArticlesRouted(onWindow) {
   if (source === 'bigquery') {
     return fetchGdeltArticlesBigQuery(onWindow);
   }
-  return fetchGdeltArticles();
+  return fetchGdeltArticles(onWindow);
 }
 
 module.exports = {