add Google News integration and enhance crawler capabilities

src/sources/gdelt.js

@@ -114,14 +114,19 @@ async function fetchWindowBigQuery(source, window, bigquery) {
   const maxRecords = Math.max(1, Math.min(Number(config.gdelt?.maxRecords) || 100, 1000));
   const domainClauses = source.website.map((d) => `LOWER(DocumentIdentifier) LIKE '%${d}%'`).join(' OR ');
 
+  // use _PARTITIONTIME (not the INTEGER DATE column) so BigQuery can prune partitions
+  // and avoid full-table scans on the ~130TB gkg table
+  const startTs = window.start.toISOString();
+  const endTs = window.end.toISOString();
+
   const query = `
     SELECT
       DocumentIdentifier AS url,
       SourceCommonName AS domain,
       CAST(DATE AS STRING) AS seendate
     FROM \`gdelt-bq.gdeltv2.gkg\`
-    WHERE DATE >= ${window.startKey}
-      AND DATE < ${window.endKey}
+    WHERE _PARTITIONTIME >= TIMESTAMP("${startTs}")
+      AND _PARTITIONTIME < TIMESTAMP("${endTs}")
       AND (${domainClauses})
     LIMIT ${maxRecords}
   `;