add Google News integration and enhance crawler capabilities
This commit is contained in:
@@ -95,7 +95,7 @@ async function fetchWindow(source, window) {
|
|||||||
title,
|
title,
|
||||||
description: item.domain || source.label,
|
description: item.domain || source.label,
|
||||||
url,
|
url,
|
||||||
source: 'gdelt',
|
source: `gdelt:${source.label}`,
|
||||||
pubDate: item.seendate || null,
|
pubDate: item.seendate || null,
|
||||||
};
|
};
|
||||||
}).filter(Boolean);
|
}).filter(Boolean);
|
||||||
@@ -138,7 +138,7 @@ async function fetchWindowBigQuery(source, window, bigquery) {
|
|||||||
title: url,
|
title: url,
|
||||||
description: row.domain || source.label,
|
description: row.domain || source.label,
|
||||||
url,
|
url,
|
||||||
source: 'gdelt',
|
source: `gdelt:${source.label}`,
|
||||||
pubDate: row.seendate || null,
|
pubDate: row.seendate || null,
|
||||||
};
|
};
|
||||||
}).filter(Boolean);
|
}).filter(Boolean);
|
||||||
|
|||||||
Reference in New Issue
Block a user