add Google News integration and enhance crawler capabilities
This commit is contained in:
+1
-2
@@ -42,8 +42,7 @@
|
|||||||
"googleNews": "0 * * * *"
|
"googleNews": "0 * * * *"
|
||||||
},
|
},
|
||||||
"contentBackfill": {
|
"contentBackfill": {
|
||||||
"cron": "0 * * * *",
|
"concurrency": 10
|
||||||
"batchSize": -1
|
|
||||||
},
|
},
|
||||||
"googleNews": {
|
"googleNews": {
|
||||||
"queries": [
|
"queries": [
|
||||||
|
|||||||
+2
-1
@@ -97,7 +97,8 @@ function startScheduler() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await backfillMissingContent();
|
const concurrency = Number(config.contentBackfill?.concurrency) || 5;
|
||||||
|
await backfillMissingContent(-1, concurrency);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('content backfill failed:', error);
|
console.error('content backfill failed:', error);
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user