add Google News integration and enhance crawler capabilities
This commit is contained in:
+1
-2
@@ -42,8 +42,7 @@
|
||||
"googleNews": "0 * * * *"
|
||||
},
|
||||
"contentBackfill": {
|
||||
"cron": "0 * * * *",
|
||||
"batchSize": -1
|
||||
"concurrency": 10
|
||||
},
|
||||
"googleNews": {
|
||||
"queries": [
|
||||
|
||||
+2
-1
@@ -97,7 +97,8 @@ function startScheduler() {
|
||||
}
|
||||
|
||||
try {
|
||||
await backfillMissingContent();
|
||||
const concurrency = Number(config.contentBackfill?.concurrency) || 5;
|
||||
await backfillMissingContent(-1, concurrency);
|
||||
} catch (error) {
|
||||
console.error('content backfill failed:', error);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user