add browser crawling capabilities and enhance configuration options

This commit is contained in:
ImBenji
2026-04-17 16:53:18 +01:00
parent c6bbe2a061
commit 1a8504389a
6 changed files with 298 additions and 49 deletions
+17
View File
@@ -394,6 +394,7 @@
"newsCrawler": {
"maxPages": -1,
"maxDepth": 10,
"pageConcurrency": 4,
"requestTimeout": 15000,
"disabledLabels": [
"Arab News",
@@ -463,6 +464,7 @@
"www.cnbc.com",
"cnbc.com"
],
"renderMode": "browser",
"seeds": [
"https://www.cnbc.com/world/",
"https://www.cnbc.com/business/",
@@ -572,6 +574,7 @@
"fortune.com",
"www.fortune.com"
],
"renderMode": "browser",
"seeds": [
"https://fortune.com/",
"https://fortune.com/section/tech/",
@@ -583,11 +586,23 @@
"www.forbes.com",
"forbes.com"
],
"renderMode": "browser",
"seeds": [
"https://www.forbes.com/business/",
"https://www.forbes.com/innovation/"
]
},
"Financial Times": {
"allowedHosts": [
"www.ft.com",
"ft.com"
],
"renderMode": "browser",
"seeds": [
"https://www.ft.com/world/us",
"https://www.ft.com/technology"
]
},
"Nikkei Asia": {
"allowedHosts": [
"asia.nikkei.com"
@@ -635,6 +650,7 @@
"www.wired.com",
"wired.com"
],
"renderMode": "browser",
"seeds": [
"https://www.wired.com/category/business/",
"https://www.wired.com/category/security/"
@@ -644,6 +660,7 @@
"allowedHosts": [
"finance.yahoo.com"
],
"renderMode": "browser",
"seeds": [
"https://finance.yahoo.com/",
"https://finance.yahoo.com/news/",