add browser crawling capabilities and enhance configuration options
This commit is contained in:
+17
@@ -394,6 +394,7 @@
|
||||
"newsCrawler": {
|
||||
"maxPages": -1,
|
||||
"maxDepth": 10,
|
||||
"pageConcurrency": 4,
|
||||
"requestTimeout": 15000,
|
||||
"disabledLabels": [
|
||||
"Arab News",
|
||||
@@ -463,6 +464,7 @@
|
||||
"www.cnbc.com",
|
||||
"cnbc.com"
|
||||
],
|
||||
"renderMode": "browser",
|
||||
"seeds": [
|
||||
"https://www.cnbc.com/world/",
|
||||
"https://www.cnbc.com/business/",
|
||||
@@ -572,6 +574,7 @@
|
||||
"fortune.com",
|
||||
"www.fortune.com"
|
||||
],
|
||||
"renderMode": "browser",
|
||||
"seeds": [
|
||||
"https://fortune.com/",
|
||||
"https://fortune.com/section/tech/",
|
||||
@@ -583,11 +586,23 @@
|
||||
"www.forbes.com",
|
||||
"forbes.com"
|
||||
],
|
||||
"renderMode": "browser",
|
||||
"seeds": [
|
||||
"https://www.forbes.com/business/",
|
||||
"https://www.forbes.com/innovation/"
|
||||
]
|
||||
},
|
||||
"Financial Times": {
|
||||
"allowedHosts": [
|
||||
"www.ft.com",
|
||||
"ft.com"
|
||||
],
|
||||
"renderMode": "browser",
|
||||
"seeds": [
|
||||
"https://www.ft.com/world/us",
|
||||
"https://www.ft.com/technology"
|
||||
]
|
||||
},
|
||||
"Nikkei Asia": {
|
||||
"allowedHosts": [
|
||||
"asia.nikkei.com"
|
||||
@@ -635,6 +650,7 @@
|
||||
"www.wired.com",
|
||||
"wired.com"
|
||||
],
|
||||
"renderMode": "browser",
|
||||
"seeds": [
|
||||
"https://www.wired.com/category/business/",
|
||||
"https://www.wired.com/category/security/"
|
||||
@@ -644,6 +660,7 @@
|
||||
"allowedHosts": [
|
||||
"finance.yahoo.com"
|
||||
],
|
||||
"renderMode": "browser",
|
||||
"seeds": [
|
||||
"https://finance.yahoo.com/",
|
||||
"https://finance.yahoo.com/news/",
|
||||
|
||||
Reference in New Issue
Block a user