enhance news crawler configuration with new sources and improved request headers
This commit is contained in:
ImBenji
+269
-13
@@ -365,6 +365,22 @@
|
||||
{
|
||||
"url": "https://elcomercio.pe/arc/outboundfeeds/rss/section/economia/",
|
||||
"label": "El Comercio Peru"
|
||||
},
|
||||
{
|
||||
"url": "https://jamaica-gleaner.com/feed/business.xml",
|
||||
"label": "Jamaica Gleaner"
|
||||
},
|
||||
{
|
||||
"url": "https://www.jamaicaobserver.com/app/business/",
|
||||
"label": "Jamaica Observer"
|
||||
},
|
||||
{
|
||||
"url": "https://www.stabroeknews.com/feed/",
|
||||
"label": "Stabroek News"
|
||||
},
|
||||
{
|
||||
"url": "https://nationnews.com/rss-feed/",
|
||||
"label": "Nation News Barbados"
|
||||
}
|
||||
],
|
||||
"gdelt": {
|
||||
@@ -376,24 +392,264 @@
|
||||
"format": "json"
|
||||
},
|
||||
"newsCrawler": {
|
||||
"sites": [
|
||||
{
|
||||
"name": "crawler_reuters",
|
||||
"maxPages": 15,
|
||||
"maxDepth": 1,
|
||||
"requestTimeout": 15000,
|
||||
"disabledLabels": [
|
||||
"Arab News",
|
||||
"Arabian Business",
|
||||
"Australian Fin Review",
|
||||
"BFM Business",
|
||||
"Business Daily Africa",
|
||||
"Business Standard IN",
|
||||
"BusinessLive SA",
|
||||
"Caixin Global",
|
||||
"Cinco Dias",
|
||||
"City A.M.",
|
||||
"El Comercio Peru",
|
||||
"El Economista ES",
|
||||
"El Economista MX",
|
||||
"FD.nl",
|
||||
"Gulf News Business",
|
||||
"Il Sole 24 Ore",
|
||||
"Infobae Economia AR",
|
||||
"Japan Times Business",
|
||||
"Korea JoongAng Daily",
|
||||
"Les Echos",
|
||||
"Live Mint",
|
||||
"Moneycontrol",
|
||||
"NZ Herald Business",
|
||||
"Portafolio Colombia",
|
||||
"Reuters",
|
||||
"The Star Malaysia",
|
||||
"This Is Money",
|
||||
"Xinhua Business"
|
||||
],
|
||||
"overrides": {
|
||||
"Al Jazeera": {
|
||||
"allowedHosts": [
|
||||
"www.reuters.com",
|
||||
"reuters.com"
|
||||
"www.aljazeera.com",
|
||||
"aljazeera.com"
|
||||
],
|
||||
"seeds": [
|
||||
"https://www.reuters.com/world/",
|
||||
"https://www.reuters.com/business/",
|
||||
"https://www.reuters.com/markets/",
|
||||
"https://www.reuters.com/technology/"
|
||||
"https://www.aljazeera.com/",
|
||||
"https://www.aljazeera.com/economy/",
|
||||
"https://www.aljazeera.com/tag/technology/"
|
||||
]
|
||||
},
|
||||
"Ars Technica": {
|
||||
"allowedHosts": [
|
||||
"arstechnica.com",
|
||||
"www.arstechnica.com"
|
||||
],
|
||||
"maxPages": 100,
|
||||
"maxDepth": 2,
|
||||
"requestTimeout": 15000
|
||||
"seeds": [
|
||||
"https://arstechnica.com/",
|
||||
"https://arstechnica.com/tech-policy/",
|
||||
"https://arstechnica.com/information-technology/"
|
||||
]
|
||||
},
|
||||
"BBC Business": {
|
||||
"allowedHosts": [
|
||||
"www.bbc.com",
|
||||
"bbc.com"
|
||||
],
|
||||
"seeds": [
|
||||
"https://www.bbc.com/news/business",
|
||||
"https://www.bbc.com/news/technology"
|
||||
]
|
||||
},
|
||||
"CNBC": {
|
||||
"allowedHosts": [
|
||||
"www.cnbc.com",
|
||||
"cnbc.com"
|
||||
],
|
||||
"seeds": [
|
||||
"https://www.cnbc.com/world/",
|
||||
"https://www.cnbc.com/business/",
|
||||
"https://www.cnbc.com/technology/"
|
||||
]
|
||||
},
|
||||
"Guardian Business": {
|
||||
"allowedHosts": [
|
||||
"www.theguardian.com",
|
||||
"theguardian.com"
|
||||
],
|
||||
"seeds": [
|
||||
"https://www.theguardian.com/",
|
||||
"https://www.theguardian.com/business",
|
||||
"https://www.theguardian.com/technology"
|
||||
]
|
||||
},
|
||||
"Jamaica Gleaner": {
|
||||
"allowedHosts": [
|
||||
"jamaica-gleaner.com",
|
||||
"www.jamaica-gleaner.com"
|
||||
],
|
||||
"seeds": [
|
||||
"https://jamaica-gleaner.com/",
|
||||
"https://jamaica-gleaner.com/news",
|
||||
"https://jamaica-gleaner.com/business"
|
||||
]
|
||||
},
|
||||
"Jamaica Observer": {
|
||||
"allowedHosts": [
|
||||
"www.jamaicaobserver.com",
|
||||
"jamaicaobserver.com"
|
||||
],
|
||||
"seeds": [
|
||||
"https://www.jamaicaobserver.com/",
|
||||
"https://www.jamaicaobserver.com/news/",
|
||||
"https://www.jamaicaobserver.com/business/"
|
||||
]
|
||||
},
|
||||
"Nation News Barbados": {
|
||||
"allowedHosts": [
|
||||
"nationnews.com",
|
||||
"www.nationnews.com"
|
||||
],
|
||||
"seeds": [
|
||||
"https://nationnews.com/",
|
||||
"https://nationnews.com/category/business/",
|
||||
"https://nationnews.com/category/news/"
|
||||
]
|
||||
},
|
||||
"NPR Business": {
|
||||
"allowedHosts": [
|
||||
"www.npr.org",
|
||||
"npr.org"
|
||||
],
|
||||
"seeds": [
|
||||
"https://www.npr.org/sections/business/",
|
||||
"https://www.npr.org/sections/technology/"
|
||||
]
|
||||
},
|
||||
"The Verge": {
|
||||
"allowedHosts": [
|
||||
"www.theverge.com",
|
||||
"theverge.com"
|
||||
],
|
||||
"seeds": [
|
||||
"https://www.theverge.com/tech",
|
||||
"https://www.theverge.com/business",
|
||||
"https://www.theverge.com/archives"
|
||||
]
|
||||
},
|
||||
"TechCrunch": {
|
||||
"allowedHosts": [
|
||||
"techcrunch.com",
|
||||
"www.techcrunch.com"
|
||||
],
|
||||
"seeds": [
|
||||
"https://techcrunch.com/",
|
||||
"https://techcrunch.com/category/startups/",
|
||||
"https://techcrunch.com/category/venture/"
|
||||
]
|
||||
},
|
||||
"The Economist": {
|
||||
"allowedHosts": [
|
||||
"www.economist.com",
|
||||
"economist.com"
|
||||
],
|
||||
"seeds": [
|
||||
"https://www.economist.com/finance-and-economics",
|
||||
"https://www.economist.com/business",
|
||||
"https://www.economist.com/science-and-technology"
|
||||
]
|
||||
},
|
||||
"Federal Reserve": {
|
||||
"allowedHosts": [
|
||||
"www.federalreserve.gov",
|
||||
"federalreserve.gov"
|
||||
],
|
||||
"seeds": [
|
||||
"https://www.federalreserve.gov/newsevents.htm",
|
||||
"https://www.federalreserve.gov/monetarypolicy.htm"
|
||||
]
|
||||
},
|
||||
"Fortune": {
|
||||
"allowedHosts": [
|
||||
"fortune.com",
|
||||
"www.fortune.com"
|
||||
],
|
||||
"seeds": [
|
||||
"https://fortune.com/",
|
||||
"https://fortune.com/section/tech/",
|
||||
"https://fortune.com/section/finance/"
|
||||
]
|
||||
},
|
||||
"Forbes Business": {
|
||||
"allowedHosts": [
|
||||
"www.forbes.com",
|
||||
"forbes.com"
|
||||
],
|
||||
"seeds": [
|
||||
"https://www.forbes.com/business/",
|
||||
"https://www.forbes.com/innovation/"
|
||||
]
|
||||
},
|
||||
"Nikkei Asia": {
|
||||
"allowedHosts": [
|
||||
"asia.nikkei.com"
|
||||
],
|
||||
"seeds": [
|
||||
"https://asia.nikkei.com/",
|
||||
"https://asia.nikkei.com/Business",
|
||||
"https://asia.nikkei.com/Technology"
|
||||
]
|
||||
},
|
||||
"South China Morning Post": {
|
||||
"allowedHosts": [
|
||||
"www.scmp.com",
|
||||
"scmp.com"
|
||||
],
|
||||
"seeds": [
|
||||
"https://www.scmp.com/",
|
||||
"https://www.scmp.com/business",
|
||||
"https://www.scmp.com/tech"
|
||||
]
|
||||
},
|
||||
"Stabroek News": {
|
||||
"allowedHosts": [
|
||||
"www.stabroeknews.com",
|
||||
"stabroeknews.com"
|
||||
],
|
||||
"seeds": [
|
||||
"https://www.stabroeknews.com/",
|
||||
"https://www.stabroeknews.com/category/business/",
|
||||
"https://www.stabroeknews.com/category/news/"
|
||||
]
|
||||
},
|
||||
"Wall Street Journal": {
|
||||
"allowedHosts": [
|
||||
"www.wsj.com",
|
||||
"wsj.com"
|
||||
],
|
||||
"seeds": [
|
||||
"https://www.wsj.com/news/business",
|
||||
"https://www.wsj.com/tech"
|
||||
]
|
||||
},
|
||||
"Wired Business": {
|
||||
"allowedHosts": [
|
||||
"www.wired.com",
|
||||
"wired.com"
|
||||
],
|
||||
"seeds": [
|
||||
"https://www.wired.com/category/business/",
|
||||
"https://www.wired.com/category/security/"
|
||||
]
|
||||
},
|
||||
"Yahoo Finance": {
|
||||
"allowedHosts": [
|
||||
"finance.yahoo.com"
|
||||
],
|
||||
"seeds": [
|
||||
"https://finance.yahoo.com/",
|
||||
"https://finance.yahoo.com/news/",
|
||||
"https://finance.yahoo.com/topic/tech/"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"scheduler": {
|
||||
"rss": "0 */6 * * *",
|
||||
|
||||
Reference in New Issue
Block a user