enhance news crawler configuration with new sources and improved request headers
This commit is contained in:
+26
-2
@@ -44,8 +44,19 @@ const blockedContentDomains = [
|
||||
];
|
||||
const loggedBlockedDomains = new Set();
|
||||
const articleFetchHeaders = {
|
||||
Accept: 'text/html,application/xhtml+xml',
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
|
||||
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'Cache-Control': 'no-cache',
|
||||
Pragma: 'no-cache',
|
||||
'Upgrade-Insecure-Requests': '1',
|
||||
'sec-ch-ua': '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"',
|
||||
'sec-ch-ua-mobile': '?0',
|
||||
'sec-ch-ua-platform': '"macOS"',
|
||||
'Sec-Fetch-Dest': 'document',
|
||||
'Sec-Fetch-Mode': 'navigate',
|
||||
'Sec-Fetch-Site': 'none',
|
||||
'Sec-Fetch-User': '?1',
|
||||
};
|
||||
|
||||
let contentBackfillRunning = false;
|
||||
@@ -78,7 +89,20 @@ function getErrorMessage(error, fallback) {
|
||||
}
|
||||
|
||||
function markArticleStatus(statement, id, message) {
|
||||
statement.run(message, new Date().toISOString(), id);
|
||||
const attemptedAt = new Date().toISOString();
|
||||
const parameterCount = statement.source.split('?').length - 1;
|
||||
|
||||
if (parameterCount === 3) {
|
||||
statement.run(message, attemptedAt, id);
|
||||
return;
|
||||
}
|
||||
|
||||
if (parameterCount === 2) {
|
||||
statement.run(attemptedAt, id);
|
||||
return;
|
||||
}
|
||||
|
||||
throw new Error(`Unexpected content status statement parameter count: ${parameterCount}`);
|
||||
}
|
||||
|
||||
async function fetchCompressedImage(url) {
|
||||
|
||||
Reference in New Issue
Block a user