enhance news crawler configuration with new sources and improved request headers

This commit is contained in:
ImBenji
2026-04-16 23:32:56 +01:00
parent c91e4ddb60
commit 11647e6a35
6 changed files with 449 additions and 33 deletions
+26 -2
View File
@@ -44,8 +44,19 @@ const blockedContentDomains = [
];
const loggedBlockedDomains = new Set();
const articleFetchHeaders = {
Accept: 'text/html,application/xhtml+xml',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.9',
'Cache-Control': 'no-cache',
Pragma: 'no-cache',
'Upgrade-Insecure-Requests': '1',
'sec-ch-ua': '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"macOS"',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-User': '?1',
};
let contentBackfillRunning = false;
@@ -78,7 +89,20 @@ function getErrorMessage(error, fallback) {
}
function markArticleStatus(statement, id, message) {
statement.run(message, new Date().toISOString(), id);
const attemptedAt = new Date().toISOString();
const parameterCount = statement.source.split('?').length - 1;
if (parameterCount === 3) {
statement.run(message, attemptedAt, id);
return;
}
if (parameterCount === 2) {
statement.run(attemptedAt, id);
return;
}
throw new Error(`Unexpected content status statement parameter count: ${parameterCount}`);
}
async function fetchCompressedImage(url) {