enhance news crawler configuration with new sources and improved request headers

src/sources/newsCrawler.js

@@ -19,6 +19,8 @@ const TRACKING_PARAM_PATTERNS = [
 const LISTING_PATH_HINT = /(archive|archives|latest|topic|topics|section|sections|category|categories|news|world|business|politics|technology|tech|markets|economy|page|tag|tags)/i;
 const ARTICLE_DATE_PATH = /\/\d{4}\/\d{2}\/\d{2}(?:\/|$)|\/\d{4}\/\d{2}(?:\/|$)/;
 const ARTICLE_PATH_HINT = /(\/article\/|\/articles\/|\/news\/|\/story\/|\/stories\/)/i;
+const ARTICLE_PATH_STRONG_HINT = /\/\d{4}\/\d{2}\/\d{2}\//;
+const LISTING_ARTICLE_FALSE_POSITIVE_PATH = /(\/category\/|\/tag\/|\/latest(?:\/|$)|\/topics?(?:\/|$)|\/sections?(?:\/|$))/i;
 const BLOCKED_PATH_HINT = /(\/search(?:\/|$)|\/login(?:\/|$)|\/account(?:\/|$)|\/video(?:\/|$)|\/videos(?:\/|$)|\/podcast(?:\/|$)|\/podcasts(?:\/|$)|\/live(?:\/|$))/i;
 
 function decodeHtmlEntities(value) {
@@ -267,30 +269,36 @@ function selectPubDate(meta, jsonLdArticle, html) {
 function scorePage(pageUrl, meta, html, jsonLdArticle, links) {
   let articleScore = 0;
   let listingScore = 0;
+  const pathname = new URL(pageUrl).pathname;
+  const hasArticleDatePath = ARTICLE_DATE_PATH.test(pageUrl);
+  const hasArticlePathHint = ARTICLE_PATH_HINT.test(pageUrl);
+  const hasStrongArticlePath = ARTICLE_PATH_STRONG_HINT.test(pathname);
+  const hasListingFalsePositivePath = LISTING_ARTICLE_FALSE_POSITIVE_PATH.test(pathname);
+  const paragraphTextLength = extractParagraphTextLength(html);
   const headlineLinks = links.filter(({ text }) => text.length >= 25 && text.length <= 180).length;
 
   if (jsonLdArticle) {
-    articleScore += 3;
+    articleScore += 4;
   }
 
-  if (String(meta.get('og:type') || '').toLowerCase() === 'article') {
-    articleScore += 2;
+  if (String(meta.get('og:type') || '').toLowerCase() === 'article' && !hasListingFalsePositivePath) {
+    articleScore += 1;
   }
 
-  if (meta.get('article:published_time') || meta.get('og:article:published_time') || extractTimeDatetime(html)) {
-    articleScore += 2;
+  if ((meta.get('article:published_time') || meta.get('og:article:published_time') || extractTimeDatetime(html)) && !hasListingFalsePositivePath) {
+    articleScore += 1;
   }
 
   if (/<article\b/i.test(html)) {
     articleScore += 1;
   }
 
-  if (ARTICLE_DATE_PATH.test(pageUrl) || ARTICLE_PATH_HINT.test(pageUrl)) {
-    articleScore += 1;
+  if (hasArticleDatePath || hasArticlePathHint) {
+    articleScore += 2;
   }
 
-  if (extractH1(html) && extractParagraphTextLength(html) >= 500) {
-    articleScore += 1;
+  if (extractH1(html) && paragraphTextLength >= 500) {
+    articleScore += 2;
   }
 
   if (links.length >= 20) {
@@ -301,15 +309,23 @@ function scorePage(pageUrl, meta, html, jsonLdArticle, links) {
     listingScore += 2;
   }
 
-  if (LISTING_PATH_HINT.test(new URL(pageUrl).pathname)) {
+  if (LISTING_PATH_HINT.test(pathname)) {
     listingScore += 1;
   }
 
-  if (articleScore > 0) {
-    listingScore -= 2;
+  if (hasListingFalsePositivePath) {
+    listingScore += 3;
   }
 
-  return { articleScore, listingScore };
+  if (articleScore > 0) {
+    listingScore -= 1;
+  }
+
+  const isArticleCandidate = articleScore >= 4
+    && articleScore > listingScore
+    && (Boolean(jsonLdArticle) || hasStrongArticlePath || hasArticlePathHint || paragraphTextLength >= 500);
+
+  return { articleScore, listingScore, isArticleCandidate };
 }
 
 function shouldQueueLink(url) {
@@ -396,6 +412,20 @@ function buildDefaultSeeds(feedUrl) {
   }
 }
 
+function normalizeLimit(value, fallback, minimum, maximum) {
+  const numeric = Number(value);
+
+  if (numeric === -1) {
+    return Number.POSITIVE_INFINITY;
+  }
+
+  if (!Number.isFinite(numeric)) {
+    return fallback;
+  }
+
+  return Math.max(minimum, Math.min(numeric, maximum));
+}
+
 function normalizeSite(site) {
   const allowedHosts = unique((site.allowedHosts || []).map((host) => String(host || '').toLowerCase()).filter(Boolean));
   const seeds = unique((site.seeds || [])
@@ -407,8 +437,8 @@ function normalizeSite(site) {
     label: String(site.label || '').trim(),
     allowedHosts,
     seeds,
-    maxPages: Math.max(1, Math.min(Number(site.maxPages) || 15, 500)),
-    maxDepth: Math.max(0, Math.min(Number(site.maxDepth) || 1, 5)),
+    maxPages: normalizeLimit(site.maxPages, 15, 1, 500),
+    maxDepth: normalizeLimit(site.maxDepth, 1, 0, 5),
     requestTimeout: Math.max(1000, Math.min(Number(site.requestTimeout) || 15000, 30000)),
   };
 }
@@ -515,9 +545,9 @@ async function crawlSite(site) {
       ? canonicalizeUrl(canonicalHref, current.url, normalizedSite.allowedHosts) || current.url
       : current.url;
     const links = extractLinks(html, canonicalUrl, normalizedSite.allowedHosts);
-    const { articleScore, listingScore } = scorePage(canonicalUrl, meta, html, jsonLdArticle, links);
+    const { listingScore, isArticleCandidate } = scorePage(canonicalUrl, meta, html, jsonLdArticle, links);
 
-    if (articleScore >= 3 && !discoveredArticleUrls.has(canonicalUrl)) {
+    if (isArticleCandidate && !discoveredArticleUrls.has(canonicalUrl)) {
       const title = normalizeText(selectTitle(meta, jsonLdArticle, html));
       if (title) {
         discoveredArticleUrls.add(canonicalUrl);