Duriin-API/src/contentValidation.js

// validates whether an extracted article is real content vs a soft-error page
// (cookie wall, cloudflare challenge, paywall, "enable javascript", etc).
//
// the rules are deliberately conservative. we'd rather let a few junk pages
// through (caught downstream when re-checked) than reject 5% of real articles.
// fingerprints are anchored to title or the first ~500 chars of body so an
// article that *mentions* cloudflare doesnt get falsely rejected.

const MIN_CONTENT_LENGTH = 400;
const MIN_SENTENCE_TERMINATORS = 3;
const BODY_SNIFF_LENGTH = 800;


// titles that ONLY appear on error/challenge pages — never on real articles.
// match is case-insensitive, exact-or-prefix only (not substring) to avoid
// false positives like a real article titled "404 reasons your startup failed"
const TITLE_BLOCKLIST = [
  "just a moment",
  "just a moment...",
  "attention required! | cloudflare",
  "attention required!",
  "access denied",
  "access to this page has been denied",
  "you have been blocked",
  "are you a robot",
  "are you a robot?",
  "verify you are human",
  "please verify you are a human",
  "page not found",
  "404 not found",
  "404 page not found",
  "403 forbidden",
  "503 service unavailable",
  "this page isn't available",
  "this page isn’t available",
  "site temporarily unavailable",
  "request unsuccessful",
];


// substrings to look for in the raw html head/early body that indicate a
// cloudflare/akamai/imperva interstitial. these are infrastructure markers
// the real site never serves
const CHALLENGE_MARKERS = [
  "cf-chl-bypass",
  "__cf_chl_",
  "cf_chl_opt",
  "/cdn-cgi/challenge-platform",
  "_incapsula_resource",
  "incap_ses_",
  "x-iinfo",
  "akamai-bm-telemetry",
  "ak_bmsc",
  "distil_r_captcha",
];


// phrases at the very start of extracted body text that mean we got a stub.
// anchored to first ~500 chars so we dont false-flag articles that discuss
// these topics later in the body
const BODY_PREFIX_BLOCKLIST = [
  "you need to enable javascript",
  "please enable javascript",
  "javascript is required",
  "please enable cookies",
  "cookies must be enabled",
  "your browser will redirect",
  "checking your browser before",
  "this site requires javascript",
  "please make sure your browser supports",

  // yahoo finance serves its global nav when the article body is js-rendered
  // and the plain fetch only gets the static shell
  "today's news us politics world weather",

  // cnbc paywall shell — no article body, just site nav
  "subscribe to cnbc pro subscribe to investing club",
];


// final-url path suffixes that indicate the request was redirected to a
// generic error/login page. we only check the pathname so query strings dont
// throw it off
const ERROR_PATH_HINTS = [
  "/404",
  "/403",
  "/error",
  "/errors",
  "/blocked",
  "/captcha",
  "/challenge",
  "/access-denied",
  "/account/login",
  "/sign-in",
  "/signin",
  "/subscribe",
  "/subscription",
];


function normalizeForMatch(value) {
  return String(value || "").trim().toLowerCase();
}

function countSentenceTerminators(text) {
  // matches . ! ? followed by whitespace or end — avoids counting decimals like 3.14
  const matches = String(text || "").match(/[.!?](?:\s|$)/g);
  return matches ? matches.length : 0;
}

function hasErrorPath(finalUrl) {
  if (!finalUrl) return false;
  try {
    const path = new URL(finalUrl).pathname.toLowerCase();
    return ERROR_PATH_HINTS.some((hint) => path === hint || path.startsWith(`${hint}/`) || path.endsWith(hint));
  } catch {
    return false;
  }
}

function hasChallengeMarker(html) {
  if (!html) return null;
  // cap the search window — challenge markers are always in head or top of body,
  // dont need to scan a full 1.5mb document
  const haystack = String(html).slice(0, 50000).toLowerCase();
  for (const marker of CHALLENGE_MARKERS) {
    if (haystack.includes(marker)) {
      return marker;
    }
  }
  return null;
}

function titleIsBlocked(title) {
  const normalized = normalizeForMatch(title);
  if (!normalized) return null;

  for (const entry of TITLE_BLOCKLIST) {
    if (normalized === entry || normalized.startsWith(`${entry} `) || normalized.startsWith(`${entry}|`)) {
      return entry;
    }
  }
  return null;
}

function bodyPrefixIsBlocked(content) {
  const sniff = normalizeForMatch(content).slice(0, BODY_SNIFF_LENGTH);
  if (!sniff) return null;

  for (const phrase of BODY_PREFIX_BLOCKLIST) {
    if (sniff.includes(phrase)) {
      return phrase;
    }
  }
  return null;
}


function validateExtractedArticle({ article, html, finalUrl }) {
  if (!article) {
    return { ok: false, reason: "extractor-returned-null", retryable: false };
  }

  const content = typeof article.content === "string" ? article.content.trim() : "";
  const title = typeof article.title === "string" ? article.title.trim() : "";

  // title-level checks first since they're the cheapest signal
  const blockedTitle = titleIsBlocked(title);
  if (blockedTitle) {
    return { ok: false, reason: `title-blocklist:${blockedTitle}`, retryable: true };
  }

  if (hasErrorPath(finalUrl)) {
    return { ok: false, reason: `error-path:${finalUrl}`, retryable: true };
  }

  const challenge = hasChallengeMarker(html);
  if (challenge) {
    return { ok: false, reason: `challenge-marker:${challenge}`, retryable: true };
  }

  if (!content) {
    return { ok: false, reason: "no-content-extracted", retryable: true };
  }

  if (content.length < MIN_CONTENT_LENGTH) {
    return { ok: false, reason: `content-too-short:${content.length}`, retryable: true };
  }

  const blockedPrefix = bodyPrefixIsBlocked(content);
  if (blockedPrefix) {
    return { ok: false, reason: `body-prefix-blocklist:${blockedPrefix}`, retryable: true };
  }

  if (countSentenceTerminators(content) < MIN_SENTENCE_TERMINATORS) {
    return { ok: false, reason: "too-few-sentences", retryable: true };
  }

  return { ok: true };
}


module.exports = {
  validateExtractedArticle,
  MIN_CONTENT_LENGTH,
};