migrate article embeddings to support multi-model architecture and enhance data integrity

src/contentValidation.js

@@ -0,0 +1,199 @@
+// validates whether an extracted article is real content vs a soft-error page
+// (cookie wall, cloudflare challenge, paywall, "enable javascript", etc).
+//
+// the rules are deliberately conservative. we'd rather let a few junk pages
+// through (caught downstream when re-checked) than reject 5% of real articles.
+// fingerprints are anchored to title or the first ~500 chars of body so an
+// article that *mentions* cloudflare doesnt get falsely rejected.
+
+const MIN_CONTENT_LENGTH = 400;
+const MIN_SENTENCE_TERMINATORS = 3;
+const BODY_SNIFF_LENGTH = 800;
+
+
+// titles that ONLY appear on error/challenge pages — never on real articles.
+// match is case-insensitive, exact-or-prefix only (not substring) to avoid
+// false positives like a real article titled "404 reasons your startup failed"
+const TITLE_BLOCKLIST = [
+  "just a moment",
+  "just a moment...",
+  "attention required! | cloudflare",
+  "attention required!",
+  "access denied",
+  "access to this page has been denied",
+  "you have been blocked",
+  "are you a robot",
+  "are you a robot?",
+  "verify you are human",
+  "please verify you are a human",
+  "page not found",
+  "404 not found",
+  "404 page not found",
+  "403 forbidden",
+  "503 service unavailable",
+  "this page isn't available",
+  "this page isn’t available",
+  "site temporarily unavailable",
+  "request unsuccessful",
+];
+
+
+// substrings to look for in the raw html head/early body that indicate a
+// cloudflare/akamai/imperva interstitial. these are infrastructure markers
+// the real site never serves
+const CHALLENGE_MARKERS = [
+  "cf-chl-bypass",
+  "__cf_chl_",
+  "cf_chl_opt",
+  "/cdn-cgi/challenge-platform",
+  "_incapsula_resource",
+  "incap_ses_",
+  "x-iinfo",
+  "akamai-bm-telemetry",
+  "ak_bmsc",
+  "distil_r_captcha",
+];
+
+
+// phrases at the very start of extracted body text that mean we got a stub.
+// anchored to first ~500 chars so we dont false-flag articles that discuss
+// these topics later in the body
+const BODY_PREFIX_BLOCKLIST = [
+  "you need to enable javascript",
+  "please enable javascript",
+  "javascript is required",
+  "please enable cookies",
+  "cookies must be enabled",
+  "your browser will redirect",
+  "checking your browser before",
+  "this site requires javascript",
+  "please make sure your browser supports",
+];
+
+
+// final-url path suffixes that indicate the request was redirected to a
+// generic error/login page. we only check the pathname so query strings dont
+// throw it off
+const ERROR_PATH_HINTS = [
+  "/404",
+  "/403",
+  "/error",
+  "/errors",
+  "/blocked",
+  "/captcha",
+  "/challenge",
+  "/access-denied",
+  "/account/login",
+  "/sign-in",
+  "/signin",
+  "/subscribe",
+  "/subscription",
+];
+
+
+function normalizeForMatch(value) {
+  return String(value || "").trim().toLowerCase();
+}
+
+function countSentenceTerminators(text) {
+  // matches . ! ? followed by whitespace or end — avoids counting decimals like 3.14
+  const matches = String(text || "").match(/[.!?](?:\s|$)/g);
+  return matches ? matches.length : 0;
+}
+
+function hasErrorPath(finalUrl) {
+  if (!finalUrl) return false;
+  try {
+    const path = new URL(finalUrl).pathname.toLowerCase();
+    return ERROR_PATH_HINTS.some((hint) => path === hint || path.startsWith(`${hint}/`) || path.endsWith(hint));
+  } catch {
+    return false;
+  }
+}
+
+function hasChallengeMarker(html) {
+  if (!html) return null;
+  // cap the search window — challenge markers are always in head or top of body,
+  // dont need to scan a full 1.5mb document
+  const haystack = String(html).slice(0, 50000).toLowerCase();
+  for (const marker of CHALLENGE_MARKERS) {
+    if (haystack.includes(marker)) {
+      return marker;
+    }
+  }
+  return null;
+}
+
+function titleIsBlocked(title) {
+  const normalized = normalizeForMatch(title);
+  if (!normalized) return null;
+
+  for (const entry of TITLE_BLOCKLIST) {
+    if (normalized === entry || normalized.startsWith(`${entry} `) || normalized.startsWith(`${entry}|`)) {
+      return entry;
+    }
+  }
+  return null;
+}
+
+function bodyPrefixIsBlocked(content) {
+  const sniff = normalizeForMatch(content).slice(0, BODY_SNIFF_LENGTH);
+  if (!sniff) return null;
+
+  for (const phrase of BODY_PREFIX_BLOCKLIST) {
+    if (sniff.includes(phrase)) {
+      return phrase;
+    }
+  }
+  return null;
+}
+
+
+function validateExtractedArticle({ article, html, finalUrl }) {
+  if (!article) {
+    return { ok: false, reason: "extractor-returned-null", retryable: false };
+  }
+
+  const content = typeof article.content === "string" ? article.content.trim() : "";
+  const title = typeof article.title === "string" ? article.title.trim() : "";
+
+  // title-level checks first since they're the cheapest signal
+  const blockedTitle = titleIsBlocked(title);
+  if (blockedTitle) {
+    return { ok: false, reason: `title-blocklist:${blockedTitle}`, retryable: true };
+  }
+
+  if (hasErrorPath(finalUrl)) {
+    return { ok: false, reason: `error-path:${finalUrl}`, retryable: true };
+  }
+
+  const challenge = hasChallengeMarker(html);
+  if (challenge) {
+    return { ok: false, reason: `challenge-marker:${challenge}`, retryable: true };
+  }
+
+  if (!content) {
+    return { ok: false, reason: "no-content-extracted", retryable: true };
+  }
+
+  if (content.length < MIN_CONTENT_LENGTH) {
+    return { ok: false, reason: `content-too-short:${content.length}`, retryable: true };
+  }
+
+  const blockedPrefix = bodyPrefixIsBlocked(content);
+  if (blockedPrefix) {
+    return { ok: false, reason: `body-prefix-blocklist:${blockedPrefix}`, retryable: true };
+  }
+
+  if (countSentenceTerminators(content) < MIN_SENTENCE_TERMINATORS) {
+    return { ok: false, reason: "too-few-sentences", retryable: true };
+  }
+
+  return { ok: true };
+}
+
+
+module.exports = {
+  validateExtractedArticle,
+  MIN_CONTENT_LENGTH,
+};