773 lines
24 KiB
JavaScript
773 lines
24 KiB
JavaScript
const config = require('../config');
|
|
const { fetchWithPolicy } = require('../http');
|
|
const { createBrowserSession, shouldUseBrowser } = require('./browserCrawler');
|
|
const { classifyPageWithLlm } = require('./crawlerClassifier');
|
|
const { getRssSources } = require('./sourceCatalog');
|
|
|
|
function sleep(ms) {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
|
|
const TRACKING_PARAM_PATTERNS = [
|
|
/^utm_/i,
|
|
/^fbclid$/i,
|
|
/^gclid$/i,
|
|
/^mkt_tok$/i,
|
|
/^mc_cid$/i,
|
|
/^mc_eid$/i,
|
|
/^ref$/i,
|
|
/^ref_src$/i,
|
|
/^s$/i,
|
|
/^cmpid$/i,
|
|
/^guccounter$/i,
|
|
/^guce_referrer$/i,
|
|
/^guce_referrer_sig$/i,
|
|
];
|
|
const LISTING_PATH_HINT = /(archive|archives|latest|topic|topics|section|sections|category|categories|news|world|business|politics|technology|tech|markets|economy|page|tag|tags)/i;
|
|
const ARTICLE_DATE_PATH = /\/\d{4}\/\d{2}\/\d{2}(?:\/|$)|\/\d{4}\/\d{2}(?:\/|$)/;
|
|
const ARTICLE_PATH_HINT = /(\/article\/|\/articles\/|\/news\/|\/story\/|\/stories\/)/i;
|
|
const ARTICLE_PATH_STRONG_HINT = /\/\d{4}\/\d{2}\/\d{2}\//;
|
|
const LISTING_ARTICLE_FALSE_POSITIVE_PATH = /(\/category\/|\/tag\/|\/latest(?:\/|$)|\/topics?(?:\/|$)|\/sections?(?:\/|$))/i;
|
|
const BLOCKED_PATH_HINT = /(\/search(?:\/|$)|\/login(?:\/|$)|\/account(?:\/|$)|\/video(?:\/|$)|\/videos(?:\/|$)|\/podcast(?:\/|$)|\/podcasts(?:\/|$)|\/live(?:\/|$)|\/subscribe(?:\/|$)|\/subscription(?:\/|$)|\/newsletters?(?:\/|$)|\/privacy(?:\/|$)|\/terms(?:\/|$)|\/about(?:\/|$)|\/contact(?:\/|$))/i;
|
|
const EXPLORATION_PATH_HINT = /(\/page\/\d+(?:\/|$)|[?&]page=\d+|\/archive(?:s)?(?:\/|$)|\/latest(?:\/|$)|\/news(?:\/|$)|\/world(?:\/|$)|\/business(?:\/|$)|\/politics(?:\/|$)|\/technology(?:\/|$)|\/tech(?:\/|$)|\/markets(?:\/|$)|\/economy(?:\/|$)|\/topic(?:s)?(?:\/|$)|\/section(?:s)?(?:\/|$)|\/category(?:ies)?(?:\/|$)|\/tag(?:s)?(?:\/|$))/i;
|
|
|
|
function decodeHtmlEntities(value) {
|
|
return String(value || '')
|
|
.replace(/&#x([0-9a-f]+);/gi, (_, hex) => String.fromCodePoint(parseInt(hex, 16)))
|
|
.replace(/&#(\d+);/g, (_, dec) => String.fromCodePoint(parseInt(dec, 10)))
|
|
.replace(/"/g, '"')
|
|
.replace(/'/g, "'")
|
|
.replace(/'/g, "'")
|
|
.replace(/&/g, '&')
|
|
.replace(/</g, '<')
|
|
.replace(/>/g, '>')
|
|
.replace(/ /g, ' ');
|
|
}
|
|
|
|
function stripTags(value) {
|
|
return decodeHtmlEntities(String(value || '').replace(/<[^>]*>/g, ' ')).replace(/\s+/g, ' ').trim();
|
|
}
|
|
|
|
function normalizeText(value) {
|
|
return stripTags(value).replace(/\s+/g, ' ').trim();
|
|
}
|
|
|
|
function isAllowedHost(hostname, allowedHosts) {
|
|
const normalized = String(hostname || '').toLowerCase();
|
|
return allowedHosts.some((allowedHost) => {
|
|
const candidate = String(allowedHost || '').toLowerCase();
|
|
return normalized === candidate || normalized.endsWith(`.${candidate}`);
|
|
});
|
|
}
|
|
|
|
function shouldDropParam(key) {
|
|
return TRACKING_PARAM_PATTERNS.some((pattern) => pattern.test(key));
|
|
}
|
|
|
|
function canonicalizeUrl(rawUrl, baseUrl, allowedHosts) {
|
|
try {
|
|
const url = new URL(rawUrl, baseUrl);
|
|
|
|
if (!['http:', 'https:'].includes(url.protocol)) {
|
|
return null;
|
|
}
|
|
|
|
if (allowedHosts && allowedHosts.length && !isAllowedHost(url.hostname, allowedHosts)) {
|
|
return null;
|
|
}
|
|
|
|
url.hash = '';
|
|
url.username = '';
|
|
url.password = '';
|
|
|
|
const params = [...url.searchParams.entries()]
|
|
.filter(([key]) => !shouldDropParam(key))
|
|
.sort(([left], [right]) => left.localeCompare(right));
|
|
|
|
url.search = '';
|
|
for (const [key, value] of params) {
|
|
url.searchParams.append(key, value);
|
|
}
|
|
|
|
if (url.pathname !== '/') {
|
|
url.pathname = url.pathname.replace(/\/+$/, '') || '/';
|
|
}
|
|
|
|
return url.toString();
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function extractAttribute(tag, name) {
|
|
const match = tag.match(new RegExp(`${name}\\s*=\\s*(["'])(.*?)\\1`, 'i'));
|
|
return match ? decodeHtmlEntities(match[2]).trim() : '';
|
|
}
|
|
|
|
function extractMetaMap(html) {
|
|
const metas = new Map();
|
|
const metaTags = html.match(/<meta\b[^>]*>/gi) || [];
|
|
|
|
for (const tag of metaTags) {
|
|
const key = extractAttribute(tag, 'property') || extractAttribute(tag, 'name');
|
|
const content = extractAttribute(tag, 'content');
|
|
|
|
if (!key || !content) {
|
|
continue;
|
|
}
|
|
|
|
metas.set(key.toLowerCase(), content);
|
|
}
|
|
|
|
return metas;
|
|
}
|
|
|
|
function extractCanonicalHref(html) {
|
|
const links = html.match(/<link\b[^>]*>/gi) || [];
|
|
|
|
for (const tag of links) {
|
|
const rel = extractAttribute(tag, 'rel').toLowerCase();
|
|
if (!rel || !rel.split(/\s+/).includes('canonical')) {
|
|
continue;
|
|
}
|
|
|
|
const href = extractAttribute(tag, 'href');
|
|
if (href) {
|
|
return href;
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
function extractTitleTag(html) {
|
|
const match = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
|
|
return match ? normalizeText(match[1]) : null;
|
|
}
|
|
|
|
function extractH1(html) {
|
|
const match = html.match(/<h1\b[^>]*>([\s\S]*?)<\/h1>/i);
|
|
return match ? normalizeText(match[1]) : null;
|
|
}
|
|
|
|
function extractTimeDatetime(html) {
|
|
const match = html.match(/<time\b[^>]*datetime\s*=\s*(["'])(.*?)\1/i);
|
|
return match ? decodeHtmlEntities(match[2]).trim() : null;
|
|
}
|
|
|
|
function extractParagraphStats(html) {
|
|
const paragraphs = html.match(/<p\b[^>]*>[\s\S]*?<\/p>/gi) || [];
|
|
const normalizedParagraphs = paragraphs.slice(0, 12).map((paragraph) => normalizeText(paragraph)).filter(Boolean);
|
|
|
|
return {
|
|
textLength: normalizedParagraphs.reduce((total, paragraph) => total + paragraph.length, 0),
|
|
substantialCount: normalizedParagraphs.filter((paragraph) => paragraph.length >= 80).length,
|
|
};
|
|
}
|
|
|
|
function extractJsonLdBlocks(html) {
|
|
const blocks = [];
|
|
const regex = /<script\b[^>]*type\s*=\s*(["'])application\/ld\+json\1[^>]*>([\s\S]*?)<\/script>/gi;
|
|
let match;
|
|
|
|
while ((match = regex.exec(html)) !== null) {
|
|
const raw = String(match[2] || '').trim();
|
|
if (!raw) {
|
|
continue;
|
|
}
|
|
|
|
try {
|
|
blocks.push(JSON.parse(raw));
|
|
} catch {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
return blocks;
|
|
}
|
|
|
|
function isArticleType(type) {
|
|
if (Array.isArray(type)) {
|
|
return type.some((entry) => isArticleType(entry));
|
|
}
|
|
|
|
return ['article', 'newsarticle'].includes(String(type || '').toLowerCase());
|
|
}
|
|
|
|
function extractArticleJsonLd(html) {
|
|
const blocks = extractJsonLdBlocks(html);
|
|
|
|
for (const block of blocks) {
|
|
if (!block || typeof block !== 'object') {
|
|
continue;
|
|
}
|
|
|
|
if (isArticleType(block['@type'])) {
|
|
return block;
|
|
}
|
|
|
|
const graph = Array.isArray(block['@graph']) ? block['@graph'] : [];
|
|
const directChildren = [
|
|
...graph,
|
|
...(Array.isArray(block.mainEntity) ? block.mainEntity : [block.mainEntity]),
|
|
...(Array.isArray(block.mainEntityOfPage) ? block.mainEntityOfPage : [block.mainEntityOfPage]),
|
|
].filter((value) => value && typeof value === 'object');
|
|
|
|
const article = directChildren.find((value) => isArticleType(value['@type']));
|
|
if (article) {
|
|
return article;
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
function extractLinks(html, pageUrl, allowedHosts) {
|
|
const links = [];
|
|
const seen = new Set();
|
|
const regex = /<a\b[^>]*href\s*=\s*(["'])(.*?)\1[^>]*>([\s\S]*?)<\/a>/gi;
|
|
let match;
|
|
|
|
while ((match = regex.exec(html)) !== null) {
|
|
const url = canonicalizeUrl(match[2], pageUrl, allowedHosts);
|
|
if (!url || seen.has(url)) {
|
|
continue;
|
|
}
|
|
|
|
const text = normalizeText(match[3]);
|
|
seen.add(url);
|
|
links.push({ url, text });
|
|
}
|
|
|
|
return links;
|
|
}
|
|
|
|
function selectTitle(meta, jsonLdArticle, html) {
|
|
return [
|
|
meta.get('og:title'),
|
|
meta.get('twitter:title'),
|
|
jsonLdArticle && jsonLdArticle.headline,
|
|
extractH1(html),
|
|
extractTitleTag(html),
|
|
].find((value) => String(value || '').trim()) || null;
|
|
}
|
|
|
|
function selectDescription(meta, jsonLdArticle) {
|
|
return [
|
|
meta.get('og:description'),
|
|
meta.get('description'),
|
|
jsonLdArticle && jsonLdArticle.description,
|
|
].find((value) => String(value || '').trim()) || null;
|
|
}
|
|
|
|
function selectPubDate(meta, jsonLdArticle, html) {
|
|
return [
|
|
jsonLdArticle && jsonLdArticle.datePublished,
|
|
meta.get('article:published_time'),
|
|
meta.get('og:article:published_time'),
|
|
extractTimeDatetime(html),
|
|
].find((value) => String(value || '').trim()) || null;
|
|
}
|
|
|
|
function scorePage(pageUrl, meta, html, jsonLdArticle, links) {
|
|
let articleScore = 0;
|
|
let listingScore = 0;
|
|
const pathname = new URL(pageUrl).pathname;
|
|
const hasArticleDatePath = ARTICLE_DATE_PATH.test(pageUrl);
|
|
const hasArticlePathHint = ARTICLE_PATH_HINT.test(pageUrl);
|
|
const hasStrongArticlePath = ARTICLE_PATH_STRONG_HINT.test(pathname);
|
|
const hasListingFalsePositivePath = LISTING_ARTICLE_FALSE_POSITIVE_PATH.test(pathname);
|
|
const { textLength: paragraphTextLength, substantialCount: substantialParagraphCount } = extractParagraphStats(html);
|
|
const headlineLinks = links.filter(({ text }) => text.length >= 25 && text.length <= 180).length;
|
|
const h1 = extractH1(html);
|
|
const titleTag = extractTitleTag(html);
|
|
const ogTitle = normalizeText(meta.get('og:title') || '');
|
|
const twitterTitle = normalizeText(meta.get('twitter:title') || '');
|
|
const titleSignalsLower = [h1, titleTag, ogTitle, twitterTitle].filter(Boolean).map((value) => value.toLowerCase());
|
|
const looksLikeSectionTitle = titleSignalsLower.some((value) => /^(category:|section:)/.test(value)
|
|
|| /(market[s]?|business|technology|tech|science|health|gaming|culture|news|world|economy)/.test(value) && value.length <= 80);
|
|
const looksLikeCommercialPage = titleSignalsLower.some((value) => /(subscribe|subscription|sign in|log in|newsletter|advertis)/.test(value));
|
|
const jsonLdHeadline = normalizeText(jsonLdArticle && jsonLdArticle.headline);
|
|
const titleSignals = [h1, ogTitle, twitterTitle, titleTag].filter(Boolean);
|
|
const matchingTitleSignals = titleSignals.filter((value) => jsonLdHeadline && (value.includes(jsonLdHeadline) || jsonLdHeadline.includes(value)));
|
|
const hasJsonLdArticle = Boolean(jsonLdArticle && matchingTitleSignals.length > 0);
|
|
const hasPublishTime = Boolean(meta.get('article:published_time') || meta.get('og:article:published_time') || extractTimeDatetime(html));
|
|
const hasOgArticle = String(meta.get('og:type') || '').toLowerCase() === 'article';
|
|
const hasArticleTag = /<article\b/i.test(html);
|
|
const hasLongBody = paragraphTextLength >= 600 && substantialParagraphCount >= 3;
|
|
const hasArticleStructure = Boolean(h1) && (hasArticleTag || hasLongBody);
|
|
const hasMetadataArticleSignal = hasOgArticle || hasPublishTime || hasJsonLdArticle;
|
|
const hasUrlArticleSignal = hasArticleDatePath || hasStrongArticlePath || hasArticlePathHint;
|
|
const hasPrimaryArticleSignal = hasMetadataArticleSignal || hasUrlArticleSignal;
|
|
const hasSecondaryArticleSignal = hasArticlePathHint || hasArticleStructure;
|
|
|
|
if (hasJsonLdArticle) {
|
|
articleScore += 3;
|
|
}
|
|
|
|
if (hasOgArticle && !hasListingFalsePositivePath) {
|
|
articleScore += 2;
|
|
}
|
|
|
|
if (hasPublishTime && !hasListingFalsePositivePath) {
|
|
articleScore += 2;
|
|
}
|
|
|
|
if (hasArticleTag) {
|
|
articleScore += 1;
|
|
}
|
|
|
|
if (hasArticleDatePath || hasArticlePathHint) {
|
|
articleScore += 2;
|
|
}
|
|
|
|
if (hasLongBody) {
|
|
articleScore += 2;
|
|
}
|
|
|
|
if (links.length >= 20) {
|
|
listingScore += 2;
|
|
}
|
|
|
|
if (headlineLinks >= 8) {
|
|
listingScore += 2;
|
|
}
|
|
|
|
if (LISTING_PATH_HINT.test(pathname)) {
|
|
listingScore += 1;
|
|
}
|
|
|
|
if (hasListingFalsePositivePath) {
|
|
listingScore += 3;
|
|
}
|
|
|
|
if (looksLikeSectionTitle) {
|
|
listingScore += 3;
|
|
}
|
|
|
|
if (looksLikeCommercialPage) {
|
|
listingScore += 4;
|
|
}
|
|
|
|
if ((pathname === '/' || LISTING_PATH_HINT.test(pathname)) && headlineLinks >= 8 && links.length >= 20) {
|
|
listingScore += 4;
|
|
}
|
|
|
|
if (headlineLinks >= 15 && !hasPrimaryArticleSignal && !hasLongBody) {
|
|
listingScore += 3;
|
|
}
|
|
|
|
if (!hasPrimaryArticleSignal && headlineLinks >= 12 && links.length >= 25) {
|
|
listingScore += 2;
|
|
}
|
|
|
|
const shouldAskLlm = !looksLikeCommercialPage && (
|
|
articleScore >= 2
|
|
|| hasPrimaryArticleSignal
|
|
|| hasSecondaryArticleSignal
|
|
|| (headlineLinks <= 10 && links.length <= 40)
|
|
);
|
|
|
|
return {
|
|
articleScore,
|
|
listingScore,
|
|
shouldAskLlm,
|
|
headlineLinks,
|
|
paragraphCount: substantialParagraphCount,
|
|
paragraphTextLength,
|
|
};
|
|
}
|
|
|
|
function shouldQueueLink(url) {
|
|
const parsed = new URL(url);
|
|
const pathname = parsed.pathname.toLowerCase();
|
|
const hostname = parsed.hostname.toLowerCase();
|
|
|
|
if (BLOCKED_PATH_HINT.test(pathname)) {
|
|
return false;
|
|
}
|
|
|
|
if (hostname.includes('consent.yahoo.com') || pathname.startsWith('/v2/') || pathname.startsWith('/redirect')) {
|
|
return false;
|
|
}
|
|
|
|
return !/\.(?:jpg|jpeg|png|gif|webp|svg|pdf|zip|xml|mp4|mp3|avi|mov|wmv|m4v)$/i.test(pathname);
|
|
}
|
|
|
|
function shouldContinueExploring(current, listingScore, links) {
|
|
if (current.depth === 0) {
|
|
return true;
|
|
}
|
|
|
|
if (listingScore >= 1) {
|
|
return true;
|
|
}
|
|
|
|
if (current.depth <= 1 && links.length >= 8) {
|
|
return true;
|
|
}
|
|
|
|
return EXPLORATION_PATH_HINT.test(current.url);
|
|
}
|
|
|
|
function slugifyLabel(label) {
|
|
return String(label || '')
|
|
.toLowerCase()
|
|
.replace(/[^a-z0-9]+/g, '_')
|
|
.replace(/^_+|_+$/g, '');
|
|
}
|
|
|
|
function unique(values) {
|
|
return [...new Set(values.filter(Boolean))];
|
|
}
|
|
|
|
function buildAllowedHosts(hostname) {
|
|
if (!hostname) {
|
|
return [];
|
|
}
|
|
|
|
const hosts = [hostname.toLowerCase()];
|
|
if (hostname.startsWith('www.')) {
|
|
hosts.push(hostname.slice(4).toLowerCase());
|
|
} else {
|
|
hosts.push(`www.${hostname}`.toLowerCase());
|
|
}
|
|
|
|
return unique(hosts);
|
|
}
|
|
|
|
function cleanFeedPath(pathname) {
|
|
const withoutIndex = pathname
|
|
.replace(/\/index\.[a-z0-9]+$/i, '/')
|
|
.replace(/\.[a-z0-9]+$/i, '')
|
|
.replace(/\/rss(?:$|\/.*$)/i, '/')
|
|
.replace(/\/feed(?:$|\/.*$)/i, '/')
|
|
.replace(/\/feeds?(?:$|\/.*$)/i, '/')
|
|
.replace(/\/xml(?:$|\/.*$)/i, '/')
|
|
.replace(/\/arc\/outboundfeeds\//i, '/')
|
|
.replace(/\/dynamo\//i, '/')
|
|
.replace(/\/id\/\d+\/device\/rss\//i, '/')
|
|
.replace(/\/contentexport\//i, '/')
|
|
.replace(/\/rssfeedstopstories$/i, '/')
|
|
.replace(/\/latest$/i, '/')
|
|
.replace(/\/+$|^$/g, '');
|
|
|
|
if (!withoutIndex) {
|
|
return '/';
|
|
}
|
|
|
|
const segments = withoutIndex
|
|
.split('/')
|
|
.filter(Boolean)
|
|
.filter((segment) => !/^(rss|feed|feeds|xml)$/i.test(segment))
|
|
.slice(0, 3);
|
|
|
|
if (!segments.length) {
|
|
return '/';
|
|
}
|
|
|
|
return `/${segments.join('/')}`;
|
|
}
|
|
|
|
function buildDefaultSeeds(feedUrl) {
|
|
try {
|
|
const parsed = new URL(feedUrl);
|
|
const origin = `${parsed.protocol}//${parsed.hostname}`;
|
|
const cleanedPath = cleanFeedPath(parsed.pathname);
|
|
|
|
return unique([
|
|
canonicalizeUrl(origin, origin),
|
|
cleanedPath === '/' ? null : canonicalizeUrl(`${origin}${cleanedPath}`, origin),
|
|
]);
|
|
} catch {
|
|
return [];
|
|
}
|
|
}
|
|
|
|
function normalizeLimit(value, fallback, minimum, maximum) {
|
|
const numeric = Number(value);
|
|
|
|
if (numeric === -1) {
|
|
return Number.POSITIVE_INFINITY;
|
|
}
|
|
|
|
if (!Number.isFinite(numeric)) {
|
|
return fallback;
|
|
}
|
|
|
|
return Math.max(minimum, Math.min(numeric, maximum));
|
|
}
|
|
|
|
function normalizeSite(site) {
|
|
const allowedHosts = unique((site.allowedHosts || []).map((host) => String(host || '').toLowerCase()).filter(Boolean));
|
|
const seeds = unique((site.seeds || [])
|
|
.map((seed) => canonicalizeUrl(seed, seed, allowedHosts))
|
|
.filter(Boolean));
|
|
const renderMode = String(site.renderMode || 'http').trim().toLowerCase() === 'browser' ? 'browser' : 'http';
|
|
const maxPages = normalizeLimit(site.maxPages, 15, 1, 500);
|
|
|
|
return {
|
|
name: String(site.name || '').trim(),
|
|
label: String(site.label || '').trim(),
|
|
allowedHosts,
|
|
seeds,
|
|
renderMode,
|
|
maxPages,
|
|
maxDepth: normalizeLimit(site.maxDepth, 1, 0, 5),
|
|
pageConcurrency: normalizeLimit(site.pageConcurrency, renderMode === 'browser' ? 2 : 4, 1, renderMode === 'browser' ? 4 : 12),
|
|
maxQueuedPages: normalizeLimit(site.maxQueuedPages, Math.min(maxPages * 3, 1000), maxPages, 2000),
|
|
memorySoftLimitMb: normalizeLimit(site.memorySoftLimitMb, 800, 128, 8192),
|
|
memoryHardLimitMb: normalizeLimit(site.memoryHardLimitMb, 1400, 256, 16384),
|
|
memoryThrottleDelayMs: normalizeLimit(site.memoryThrottleDelayMs, 1500, 100, 10000),
|
|
llmPatternMinHits: normalizeLimit(site.llmPatternMinHits, 3, 1, 20),
|
|
requestTimeout: Math.max(1000, Math.min(Number(site.requestTimeout) || 15000, 30000)),
|
|
};
|
|
}
|
|
|
|
function getCrawlerSiteOverrides(label) {
|
|
return config.newsCrawler?.overrides?.[label] || null;
|
|
}
|
|
|
|
function getConfiguredCrawlerSites() {
|
|
const defaults = config.newsCrawler || {};
|
|
const disabledLabels = new Set((defaults.disabledLabels || []).map((label) => String(label || '').trim()));
|
|
const explicitSites = (defaults.sites || []).map((site) => normalizeSite(site));
|
|
const explicitLabels = new Set(explicitSites.map((site) => site.label).filter(Boolean));
|
|
const derivedSites = [];
|
|
|
|
for (const feed of getRssSources()) {
|
|
const label = String(feed.label || '').trim();
|
|
if (!label || disabledLabels.has(label) || explicitLabels.has(label)) {
|
|
continue;
|
|
}
|
|
|
|
let hostname = '';
|
|
try {
|
|
hostname = new URL(feed.feedUrl).hostname;
|
|
} catch {
|
|
continue;
|
|
}
|
|
|
|
const override = getCrawlerSiteOverrides(label) || {};
|
|
const site = normalizeSite({
|
|
label,
|
|
name: override.name || `crawler_${slugifyLabel(label)}`,
|
|
allowedHosts: override.allowedHosts || buildAllowedHosts(hostname),
|
|
seeds: override.seeds || buildDefaultSeeds(feed.feedUrl),
|
|
renderMode: override.renderMode || defaults.renderMode,
|
|
maxPages: override.maxPages || defaults.maxPages,
|
|
maxDepth: override.maxDepth || defaults.maxDepth,
|
|
pageConcurrency: override.pageConcurrency,
|
|
maxQueuedPages: override.maxQueuedPages,
|
|
memorySoftLimitMb: override.memorySoftLimitMb || defaults.memorySoftLimitMb,
|
|
memoryHardLimitMb: override.memoryHardLimitMb || defaults.memoryHardLimitMb,
|
|
memoryThrottleDelayMs: override.memoryThrottleDelayMs || defaults.memoryThrottleDelayMs,
|
|
llmPatternMinHits: override.llmPatternMinHits || defaults.llmPatternMinHits,
|
|
requestTimeout: override.requestTimeout || defaults.requestTimeout,
|
|
});
|
|
|
|
if (site.name && site.allowedHosts.length && site.seeds.length) {
|
|
derivedSites.push(site);
|
|
}
|
|
}
|
|
|
|
return [...explicitSites.filter((site) => site.name && site.allowedHosts.length && site.seeds.length), ...derivedSites];
|
|
}
|
|
|
|
async function fetchHtml(url, site, browserSession) {
|
|
if (browserSession) {
|
|
return browserSession.fetchRenderedHtml(url);
|
|
}
|
|
|
|
const response = await fetchWithPolicy(url, {
|
|
timeout: site.requestTimeout,
|
|
retries: 1,
|
|
});
|
|
|
|
if (!response.ok) {
|
|
return null;
|
|
}
|
|
|
|
const contentType = String(response.headers.get('content-type') || '').toLowerCase();
|
|
if (!contentType.includes('text/html') && !contentType.includes('application/xhtml+xml')) {
|
|
return null;
|
|
}
|
|
|
|
return response.text();
|
|
}
|
|
|
|
async function crawlSite(site) {
|
|
const normalizedSite = normalizeSite(site);
|
|
|
|
if (!normalizedSite.name || !normalizedSite.allowedHosts.length || !normalizedSite.seeds.length) {
|
|
return [];
|
|
}
|
|
|
|
const browserSession = shouldUseBrowser(normalizedSite)
|
|
? await createBrowserSession(normalizedSite)
|
|
: null;
|
|
const queue = normalizedSite.seeds.map((url) => ({ url, depth: 0 }));
|
|
const queuedUrls = new Set(normalizedSite.seeds);
|
|
const visitedUrls = new Set();
|
|
const discoveredArticleUrls = new Set();
|
|
const articles = [];
|
|
|
|
function getResidentSetMb() {
|
|
return Math.round(process.memoryUsage().rss / (1024 * 1024));
|
|
}
|
|
|
|
async function throttleForMemory() {
|
|
let residentSetMb = getResidentSetMb();
|
|
|
|
while (residentSetMb >= normalizedSite.memorySoftLimitMb) {
|
|
if (residentSetMb >= normalizedSite.memoryHardLimitMb) {
|
|
console.error(`Crawler memory hard limit reached for ${normalizedSite.name}: ${residentSetMb}MB >= ${normalizedSite.memoryHardLimitMb}MB`);
|
|
queue.length = 0;
|
|
return false;
|
|
}
|
|
|
|
console.warn(`Crawler memory throttle for ${normalizedSite.name}: ${residentSetMb}MB >= ${normalizedSite.memorySoftLimitMb}MB`);
|
|
await sleep(normalizedSite.memoryThrottleDelayMs);
|
|
residentSetMb = getResidentSetMb();
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
async function processPage(current) {
|
|
let html;
|
|
try {
|
|
html = await fetchHtml(current.url, normalizedSite, browserSession);
|
|
} catch (error) {
|
|
console.error(`Crawler fetch failed for ${normalizedSite.name}: ${current.url}`, error);
|
|
return;
|
|
}
|
|
|
|
if (!html) {
|
|
return;
|
|
}
|
|
|
|
const meta = extractMetaMap(html);
|
|
const jsonLdArticle = extractArticleJsonLd(html);
|
|
const canonicalHref = extractCanonicalHref(html);
|
|
const canonicalUrl = canonicalHref
|
|
? canonicalizeUrl(canonicalHref, current.url, normalizedSite.allowedHosts) || current.url
|
|
: current.url;
|
|
const links = extractLinks(html, canonicalUrl, normalizedSite.allowedHosts);
|
|
const heuristic = scorePage(canonicalUrl, meta, html, jsonLdArticle, links);
|
|
let isArticleCandidate = false;
|
|
let effectiveListingScore = heuristic.listingScore;
|
|
|
|
if (heuristic.shouldAskLlm) {
|
|
try {
|
|
const llmDecision = await classifyPageWithLlm({
|
|
siteName: normalizedSite.name,
|
|
url: canonicalUrl,
|
|
html,
|
|
meta,
|
|
jsonLdArticle,
|
|
heuristic,
|
|
links,
|
|
minPatternHits: normalizedSite.llmPatternMinHits,
|
|
});
|
|
|
|
if (llmDecision.classification === 'article') {
|
|
isArticleCandidate = true;
|
|
effectiveListingScore = Math.max(0, heuristic.listingScore - 2);
|
|
} else if (llmDecision.classification === 'listing') {
|
|
effectiveListingScore = Math.max(heuristic.listingScore, 3);
|
|
} else if (llmDecision.classification === 'other') {
|
|
effectiveListingScore = Math.max(0, heuristic.listingScore - 1);
|
|
}
|
|
} catch (error) {
|
|
console.error(`Crawler LLM classification failed for ${normalizedSite.name}: ${canonicalUrl}`, error);
|
|
}
|
|
}
|
|
|
|
if (isArticleCandidate && !discoveredArticleUrls.has(canonicalUrl)) {
|
|
const title = normalizeText(selectTitle(meta, jsonLdArticle, html));
|
|
if (title) {
|
|
discoveredArticleUrls.add(canonicalUrl);
|
|
articles.push({
|
|
title,
|
|
description: normalizeText(selectDescription(meta, jsonLdArticle)) || null,
|
|
url: canonicalUrl,
|
|
source: normalizedSite.name,
|
|
pubDate: selectPubDate(meta, jsonLdArticle, html),
|
|
isIndexPage: false,
|
|
});
|
|
}
|
|
}
|
|
|
|
if (current.depth >= normalizedSite.maxDepth || !shouldContinueExploring(current, effectiveListingScore, links)) {
|
|
return;
|
|
}
|
|
|
|
for (const link of links) {
|
|
if (queuedUrls.size >= normalizedSite.maxQueuedPages) {
|
|
break;
|
|
}
|
|
|
|
if (!shouldQueueLink(link.url) || visitedUrls.has(link.url) || queuedUrls.has(link.url)) {
|
|
continue;
|
|
}
|
|
|
|
queuedUrls.add(link.url);
|
|
queue.push({ url: link.url, depth: current.depth + 1 });
|
|
}
|
|
}
|
|
|
|
try {
|
|
while (queue.length && visitedUrls.size < normalizedSite.maxPages) {
|
|
if (!await throttleForMemory()) {
|
|
break;
|
|
}
|
|
|
|
const batch = [];
|
|
|
|
while (queue.length && batch.length < normalizedSite.pageConcurrency && visitedUrls.size + batch.length < normalizedSite.maxPages) {
|
|
const current = queue.shift();
|
|
if (!current || visitedUrls.has(current.url)) {
|
|
continue;
|
|
}
|
|
|
|
visitedUrls.add(current.url);
|
|
batch.push(current);
|
|
}
|
|
|
|
if (!batch.length) {
|
|
continue;
|
|
}
|
|
|
|
await Promise.all(batch.map(processPage));
|
|
}
|
|
|
|
return articles;
|
|
} finally {
|
|
if (browserSession) {
|
|
await browserSession.close();
|
|
}
|
|
}
|
|
}
|
|
|
|
async function fetchCrawlerArticles() {
|
|
const articles = [];
|
|
|
|
for (const site of getConfiguredCrawlerSites()) {
|
|
try {
|
|
articles.push(...await crawlSite(site));
|
|
} catch (error) {
|
|
console.error(`Crawler failed for ${site && site.name ? site.name : 'unknown_site'}`, error);
|
|
}
|
|
}
|
|
|
|
return articles;
|
|
}
|
|
|
|
module.exports = {
|
|
fetchCrawlerArticles,
|
|
crawlSite,
|
|
canonicalizeUrl,
|
|
getConfiguredCrawlerSites,
|
|
};
|