add Google News integration and enhance crawler capabilities

src/sources/browserCrawler.js

@@ -1,43 +1,131 @@
 const { chromium } = require('playwright');
 
 const BROWSER_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36';
-let browserPromise = null;
+const MAX_RENDERED_HTML_LENGTH = 1_500_000;
+const DEFAULT_REQUEST_TIMEOUT = 20000;
+const CONSENT_BUTTON_SELECTORS = [
+  'button[name="agree"]',
+  'input[name="agree"]',
+  'button:has-text("Accept all")',
+  'button:has-text("Accept All")',
+  'button:has-text("Accept")',
+  'button:has-text("Accept cookies")',
+  'button:has-text("I agree")',
+  'button:has-text("Agree")',
+  'button:has-text("Consent")',
+  '[data-action="agree"]',
+  '[data-testid="consent-accept"]',
+  '[data-testid="accept-button"]',
+];
 
-async function getBrowser() {
-  if (!browserPromise) {
-    browserPromise = chromium.launch({
-      headless: false,
-    });
-  }
+let sharedBrowserSessionPromise = null;
+let sharedBrowserShutdownInstalled = false;
 
-  return browserPromise;
+function normalizeTimeout(value) {
+  return Math.max(1000, Math.min(Number(value) || DEFAULT_REQUEST_TIMEOUT, 30000));
 }
 
-async function waitForUsefulDom(page, site) {
+async function waitForUsefulDom(page, requestTimeout) {
+  const timeout = Math.min(normalizeTimeout(requestTimeout), 5000);
+
   try {
-    await page.waitForLoadState('networkidle', { timeout: Math.min(site.requestTimeout, 5000) });
+    await page.waitForLoadState('networkidle', { timeout });
   } catch {
   }
 
   try {
-    await page.waitForFunction(() => document.querySelectorAll('a[href]').length > 20, {
-      timeout: Math.min(site.requestTimeout, 5000),
-    });
+    await page.waitForFunction(() => document.querySelectorAll('a[href]').length > 20, { timeout });
   } catch {
   }
 }
 
-async function createBrowserSession(site) {
-  const browser = await getBrowser();
+async function acceptConsentIfPresent(page) {
+  let url;
+  try {
+    url = new URL(page.url());
+  } catch {
+    return;
+  }
+
+  const hostname = url.hostname.toLowerCase();
+  const pathname = url.pathname.toLowerCase();
+  const shouldCheckConsent = hostname.includes('consent.')
+    || hostname.includes('yahoo.com')
+    || /cookie|consent|privacy/.test(hostname)
+    || /cookie|consent|privacy/.test(pathname);
+
+  if (!shouldCheckConsent) {
+    return;
+  }
+
+  for (const selector of CONSENT_BUTTON_SELECTORS) {
+    const locator = page.locator(selector).first();
+
+    try {
+      if (await locator.isVisible({ timeout: 750 })) {
+        await locator.click({ timeout: 2000 });
+        try {
+          await page.waitForLoadState('domcontentloaded', { timeout: 5000 });
+        } catch {
+        }
+        return;
+      }
+    } catch {
+    }
+  }
+}
+
+async function buildBrowserSession(options = {}) {
+  const requestTimeout = normalizeTimeout(options.requestTimeout);
+  const maxConcurrentPages = Math.max(1, Math.min(Number(options.maxConcurrentPages) || 2, 8));
+  const browser = await chromium.launch({
+    headless: true,
+    args: [
+      '--disable-blink-features=AutomationControlled',
+      '--no-sandbox',
+      '--disable-dev-shm-usage',
+    ],
+  });
   const context = await browser.newContext({
     userAgent: BROWSER_USER_AGENT,
     viewport: { width: 1440, height: 1200 },
     javaScriptEnabled: true,
+    extraHTTPHeaders: {
+      'Accept-Language': 'en-US,en;q=0.9',
+    },
   });
 
+  await context.addInitScript(() => {
+    Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
+    Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
+    Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3] });
+  });
+  const waiters = [];
+  let activePages = 0;
+  let closed = false;
+
+  async function acquirePageSlot() {
+    if (activePages < maxConcurrentPages) {
+      activePages += 1;
+      return;
+    }
+
+    await new Promise((resolve) => {
+      waiters.push(resolve);
+    });
+    activePages += 1;
+  }
+
+  function releasePageSlot() {
+    activePages = Math.max(0, activePages - 1);
+    const next = waiters.shift();
+    if (next) {
+      next();
+    }
+  }
+
   await context.route('**/*', async (route) => {
-    const request = route.request();
-    const resourceType = request.resourceType();
+    const resourceType = route.request().resourceType();
 
     if (['image', 'media', 'font'].includes(resourceType)) {
       await route.abort();
@@ -49,16 +137,23 @@ async function createBrowserSession(site) {
 
   return {
     async fetchRenderedHtml(url, options = {}) {
+      if (closed) {
+        throw new Error('browser session is closed');
+      }
+
+      await acquirePageSlot();
       const page = await context.newPage();
+      const timeout = normalizeTimeout(options.timeout || requestTimeout);
 
       try {
         await page.goto(url, {
           waitUntil: 'domcontentloaded',
-          timeout: site.requestTimeout,
+          timeout,
         });
 
-        await waitForUsefulDom(page, site);
-        const html = await page.content();
+        await acceptConsentIfPresent(page);
+        await waitForUsefulDom(page, timeout);
+        const html = (await page.content()).slice(0, MAX_RENDERED_HTML_LENGTH);
 
         if (options.includeDebug) {
           return {
@@ -71,20 +166,73 @@ async function createBrowserSession(site) {
 
         return html;
       } finally {
-        await page.close();
+        try {
+          await page.close();
+        } finally {
+          releasePageSlot();
+        }
       }
     },
     async close() {
+      if (closed) {
+        return;
+      }
+
+      closed = true;
       await context.close();
+      await browser.close();
     },
   };
 }
 
-function shouldUseBrowser(site) {
-  return site.renderMode === 'browser';
+function installSharedBrowserShutdown() {
+  if (sharedBrowserShutdownInstalled) {
+    return;
+  }
+
+  sharedBrowserShutdownInstalled = true;
+  const shutdown = () => {
+    if (!sharedBrowserSessionPromise) {
+      return;
+    }
+
+    const sessionPromise = sharedBrowserSessionPromise;
+    sharedBrowserSessionPromise = null;
+    sessionPromise
+      .then((session) => session.close())
+      .catch((error) => console.error('shared browser shutdown failed:', error));
+  };
+
+  process.once('beforeExit', shutdown);
+  process.once('SIGINT', shutdown);
+  process.once('SIGTERM', shutdown);
+}
+
+async function createBrowserSession(site = {}) {
+  return buildBrowserSession({
+    requestTimeout: site.requestTimeout,
+    maxConcurrentPages: site.pageConcurrency,
+  });
+}
+
+async function getSharedBrowserSession(options = {}) {
+  if (!sharedBrowserSessionPromise) {
+    installSharedBrowserShutdown();
+    sharedBrowserSessionPromise = buildBrowserSession({
+      requestTimeout: options.requestTimeout,
+      maxConcurrentPages: options.maxConcurrentPages || 2,
+    });
+  }
+
+  return sharedBrowserSessionPromise;
+}
+
+function shouldUseBrowser() {
+  return true;
 }
 
 module.exports = {
   createBrowserSession,
+  getSharedBrowserSession,
   shouldUseBrowser,
 };