Duriin-API/src/sources/browserCrawler.js

const { chromium } = require('playwright');

const BROWSER_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36';
const MAX_RENDERED_HTML_LENGTH = 1_500_000;
const DEFAULT_REQUEST_TIMEOUT = 20000;
// generous, this is the "something has gone wrong" bound rather than a normal wait
const PAGE_SLOT_WAIT_MS = 120000;
const PAGE_CLOSE_TIMEOUT_MS = 10000;

function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); }
const CONSENT_BUTTON_SELECTORS = [
  'button[name="agree"]',
  'input[name="agree"]',
  'button:has-text("Accept all")',
  'button:has-text("Accept All")',
  'button:has-text("Accept")',
  'button:has-text("Accept cookies")',
  'button:has-text("I agree")',
  'button:has-text("Agree")',
  'button:has-text("Consent")',
  '[data-action="agree"]',
  '[data-testid="consent-accept"]',
  '[data-testid="accept-button"]',
];

let sharedBrowserSessionPromise = null;
let sharedBrowserShutdownInstalled = false;

function normalizeTimeout(value) {
  return Math.max(1000, Math.min(Number(value) || DEFAULT_REQUEST_TIMEOUT, 30000));
}

async function waitForUsefulDom(page, requestTimeout) {
  const timeout = Math.min(normalizeTimeout(requestTimeout), 5000);

  try {
    await page.waitForLoadState('networkidle', { timeout });
  } catch {
  }

  try {
    await page.waitForFunction(() => document.querySelectorAll('a[href]').length > 20, { timeout });
  } catch {
  }
}

async function acceptConsentIfPresent(page) {
  let url;
  try {
    url = new URL(page.url());
  } catch {
    return;
  }

  const hostname = url.hostname.toLowerCase();
  const pathname = url.pathname.toLowerCase();
  const shouldCheckConsent = hostname.includes('consent.')
    || hostname.includes('yahoo.com')
    || /cookie|consent|privacy/.test(hostname)
    || /cookie|consent|privacy/.test(pathname);

  if (!shouldCheckConsent) {
    return;
  }

  for (const selector of CONSENT_BUTTON_SELECTORS) {
    const locator = page.locator(selector).first();

    try {
      if (await locator.isVisible({ timeout: 750 })) {
        await locator.click({ timeout: 2000 });
        try {
          await page.waitForLoadState('domcontentloaded', { timeout: 5000 });
        } catch {
        }
        return;
      }
    } catch {
    }
  }
}

async function buildBrowserSession(options = {}) {
  const requestTimeout = normalizeTimeout(options.requestTimeout);
  const maxConcurrentPages = Math.max(1, Math.min(Number(options.maxConcurrentPages) || 2, 8));
  const browser = await chromium.launch({
    headless: true,
    args: [
      '--disable-blink-features=AutomationControlled',
      '--no-sandbox',
      '--disable-dev-shm-usage',
    ],
  });
  const context = await browser.newContext({
    userAgent: BROWSER_USER_AGENT,
    viewport: { width: 1440, height: 1200 },
    javaScriptEnabled: true,
    extraHTTPHeaders: {
      'Accept-Language': 'en-US,en;q=0.9',
    },
  });

  await context.addInitScript(() => {
    Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
    Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
    Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3] });
  });
  const waiters = [];
  let activePages = 0;
  let closed = false;

  // A slot is never waited on forever. If every slot has leaked the callers used
  // to park here silently with no log and no progress, which looks exactly like a
  // dead worker, so time out and let the caller fail loudly instead.
  async function acquirePageSlot(waitMs = PAGE_SLOT_WAIT_MS) {
    if (activePages < maxConcurrentPages) {
      activePages += 1;
      return;
    }

    let waiter;
    let timer;
    try {
      await new Promise((resolve, reject) => {
        waiter = resolve;
        waiters.push(waiter);
        timer = setTimeout(() => {
          const index = waiters.indexOf(waiter);
          if (index !== -1) waiters.splice(index, 1);
          reject(new Error(`timed out after ${waitMs}ms waiting for a browser page slot`
            + ` (${activePages}/${maxConcurrentPages} active)`));
        }, waitMs);
      });
    } finally {
      clearTimeout(timer);
    }
    activePages += 1;
  }

  function releasePageSlot() {
    activePages = Math.max(0, activePages - 1);
    const next = waiters.shift();
    if (next) {
      next();
    }
  }

  await context.route('**/*', async (route) => {
    const resourceType = route.request().resourceType();

    if (['image', 'media', 'font'].includes(resourceType)) {
      await route.abort();
      return;
    }

    await route.continue();
  });

  return {
    async fetchRenderedHtml(url, options = {}) {
      if (closed) {
        throw new Error('browser session is closed');
      }

      await acquirePageSlot();
      // newPage() used to sit out here. When it threw or hung the slot was gone for
      // good, and after maxConcurrentPages of those every caller blocked forever.
      let page = null;
      const timeout = normalizeTimeout(options.timeout || requestTimeout);

      try {
        page = await context.newPage();
        await page.goto(url, {
          waitUntil: 'domcontentloaded',
          timeout,
        });

        await acceptConsentIfPresent(page);
        await waitForUsefulDom(page, timeout);
        const html = (await page.content()).slice(0, MAX_RENDERED_HTML_LENGTH);

        if (options.includeDebug) {
          return {
            html,
            finalUrl: page.url(),
            title: await page.title(),
            linkCount: await page.locator('a[href]').count(),
          };
        }

        return html;
      } finally {
        try {
          // a wedged renderer can make close() hang too, and that would strand the
          // slot just as badly as the original leak did
          if (page) await Promise.race([page.close(), sleep(PAGE_CLOSE_TIMEOUT_MS)]);
        } catch (error) {
          console.error(`[browser] page close failed for ${url}:`, error.message);
        } finally {
          releasePageSlot();
        }
      }
    },
    async close() {
      if (closed) {
        return;
      }

      closed = true;
      await context.close();
      await browser.close();
    },
  };
}

function installSharedBrowserShutdown() {
  if (sharedBrowserShutdownInstalled) {
    return;
  }

  sharedBrowserShutdownInstalled = true;
  const shutdown = () => {
    if (!sharedBrowserSessionPromise) {
      return;
    }

    const sessionPromise = sharedBrowserSessionPromise;
    sharedBrowserSessionPromise = null;
    sessionPromise
      .then((session) => session.close())
      .catch((error) => console.error('shared browser shutdown failed:', error));
  };

  process.once('beforeExit', shutdown);
  process.once('SIGINT', shutdown);
  process.once('SIGTERM', shutdown);
}

async function createBrowserSession(site = {}) {
  return buildBrowserSession({
    requestTimeout: site.requestTimeout,
    maxConcurrentPages: site.pageConcurrency,
  });
}

async function getSharedBrowserSession(options = {}) {
  if (!sharedBrowserSessionPromise) {
    installSharedBrowserShutdown();
    sharedBrowserSessionPromise = buildBrowserSession({
      requestTimeout: options.requestTimeout,
      maxConcurrentPages: options.maxConcurrentPages || 2,
    });
  }

  return sharedBrowserSessionPromise;
}

function shouldUseBrowser() {
  return true;
}

module.exports = {
  createBrowserSession,
  getSharedBrowserSession,
  shouldUseBrowser,
};