const { chromium } = require('playwright'); const BROWSER_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36'; const MAX_RENDERED_HTML_LENGTH = 1_500_000; const DEFAULT_REQUEST_TIMEOUT = 20000; // generous, this is the "something has gone wrong" bound rather than a normal wait const PAGE_SLOT_WAIT_MS = 120000; const PAGE_CLOSE_TIMEOUT_MS = 10000; function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } const CONSENT_BUTTON_SELECTORS = [ 'button[name="agree"]', 'input[name="agree"]', 'button:has-text("Accept all")', 'button:has-text("Accept All")', 'button:has-text("Accept")', 'button:has-text("Accept cookies")', 'button:has-text("I agree")', 'button:has-text("Agree")', 'button:has-text("Consent")', '[data-action="agree"]', '[data-testid="consent-accept"]', '[data-testid="accept-button"]', ]; let sharedBrowserSessionPromise = null; let sharedBrowserShutdownInstalled = false; function normalizeTimeout(value) { return Math.max(1000, Math.min(Number(value) || DEFAULT_REQUEST_TIMEOUT, 30000)); } async function waitForUsefulDom(page, requestTimeout) { const timeout = Math.min(normalizeTimeout(requestTimeout), 5000); try { await page.waitForLoadState('networkidle', { timeout }); } catch { } try { await page.waitForFunction(() => document.querySelectorAll('a[href]').length > 20, { timeout }); } catch { } } async function acceptConsentIfPresent(page) { let url; try { url = new URL(page.url()); } catch { return; } const hostname = url.hostname.toLowerCase(); const pathname = url.pathname.toLowerCase(); const shouldCheckConsent = hostname.includes('consent.') || hostname.includes('yahoo.com') || /cookie|consent|privacy/.test(hostname) || /cookie|consent|privacy/.test(pathname); if (!shouldCheckConsent) { return; } for (const selector of CONSENT_BUTTON_SELECTORS) { const locator = page.locator(selector).first(); try { if (await locator.isVisible({ timeout: 750 })) { await locator.click({ timeout: 2000 }); try { await page.waitForLoadState('domcontentloaded', { timeout: 5000 }); } catch { } return; } } catch { } } } async function buildBrowserSession(options = {}) { const requestTimeout = normalizeTimeout(options.requestTimeout); const maxConcurrentPages = Math.max(1, Math.min(Number(options.maxConcurrentPages) || 2, 8)); const browser = await chromium.launch({ headless: true, args: [ '--disable-blink-features=AutomationControlled', '--no-sandbox', '--disable-dev-shm-usage', ], }); const context = await browser.newContext({ userAgent: BROWSER_USER_AGENT, viewport: { width: 1440, height: 1200 }, javaScriptEnabled: true, extraHTTPHeaders: { 'Accept-Language': 'en-US,en;q=0.9', }, }); await context.addInitScript(() => { Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] }); Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3] }); }); const waiters = []; let activePages = 0; let closed = false; // A slot is never waited on forever. If every slot has leaked the callers used // to park here silently with no log and no progress, which looks exactly like a // dead worker, so time out and let the caller fail loudly instead. async function acquirePageSlot(waitMs = PAGE_SLOT_WAIT_MS) { if (activePages < maxConcurrentPages) { activePages += 1; return; } let waiter; let timer; try { await new Promise((resolve, reject) => { waiter = resolve; waiters.push(waiter); timer = setTimeout(() => { const index = waiters.indexOf(waiter); if (index !== -1) waiters.splice(index, 1); reject(new Error(`timed out after ${waitMs}ms waiting for a browser page slot` + ` (${activePages}/${maxConcurrentPages} active)`)); }, waitMs); }); } finally { clearTimeout(timer); } activePages += 1; } function releasePageSlot() { activePages = Math.max(0, activePages - 1); const next = waiters.shift(); if (next) { next(); } } await context.route('**/*', async (route) => { const resourceType = route.request().resourceType(); if (['image', 'media', 'font'].includes(resourceType)) { await route.abort(); return; } await route.continue(); }); return { async fetchRenderedHtml(url, options = {}) { if (closed) { throw new Error('browser session is closed'); } await acquirePageSlot(); // newPage() used to sit out here. When it threw or hung the slot was gone for // good, and after maxConcurrentPages of those every caller blocked forever. let page = null; const timeout = normalizeTimeout(options.timeout || requestTimeout); try { page = await context.newPage(); await page.goto(url, { waitUntil: 'domcontentloaded', timeout, }); await acceptConsentIfPresent(page); await waitForUsefulDom(page, timeout); const html = (await page.content()).slice(0, MAX_RENDERED_HTML_LENGTH); if (options.includeDebug) { return { html, finalUrl: page.url(), title: await page.title(), linkCount: await page.locator('a[href]').count(), }; } return html; } finally { try { // a wedged renderer can make close() hang too, and that would strand the // slot just as badly as the original leak did if (page) await Promise.race([page.close(), sleep(PAGE_CLOSE_TIMEOUT_MS)]); } catch (error) { console.error(`[browser] page close failed for ${url}:`, error.message); } finally { releasePageSlot(); } } }, async close() { if (closed) { return; } closed = true; await context.close(); await browser.close(); }, }; } function installSharedBrowserShutdown() { if (sharedBrowserShutdownInstalled) { return; } sharedBrowserShutdownInstalled = true; const shutdown = () => { if (!sharedBrowserSessionPromise) { return; } const sessionPromise = sharedBrowserSessionPromise; sharedBrowserSessionPromise = null; sessionPromise .then((session) => session.close()) .catch((error) => console.error('shared browser shutdown failed:', error)); }; process.once('beforeExit', shutdown); process.once('SIGINT', shutdown); process.once('SIGTERM', shutdown); } async function createBrowserSession(site = {}) { return buildBrowserSession({ requestTimeout: site.requestTimeout, maxConcurrentPages: site.pageConcurrency, }); } async function getSharedBrowserSession(options = {}) { if (!sharedBrowserSessionPromise) { installSharedBrowserShutdown(); sharedBrowserSessionPromise = buildBrowserSession({ requestTimeout: options.requestTimeout, maxConcurrentPages: options.maxConcurrentPages || 2, }); } return sharedBrowserSessionPromise; } function shouldUseBrowser() { return true; } module.exports = { createBrowserSession, getSharedBrowserSession, shouldUseBrowser, };