add browser crawling capabilities and enhance configuration options

src/sources/browserCrawler.js

@@ -0,0 +1,90 @@
+const { chromium } = require('playwright');
+
+const BROWSER_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36';
+let browserPromise = null;
+
+async function getBrowser() {
+  if (!browserPromise) {
+    browserPromise = chromium.launch({
+      headless: false,
+    });
+  }
+
+  return browserPromise;
+}
+
+async function waitForUsefulDom(page, site) {
+  try {
+    await page.waitForLoadState('networkidle', { timeout: Math.min(site.requestTimeout, 5000) });
+  } catch {
+  }
+
+  try {
+    await page.waitForFunction(() => document.querySelectorAll('a[href]').length > 20, {
+      timeout: Math.min(site.requestTimeout, 5000),
+    });
+  } catch {
+  }
+}
+
+async function createBrowserSession(site) {
+  const browser = await getBrowser();
+  const context = await browser.newContext({
+    userAgent: BROWSER_USER_AGENT,
+    viewport: { width: 1440, height: 1200 },
+    javaScriptEnabled: true,
+  });
+
+  await context.route('**/*', async (route) => {
+    const request = route.request();
+    const resourceType = request.resourceType();
+
+    if (['image', 'media', 'font'].includes(resourceType)) {
+      await route.abort();
+      return;
+    }
+
+    await route.continue();
+  });
+
+  return {
+    async fetchRenderedHtml(url, options = {}) {
+      const page = await context.newPage();
+
+      try {
+        await page.goto(url, {
+          waitUntil: 'domcontentloaded',
+          timeout: site.requestTimeout,
+        });
+
+        await waitForUsefulDom(page, site);
+        const html = await page.content();
+
+        if (options.includeDebug) {
+          return {
+            html,
+            finalUrl: page.url(),
+            title: await page.title(),
+            linkCount: await page.locator('a[href]').count(),
+          };
+        }
+
+        return html;
+      } finally {
+        await page.close();
+      }
+    },
+    async close() {
+      await context.close();
+    },
+  };
+}
+
+function shouldUseBrowser(site) {
+  return site.renderMode === 'browser';
+}
+
+module.exports = {
+  createBrowserSession,
+  shouldUseBrowser,
+};