diff --git a/src/content.js b/src/content.js index 188eba0..8f2dcef 100644 --- a/src/content.js +++ b/src/content.js @@ -182,17 +182,39 @@ async function fetchPlainHtml(url) { } +// Every individual step below has its own timeout, but acquiring the shared +// session does not, and it is awaited while holding a browser slot. A wedged +// chromium therefore parks all eight slots forever, and because nothing throws +// there is not a single line in the log to say so: content fetching simply stops. +// That is exactly what happened after the 4 Sept restart, ~328 articles in. +// This is the outer bound that guarantees the slot always comes back. +const BROWSER_HARD_TIMEOUT = 90000; + async function fetchBrowserHtml(url) { await browserSemaphore.acquire(); try { const maxConcurrentPages = Number(config.browser?.maxConcurrentPages) || 8; - const session = await getSharedBrowserSession({ - requestTimeout: BROWSER_FETCH_TIMEOUT, - maxConcurrentPages, + let timer; + const expired = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error( + `browser fetch exceeded ${BROWSER_HARD_TIMEOUT}ms for ${url}, the session is probably wedged`)), + BROWSER_HARD_TIMEOUT); }); - - const html = await session.fetchRenderedHtml(url, { timeout: BROWSER_FETCH_TIMEOUT }); - return { html, finalUrl: url }; + try { + return await Promise.race([ + (async () => { + const session = await getSharedBrowserSession({ + requestTimeout: BROWSER_FETCH_TIMEOUT, + maxConcurrentPages, + }); + const html = await session.fetchRenderedHtml(url, { timeout: BROWSER_FETCH_TIMEOUT }); + return { html, finalUrl: url }; + })(), + expired, + ]); + } finally { + clearTimeout(timer); + } } finally { browserSemaphore.release(); } diff --git a/src/scheduler.js b/src/scheduler.js index 639a38b..8b73cee 100644 --- a/src/scheduler.js +++ b/src/scheduler.js @@ -129,7 +129,14 @@ function startScheduler() { try { const perSource = Number(config.contentBackfill?.perSource) || 50; const batchSize = Number(config.contentBackfill?.batchSize) || 25; + // A round is long and silent. When content fetching wedged on 4 Sept there + // was not one line anywhere saying so, which is why it went unnoticed for + // three days while the live lane starved for want of enriched articles. + const startedAt = Date.now(); + console.log(`[content] worker ${workerIndex} starting a round`); const processed = await runBackfillWorker({ workerIndex, workerCount, perSource, batchSize }); + console.log(`[content] worker ${workerIndex} finished ${processed} articles` + + ` in ${Math.round((Date.now() - startedAt) / 1000)}s`); // if a worker found nothing in its partition, brief sleep so we dont // hammer the db with empty selects