add Google News integration and enhance crawler capabilities

src/content.js

@@ -171,7 +171,7 @@ async function fetchAndStoreContent(id, url) {
   }
 }
 
-async function backfillMissingContent(limit = 10) {
+async function backfillMissingContent(limit = 100, concurrency = 5) {
   if (contentBackfillRunning) {
     return;
   }
@@ -183,15 +183,26 @@ async function backfillMissingContent(limit = 10) {
       ? selectAllArticlesMissingContent.all()
       : selectArticlesMissingContent.all(limit);
 
-    for (const row of rows) {
-      await fetchAndStoreContent(row.id, row.url);
+    for (let i = 0; i < rows.length; i += concurrency) {
+      const batch = rows.slice(i, i + concurrency);
+      await Promise.all(batch.map((row) => fetchAndStoreContent(row.id, row.url)));
     }
   } finally {
     contentBackfillRunning = false;
   }
 }
 
+function hasPendingContent() {
+  return Boolean(db.prepare(`
+    SELECT 1 FROM articles
+    WHERE (content IS NULL OR TRIM(content) = '')
+      AND (content_status IS NULL OR content_status = 'pending')
+    LIMIT 1
+  `).get());
+}
+
 module.exports = {
   fetchAndStoreContent,
   backfillMissingContent,
+  hasPendingContent,
 };