add Google News integration and enhance crawler capabilities

src/sources/sourceCatalog.js

@@ -0,0 +1,133 @@
+const fs = require('fs');
+const path = require('path');
+
+const catalogPath = path.join(__dirname, '..', '..', 'sources.json');
+
+function normalizeHostname(value) {
+  const input = String(value || '').trim().toLowerCase();
+  if (!input) {
+    return null;
+  }
+
+  try {
+    const url = new URL(input.includes('://') ? input : `https://${input}`);
+    return url.hostname.toLowerCase();
+  } catch {
+    return input.replace(/^[a-z]+:\/\//i, '').split('/')[0].trim().toLowerCase() || null;
+  }
+}
+
+function normalizeWebsites(value) {
+  const rawValues = Array.isArray(value) ? value : value == null ? [] : [value];
+  return [...new Set(rawValues.map((entry) => normalizeHostname(entry)).filter(Boolean))];
+}
+
+function normalizeSource(source, index) {
+  if (!source || typeof source !== 'object' || Array.isArray(source)) {
+    throw new Error(`sources.json entry ${index} must be an object`);
+  }
+
+  const id = String(source.id || '').trim();
+  const label = String(source.label || '').trim();
+  const rawFeedUrls = Array.isArray(source.feedUrl)
+    ? source.feedUrl
+    : source.feedUrl == null ? [] : [source.feedUrl];
+  const feedUrls = rawFeedUrls.map((u) => String(u).trim()).filter(Boolean);
+  const website = normalizeWebsites(source.website);
+  const backfill = Boolean(source.backfill);
+
+  if (!id) {
+    throw new Error(`sources.json entry ${index} is missing id`);
+  }
+
+  if (!label) {
+    throw new Error(`sources.json entry ${index} is missing label`);
+  }
+
+  for (const u of feedUrls) {
+    if (u.startsWith('[FAILED] ')) continue;
+    try {
+      new URL(u);
+    } catch {
+      throw new Error(`sources.json entry ${index} has invalid feedUrl "${u}" for ${id}`);
+    }
+  }
+
+  if (backfill && website.length === 0) {
+    throw new Error(`sources.json entry ${index} has backfill enabled but no website for ${id}`);
+  }
+
+  return {
+    id,
+    label,
+    feedUrls,
+    website,
+    backfill,
+  };
+}
+
+function loadCatalog() {
+  const parsed = JSON.parse(fs.readFileSync(catalogPath, 'utf8'));
+  if (!Array.isArray(parsed)) {
+    throw new Error('sources.json must contain an array');
+  }
+
+  const sources = parsed.map((source, index) => normalizeSource(source, index));
+  const seenIds = new Set();
+
+  for (const source of sources) {
+    if (seenIds.has(source.id)) {
+      throw new Error(`sources.json contains duplicate id ${source.id}`);
+    }
+
+    seenIds.add(source.id);
+  }
+
+  return sources;
+}
+
+const sourceCatalog = loadCatalog();
+
+function getSourceCatalog() {
+  return sourceCatalog;
+}
+
+function getRssSources() {
+  return sourceCatalog.filter((source) => source.feedUrls.length > 0);
+}
+
+function getBackfillSources() {
+  return sourceCatalog.filter((source) => source.backfill && source.website.length > 0);
+}
+
+function markFeedFailed(feedUrl) {
+  try {
+    const raw = JSON.parse(fs.readFileSync(catalogPath, 'utf8'));
+    let changed = false;
+
+    for (const entry of raw) {
+      const urls = Array.isArray(entry.feedUrl) ? entry.feedUrl : entry.feedUrl ? [entry.feedUrl] : [];
+      const idx = urls.findIndex((u) => u === feedUrl && !u.startsWith('[FAILED] '));
+
+      if (idx !== -1) {
+        urls[idx] = `[FAILED] ${urls[idx]}`;
+        entry.feedUrl = urls.length === 1 ? urls[0] : urls;
+        changed = true;
+        break;
+      }
+    }
+
+    if (changed) {
+      fs.writeFileSync(catalogPath, JSON.stringify(raw, null, 2) + '\n', 'utf8');
+    }
+  } catch (error) {
+    console.error('Failed to mark feed as failed in sources.json:', error);
+  }
+}
+
+module.exports = {
+  getSourceCatalog,
+  getRssSources,
+  getBackfillSources,
+  markFeedFailed,
+};