const fs = require('fs'); const path = require('path'); const catalogPath = path.join(__dirname, '..', '..', 'sources.json'); function normalizeHostname(value) { const input = String(value || '').trim().toLowerCase(); if (!input) { return null; } try { const url = new URL(input.includes('://') ? input : `https://${input}`); return url.hostname.toLowerCase(); } catch { return input.replace(/^[a-z]+:\/\//i, '').split('/')[0].trim().toLowerCase() || null; } } function normalizeWebsites(value) { const rawValues = Array.isArray(value) ? value : value == null ? [] : [value]; return [...new Set(rawValues.map((entry) => normalizeHostname(entry)).filter(Boolean))]; } function normalizeSource(source, index) { if (!source || typeof source !== 'object' || Array.isArray(source)) { throw new Error(`sources.json entry ${index} must be an object`); } const id = String(source.id || '').trim(); const label = String(source.label || '').trim(); const rawFeedUrls = Array.isArray(source.feedUrl) ? source.feedUrl : source.feedUrl == null ? [] : [source.feedUrl]; const feedUrls = rawFeedUrls.map((u) => String(u).trim()).filter(Boolean); const website = normalizeWebsites(source.website); const backfill = Boolean(source.backfill); if (!id) { throw new Error(`sources.json entry ${index} is missing id`); } if (!label) { throw new Error(`sources.json entry ${index} is missing label`); } for (const u of feedUrls) { if (u.startsWith('[FAILED] ')) continue; try { new URL(u); } catch { throw new Error(`sources.json entry ${index} has invalid feedUrl "${u}" for ${id}`); } } if (backfill && website.length === 0) { throw new Error(`sources.json entry ${index} has backfill enabled but no website for ${id}`); } return { id, label, feedUrls, website, backfill, }; } function loadCatalog() { const parsed = JSON.parse(fs.readFileSync(catalogPath, 'utf8')); if (!Array.isArray(parsed)) { throw new Error('sources.json must contain an array'); } const sources = parsed.map((source, index) => normalizeSource(source, index)); const seenIds = new Set(); for (const source of sources) { if (seenIds.has(source.id)) { throw new Error(`sources.json contains duplicate id ${source.id}`); } seenIds.add(source.id); } return sources; } const sourceCatalog = loadCatalog(); function getSourceCatalog() { return sourceCatalog; } function getRssSources() { return sourceCatalog.filter((source) => source.feedUrls.length > 0); } function getBackfillSources() { return sourceCatalog.filter((source) => source.backfill && source.website.length > 0); } function markFeedFailed(feedUrl) { try { const raw = JSON.parse(fs.readFileSync(catalogPath, 'utf8')); let changed = false; for (const entry of raw) { const urls = Array.isArray(entry.feedUrl) ? entry.feedUrl : entry.feedUrl ? [entry.feedUrl] : []; const idx = urls.findIndex((u) => u === feedUrl && !u.startsWith('[FAILED] ')); if (idx !== -1) { urls[idx] = `[FAILED] ${urls[idx]}`; entry.feedUrl = urls.length === 1 ? urls[0] : urls; changed = true; break; } } if (changed) { fs.writeFileSync(catalogPath, JSON.stringify(raw, null, 2) + '\n', 'utf8'); } } catch (error) { console.error('Failed to mark feed as failed in sources.json:', error); } } module.exports = { getSourceCatalog, getRssSources, getBackfillSources, markFeedFailed, };