134 lines
3.5 KiB
JavaScript
134 lines
3.5 KiB
JavaScript
const fs = require('fs');
|
|
const path = require('path');
|
|
|
|
const catalogPath = path.join(__dirname, '..', '..', 'sources.json');
|
|
|
|
function normalizeHostname(value) {
|
|
const input = String(value || '').trim().toLowerCase();
|
|
if (!input) {
|
|
return null;
|
|
}
|
|
|
|
try {
|
|
const url = new URL(input.includes('://') ? input : `https://${input}`);
|
|
return url.hostname.toLowerCase();
|
|
} catch {
|
|
return input.replace(/^[a-z]+:\/\//i, '').split('/')[0].trim().toLowerCase() || null;
|
|
}
|
|
}
|
|
|
|
function normalizeWebsites(value) {
|
|
const rawValues = Array.isArray(value) ? value : value == null ? [] : [value];
|
|
return [...new Set(rawValues.map((entry) => normalizeHostname(entry)).filter(Boolean))];
|
|
}
|
|
|
|
function normalizeSource(source, index) {
|
|
if (!source || typeof source !== 'object' || Array.isArray(source)) {
|
|
throw new Error(`sources.json entry ${index} must be an object`);
|
|
}
|
|
|
|
const id = String(source.id || '').trim();
|
|
const label = String(source.label || '').trim();
|
|
const rawFeedUrls = Array.isArray(source.feedUrl)
|
|
? source.feedUrl
|
|
: source.feedUrl == null ? [] : [source.feedUrl];
|
|
const feedUrls = rawFeedUrls.map((u) => String(u).trim()).filter(Boolean);
|
|
const website = normalizeWebsites(source.website);
|
|
const backfill = Boolean(source.backfill);
|
|
|
|
if (!id) {
|
|
throw new Error(`sources.json entry ${index} is missing id`);
|
|
}
|
|
|
|
if (!label) {
|
|
throw new Error(`sources.json entry ${index} is missing label`);
|
|
}
|
|
|
|
for (const u of feedUrls) {
|
|
if (u.startsWith('[FAILED] ')) continue;
|
|
try {
|
|
new URL(u);
|
|
} catch {
|
|
throw new Error(`sources.json entry ${index} has invalid feedUrl "${u}" for ${id}`);
|
|
}
|
|
}
|
|
|
|
if (backfill && website.length === 0) {
|
|
throw new Error(`sources.json entry ${index} has backfill enabled but no website for ${id}`);
|
|
}
|
|
|
|
return {
|
|
id,
|
|
label,
|
|
feedUrls,
|
|
website,
|
|
backfill,
|
|
};
|
|
}
|
|
|
|
function loadCatalog() {
|
|
const parsed = JSON.parse(fs.readFileSync(catalogPath, 'utf8'));
|
|
if (!Array.isArray(parsed)) {
|
|
throw new Error('sources.json must contain an array');
|
|
}
|
|
|
|
const sources = parsed.map((source, index) => normalizeSource(source, index));
|
|
const seenIds = new Set();
|
|
|
|
for (const source of sources) {
|
|
if (seenIds.has(source.id)) {
|
|
throw new Error(`sources.json contains duplicate id ${source.id}`);
|
|
}
|
|
|
|
seenIds.add(source.id);
|
|
}
|
|
|
|
return sources;
|
|
}
|
|
|
|
const sourceCatalog = loadCatalog();
|
|
|
|
function getSourceCatalog() {
|
|
return sourceCatalog;
|
|
}
|
|
|
|
function getRssSources() {
|
|
return sourceCatalog.filter((source) => source.feedUrls.length > 0);
|
|
}
|
|
|
|
function getBackfillSources() {
|
|
return sourceCatalog.filter((source) => source.backfill && source.website.length > 0);
|
|
}
|
|
|
|
function markFeedFailed(feedUrl) {
|
|
try {
|
|
const raw = JSON.parse(fs.readFileSync(catalogPath, 'utf8'));
|
|
let changed = false;
|
|
|
|
for (const entry of raw) {
|
|
const urls = Array.isArray(entry.feedUrl) ? entry.feedUrl : entry.feedUrl ? [entry.feedUrl] : [];
|
|
const idx = urls.findIndex((u) => u === feedUrl && !u.startsWith('[FAILED] '));
|
|
|
|
if (idx !== -1) {
|
|
urls[idx] = `[FAILED] ${urls[idx]}`;
|
|
entry.feedUrl = urls.length === 1 ? urls[0] : urls;
|
|
changed = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (changed) {
|
|
fs.writeFileSync(catalogPath, JSON.stringify(raw, null, 2) + '\n', 'utf8');
|
|
}
|
|
} catch (error) {
|
|
console.error('Failed to mark feed as failed in sources.json:', error);
|
|
}
|
|
}
|
|
|
|
module.exports = {
|
|
getSourceCatalog,
|
|
getRssSources,
|
|
getBackfillSources,
|
|
markFeedFailed,
|
|
};
|