initialize project structure with core modules and configuration

2026-04-16 22:47:34 +01:00
parent 82d856a21d
commit 7724fafbdc
20 changed files with 3644 additions and 0 deletions
@@ -0,0 +1,32 @@
+const config = require('../config');
+const { fetchJson } = require('../http');
+
+async function fetchAlphaVantageArticles() {
+  if (!config.alphaVantage?.apiKey) {
+    return [];
+  }
+
+  const tickers = (config.alphaVantage.tickers || []).join(',');
+  const params = new URLSearchParams({
+    function: 'NEWS_SENTIMENT',
+    apikey: config.alphaVantage.apiKey,
+  });
+
+  if (tickers) {
+    params.set('tickers', tickers);
+  }
+
+  const data = await fetchJson(`https://www.alphavantage.co/query?${params.toString()}`);
+
+  return (data.feed || []).map((item) => ({
+    title: item.title,
+    description: item.summary || null,
+    url: item.url,
+    source: 'alphavantage',
+    pubDate: item.time_published || null,
+  })).filter((item) => item.title && item.url);
+}
+
+module.exports = {
+  fetchAlphaVantageArticles,
+};
@@ -0,0 +1,78 @@
+const config = require('../config');
+const { fetchJson } = require('../http');
+
+let tickerMap;
+
+function getHeaders() {
+  return {
+    headers: {
+      'User-Agent': config.sec.userAgent,
+      Accept: 'application/json',
+    },
+  };
+}
+
+async function getTickerMap() {
+  if (tickerMap) {
+    return tickerMap;
+  }
+
+  const data = await fetchJson('https://www.sec.gov/files/company_tickers.json', getHeaders());
+  tickerMap = Object.values(data).reduce((acc, company) => {
+    acc[String(company.ticker || '').toUpperCase()] = String(company.cik_str || '').padStart(10, '0');
+    return acc;
+  }, {});
+
+  return tickerMap;
+}
+
+async function fetchEdgarArticles() {
+  const tickers = config.sec?.tickers || [];
+  if (tickers.length === 0) {
+    return [];
+  }
+
+  const map = await getTickerMap();
+  const articles = [];
+
+  for (const ticker of tickers) {
+    const cik = map[String(ticker).toUpperCase()];
+    if (!cik) {
+      continue;
+    }
+
+    const data = await fetchJson(`https://data.sec.gov/submissions/CIK${cik}.json`, getHeaders());
+    const recent = data.filings?.recent;
+    if (!recent) {
+      continue;
+    }
+
+    for (let i = 0; i < recent.form.length; i += 1) {
+      if (recent.form[i] !== '8-K') {
+        continue;
+      }
+
+      const accession = String(recent.accessionNumber[i] || '').replace(/-/g, '');
+      const primaryDocument = String(recent.primaryDocument[i] || '').trim();
+      const filingDate = recent.filingDate[i] || null;
+
+      if (!accession || !primaryDocument) {
+        continue;
+      }
+
+      articles.push({
+        title: `${String(ticker).toUpperCase()} 8-K ${filingDate || accession}`,
+        description: `${String(data.name || ticker).trim()} filed Form 8-K`,
+        url: `https://www.sec.gov/Archives/edgar/data/${Number(cik)}/${accession}/${primaryDocument}`,
+        source: 'edgar',
+        pubDate: filingDate,
+      });
+    }
+  }
+
+  return articles;
+}
+
+module.exports = {
+  fetchEdgarArticles,
+};
@@ -0,0 +1,55 @@
+const config = require('../config');
+const { fetchJson } = require('../http');
+
+function getDateRange() {
+  const to = new Date();
+  const from = new Date(to);
+  from.setDate(from.getDate() - 1);
+
+  return {
+    from: from.toISOString().slice(0, 10),
+    to: to.toISOString().slice(0, 10),
+  };
+}
+
+async function fetchFinnhubArticles() {
+  if (!config.finnhub?.apiKey) {
+    return [];
+  }
+
+  const { from, to } = getDateRange();
+  const articles = [];
+
+  for (const symbol of config.finnhub.tickers || []) {
+    const params = new URLSearchParams({
+      symbol,
+      from,
+      to,
+      token: config.finnhub.apiKey,
+    });
+
+    const data = await fetchJson(`https://finnhub.io/api/v1/company-news?${params.toString()}`);
+    for (const item of data || []) {
+      const title = String(item.headline || '').trim();
+      const url = String(item.url || '').trim();
+
+      if (!title || !url) {
+        continue;
+      }
+
+      articles.push({
+        title,
+        description: item.summary || null,
+        url,
+        source: 'finnhub',
+        pubDate: item.datetime ? new Date(item.datetime * 1000).toISOString() : null,
+      });
+    }
+  }
+
+  return articles;
+}
+
+module.exports = {
+  fetchFinnhubArticles,
+};
@@ -0,0 +1,48 @@
+const config = require('../config');
+const { fetchJson } = require('../http');
+
+async function fetchGdeltArticles() {
+  const articles = [];
+
+  for (const query of config.gdelt?.queries || []) {
+    try {
+      const params = new URLSearchParams({
+        query,
+        mode: config.gdelt.mode || 'ArtList',
+        maxrecords: String(Math.min(config.gdelt.maxRecords || 10, 10)),
+        format: config.gdelt.format || 'json',
+      });
+
+      const data = await fetchJson(`https://api.gdeltproject.org/api/v2/doc/doc?${params.toString()}`);
+      for (const item of data.articles || []) {
+        const title = String(item.title || '').trim();
+        const url = String(item.url || '').trim();
+
+        if (!title || !url) {
+          continue;
+        }
+
+        articles.push({
+          title,
+          description: item.domain || null,
+          url,
+          source: 'gdelt',
+          pubDate: item.seendate || null,
+        });
+      }
+    } catch (error) {
+      if (error && error.status === 429) {
+        console.warn(`GDELT query skipped for rate limit: ${query}`);
+        continue;
+      }
+
+      console.error(`Failed to fetch GDELT query: ${query}`, error);
+    }
+  }
+
+  return articles;
+}
+
+module.exports = {
+  fetchGdeltArticles,
+};
@@ -0,0 +1,127 @@
+const Parser = require('rss-parser');
+const config = require('../config');
+
+const parser = new Parser({
+  timeout: 10000,
+  headers: {
+    'User-Agent': 'Mozilla/5.0',
+    Accept: 'application/rss+xml, application/xml, text/xml;q=0.9, */*;q=0.8',
+  },
+});
+
+const blockedFeedDomains = [
+  'arabnews.com',
+  'arabianbusiness.com',
+  'business-standard.com',
+  'cityam.com',
+  'eleconomista.com.mx',
+  'eleconomista.es',
+  'moneycontrol.com',
+  'thisismoney.co.uk',
+];
+const invalidFeedLabels = new Set([
+  'ABC Business AU',
+  'Australian Fin Review',
+  'Business Daily Africa',
+  'BusinessLive SA',
+  'Caixin Global',
+  'Cinco Dias',
+  'El Comercio Peru',
+  'FD.nl',
+  'Gulf News Business',
+  'Il Sole 24 Ore',
+  'Infobae Economia AR',
+  'Japan Times Business',
+  'Korea JoongAng Daily',
+  'Les Echos',
+  'Live Mint',
+  'NZ Herald Business',
+  'Portafolio Colombia',
+  'The Star Malaysia',
+  'Xinhua Business',
+]);
+const malformedFeedLabels = new Set([
+  'BFM Business',
+  'Business Daily Africa',
+]);
+const loggedBlockedFeeds = new Set();
+const loggedInvalidFeeds = new Set();
+
+function getHostname(url) {
+  try {
+    return new URL(url).hostname.toLowerCase();
+  } catch {
+    return '';
+  }
+}
+
+function isBlockedFeed(feed) {
+  const hostname = getHostname(feed.url);
+  return blockedFeedDomains.some((domain) => hostname === domain || hostname.endsWith(`.${domain}`));
+}
+
+function isMalformedFeedError(error) {
+  const message = String(error && error.message || '');
+  return message.includes('Invalid character in entity name') || message.includes('Attribute without value');
+}
+
+async function fetchRssArticles() {
+  const articles = [];
+
+  for (const feed of config.rssFeeds || []) {
+    const label = feed.label || feed.url;
+
+    if (invalidFeedLabels.has(label)) {
+      if (!loggedInvalidFeeds.has(label)) {
+        loggedInvalidFeeds.add(label);
+        console.warn(`RSS feed skipped for invalid endpoint ${label}`);
+      }
+      continue;
+    }
+
+    if (isBlockedFeed(feed)) {
+      const hostname = getHostname(feed.url);
+      if (!loggedBlockedFeeds.has(hostname)) {
+        loggedBlockedFeeds.add(hostname);
+        console.warn(`RSS feed skipped for blocked domain ${hostname}`);
+      }
+      continue;
+    }
+
+    try {
+      const parsed = await parser.parseURL(feed.url);
+      for (const item of parsed.items || []) {
+        const title = String(item.title || '').trim();
+        const url = String(item.link || item.guid || '').trim();
+
+        if (!title || !url) {
+          continue;
+        }
+
+        articles.push({
+          title,
+          description: item.contentSnippet || item.content || item.summary || null,
+          url,
+          source: feed.label ? `rss:${feed.label}` : 'rss',
+          pubDate: item.isoDate || item.pubDate || null,
+        });
+      }
+    } catch (error) {
+      if (malformedFeedLabels.has(label) && isMalformedFeedError(error)) {
+        if (!loggedInvalidFeeds.has(label)) {
+          loggedInvalidFeeds.add(label);
+          console.warn(`RSS feed skipped for malformed XML ${label}`);
+        }
+        continue;
+      }
+
+      console.error(`Failed to fetch RSS feed: ${label}`, error);
+    }
+  }
+
+  return articles;
+}
+
+module.exports = {
+  fetchRssArticles,
+};