initialize project structure with core modules and configuration

2026-04-16 22:47:34 +01:00
parent 82d856a21d
commit 7724fafbdc
20 changed files with 3644 additions and 0 deletions
@@ -0,0 +1,155 @@
+const db = require('../db');
+const {
+  findArticlesByEmbedding,
+  findSimilarArticles,
+  getEmbeddingBuffer,
+  getOrCreateQueryEmbedding,
+} = require('../embeddings');
+
+function buildArticlesQuery(query) {
+  const conditions = [];
+  const params = [];
+  const includeEmbedding = String(query.include_embedding || '').toLowerCase() === 'true';
+
+  if (query.q) {
+    conditions.push('(title LIKE ? OR description LIKE ? OR content LIKE ?)');
+    const keyword = `%${query.q}%`;
+    params.push(keyword, keyword, keyword);
+  }
+
+  if (query.source) {
+    conditions.push('source = ?');
+    params.push(query.source);
+  }
+
+  if (query.from) {
+    conditions.push('pub_date >= ?');
+    params.push(query.from);
+  }
+
+  if (query.to) {
+    conditions.push('pub_date <= ?');
+    params.push(query.to);
+  }
+
+  const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
+  const limit = Number.parseInt(query.limit, 10);
+  const offset = Number.parseInt(query.offset, 10);
+
+  params.push(Number.isFinite(limit) && limit > 0 ? Math.min(limit, 100) : 20);
+  params.push(Number.isFinite(offset) && offset >= 0 ? offset : 0);
+
+  return {
+    sql: `
+      SELECT id, title, description, content, image, ${includeEmbedding ? 'embedding,' : ''} url, normalized_title, source, pub_date, ingested_at
+      FROM articles
+      ${whereClause}
+      ORDER BY COALESCE(pub_date, ingested_at) DESC, id DESC
+      LIMIT ? OFFSET ?
+    `,
+    params,
+  };
+}
+
+async function articleRoutes(fastify) {
+  fastify.get('/articles', async (request, reply) => {
+    const query = request.query || {};
+    if (query.include_embedding) {
+      reply.code(400);
+      return { error: 'Embeddings are not returned directly. Use similar_to for vector search.' };
+    }
+
+    if (query.topic !== undefined) {
+      const limit = Number.parseInt(query.limit, 10);
+      const embedding = await getOrCreateQueryEmbedding(query.topic);
+
+      if (!embedding) {
+        reply.code(400);
+        return { error: 'Topic must not be empty' };
+      }
+
+      const neighbors = findArticlesByEmbedding(
+        embedding,
+        Number.isFinite(limit) && limit > 0 ? Math.min(limit, 100) : 20
+      );
+      const ids = neighbors.map((row) => row.articleId);
+      if (ids.length === 0) {
+        return [];
+      }
+
+      const placeholders = ids.map(() => '?').join(', ');
+      const articles = db.prepare(`
+        SELECT id, title, description, content, image, url, normalized_title, source, pub_date, ingested_at
+        FROM articles
+        WHERE id IN (${placeholders})
+      `).all(...ids);
+      const byId = new Map(articles.map((article) => [article.id, article]));
+
+      return neighbors
+        .map((row) => {
+          const article = byId.get(row.articleId);
+          return article ? { ...article, distance: row.distance } : null;
+        })
+        .filter(Boolean);
+    }
+
+    if (query.similar_to) {
+      const limit = Number.parseInt(query.limit, 10);
+      const articleId = Number.parseInt(query.similar_to, 10);
+      const neighbors = findSimilarArticles(
+        articleId,
+        Number.isFinite(limit) && limit > 0 ? Math.min(limit, 100) : 20
+      );
+
+      if (neighbors.length === 0 && !getEmbeddingBuffer(articleId)) {
+        reply.code(404);
+        return { error: 'Embedding not found for article' };
+      }
+
+      const ids = neighbors.map((row) => row.articleId);
+      if (ids.length === 0) {
+        return [];
+      }
+
+      const placeholders = ids.map(() => '?').join(', ');
+      const articles = db.prepare(`
+        SELECT id, title, description, content, image, url, normalized_title, source, pub_date, ingested_at
+        FROM articles
+        WHERE id IN (${placeholders})
+      `).all(...ids);
+      const byId = new Map(articles.map((article) => [article.id, article]));
+
+      return neighbors
+        .map((row) => {
+          const article = byId.get(row.articleId);
+          return article ? { ...article, distance: row.distance } : null;
+        })
+        .filter(Boolean);
+    }
+
+    const { sql, params } = buildArticlesQuery(query);
+    return db.prepare(sql).all(...params);
+  });
+
+  fastify.get('/articles/:id', async (request, reply) => {
+    if (String((request.query || {}).include_embedding || '').toLowerCase() === 'true') {
+      reply.code(400);
+      return { error: 'Embeddings are not returned directly. Use similar_to for vector search.' };
+    }
+
+    const article = db.prepare(`
+      SELECT id, title, description, content, image, url, normalized_title, source, pub_date, ingested_at
+      FROM articles
+      WHERE id = ?
+    `).get(request.params.id);
+
+    if (!article) {
+      reply.code(404);
+      return { error: 'Article not found' };
+    }
+
+    return article;
+  });
+}
+
+module.exports = articleRoutes;
@@ -0,0 +1,34 @@
+const db = require('../db');
+const { getLastIngestionBySource } = require('../state');
+
+async function statusRoutes(fastify) {
+  fastify.get('/status', async () => {
+    const total = db.prepare('SELECT COUNT(*) AS count FROM articles').get().count;
+    const bySourceRows = db.prepare('SELECT source, COUNT(*) AS count FROM articles GROUP BY source ORDER BY source').all();
+    const contentCoverage = db.prepare(`
+      SELECT
+        COUNT(*) AS total,
+        SUM(CASE WHEN content IS NOT NULL AND TRIM(content) <> '' THEN 1 ELSE 0 END) AS with_content,
+        SUM(CASE WHEN image IS NOT NULL AND TRIM(image) <> '' THEN 1 ELSE 0 END) AS with_image
+      FROM articles
+    `).get();
+    const embeddingCoverage = db.prepare('SELECT COUNT(*) AS count FROM article_embeddings').get();
+
+    return {
+      totalArticles: total,
+      countsBySource: Object.fromEntries(bySourceRows.map((row) => [row.source, row.count])),
+      lastIngestionBySource: getLastIngestionBySource(),
+      contentFetchCoverage: {
+        withContent: contentCoverage.with_content || 0,
+        withImage: contentCoverage.with_image || 0,
+        withEmbedding: embeddingCoverage.count || 0,
+        total: contentCoverage.total || 0,
+        contentRatio: contentCoverage.total ? Number((contentCoverage.with_content / contentCoverage.total).toFixed(4)) : 0,
+        imageRatio: contentCoverage.total ? Number((contentCoverage.with_image / contentCoverage.total).toFixed(4)) : 0,
+        embeddingRatio: contentCoverage.total ? Number((embeddingCoverage.count / contentCoverage.total).toFixed(4)) : 0,
+      },
+    };
+  });
+}
+
+module.exports = statusRoutes;