IMBENJI.NET/Duriin-API
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

enhance article handling with index page detection and query improvements

Browse Source
This commit is contained in:
ImBenji
2026-04-17 01:02:39 +01:00
parent b298be6108
commit 01bc624a24
4 changed files with 86 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/routes/articles.js
+41 -41
View File
@@ -51,6 +51,39 @@ function buildArticlesQuery(query) {
};
}
function getRequestedLimit(value) {
const limit = Number.parseInt(value, 10);
return Number.isFinite(limit) && limit > 0 ? Math.min(limit, 100) : 20;
}
function shouldExcludeIndexPages(query) {
return String(query.exclude_index_pages || '').toLowerCase() !== 'false';
}
function mapNeighborsToArticles(neighbors, excludeIndexPages, limit) {
const ids = neighbors.map((row) => row.articleId);
if (ids.length === 0) {
return [];
}
const placeholders = ids.map(() => '?').join(', ');
const articles = db.prepare(`
SELECT id, title, description, content, image, url, normalized_title, source, pub_date, ingested_at
FROM articles
WHERE id IN (${placeholders})
${excludeIndexPages ? 'AND is_index_page = 0' : ''}
`).all(...ids);
const byId = new Map(articles.map((article) => [article.id, article]));
return neighbors
.map((row) => {
const article = byId.get(row.articleId);
return article ? { ...article, distance: row.distance } : null;
})
.filter(Boolean)
.slice(0, limit);
}
async function articleRoutes(fastify) {
fastify.get('/articles', async (request, reply) => {
const query = request.query || {};
@@ -60,7 +93,8 @@ async function articleRoutes(fastify) {
}
if (query.semantic !== undefined) {
const limit = Number.parseInt(query.limit, 10);
const limit = getRequestedLimit(query.limit);
const excludeIndexPages = shouldExcludeIndexPages(query);
const embedding = await getOrCreateQueryEmbedding(query.semantic);
if (!embedding) {
@@ -70,35 +104,19 @@ async function articleRoutes(fastify) {
const neighbors = findArticlesByEmbedding(
embedding,
Number.isFinite(limit) && limit > 0 ? Math.min(limit, 100) : 20
Math.min(limit * 5, 500)
);
const ids = neighbors.map((row) => row.articleId);
if (ids.length === 0) {
return [];
}
const placeholders = ids.map(() => '?').join(', ');
const articles = db.prepare(`
SELECT id, title, description, content, image, url, normalized_title, source, pub_date, ingested_at
FROM articles
WHERE id IN (${placeholders})
`).all(...ids);
const byId = new Map(articles.map((article) => [article.id, article]));
return neighbors
.map((row) => {
const article = byId.get(row.articleId);
return article ? { ...article, distance: row.distance } : null;
})
.filter(Boolean);
return mapNeighborsToArticles(neighbors, excludeIndexPages, limit);
}
if (query.similar_to_article) {
const limit = Number.parseInt(query.limit, 10);
const limit = getRequestedLimit(query.limit);
const excludeIndexPages = shouldExcludeIndexPages(query);
const articleId = Number.parseInt(query.similar_to_article, 10);
const neighbors = findSimilarArticles(
articleId,
Number.isFinite(limit) && limit > 0 ? Math.min(limit, 100) : 20
Math.min(limit * 5, 500)
);
if (neighbors.length === 0 && !getEmbeddingBuffer(articleId)) {
@@ -106,25 +124,7 @@ async function articleRoutes(fastify) {
return { error: 'Embedding not found for article' };
}
const ids = neighbors.map((row) => row.articleId);
if (ids.length === 0) {
return [];
}
const placeholders = ids.map(() => '?').join(', ');
const articles = db.prepare(`
SELECT id, title, description, content, image, url, normalized_title, source, pub_date, ingested_at
FROM articles
WHERE id IN (${placeholders})
`).all(...ids);
const byId = new Map(articles.map((article) => [article.id, article]));
return neighbors
.map((row) => {
const article = byId.get(row.articleId);
return article ? { ...article, distance: row.distance } : null;
})
.filter(Boolean);
return mapNeighborsToArticles(neighbors, excludeIndexPages, limit);
}
const { sql, params } = buildArticlesQuery(query);