enhance article processing by adding language support and adjusting embedding parameters

This commit is contained in:
ImBenji
2026-04-20 03:41:10 +01:00
parent 37d9dfb083
commit 8805d3a3fc
7 changed files with 129 additions and 9 deletions
+3 -3
View File
@@ -65,8 +65,8 @@ const markContentPending = db.prepare(`
const selectPartitionedArticlesMissingContent = db.prepare(`
SELECT id, url, title, description
FROM (
SELECT id, url, title, description, source,
ROW_NUMBER() OVER (PARTITION BY source ORDER BY ingested_at DESC, id DESC) AS rn
SELECT id, url, title, description, source, pub_date_effective,
ROW_NUMBER() OVER (PARTITION BY source ORDER BY pub_date_effective DESC, id DESC) AS rn
FROM articles
WHERE (content IS NULL OR TRIM(content) = '')
AND (content_status IS NULL OR content_status = 'pending')
@@ -74,7 +74,7 @@ const selectPartitionedArticlesMissingContent = db.prepare(`
AND (id % ?) = ?
)
WHERE rn <= ?
ORDER BY rn, source
ORDER BY pub_date_effective DESC, rn, source
`);
const selectAttemptCount = db.prepare(`