refactor content fetching and embedding processes for improved concurrency and error handling

2026-04-19 18:51:42 +01:00
parent b4b2fe2ac7
commit 6bf3a9282f
8 changed files with 348 additions and 157 deletions
@@ -43,11 +43,18 @@
    "googleNews": "0 * * * *"
  },
  "contentBackfill": {
-    "concurrency": 100,
-    "perSource": 50
+    "workers": 3,
+    "perSource": 50,
+    "batchSize": 25,
+    "plainConcurrency": 50,
+    "browserConcurrency": 8
+  },
+  "embeddingBackfill": {
+    "perRound": 256,
+    "batchSize": 16
  },
  "browser": {
-    "maxConcurrentPages": 25
+    "maxConcurrentPages": 8
  },
  "googleNews": {
    "queries": [