From 7724fafbdc9149c4626b1ae16f335de93535c8bd Mon Sep 17 00:00:00 2001 From: ImBenji Date: Thu, 16 Apr 2026 22:47:34 +0100 Subject: [PATCH] initialize project structure with core modules and configuration --- .gitignore | 14 + package-lock.json | 2133 +++++++++++++++++++++++++++++++++++ package.json | 22 + server.js | 27 + src/config.js | 7 + src/content.js | 203 ++++ src/db.js | 141 +++ src/dedup.js | 11 + src/embeddings.js | 242 ++++ src/http.js | 96 ++ src/ingest.js | 120 ++ src/routes/articles.js | 155 +++ src/routes/status.js | 34 + src/scheduler.js | 85 ++ src/sources/alphavantage.js | 32 + src/sources/edgar.js | 78 ++ src/sources/finnhub.js | 55 + src/sources/gdelt.js | 48 + src/sources/rss.js | 127 +++ src/state.js | 14 + 20 files changed, 3644 insertions(+) create mode 100644 .gitignore create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 server.js create mode 100644 src/config.js create mode 100644 src/content.js create mode 100644 src/db.js create mode 100644 src/dedup.js create mode 100644 src/embeddings.js create mode 100644 src/http.js create mode 100644 src/ingest.js create mode 100644 src/routes/articles.js create mode 100644 src/routes/status.js create mode 100644 src/scheduler.js create mode 100644 src/sources/alphavantage.js create mode 100644 src/sources/edgar.js create mode 100644 src/sources/finnhub.js create mode 100644 src/sources/gdelt.js create mode 100644 src/sources/rss.js create mode 100644 src/state.js diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..61317ae --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +node_modules/ +.env +.env.* + +config.json + +*.sqlite +*.sqlite-shm +*.sqlite-wal + +.idea/ +.DS_Store + +.the_agency/ \ No newline at end of file diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..bd8ffd7 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,2133 @@ +{ + "name": "duriin_api", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "duriin_api", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "@extractus/article-extractor": "^8.0.18", + "better-sqlite3": "^12.4.1", + "fastify": "^5.6.1", + "node-cron": "^4.2.1", + "rss-parser": "^3.13.0", + "sharp": "^0.34.5", + "sqlite-vec": "^0.1.9" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", + "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@extractus/article-extractor": { + "version": "8.0.20", + "resolved": "https://registry.npmjs.org/@extractus/article-extractor/-/article-extractor-8.0.20.tgz", + "integrity": "sha512-oxHLZ3X5ctLVkQfFkOLf8afvQq6aJ2VBxwQhAaV6ZypaaMJboFz8uwpCGy7QBehmQIvzgWhCwuu8j4ayJFvPcg==", + "license": "MIT", + "dependencies": { + "@mozilla/readability": "^0.6.0", + "@ndaidong/bellajs": "^12.0.1", + "cross-fetch": "^4.1.0", + "linkedom": "^0.18.12", + "sanitize-html": "2.17.0" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/@fastify/ajv-compiler": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@fastify/ajv-compiler/-/ajv-compiler-4.0.5.tgz", + "integrity": "sha512-KoWKW+MhvfTRWL4qrhUwAAZoaChluo0m0vbiJlGMt2GXvL4LVPQEjt8kSpHI3IBq5Rez8fg+XeH3cneztq+C7A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT", + "dependencies": { + "ajv": "^8.12.0", + "ajv-formats": "^3.0.1", + "fast-uri": "^3.0.0" + } + }, + "node_modules/@fastify/error": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@fastify/error/-/error-4.2.0.tgz", + "integrity": "sha512-RSo3sVDXfHskiBZKBPRgnQTtIqpi/7zhJOEmAxCiBcM7d0uwdGdxLlsCaLzGs8v8NnxIRlfG0N51p5yFaOentQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT" + }, + "node_modules/@fastify/fast-json-stringify-compiler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/@fastify/fast-json-stringify-compiler/-/fast-json-stringify-compiler-5.0.3.tgz", + "integrity": "sha512-uik7yYHkLr6fxd8hJSZ8c+xF4WafPK+XzneQDPU+D10r5X19GW8lJcom2YijX2+qtFF1ENJlHXKFM9ouXNJYgQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT", + "dependencies": { + "fast-json-stringify": "^6.0.0" + } + }, + "node_modules/@fastify/forwarded": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@fastify/forwarded/-/forwarded-3.0.1.tgz", + "integrity": "sha512-JqDochHFqXs3C3Ml3gOY58zM7OqO9ENqPo0UqAjAjH8L01fRZqwX9iLeX34//kiJubF7r2ZQHtBRU36vONbLlw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT" + }, + "node_modules/@fastify/merge-json-schemas": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/@fastify/merge-json-schemas/-/merge-json-schemas-0.2.1.tgz", + "integrity": "sha512-OA3KGBCy6KtIvLf8DINC5880o5iBlDX4SxzLQS8HorJAbqluzLRn80UXU0bxZn7UOFhFgpRJDasfwn9nG4FG4A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT", + "dependencies": { + "dequal": "^2.0.3" + } + }, + "node_modules/@fastify/proxy-addr": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@fastify/proxy-addr/-/proxy-addr-5.1.0.tgz", + "integrity": "sha512-INS+6gh91cLUjB+PVHfu1UqcB76Sqtpyp7bnL+FYojhjygvOPA9ctiD/JDKsyD9Xgu4hUhCSJBPig/w7duNajw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT", + "dependencies": { + "@fastify/forwarded": "^3.0.0", + "ipaddr.js": "^2.1.0" + } + }, + "node_modules/@img/colour": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", + "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", + "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", + "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", + "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", + "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", + "cpu": [ + "arm" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", + "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", + "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", + "cpu": [ + "ppc64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", + "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", + "cpu": [ + "riscv64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", + "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", + "cpu": [ + "s390x" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", + "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", + "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", + "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", + "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", + "cpu": [ + "arm" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", + "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", + "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", + "cpu": [ + "ppc64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", + "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", + "cpu": [ + "riscv64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", + "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", + "cpu": [ + "s390x" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", + "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", + "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", + "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", + "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.7.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", + "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", + "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", + "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@mozilla/readability": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@mozilla/readability/-/readability-0.6.0.tgz", + "integrity": "sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@ndaidong/bellajs": { + "version": "12.0.1", + "resolved": "https://registry.npmjs.org/@ndaidong/bellajs/-/bellajs-12.0.1.tgz", + "integrity": "sha512-1iY42uiHz0cxNMbde7O3zVN+ZX1viOOUOBRt6ht6lkRZbSjwOnFV34Zv4URp3hGzEe6L9Byk7BOq/41H0PzAOQ==", + "license": "MIT" + }, + "node_modules/@pinojs/redact": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz", + "integrity": "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==", + "license": "MIT" + }, + "node_modules/abstract-logging": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/abstract-logging/-/abstract-logging-2.0.1.tgz", + "integrity": "sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA==", + "license": "MIT" + }, + "node_modules/ajv": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", + "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/atomic-sleep": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", + "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/avvio": { + "version": "9.2.0", + "resolved": "https://registry.npmjs.org/avvio/-/avvio-9.2.0.tgz", + "integrity": "sha512-2t/sy01ArdHHE0vRH5Hsay+RtCZt3dLPji7W7/MMOCEgze5b7SNDC4j5H6FnVgPkI1MTNFGzHdHrVXDDl7QSSQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT", + "dependencies": { + "@fastify/error": "^4.0.0", + "fastq": "^1.17.1" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/better-sqlite3": { + "version": "12.9.0", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-12.9.0.tgz", + "integrity": "sha512-wqUv4Gm3toFpHDQmaKD4QhZm3g1DjUBI0yzS4UBl6lElUmXFYdTQmmEDpAFa5o8FiFiymURypEnfVHzILKaxqQ==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "bindings": "^1.5.0", + "prebuild-install": "^7.1.1" + }, + "engines": { + "node": "20.x || 22.x || 23.x || 24.x || 25.x" + } + }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "license": "MIT", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "license": "ISC" + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "license": "ISC" + }, + "node_modules/cookie": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", + "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/cross-fetch": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-4.1.0.tgz", + "integrity": "sha512-uKm5PU+MHTootlWEY+mZ4vvXoCn4fLQxT9dSc1sXVMSFkINTJVN8cAQROpwcKm8bJ/c7rgZVIBWzH5T78sNZZw==", + "license": "MIT", + "dependencies": { + "node-fetch": "^2.7.0" + } + }, + "node_modules/css-select": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz", + "integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz", + "integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/cssom": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz", + "integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==", + "license": "MIT" + }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "license": "MIT", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/deepmerge": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", + "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "license": "(MIT OR WTFPL)", + "engines": { + "node": ">=6" + } + }, + "node_modules/fast-decode-uri-component": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/fast-decode-uri-component/-/fast-decode-uri-component-1.0.1.tgz", + "integrity": "sha512-WKgKWg5eUxvRZGwW8FvfbaH7AXSh2cL+3j5fMGzUMCxWBJ3dV3a7Wz8y2f/uQ0e3B6WmodD3oS54jTQ9HVTIIg==", + "license": "MIT" + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-json-stringify": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/fast-json-stringify/-/fast-json-stringify-6.3.0.tgz", + "integrity": "sha512-oRCntNDY/329HJPlmdNLIdogNtt6Vyjb1WuT01Soss3slIdyUp8kAcDU3saQTOquEK8KFVfwIIF7FebxUAu+yA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT", + "dependencies": { + "@fastify/merge-json-schemas": "^0.2.0", + "ajv": "^8.12.0", + "ajv-formats": "^3.0.1", + "fast-uri": "^3.0.0", + "json-schema-ref-resolver": "^3.0.0", + "rfdc": "^1.2.0" + } + }, + "node_modules/fast-querystring": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/fast-querystring/-/fast-querystring-1.1.2.tgz", + "integrity": "sha512-g6KuKWmFXc0fID8WWH0jit4g0AGBoJhCkJMb1RmbsSEUNvQ+ZC8D6CUZ+GtF8nMzSPXnhiePyyqqipzNNEnHjg==", + "license": "MIT", + "dependencies": { + "fast-decode-uri-component": "^1.0.1" + } + }, + "node_modules/fast-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", + "integrity": "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/fastify": { + "version": "5.8.5", + "resolved": "https://registry.npmjs.org/fastify/-/fastify-5.8.5.tgz", + "integrity": "sha512-Yqptv59pQzPgQUSIm87hMqHJmdkb1+GPxdE6vW6FRyVE9G86mt7rOghitiU4JHRaTyDUk9pfeKmDeu70lAwM4Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT", + "dependencies": { + "@fastify/ajv-compiler": "^4.0.5", + "@fastify/error": "^4.0.0", + "@fastify/fast-json-stringify-compiler": "^5.0.0", + "@fastify/proxy-addr": "^5.0.0", + "abstract-logging": "^2.0.1", + "avvio": "^9.0.0", + "fast-json-stringify": "^6.0.0", + "find-my-way": "^9.0.0", + "light-my-request": "^6.0.0", + "pino": "^9.14.0 || ^10.1.0", + "process-warning": "^5.0.0", + "rfdc": "^1.3.1", + "secure-json-parse": "^4.0.0", + "semver": "^7.6.0", + "toad-cache": "^3.7.0" + } + }, + "node_modules/fastq": { + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", + "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==", + "license": "ISC", + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "license": "MIT" + }, + "node_modules/find-my-way": { + "version": "9.5.0", + "resolved": "https://registry.npmjs.org/find-my-way/-/find-my-way-9.5.0.tgz", + "integrity": "sha512-VW2RfnmscZO5KgBY5XVyKREMW5nMZcxDy+buTOsL+zIPnBlbKm+00sgzoQzq1EVh4aALZLfKdwv6atBGcjvjrQ==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-querystring": "^1.0.0", + "safe-regex2": "^5.0.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "license": "MIT" + }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", + "license": "MIT" + }, + "node_modules/html-escaper": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-3.0.3.tgz", + "integrity": "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==", + "license": "MIT" + }, + "node_modules/htmlparser2": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", + "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.2", + "entities": "^7.0.1" + } + }, + "node_modules/htmlparser2/node_modules/entities": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", + "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.3.0.tgz", + "integrity": "sha512-Zv/pA+ciVFbCSBBjGfaKUya/CcGmUHzTydLMaTwrUUEM2DIEO3iZvueGxmacvmN50fGpGVKeTXpb2LcYQxeVdg==", + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, + "node_modules/is-plain-object": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", + "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/json-schema-ref-resolver": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/json-schema-ref-resolver/-/json-schema-ref-resolver-3.0.0.tgz", + "integrity": "sha512-hOrZIVL5jyYFjzk7+y7n5JDzGlU8rfWDuYyHwGa2WA8/pcmMHezp2xsVwxrebD/Q9t8Nc5DboieySDpCp4WG4A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT", + "dependencies": { + "dequal": "^2.0.3" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/light-my-request": { + "version": "6.6.0", + "resolved": "https://registry.npmjs.org/light-my-request/-/light-my-request-6.6.0.tgz", + "integrity": "sha512-CHYbu8RtboSIoVsHZ6Ye4cj4Aw/yg2oAFimlF7mNvfDV192LR7nDiKtSIfCuLT7KokPSTn/9kfVLm5OGN0A28A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause", + "dependencies": { + "cookie": "^1.0.1", + "process-warning": "^4.0.0", + "set-cookie-parser": "^2.6.0" + } + }, + "node_modules/light-my-request/node_modules/process-warning": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-4.0.1.tgz", + "integrity": "sha512-3c2LzQ3rY9d0hc1emcsHhfT9Jwz0cChib/QN89oME2R451w5fy3f0afAhERFZAwrbDU43wk12d0ORBpDVME50Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT" + }, + "node_modules/linkedom": { + "version": "0.18.12", + "resolved": "https://registry.npmjs.org/linkedom/-/linkedom-0.18.12.tgz", + "integrity": "sha512-jalJsOwIKuQJSeTvsgzPe9iJzyfVaEJiEXl+25EkKevsULHvMJzpNqwvj1jOESWdmgKDiXObyjOYwlUqG7wo1Q==", + "license": "ISC", + "dependencies": { + "css-select": "^5.1.0", + "cssom": "^0.5.0", + "html-escaper": "^3.0.3", + "htmlparser2": "^10.0.0", + "uhyphen": "^0.2.0" + }, + "engines": { + "node": ">=16" + }, + "peerDependencies": { + "canvas": ">= 2" + }, + "peerDependenciesMeta": { + "canvas": { + "optional": true + } + } + }, + "node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/napi-build-utils": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", + "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==", + "license": "MIT" + }, + "node_modules/node-abi": { + "version": "3.89.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.89.0.tgz", + "integrity": "sha512-6u9UwL0HlAl21+agMN3YAMXcKByMqwGx+pq+P76vii5f7hTPtKDp08/H9py6DY+cfDw7kQNTGEj/rly3IgbNQA==", + "license": "MIT", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/node-cron": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-4.2.1.tgz", + "integrity": "sha512-lgimEHPE/QDgFlywTd8yTR61ptugX3Qer29efeyWw2rv259HtGBNn1vZVmp8lB9uo9wC0t/AT4iGqXxia+CJFg==", + "license": "ISC", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, + "node_modules/on-exit-leak-free": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", + "integrity": "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==", + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/parse-srcset": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/parse-srcset/-/parse-srcset-1.0.2.tgz", + "integrity": "sha512-/2qh0lav6CmI15FzA3i/2Bzk2zCgQhGMkvhOhKNcBVQ1ldgpbfiNTVslmooUmWJcADi1f1kIeynbDRVzNlfR6Q==", + "license": "MIT" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/pino": { + "version": "10.3.1", + "resolved": "https://registry.npmjs.org/pino/-/pino-10.3.1.tgz", + "integrity": "sha512-r34yH/GlQpKZbU1BvFFqOjhISRo1MNx1tWYsYvmj6KIRHSPMT2+yHOEb1SG6NMvRoHRF0a07kCOox/9yakl1vg==", + "license": "MIT", + "dependencies": { + "@pinojs/redact": "^0.4.0", + "atomic-sleep": "^1.0.0", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^3.0.0", + "pino-std-serializers": "^7.0.0", + "process-warning": "^5.0.0", + "quick-format-unescaped": "^4.0.3", + "real-require": "^0.2.0", + "safe-stable-stringify": "^2.3.1", + "sonic-boom": "^4.0.1", + "thread-stream": "^4.0.0" + }, + "bin": { + "pino": "bin.js" + } + }, + "node_modules/pino-abstract-transport": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-3.0.0.tgz", + "integrity": "sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==", + "license": "MIT", + "dependencies": { + "split2": "^4.0.0" + } + }, + "node_modules/pino-std-serializers": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-7.1.0.tgz", + "integrity": "sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==", + "license": "MIT" + }, + "node_modules/postcss": { + "version": "8.5.10", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.10.tgz", + "integrity": "sha512-pMMHxBOZKFU6HgAZ4eyGnwXF/EvPGGqUr0MnZ5+99485wwW41kW91A4LOGxSHhgugZmSChL5AlElNdwlNgcnLQ==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.11", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/prebuild-install": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", + "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==", + "deprecated": "No longer maintained. Please contact the author of the relevant native addon; alternatives are available.", + "license": "MIT", + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^2.0.0", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/process-warning": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.0.0.tgz", + "integrity": "sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT" + }, + "node_modules/pump": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/quick-format-unescaped": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", + "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==", + "license": "MIT" + }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/real-require": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", + "integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==", + "license": "MIT", + "engines": { + "node": ">= 12.13.0" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ret": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/ret/-/ret-0.5.0.tgz", + "integrity": "sha512-I1XxrZSQ+oErkRR4jYbAyEEu2I0avBvvMM5JN+6EBprOGRCs63ENqZ3vjavq8fBw2+62G5LF5XelKwuJpcvcxw==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/reusify": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", + "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", + "license": "MIT", + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/rfdc": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.4.1.tgz", + "integrity": "sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==", + "license": "MIT" + }, + "node_modules/rss-parser": { + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/rss-parser/-/rss-parser-3.13.0.tgz", + "integrity": "sha512-7jWUBV5yGN3rqMMj7CZufl/291QAhvrrGpDNE4k/02ZchL0npisiYYqULF71jCEKoIiHvK/Q2e6IkDwPziT7+w==", + "license": "MIT", + "dependencies": { + "entities": "^2.0.3", + "xml2js": "^0.5.0" + } + }, + "node_modules/rss-parser/node_modules/entities": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz", + "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==", + "license": "BSD-2-Clause", + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safe-regex2": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/safe-regex2/-/safe-regex2-5.1.0.tgz", + "integrity": "sha512-pNHAuBW7TrcleFHsxBr5QMi/Iyp0ENjUKz7GCcX1UO7cMh+NmVK6HxQckNL1tJp1XAJVjG6B8OKIPqodqj9rtw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT", + "dependencies": { + "ret": "~0.5.0" + }, + "bin": { + "safe-regex2": "bin/safe-regex2.js" + } + }, + "node_modules/safe-stable-stringify": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", + "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/sanitize-html": { + "version": "2.17.0", + "resolved": "https://registry.npmjs.org/sanitize-html/-/sanitize-html-2.17.0.tgz", + "integrity": "sha512-dLAADUSS8rBwhaevT12yCezvioCA+bmUTPH/u57xKPT8d++voeYE6HeluA/bPbQ15TwDBG2ii+QZIEmYx8VdxA==", + "license": "MIT", + "dependencies": { + "deepmerge": "^4.2.2", + "escape-string-regexp": "^4.0.0", + "htmlparser2": "^8.0.0", + "is-plain-object": "^5.0.0", + "parse-srcset": "^1.0.2", + "postcss": "^8.3.11" + } + }, + "node_modules/sanitize-html/node_modules/htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } + }, + "node_modules/sax": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.6.0.tgz", + "integrity": "sha512-6R3J5M4AcbtLUdZmRv2SygeVaM7IhrLXu9BmnOGmmACak8fiUtOsYNWUS4uK7upbmHIBbLBeFeI//477BKLBzA==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=11.0.0" + } + }, + "node_modules/secure-json-parse": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-4.1.0.tgz", + "integrity": "sha512-l4KnYfEyqYJxDwlNVyRfO2E4NTHfMKAWdUuA8J0yve2Dz/E/PdBepY03RvyJpssIpRFwJoCD55wA+mEDs6ByWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/set-cookie-parser": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.7.2.tgz", + "integrity": "sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==", + "license": "MIT" + }, + "node_modules/sharp": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", + "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@img/colour": "^1.0.0", + "detect-libc": "^2.1.2", + "semver": "^7.7.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.34.5", + "@img/sharp-darwin-x64": "0.34.5", + "@img/sharp-libvips-darwin-arm64": "1.2.4", + "@img/sharp-libvips-darwin-x64": "1.2.4", + "@img/sharp-libvips-linux-arm": "1.2.4", + "@img/sharp-libvips-linux-arm64": "1.2.4", + "@img/sharp-libvips-linux-ppc64": "1.2.4", + "@img/sharp-libvips-linux-riscv64": "1.2.4", + "@img/sharp-libvips-linux-s390x": "1.2.4", + "@img/sharp-libvips-linux-x64": "1.2.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", + "@img/sharp-libvips-linuxmusl-x64": "1.2.4", + "@img/sharp-linux-arm": "0.34.5", + "@img/sharp-linux-arm64": "0.34.5", + "@img/sharp-linux-ppc64": "0.34.5", + "@img/sharp-linux-riscv64": "0.34.5", + "@img/sharp-linux-s390x": "0.34.5", + "@img/sharp-linux-x64": "0.34.5", + "@img/sharp-linuxmusl-arm64": "0.34.5", + "@img/sharp-linuxmusl-x64": "0.34.5", + "@img/sharp-wasm32": "0.34.5", + "@img/sharp-win32-arm64": "0.34.5", + "@img/sharp-win32-ia32": "0.34.5", + "@img/sharp-win32-x64": "0.34.5" + } + }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, + "node_modules/sonic-boom": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.1.tgz", + "integrity": "sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==", + "license": "MIT", + "dependencies": { + "atomic-sleep": "^1.0.0" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, + "node_modules/sqlite-vec": { + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/sqlite-vec/-/sqlite-vec-0.1.9.tgz", + "integrity": "sha512-L7XJWRIBNvR9O5+vh1FQ+IGkh/3D2AzVksW5gdtk28m78Hy8skFD0pqReKH1Yp0/BUKRGcffgKvyO/EON5JXpA==", + "license": "MIT OR Apache", + "optionalDependencies": { + "sqlite-vec-darwin-arm64": "0.1.9", + "sqlite-vec-darwin-x64": "0.1.9", + "sqlite-vec-linux-arm64": "0.1.9", + "sqlite-vec-linux-x64": "0.1.9", + "sqlite-vec-windows-x64": "0.1.9" + } + }, + "node_modules/sqlite-vec-darwin-arm64": { + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/sqlite-vec-darwin-arm64/-/sqlite-vec-darwin-arm64-0.1.9.tgz", + "integrity": "sha512-jSsZpE42OfBkGL/ItyJTVCUwl6o6Ka3U5rc4j+UBDIQzC1ulSSKMEhQLthsOnF/MdAf1MuAkYhkdKmmcjaIZQg==", + "cpu": [ + "arm64" + ], + "license": "MIT OR Apache", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/sqlite-vec-darwin-x64": { + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/sqlite-vec-darwin-x64/-/sqlite-vec-darwin-x64-0.1.9.tgz", + "integrity": "sha512-KDlVyqQT7pnOhU1ymB9gs7dMbSoVmKHitT+k1/xkjarcX8bBqPxWrGlK/R+C5WmWkfvWwyq5FfXfiBYCBs6PlA==", + "cpu": [ + "x64" + ], + "license": "MIT OR Apache", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/sqlite-vec-linux-arm64": { + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/sqlite-vec-linux-arm64/-/sqlite-vec-linux-arm64-0.1.9.tgz", + "integrity": "sha512-5wXVJ9c9kR4CHm/wVqXb/R+XUHTdpZ4nWbPHlS+gc9qQFVHs92Km4bPnCKX4rtcPMzvNis+SIzMJR1SCEwpuUw==", + "cpu": [ + "arm64" + ], + "license": "MIT OR Apache", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/sqlite-vec-linux-x64": { + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/sqlite-vec-linux-x64/-/sqlite-vec-linux-x64-0.1.9.tgz", + "integrity": "sha512-w3tCH8xK2finW8fQJ/m8uqKodXUZ9KAuAar2UIhz4BHILfpE0WM/MTGCRfa7RjYbrYim5Luk3guvMOGI7T7JQA==", + "cpu": [ + "x64" + ], + "license": "MIT OR Apache", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/sqlite-vec-windows-x64": { + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/sqlite-vec-windows-x64/-/sqlite-vec-windows-x64-0.1.9.tgz", + "integrity": "sha512-y3gEIyy/17bq2QFPQOWLE68TYWcRZkBQVA2XLrTPHNTOp55xJi/BBBmOm40tVMDMjtP+Elpk6UBUXdaq+46b0Q==", + "cpu": [ + "x64" + ], + "license": "MIT OR Apache", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tar-fs": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", + "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==", + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/thread-stream": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-4.0.0.tgz", + "integrity": "sha512-4iMVL6HAINXWf1ZKZjIPcz5wYaOdPhtO8ATvZ+Xqp3BTdaqtAwQkNmKORqcIo5YkQqGXq5cwfswDwMqqQNrpJA==", + "license": "MIT", + "dependencies": { + "real-require": "^0.2.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/toad-cache": { + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/toad-cache/-/toad-cache-3.7.0.tgz", + "integrity": "sha512-/m8M+2BJUpoJdgAHoG+baCwBT+tf2VraSfkBgl0Y00qIWt41DJ8R5B8nsEw0I58YwF5IZH6z24/2TobDKnqSWw==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD", + "optional": true + }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, + "node_modules/uhyphen": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/uhyphen/-/uhyphen-0.2.0.tgz", + "integrity": "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA==", + "license": "ISC" + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/xml2js": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz", + "integrity": "sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==", + "license": "MIT", + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", + "license": "MIT", + "engines": { + "node": ">=4.0" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..cc1b8f8 --- /dev/null +++ b/package.json @@ -0,0 +1,22 @@ +{ + "name": "duriin_api", + "version": "1.0.0", + "description": "News ingestion archive server", + "main": "server.js", + "scripts": { + "start": "node server.js" + }, + "keywords": [], + "author": "", + "license": "ISC", + "type": "commonjs", + "dependencies": { + "@extractus/article-extractor": "^8.0.18", + "better-sqlite3": "^12.4.1", + "fastify": "^5.6.1", + "node-cron": "^4.2.1", + "rss-parser": "^3.13.0", + "sharp": "^0.34.5", + "sqlite-vec": "^0.1.9" + } +} diff --git a/server.js b/server.js new file mode 100644 index 0000000..a930480 --- /dev/null +++ b/server.js @@ -0,0 +1,27 @@ +const Fastify = require('fastify'); +const articleRoutes = require('./src/routes/articles'); +const statusRoutes = require('./src/routes/status'); +const config = require('./src/config'); +const { startScheduler, runAllIngestions } = require('./src/scheduler'); + +const app = Fastify({ logger: true }); + +app.register(articleRoutes); +app.register(statusRoutes); + +app.get('/', async () => ({ ok: true })); + +async function start() { + await app.listen({ port: config.server.port, host: config.server.host }); + + runAllIngestions().catch((error) => { + app.log.error(error); + }); + + startScheduler(); +} + +start().catch((error) => { + app.log.error(error); + process.exit(1); +}); diff --git a/src/config.js b/src/config.js new file mode 100644 index 0000000..a7c9689 --- /dev/null +++ b/src/config.js @@ -0,0 +1,7 @@ +const fs = require('fs'); +const path = require('path'); + +const configPath = path.join(__dirname, '..', 'config.json'); +const config = JSON.parse(fs.readFileSync(configPath, 'utf8')); + +module.exports = config; diff --git a/src/content.js b/src/content.js new file mode 100644 index 0000000..51b2cc6 --- /dev/null +++ b/src/content.js @@ -0,0 +1,203 @@ +const { extract } = require('@extractus/article-extractor'); +const sharp = require('sharp'); +const db = require('./db'); +const { generateAndStoreEmbedding } = require('./embeddings'); +const { fetchWithPolicy } = require('./http'); + +const updateArticleAssets = db.prepare(` + UPDATE articles + SET content = ?, image = ?, content_status = 'ready', content_error = NULL, content_attempted_at = ? + WHERE id = ? +`); +const markContentSkipped = db.prepare(` + UPDATE articles + SET content_status = 'skipped', content_error = ?, content_attempted_at = ? + WHERE id = ? +`); +const markContentFailed = db.prepare(` + UPDATE articles + SET content_status = 'failed', content_error = ?, content_attempted_at = ? + WHERE id = ? +`); +const markContentPending = db.prepare(` + UPDATE articles + SET content_status = NULL, content_error = NULL, content_attempted_at = ? + WHERE id = ? +`); +const selectArticlesMissingContent = db.prepare(` + SELECT id, url + FROM articles + WHERE (content IS NULL OR TRIM(content) = '') + AND (content_status IS NULL OR content_status = 'pending') + ORDER BY ingested_at DESC, id DESC + LIMIT ? +`); + +const blockedContentDomains = [ + 'axios.com', + 'bizjournals.com', + 'fastcompany.com', + 'gurufocus.com', + 'investing.com', + 'rbc.ru', + 'stocktitan.net', +]; +const loggedBlockedDomains = new Set(); +const articleFetchHeaders = { + Accept: 'text/html,application/xhtml+xml', + 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36', +}; + +let contentBackfillRunning = false; + +function getHostname(url) { + try { + return new URL(url).hostname.toLowerCase(); + } catch { + return ''; + } +} + +function isBlockedContentUrl(url) { + const hostname = getHostname(url); + return blockedContentDomains.some((domain) => hostname === domain || hostname.endsWith(`.${domain}`)); +} + +function getErrorStatus(error) { + if (error && Number.isInteger(error.status)) { + return error.status; + } + + const match = String(error && error.message || '').match(/\b(401|403|404|408|429|5\d\d)\b/); + return match ? Number(match[1]) : null; +} + +function getErrorMessage(error, fallback) { + const message = String(error && error.message || fallback || '').trim(); + return message ? message.slice(0, 500) : null; +} + +function markArticleStatus(statement, id, message) { + statement.run(message, new Date().toISOString(), id); +} + +async function fetchCompressedImage(url) { + const response = await fetchWithPolicy(url, { + retries: 1, + headers: { + Accept: 'image/*', + }, + }); + + if (!response.ok) { + const error = new Error(`image request failed with ${response.status}`); + error.status = response.status; + throw error; + } + + const contentType = String(response.headers.get('content-type') || '').toLowerCase(); + if (!contentType.startsWith('image/')) { + throw new Error(`image request returned ${contentType || 'unknown content-type'}`); + } + + const input = Buffer.from(await response.arrayBuffer()); + if (input.length === 0) { + throw new Error('image request returned an empty body'); + } + + const output = await sharp(input) + .rotate() + .resize({ width: 320, height: 320, fit: 'inside', withoutEnlargement: true }) + .webp({ quality: 25 }) + .toBuffer(); + + return output.toString('base64'); +} + +async function fetchAndStoreContent(id, url) { + try { + if (isBlockedContentUrl(url)) { + const hostname = getHostname(url); + if (hostname && !loggedBlockedDomains.has(hostname)) { + loggedBlockedDomains.add(hostname); + console.warn(`content extraction skipped for blocked domain ${hostname}`); + } + markArticleStatus(markContentSkipped, id, `blocked domain: ${hostname || 'unknown'}`); + return; + } + + const article = await extract(url, {}, { + headers: articleFetchHeaders, + signal: AbortSignal.timeout(20000), + }); + if (!article) { + markArticleStatus(markContentSkipped, id, 'extractor returned no article'); + return; + } + + const content = typeof article.content === 'string' + ? article.content.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim() || null + : null; + + let image = null; + if (article.image) { + try { + image = await fetchCompressedImage(article.image); + } catch (error) { + const status = getErrorStatus(error); + if (status === 401 || status === 403 || status === 404 || status === 429) { + console.warn(`image fetch skipped for ${url}: upstream returned ${status}`); + } else { + console.error(`image fetch failed for ${url}:`, error); + } + } + } + + if (!content && !image) { + markArticleStatus(markContentSkipped, id, 'article had no extractable content or image'); + return; + } + + updateArticleAssets.run(content, image, new Date().toISOString(), id); + await generateAndStoreEmbedding(id); + } catch (error) { + const status = getErrorStatus(error); + if (status === 401 || status === 403 || status === 404) { + console.warn(`content fetch skipped for ${url}: upstream returned ${status}`); + markArticleStatus(markContentSkipped, id, `upstream returned ${status}`); + return; + } + + if (status === 408 || status === 429 || (status && status >= 500)) { + console.warn(`content fetch deferred for ${url}: upstream returned ${status}`); + markArticleStatus(markContentPending, id, null); + return; + } + + markArticleStatus(markContentFailed, id, getErrorMessage(error, 'content fetch failed')); + console.error(`content fetch failed for ${url}:`, error); + } +} + +async function backfillMissingContent(limit = 10) { + if (contentBackfillRunning) { + return; + } + + contentBackfillRunning = true; + + try { + const rows = selectArticlesMissingContent.all(limit); + + for (const row of rows) { + await fetchAndStoreContent(row.id, row.url); + } + } finally { + contentBackfillRunning = false; + } +} + +module.exports = { + fetchAndStoreContent, + backfillMissingContent, +}; diff --git a/src/db.js b/src/db.js new file mode 100644 index 0000000..b74157f --- /dev/null +++ b/src/db.js @@ -0,0 +1,141 @@ +const path = require('path'); +const Database = require('better-sqlite3'); +const sqliteVec = require('sqlite-vec'); +const config = require('./config'); + +const dbPath = path.resolve(__dirname, '..', config.database.path || './archive.sqlite'); +const db = new Database(dbPath); +sqliteVec.load(db); + +db.pragma('journal_mode = WAL'); + +db.exec(` + CREATE TABLE IF NOT EXISTS articles ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + title TEXT NOT NULL, + description TEXT, + content TEXT, + image TEXT, + content_status TEXT, + content_error TEXT, + content_attempted_at TEXT, + url TEXT NOT NULL UNIQUE, + normalized_title TEXT NOT NULL, + source TEXT NOT NULL, + pub_date TEXT, + ingested_at TEXT NOT NULL DEFAULT (datetime('now')) + ); +`); + +function rebuildArticlesTableIfNeeded() { + const indexes = db.prepare(`PRAGMA index_list('articles')`).all(); + const hasUniqueNormalizedTitleIndex = indexes.some((index) => { + if (index.origin !== 'u' || !index.name) { + return false; + } + + const columns = db.prepare(`PRAGMA index_info('${index.name.replace(/'/g, "''")}')`).all(); + return columns.length === 1 && columns[0].name === 'normalized_title'; + }); + + if (!hasUniqueNormalizedTitleIndex) { + return; + } + + db.exec(` + BEGIN; + + CREATE TABLE articles_rebuild ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + title TEXT NOT NULL, + description TEXT, + content TEXT, + image TEXT, + content_status TEXT, + content_error TEXT, + content_attempted_at TEXT, + url TEXT NOT NULL UNIQUE, + normalized_title TEXT NOT NULL, + source TEXT NOT NULL, + pub_date TEXT, + ingested_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + + INSERT INTO articles_rebuild ( + id, + title, + description, + content, + image, + content_status, + content_error, + content_attempted_at, + url, + normalized_title, + source, + pub_date, + ingested_at + ) + SELECT + id, + title, + description, + content, + image, + content_status, + content_error, + content_attempted_at, + url, + normalized_title, + source, + pub_date, + ingested_at + FROM articles; + + DROP TABLE articles; + ALTER TABLE articles_rebuild RENAME TO articles; + + COMMIT; + `); +} + +rebuildArticlesTableIfNeeded(); + +db.exec(` + CREATE INDEX IF NOT EXISTS idx_articles_source ON articles(source); + CREATE INDEX IF NOT EXISTS idx_articles_pub_date ON articles(pub_date); + CREATE INDEX IF NOT EXISTS idx_articles_ingested_at ON articles(ingested_at); + CREATE INDEX IF NOT EXISTS idx_articles_normalized_title ON articles(normalized_title); +`); + +db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS article_embeddings USING vec0( + article_id INTEGER PRIMARY KEY, + embedding FLOAT[1024] + ); +`); + +db.exec(` + CREATE TABLE IF NOT EXISTS query_embeddings ( + query TEXT PRIMARY KEY, + embedding BLOB NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')) + ); +`); + +for (const statement of [ + 'ALTER TABLE articles ADD COLUMN image TEXT', + 'ALTER TABLE articles ADD COLUMN content_status TEXT', + 'ALTER TABLE articles ADD COLUMN content_error TEXT', + 'ALTER TABLE articles ADD COLUMN content_attempted_at TEXT' +]) { + try { + db.exec(statement); + } catch (error) { + if (!String(error.message).includes('duplicate column name')) { + throw error; + } + } +} + +module.exports = db; diff --git a/src/dedup.js b/src/dedup.js new file mode 100644 index 0000000..4240b61 --- /dev/null +++ b/src/dedup.js @@ -0,0 +1,11 @@ +function normalizeTitle(title) { + return String(title || '') + .toLowerCase() + .replace(/[^a-z0-9\s]/g, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +module.exports = { + normalizeTitle, +}; diff --git a/src/embeddings.js b/src/embeddings.js new file mode 100644 index 0000000..47f1054 --- /dev/null +++ b/src/embeddings.js @@ -0,0 +1,242 @@ +const db = require('./db'); +const config = require('./config'); + +const selectArticleForEmbedding = db.prepare(` + SELECT id, title, description, content + FROM articles + WHERE id = ? +`); +const insertEmbedding = db.prepare(` + INSERT INTO article_embeddings (article_id, embedding) + VALUES (?, ?) +`); +const deleteEmbedding = db.prepare(` + DELETE FROM article_embeddings + WHERE article_id = ? +`); +const selectEmbeddingBuffer = db.prepare(` + SELECT embedding + FROM article_embeddings + WHERE article_id = ? +`); +const nearestNeighbors = db.prepare(` + SELECT article_id, distance + FROM article_embeddings + WHERE embedding MATCH ? + AND k = ? + ORDER BY distance +`); +const selectArticlesMissingEmbeddings = db.prepare(` + SELECT a.id + FROM articles a + WHERE a.title IS NOT NULL + AND TRIM(a.title) != '' + AND a.description IS NOT NULL + AND TRIM(a.description) != '' + AND a.content IS NOT NULL + AND TRIM(a.content) != '' + AND NOT EXISTS ( + SELECT 1 + FROM article_embeddings e + WHERE e.article_id = a.id + ) + ORDER BY a.ingested_at ASC, a.id ASC + LIMIT ? +`); +const selectQueryEmbedding = db.prepare(` + SELECT embedding + FROM query_embeddings + WHERE query = ? +`); +const upsertQueryEmbedding = db.prepare(` + INSERT INTO query_embeddings (query, embedding) + VALUES (?, ?) + ON CONFLICT(query) DO UPDATE SET + embedding = excluded.embedding, + created_at = datetime('now') +`); + +let embeddingBackfillRunning = false; +const embeddingJobsRunning = new Set(); + +function buildEmbeddingInput(article) { + const title = typeof article.title === 'string' ? article.title.trim() : ''; + const description = typeof article.description === 'string' ? article.description.trim() : ''; + const content = typeof article.content === 'string' ? article.content.trim() : ''; + + if (!title || !description || !content) { + return ''; + } + + return [title, description, content].join('\n\n'); +} + +function serializeEmbedding(values) { + return Buffer.from(new Float32Array(values).buffer); +} + +function normalizeQuery(input) { + return String(input || '') + .trim() + .toLowerCase() + .replace(/\s+/g, ' '); +} + +async function requestEmbedding(input) { + const response = await fetch('https://openrouter.ai/api/v1/embeddings', { + method: 'POST', + headers: { + Authorization: `Bearer ${String(config.openRouter.apiKey).trim()}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + model: 'perplexity/pplx-embed-v1-0.6b', + input, + }), + }); + + if (!response.ok) { + let message = `embedding request failed with ${response.status}`; + + try { + const payload = await response.json(); + const errorMessage = payload && payload.error && payload.error.message; + if (errorMessage) { + message = errorMessage; + } + } catch (error) { + console.error('failed to parse embedding error response:', error); + } + + const requestError = new Error(message); + requestError.status = response.status; + throw requestError; + } + + const payload = await response.json(); + const embedding = payload && payload.data && payload.data[0] && payload.data[0].embedding; + if (!Array.isArray(embedding) || embedding.length !== 1024) { + throw new Error(`unexpected embedding length: ${Array.isArray(embedding) ? embedding.length : 'missing'}`); + } + + return embedding; +} + +async function generateAndStoreEmbedding(id) { + const apiKey = config.openRouter && config.openRouter.apiKey + ? String(config.openRouter.apiKey).trim() + : ''; + + if (!apiKey) { + return { stored: false, shouldPauseBatch: false }; + } + + if (embeddingJobsRunning.has(id)) { + return { stored: false, shouldPauseBatch: false }; + } + + if (selectEmbeddingBuffer.get(id)) { + return { stored: false, shouldPauseBatch: false }; + } + + const article = selectArticleForEmbedding.get(id); + if (!article) { + return { stored: false, shouldPauseBatch: false }; + } + + const input = buildEmbeddingInput(article); + if (!input) { + return { stored: false, shouldPauseBatch: false }; + } + + embeddingJobsRunning.add(id); + + try { + if (selectEmbeddingBuffer.get(id)) { + return { stored: false, shouldPauseBatch: false }; + } + + const embedding = await requestEmbedding(input); + deleteEmbedding.run(BigInt(id)); + insertEmbedding.run(BigInt(id), serializeEmbedding(embedding)); + return { stored: true, shouldPauseBatch: false }; + } catch (error) { + console.error(`embedding generation failed for article ${id}:`, error); + return { + stored: false, + shouldPauseBatch: error && error.status === 402, + }; + } finally { + embeddingJobsRunning.delete(id); + } +} + +async function backfillMissingEmbeddings(limit = 100) { + if (embeddingBackfillRunning) { + return; + } + + embeddingBackfillRunning = true; + + try { + const rows = selectArticlesMissingEmbeddings.all(limit); + + for (const row of rows) { + const result = await generateAndStoreEmbedding(row.id); + if (result.shouldPauseBatch) { + break; + } + } + } finally { + embeddingBackfillRunning = false; + } +} + +function getEmbeddingBuffer(articleId) { + const row = selectEmbeddingBuffer.get(articleId); + return row ? row.embedding : null; +} + +function findArticlesByEmbedding(embedding, limit) { + if (!embedding) { + return []; + } + + return nearestNeighbors.all(embedding, limit) + .map((row) => ({ + articleId: Number(row.article_id), + distance: row.distance, + })); +} + +async function getOrCreateQueryEmbedding(query) { + const normalizedQuery = normalizeQuery(query); + if (!normalizedQuery) { + return null; + } + + const cached = selectQueryEmbedding.get(normalizedQuery); + if (cached) { + return cached.embedding; + } + + const embedding = await requestEmbedding(normalizedQuery); + const buffer = serializeEmbedding(embedding); + upsertQueryEmbedding.run(normalizedQuery, buffer); + return buffer; +} + +function findSimilarArticles(articleId, limit) { + return findArticlesByEmbedding(getEmbeddingBuffer(articleId), limit + 1) + .filter((row) => row.articleId !== Number(articleId)) + .slice(0, limit); +} + +module.exports = { + backfillMissingEmbeddings, + findArticlesByEmbedding, + generateAndStoreEmbedding, + findSimilarArticles, + getEmbeddingBuffer, + getOrCreateQueryEmbedding, +}; diff --git a/src/http.js b/src/http.js new file mode 100644 index 0000000..c459fbe --- /dev/null +++ b/src/http.js @@ -0,0 +1,96 @@ +const DEFAULT_HEADERS = { + 'User-Agent': 'duriin_api/1.0', + Accept: 'application/json, text/plain, */*', +}; + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function isRetryableStatus(status) { + return status === 429 || status >= 500; +} + +function getRetryDelay(attempt, response) { + const retryAfter = response && response.headers ? response.headers.get('retry-after') : null; + + if (retryAfter) { + const seconds = Number(retryAfter); + if (Number.isFinite(seconds) && seconds >= 0) { + return Math.min(seconds * 1000, 30000); + } + + const retryDate = new Date(retryAfter).getTime(); + if (!Number.isNaN(retryDate)) { + return Math.max(Math.min(retryDate - Date.now(), 30000), 0); + } + } + + const baseDelay = Math.min(1000 * (2 ** attempt), 10000); + return baseDelay + Math.floor(Math.random() * 250); +} + +async function fetchWithPolicy(url, options = {}) { + const { + timeout = 20000, + retries = 2, + headers = {}, + ...fetchOptions + } = options; + + let lastError; + + for (let attempt = 0; attempt <= retries; attempt += 1) { + let response; + + try { + response = await fetch(url, { + ...fetchOptions, + signal: fetchOptions.signal || AbortSignal.timeout(timeout), + headers: { + ...DEFAULT_HEADERS, + ...headers, + }, + }); + + if (response.ok || !isRetryableStatus(response.status)) { + return response; + } + + const error = new Error(`Request failed with ${response.status} for ${url}`); + error.status = response.status; + lastError = error; + } catch (error) { + lastError = error; + } + + if (attempt < retries) { + await sleep(getRetryDelay(attempt, response)); + } + } + + throw lastError; +} + +async function fetchJson(url, options = {}) { + const response = await fetchWithPolicy(url, { + ...options, + headers: { + Accept: 'application/json', + ...(options.headers || {}), + }, + }); + + if (!response.ok) { + const error = new Error(`Request failed with ${response.status} for ${url}`); + error.status = response.status; + throw error; + } + + return response.json(); +} + +module.exports = { + fetchJson, + fetchWithPolicy, +}; diff --git a/src/ingest.js b/src/ingest.js new file mode 100644 index 0000000..c915eb1 --- /dev/null +++ b/src/ingest.js @@ -0,0 +1,120 @@ +const db = require('./db'); +const { normalizeTitle } = require('./dedup'); +const { fetchAndStoreContent } = require('./content'); +const { markSourceRun } = require('./state'); + +const insertArticle = db.prepare(` + INSERT INTO articles ( + title, + description, + content, + image, + url, + normalized_title, + source, + pub_date, + ingested_at + ) VALUES (?, ?, NULL, NULL, ?, ?, ?, ?, ?) +`); +const findByUrl = db.prepare('SELECT id FROM articles WHERE url = ?'); + +function normalizePubDate(value) { + if (!value) { + return null; + } + + if (typeof value === 'number') { + const parsed = new Date(value); + return Number.isNaN(parsed.getTime()) ? null : parsed.toISOString(); + } + + const input = String(value).trim(); + if (!input) { + return null; + } + + if (/^\d{8}T\d{6}$/.test(input)) { + const normalized = `${input.slice(0, 4)}-${input.slice(4, 6)}-${input.slice(6, 8)}T${input.slice(9, 11)}:${input.slice(11, 13)}:${input.slice(13, 15)}Z`; + const parsed = new Date(normalized); + return Number.isNaN(parsed.getTime()) ? null : parsed.toISOString(); + } + + if (/^\d{8}T\d{6}Z$/.test(input)) { + const normalized = `${input.slice(0, 4)}-${input.slice(4, 6)}-${input.slice(6, 8)}T${input.slice(9, 11)}:${input.slice(11, 13)}:${input.slice(13, 15)}Z`; + const parsed = new Date(normalized); + return Number.isNaN(parsed.getTime()) ? null : parsed.toISOString(); + } + + if (/^\d{4}-\d{2}-\d{2}$/.test(input)) { + return `${input}T00:00:00.000Z`; + } + + const parsed = new Date(input); + return Number.isNaN(parsed.getTime()) ? null : parsed.toISOString(); +} + +function ingestArticle(article) { + const title = String(article.title || '').trim(); + const url = String(article.url || '').trim(); + const source = String(article.source || '').trim(); + + if (!title || !url || !source) { + return { inserted: false, reason: 'missing_required_fields' }; + } + + const normalizedTitle = normalizeTitle(title); + if (!normalizedTitle) { + return { inserted: false, reason: 'empty_normalized_title' }; + } + + const description = article.description == null ? null : String(article.description).trim() || null; + const pubDate = normalizePubDate(article.pubDate); + const ingestedAt = new Date().toISOString(); + + try { + const result = insertArticle.run( + title, + description, + url, + normalizedTitle, + source, + pubDate, + ingestedAt + ); + + fetchAndStoreContent(result.lastInsertRowid, url); + + return { inserted: true, id: result.lastInsertRowid }; + } catch (error) { + if (error.code === 'SQLITE_CONSTRAINT_UNIQUE') { + const duplicateByUrl = findByUrl.get(url); + if (duplicateByUrl) { + return { inserted: false, reason: 'duplicate_url', id: duplicateByUrl.id }; + } + + return { inserted: false, reason: 'duplicate' }; + } + + throw error; + } +} + +async function ingestBatch(source, articles) { + let inserted = 0; + + for (const article of articles) { + const result = ingestArticle({ ...article, source: article.source || source }); + if (result.inserted) { + inserted += 1; + } + } + + markSourceRun(source); + + return { source, inserted, total: articles.length }; +} + +module.exports = { + ingestArticle, + ingestBatch, +}; diff --git a/src/routes/articles.js b/src/routes/articles.js new file mode 100644 index 0000000..4f0b9b9 --- /dev/null +++ b/src/routes/articles.js @@ -0,0 +1,155 @@ +const db = require('../db'); +const { + findArticlesByEmbedding, + findSimilarArticles, + getEmbeddingBuffer, + getOrCreateQueryEmbedding, +} = require('../embeddings'); + +function buildArticlesQuery(query) { + const conditions = []; + const params = []; + const includeEmbedding = String(query.include_embedding || '').toLowerCase() === 'true'; + + if (query.q) { + conditions.push('(title LIKE ? OR description LIKE ? OR content LIKE ?)'); + const keyword = `%${query.q}%`; + params.push(keyword, keyword, keyword); + } + + if (query.source) { + conditions.push('source = ?'); + params.push(query.source); + } + + if (query.from) { + conditions.push('pub_date >= ?'); + params.push(query.from); + } + + if (query.to) { + conditions.push('pub_date <= ?'); + params.push(query.to); + } + + const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''; + const limit = Number.parseInt(query.limit, 10); + const offset = Number.parseInt(query.offset, 10); + + params.push(Number.isFinite(limit) && limit > 0 ? Math.min(limit, 100) : 20); + params.push(Number.isFinite(offset) && offset >= 0 ? offset : 0); + + return { + sql: ` + SELECT id, title, description, content, image, ${includeEmbedding ? 'embedding,' : ''} url, normalized_title, source, pub_date, ingested_at + FROM articles + ${whereClause} + ORDER BY COALESCE(pub_date, ingested_at) DESC, id DESC + LIMIT ? OFFSET ? + `, + params, + }; +} + +async function articleRoutes(fastify) { + fastify.get('/articles', async (request, reply) => { + const query = request.query || {}; + if (query.include_embedding) { + reply.code(400); + return { error: 'Embeddings are not returned directly. Use similar_to for vector search.' }; + } + + if (query.topic !== undefined) { + const limit = Number.parseInt(query.limit, 10); + const embedding = await getOrCreateQueryEmbedding(query.topic); + + if (!embedding) { + reply.code(400); + return { error: 'Topic must not be empty' }; + } + + const neighbors = findArticlesByEmbedding( + embedding, + Number.isFinite(limit) && limit > 0 ? Math.min(limit, 100) : 20 + ); + const ids = neighbors.map((row) => row.articleId); + if (ids.length === 0) { + return []; + } + + const placeholders = ids.map(() => '?').join(', '); + const articles = db.prepare(` + SELECT id, title, description, content, image, url, normalized_title, source, pub_date, ingested_at + FROM articles + WHERE id IN (${placeholders}) + `).all(...ids); + const byId = new Map(articles.map((article) => [article.id, article])); + + return neighbors + .map((row) => { + const article = byId.get(row.articleId); + return article ? { ...article, distance: row.distance } : null; + }) + .filter(Boolean); + } + + if (query.similar_to) { + const limit = Number.parseInt(query.limit, 10); + const articleId = Number.parseInt(query.similar_to, 10); + const neighbors = findSimilarArticles( + articleId, + Number.isFinite(limit) && limit > 0 ? Math.min(limit, 100) : 20 + ); + + if (neighbors.length === 0 && !getEmbeddingBuffer(articleId)) { + reply.code(404); + return { error: 'Embedding not found for article' }; + } + + const ids = neighbors.map((row) => row.articleId); + if (ids.length === 0) { + return []; + } + + const placeholders = ids.map(() => '?').join(', '); + const articles = db.prepare(` + SELECT id, title, description, content, image, url, normalized_title, source, pub_date, ingested_at + FROM articles + WHERE id IN (${placeholders}) + `).all(...ids); + const byId = new Map(articles.map((article) => [article.id, article])); + + return neighbors + .map((row) => { + const article = byId.get(row.articleId); + return article ? { ...article, distance: row.distance } : null; + }) + .filter(Boolean); + } + + const { sql, params } = buildArticlesQuery(query); + return db.prepare(sql).all(...params); + }); + + fastify.get('/articles/:id', async (request, reply) => { + if (String((request.query || {}).include_embedding || '').toLowerCase() === 'true') { + reply.code(400); + return { error: 'Embeddings are not returned directly. Use similar_to for vector search.' }; + } + + const article = db.prepare(` + SELECT id, title, description, content, image, url, normalized_title, source, pub_date, ingested_at + FROM articles + WHERE id = ? + `).get(request.params.id); + + if (!article) { + reply.code(404); + return { error: 'Article not found' }; + } + + return article; + }); +} + +module.exports = articleRoutes; diff --git a/src/routes/status.js b/src/routes/status.js new file mode 100644 index 0000000..62799a7 --- /dev/null +++ b/src/routes/status.js @@ -0,0 +1,34 @@ +const db = require('../db'); +const { getLastIngestionBySource } = require('../state'); + +async function statusRoutes(fastify) { + fastify.get('/status', async () => { + const total = db.prepare('SELECT COUNT(*) AS count FROM articles').get().count; + const bySourceRows = db.prepare('SELECT source, COUNT(*) AS count FROM articles GROUP BY source ORDER BY source').all(); + const contentCoverage = db.prepare(` + SELECT + COUNT(*) AS total, + SUM(CASE WHEN content IS NOT NULL AND TRIM(content) <> '' THEN 1 ELSE 0 END) AS with_content, + SUM(CASE WHEN image IS NOT NULL AND TRIM(image) <> '' THEN 1 ELSE 0 END) AS with_image + FROM articles + `).get(); + const embeddingCoverage = db.prepare('SELECT COUNT(*) AS count FROM article_embeddings').get(); + + return { + totalArticles: total, + countsBySource: Object.fromEntries(bySourceRows.map((row) => [row.source, row.count])), + lastIngestionBySource: getLastIngestionBySource(), + contentFetchCoverage: { + withContent: contentCoverage.with_content || 0, + withImage: contentCoverage.with_image || 0, + withEmbedding: embeddingCoverage.count || 0, + total: contentCoverage.total || 0, + contentRatio: contentCoverage.total ? Number((contentCoverage.with_content / contentCoverage.total).toFixed(4)) : 0, + imageRatio: contentCoverage.total ? Number((contentCoverage.with_image / contentCoverage.total).toFixed(4)) : 0, + embeddingRatio: contentCoverage.total ? Number((embeddingCoverage.count / contentCoverage.total).toFixed(4)) : 0, + }, + }; + }); +} + +module.exports = statusRoutes; diff --git a/src/scheduler.js b/src/scheduler.js new file mode 100644 index 0000000..9f331f7 --- /dev/null +++ b/src/scheduler.js @@ -0,0 +1,85 @@ +const cron = require('node-cron'); +const config = require('./config'); +const { ingestBatch } = require('./ingest'); +const { fetchRssArticles } = require('./sources/rss'); +const { fetchGdeltArticles } = require('./sources/gdelt'); +const { fetchEdgarArticles } = require('./sources/edgar'); +const { fetchAlphaVantageArticles } = require('./sources/alphavantage'); +const { fetchFinnhubArticles } = require('./sources/finnhub'); +const { backfillMissingContent } = require('./content'); +const { backfillMissingEmbeddings } = require('./embeddings'); + +async function runSource(source, fetcher) { + try { + const articles = await fetcher(); + return await ingestBatch(source, articles); + } catch (error) { + console.error(`${source} ingestion failed:`, error); + return { source, inserted: 0, total: 0, error: error.message }; + } +} + +async function runAllIngestions() { + const results = []; + + results.push(await runSource('rss', fetchRssArticles)); + results.push(await runSource('gdelt', fetchGdeltArticles)); + results.push(await runSource('edgar', fetchEdgarArticles)); + results.push(await runSource('alphavantage', fetchAlphaVantageArticles)); + results.push(await runSource('finnhub', fetchFinnhubArticles)); + + try { + await backfillMissingContent(); + } catch (error) { + console.error('content backfill failed:', error); + } + + try { + await backfillMissingEmbeddings(); + } catch (error) { + console.error('embedding backfill failed:', error); + } + + return results; +} + +function startScheduler() { + cron.schedule(config.scheduler.rss, async () => { + await runSource('rss', fetchRssArticles); + }); + + cron.schedule(config.scheduler.gdelt, async () => { + await runSource('gdelt', fetchGdeltArticles); + }); + + cron.schedule(config.scheduler.edgar, async () => { + await runSource('edgar', fetchEdgarArticles); + }); + + cron.schedule(config.scheduler.alphaVantage, async () => { + await runSource('alphavantage', fetchAlphaVantageArticles); + }); + + cron.schedule(config.scheduler.finnhub, async () => { + await runSource('finnhub', fetchFinnhubArticles); + }); + + cron.schedule('0 * * * *', async () => { + try { + await backfillMissingContent(); + } catch (error) { + console.error('content backfill failed:', error); + } + + try { + await backfillMissingEmbeddings(); + } catch (error) { + console.error('embedding backfill failed:', error); + } + }); +} + +module.exports = { + startScheduler, + runAllIngestions, +}; diff --git a/src/sources/alphavantage.js b/src/sources/alphavantage.js new file mode 100644 index 0000000..ae2589d --- /dev/null +++ b/src/sources/alphavantage.js @@ -0,0 +1,32 @@ +const config = require('../config'); +const { fetchJson } = require('../http'); + +async function fetchAlphaVantageArticles() { + if (!config.alphaVantage?.apiKey) { + return []; + } + + const tickers = (config.alphaVantage.tickers || []).join(','); + const params = new URLSearchParams({ + function: 'NEWS_SENTIMENT', + apikey: config.alphaVantage.apiKey, + }); + + if (tickers) { + params.set('tickers', tickers); + } + + const data = await fetchJson(`https://www.alphavantage.co/query?${params.toString()}`); + + return (data.feed || []).map((item) => ({ + title: item.title, + description: item.summary || null, + url: item.url, + source: 'alphavantage', + pubDate: item.time_published || null, + })).filter((item) => item.title && item.url); +} + +module.exports = { + fetchAlphaVantageArticles, +}; diff --git a/src/sources/edgar.js b/src/sources/edgar.js new file mode 100644 index 0000000..fe0d89b --- /dev/null +++ b/src/sources/edgar.js @@ -0,0 +1,78 @@ +const config = require('../config'); +const { fetchJson } = require('../http'); + +let tickerMap; + +function getHeaders() { + return { + headers: { + 'User-Agent': config.sec.userAgent, + Accept: 'application/json', + }, + }; +} + +async function getTickerMap() { + if (tickerMap) { + return tickerMap; + } + + const data = await fetchJson('https://www.sec.gov/files/company_tickers.json', getHeaders()); + tickerMap = Object.values(data).reduce((acc, company) => { + acc[String(company.ticker || '').toUpperCase()] = String(company.cik_str || '').padStart(10, '0'); + return acc; + }, {}); + + return tickerMap; +} + +async function fetchEdgarArticles() { + const tickers = config.sec?.tickers || []; + if (tickers.length === 0) { + return []; + } + + const map = await getTickerMap(); + const articles = []; + + for (const ticker of tickers) { + const cik = map[String(ticker).toUpperCase()]; + if (!cik) { + continue; + } + + const data = await fetchJson(`https://data.sec.gov/submissions/CIK${cik}.json`, getHeaders()); + const recent = data.filings?.recent; + if (!recent) { + continue; + } + + for (let i = 0; i < recent.form.length; i += 1) { + if (recent.form[i] !== '8-K') { + continue; + } + + const accession = String(recent.accessionNumber[i] || '').replace(/-/g, ''); + const primaryDocument = String(recent.primaryDocument[i] || '').trim(); + const filingDate = recent.filingDate[i] || null; + + if (!accession || !primaryDocument) { + continue; + } + + articles.push({ + title: `${String(ticker).toUpperCase()} 8-K ${filingDate || accession}`, + description: `${String(data.name || ticker).trim()} filed Form 8-K`, + url: `https://www.sec.gov/Archives/edgar/data/${Number(cik)}/${accession}/${primaryDocument}`, + source: 'edgar', + pubDate: filingDate, + }); + } + } + + return articles; +} + +module.exports = { + fetchEdgarArticles, +}; diff --git a/src/sources/finnhub.js b/src/sources/finnhub.js new file mode 100644 index 0000000..846751a --- /dev/null +++ b/src/sources/finnhub.js @@ -0,0 +1,55 @@ +const config = require('../config'); +const { fetchJson } = require('../http'); + +function getDateRange() { + const to = new Date(); + const from = new Date(to); + from.setDate(from.getDate() - 1); + + return { + from: from.toISOString().slice(0, 10), + to: to.toISOString().slice(0, 10), + }; +} + +async function fetchFinnhubArticles() { + if (!config.finnhub?.apiKey) { + return []; + } + + const { from, to } = getDateRange(); + const articles = []; + + for (const symbol of config.finnhub.tickers || []) { + const params = new URLSearchParams({ + symbol, + from, + to, + token: config.finnhub.apiKey, + }); + + const data = await fetchJson(`https://finnhub.io/api/v1/company-news?${params.toString()}`); + for (const item of data || []) { + const title = String(item.headline || '').trim(); + const url = String(item.url || '').trim(); + + if (!title || !url) { + continue; + } + + articles.push({ + title, + description: item.summary || null, + url, + source: 'finnhub', + pubDate: item.datetime ? new Date(item.datetime * 1000).toISOString() : null, + }); + } + } + + return articles; +} + +module.exports = { + fetchFinnhubArticles, +}; diff --git a/src/sources/gdelt.js b/src/sources/gdelt.js new file mode 100644 index 0000000..226d531 --- /dev/null +++ b/src/sources/gdelt.js @@ -0,0 +1,48 @@ +const config = require('../config'); +const { fetchJson } = require('../http'); + +async function fetchGdeltArticles() { + const articles = []; + + for (const query of config.gdelt?.queries || []) { + try { + const params = new URLSearchParams({ + query, + mode: config.gdelt.mode || 'ArtList', + maxrecords: String(Math.min(config.gdelt.maxRecords || 10, 10)), + format: config.gdelt.format || 'json', + }); + + const data = await fetchJson(`https://api.gdeltproject.org/api/v2/doc/doc?${params.toString()}`); + for (const item of data.articles || []) { + const title = String(item.title || '').trim(); + const url = String(item.url || '').trim(); + + if (!title || !url) { + continue; + } + + articles.push({ + title, + description: item.domain || null, + url, + source: 'gdelt', + pubDate: item.seendate || null, + }); + } + } catch (error) { + if (error && error.status === 429) { + console.warn(`GDELT query skipped for rate limit: ${query}`); + continue; + } + + console.error(`Failed to fetch GDELT query: ${query}`, error); + } + } + + return articles; +} + +module.exports = { + fetchGdeltArticles, +}; diff --git a/src/sources/rss.js b/src/sources/rss.js new file mode 100644 index 0000000..b7e6243 --- /dev/null +++ b/src/sources/rss.js @@ -0,0 +1,127 @@ +const Parser = require('rss-parser'); +const config = require('../config'); + +const parser = new Parser({ + timeout: 10000, + headers: { + 'User-Agent': 'Mozilla/5.0', + Accept: 'application/rss+xml, application/xml, text/xml;q=0.9, */*;q=0.8', + }, +}); + +const blockedFeedDomains = [ + 'arabnews.com', + 'arabianbusiness.com', + 'business-standard.com', + 'cityam.com', + 'eleconomista.com.mx', + 'eleconomista.es', + 'moneycontrol.com', + 'thisismoney.co.uk', +]; +const invalidFeedLabels = new Set([ + 'ABC Business AU', + 'Australian Fin Review', + 'Business Daily Africa', + 'BusinessLive SA', + 'Caixin Global', + 'Cinco Dias', + 'El Comercio Peru', + 'FD.nl', + 'Gulf News Business', + 'Il Sole 24 Ore', + 'Infobae Economia AR', + 'Japan Times Business', + 'Korea JoongAng Daily', + 'Les Echos', + 'Live Mint', + 'NZ Herald Business', + 'Portafolio Colombia', + 'The Star Malaysia', + 'Xinhua Business', +]); +const malformedFeedLabels = new Set([ + 'BFM Business', + 'Business Daily Africa', +]); +const loggedBlockedFeeds = new Set(); +const loggedInvalidFeeds = new Set(); + +function getHostname(url) { + try { + return new URL(url).hostname.toLowerCase(); + } catch { + return ''; + } +} + +function isBlockedFeed(feed) { + const hostname = getHostname(feed.url); + return blockedFeedDomains.some((domain) => hostname === domain || hostname.endsWith(`.${domain}`)); +} + +function isMalformedFeedError(error) { + const message = String(error && error.message || ''); + return message.includes('Invalid character in entity name') || message.includes('Attribute without value'); +} + +async function fetchRssArticles() { + const articles = []; + + for (const feed of config.rssFeeds || []) { + const label = feed.label || feed.url; + + if (invalidFeedLabels.has(label)) { + if (!loggedInvalidFeeds.has(label)) { + loggedInvalidFeeds.add(label); + console.warn(`RSS feed skipped for invalid endpoint ${label}`); + } + continue; + } + + if (isBlockedFeed(feed)) { + const hostname = getHostname(feed.url); + if (!loggedBlockedFeeds.has(hostname)) { + loggedBlockedFeeds.add(hostname); + console.warn(`RSS feed skipped for blocked domain ${hostname}`); + } + continue; + } + + try { + const parsed = await parser.parseURL(feed.url); + for (const item of parsed.items || []) { + const title = String(item.title || '').trim(); + const url = String(item.link || item.guid || '').trim(); + + if (!title || !url) { + continue; + } + + articles.push({ + title, + description: item.contentSnippet || item.content || item.summary || null, + url, + source: feed.label ? `rss:${feed.label}` : 'rss', + pubDate: item.isoDate || item.pubDate || null, + }); + } + } catch (error) { + if (malformedFeedLabels.has(label) && isMalformedFeedError(error)) { + if (!loggedInvalidFeeds.has(label)) { + loggedInvalidFeeds.add(label); + console.warn(`RSS feed skipped for malformed XML ${label}`); + } + continue; + } + + console.error(`Failed to fetch RSS feed: ${label}`, error); + } + } + + return articles; +} + +module.exports = { + fetchRssArticles, +}; diff --git a/src/state.js b/src/state.js new file mode 100644 index 0000000..653930b --- /dev/null +++ b/src/state.js @@ -0,0 +1,14 @@ +const ingestionState = new Map(); + +function markSourceRun(source) { + ingestionState.set(source, new Date().toISOString()); +} + +function getLastIngestionBySource() { + return Object.fromEntries(ingestionState.entries()); +} + +module.exports = { + markSourceRun, + getLastIngestionBySource, +};