add browser crawling capabilities and enhance configuration options

This commit is contained in:
ImBenji
2026-04-17 16:53:18 +01:00
parent c6bbe2a061
commit 1a8504389a
6 changed files with 298 additions and 49 deletions
+82
View File
@@ -10,9 +10,11 @@
"license": "ISC",
"dependencies": {
"@extractus/article-extractor": "^8.0.18",
"@fastify/cors": "^11.2.0",
"better-sqlite3": "^12.4.1",
"fastify": "^5.6.1",
"node-cron": "^4.2.1",
"playwright": "^1.59.1",
"rss-parser": "^3.13.0",
"sharp": "^0.34.5",
"sqlite-vec": "^0.1.9"
@@ -65,6 +67,26 @@
"fast-uri": "^3.0.0"
}
},
"node_modules/@fastify/cors": {
"version": "11.2.0",
"resolved": "https://registry.npmjs.org/@fastify/cors/-/cors-11.2.0.tgz",
"integrity": "sha512-LbLHBuSAdGdSFZYTLVA3+Ch2t+sA6nq3Ejc6XLAKiQ6ViS2qFnvicpj0htsx03FyYeLs04HfRNBsz/a8SvbcUw==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/fastify"
},
{
"type": "opencollective",
"url": "https://opencollective.com/fastify"
}
],
"license": "MIT",
"dependencies": {
"fastify-plugin": "^5.0.0",
"toad-cache": "^3.7.0"
}
},
"node_modules/@fastify/error": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/@fastify/error/-/error-4.2.0.tgz",
@@ -1097,6 +1119,22 @@
"toad-cache": "^3.7.0"
}
},
"node_modules/fastify-plugin": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/fastify-plugin/-/fastify-plugin-5.1.0.tgz",
"integrity": "sha512-FAIDA8eovSt5qcDgcBvDuX/v0Cjz0ohGhENZ/wpc3y+oZCY2afZ9Baqql3g/lC+OHRnciQol4ww7tuthOb9idw==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/fastify"
},
{
"type": "opencollective",
"url": "https://opencollective.com/fastify"
}
],
"license": "MIT"
},
"node_modules/fastq": {
"version": "1.20.1",
"resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz",
@@ -1132,6 +1170,20 @@
"integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==",
"license": "MIT"
},
"node_modules/fsevents": {
"version": "2.3.2",
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
"hasInstallScript": true,
"license": "MIT",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
}
},
"node_modules/github-from-package": {
"version": "0.0.0",
"resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
@@ -1482,6 +1534,36 @@
"integrity": "sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==",
"license": "MIT"
},
"node_modules/playwright": {
"version": "1.59.1",
"resolved": "https://registry.npmjs.org/playwright/-/playwright-1.59.1.tgz",
"integrity": "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw==",
"license": "Apache-2.0",
"dependencies": {
"playwright-core": "1.59.1"
},
"bin": {
"playwright": "cli.js"
},
"engines": {
"node": ">=18"
},
"optionalDependencies": {
"fsevents": "2.3.2"
}
},
"node_modules/playwright-core": {
"version": "1.59.1",
"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.59.1.tgz",
"integrity": "sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg==",
"license": "Apache-2.0",
"bin": {
"playwright-core": "cli.js"
},
"engines": {
"node": ">=18"
}
},
"node_modules/postcss": {
"version": "8.5.10",
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.10.tgz",