diff --git a/.gitignore b/.gitignore index b07a0e5..bab31e5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /keys /node_modules /dist +/data .env /*.log \ No newline at end of file diff --git a/data/sessions.db b/data/sessions.db deleted file mode 100644 index 9e5ec56..0000000 Binary files a/data/sessions.db and /dev/null differ diff --git a/package-lock.json b/package-lock.json index abbad05..dac9efb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0", "license": "ISC", "dependencies": { + "@mozilla/readability": "^0.6.0", "@nestjs/common": "^11.1.18", "@nestjs/config": "^4.0.3", "@nestjs/core": "^11.1.18", @@ -18,6 +19,7 @@ "better-sqlite3": "^12.8.0", "class-transformer": "^0.5.1", "class-validator": "^0.14.4", + "jsdom": "^29.0.2", "playwright": "^1.59.1", "reflect-metadata": "^0.2.2", "rxjs": "^7.8.2", @@ -27,6 +29,8 @@ "devDependencies": { "@nestjs/cli": "^11.0.17", "@types/better-sqlite3": "^7.6.13", + "@types/jsdom": "^28.0.1", + "@types/mozilla__readability": "^0.4.2", "@types/node": "^25.5.2", "typescript": "^6.0.2" } @@ -151,6 +155,42 @@ "tslib": "^2.1.0" } }, + "node_modules/@asamuzakjp/css-color": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-5.1.6.tgz", + "integrity": "sha512-BXWCh8dHs9GOfpo/fWGDJtDmleta2VePN9rn6WQt3GjEbxzutVF4t0x2pmH+7dbMCLtuv3MlwqRsAuxlzFXqFg==", + "license": "MIT", + "dependencies": { + "@csstools/css-calc": "^3.1.1", + "@csstools/css-color-parser": "^4.0.2", + "@csstools/css-parser-algorithms": "^4.0.0", + "@csstools/css-tokenizer": "^4.0.0" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, + "node_modules/@asamuzakjp/dom-selector": { + "version": "7.0.7", + "resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-7.0.7.tgz", + "integrity": "sha512-d2BgqDUOS1Hfp4IzKUZqCNz+Kg3Y88AkaBvJK/ZVSQPU1f7OpPNi7nQTH6/oI47Dkdg+Z3e8Yp6ynOu4UMINAQ==", + "license": "MIT", + "dependencies": { + "@asamuzakjp/nwsapi": "^2.3.9", + "bidi-js": "^1.0.3", + "css-tree": "^3.2.1", + "is-potential-custom-element-name": "^1.0.1" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, + "node_modules/@asamuzakjp/nwsapi": { + "version": "2.3.9", + "resolved": "https://registry.npmjs.org/@asamuzakjp/nwsapi/-/nwsapi-2.3.9.tgz", + "integrity": "sha512-n8GuYSrI9bF7FFZ/SjhwevlHc8xaVlb/7HmHelnc/PZXBD2ZR49NnN9sMMuDdEGPeeRQ5d0hqlSlEpgCX3Wl0Q==", + "license": "MIT" + }, "node_modules/@babel/code-frame": { "version": "7.29.0", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.0.tgz", @@ -186,6 +226,18 @@ "url": "https://github.com/sponsors/Borewit" } }, + "node_modules/@bramus/specificity": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/@bramus/specificity/-/specificity-2.4.2.tgz", + "integrity": "sha512-ctxtJ/eA+t+6q2++vj5j7FYX3nRu311q1wfYH3xjlLOsczhlhxAg2FWNUXhpGvAw3BWo1xBcvOV6/YLc2r5FJw==", + "license": "MIT", + "dependencies": { + "css-tree": "^3.0.0" + }, + "bin": { + "specificity": "bin/cli.js" + } + }, "node_modules/@colors/colors": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz", @@ -197,6 +249,157 @@ "node": ">=0.1.90" } }, + "node_modules/@csstools/color-helpers": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-6.0.2.tgz", + "integrity": "sha512-LMGQLS9EuADloEFkcTBR3BwV/CGHV7zyDxVRtVDTwdI2Ca4it0CCVTT9wCkxSgokjE5Ho41hEPgb8OEUwoXr6Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/@csstools/css-calc": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-3.1.1.tgz", + "integrity": "sha512-HJ26Z/vmsZQqs/o3a6bgKslXGFAungXGbinULZO3eMsOyNJHeBBZfup5FiZInOghgoM4Hwnmw+OgbJCNg1wwUQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^4.0.0", + "@csstools/css-tokenizer": "^4.0.0" + } + }, + "node_modules/@csstools/css-color-parser": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-4.0.2.tgz", + "integrity": "sha512-0GEfbBLmTFf0dJlpsNU7zwxRIH0/BGEMuXLTCvFYxuL1tNhqzTbtnFICyJLTNK4a+RechKP75e7w42ClXSnJQw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "dependencies": { + "@csstools/color-helpers": "^6.0.2", + "@csstools/css-calc": "^3.1.1" + }, + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^4.0.0", + "@csstools/css-tokenizer": "^4.0.0" + } + }, + "node_modules/@csstools/css-parser-algorithms": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-4.0.0.tgz", + "integrity": "sha512-+B87qS7fIG3L5h3qwJ/IFbjoVoOe/bpOdh9hAjXbvx0o8ImEmUsGXN0inFOnk2ChCFgqkkGFQ+TpM5rbhkKe4w==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "@csstools/css-tokenizer": "^4.0.0" + } + }, + "node_modules/@csstools/css-syntax-patches-for-csstree": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@csstools/css-syntax-patches-for-csstree/-/css-syntax-patches-for-csstree-1.1.2.tgz", + "integrity": "sha512-5GkLzz4prTIpoyeUiIu3iV6CSG3Plo7xRVOFPKI7FVEJ3mZ0A8SwK0XU3Gl7xAkiQ+mDyam+NNp875/C5y+jSA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "peerDependencies": { + "css-tree": "^3.2.1" + }, + "peerDependenciesMeta": { + "css-tree": { + "optional": true + } + } + }, + "node_modules/@csstools/css-tokenizer": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-4.0.0.tgz", + "integrity": "sha512-QxULHAm7cNu72w97JUNCBFODFaXpbDg+dP8b/oWFAZ2MTRppA3U00Y2L1HqaS4J6yBqxwa/Y3nMBaxVKbB/NsA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/@exodus/bytes": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/@exodus/bytes/-/bytes-1.15.0.tgz", + "integrity": "sha512-UY0nlA+feH81UGSHv92sLEPLCeZFjXOuHhrIo0HQydScuQc8s0A7kL/UdgwgDq8g8ilksmuoF35YVTNphV2aBQ==", + "license": "MIT", + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + }, + "peerDependencies": { + "@noble/hashes": "^1.8.0 || ^2.0.0" + }, + "peerDependenciesMeta": { + "@noble/hashes": { + "optional": true + } + } + }, "node_modules/@inquirer/ansi": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@inquirer/ansi/-/ansi-1.0.2.tgz", @@ -719,6 +922,15 @@ "integrity": "sha512-xgAyonlVVS+q7Vc7qLW0UrJU7rSFcETRWsqdXZtjzRU8dF+6CkozTK4V4y1LwOX7j8r/vHphjDeMeGI4tNGeGA==", "license": "MIT" }, + "node_modules/@mozilla/readability": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@mozilla/readability/-/readability-0.6.0.tgz", + "integrity": "sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/@nestjs/cli": { "version": "11.0.17", "resolved": "https://registry.npmjs.org/@nestjs/cli/-/cli-11.0.17.tgz", @@ -1150,6 +1362,39 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/jsdom": { + "version": "28.0.1", + "resolved": "https://registry.npmjs.org/@types/jsdom/-/jsdom-28.0.1.tgz", + "integrity": "sha512-GJq2QE4TAZ5ajSoCasn5DOFm8u1mI3tIFvM5tIq3W5U/RTB6gsHwc6Yhpl91X9VSDOUVblgXmG+2+sSvFQrdlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/tough-cookie": "*", + "parse5": "^7.0.0", + "undici-types": "^7.21.0" + } + }, + "node_modules/@types/jsdom/node_modules/parse5": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", + "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "entities": "^6.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/@types/jsdom/node_modules/undici-types": { + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.7.tgz", + "integrity": "sha512-XA+gOBkzYD3C74sZowtCLTpgtaCdqZhqCvR6y9LXvrKTt/IVU6bz49T4D+BPi475scshCCkb0IklJRw6T1ZlgQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/json-schema": { "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", @@ -1157,6 +1402,13 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/mozilla__readability": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/@types/mozilla__readability/-/mozilla__readability-0.4.2.tgz", + "integrity": "sha512-mXHoZZ/Knps7GABAG9D512yDLCMQNOvQBqzIZpFJnRJveHc528rltnXh6EZTi1RiB1I2Z8yV35Dzs6DhE2WaYw==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/node": { "version": "25.5.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-25.5.2.tgz", @@ -1167,6 +1419,13 @@ "undici-types": "~7.18.0" } }, + "node_modules/@types/tough-cookie": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", + "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/validator": { "version": "13.15.10", "resolved": "https://registry.npmjs.org/@types/validator/-/validator-13.15.10.tgz", @@ -1574,6 +1833,15 @@ "node": "20.x || 22.x || 23.x || 24.x || 25.x" } }, + "node_modules/bidi-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/bidi-js/-/bidi-js-1.0.3.tgz", + "integrity": "sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw==", + "license": "MIT", + "dependencies": { + "require-from-string": "^2.0.2" + } + }, "node_modules/bindings": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", @@ -2128,6 +2396,32 @@ "node": ">= 8" } }, + "node_modules/css-tree": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-3.2.1.tgz", + "integrity": "sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA==", + "license": "MIT", + "dependencies": { + "mdn-data": "2.27.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0" + } + }, + "node_modules/data-urls": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-7.0.0.tgz", + "integrity": "sha512-23XHcCF+coGYevirZceTVD7NdJOqVn+49IHyxgszm+JIiHLoB2TkmPtsYkNWT1pvRSGkc35L6NHs0yHkN2SumA==", + "license": "MIT", + "dependencies": { + "whatwg-mimetype": "^5.0.0", + "whatwg-url": "^16.0.0" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, "node_modules/dayjs": { "version": "1.11.20", "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.20.tgz", @@ -2151,6 +2445,12 @@ } } }, + "node_modules/decimal.js": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.6.0.tgz", + "integrity": "sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==", + "license": "MIT" + }, "node_modules/decompress-response": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", @@ -2357,6 +2657,18 @@ "node": ">=10.13.0" } }, + "node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/error-ex": { "version": "1.3.4", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", @@ -2957,6 +3269,18 @@ "node": ">= 0.4" } }, + "node_modules/html-encoding-sniffer": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-6.0.0.tgz", + "integrity": "sha512-CV9TW3Y3f8/wT0BRFc1/KAVQ3TUHiXmaAb6VW9vtiMFf7SLoMd1PdAc4W3KFOFETBJUb90KatHqlsZMWV+R9Gg==", + "license": "MIT", + "dependencies": { + "@exodus/bytes": "^1.6.0" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, "node_modules/http-errors": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", @@ -3089,6 +3413,12 @@ "node": ">=8" } }, + "node_modules/is-potential-custom-element-name": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", + "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==", + "license": "MIT" + }, "node_modules/is-promise": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", @@ -3209,6 +3539,46 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/jsdom": { + "version": "29.0.2", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-29.0.2.tgz", + "integrity": "sha512-9VnGEBosc/ZpwyOsJBCQ/3I5p7Q5ngOY14a9bf5btenAORmZfDse1ZEheMiWcJ3h81+Fv7HmJFdS0szo/waF2w==", + "license": "MIT", + "dependencies": { + "@asamuzakjp/css-color": "^5.1.5", + "@asamuzakjp/dom-selector": "^7.0.6", + "@bramus/specificity": "^2.4.2", + "@csstools/css-syntax-patches-for-csstree": "^1.1.1", + "@exodus/bytes": "^1.15.0", + "css-tree": "^3.2.1", + "data-urls": "^7.0.0", + "decimal.js": "^10.6.0", + "html-encoding-sniffer": "^6.0.0", + "is-potential-custom-element-name": "^1.0.1", + "lru-cache": "^11.2.7", + "parse5": "^8.0.0", + "saxes": "^6.0.0", + "symbol-tree": "^3.2.4", + "tough-cookie": "^6.0.1", + "undici": "^7.24.5", + "w3c-xmlserializer": "^5.0.0", + "webidl-conversions": "^8.0.1", + "whatwg-mimetype": "^5.0.0", + "whatwg-url": "^16.0.1", + "xml-name-validator": "^5.0.0" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24.0.0" + }, + "peerDependencies": { + "canvas": "^3.0.0" + }, + "peerDependenciesMeta": { + "canvas": { + "optional": true + } + } + }, "node_modules/json-parse-even-better-errors": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", @@ -3329,7 +3699,6 @@ "version": "11.3.2", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.3.2.tgz", "integrity": "sha512-wgWa6FWQ3QRRJbIjbsldRJZxdxYngT/dO0I5Ynmlnin8qy7tC6xYzbcJjtN4wHLXtkbVwHzk0C+OejVw1XM+DQ==", - "dev": true, "license": "BlueOak-1.0.0", "engines": { "node": "20 || >=22" @@ -3354,6 +3723,12 @@ "node": ">= 0.4" } }, + "node_modules/mdn-data": { + "version": "2.27.1", + "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.27.1.tgz", + "integrity": "sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==", + "license": "CC0-1.0" + }, "node_modules/media-typer": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", @@ -3735,6 +4110,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/parse5": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.0.tgz", + "integrity": "sha512-9m4m5GSgXjL4AjumKzq1Fgfp3Z8rsvjRNbnkVwfu2ImRqE5D0LnY2QfDen18FSY9C573YU5XxSapdHZTZ2WolA==", + "license": "MIT", + "dependencies": { + "entities": "^6.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -3913,7 +4300,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -4020,7 +4406,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -4108,6 +4493,18 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "license": "MIT" }, + "node_modules/saxes": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz", + "integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==", + "license": "ISC", + "dependencies": { + "xmlchars": "^2.2.0" + }, + "engines": { + "node": ">=v12.22.7" + } + }, "node_modules/schema-utils": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz", @@ -4421,6 +4818,15 @@ "node": ">= 8" } }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/source-map-support": { "version": "0.5.21", "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", @@ -4620,6 +5026,12 @@ "node": ">=0.10" } }, + "node_modules/symbol-tree": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", + "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", + "license": "MIT" + }, "node_modules/tapable": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.2.tgz", @@ -4771,6 +5183,24 @@ "dev": true, "license": "MIT" }, + "node_modules/tldts": { + "version": "7.0.28", + "resolved": "https://registry.npmjs.org/tldts/-/tldts-7.0.28.tgz", + "integrity": "sha512-+Zg3vWhRUv8B1maGSTFdev9mjoo8Etn2Ayfs4cnjlD3CsGkxXX4QyW3j2WJ0wdjYcYmy7Lx2RDsZMhgCWafKIw==", + "license": "MIT", + "dependencies": { + "tldts-core": "^7.0.28" + }, + "bin": { + "tldts": "bin/cli.js" + } + }, + "node_modules/tldts-core": { + "version": "7.0.28", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.0.28.tgz", + "integrity": "sha512-7W5Efjhsc3chVdFhqtaU0KtK32J37Zcr9RKtID54nG+tIpcY79CQK/veYPODxtD/LJ4Lue66jvrQzIX2Z2/pUQ==", + "license": "MIT" + }, "node_modules/to-buffer": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.2.2.tgz", @@ -4812,6 +5242,30 @@ "url": "https://github.com/sponsors/Borewit" } }, + "node_modules/tough-cookie": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.1.tgz", + "integrity": "sha512-LktZQb3IeoUWB9lqR5EWTHgW/VTITCXg4D21M+lvybRVdylLrRMnqaIONLVb5mav8vM19m44HIcGq4qASeu2Qw==", + "license": "BSD-3-Clause", + "dependencies": { + "tldts": "^7.0.5" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/tr46": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-6.0.0.tgz", + "integrity": "sha512-bLVMLPtstlZ4iMQHpFHTR7GAGj2jxi8Dg0s2h2MafAE4uSWF98FC/3MomU51iQAMf8/qDUbKWf5GxuvvVcXEhw==", + "license": "MIT", + "dependencies": { + "punycode": "^2.3.1" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/tsconfig-paths": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-4.2.0.tgz", @@ -5138,6 +5592,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/undici": { + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.24.7.tgz", + "integrity": "sha512-H/nlJ/h0ggGC+uRL3ovD+G0i4bqhvsDOpbDv7At5eFLlj2b41L8QliGbnl2H7SnDiYhENphh1tQFJZf+MyfLsQ==", + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, "node_modules/undici-types": { "version": "7.18.2", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", @@ -5242,6 +5705,18 @@ "node": ">= 0.8" } }, + "node_modules/w3c-xmlserializer": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-5.0.0.tgz", + "integrity": "sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==", + "license": "MIT", + "dependencies": { + "xml-name-validator": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/watchpack": { "version": "2.5.1", "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.5.1.tgz", @@ -5266,6 +5741,15 @@ "defaults": "^1.0.3" } }, + "node_modules/webidl-conversions": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-8.0.1.tgz", + "integrity": "sha512-BMhLD/Sw+GbJC21C/UgyaZX41nPt8bUTg+jWyDeg7e7YN4xOM05YPSIXceACnXVtqyEw/LMClUQMtMZ+PGGpqQ==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=20" + } + }, "node_modules/webpack": { "version": "5.105.4", "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.105.4.tgz", @@ -5396,6 +5880,29 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/whatwg-mimetype": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-5.0.0.tgz", + "integrity": "sha512-sXcNcHOC51uPGF0P/D4NVtrkjSU2fNsm9iog4ZvZJsL3rjoDAzXZhkm2MWt1y+PUdggKAYVoMAIYcs78wJ51Cw==", + "license": "MIT", + "engines": { + "node": ">=20" + } + }, + "node_modules/whatwg-url": { + "version": "16.0.1", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-16.0.1.tgz", + "integrity": "sha512-1to4zXBxmXHV3IiSSEInrreIlu02vUOvrhxJJH5vcxYTBDAx51cqZiKdyTxlecdKNSjj8EcxGBxNf6Vg+945gw==", + "license": "MIT", + "dependencies": { + "@exodus/bytes": "^1.11.0", + "tr46": "^6.0.0", + "webidl-conversions": "^8.0.1" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -5471,6 +5978,21 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", "license": "ISC" }, + "node_modules/xml-name-validator": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-5.0.0.tgz", + "integrity": "sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==", + "license": "Apache-2.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/xmlchars": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", + "license": "MIT" + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/package.json b/package.json index e9db800..5259f37 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "author": "", "license": "ISC", "dependencies": { + "@mozilla/readability": "^0.6.0", "@nestjs/common": "^11.1.18", "@nestjs/config": "^4.0.3", "@nestjs/core": "^11.1.18", @@ -22,6 +23,7 @@ "better-sqlite3": "^12.8.0", "class-transformer": "^0.5.1", "class-validator": "^0.14.4", + "jsdom": "^29.0.2", "playwright": "^1.59.1", "reflect-metadata": "^0.2.2", "rxjs": "^7.8.2", @@ -31,6 +33,8 @@ "devDependencies": { "@nestjs/cli": "^11.0.17", "@types/better-sqlite3": "^7.6.13", + "@types/jsdom": "^28.0.1", + "@types/mozilla__readability": "^0.4.2", "@types/node": "^25.5.2", "typescript": "^6.0.2" } diff --git a/src/app.module.ts b/src/app.module.ts index eb78044..cd42142 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -3,6 +3,7 @@ import { ConfigModule, ConfigService } from '@nestjs/config'; import { TypeOrmModule } from '@nestjs/typeorm'; import { HealthController } from './health/health.controller'; import { AuthModule } from './auth/auth.module'; +import { BrowserModule } from './browser/browser.module'; import { SessionEntity } from './session/session.entity'; @Module({ @@ -21,6 +22,7 @@ import { SessionEntity } from './session/session.entity'; }), }), AuthModule, + BrowserModule, ], controllers: [HealthController], }) diff --git a/src/browser/browser.controller.ts b/src/browser/browser.controller.ts new file mode 100644 index 0000000..7ee4618 --- /dev/null +++ b/src/browser/browser.controller.ts @@ -0,0 +1,29 @@ +import { Body, Controller, Post } from '@nestjs/common'; +import { ApiOperation, ApiResponse, ApiTags } from '@nestjs/swagger'; +import { BrowserService, OpenResult } from './browser.service'; +import { OpenDto } from './dto/open.dto'; + +@ApiTags('browser') +@Controller() +export class BrowserController { + constructor(private readonly browserService: BrowserService) {} + + @Post('open') + @ApiOperation({ summary: 'Open a URL with a stored session (cookies + localStorage)' }) + @ApiResponse({ + status: 201, + description: 'Page loaded successfully', + schema: { + properties: { + url: { type: 'string' }, + title: { type: 'string' }, + content: { type: 'string' }, + }, + }, + }) + @ApiResponse({ status: 400, description: 'Session not found or invalid input' }) + @ApiResponse({ status: 500, description: 'Browser automation failed' }) + open(@Body() dto: OpenDto): Promise { + return this.browserService.open(dto.sessionName, dto.url, dto.readerMode ?? false); + } +} diff --git a/src/browser/browser.module.ts b/src/browser/browser.module.ts new file mode 100644 index 0000000..5c2e562 --- /dev/null +++ b/src/browser/browser.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; +import { BrowserController } from './browser.controller'; +import { BrowserService } from './browser.service'; +import { SessionModule } from '../session/session.module'; + +@Module({ + imports: [SessionModule], + controllers: [BrowserController], + providers: [BrowserService], +}) +export class BrowserModule {} diff --git a/src/browser/browser.service.ts b/src/browser/browser.service.ts new file mode 100644 index 0000000..2e35d77 --- /dev/null +++ b/src/browser/browser.service.ts @@ -0,0 +1,89 @@ +import { + Injectable, + BadRequestException, + InternalServerErrorException, + Logger, +} from '@nestjs/common'; +import { chromium } from 'playwright'; +import { Readability } from '@mozilla/readability'; +import { JSDOM } from 'jsdom'; +import { SessionService } from '../session/session.service'; + +export interface OpenResult { + url: string; + title: string; + content: string; +} + +@Injectable() +export class BrowserService { + private readonly logger = new Logger(BrowserService.name); + + constructor(private readonly sessionService: SessionService) {} + + async open(sessionName: string, url: string, readerMode = false): Promise { + const session = await this.sessionService.findBySessionName(sessionName); + if (!session) { + throw new BadRequestException(`Session not found: ${sessionName}`); + } + + let cookies: any[]; + let localStorageData: Record; + try { + cookies = JSON.parse(session.cookies); + localStorageData = JSON.parse(session.localStorage); + } catch { + throw new InternalServerErrorException('Failed to deserialize session data'); + } + + const browser = await chromium.launch({ headless: true }); + try { + const context = await browser.newContext(); + + // Restore cookies + await context.addCookies(cookies); + + const page = await context.newPage(); + + // Restore localStorage before the page makes any authenticated requests + // by injecting it via init script + await context.addInitScript((entries: Record) => { + for (const [k, v] of Object.entries(entries)) { + window.localStorage.setItem(k, v); + } + }, localStorageData); + + this.logger.log(`[${sessionName}] Opening ${url}`); + await page.goto(url, { waitUntil: 'networkidle' }); + + const finalUrl = page.url(); + const title = await page.title(); + const rawHtml = await page.content(); + + let content: string; + if (readerMode) { + const dom = new JSDOM(rawHtml, { url: finalUrl }); + const article = new Readability(dom.window.document).parse(); + content = article ? article.textContent.replace(/\s+/g, ' ').trim() : rawHtml; + } else { + content = rawHtml; + } + + this.logger.log(`[${sessionName}] Loaded: ${finalUrl} — "${title}"${readerMode ? ' (reader mode)' : ''}`); + return { url: finalUrl, title, content }; + } catch (err) { + if ( + err instanceof BadRequestException || + err instanceof InternalServerErrorException + ) { + throw err; + } + this.logger.error(`[${sessionName}] Open failed: ${(err as Error).message}`); + throw new InternalServerErrorException( + `Browser open failed: ${(err as Error).message}`, + ); + } finally { + await browser.close(); + } + } +} diff --git a/src/browser/dto/open.dto.ts b/src/browser/dto/open.dto.ts new file mode 100644 index 0000000..eb9bb87 --- /dev/null +++ b/src/browser/dto/open.dto.ts @@ -0,0 +1,26 @@ +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; +import { IsBoolean, IsOptional, IsString, IsUrl } from 'class-validator'; + +export class OpenDto { + @ApiProperty({ + description: 'Session name previously created by POST /login', + example: 'test-session-1', + }) + @IsString() + sessionName: string; + + @ApiProperty({ + description: 'URL to open with the authenticated session', + example: 'https://cabinet-liquio-diia-stg.kitsoft.ua/messages', + }) + @IsUrl({ require_tld: true, require_protocol: true }) + url: string; + + @ApiPropertyOptional({ + description: 'When true, return a plain-text reader-mode summary instead of raw HTML', + default: false, + }) + @IsOptional() + @IsBoolean() + readerMode?: boolean; +} diff --git a/src/session/session.entity.ts b/src/session/session.entity.ts index 854435d..8d9ceb2 100644 --- a/src/session/session.entity.ts +++ b/src/session/session.entity.ts @@ -20,7 +20,7 @@ export class SessionEntity { @Column('text') cookies: string; // JSON-serialised Cookie[] from Playwright - @Column('text') + @Column('text', { default: '{}' }) localStorage: string; // JSON-serialised Record from Playwright @CreateDateColumn()