diff --git a/README.md b/README.md index 3762d7d..50449d6 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ ------ -[![npm package](https://nodei.co/npm/crawler.png?downloads=true&downloadRank=true&stars=true)](https://www.npmjs.com/package/crawler/v/2.0.0) +[![npm package](https://nodei.co/npm/crawler.png?downloads=true&downloadRank=true&stars=true)](https://www.npmjs.com/package/crawler/v/2.0.2) [![CircleCI](https://circleci.com/gh/bda-research/node-crawler/tree/master.svg?style=svg)](https://circleci.com/gh/bda-research/node-crawler/tree/master) [![NPM download][download-image]][download-url] @@ -246,6 +246,7 @@ crawler.send({ - [crawler.queueSize](#crawlerqueuesize) - [Options](#options) - [Global only options](#global-only-options) + - [`silence`](#silence) - [`maxConnections`](#maxconnections) - [`priorityLevels`](#prioritylevels) - [`rateLimit`](#ratelimit) @@ -520,13 +521,13 @@ items in the **crawler.add()** calls if you want them to be specific to that ite #### `retryInterval` - **Type:** `number` -- **Default** : 2000 +- **Default** : 3000 - The number of milliseconds to wait before retrying. #### `timeout` - **Type:** `number` -- **Default** : 15000 +- **Default** : 20000 - The number of milliseconds to wait before the request times out. #### `priority` diff --git a/eslint.config.js b/eslint.config.js index 20295b3..00e5bcd 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -21,7 +21,6 @@ const options = [ files: ["src/**/*.ts"], rules: { "@typescript-eslint/no-explicit-any": "warn", - "@typescript-eslint/ban-types": "warn", "@typescript-eslint/no-unused-vars": [ "error", { diff --git a/package.json b/package.json index c8f9ee3..0eb2c60 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "crawler", - "version": "2.0.1", + "version": "2.0.2", "description": "Crawler is a ready-to-use web spider that works with proxies, asynchrony, rate limit, configurable request pools, jQuery, and HTTP/2 support.", "repository": { "type": "git", @@ -36,7 +36,7 @@ "license": "MIT", "dependencies": { "cheerio": "1.0.0-rc.12", - "got": "^14.4.1", + "got": "^14.4.2", "hpagent": "^1.2.0", "http2-wrapper": "^2.2.1", "iconv-lite": "^0.6.3", @@ -44,19 +44,19 @@ "tslog": "^4.9.3" }, "devDependencies": { - "@eslint/js": "^9.5.0", + "@eslint/js": "^9.8.0", "@types/got": "^9.6.12", - "@types/node": "^20.14.8", + "@types/node": "^20.14.13", "ava": "^6.1.3", "c8": "^10.1.2", - "eslint": "~9.4.0", - "globals": "^15.6.0", + "eslint": "^9.8.0", + "globals": "^15.8.0", "nock": "^13.5.4", "sinon": "^18.0.0", "tough-cookie": "^4.1.4", - "tsx": "^4.15.7", - "typescript": "^5.4.5", - "typescript-eslint": "8.0.0-alpha.30" + "tsx": "^4.16.3", + "typescript": "^5.5.4", + "typescript-eslint": "^8.0.0" }, "ava": { "files": [ @@ -77,4 +77,4 @@ ], "clean": true } -} +} \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3d41681..1962f40 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,8 +12,8 @@ importers: specifier: 1.0.0-rc.12 version: 1.0.0-rc.12 got: - specifier: ^14.4.1 - version: 14.4.1 + specifier: ^14.4.2 + version: 14.4.2 hpagent: specifier: ^1.2.0 version: 1.2.0 @@ -31,14 +31,14 @@ importers: version: 4.9.3 devDependencies: '@eslint/js': - specifier: ^9.5.0 - version: 9.5.0 + specifier: ^9.8.0 + version: 9.8.0 '@types/got': specifier: ^9.6.12 version: 9.6.12 '@types/node': - specifier: ^20.14.8 - version: 20.14.8 + specifier: ^20.14.13 + version: 20.14.13 ava: specifier: ^6.1.3 version: 6.1.3 @@ -46,11 +46,11 @@ importers: specifier: ^10.1.2 version: 10.1.2 eslint: - specifier: ~9.4.0 - version: 9.4.0 + specifier: ^9.8.0 + version: 9.8.0 globals: - specifier: ^15.6.0 - version: 15.6.0 + specifier: ^15.8.0 + version: 15.8.0 nock: specifier: ^13.5.4 version: 13.5.4 @@ -61,14 +61,14 @@ importers: specifier: ^4.1.4 version: 4.1.4 tsx: - specifier: ^4.15.7 - version: 4.15.7 + specifier: ^4.16.3 + version: 4.16.3 typescript: - specifier: ^5.4.5 - version: 5.4.5 + specifier: ^5.5.4 + version: 5.5.4 typescript-eslint: - specifier: 8.0.0-alpha.30 - version: 8.0.0-alpha.30(eslint@9.4.0)(typescript@5.4.5) + specifier: ^8.0.0 + version: 8.0.0(eslint@9.8.0)(typescript@5.5.4) packages: @@ -219,24 +219,20 @@ packages: peerDependencies: eslint: ^6.0.0 || ^7.0.0 || >=8.0.0 - '@eslint-community/regexpp@4.10.1': - resolution: {integrity: sha512-Zm2NGpWELsQAD1xsJzGQpYfvICSsFkEpU0jxBjfdC6uNEWXcHnfs9hScFWtXVDVl+rBQJGrl4g1vcKIejpH9dA==} + '@eslint-community/regexpp@4.11.0': + resolution: {integrity: sha512-G/M/tIiMrTAxEWRfLfQJMmGNX28IxBg4PBz8XqQhqUHLFI6TL2htpIB1iQCj144V5ee/JaKyT9/WZ0MGZWfA7A==} engines: {node: ^12.0.0 || ^14.0.0 || >=16.0.0} - '@eslint/config-array@0.15.1': - resolution: {integrity: sha512-K4gzNq+yymn/EVsXYmf+SBcBro8MTf+aXJZUphM96CdzUEr+ClGDvAbpmaEK+cGVigVXIgs9gNmvHAlrzzY5JQ==} + '@eslint/config-array@0.17.1': + resolution: {integrity: sha512-BlYOpej8AQ8Ev9xVqroV7a02JK3SkBAaN9GfMMH9W6Ch8FlQlkjGw4Ir7+FgYwfirivAf4t+GtzuAxqfukmISA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} '@eslint/eslintrc@3.1.0': resolution: {integrity: sha512-4Bfj15dVJdoy3RfZmmo86RK1Fwzn6SstsvK9JS+BaVKqC6QQQQyXekNaC+g+LKNgkQ+2VhGAzm6hO40AhMR3zQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - '@eslint/js@9.4.0': - resolution: {integrity: sha512-fdI7VJjP3Rvc70lC4xkFXHB0fiPeojiL1PxVG6t1ZvXQrarj893PweuBTujxDUFk0Fxj4R7PIIAZ/aiiyZPZcg==} - engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - - '@eslint/js@9.5.0': - resolution: {integrity: sha512-A7+AOT2ICkodvtsWnxZP4Xxk3NbZ3VMHd8oihydLRGrJgqqdEz1qSeEgXYyT/Cu8h1TWWsQRejIx48mtjZ5y1w==} + '@eslint/js@9.8.0': + resolution: {integrity: sha512-MfluB7EUfxXtv3i/++oh89uzAr4PDI4nn201hsp+qaXqsjAWzinlZEHEfPgAX4doIlKvPG/i0A9dpKxOLII8yA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} '@eslint/object-schema@2.1.4': @@ -263,8 +259,8 @@ packages: resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} engines: {node: '>=6.0.0'} - '@jridgewell/sourcemap-codec@1.4.15': - resolution: {integrity: sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==} + '@jridgewell/sourcemap-codec@1.5.0': + resolution: {integrity: sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==} '@jridgewell/trace-mapping@0.3.25': resolution: {integrity: sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==} @@ -296,9 +292,9 @@ packages: '@sec-ant/readable-stream@0.4.1': resolution: {integrity: sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==} - '@sindresorhus/is@6.3.1': - resolution: {integrity: sha512-FX4MfcifwJyFOI2lPoX7PQxCqx8BG1HCho7WdiXwpEQx1Ycij0JxkfYtGK7yqNScrZGSlt6RE6sw8QYoH7eKnQ==} - engines: {node: '>=16'} + '@sindresorhus/is@7.0.0': + resolution: {integrity: sha512-WDTlVTyvFivSOuyvMeedzg2hdoBLZ3f1uNVuEida2Rl9BrfjrIRjWA/VZIrMRLvSwJYCAlCRA3usDt1THytxWQ==} + engines: {node: '>=18'} '@sindresorhus/merge-streams@2.3.0': resolution: {integrity: sha512-LtoMMhxAlorcGhmFYI+LhPgbPZCkgP6ra1YL604EeF6U98pLlQ3iWIGMdWSC+vWmPBWBNgmDBAhnAobLROJmwg==} @@ -332,14 +328,14 @@ packages: '@types/istanbul-lib-coverage@2.0.6': resolution: {integrity: sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==} - '@types/node@20.14.8': - resolution: {integrity: sha512-DO+2/jZinXfROG7j7WKFn/3C6nFwxy2lLpgLjEXJz+0XKphZlTLJ14mo8Vfg8X5BWN6XjyESXq+LcYdT7tR3bA==} + '@types/node@20.14.13': + resolution: {integrity: sha512-+bHoGiZb8UiQ0+WEtmph2IWQCjIqg8MDZMAV+ppRRhUZnquF5mQkP/9vpSwJClEiSM/C7fZZExPzfU0vJTyp8w==} '@types/tough-cookie@4.0.5': resolution: {integrity: sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==} - '@typescript-eslint/eslint-plugin@8.0.0-alpha.30': - resolution: {integrity: sha512-2CBUupdkfbE3eATph4QeZejvT+M+1bVur+zXlVx09WN31phap51ps/qemeclnCbGEz6kTgBDmScrr9XmmF8/Pg==} + '@typescript-eslint/eslint-plugin@8.0.0': + resolution: {integrity: sha512-STIZdwEQRXAHvNUS6ILDf5z3u95Gc8jzywunxSNqX00OooIemaaNIA0vEgynJlycL5AjabYLLrIyHd4iazyvtg==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: '@typescript-eslint/parser': ^8.0.0 || ^8.0.0-alpha.0 @@ -349,8 +345,8 @@ packages: typescript: optional: true - '@typescript-eslint/parser@8.0.0-alpha.30': - resolution: {integrity: sha512-tAYgFmgXU1MlCK3nbblUvJlDSibBvxtAQXGrF3IG0KmnRza9FXILZifHWL0rrwacDn40K53K607Fk2QkMjiGgw==} + '@typescript-eslint/parser@8.0.0': + resolution: {integrity: sha512-pS1hdZ+vnrpDIxuFXYQpLTILglTjSYJ9MbetZctrUawogUsPdz31DIIRZ9+rab0LhYNTsk88w4fIzVheiTbWOQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: eslint: ^8.57.0 || ^9.0.0 @@ -359,12 +355,12 @@ packages: typescript: optional: true - '@typescript-eslint/scope-manager@8.0.0-alpha.30': - resolution: {integrity: sha512-FGW/iPWGyPFamAVZ60oCAthMqQrqafUGebF8UKuq/ha+e9SVG6YhJoRzurlQXOVf8dHfOhJ0ADMXyFnMc53clg==} + '@typescript-eslint/scope-manager@8.0.0': + resolution: {integrity: sha512-V0aa9Csx/ZWWv2IPgTfY7T4agYwJyILESu/PVqFtTFz9RIS823mAze+NbnBI8xiwdX3iqeQbcTYlvB04G9wyQw==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - '@typescript-eslint/type-utils@8.0.0-alpha.30': - resolution: {integrity: sha512-FrnhlCKEKZKRbpDviHkIU9tayIUGTOfa+SjvrRv6p/AJIUv6QT8oRboRjLH/cCuwUEbM0k5UtRWYug4albHUqQ==} + '@typescript-eslint/type-utils@8.0.0': + resolution: {integrity: sha512-mJAFP2mZLTBwAn5WI4PMakpywfWFH5nQZezUQdSKV23Pqo6o9iShQg1hP2+0hJJXP2LnZkWPphdIq4juYYwCeg==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: typescript: '*' @@ -372,12 +368,12 @@ packages: typescript: optional: true - '@typescript-eslint/types@8.0.0-alpha.30': - resolution: {integrity: sha512-4WzLlw27SO9pK9UFj/Hu7WGo8WveT0SEiIpFVsV2WwtQmLps6kouwtVCB8GJPZKJyurhZhcqCoQVQFmpv441Vg==} + '@typescript-eslint/types@8.0.0': + resolution: {integrity: sha512-wgdSGs9BTMWQ7ooeHtu5quddKKs5Z5dS+fHLbrQI+ID0XWJLODGMHRfhwImiHoeO2S5Wir2yXuadJN6/l4JRxw==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - '@typescript-eslint/typescript-estree@8.0.0-alpha.30': - resolution: {integrity: sha512-WSXbc9ZcXI+7yC+6q95u77i8FXz6HOLsw3ST+vMUlFy1lFbXyFL/3e6HDKQCm2Clt0krnoCPiTGvIn+GkYPn4Q==} + '@typescript-eslint/typescript-estree@8.0.0': + resolution: {integrity: sha512-5b97WpKMX+Y43YKi4zVcCVLtK5F98dFls3Oxui8LbnmRsseKenbbDinmvxrWegKDMmlkIq/XHuyy0UGLtpCDKg==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: typescript: '*' @@ -385,14 +381,14 @@ packages: typescript: optional: true - '@typescript-eslint/utils@8.0.0-alpha.30': - resolution: {integrity: sha512-rfhqfLqFyXhHNDwMnHiVGxl/Z2q/3guQ1jLlGQ0hi9Rb7inmwz42crM+NnLPR+2vEnwyw1P/g7fnQgQ3qvFx4g==} + '@typescript-eslint/utils@8.0.0': + resolution: {integrity: sha512-k/oS/A/3QeGLRvOWCg6/9rATJL5rec7/5s1YmdS0ZU6LHveJyGFwBvLhSRBv6i9xaj7etmosp+l+ViN1I9Aj/Q==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: eslint: ^8.57.0 || ^9.0.0 - '@typescript-eslint/visitor-keys@8.0.0-alpha.30': - resolution: {integrity: sha512-XZuNurZxBqmr6ZIRIwWFq7j5RZd6ZlkId/HZEWyfciK+CWoyOxSF9Pv2VXH9Rlu2ZG2PfbhLz2Veszl4Pfn7yA==} + '@typescript-eslint/visitor-keys@8.0.0': + resolution: {integrity: sha512-oN0K4nkHuOyF3PVMyETbpP5zp6wfyOvm7tWhTMfoqxSSsPmJIh6JNASuZDlODE8eE+0EB9uar+6+vxr9DBTYOA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} '@vercel/nft@0.26.5': @@ -417,8 +413,8 @@ packages: resolution: {integrity: sha512-MxXdReSRhGO7VlFe1bRG/oI7/mdLV9B9JJT0N8vZOhF7gFRR5l3M8W9G8JxmKV+JC5mGqJ0QvqfSOLsCPa4nUw==} engines: {node: '>=0.4.0'} - acorn@8.12.0: - resolution: {integrity: sha512-RTvkC4w+KNXrM39/lWCUaG0IbRkWdCv7W/IOW9oU6SawyxulvkQy5HQPVTKxEjczcUvapcrw3cFx/60VN/NRNw==} + acorn@8.12.1: + resolution: {integrity: sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==} engines: {node: '>=0.4.0'} hasBin: true @@ -535,8 +531,8 @@ packages: resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} engines: {node: '>=6'} - callsites@4.1.0: - resolution: {integrity: sha512-aBMbD1Xxay75ViYezwT40aQONfr+pSXTHwNKvIXhXD6+LY3F1dLIcceoC5OZKBVHbXcysz1hL9D2w0JJIMXpUw==} + callsites@4.2.0: + resolution: {integrity: sha512-kfzR4zzQtAE9PC7CzZsjl3aBNbXWuXiSeOCdLcPpBfGW8YuCqQHcRPFDbr/BPVmd3EEPVpuFzLyuT/cUhPr4OQ==} engines: {node: '>=12.20'} cbor@9.0.2: @@ -638,8 +634,8 @@ packages: resolution: {integrity: sha512-uqCUKXE5q1PNBXjPqvwhwJf9SwMoAHBgWJ6DcrnS5o+W2JOiIILl0JEdVD8SGujrNS02GGxgwAg2PN2zONgtjg==} engines: {node: '>=6'} - debug@4.3.5: - resolution: {integrity: sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==} + debug@4.3.6: + resolution: {integrity: sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg==} engines: {node: '>=6.0'} peerDependencies: supports-color: '*' @@ -731,8 +727,8 @@ packages: resolution: {integrity: sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==} engines: {node: '>=12'} - eslint-scope@8.0.1: - resolution: {integrity: sha512-pL8XjgP4ZOmmwfFE8mEhSxA7ZY4C+LWyqjQ3o4yWkkmD0qcMT9kkW3zWHOczhWcjTSgqycYAgwSlXvZltv65og==} + eslint-scope@8.0.2: + resolution: {integrity: sha512-6E4xmrTw5wtxnLA5wYL3WDfhZ/1bUBGOXV0zQvVRDOtrR8D0p6W7fs3JweNYhwRYeGvd/1CKX2se0/2s7Q/nJA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} eslint-visitor-keys@3.4.3: @@ -743,8 +739,8 @@ packages: resolution: {integrity: sha512-OtIRv/2GyiF6o/d8K7MYKKbXrOUBIK6SfkIRM4Z0dY3w+LiQ0vy3F57m0Z71bjbyeiWFiHJ8brqnmE6H6/jEuw==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - eslint@9.4.0: - resolution: {integrity: sha512-sjc7Y8cUD1IlwYcTS9qPSvGjAC8Ne9LctpxKKu3x/1IC9bnOg98Zy6GxEJUfr1NojMgVPlyANXYns8oE2c1TAA==} + eslint@9.8.0: + resolution: {integrity: sha512-K8qnZ/QJzT2dLKdZJVX6W4XOwBzutMYmt0lqUS+JdXgd+HTYFlonFgkJ8s44d/zMPPCnOOk0kMWCApCPhiOy9A==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} hasBin: true @@ -757,8 +753,8 @@ packages: engines: {node: '>=4'} hasBin: true - esquery@1.5.0: - resolution: {integrity: sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==} + esquery@1.6.0: + resolution: {integrity: sha512-ca9pw9fomFcKPvFLXhBKUK90ZvGibiGOvRJNbjljY7s7uq/5YO4BOzcYtJqExdx99rF6aAcnRxHmcUHcz6sQsg==} engines: {node: '>=0.10'} esrecurse@4.3.0: @@ -862,16 +858,12 @@ packages: resolution: {integrity: sha512-2nk+7SIVb14QrgXFHcm84tD4bKQz0RxPuMT8Ag5KPOq7J5fEmAg0UbXdTOSHqNuHSU28k55qnceesxXRZGzKWA==} engines: {node: '>=18'} - get-stream@8.0.1: - resolution: {integrity: sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==} - engines: {node: '>=16'} - get-stream@9.0.1: resolution: {integrity: sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA==} engines: {node: '>=18'} - get-tsconfig@4.7.5: - resolution: {integrity: sha512-ZCuZCnlqNzjb4QprAzXKdpp/gh6KTxSJuw3IBsPnV/7fV4NxC9ckB+vPTt8w7fJA0TaSD7c55BR47JD6MEDyDw==} + get-tsconfig@4.7.6: + resolution: {integrity: sha512-ZAqrLlu18NbDdRaHq+AKXzAmqIUPswPWKUchfytdAjiRFnCe5ojG2bstg6mRiZabkKfCoL/e98pbBELIV/YCeA==} glob-parent@5.1.2: resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} @@ -881,9 +873,8 @@ packages: resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} engines: {node: '>=10.13.0'} - glob@10.4.2: - resolution: {integrity: sha512-GwMlUF6PkPo3Gk21UxkCohOv0PLcIXVtKyLlpEI28R/cO/4eNOdmLk3CMW1wROV/WR/EsZOWAfBbBOqYvs88/w==} - engines: {node: '>=16 || 14 >=14.18'} + glob@10.4.5: + resolution: {integrity: sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==} hasBin: true glob@7.2.3: @@ -894,20 +885,20 @@ packages: resolution: {integrity: sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==} engines: {node: '>=18'} - globals@15.6.0: - resolution: {integrity: sha512-UzcJi88Hw//CurUIRa9Jxb0vgOCcuD/MNjwmXp633cyaRKkCWACkoqHCtfZv43b1kqXGg/fpOa8bwgacCeXsVg==} + globals@15.8.0: + resolution: {integrity: sha512-VZAJ4cewHTExBWDHR6yptdIBlx9YSSZuwojj9Nt5mBRXQzrKakDsVKQ1J63sklLvzAJm0X5+RpO4i3Y2hcOnFw==} engines: {node: '>=18'} globby@11.1.0: resolution: {integrity: sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==} engines: {node: '>=10'} - globby@14.0.1: - resolution: {integrity: sha512-jOMLD2Z7MAhyG8aJpNOpmziMOP4rPLcc95oQPKXBazW82z+CEgPFBQvEpRUa1KeIMUJo4Wsm+q6uzO/Q/4BksQ==} + globby@14.0.2: + resolution: {integrity: sha512-s3Fq41ZVh7vbbe2PN3nrW7yC7U7MFVc5c98/iTl9c2GawNMKx/J648KQRW6WKkuU8GIbbh2IXfIRQjOZnXcTnw==} engines: {node: '>=18'} - got@14.4.1: - resolution: {integrity: sha512-IvDJbJBUeexX74xNQuMIVgCRRuNOm5wuK+OC3Dc2pnSoh1AOmgc7JVj7WC+cJ4u0aPcO9KZ2frTXcqK4W/5qTQ==} + got@14.4.2: + resolution: {integrity: sha512-+Te/qEZ6hr7i+f0FNgXx/6WQteSM/QqueGvxeYQQFm0GDfoxLVJ/oiwUKYMTeioColWUTdewZ06hmrBjw6F7tw==} engines: {node: '>=20'} graceful-fs@4.2.11: @@ -1033,9 +1024,8 @@ packages: resolution: {integrity: sha512-BewmUXImeuRk2YY0PVbxgKAysvhRPUQE0h5QRM++nVWyubKGV0l8qQ5op8+B2DOmwSe63Jivj0BjkPQVf8fP5g==} engines: {node: '>=8'} - jackspeak@3.4.0: - resolution: {integrity: sha512-JVYhQnN59LVPFCEcVa2C3CrEKYacvjRfqIQl+h8oi91aLYQVWRYbxjPcv1bUiUy/kLmQaANrYfNMCO3kuEDHfw==} - engines: {node: '>=14'} + jackspeak@3.4.3: + resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} js-string-escape@1.0.1: resolution: {integrity: sha512-Smw4xcfIQ5LVjAOuJCvN/zIodzA/BBSsluuoSykP+lUvScIi4U6RJLfwHet5cxFnCswUjISV8oAXaqaJDY3chg==} @@ -1092,9 +1082,8 @@ packages: resolution: {integrity: sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - lru-cache@10.2.2: - resolution: {integrity: sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==} - engines: {node: 14 || >=16.14} + lru-cache@10.4.3: + resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} make-dir@3.1.0: resolution: {integrity: sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==} @@ -1147,8 +1136,8 @@ packages: minimatch@3.1.2: resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} - minimatch@9.0.4: - resolution: {integrity: sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==} + minimatch@9.0.5: + resolution: {integrity: sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==} engines: {node: '>=16 || 14 >=14.17'} minipass@3.3.6: @@ -1316,8 +1305,8 @@ packages: resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} engines: {node: '>= 0.8.0'} - pretty-ms@9.0.0: - resolution: {integrity: sha512-E9e9HJ9R9NasGOgPaPE8VMeiPKAyWR5jcFpNnwIejslIhWqdqOrb2wShBsncMPUb+BcCd2OPYfh7p2W6oemTng==} + pretty-ms@9.1.0: + resolution: {integrity: sha512-o1piW0n3tgKIKCwk2vpM/vOV13zjJzvP37Ioze54YlTHE06m4tjEbzg9WsKkvTuyYln2DHjo5pY4qrZGI0otpw==} engines: {node: '>=18'} propagate@2.0.1: @@ -1399,8 +1388,8 @@ packages: resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} hasBin: true - semver@7.6.2: - resolution: {integrity: sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==} + semver@7.6.3: + resolution: {integrity: sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==} engines: {node: '>=10'} hasBin: true @@ -1456,8 +1445,8 @@ packages: resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==} engines: {node: '>=12'} - string-width@7.1.0: - resolution: {integrity: sha512-SEIJCWiX7Kg4c129n48aDRwLbFb2LJmXXFrWBG4NGaRtMQ3myKPKbwrD1BKqQn74oCoNMBVrfDEr5M9YxCsrkw==} + string-width@7.2.0: + resolution: {integrity: sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==} engines: {node: '>=18'} string_decoder@1.3.0: @@ -1523,8 +1512,8 @@ packages: resolution: {integrity: sha512-oDWuGVONxhVEBtschLf2cs/Jy8i7h1T+CpdkTNWQgdAF7DhRo2G8vMCgILKe7ojdEkLhICWgI1LYSSKaJsRgcw==} engines: {node: '>=16'} - tsx@4.15.7: - resolution: {integrity: sha512-u3H0iSFDZM3za+VxkZ1kywdCeHCn+8/qHQS1MNoO2sONDgD95HlWtt8aB23OzeTmFP9IU4/8bZUdg58Uu5J4cg==} + tsx@4.16.3: + resolution: {integrity: sha512-MP8AEUxVnboD2rCC6kDLxnpDBNWN9k3BSVU/0/nNxgm70bPBnfn+yCKcnOsIVPQwdkbKYoFOlKjjWZWJ2XCXUg==} engines: {node: '>=18.0.0'} hasBin: true @@ -1536,16 +1525,20 @@ packages: resolution: {integrity: sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==} engines: {node: '>=4'} + type-detect@4.1.0: + resolution: {integrity: sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw==} + engines: {node: '>=4'} + type-fest@0.13.1: resolution: {integrity: sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==} engines: {node: '>=10'} - type-fest@4.20.1: - resolution: {integrity: sha512-R6wDsVsoS9xYOpy8vgeBlqpdOyzJ12HNfQhC/aAKWM3YoCV9TtunJzh/QpkMgeDhkoynDcw5f1y+qF9yc/HHyg==} + type-fest@4.23.0: + resolution: {integrity: sha512-ZiBujro2ohr5+Z/hZWHESLz3g08BBdrdLMieYFULJO+tWc437sn8kQsWLJoZErY8alNhxre9K4p3GURAG11n+w==} engines: {node: '>=16'} - typescript-eslint@8.0.0-alpha.30: - resolution: {integrity: sha512-/vGhBMsK1TpadQh1eQ02c5pyiPGmKR9cVzX5C9plZ+LC0HPLpWoJbbTVfQN7BkIK7tUxDt2BFr3pFL5hDDrx7g==} + typescript-eslint@8.0.0: + resolution: {integrity: sha512-yQWBJutWL1PmpmDddIOl9/Mi6vZjqNCjqSGBMQ4vsc2Aiodk0SnbQQWPXbSy0HNuKCuGkw1+u4aQ2mO40TdhDQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: typescript: '*' @@ -1553,8 +1546,8 @@ packages: typescript: optional: true - typescript@5.4.5: - resolution: {integrity: sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ==} + typescript@5.5.4: + resolution: {integrity: sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==} engines: {node: '>=14.17'} hasBin: true @@ -1711,17 +1704,17 @@ snapshots: '@esbuild/win32-x64@0.21.5': optional: true - '@eslint-community/eslint-utils@4.4.0(eslint@9.4.0)': + '@eslint-community/eslint-utils@4.4.0(eslint@9.8.0)': dependencies: - eslint: 9.4.0 + eslint: 9.8.0 eslint-visitor-keys: 3.4.3 - '@eslint-community/regexpp@4.10.1': {} + '@eslint-community/regexpp@4.11.0': {} - '@eslint/config-array@0.15.1': + '@eslint/config-array@0.17.1': dependencies: '@eslint/object-schema': 2.1.4 - debug: 4.3.5 + debug: 4.3.6 minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -1729,7 +1722,7 @@ snapshots: '@eslint/eslintrc@3.1.0': dependencies: ajv: 6.12.6 - debug: 4.3.5 + debug: 4.3.6 espree: 10.1.0 globals: 14.0.0 ignore: 5.3.1 @@ -1740,9 +1733,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@eslint/js@9.4.0': {} - - '@eslint/js@9.5.0': {} + '@eslint/js@9.8.0': {} '@eslint/object-schema@2.1.4': {} @@ -1763,12 +1754,12 @@ snapshots: '@jridgewell/resolve-uri@3.1.2': {} - '@jridgewell/sourcemap-codec@1.4.15': {} + '@jridgewell/sourcemap-codec@1.5.0': {} '@jridgewell/trace-mapping@0.3.25': dependencies: '@jridgewell/resolve-uri': 3.1.2 - '@jridgewell/sourcemap-codec': 1.4.15 + '@jridgewell/sourcemap-codec': 1.5.0 '@mapbox/node-pre-gyp@1.0.11': dependencies: @@ -1779,7 +1770,7 @@ snapshots: nopt: 5.0.0 npmlog: 5.0.1 rimraf: 3.0.2 - semver: 7.6.2 + semver: 7.6.3 tar: 6.2.1 transitivePeerDependencies: - encoding @@ -1807,7 +1798,7 @@ snapshots: '@sec-ant/readable-stream@0.4.1': {} - '@sindresorhus/is@6.3.1': {} + '@sindresorhus/is@7.0.0': {} '@sindresorhus/merge-streams@2.3.0': {} @@ -1827,7 +1818,7 @@ snapshots: dependencies: '@sinonjs/commons': 2.0.0 lodash.get: 4.4.2 - type-detect: 4.0.8 + type-detect: 4.1.0 '@sinonjs/text-encoding@0.7.2': {} @@ -1837,7 +1828,7 @@ snapshots: '@types/got@9.6.12': dependencies: - '@types/node': 20.14.8 + '@types/node': 20.14.13 '@types/tough-cookie': 4.0.5 form-data: 2.5.1 @@ -1845,99 +1836,99 @@ snapshots: '@types/istanbul-lib-coverage@2.0.6': {} - '@types/node@20.14.8': + '@types/node@20.14.13': dependencies: undici-types: 5.26.5 '@types/tough-cookie@4.0.5': {} - '@typescript-eslint/eslint-plugin@8.0.0-alpha.30(@typescript-eslint/parser@8.0.0-alpha.30(eslint@9.4.0)(typescript@5.4.5))(eslint@9.4.0)(typescript@5.4.5)': + '@typescript-eslint/eslint-plugin@8.0.0(@typescript-eslint/parser@8.0.0(eslint@9.8.0)(typescript@5.5.4))(eslint@9.8.0)(typescript@5.5.4)': dependencies: - '@eslint-community/regexpp': 4.10.1 - '@typescript-eslint/parser': 8.0.0-alpha.30(eslint@9.4.0)(typescript@5.4.5) - '@typescript-eslint/scope-manager': 8.0.0-alpha.30 - '@typescript-eslint/type-utils': 8.0.0-alpha.30(eslint@9.4.0)(typescript@5.4.5) - '@typescript-eslint/utils': 8.0.0-alpha.30(eslint@9.4.0)(typescript@5.4.5) - '@typescript-eslint/visitor-keys': 8.0.0-alpha.30 - eslint: 9.4.0 + '@eslint-community/regexpp': 4.11.0 + '@typescript-eslint/parser': 8.0.0(eslint@9.8.0)(typescript@5.5.4) + '@typescript-eslint/scope-manager': 8.0.0 + '@typescript-eslint/type-utils': 8.0.0(eslint@9.8.0)(typescript@5.5.4) + '@typescript-eslint/utils': 8.0.0(eslint@9.8.0)(typescript@5.5.4) + '@typescript-eslint/visitor-keys': 8.0.0 + eslint: 9.8.0 graphemer: 1.4.0 ignore: 5.3.1 natural-compare: 1.4.0 - ts-api-utils: 1.3.0(typescript@5.4.5) + ts-api-utils: 1.3.0(typescript@5.5.4) optionalDependencies: - typescript: 5.4.5 + typescript: 5.5.4 transitivePeerDependencies: - supports-color - '@typescript-eslint/parser@8.0.0-alpha.30(eslint@9.4.0)(typescript@5.4.5)': + '@typescript-eslint/parser@8.0.0(eslint@9.8.0)(typescript@5.5.4)': dependencies: - '@typescript-eslint/scope-manager': 8.0.0-alpha.30 - '@typescript-eslint/types': 8.0.0-alpha.30 - '@typescript-eslint/typescript-estree': 8.0.0-alpha.30(typescript@5.4.5) - '@typescript-eslint/visitor-keys': 8.0.0-alpha.30 - debug: 4.3.5 - eslint: 9.4.0 + '@typescript-eslint/scope-manager': 8.0.0 + '@typescript-eslint/types': 8.0.0 + '@typescript-eslint/typescript-estree': 8.0.0(typescript@5.5.4) + '@typescript-eslint/visitor-keys': 8.0.0 + debug: 4.3.6 + eslint: 9.8.0 optionalDependencies: - typescript: 5.4.5 + typescript: 5.5.4 transitivePeerDependencies: - supports-color - '@typescript-eslint/scope-manager@8.0.0-alpha.30': + '@typescript-eslint/scope-manager@8.0.0': dependencies: - '@typescript-eslint/types': 8.0.0-alpha.30 - '@typescript-eslint/visitor-keys': 8.0.0-alpha.30 + '@typescript-eslint/types': 8.0.0 + '@typescript-eslint/visitor-keys': 8.0.0 - '@typescript-eslint/type-utils@8.0.0-alpha.30(eslint@9.4.0)(typescript@5.4.5)': + '@typescript-eslint/type-utils@8.0.0(eslint@9.8.0)(typescript@5.5.4)': dependencies: - '@typescript-eslint/typescript-estree': 8.0.0-alpha.30(typescript@5.4.5) - '@typescript-eslint/utils': 8.0.0-alpha.30(eslint@9.4.0)(typescript@5.4.5) - debug: 4.3.5 - ts-api-utils: 1.3.0(typescript@5.4.5) + '@typescript-eslint/typescript-estree': 8.0.0(typescript@5.5.4) + '@typescript-eslint/utils': 8.0.0(eslint@9.8.0)(typescript@5.5.4) + debug: 4.3.6 + ts-api-utils: 1.3.0(typescript@5.5.4) optionalDependencies: - typescript: 5.4.5 + typescript: 5.5.4 transitivePeerDependencies: - eslint - supports-color - '@typescript-eslint/types@8.0.0-alpha.30': {} + '@typescript-eslint/types@8.0.0': {} - '@typescript-eslint/typescript-estree@8.0.0-alpha.30(typescript@5.4.5)': + '@typescript-eslint/typescript-estree@8.0.0(typescript@5.5.4)': dependencies: - '@typescript-eslint/types': 8.0.0-alpha.30 - '@typescript-eslint/visitor-keys': 8.0.0-alpha.30 - debug: 4.3.5 + '@typescript-eslint/types': 8.0.0 + '@typescript-eslint/visitor-keys': 8.0.0 + debug: 4.3.6 globby: 11.1.0 is-glob: 4.0.3 - minimatch: 9.0.4 - semver: 7.6.2 - ts-api-utils: 1.3.0(typescript@5.4.5) + minimatch: 9.0.5 + semver: 7.6.3 + ts-api-utils: 1.3.0(typescript@5.5.4) optionalDependencies: - typescript: 5.4.5 + typescript: 5.5.4 transitivePeerDependencies: - supports-color - '@typescript-eslint/utils@8.0.0-alpha.30(eslint@9.4.0)(typescript@5.4.5)': + '@typescript-eslint/utils@8.0.0(eslint@9.8.0)(typescript@5.5.4)': dependencies: - '@eslint-community/eslint-utils': 4.4.0(eslint@9.4.0) - '@typescript-eslint/scope-manager': 8.0.0-alpha.30 - '@typescript-eslint/types': 8.0.0-alpha.30 - '@typescript-eslint/typescript-estree': 8.0.0-alpha.30(typescript@5.4.5) - eslint: 9.4.0 + '@eslint-community/eslint-utils': 4.4.0(eslint@9.8.0) + '@typescript-eslint/scope-manager': 8.0.0 + '@typescript-eslint/types': 8.0.0 + '@typescript-eslint/typescript-estree': 8.0.0(typescript@5.5.4) + eslint: 9.8.0 transitivePeerDependencies: - supports-color - typescript - '@typescript-eslint/visitor-keys@8.0.0-alpha.30': + '@typescript-eslint/visitor-keys@8.0.0': dependencies: - '@typescript-eslint/types': 8.0.0-alpha.30 + '@typescript-eslint/types': 8.0.0 eslint-visitor-keys: 3.4.3 '@vercel/nft@0.26.5': dependencies: '@mapbox/node-pre-gyp': 1.0.11 '@rollup/pluginutils': 4.2.1 - acorn: 8.12.0 - acorn-import-attributes: 1.9.5(acorn@8.12.0) + acorn: 8.12.1 + acorn-import-attributes: 1.9.5(acorn@8.12.1) async-sema: 3.1.1 bindings: 1.5.0 estree-walker: 2.0.2 @@ -1952,23 +1943,23 @@ snapshots: abbrev@1.1.1: {} - acorn-import-attributes@1.9.5(acorn@8.12.0): + acorn-import-attributes@1.9.5(acorn@8.12.1): dependencies: - acorn: 8.12.0 + acorn: 8.12.1 - acorn-jsx@5.3.2(acorn@8.12.0): + acorn-jsx@5.3.2(acorn@8.12.1): dependencies: - acorn: 8.12.0 + acorn: 8.12.1 acorn-walk@8.3.3: dependencies: - acorn: 8.12.0 + acorn: 8.12.1 - acorn@8.12.0: {} + acorn@8.12.1: {} agent-base@6.0.2: dependencies: - debug: 4.3.5 + debug: 4.3.6 transitivePeerDependencies: - supports-color @@ -2017,12 +2008,12 @@ snapshots: ava@6.1.3: dependencies: '@vercel/nft': 0.26.5 - acorn: 8.12.0 + acorn: 8.12.1 acorn-walk: 8.3.3 ansi-styles: 6.2.1 arrgv: 1.0.2 arrify: 3.0.0 - callsites: 4.1.0 + callsites: 4.2.0 cbor: 9.0.2 chalk: 5.3.0 chunkd: 2.0.1 @@ -2033,10 +2024,10 @@ snapshots: common-path-prefix: 3.0.0 concordance: 5.0.4 currently-unhandled: 0.4.1 - debug: 4.3.5 + debug: 4.3.6 emittery: 1.0.3 figures: 6.1.0 - globby: 14.0.1 + globby: 14.0.2 ignore-by-default: 2.1.0 indent-string: 5.0.0 is-plain-object: 5.0.0 @@ -2048,7 +2039,7 @@ snapshots: package-config: 5.0.0 picomatch: 3.0.1 plur: 5.1.0 - pretty-ms: 9.0.0 + pretty-ms: 9.1.0 resolve-cwd: 3.0.0 stack-utils: 2.0.6 strip-ansi: 7.1.0 @@ -2111,7 +2102,7 @@ snapshots: callsites@3.1.0: {} - callsites@4.1.0: {} + callsites@4.2.0: {} cbor@9.0.2: dependencies: @@ -2154,7 +2145,7 @@ snapshots: cli-truncate@4.0.0: dependencies: slice-ansi: 5.0.0 - string-width: 7.1.0 + string-width: 7.2.0 cliui@8.0.1: dependencies: @@ -2190,7 +2181,7 @@ snapshots: js-string-escape: 1.0.1 lodash: 4.17.21 md5-hex: 3.0.1 - semver: 7.6.2 + semver: 7.6.3 well-known-symbols: 2.0.0 console-control-strings@1.1.0: {} @@ -2223,7 +2214,7 @@ snapshots: dependencies: time-zone: 1.0.0 - debug@4.3.5: + debug@4.3.6: dependencies: ms: 2.1.2 @@ -2311,7 +2302,7 @@ snapshots: escape-string-regexp@5.0.0: {} - eslint-scope@8.0.1: + eslint-scope@8.0.2: dependencies: esrecurse: 4.3.0 estraverse: 5.3.0 @@ -2320,25 +2311,25 @@ snapshots: eslint-visitor-keys@4.0.0: {} - eslint@9.4.0: + eslint@9.8.0: dependencies: - '@eslint-community/eslint-utils': 4.4.0(eslint@9.4.0) - '@eslint-community/regexpp': 4.10.1 - '@eslint/config-array': 0.15.1 + '@eslint-community/eslint-utils': 4.4.0(eslint@9.8.0) + '@eslint-community/regexpp': 4.11.0 + '@eslint/config-array': 0.17.1 '@eslint/eslintrc': 3.1.0 - '@eslint/js': 9.4.0 + '@eslint/js': 9.8.0 '@humanwhocodes/module-importer': 1.0.1 '@humanwhocodes/retry': 0.3.0 '@nodelib/fs.walk': 1.2.8 ajv: 6.12.6 chalk: 4.1.2 cross-spawn: 7.0.3 - debug: 4.3.5 + debug: 4.3.6 escape-string-regexp: 4.0.0 - eslint-scope: 8.0.1 + eslint-scope: 8.0.2 eslint-visitor-keys: 4.0.0 espree: 10.1.0 - esquery: 1.5.0 + esquery: 1.6.0 esutils: 2.0.3 fast-deep-equal: 3.1.3 file-entry-cache: 8.0.0 @@ -2361,13 +2352,13 @@ snapshots: espree@10.1.0: dependencies: - acorn: 8.12.0 - acorn-jsx: 5.3.2(acorn@8.12.0) + acorn: 8.12.1 + acorn-jsx: 5.3.2(acorn@8.12.1) eslint-visitor-keys: 4.0.0 esprima@4.0.1: {} - esquery@1.5.0: + esquery@1.6.0: dependencies: estraverse: 5.3.0 @@ -2467,14 +2458,12 @@ snapshots: get-east-asian-width@1.2.0: {} - get-stream@8.0.1: {} - get-stream@9.0.1: dependencies: '@sec-ant/readable-stream': 0.4.1 is-stream: 4.0.1 - get-tsconfig@4.7.5: + get-tsconfig@4.7.6: dependencies: resolve-pkg-maps: 1.0.0 @@ -2486,11 +2475,11 @@ snapshots: dependencies: is-glob: 4.0.3 - glob@10.4.2: + glob@10.4.5: dependencies: foreground-child: 3.2.1 - jackspeak: 3.4.0 - minimatch: 9.0.4 + jackspeak: 3.4.3 + minimatch: 9.0.5 minipass: 7.1.2 package-json-from-dist: 1.0.0 path-scurry: 1.11.1 @@ -2506,7 +2495,7 @@ snapshots: globals@14.0.0: {} - globals@15.6.0: {} + globals@15.8.0: {} globby@11.1.0: dependencies: @@ -2517,7 +2506,7 @@ snapshots: merge2: 1.4.1 slash: 3.0.0 - globby@14.0.1: + globby@14.0.2: dependencies: '@sindresorhus/merge-streams': 2.3.0 fast-glob: 3.3.2 @@ -2526,20 +2515,19 @@ snapshots: slash: 5.1.0 unicorn-magic: 0.1.0 - got@14.4.1: + got@14.4.2: dependencies: - '@sindresorhus/is': 6.3.1 + '@sindresorhus/is': 7.0.0 '@szmarczak/http-timer': 5.0.1 cacheable-lookup: 7.0.0 cacheable-request: 12.0.1 decompress-response: 6.0.0 form-data-encoder: 4.0.2 - get-stream: 8.0.1 http2-wrapper: 2.2.1 lowercase-keys: 3.0.0 p-cancelable: 4.0.1 responselike: 3.0.0 - type-fest: 4.20.1 + type-fest: 4.23.0 graceful-fs@4.2.11: {} @@ -2570,7 +2558,7 @@ snapshots: https-proxy-agent@5.0.1: dependencies: agent-base: 6.0.2 - debug: 4.3.5 + debug: 4.3.6 transitivePeerDependencies: - supports-color @@ -2637,7 +2625,7 @@ snapshots: html-escaper: 2.0.2 istanbul-lib-report: 3.0.1 - jackspeak@3.4.0: + jackspeak@3.4.3: dependencies: '@isaacs/cliui': 8.0.2 optionalDependencies: @@ -2687,7 +2675,7 @@ snapshots: lowercase-keys@3.0.0: {} - lru-cache@10.2.2: {} + lru-cache@10.4.3: {} make-dir@3.1.0: dependencies: @@ -2695,7 +2683,7 @@ snapshots: make-dir@4.0.0: dependencies: - semver: 7.6.2 + semver: 7.6.3 matcher@5.0.0: dependencies: @@ -2732,7 +2720,7 @@ snapshots: dependencies: brace-expansion: 1.1.11 - minimatch@9.0.4: + minimatch@9.0.5: dependencies: brace-expansion: 2.0.1 @@ -2767,7 +2755,7 @@ snapshots: nock@13.5.4: dependencies: - debug: 4.3.5 + debug: 4.3.6 json-stringify-safe: 5.0.1 propagate: 2.0.1 transitivePeerDependencies: @@ -2857,7 +2845,7 @@ snapshots: path-scurry@1.11.1: dependencies: - lru-cache: 10.2.2 + lru-cache: 10.4.3 minipass: 7.1.2 path-to-regexp@6.2.2: {} @@ -2876,7 +2864,7 @@ snapshots: prelude-ls@1.2.1: {} - pretty-ms@9.0.0: + pretty-ms@9.1.0: dependencies: parse-ms: 4.0.0 @@ -2938,7 +2926,7 @@ snapshots: semver@6.3.1: {} - semver@7.6.2: {} + semver@7.6.3: {} serialize-error@7.0.1: dependencies: @@ -2992,7 +2980,7 @@ snapshots: emoji-regex: 9.2.2 strip-ansi: 7.1.0 - string-width@7.1.0: + string-width@7.2.0: dependencies: emoji-regex: 10.3.0 get-east-asian-width: 1.2.0 @@ -3037,8 +3025,8 @@ snapshots: test-exclude@7.0.1: dependencies: '@istanbuljs/schema': 0.1.3 - glob: 10.4.2 - minimatch: 9.0.4 + glob: 10.4.5 + minimatch: 9.0.5 text-table@0.2.0: {} @@ -3057,16 +3045,16 @@ snapshots: tr46@0.0.3: {} - ts-api-utils@1.3.0(typescript@5.4.5): + ts-api-utils@1.3.0(typescript@5.5.4): dependencies: - typescript: 5.4.5 + typescript: 5.5.4 tslog@4.9.3: {} - tsx@4.15.7: + tsx@4.16.3: dependencies: esbuild: 0.21.5 - get-tsconfig: 4.7.5 + get-tsconfig: 4.7.6 optionalDependencies: fsevents: 2.3.3 @@ -3076,22 +3064,24 @@ snapshots: type-detect@4.0.8: {} + type-detect@4.1.0: {} + type-fest@0.13.1: {} - type-fest@4.20.1: {} + type-fest@4.23.0: {} - typescript-eslint@8.0.0-alpha.30(eslint@9.4.0)(typescript@5.4.5): + typescript-eslint@8.0.0(eslint@9.8.0)(typescript@5.5.4): dependencies: - '@typescript-eslint/eslint-plugin': 8.0.0-alpha.30(@typescript-eslint/parser@8.0.0-alpha.30(eslint@9.4.0)(typescript@5.4.5))(eslint@9.4.0)(typescript@5.4.5) - '@typescript-eslint/parser': 8.0.0-alpha.30(eslint@9.4.0)(typescript@5.4.5) - '@typescript-eslint/utils': 8.0.0-alpha.30(eslint@9.4.0)(typescript@5.4.5) + '@typescript-eslint/eslint-plugin': 8.0.0(@typescript-eslint/parser@8.0.0(eslint@9.8.0)(typescript@5.5.4))(eslint@9.8.0)(typescript@5.5.4) + '@typescript-eslint/parser': 8.0.0(eslint@9.8.0)(typescript@5.5.4) + '@typescript-eslint/utils': 8.0.0(eslint@9.8.0)(typescript@5.5.4) optionalDependencies: - typescript: 5.4.5 + typescript: 5.5.4 transitivePeerDependencies: - eslint - supports-color - typescript@5.4.5: {} + typescript@5.5.4: {} undici-types@5.26.5: {} diff --git a/src/crawler.ts b/src/crawler.ts index 84bf0ca..2ad3bd6 100644 --- a/src/crawler.ts +++ b/src/crawler.ts @@ -2,18 +2,15 @@ import { EventEmitter } from "events"; import { Cluster } from "./rateLimiter/index.js"; import { isBoolean, isFunction, setDefaults, flattenDeep, lowerObjectKeys, isNumber } from "./lib/utils.js"; import { getValidOptions, alignOptions, getCharset } from "./options.js"; -import { logOptions } from "./logger.js"; +import { getLogger } from "./logger.js"; import type { CrawlerOptions, RequestOptions, RequestConfig, CrawlerResponse } from "./types/crawler.js"; import { load } from "cheerio"; import got from "got"; import seenreq from "seenreq"; import iconv from "iconv-lite"; -import { Logger } from "tslog"; // @todo: remove seenreq dependency -// process.env.NODE_ENV = process.env.NODE_ENV ?? process.argv[2]; -logOptions.minLevel = process.env.NODE_ENV === "debug" ? 0 : process.env.NODE_ENV === "test" ? 7 : 3; -const log = new Logger(logOptions); +const log = getLogger(); class Crawler extends EventEmitter { private _limiters: Cluster; @@ -36,8 +33,8 @@ class Crawler extends EventEmitter { jQuery: true, priority: 5, retries: 2, - retryInterval: 2000, - timeout: 15000, + retryInterval: 3000, + timeout: 20000, isJson: false, silence: false, }; diff --git a/src/lib/utils.ts b/src/lib/utils.ts index ef81180..5bd322b 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -44,8 +44,8 @@ export const isValidUrl = (url: string): boolean => { * Flattens an array of arrays recursively. * */ -export function flattenDeep(array: any[]): any[] { - const result: any[] = []; +export function flattenDeep(array: (T | T[])[]): T[] { + const result: T[] = []; array.forEach(element => { if (Array.isArray(element)) { result.push(...flattenDeep(element)); @@ -56,8 +56,8 @@ export function flattenDeep(array: any[]): any[] { return result; } -export function pick(target: T, keys: (keyof T)[]): Pick { - const result: any = {}; +export function pick(target: T, keys: K[]): Pick { + const result = {} as Pick; keys.forEach(key => { if (target[key] !== undefined) { result[key] = target[key]; diff --git a/src/logger.ts b/src/logger.ts index 0b31075..fe57f59 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -1,3 +1,11 @@ +import { Logger } from "tslog"; + +const logLevelsByEnv: Record = { + "debug": 0, + "production": 3, + "test": 7, +}; + export const logOptions = { type: "pretty" as "json" | "pretty" | "hidden", name: "Crawler", @@ -23,3 +31,7 @@ export const logOptions = { }, minLevel: 0, }; + +logOptions.minLevel = process.env.NODE_ENV ? logLevelsByEnv[process.env.NODE_ENV] : 3; + +export const getLogger = () => new Logger(logOptions); \ No newline at end of file diff --git a/src/options.ts b/src/options.ts index dea0189..7161ed7 100644 --- a/src/options.ts +++ b/src/options.ts @@ -1,3 +1,4 @@ +import { GotUrl } from "got"; import { HttpProxyAgent, HttpsProxyAgent } from "hpagent"; import http2Wrapper from "http2-wrapper"; import { cleanObject, getType, isValidUrl } from "./lib/utils.js"; @@ -5,8 +6,8 @@ import { RequestConfig, RequestOptions } from "./types/crawler.js"; export const globalOnlyOptions = [ "maxConnections", - "priorityLevels", "rateLimit", + "priorityLevels", "skipDuplicates", "homogeneous", "userAgents", @@ -69,7 +70,7 @@ export const getValidOptions = (options: RequestConfig): RequestOptions => { throw new TypeError(`Invalid options: ${JSON.stringify(options)}`); }; -export const alignOptions = (options: RequestOptions): any => { +export const alignOptions = (options: RequestOptions): GotUrl => { const gotOptions = { ...options, url: options.url ?? options.uri, diff --git a/src/types/crawler.ts b/src/types/crawler.ts index bbafb47..ee960f9 100644 --- a/src/types/crawler.ts +++ b/src/types/crawler.ts @@ -72,8 +72,20 @@ export type RequestOptions = { * @description rateLimiter ID */ rateLimiterId?: number; + /** + * @default 2 + * @description The retry count of the request. + */ retries?: number; + /** + * @default 3000 + * @description The interval between retries in milliseconds. + */ retryInterval?: number; + /** + * @default 20000 + * @description The global timeout of the request in milliseconds. + */ timeout?: number; priority?: number; seenreq?: any; @@ -91,8 +103,8 @@ export type RequestOptions = { /** * @deprecated Please use "url" instead. */ - uri?: string | Function; - url?: string | Function; + uri?: string | ((urlFn: (url: string) => void) => void); + url?: string | ((urlFn: (url: string) => void) => void); /** * @deprecated Please use "searchParams" instead. @@ -124,8 +136,10 @@ export type RequestOptions = { cookieJar?: object; /** - * @description If true, the crawler will parse the response body as JSON. * @default false + * + * If true, the crawler will parse the response body as JSON. + * This will set 'jQuery' to false. */ isJson?: boolean; @@ -134,14 +148,14 @@ export type RequestOptions = { /** * @deprecated Please use "parseJson" instead. */ - jsonReviver?: Function; - parseJson?: Function; + jsonReviver?: (text: string) => unknown; + parseJson?: (text: string) => unknown; /** * @deprecated Please use "stringifyJson" instead. */ - jsonReplacer?: Function; - stringifyJson?: Function; + jsonReplacer?: (object: unknown) => string; + stringifyJson?: (object: unknown) => string; preRequest?: (options: RequestOptions, done?: (error?: Error | null) => void) => void; release?: () => void; @@ -150,4 +164,4 @@ export type RequestOptions = { export type RequestConfig = string | RequestOptions | RequestOptions[]; export type CrawlerOptions = Partial & RequestOptions; -export type CrawlerResponse = any +export type CrawlerResponse = any; diff --git a/test/cacheOptions.js b/test/cacheOptions.js index 45765fa..21101f8 100644 --- a/test/cacheOptions.js +++ b/test/cacheOptions.js @@ -1,5 +1,5 @@ import test from "ava"; -import { testCb, testCbSync } from "./lib/avaTestCb.js"; +import { testCb } from "./lib/avaTestCb.js"; import nock from "nock"; import Crawler from "../dist/index.js"; @@ -10,7 +10,7 @@ test.afterEach(t => { t.context.c = {}; }); -testCbSync(test, "Should't skip one single url if duplicates are active.", async t => { +testCb(test, "Should't skip one single url if duplicates are active.", async t => { t.context.scope.get("/").reply(200); t.context.c = new Crawler({ // silence: true, @@ -25,7 +25,7 @@ testCbSync(test, "Should't skip one single url if duplicates are active.", async t.context.c.add("http://target.com"); }); -testCbSync(test, "Should notify the callback when an error occurs and 'retries' is disabled.", async t => { +testCb(test, "Should notify the callback when an error occurs and 'retries' is disabled.", async t => { t.context.scope.get("/").replyWithError("Bad request."); t.context.c = new Crawler({ // silence: true, @@ -41,7 +41,7 @@ testCbSync(test, "Should notify the callback when an error occurs and 'retries' t.context.c.add("http://target.com"); }); -testCbSync(test, "Should retry and notify the callback when an error occurs and 'retries' is enabled.", async t => { +testCb(test, "Should retry and notify the callback when an error occurs and 'retries' is enabled.", async t => { t.context.scope.get("/").replyWithError("Bad request.").persist(); t.context.c = new Crawler({ jQuery: false, @@ -58,7 +58,7 @@ testCbSync(test, "Should retry and notify the callback when an error occurs and t.context.c.add("http://target.com"); }); -testCbSync(test, "Should skip previously crawled urls when 'skipDuplicates' is active.", async t => { +testCb(test, "Should skip previously crawled urls when 'skipDuplicates' is active.", async t => { t.context.scope.get("/").reply(200).persist(); t.plan(3); t.context.c = new Crawler({ diff --git a/test/callback.js b/test/callback.js index dbad52a..cb40899 100644 --- a/test/callback.js +++ b/test/callback.js @@ -1,5 +1,5 @@ import test from "ava"; -import { testCb, testCbSync } from "./lib/avaTestCb.js"; +import { testCb } from "./lib/avaTestCb.js"; import nock from "nock"; import Crawler from "../dist/index.js"; @@ -19,7 +19,7 @@ test.afterEach(t => { t.context.crawler = null; }); -testCbSync(test, "should end as expected without callback", async t => { +testCb(test, "should end as expected without callback", async t => { t.context.scope = nock(url).get("/get").reply(200, "", { "Content-Type": "text/html", }); @@ -30,7 +30,7 @@ testCbSync(test, "should end as expected without callback", async t => { t.context.crawler.add(`${url}/get`); }); -testCbSync(test, "should end as expected without callback when timedout", async t => { +testCb(test, "should end as expected without callback when timedout", async t => { t.context.scope = nock(url).get("/delay").delayBody(500).reply(200, "", { "Content-Type": "text/html", }); diff --git a/test/cookieJar.js b/test/cookieJar.js index 952b1f1..7a5fa3b 100644 --- a/test/cookieJar.js +++ b/test/cookieJar.js @@ -1,5 +1,5 @@ import test from "ava"; -import { testCb, testCbSync } from "./lib/avaTestCb.js"; +import { testCb } from "./lib/avaTestCb.js"; import nock from "nock"; import Crawler from "../dist/index.js"; import { CookieJar } from "tough-cookie"; @@ -29,7 +29,7 @@ test.before(t => { }); }); -testCbSync(test, "should send with cookie when setting jar options", async t => { +testCb(test, "should send with cookie when setting jar options", async t => { t.context.crawler.add({ url: "http://test.crawler.com/getCookie", callback: (error, response, done) => { @@ -41,7 +41,7 @@ testCbSync(test, "should send with cookie when setting jar options", async t => }); }); -testCbSync(test, "should set cookie when response set-cookie headers exist", async t => { +testCb(test, "should set cookie when response set-cookie headers exist", async t => { t.context.crawler.add({ url: "http://test.crawler.com/setCookie", callback: (error, response, done) => { diff --git a/test/direct.js b/test/direct.js index 9cc0e3b..2712bcd 100644 --- a/test/direct.js +++ b/test/direct.js @@ -1,5 +1,5 @@ import test from "ava"; -import { testCb, testCbSync } from "./lib/avaTestCb.js"; +import { testCb } from "./lib/avaTestCb.js"; import nock from "nock"; import Crawler from "../dist/index.js"; import sinon from "sinon"; @@ -35,7 +35,7 @@ test.afterEach(t => { t.context.crawler = null; }); -testCbSync(test, "should not trigger preRequest or callback of crawler instance", async t => { +testCb(test, "should not trigger preRequest or callback of crawler instance", async t => { t.context.crawler.send({ url: "http://test.crawler.com/", callback: (error, res) => { @@ -48,7 +48,7 @@ testCbSync(test, "should not trigger preRequest or callback of crawler instance" }); }); -testCbSync(test, "should be sent directly regardless of current queue of crawler", async t => { +testCb(test, "should be sent directly regardless of current queue of crawler", async t => { t.context.crawler.add({ url: "http://test.crawler.com/", callback: (error, res, done) => { @@ -91,7 +91,7 @@ testCbSync(test, "should be sent directly regardless of current queue of crawler }); }); -testCbSync(test, "should not trigger Event:request by default.", async t => { +testCb(test, "should not trigger Event:request by default.", async t => { t.context.crawler.send({ url: "http://test.crawler.com/", callback: (error, res) => { @@ -104,7 +104,7 @@ testCbSync(test, "should not trigger Event:request by default.", async t => { }); }); -testCbSync(test, "should trigger Event:request if set.", async t => { +testCb(test, "should trigger Event:request if set.", async t => { t.context.crawler.send({ url: "http://test.crawler.com/", skipEventRequest: false, diff --git a/test/encoding.js b/test/encoding.js index cb699d8..e3add29 100644 --- a/test/encoding.js +++ b/test/encoding.js @@ -1,5 +1,5 @@ import test from "ava"; -import { testCb, testCbSync } from "./lib/avaTestCb.js"; +import { testCb } from "./lib/avaTestCb.js"; import nock from "nock"; import Crawler from "../dist/index.js"; @@ -26,7 +26,7 @@ test.afterEach(t => { t.context.crawler = null; }); -testCbSync(test, "should parse latin-1", async t => { +testCb(test, "should parse latin-1", async t => { t.context.crawler.add({ url, callback: (error, result) => { @@ -38,7 +38,7 @@ testCbSync(test, "should parse latin-1", async t => { }); }); -testCbSync(test, "should return buffer if encoding = null", async t => { +testCb(test, "should return buffer if encoding = null", async t => { t.context.crawler.add({ url, encoding: null, @@ -50,7 +50,7 @@ testCbSync(test, "should return buffer if encoding = null", async t => { }); }); -testCbSync(test, "should parse latin-1 if encoding = ISO-8859-1", async t => { +testCb(test, "should parse latin-1 if encoding = ISO-8859-1", async t => { t.context.crawler.add({ url, encoding: charsetName, @@ -63,7 +63,7 @@ testCbSync(test, "should parse latin-1 if encoding = ISO-8859-1", async t => { }); }); -testCbSync(test, "could not parse latin-1 if encoding = gb2312", async t => { +testCb(test, "could not parse latin-1 if encoding = gb2312", async t => { t.context.crawler.add({ url, encoding: "gb2312", @@ -75,7 +75,7 @@ testCbSync(test, "could not parse latin-1 if encoding = gb2312", async t => { }); }); -testCbSync(test, "should parse charset from header", async t => { +testCb(test, "should parse charset from header", async t => { t.context.crawler.add({ url, callback: (error, result) => { @@ -87,7 +87,7 @@ testCbSync(test, "should parse charset from header", async t => { }); }); -testCbSync(test, "should parse charset from meta tag in html if header does not contain content-type key", async t => { +testCb(test, "should parse charset from meta tag in html if header does not contain content-type key", async t => { t.context.crawler.add({ url: urlWithoutCharsetHeader, callback: (error, result) => { diff --git a/test/errorHandling.js b/test/errorHandling.js index 46d9c1a..4a989af 100644 --- a/test/errorHandling.js +++ b/test/errorHandling.js @@ -1,5 +1,5 @@ import test from "ava"; -import { testCb, testCbSync } from "./lib/avaTestCb.js"; +import { testCb } from "./lib/avaTestCb.js"; import nock from "nock"; import Crawler from "../dist/index.js"; @@ -26,7 +26,7 @@ test.afterEach(t => { t.context.crawler = null; }); -testCbSync(test, "should retry after timeout", async t => { +testCb(test, "should retry after timeout", async t => { let options = { url: "http://test.crawler.com/delay/1", callback: (error, response, done) => { @@ -39,7 +39,7 @@ testCbSync(test, "should retry after timeout", async t => { t.is(options.retries, 2); }); -testCbSync(test, "should return a timeout error after ~2sec", async t => { +testCb(test, "should return a timeout error after ~2sec", async t => { t.context.crawler.add({ url: "http://test.crawler.com/delay/1", callback: (error, response, done) => { @@ -50,7 +50,7 @@ testCbSync(test, "should return a timeout error after ~2sec", async t => { }); }); -testCbSync(test, "should not failed on empty response", async t => { +testCb(test, "should not failed on empty response", async t => { t.context.crawler.add({ url: "http://test.crawler.com/status/204", callback: (error, response, done) => { @@ -61,7 +61,7 @@ testCbSync(test, "should not failed on empty response", async t => { }); }); -testCbSync(test, "should not failed on a malformed html if jQuery is false", async t => { +testCb(test, "should not failed on a malformed html if jQuery is false", async t => { t.context.crawler.add({ html: "

hello

dude

", callback: (error, response, done) => { diff --git a/test/examples.js b/test/examples.js index 79bc897..dc526c6 100644 --- a/test/examples.js +++ b/test/examples.js @@ -1,5 +1,5 @@ import test from "ava"; -import { testCb, testCbSync } from "./lib/avaTestCb.js"; +import { testCb } from "./lib/avaTestCb.js"; import nock from "nock"; import Crawler from "../dist/index.js"; import sinon from "sinon"; diff --git a/test/http2ErrorHanding.js b/test/http2ErrorHanding.js index 10dbe53..590b4a2 100644 --- a/test/http2ErrorHanding.js +++ b/test/http2ErrorHanding.js @@ -1,5 +1,5 @@ import test from "ava"; -import { testCb, testCbSync } from "./lib/avaTestCb.js"; +import { testCb } from "./lib/avaTestCb.js"; import Crawler from "../dist/index.js"; test.before(t => { @@ -13,7 +13,7 @@ test.before(t => { }); }); -testCbSync(test, "http2: should retry after timeout.", async t => { +testCb(test, "http2: should retry after timeout.", async t => { const options = { url: "https://nghttp2.org/httpbin/delay/4", callback: (error, response, done) => { @@ -27,7 +27,7 @@ testCbSync(test, "http2: should retry after timeout.", async t => { t.is(options.retries, 2); }); -testCbSync(test, "http2: should return a timeout error after ~3sec.", async t => { +testCb(test, "http2: should return a timeout error after ~3sec.", async t => { t.context.crawler.add({ url: "https://nghttp2.org/httpbin/delay/4", callback: (error, response, done) => { diff --git a/test/http2Response.js b/test/http2Response.js index eb88f37..3c68cb1 100644 --- a/test/http2Response.js +++ b/test/http2Response.js @@ -1,13 +1,12 @@ import test from "ava"; -import { testCb, testCbSync } from "./lib/avaTestCb.js"; +import { testCb } from "./lib/avaTestCb.js"; import Crawler from "../dist/index.js"; -import fs from "fs"; test.afterEach(t => { t.context.crawler = null; }); -testCbSync(test, "response statusCode.", async t => { +testCb(test, "response statusCode.", async t => { t.context.crawler = new Crawler({ // silence: true, timeout: 10000, @@ -26,7 +25,7 @@ testCbSync(test, "response statusCode.", async t => { }); }); -testCbSync(test, "response headers.", async t => { +testCb(test, "response headers.", async t => { t.context.crawler = new Crawler({ // silence: true, retryInterval: 1000, @@ -46,7 +45,7 @@ testCbSync(test, "response headers.", async t => { }); }); -testCbSync(test, "html response body.", async t => { +testCb(test, "html response body.", async t => { t.context.crawler = new Crawler({ // silence: true, retryInterval: 1000, diff --git a/test/lib/avaTestCb.js b/test/lib/avaTestCb.js index 5ec5180..fee7008 100644 --- a/test/lib/avaTestCb.js +++ b/test/lib/avaTestCb.js @@ -1,4 +1,4 @@ -export const testCb = (test, description, assertions) => { +export const testCbAsync = (test, description, assertions) => { test(description, async t => { await new Promise(resolve => { // eslint-disable-next-linse @typescript-eslint/no-explicit-any @@ -22,3 +22,6 @@ export const testCbSync = (test, description, assertions) => { }); }); }; + +// @todo: add async test version +export const testCb = testCbSync; diff --git a/test/limiter.js b/test/limiter.js index 07d4ca9..5163c65 100644 --- a/test/limiter.js +++ b/test/limiter.js @@ -1,5 +1,5 @@ import test from "ava"; -import { testCb, testCbSync } from "./lib/avaTestCb.js"; +import { testCb } from "./lib/avaTestCb.js"; import nock from "nock"; import Crawler from "../dist/index.js"; @@ -32,7 +32,7 @@ test.afterEach(t => { t.context.tsArrs = []; }); -testCbSync(test, "One limiter, tasks should execute one by one", async t => { +testCb(test, "One limiter, tasks should execute one by one", async t => { for (let i = 0; i < 5; i++) { t.context.crawler.add({ url: "http://nockHost/status/200" }); } @@ -46,7 +46,7 @@ testCbSync(test, "One limiter, tasks should execute one by one", async t => { }); }); -testCbSync(test, "Multiple limiters, tasks should execute in parallel", async t => { +testCb(test, "Multiple limiters, tasks should execute in parallel", async t => { for (let i = 0; i < 5; i++) { t.context.crawler.add({ url: "http://nockHost/status/200", rateLimiterId: i }); } @@ -59,7 +59,7 @@ testCbSync(test, "Multiple limiters, tasks should execute in parallel", async t }); }); -testCbSync(test, "Multiple limiters are mutual independent", async t => { +testCb(test, "Multiple limiters are mutual independent", async t => { for (let i = 0; i < 5; i++) { const limiter = i === 4 ? "second" : "default"; t.context.crawler.add({ url: "http://nockHost/status/200", rateLimiterId: limiter }); @@ -74,7 +74,7 @@ testCbSync(test, "Multiple limiters are mutual independent", async t => { }); }); -testCbSync(test, "should modify maxConnections when rateLimit is set", async t => { +testCb(test, "should modify maxConnections when rateLimit is set", async t => { nock.cleanAll(); nock("http://nockHost").get(url => url.indexOf("status") >= 0).times(1).reply(200, "Yes"); t.context.crawler.add({ diff --git a/test/preRequest.js b/test/preRequest.js index 5ebf8c6..cf96fe8 100644 --- a/test/preRequest.js +++ b/test/preRequest.js @@ -1,5 +1,5 @@ import test from "ava"; -import { testCb, testCbSync } from "./lib/avaTestCb.js"; +import { testCb } from "./lib/avaTestCb.js"; import nock from "nock"; import Crawler from "../dist/index.js"; import sinon from "sinon"; @@ -12,7 +12,7 @@ test.beforeEach(t => { t.context.cb = sinon.spy(); }); -testCbSync(test, "Should do preRequest before request when preRequest defined in crawler options.", async t => { +testCb(test, "Should do preRequest before request when preRequest defined in crawler options.", async t => { t.context.crawler = new Crawler({ // silence: true, jQuery: false, @@ -35,7 +35,7 @@ testCbSync(test, "Should do preRequest before request when preRequest defined in }); }); -testCbSync(test, "Should do preRequest before request when preRequest defined in add options.", async t => { +testCb(test, "Should do preRequest before request when preRequest defined in add options.", async t => { t.context.crawler = new Crawler({ // silence: true, jQuery: false @@ -58,7 +58,7 @@ testCbSync(test, "Should do preRequest before request when preRequest defined in }); }); -testCbSync(test, "preRequest should be executed the same times as request.", async t => { +testCb(test, "preRequest should be executed the same times as request.", async t => { t.context.crawler = new Crawler({ // silence: true, jQuery: false, @@ -94,7 +94,7 @@ testCbSync(test, "preRequest should be executed the same times as request.", asy }); }); -testCbSync(test, "when preRequest fail, should retry two times by default.", async t => { +testCb(test, "when preRequest fail, should retry two times by default.", async t => { t.context.crawler = new Crawler({ // silence: true, jQuery: false, diff --git a/test/priority.js b/test/priority.js index db7c5b1..ad838fc 100644 --- a/test/priority.js +++ b/test/priority.js @@ -1,5 +1,5 @@ import test from "ava"; -import { testCb, testCbSync } from "./lib/avaTestCb.js"; +import { testCb } from "./lib/avaTestCb.js"; import nock from "nock"; import Crawler from "../dist/index.js"; @@ -9,7 +9,7 @@ test.before(t => { t.context.crawler = new Crawler({ jQuery: false, maxConnections: 1 }); }); -testCbSync(test, "should execute requests in the correct order", async t => { +testCb(test, "should execute requests in the correct order", async t => { t.context.spf = []; let cnt = 0; t.context.crawler.add([{ diff --git a/test/rateLimit.js b/test/rateLimit.js index 590336d..dda8abe 100644 --- a/test/rateLimit.js +++ b/test/rateLimit.js @@ -1,5 +1,5 @@ import test from "ava"; -import { testCb, testCbSync } from "./lib/avaTestCb.js"; +import { testCb } from "./lib/avaTestCb.js"; import nock from "nock"; import Crawler from "../dist/index.js"; @@ -26,7 +26,7 @@ test.afterEach(t => { t.context.tsArrs = []; }); -testCbSync(test, "Interval of two requests should be no less than 500ms", async t => { +testCb(test, "Interval of two requests should be no less than 500ms", async t => { nock('http://nockHost').get(url => url.includes('status')).times(2).delay(500).reply(200, 'Yes'); t.context.c.add({ url: 'http://nockHost/status/200' }); t.context.c.add({ @@ -43,7 +43,7 @@ testCbSync(test, "Interval of two requests should be no less than 500ms", async t.context.c.on("drain", t.end); }); -testCbSync(test, "request speed should abide by rateLimit", async t => { +testCb(test, "request speed should abide by rateLimit", async t => { nock('http://nockHost').get(url => url.includes('status')).times(5).reply(200, 'Yes'); for (let i = 0; i < 5; i++) { t.context.c.add('http://nockHost/status/200'); @@ -58,7 +58,7 @@ testCbSync(test, "request speed should abide by rateLimit", async t => { }); }); -testCbSync(test, "should be able to change rateLimit", async t => { +testCb(test, "should be able to change rateLimit", async t => { nock('http://nockHost').get(url => url.includes('status')).times(5).reply(200, 'Yes'); t.context.c.setLimiter(0, 'rateLimit', 300); for (let i = 0; i < 5; i++) { diff --git a/test/requests.js b/test/requests.js index 8144a22..82936f7 100644 --- a/test/requests.js +++ b/test/requests.js @@ -1,5 +1,5 @@ import test from "ava"; -import { testCb, testCbSync } from "./lib/avaTestCb.js"; +import { testCb } from "./lib/avaTestCb.js"; import nock from "nock"; import Crawler from "../dist/index.js"; @@ -61,7 +61,7 @@ testCb(test, "should crawl two request request and emit the drain event.", async }); }); -testCbSync(test, "should use the provided user-agent", async t => { +testCb(test, "should use the provided user-agent", async t => { const userAgent = 'test/1.2'; t.context.crawler.add({ url: `${origin}${path}`, @@ -76,7 +76,7 @@ testCbSync(test, "should use the provided user-agent", async t => { }); }); -testCbSync(test, "should replace the global default user-agent", async t => { +testCb(test, "should replace the global default user-agent", async t => { t.context.crawler = new Crawler({ // silence: true, isJson: true, @@ -94,7 +94,7 @@ testCbSync(test, "should replace the global default user-agent", async t => { }); }); -testCbSync(test, "should spoof the referrer", async t => { +testCb(test, "should spoof the referrer", async t => { const referer = 'http://spoofed.com'; t.context.crawler.add({ url: `${origin}${path}`, diff --git a/test/urlOptions.js b/test/urlOptions.js index 709009b..0e8e414 100644 --- a/test/urlOptions.js +++ b/test/urlOptions.js @@ -1,5 +1,5 @@ import test from "ava"; -import { testCb, testCbSync } from "./lib/avaTestCb.js"; +import { testCb } from "./lib/avaTestCb.js"; import nock from "nock"; import Crawler from "../dist/index.js"; import sinon from "sinon"; @@ -13,7 +13,7 @@ test.before(t => { }); }); -testCbSync(test, "should work if url is string", t => { +testCb(test, "should work if url is string", t => { t.context.crawler.add({ url: 'http://test.crawler.com/', callback: (error, response, done) => { @@ -24,7 +24,7 @@ testCbSync(test, "should work if url is string", t => { }); }); -testCbSync(test, "should work if url is a function", t => { +testCb(test, "should work if url is a function", t => { function urlFn(onUrl) { onUrl('http://test.crawler.com/'); } @@ -38,7 +38,7 @@ testCbSync(test, "should work if url is a function", t => { }); }); -testCbSync(test, "should skip if the url is undefined or an empty string", t => { +testCb(test, "should skip if the url is undefined or an empty string", t => { const push = sinon.spy(t.context.crawler, '_schedule'); t.context.crawler.add([undefined, null, []]); t.context.crawler.add({ diff --git a/test/userAgent.js b/test/userAgent.js index 338ec0f..6d760d5 100644 --- a/test/userAgent.js +++ b/test/userAgent.js @@ -1,5 +1,5 @@ import test from "ava"; -import { testCb, testCbSync } from "./lib/avaTestCb.js"; +import { testCb } from "./lib/avaTestCb.js"; import nock from "nock"; import Crawler from "../dist/index.js"; @@ -23,7 +23,7 @@ test.before(t => { }); }); -testCbSync(test, "should rotate user agents if userAgents is set.", async t => { +testCb(test, "should rotate user agents if userAgents is set.", async t => { t.context.crawler.add([ "http://nockhost/status1", "http://nockhost/status2",