From a24d1c577239a77c038e1f6763de32d033fa731a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Tue, 16 Jun 2026 14:33:44 +0200 Subject: [PATCH 1/2] feat: implement crawlee v4 RequestQueueClient over apify-client MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Crawlee v4 (apify/crawlee#3729) redesigned `RequestQueueClient` into a stateful, pull-based interface (`addBatchOfRequests`, `fetchNextRequest`, `markRequestAsHandled`, `reclaimRequest`, `isEmpty`, `isFinished`) modeled on the Python SDK — replacing the old thin REST wrapper. The old name-remapping proxy can't satisfy it, so this adds a real `ApifyRequestQueueClient` implementing the interface on apify-client's REST API: - `fetchNextRequest` locks the head server-side (`listAndLockHead`) so the queue is safe to share across consumers; `markRequestAsHandled` / `reclaimRequest` update the request and release the lock. - request ids are derived from the unique key the same way as the platform. - `getMetadata`/`drop`, no-op `purge`. Also adds `ApifyStorageClient.setStatusMessage` (Crawlee v4 calls it on the storage client to surface crawl progress) and bumps `@crawlee/*` to `^4.0.0-beta.66`. Validated end-to-end on the Apify platform: a CheerioCrawler run through Apify Proxy completed 29/29 requests driven entirely by the new request-queue client. Note: published `@crawlee/types@beta.66` still ships the pre-redesign interface in its `.d.ts` even though `@crawlee/core`'s runtime calls the new methods, so the client implements a local copy of the new interface (`RequestQueueClientV4`) until crawlee republishes regenerated types. --- package.json | 10 +- pnpm-lock.yaml | 244 +++++++++++++++--------------- src/actor.ts | 3 - src/apify_request_queue_client.ts | 209 +++++++++++++++++++++++++ src/apify_storage_client.ts | 21 ++- test/apify/actor.test.ts | 10 +- 6 files changed, 358 insertions(+), 139 deletions(-) create mode 100644 src/apify_request_queue_client.ts diff --git a/package.json b/package.json index d92aee24b0..996e0363ab 100644 --- a/package.json +++ b/package.json @@ -77,9 +77,9 @@ "@apify/log": "^2.4.3", "@apify/timeout": "^0.3.0", "@apify/utilities": "^2.13.0", - "@crawlee/core": "^4.0.0-beta.61", - "@crawlee/types": "^4.0.0-beta.61", - "@crawlee/utils": "^4.0.0-beta.61", + "@crawlee/core": "^4.0.0-beta.66", + "@crawlee/types": "^4.0.0-beta.66", + "@crawlee/utils": "^4.0.0-beta.66", "apify-client": "^2.23.4", "semver": "^7.5.4", "tslib": "^2.6.2", @@ -90,7 +90,7 @@ "@apify/oxlint-config": "^0.2.5", "@apify/tsconfig": "^0.1.2", "@commitlint/config-conventional": "^21.0.0", - "@crawlee/memory-storage": "^4.0.0-beta.61", + "@crawlee/memory-storage": "^4.0.0-beta.66", "@playwright/browser-chromium": "^1.60.0", "@types/content-type": "^1.1.8", "@types/node": "^24.0.0", @@ -98,7 +98,7 @@ "@types/tough-cookie": "^4.0.5", "@types/ws": "^8.5.12", "commitlint": "^21.0.0", - "crawlee": "^4.0.0-beta.61", + "crawlee": "^4.0.0-beta.66", "globby": "^16.0.0", "husky": "^9.1.7", "lint-staged": "^17.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a188cbcb5c..7b3eaf6781 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -29,14 +29,14 @@ importers: specifier: ^2.13.0 version: 2.26.0 '@crawlee/core': - specifier: ^4.0.0-beta.61 - version: 4.0.0-beta.61 + specifier: ^4.0.0-beta.66 + version: 4.0.0-beta.66 '@crawlee/types': - specifier: ^4.0.0-beta.61 - version: 4.0.0-beta.61 + specifier: ^4.0.0-beta.66 + version: 4.0.0-beta.66 '@crawlee/utils': - specifier: ^4.0.0-beta.61 - version: 4.0.0-beta.61 + specifier: ^4.0.0-beta.66 + version: 4.0.0-beta.66 apify-client: specifier: ^2.23.4 version: 2.23.4 @@ -63,8 +63,8 @@ importers: specifier: ^21.0.0 version: 21.0.1 '@crawlee/memory-storage': - specifier: ^4.0.0-beta.61 - version: 4.0.0-beta.61 + specifier: ^4.0.0-beta.66 + version: 4.0.0-beta.66 '@playwright/browser-chromium': specifier: ^1.60.0 version: 1.60.0 @@ -87,8 +87,8 @@ importers: specifier: ^21.0.0 version: 21.0.1(@types/node@24.12.2)(conventional-commits-parser@6.4.0)(typescript@6.0.3) crawlee: - specifier: ^4.0.0-beta.61 - version: 4.0.0-beta.61(@types/node@24.12.2)(playwright@1.60.0)(puppeteer@25.1.0) + specifier: ^4.0.0-beta.66 + version: 4.0.0-beta.66(@types/node@24.12.2)(playwright@1.60.0)(puppeteer@25.1.0) globby: specifier: ^16.0.0 version: 16.2.0 @@ -1076,12 +1076,12 @@ packages: conventional-commits-parser: optional: true - '@crawlee/basic@4.0.0-beta.61': - resolution: {integrity: sha512-HZCbqAxv6h6AUNTkPMYUuMKMyKKC7vk6Or4p4GCj+eQvqUcWTLwLt6/jdr0LmG+yiepQ4dfK4hzCsexhDicvSQ==} + '@crawlee/basic@4.0.0-beta.66': + resolution: {integrity: sha512-8N3YelWVtZ3UrOZo3FryvKhFu/2s7TK4WaY/49DabO9Y27E+IkHPxxCyji1vaBt5a5BAnpmNRf17wXenPIGgIA==} engines: {node: '>=22.0.0'} - '@crawlee/browser-pool@4.0.0-beta.61': - resolution: {integrity: sha512-tCV94HnI12TV4+94nqtNDlkoSSVmp4y0l9Z18fkTeVuOOsqdx3U3E9oSc0H1j/mfK8bkWcggDbcaxj4pVmGjxQ==} + '@crawlee/browser-pool@4.0.0-beta.66': + resolution: {integrity: sha512-YyBm3oszlQqoPiSGaOHtgCR8i6xLdZmNcmAqMbi3CnFSpW3+VufNE57MadHeJaqAbQBIj0ZmX0Ng6vXBNhTSGg==} engines: {node: '>=22.0.0'} peerDependencies: playwright: '*' @@ -1092,8 +1092,8 @@ packages: puppeteer: optional: true - '@crawlee/browser@4.0.0-beta.61': - resolution: {integrity: sha512-JwsgVfnR0uPSlxRXZcpdXYtTUZJUOHSabeIe3xAF64KzOTfoRMVo+38079dVF7vZ8252VD6KhjUAMdtuHRvJGg==} + '@crawlee/browser@4.0.0-beta.66': + resolution: {integrity: sha512-/qZC1TqX3w0nBqC+WFK1WLAgQXtIfJq8SrSCDx2hqKesKvq0Qyb5veJs+96myOBSz1r/jN5z65fgNJjs+nS6Vg==} engines: {node: '>=22.0.0'} peerDependencies: playwright: '*' @@ -1104,45 +1104,45 @@ packages: puppeteer: optional: true - '@crawlee/cheerio@4.0.0-beta.61': - resolution: {integrity: sha512-dOq2zhQV40Ck3vLTmGFTNhu3XlwVeoPEd4u9e3b3qiJFqHrMeHPgWkUCL15kcYg5wtvZ9U6oPLjwhN3D99B4XA==} + '@crawlee/cheerio@4.0.0-beta.66': + resolution: {integrity: sha512-SBOvy1YQ9neDExUrw0LhXAnWSYrkqoxCDLwKLYzvSkYFsrdTMW8oIAvC65yMXjIocDF2YTinCvS6zt0O2Thobg==} engines: {node: '>=22.0.0'} - '@crawlee/cli@4.0.0-beta.61': - resolution: {integrity: sha512-c9CvwDIj3qMnMFJo41cdgWBcWtZkACkBcLoJ2CA/ysdhOPx4XvGSJGGDVZXT5cX4psJVmKUF2jFiWfmC/Jbfaw==} + '@crawlee/cli@4.0.0-beta.66': + resolution: {integrity: sha512-/bbwT594gyBCzkQp0EAb4oaIOkrZjR9lfH5JX/6A/x0Trp6Wh8qC7RFCCtCEqcWwtS2+ggtgeTH0fdTWJHVXpQ==} engines: {node: '>=22.0.0'} hasBin: true - '@crawlee/core@4.0.0-beta.61': - resolution: {integrity: sha512-iuJlwScQ5yZW6sf9/hNoRkVeRSED8d0wsuQp3/3tpKLS2Y/er1mhUrtng16RstQWrN31hi5ea1Fv4zp8e0swdQ==} + '@crawlee/core@4.0.0-beta.66': + resolution: {integrity: sha512-4ONOe8XMBtlMKFPCYdVJImXjZF1a+R1Hw0K7KEByTwVTx8sunZiTX7ac0AjRKIcOYS3mp5+lHhUqSMLEm15vdg==} engines: {node: '>=22.0.0'} - '@crawlee/got-scraping-client@4.0.0-beta.61': - resolution: {integrity: sha512-xz2PM7urfL+EWOG7kvy/HKTyfJZNjV7GLfm7PL1d9aSwdTHATtc4n3E4pPB4Jt4o9ZG5uuRwFMkiCWKSJnR2CA==} + '@crawlee/got-scraping-client@4.0.0-beta.66': + resolution: {integrity: sha512-ohCx0ClWp9UX+KZmfs3R0IKtcGllwp1kJblf7nxqZRXiZuIt7CwCuWFtSD4rVXkFzlGd7lR3165r+7jVyKIM+g==} engines: {node: '>=22.0.0'} - '@crawlee/http-client@4.0.0-beta.61': - resolution: {integrity: sha512-oShefU7Tzy4f8oqFYCgolQKOad6bWZQ59SGXJjTa+TZb+jj7ETQfF29R+RwmH/AZeDgzXBzkEqmFQi4iOuYFDg==} + '@crawlee/http-client@4.0.0-beta.66': + resolution: {integrity: sha512-it1G1UxEH2/gMZ5Mt+rkVOe+vRbEcaE8xBeb+Mhx5I+BfC9uiqz8oGDuX37v4UwoFqeNjBXPAqSHKHae7nd9CA==} engines: {node: '>=22.0.0'} - '@crawlee/http@4.0.0-beta.61': - resolution: {integrity: sha512-SuTCpfyCzg2zQLc2WnDQlE794m4EC4Lqx+zIS7am0hw4mNXQqoRbsnwabN3Wo/tvvhBfRCso7sQ1xcN5NGF5AQ==} + '@crawlee/http@4.0.0-beta.66': + resolution: {integrity: sha512-SoRzYId/3WFVSZECxem9ea7OznRHOU2TKK7ymsRWphfFxayr1EVGr2p2HBZveCuziQzcicDURLEIXTYxMl25Sw==} engines: {node: '>=22.0.0'} - '@crawlee/jsdom@4.0.0-beta.61': - resolution: {integrity: sha512-ysuWxuRcriD6+taxu2Y7lOLX1Jwp8u3zcfc/SxSgdjIQQouDPNFs2WTFycPt2+TB9HRSalTBRCZ68QxHfBk2oQ==} + '@crawlee/jsdom@4.0.0-beta.66': + resolution: {integrity: sha512-BeDJDIc+XD6srSPjR07wJQxyIiRLoMbsgsOtlu1UFvrNWG/OoM9pEPvNdCdltPDFwTzj5kK1H4VjYgrttVYS3Q==} engines: {node: '>=22.0.0'} - '@crawlee/linkedom@4.0.0-beta.61': - resolution: {integrity: sha512-TN1s73PfN+Ws6GIcpjK9dN+EdUTZ4hy5B7TApoidWZJGHOdR0NHrpV/dHk2ug6KnyKIa5ELxdZaLJM9G2rnU5A==} + '@crawlee/linkedom@4.0.0-beta.66': + resolution: {integrity: sha512-6w7fXBQZxjDBfacnULnaicjguDpcofi6zROScaQawYMzZw5pyrRr84fcUK4ii2kYq6a4ESxltQP4UrfR6t1ksg==} engines: {node: '>=22.0.0'} - '@crawlee/memory-storage@4.0.0-beta.61': - resolution: {integrity: sha512-4fQI5GHfMIkz52N3b3DMDQg8lFmDmF2xbSoszx/s79eBzmlmKD22njcgDxgIg/G8uEQfIfk69he7oJDtBwY2QA==} + '@crawlee/memory-storage@4.0.0-beta.66': + resolution: {integrity: sha512-iEcbpCRqjErmm2EFKuNhWEjNndR2JJN8bP82vwXlFaKPsR4yGywP/XSCG3nqQrg+YCnUOmGjA1R3db9C83vk6A==} engines: {node: '>=22.0.0'} - '@crawlee/playwright@4.0.0-beta.61': - resolution: {integrity: sha512-lf/y19YAqvlrEWI86Fu7E3lMk/d7zlgRkWVbZFEOXvoVCt48Uxu9lu/s94jfpvxwzjNgHeUfTN1DONO8PFJtsA==} + '@crawlee/playwright@4.0.0-beta.66': + resolution: {integrity: sha512-rZaUascscjOGPaIpw9eNG/AmVHSPlL14pw9haNi6XmcT2cSvJvE1hosqqyT8NrfoAn434XAjwh7eAprehhCXyw==} engines: {node: '>=22.0.0'} peerDependencies: idcac-playwright: ^0.2.0 @@ -1153,8 +1153,8 @@ packages: playwright: optional: true - '@crawlee/puppeteer@4.0.0-beta.61': - resolution: {integrity: sha512-fiGkTU+xu2JU0IN7R96cAkDFe5iNHp13VjQPgXL+gvS1oNOSthVue8+j3Be5ZxvmyqnCb3qGMhqf8pn2knym7w==} + '@crawlee/puppeteer@4.0.0-beta.66': + resolution: {integrity: sha512-fFlu86g/On7L7GX1tnA3Joz/gKBnwI9rjcH8pvD23Wfo6Gcy7jQPduuNk5rjPV2mZWIxG2M+thiScm/S3+IYdg==} engines: {node: '>=22.0.0'} peerDependencies: idcac-playwright: ^0.2.0 @@ -1165,20 +1165,20 @@ packages: puppeteer: optional: true - '@crawlee/templates@4.0.0-beta.61': - resolution: {integrity: sha512-5XXPefaxHlNHJE8T9V5+BV6oiOj07bKsM+kA+jF/U1uSw+doKumrVrQ5INjHuOriwRHKSpDuIOWVYekgSgs9yw==} + '@crawlee/templates@4.0.0-beta.66': + resolution: {integrity: sha512-efOO/f749Z6teELNeFq5iL79qSBJY9s/LaFXkRCaiHWVpgQU7yHP9TMSHAdAQS38czvRR2dmyy2BdbOhz6YY5Q==} engines: {node: '>=22.0.0'} '@crawlee/types@3.16.0': resolution: {integrity: sha512-CcIM+JDVx4gzQzMPl+9RJiEeqdzTrx2RLPA7y4IMJSyfZm3J/VrEunielKA3NQrk095j9OuvS/rQL2y8mBV1qw==} engines: {node: '>=16.0.0'} - '@crawlee/types@4.0.0-beta.61': - resolution: {integrity: sha512-eCCGO2eoDvjNgZcIHUtY0bP5bXfxxUCNQm0W1eZciC0X3kFp1Ouei0RcZOLOqdUTRx69trJnAwWZdOvDiUL3lA==} + '@crawlee/types@4.0.0-beta.66': + resolution: {integrity: sha512-holROsQQpWAwQ6DyvgjsaegSnoQZstnyNvy7DfEDeiu7S8i2jsgrkeQnF5ZT6fxDHsDwzwNpbVlo5O+CI+pf9Q==} engines: {node: '>=22.0.0'} - '@crawlee/utils@4.0.0-beta.61': - resolution: {integrity: sha512-2IloaTvA1W5reQ0tD94amOpeFwONsMPTaHWMKnx/KZABOTnzqv73dYVuYlfkPNatGc4Kqr1Sjwx1iF+tQB22fw==} + '@crawlee/utils@4.0.0-beta.66': + resolution: {integrity: sha512-MJMg2Vf0a27YsY53Mp+cX0NpzFyyjHRoqBIJrxt0kPvvLJB6t49l85VjUOYKTTPScFW/4KmFQBKZ2dvEWuejnw==} engines: {node: '>=22.0.0'} '@csstools/cascade-layer-name-parser@2.0.5': @@ -4571,8 +4571,8 @@ packages: typescript: optional: true - crawlee@4.0.0-beta.61: - resolution: {integrity: sha512-Ravqv5dni2uoTM8GOoV6amKhpY0VZaeZWhf3H57l/OrUVkG8IXezYvfZal2IvkMm+shXjW1njty4UqLkJOgGng==} + crawlee@4.0.0-beta.66: + resolution: {integrity: sha512-ebhJOZKw9VDVGyt1Pslm0VwcJo382KNg0TL0KbRmej9AfobOwDq2MdVuPnIz0g2Bcm96whYaB9OLt7zW4KsI1A==} engines: {node: '>=22.0.0'} hasBin: true peerDependencies: @@ -10253,14 +10253,14 @@ snapshots: optionalDependencies: conventional-commits-parser: 6.4.0 - '@crawlee/basic@4.0.0-beta.61': + '@crawlee/basic@4.0.0-beta.66': dependencies: '@apify/timeout': 0.3.2 '@apify/utilities': 2.26.0 - '@crawlee/core': 4.0.0-beta.61 - '@crawlee/got-scraping-client': 4.0.0-beta.61 - '@crawlee/types': 4.0.0-beta.61 - '@crawlee/utils': 4.0.0-beta.61 + '@crawlee/core': 4.0.0-beta.66 + '@crawlee/got-scraping-client': 4.0.0-beta.66 + '@crawlee/types': 4.0.0-beta.66 + '@crawlee/utils': 4.0.0-beta.66 csv-stringify: 6.7.0 fs-extra: 11.3.4 ow: 2.0.0 @@ -10270,11 +10270,11 @@ snapshots: transitivePeerDependencies: - supports-color - '@crawlee/browser-pool@4.0.0-beta.61(playwright@1.60.0)(puppeteer@25.1.0)': + '@crawlee/browser-pool@4.0.0-beta.66(playwright@1.60.0)(puppeteer@25.1.0)': dependencies: '@apify/timeout': 0.3.2 - '@crawlee/core': 4.0.0-beta.61 - '@crawlee/types': 4.0.0-beta.61 + '@crawlee/core': 4.0.0-beta.66 + '@crawlee/types': 4.0.0-beta.66 fingerprint-generator: 2.1.82 fingerprint-injector: 2.1.82(playwright@1.60.0)(puppeteer@25.1.0) lodash.merge: 4.6.2 @@ -10291,13 +10291,13 @@ snapshots: transitivePeerDependencies: - supports-color - '@crawlee/browser@4.0.0-beta.61(playwright@1.60.0)(puppeteer@25.1.0)': + '@crawlee/browser@4.0.0-beta.66(playwright@1.60.0)(puppeteer@25.1.0)': dependencies: '@apify/timeout': 0.3.2 - '@crawlee/basic': 4.0.0-beta.61 - '@crawlee/browser-pool': 4.0.0-beta.61(playwright@1.60.0)(puppeteer@25.1.0) - '@crawlee/types': 4.0.0-beta.61 - '@crawlee/utils': 4.0.0-beta.61 + '@crawlee/basic': 4.0.0-beta.66 + '@crawlee/browser-pool': 4.0.0-beta.66(playwright@1.60.0)(puppeteer@25.1.0) + '@crawlee/types': 4.0.0-beta.66 + '@crawlee/utils': 4.0.0-beta.66 ow: 2.0.0 tslib: 2.8.1 type-fest: 4.41.0 @@ -10307,20 +10307,20 @@ snapshots: transitivePeerDependencies: - supports-color - '@crawlee/cheerio@4.0.0-beta.61': + '@crawlee/cheerio@4.0.0-beta.66': dependencies: - '@crawlee/http': 4.0.0-beta.61 - '@crawlee/types': 4.0.0-beta.61 - '@crawlee/utils': 4.0.0-beta.61 + '@crawlee/http': 4.0.0-beta.66 + '@crawlee/types': 4.0.0-beta.66 + '@crawlee/utils': 4.0.0-beta.66 cheerio: 1.2.0 htmlparser2: 10.1.0 tslib: 2.8.1 transitivePeerDependencies: - supports-color - '@crawlee/cli@4.0.0-beta.61(@types/node@24.12.2)': + '@crawlee/cli@4.0.0-beta.66(@types/node@24.12.2)': dependencies: - '@crawlee/templates': 4.0.0-beta.61 + '@crawlee/templates': 4.0.0-beta.66 '@inquirer/prompts': 7.10.1(@types/node@24.12.2) ansi-colors: 4.1.3 fs-extra: 11.3.4 @@ -10329,7 +10329,7 @@ snapshots: transitivePeerDependencies: - '@types/node' - '@crawlee/core@4.0.0-beta.61': + '@crawlee/core@4.0.0-beta.66': dependencies: '@apify/consts': 2.52.0 '@apify/datastructures': 2.0.3 @@ -10337,9 +10337,9 @@ snapshots: '@apify/pseudo_url': 2.0.75 '@apify/timeout': 0.3.2 '@apify/utilities': 2.26.0 - '@crawlee/memory-storage': 4.0.0-beta.61 - '@crawlee/types': 4.0.0-beta.61 - '@crawlee/utils': 4.0.0-beta.61 + '@crawlee/memory-storage': 4.0.0-beta.66 + '@crawlee/types': 4.0.0-beta.66 + '@crawlee/utils': 4.0.0-beta.66 '@sapphire/async-queue': 1.5.5 '@vladfrangu/async_event_emitter': 2.4.7 csv-stringify: 6.7.0 @@ -10355,25 +10355,25 @@ snapshots: transitivePeerDependencies: - supports-color - '@crawlee/got-scraping-client@4.0.0-beta.61': + '@crawlee/got-scraping-client@4.0.0-beta.66': dependencies: - '@crawlee/http-client': 4.0.0-beta.61 + '@crawlee/http-client': 4.0.0-beta.66 got-scraping: 4.2.1 - '@crawlee/http-client@4.0.0-beta.61': + '@crawlee/http-client@4.0.0-beta.66': dependencies: - '@crawlee/types': 4.0.0-beta.61 + '@crawlee/types': 4.0.0-beta.66 tough-cookie: 6.0.1 - '@crawlee/http@4.0.0-beta.61': + '@crawlee/http@4.0.0-beta.66': dependencies: '@apify/timeout': 0.3.2 '@apify/utilities': 2.26.0 - '@crawlee/basic': 4.0.0-beta.61 - '@crawlee/core': 4.0.0-beta.61 - '@crawlee/http-client': 4.0.0-beta.61 - '@crawlee/types': 4.0.0-beta.61 - '@crawlee/utils': 4.0.0-beta.61 + '@crawlee/basic': 4.0.0-beta.66 + '@crawlee/core': 4.0.0-beta.66 + '@crawlee/http-client': 4.0.0-beta.66 + '@crawlee/types': 4.0.0-beta.66 + '@crawlee/utils': 4.0.0-beta.66 '@types/content-type': 1.1.9 cheerio: 1.2.0 content-type: 1.0.5 @@ -10385,13 +10385,13 @@ snapshots: transitivePeerDependencies: - supports-color - '@crawlee/jsdom@4.0.0-beta.61': + '@crawlee/jsdom@4.0.0-beta.66': dependencies: '@apify/timeout': 0.3.2 '@apify/utilities': 2.26.0 - '@crawlee/http': 4.0.0-beta.61 - '@crawlee/types': 4.0.0-beta.61 - '@crawlee/utils': 4.0.0-beta.61 + '@crawlee/http': 4.0.0-beta.66 + '@crawlee/types': 4.0.0-beta.66 + '@crawlee/utils': 4.0.0-beta.66 '@types/jsdom': 21.1.7 cheerio: 1.2.0 jsdom: 26.1.0 @@ -10403,13 +10403,13 @@ snapshots: - supports-color - utf-8-validate - '@crawlee/linkedom@4.0.0-beta.61': + '@crawlee/linkedom@4.0.0-beta.66': dependencies: '@apify/timeout': 0.3.2 '@apify/utilities': 2.26.0 - '@crawlee/http': 4.0.0-beta.61 - '@crawlee/types': 4.0.0-beta.61 - '@crawlee/utils': 4.0.0-beta.61 + '@crawlee/http': 4.0.0-beta.66 + '@crawlee/types': 4.0.0-beta.66 + '@crawlee/utils': 4.0.0-beta.66 cheerio: 1.2.0 linkedom: 0.18.12 ow: 2.0.0 @@ -10418,9 +10418,9 @@ snapshots: - canvas - supports-color - '@crawlee/memory-storage@4.0.0-beta.61': + '@crawlee/memory-storage@4.0.0-beta.66': dependencies: - '@crawlee/types': 4.0.0-beta.61 + '@crawlee/types': 4.0.0-beta.66 '@sapphire/async-queue': 1.5.5 '@sapphire/shapeshift': 4.0.0 content-type: 1.0.5 @@ -10431,17 +10431,17 @@ snapshots: proper-lockfile: 4.1.2 tslib: 2.8.1 - '@crawlee/playwright@4.0.0-beta.61(playwright@1.60.0)(puppeteer@25.1.0)': + '@crawlee/playwright@4.0.0-beta.66(playwright@1.60.0)(puppeteer@25.1.0)': dependencies: '@apify/datastructures': 2.0.3 '@apify/timeout': 0.3.2 - '@crawlee/basic': 4.0.0-beta.61 - '@crawlee/browser': 4.0.0-beta.61(playwright@1.60.0)(puppeteer@25.1.0) - '@crawlee/browser-pool': 4.0.0-beta.61(playwright@1.60.0)(puppeteer@25.1.0) - '@crawlee/cheerio': 4.0.0-beta.61 - '@crawlee/core': 4.0.0-beta.61 - '@crawlee/types': 4.0.0-beta.61 - '@crawlee/utils': 4.0.0-beta.61 + '@crawlee/basic': 4.0.0-beta.66 + '@crawlee/browser': 4.0.0-beta.66(playwright@1.60.0)(puppeteer@25.1.0) + '@crawlee/browser-pool': 4.0.0-beta.66(playwright@1.60.0)(puppeteer@25.1.0) + '@crawlee/cheerio': 4.0.0-beta.66 + '@crawlee/core': 4.0.0-beta.66 + '@crawlee/types': 4.0.0-beta.66 + '@crawlee/utils': 4.0.0-beta.66 cheerio: 1.2.0 jquery: 3.7.1 ml-logistic-regression: 2.0.0 @@ -10455,14 +10455,14 @@ snapshots: - puppeteer - supports-color - '@crawlee/puppeteer@4.0.0-beta.61(playwright@1.60.0)(puppeteer@25.1.0)': + '@crawlee/puppeteer@4.0.0-beta.66(playwright@1.60.0)(puppeteer@25.1.0)': dependencies: '@apify/datastructures': 2.0.3 - '@crawlee/browser': 4.0.0-beta.61(playwright@1.60.0)(puppeteer@25.1.0) - '@crawlee/browser-pool': 4.0.0-beta.61(playwright@1.60.0)(puppeteer@25.1.0) - '@crawlee/core': 4.0.0-beta.61 - '@crawlee/types': 4.0.0-beta.61 - '@crawlee/utils': 4.0.0-beta.61 + '@crawlee/browser': 4.0.0-beta.66(playwright@1.60.0)(puppeteer@25.1.0) + '@crawlee/browser-pool': 4.0.0-beta.66(playwright@1.60.0)(puppeteer@25.1.0) + '@crawlee/core': 4.0.0-beta.66 + '@crawlee/types': 4.0.0-beta.66 + '@crawlee/utils': 4.0.0-beta.66 cheerio: 1.2.0 devtools-protocol: 0.0.1624250 jquery: 3.7.1 @@ -10474,7 +10474,7 @@ snapshots: - playwright - supports-color - '@crawlee/templates@4.0.0-beta.61': + '@crawlee/templates@4.0.0-beta.66': dependencies: tslib: 2.8.1 @@ -10482,16 +10482,16 @@ snapshots: dependencies: tslib: 2.8.1 - '@crawlee/types@4.0.0-beta.61': + '@crawlee/types@4.0.0-beta.66': dependencies: tough-cookie: 6.0.1 tslib: 2.8.1 - '@crawlee/utils@4.0.0-beta.61': + '@crawlee/utils@4.0.0-beta.66': dependencies: '@apify/ps-tree': 1.2.0 - '@crawlee/http-client': 4.0.0-beta.61 - '@crawlee/types': 4.0.0-beta.61 + '@crawlee/http-client': 4.0.0-beta.66 + '@crawlee/types': 4.0.0-beta.66 '@types/sax': 1.2.7 cheerio: 1.2.0 domhandler: 5.0.3 @@ -14430,20 +14430,20 @@ snapshots: optionalDependencies: typescript: 6.0.3 - crawlee@4.0.0-beta.61(@types/node@24.12.2)(playwright@1.60.0)(puppeteer@25.1.0): - dependencies: - '@crawlee/basic': 4.0.0-beta.61 - '@crawlee/browser': 4.0.0-beta.61(playwright@1.60.0)(puppeteer@25.1.0) - '@crawlee/browser-pool': 4.0.0-beta.61(playwright@1.60.0)(puppeteer@25.1.0) - '@crawlee/cheerio': 4.0.0-beta.61 - '@crawlee/cli': 4.0.0-beta.61(@types/node@24.12.2) - '@crawlee/core': 4.0.0-beta.61 - '@crawlee/http': 4.0.0-beta.61 - '@crawlee/jsdom': 4.0.0-beta.61 - '@crawlee/linkedom': 4.0.0-beta.61 - '@crawlee/playwright': 4.0.0-beta.61(playwright@1.60.0)(puppeteer@25.1.0) - '@crawlee/puppeteer': 4.0.0-beta.61(playwright@1.60.0)(puppeteer@25.1.0) - '@crawlee/utils': 4.0.0-beta.61 + crawlee@4.0.0-beta.66(@types/node@24.12.2)(playwright@1.60.0)(puppeteer@25.1.0): + dependencies: + '@crawlee/basic': 4.0.0-beta.66 + '@crawlee/browser': 4.0.0-beta.66(playwright@1.60.0)(puppeteer@25.1.0) + '@crawlee/browser-pool': 4.0.0-beta.66(playwright@1.60.0)(puppeteer@25.1.0) + '@crawlee/cheerio': 4.0.0-beta.66 + '@crawlee/cli': 4.0.0-beta.66(@types/node@24.12.2) + '@crawlee/core': 4.0.0-beta.66 + '@crawlee/http': 4.0.0-beta.66 + '@crawlee/jsdom': 4.0.0-beta.66 + '@crawlee/linkedom': 4.0.0-beta.66 + '@crawlee/playwright': 4.0.0-beta.66(playwright@1.60.0)(puppeteer@25.1.0) + '@crawlee/puppeteer': 4.0.0-beta.66(playwright@1.60.0)(puppeteer@25.1.0) + '@crawlee/utils': 4.0.0-beta.66 import-local: 3.2.0 tslib: 2.8.1 optionalDependencies: diff --git a/src/actor.ts b/src/actor.ts index f9941488ff..98b43dbffb 100644 --- a/src/actor.ts +++ b/src/actor.ts @@ -1332,9 +1332,6 @@ export class Actor { const queue = await this._openStorage(RequestQueue, queueIdOrName, options); - // eslint-disable-next-line dot-notation - queue['initialCount'] = (await queue.client.getMetadata())?.totalRequestCount ?? 0; - return queue; } diff --git a/src/apify_request_queue_client.ts b/src/apify_request_queue_client.ts new file mode 100644 index 0000000000..501527fb39 --- /dev/null +++ b/src/apify_request_queue_client.ts @@ -0,0 +1,209 @@ +import { createHash } from 'node:crypto'; + +import type { + BatchAddRequestsResult, + QueueOperationInfo, + RequestOptions, + RequestQueueInfo, + RequestSchema, + UpdateRequestSchema, +} from '@crawlee/types'; +import type { RequestQueueClient as ApifyRequestQueueResourceClient } from 'apify-client'; + +/** + * Crawlee v4's redesigned (pull-based) `RequestQueueClient` interface. + * + * This is a local copy because the published `@crawlee/types` still ships the + * pre-redesign interface in its `.d.ts` even though `@crawlee/core`'s runtime + * already calls these methods (`addBatchOfRequests`/`fetchNextRequest`/…). Once + * crawlee publishes the regenerated types, drop this and `implements` + * `@crawlee/types`' `RequestQueueClient` directly. + */ +interface RequestQueueClientV4 { + getMetadata(): Promise; + drop(): Promise; + purge(): Promise; + addBatchOfRequests(requests: RequestSchema[], options?: RequestOptions): Promise; + getRequest(uniqueKey: string): Promise; + fetchNextRequest(): Promise; + markRequestAsHandled(request: UpdateRequestSchema): Promise; + reclaimRequest(request: UpdateRequestSchema, options?: RequestOptions): Promise; + isEmpty(): Promise; + isFinished(): Promise; + setExpectedRequestProcessingTimeSecs?(secs: number): void; +} + +/** Apify request IDs are the first 15 chars of a base64url-ish SHA-256 of the unique key. */ +const REQUEST_ID_LENGTH = 15; + +/** Default lock duration for requests fetched via {@link ApifyRequestQueueClient.fetchNextRequest}. */ +const DEFAULT_REQUEST_LOCK_SECS = 3 * 60; + +/** How many head requests to lock per `listAndLockHead` round-trip. */ +const HEAD_LOCK_BATCH_SIZE = 25; + +/** + * Derives a request id from its unique key, exactly as the Apify platform does + * (`sha256(uniqueKey)` → base64 → strip `+`/`/`/`=` → first 15 chars). Lets us + * address a request by unique key without an extra round-trip. + */ +function uniqueKeyToRequestId(uniqueKey: string): string { + const hash = createHash('sha256').update(uniqueKey).digest('base64').replace(/[+/=]/g, ''); + return hash.slice(0, REQUEST_ID_LENGTH); +} + +/** + * Implements Crawlee v4's stateful, pull-based {@link RequestQueueClient} interface + * on top of `apify-client`'s REST request-queue API. + * + * `fetchNextRequest` locks requests server-side (`listAndLockHead`) so the queue can + * be safely shared across multiple consumers; `markRequestAsHandled` / `reclaimRequest` + * update the request and release its lock. Modeled on the Apify Python SDK's + * request-queue client. + * + * @internal + */ +export class ApifyRequestQueueClient implements RequestQueueClientV4 { + /** Locked request ids waiting to be handed out by {@link fetchNextRequest}, in queue order. */ + private readonly queueHead: string[] = []; + + /** Lock duration applied to fetched requests; adjustable via {@link setExpectedRequestProcessingTimeSecs}. */ + private lockSecs = DEFAULT_REQUEST_LOCK_SECS; + + constructor(private readonly client: ApifyRequestQueueResourceClient) {} + + async getMetadata(): Promise { + const metadata = await this.client.get(); + if (!metadata) { + throw new Error('Request queue not found or has been deleted.'); + } + return metadata as unknown as RequestQueueInfo; + } + + async drop(): Promise { + await this.client.delete(); + } + + async purge(): Promise { + // The Apify platform has no "empty but keep" endpoint, and a run's queue is + // already fresh, so purge is a no-op here (use drop() to remove it entirely). + } + + setExpectedRequestProcessingTimeSecs(secs: number): void { + this.lockSecs = secs; + } + + async addBatchOfRequests(requests: RequestSchema[], options: RequestOptions = {}): Promise { + const { forefront } = options; + // The API assigns ids itself; strip any incoming id so its strict input validation passes. + const apiRequests = requests.map((request) => { + const { id: _id, ...rest } = request; + return rest; + }); + + const result = await this.client.batchAddRequests( + apiRequests as Parameters[0], + { forefront }, + ); + + // A forefront insert changes the head order; drop our cached head so the next + // fetch re-reads (and locks) the new front of the queue. + if (forefront) { + this.queueHead.length = 0; + } + return result as unknown as BatchAddRequestsResult; + } + + async getRequest(uniqueKey: string): Promise { + const request = await this.client.getRequest(uniqueKeyToRequestId(uniqueKey)); + return (request as RequestOptions | undefined) ?? undefined; + } + + async fetchNextRequest(): Promise { + await this.ensureHeadIsNonEmpty(); + + while (this.queueHead.length > 0) { + const requestId = this.queueHead.shift()!; + // Head items lack userData/payload, so hydrate the full request. + const request = await this.client.getRequest(requestId); + // Skip requests that vanished or were already handled (e.g. by another consumer). + if (!request || (request as { handledAt?: string }).handledAt) { + continue; + } + return request as RequestOptions; + } + return null; + } + + async markRequestAsHandled(request: UpdateRequestSchema): Promise { + const result = await this.client.updateRequest({ + ...this.toApiRequest(request), + handledAt: request.handledAt ?? new Date().toISOString(), + }); + // Marking handled takes the request out of the head; its lock becomes moot. + return this.toQueueOperationInfo(result); + } + + async reclaimRequest( + request: UpdateRequestSchema, + options: RequestOptions = {}, + ): Promise { + const { forefront } = options; + const result = await this.client.updateRequest(this.toApiRequest(request), { forefront }); + + // Release the server-side lock so the request is fetchable again immediately, + // rather than only once the lock expires. + await this.client.deleteRequestLock(request.id, { forefront }).catch(() => {}); + if (forefront) { + this.queueHead.length = 0; + } + return this.toQueueOperationInfo(result); + } + + async isEmpty(): Promise { + if (this.queueHead.length > 0) { + return false; + } + const head = await this.client.listHead({ limit: 1 }); + return head.items.length === 0; + } + + async isFinished(): Promise { + if (this.queueHead.length > 0) { + return false; + } + // `pendingRequestCount` counts every not-yet-handled request, including ones + // currently locked/in-progress — so 0 means there is genuinely nothing left. + const metadata = await this.client.get(); + return (metadata?.pendingRequestCount ?? 0) === 0; + } + + private async ensureHeadIsNonEmpty(): Promise { + if (this.queueHead.length > 0) { + return; + } + const head = await this.client.listAndLockHead({ limit: HEAD_LOCK_BATCH_SIZE, lockSecs: this.lockSecs }); + for (const item of head.items) { + this.queueHead.push(item.id); + } + } + + /** Crawlee's `RequestSchema`/`UpdateRequestSchema` is structurally the apify-client request shape. */ + private toApiRequest( + request: UpdateRequestSchema, + ): Parameters[0] { + return request as unknown as Parameters[0]; + } + + private toQueueOperationInfo(result: { + requestId: string; + wasAlreadyPresent: boolean; + wasAlreadyHandled: boolean; + }): QueueOperationInfo { + return { + requestId: result.requestId, + wasAlreadyPresent: result.wasAlreadyPresent, + wasAlreadyHandled: result.wasAlreadyHandled, + }; + } +} diff --git a/src/apify_storage_client.ts b/src/apify_storage_client.ts index 749fb70e0f..e915049b50 100644 --- a/src/apify_storage_client.ts +++ b/src/apify_storage_client.ts @@ -8,11 +8,13 @@ import type { DatasetClient, KeyValueStoreClient, RequestQueueClient, + SetStatusMessageOptions, StorageClient, } from '@crawlee/types'; import type { ApifyClient } from 'apify-client'; import { DatasetClient as ApifyDatasetClient } from 'apify-client'; +import { ApifyRequestQueueClient } from './apify_request_queue_client.js'; import { type ChargeResult, type ChargingManager, @@ -203,10 +205,27 @@ export class ApifyStorageClient implements StorageClient { ) as unknown as KeyValueStoreClient; } + /** + * Sets the run's status message (shown in the Apify console). Crawlee v4 calls + * this on the storage client to surface crawl progress. No-op when not running + * on the platform (no run id). + */ + async setStatusMessage(message: string, options?: SetStatusMessageOptions): Promise { + const runId = this.config?.actorRunId; + if (!runId) return; + await this.client.run(runId).update({ + statusMessage: message, + isStatusMessageTerminal: options?.isStatusMessageTerminal, + }); + } + async createRequestQueueClient(options?: CreateRequestQueueClientOptions): Promise { const id = await this.resolveId(options, 'RequestQueue'); const client = this.client.requestQueue(id, options?.clientKey ? { clientKey: options.clientKey } : undefined); - return adapt(client, { getMetadata: 'get', drop: 'delete' }, noPurge) as unknown as RequestQueueClient; + // Crawlee v4's RequestQueueClient is a stateful, pull-based interface that + // can't be satisfied by name-remapping; it's implemented on apify-client. + // (cast: published @crawlee/types still ships the pre-redesign interface.) + return new ApifyRequestQueueClient(client) as unknown as RequestQueueClient; } /** diff --git a/test/apify/actor.test.ts b/test/apify/actor.test.ts index 45691440a9..4f08898f83 100644 --- a/test/apify/actor.test.ts +++ b/test/apify/actor.test.ts @@ -674,20 +674,14 @@ describe('Actor', () => { const options = { forceCloud: true }; // crawlee v4 opens storages via `.open(id, { storageClient })`; // `forceCloud` is expressed by passing an explicit (Apify) client. - const mockRQ = { - client: { - getMetadata: async () => ({ totalRequestCount: 10 }), - }, - } as unknown as RequestQueue; + const mockRQ = {} as unknown as RequestQueue; const openSpy = vitest.spyOn(RequestQueue, 'open').mockResolvedValueOnce(mockRQ); const queue = await sdk.openRequestQueue(queueId, options); expect(openSpy).toBeCalledTimes(1); expect(openSpy.mock.calls[0][0]).toBe(queueId); expect(openSpy.mock.calls[0][1]?.storageClient).toBeDefined(); - - // @ts-expect-error private prop - expect(queue.initialCount).toBe(10); + expect(queue).toBe(mockRQ); }); test('openDataset should open storage', async () => { From 8212ea4c58b863cdd733b8bd3d030eedae8e7e79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 16 Jul 2026 16:19:11 +0200 Subject: [PATCH 2/2] refactor: adapt to crawlee v4 StorageBackend rename (beta.71) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Crawlee v4 (apify/crawlee#3740) renamed the storage abstraction — `StorageClient` → `StorageBackend`, `RequestQueueClient` → `RequestQueueBackend`, `createXClient` → `createXBackend` — moved the in-memory client into `@crawlee/core` (`MemoryStorageBackend`, dropping the `@crawlee/memory-storage` package), and further refined the request-queue signatures (`fetchNextRequest`/`getRequest` now return `UpdateRequestSchema | undefined`, `null` → `undefined`, async `setExpectedRequestProcessingTimeSecs`). - `ApifyStorageClient` now implements `StorageBackend` (`createDatasetBackend` / `createKeyValueStoreBackend` / `createRequestQueueBackend`). - `ApifyRequestQueueClient` implements the real `RequestQueueBackend` directly — the temporary local interface copy is gone. - `StorageBackend` also dropped `setStatusMessage`, so `Actor.setStatusMessage` now sets the run status directly via apify-client. - Bumps `@crawlee/*` to `^4.0.0-beta.71`. Compile, 123 unit tests, and a local (memory-backend) smoke run on crawlee@71 all pass. The request-queue logic itself was validated end-to-end on the Apify platform against crawlee@66; on-platform validation on @71 is currently blocked by a crawlee packaging gap (its `@crawlee/fs-storage` native binding is unpublished for musl). --- package.json | 9 +- pnpm-lock.yaml | 563 +++++++++++++++--------------- src/actor.ts | 36 +- src/apify_request_queue_client.ts | 55 +-- src/apify_storage_client.ts | 46 +-- src/storage.ts | 8 +- test/apify/actor.test.ts | 9 +- test/createIsolatedActor.ts | 13 +- 8 files changed, 340 insertions(+), 399 deletions(-) diff --git a/package.json b/package.json index 996e0363ab..f85145c5b0 100644 --- a/package.json +++ b/package.json @@ -77,9 +77,9 @@ "@apify/log": "^2.4.3", "@apify/timeout": "^0.3.0", "@apify/utilities": "^2.13.0", - "@crawlee/core": "^4.0.0-beta.66", - "@crawlee/types": "^4.0.0-beta.66", - "@crawlee/utils": "^4.0.0-beta.66", + "@crawlee/core": "^4.0.0-beta.71", + "@crawlee/types": "^4.0.0-beta.71", + "@crawlee/utils": "^4.0.0-beta.71", "apify-client": "^2.23.4", "semver": "^7.5.4", "tslib": "^2.6.2", @@ -90,7 +90,6 @@ "@apify/oxlint-config": "^0.2.5", "@apify/tsconfig": "^0.1.2", "@commitlint/config-conventional": "^21.0.0", - "@crawlee/memory-storage": "^4.0.0-beta.66", "@playwright/browser-chromium": "^1.60.0", "@types/content-type": "^1.1.8", "@types/node": "^24.0.0", @@ -98,7 +97,7 @@ "@types/tough-cookie": "^4.0.5", "@types/ws": "^8.5.12", "commitlint": "^21.0.0", - "crawlee": "^4.0.0-beta.66", + "crawlee": "^4.0.0-beta.71", "globby": "^16.0.0", "husky": "^9.1.7", "lint-staged": "^17.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7b3eaf6781..961312122d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -29,14 +29,14 @@ importers: specifier: ^2.13.0 version: 2.26.0 '@crawlee/core': - specifier: ^4.0.0-beta.66 - version: 4.0.0-beta.66 + specifier: ^4.0.0-beta.71 + version: 4.0.0-beta.71 '@crawlee/types': - specifier: ^4.0.0-beta.66 - version: 4.0.0-beta.66 + specifier: ^4.0.0-beta.71 + version: 4.0.0-beta.71 '@crawlee/utils': - specifier: ^4.0.0-beta.66 - version: 4.0.0-beta.66 + specifier: ^4.0.0-beta.71 + version: 4.0.0-beta.71 apify-client: specifier: ^2.23.4 version: 2.23.4 @@ -62,9 +62,6 @@ importers: '@commitlint/config-conventional': specifier: ^21.0.0 version: 21.0.1 - '@crawlee/memory-storage': - specifier: ^4.0.0-beta.66 - version: 4.0.0-beta.66 '@playwright/browser-chromium': specifier: ^1.60.0 version: 1.60.0 @@ -87,8 +84,8 @@ importers: specifier: ^21.0.0 version: 21.0.1(@types/node@24.12.2)(conventional-commits-parser@6.4.0)(typescript@6.0.3) crawlee: - specifier: ^4.0.0-beta.66 - version: 4.0.0-beta.66(@types/node@24.12.2)(playwright@1.60.0)(puppeteer@25.1.0) + specifier: ^4.0.0-beta.71 + version: 4.0.0-beta.71(@types/node@24.12.2)(playwright@1.60.0)(puppeteer@25.1.0) globby: specifier: ^16.0.0 version: 16.2.0 @@ -1076,12 +1073,12 @@ packages: conventional-commits-parser: optional: true - '@crawlee/basic@4.0.0-beta.66': - resolution: {integrity: sha512-8N3YelWVtZ3UrOZo3FryvKhFu/2s7TK4WaY/49DabO9Y27E+IkHPxxCyji1vaBt5a5BAnpmNRf17wXenPIGgIA==} + '@crawlee/basic@4.0.0-beta.71': + resolution: {integrity: sha512-slFKdCogVt16aqCtBOZVP3/IVs2YJRF6r9IE0nk9TwvoLRHl5CgcFGoHkT6NEfoTEW1Ase9qcTt7MRSUuNOGYQ==} engines: {node: '>=22.0.0'} - '@crawlee/browser-pool@4.0.0-beta.66': - resolution: {integrity: sha512-YyBm3oszlQqoPiSGaOHtgCR8i6xLdZmNcmAqMbi3CnFSpW3+VufNE57MadHeJaqAbQBIj0ZmX0Ng6vXBNhTSGg==} + '@crawlee/browser-pool@4.0.0-beta.71': + resolution: {integrity: sha512-ZgQshknfypguXBSb06R3XYqCgxI0rF76PIiTiZQK2mm8C9mwtzPJilCOdHX8ZI66tU70rcOy6Ca6UNWuHQJwuw==} engines: {node: '>=22.0.0'} peerDependencies: playwright: '*' @@ -1092,8 +1089,8 @@ packages: puppeteer: optional: true - '@crawlee/browser@4.0.0-beta.66': - resolution: {integrity: sha512-/qZC1TqX3w0nBqC+WFK1WLAgQXtIfJq8SrSCDx2hqKesKvq0Qyb5veJs+96myOBSz1r/jN5z65fgNJjs+nS6Vg==} + '@crawlee/browser@4.0.0-beta.71': + resolution: {integrity: sha512-3RonFghN8Mtu09APGwiSAiC9Vb3L8vSCkuCr+dqqUQDLDx5GeHkbm/iL4RpUl6ndtkyOkX2PXzj8Butgnk9HwQ==} engines: {node: '>=22.0.0'} peerDependencies: playwright: '*' @@ -1104,45 +1101,74 @@ packages: puppeteer: optional: true - '@crawlee/cheerio@4.0.0-beta.66': - resolution: {integrity: sha512-SBOvy1YQ9neDExUrw0LhXAnWSYrkqoxCDLwKLYzvSkYFsrdTMW8oIAvC65yMXjIocDF2YTinCvS6zt0O2Thobg==} + '@crawlee/cheerio@4.0.0-beta.71': + resolution: {integrity: sha512-OI4B5Q6ikdTARnLKxfQIH8CcVTTgsl78S1Vpuv5T0qYoxhBul0HF+smeieZ9GdOv4mS5zqbttA/6PsbSShkFNQ==} engines: {node: '>=22.0.0'} - '@crawlee/cli@4.0.0-beta.66': - resolution: {integrity: sha512-/bbwT594gyBCzkQp0EAb4oaIOkrZjR9lfH5JX/6A/x0Trp6Wh8qC7RFCCtCEqcWwtS2+ggtgeTH0fdTWJHVXpQ==} + '@crawlee/cli@4.0.0-beta.71': + resolution: {integrity: sha512-ixJp11jeL5NpprMwNnCgQGfTpQ2U9vJs9TvU07hLemJxufZCEpgMHq8VparF2pspWGP4rQ6oUncIpPT7AKEO9Q==} engines: {node: '>=22.0.0'} hasBin: true - '@crawlee/core@4.0.0-beta.66': - resolution: {integrity: sha512-4ONOe8XMBtlMKFPCYdVJImXjZF1a+R1Hw0K7KEByTwVTx8sunZiTX7ac0AjRKIcOYS3mp5+lHhUqSMLEm15vdg==} + '@crawlee/core@4.0.0-beta.71': + resolution: {integrity: sha512-tgitInl62vB++QYhsv85W41uLmuH3nkERU/mdKsDN6SfPWZsOrVcA4ZX1t5K/cR1o8E1aWP6O6hvmhFBmea7HA==} engines: {node: '>=22.0.0'} - '@crawlee/got-scraping-client@4.0.0-beta.66': - resolution: {integrity: sha512-ohCx0ClWp9UX+KZmfs3R0IKtcGllwp1kJblf7nxqZRXiZuIt7CwCuWFtSD4rVXkFzlGd7lR3165r+7jVyKIM+g==} + '@crawlee/fs-storage-native-darwin-arm64@0.1.5-beta.18': + resolution: {integrity: sha512-fYZbU61GoMw+3q1JXRvARa8f/A8qZo0BxBpGogzq8duhiOChs4v+uHjffuZjL4E6goFR5qjX2kwOAJzoEp2kZQ==} + engines: {node: '>= 20'} + cpu: [arm64] + os: [darwin] + + '@crawlee/fs-storage-native-darwin-x64@0.1.5-beta.18': + resolution: {integrity: sha512-Wb/d6PLor34790ixEFsRADvTUXKzR846/1E2LUaDTLA3KQ5QUCwjQhs7VpfrgY9AzzgM/s8G+3TO2UI6L3eYkw==} + engines: {node: '>= 20'} + cpu: [x64] + os: [darwin] + + '@crawlee/fs-storage-native-linux-x64-gnu@0.1.5-beta.18': + resolution: {integrity: sha512-J8qfCkj3i6ic87fiD4f05WSXLXaAFLZlboXeeH9nDhJv/6RLxjiwTqec/Ig9H3yon+v4VQUbSHIh3Mm6B0g3XQ==} + engines: {node: '>= 20'} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@crawlee/fs-storage-native-win32-x64-msvc@0.1.5-beta.18': + resolution: {integrity: sha512-76uxjzcsD2E+mWqZ+HENpcSpJJAZVdWf3osKR3pA3wld+yzR9o0cckRysu4QTDhUeNtzL1KIq2NzOF1GPNLqiw==} + engines: {node: '>= 20'} + cpu: [x64] + os: [win32] + + '@crawlee/fs-storage-native@0.1.5-beta.18': + resolution: {integrity: sha512-G5+Alb5GDAZomtu+0mB8oeYA0BCMqOQIqNoTPtZxh2wLcb17CpMfJJvq6D/nT8vANAaBkrMR49bbrodtROV2ug==} + engines: {node: '>= 20'} + + '@crawlee/fs-storage@4.0.0-beta.71': + resolution: {integrity: sha512-aLhSYWhFPlw1b4gzOMKx3IJ5aZgF+a9oxpX1pfQO5jXuwST0toMR4sqfXGYbnvp/mzYnNSqvVrRGd711NDhL9g==} engines: {node: '>=22.0.0'} - '@crawlee/http-client@4.0.0-beta.66': - resolution: {integrity: sha512-it1G1UxEH2/gMZ5Mt+rkVOe+vRbEcaE8xBeb+Mhx5I+BfC9uiqz8oGDuX37v4UwoFqeNjBXPAqSHKHae7nd9CA==} + '@crawlee/http-client@4.0.0-beta.71': + resolution: {integrity: sha512-0xnu5aik6o0DYycQMxuzNjdnR/T4AjYkt4Kl3lNh0DX4Igu2DsEClycwmk5P4K2hQ65lQYq0nRELpUzKy3hhmA==} engines: {node: '>=22.0.0'} - '@crawlee/http@4.0.0-beta.66': - resolution: {integrity: sha512-SoRzYId/3WFVSZECxem9ea7OznRHOU2TKK7ymsRWphfFxayr1EVGr2p2HBZveCuziQzcicDURLEIXTYxMl25Sw==} + '@crawlee/http@4.0.0-beta.71': + resolution: {integrity: sha512-EnGMXsu7gr0Kl0g8cTzLrOit27cyY1eP/bYnH+HJe6jMlU7bWCLwmWIJOSftRnIPFbjuvss2Vx1DqWNw+iynKw==} engines: {node: '>=22.0.0'} - '@crawlee/jsdom@4.0.0-beta.66': - resolution: {integrity: sha512-BeDJDIc+XD6srSPjR07wJQxyIiRLoMbsgsOtlu1UFvrNWG/OoM9pEPvNdCdltPDFwTzj5kK1H4VjYgrttVYS3Q==} + '@crawlee/impit-client@4.0.0-beta.71': + resolution: {integrity: sha512-yi/z9Z4vmD6LR4fdF7ECJAZJf5q7ZPu1+lMA3S62OFgpJ0qceCgOnUFahaZZMaMiX6pvKvaRYo4VIoDuojaUPg==} engines: {node: '>=22.0.0'} - '@crawlee/linkedom@4.0.0-beta.66': - resolution: {integrity: sha512-6w7fXBQZxjDBfacnULnaicjguDpcofi6zROScaQawYMzZw5pyrRr84fcUK4ii2kYq6a4ESxltQP4UrfR6t1ksg==} + '@crawlee/jsdom@4.0.0-beta.71': + resolution: {integrity: sha512-6iUH9csLfpBziwAEDQXu60n2UWbandSOm0uMgqgeMyJg63KUmaeOX/DbiYY85bSTgmQsmKkH60r3SJk6iH0u/Q==} engines: {node: '>=22.0.0'} - '@crawlee/memory-storage@4.0.0-beta.66': - resolution: {integrity: sha512-iEcbpCRqjErmm2EFKuNhWEjNndR2JJN8bP82vwXlFaKPsR4yGywP/XSCG3nqQrg+YCnUOmGjA1R3db9C83vk6A==} + '@crawlee/linkedom@4.0.0-beta.71': + resolution: {integrity: sha512-cabASgJsWEggLHdbOAlxfsc9Yvuo25R5eMbN1xe35jOJl9W637JtutSzllfT4xA1/hMZlsG0L7lO3aap0eww8Q==} engines: {node: '>=22.0.0'} - '@crawlee/playwright@4.0.0-beta.66': - resolution: {integrity: sha512-rZaUascscjOGPaIpw9eNG/AmVHSPlL14pw9haNi6XmcT2cSvJvE1hosqqyT8NrfoAn434XAjwh7eAprehhCXyw==} + '@crawlee/playwright@4.0.0-beta.71': + resolution: {integrity: sha512-t80a/+q2YA26ulAA1kG7pjhgiFUTsZJ422RM1b/XEQdO7XKznZw/n5FS1OVWUBZEyUd1cL/vhrtibQP2BRi0HA==} engines: {node: '>=22.0.0'} peerDependencies: idcac-playwright: ^0.2.0 @@ -1153,8 +1179,8 @@ packages: playwright: optional: true - '@crawlee/puppeteer@4.0.0-beta.66': - resolution: {integrity: sha512-fFlu86g/On7L7GX1tnA3Joz/gKBnwI9rjcH8pvD23Wfo6Gcy7jQPduuNk5rjPV2mZWIxG2M+thiScm/S3+IYdg==} + '@crawlee/puppeteer@4.0.0-beta.71': + resolution: {integrity: sha512-cuw+ft0b1sXOfOAUm4/czulbQbYLE3MEbNkrKcYwDbNa5TA8e9pcyhrUy/XYPYG6na4jOdGbb3leHCOZgQS7qg==} engines: {node: '>=22.0.0'} peerDependencies: idcac-playwright: ^0.2.0 @@ -1165,20 +1191,20 @@ packages: puppeteer: optional: true - '@crawlee/templates@4.0.0-beta.66': - resolution: {integrity: sha512-efOO/f749Z6teELNeFq5iL79qSBJY9s/LaFXkRCaiHWVpgQU7yHP9TMSHAdAQS38czvRR2dmyy2BdbOhz6YY5Q==} + '@crawlee/templates@4.0.0-beta.71': + resolution: {integrity: sha512-4W5juEPi7GcY3KmFzx/JvKI8L+OMa9YNHHpeJb1QOTtNJ3gQl3PY/SNj9kE2V5qL+Tp176CpoMxui0euVB7UuQ==} engines: {node: '>=22.0.0'} '@crawlee/types@3.16.0': resolution: {integrity: sha512-CcIM+JDVx4gzQzMPl+9RJiEeqdzTrx2RLPA7y4IMJSyfZm3J/VrEunielKA3NQrk095j9OuvS/rQL2y8mBV1qw==} engines: {node: '>=16.0.0'} - '@crawlee/types@4.0.0-beta.66': - resolution: {integrity: sha512-holROsQQpWAwQ6DyvgjsaegSnoQZstnyNvy7DfEDeiu7S8i2jsgrkeQnF5ZT6fxDHsDwzwNpbVlo5O+CI+pf9Q==} + '@crawlee/types@4.0.0-beta.71': + resolution: {integrity: sha512-fgU2xiwjOoSEWDhDY/n7jVhAvlp+3qFVNV0ip4SGHNqMFCgjUYdJVXzRgsH50+jr1N/iYT02CfMkqJM60idczg==} engines: {node: '>=22.0.0'} - '@crawlee/utils@4.0.0-beta.66': - resolution: {integrity: sha512-MJMg2Vf0a27YsY53Mp+cX0NpzFyyjHRoqBIJrxt0kPvvLJB6t49l85VjUOYKTTPScFW/4KmFQBKZ2dvEWuejnw==} + '@crawlee/utils@4.0.0-beta.71': + resolution: {integrity: sha512-LzMahqRvsLePo6C2iUXaf2VoT9otjmT/VTjTRPHKziS/tmykdx5FfVwJb1qPA/8W/GQLnRpOmbnulS25iz79eQ==} engines: {node: '>=22.0.0'} '@csstools/cascade-layer-name-parser@2.0.5': @@ -2538,9 +2564,6 @@ packages: peerDependencies: tslib: '2' - '@keyv/serialize@1.1.1': - resolution: {integrity: sha512-dXn3FZhPv0US+7dtJsIi2R+c7qWYiReoEh5zUntWCf4oSpMNib8FDhSoed6m3QyZdx5hK7iLFkYk3rNxwt8vTA==} - '@leichtgewicht/ip-codec@2.0.5': resolution: {integrity: sha512-Vo+PSpZG2/fmgmiNzYK9qWRh8h/CHrwD0mo1h1DzL4yzHNSfWYujGTYsWGreD000gcgmZ7K4Ys6Tx9TxtsKdDw==} @@ -3305,9 +3328,6 @@ packages: resolution: {integrity: sha512-d9dUmWVA7MMiKobL3VpLF8P2aeanRTu6ypG2OIaEv/ZHH/SUQ2iHOVyi5wAPjQ+HmnMuL0whK9ez8I/raWbtIg==} engines: {node: '>=v16'} - '@sec-ant/readable-stream@0.4.1': - resolution: {integrity: sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==} - '@shikijs/core@1.29.2': resolution: {integrity: sha512-vju0lY9r27jJfOY4Z7+Rt/nIOjzJpZ3y+nYpqtUZInVoXQ/TJZcfGnNOGnKjFdVZb8qexiCuSlZRKcGfhhTTZQ==} @@ -3367,10 +3387,6 @@ packages: resolution: {integrity: sha512-FX4MfcifwJyFOI2lPoX7PQxCqx8BG1HCho7WdiXwpEQx1Ycij0JxkfYtGK7yqNScrZGSlt6RE6sw8QYoH7eKnQ==} engines: {node: '>=16'} - '@sindresorhus/is@7.2.0': - resolution: {integrity: sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==} - engines: {node: '>=18'} - '@sindresorhus/merge-streams@4.0.0': resolution: {integrity: sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==} engines: {node: '>=18'} @@ -4222,10 +4238,6 @@ packages: resolution: {integrity: sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q==} engines: {node: '>=18'} - byte-counter@0.1.0: - resolution: {integrity: sha512-jheRLVMeUKrDBjVw2O5+k4EvR4t9wtxHL+bo/LxfkxsVeuGMy3a5SEGgXdAFA4FSzTrU8rQXQIrsZ3oBq5a0pQ==} - engines: {node: '>=20'} - bytes@3.0.0: resolution: {integrity: sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==} engines: {node: '>= 0.8'} @@ -4246,10 +4258,6 @@ packages: resolution: {integrity: sha512-zkDT5WAF4hSSoUgyfg5tFIxz8XQK+25W/TLVojJTMKBaxevLBBtLxgqguAuVQB8PVW79FVjHcU+GJ9tVbDZ9mQ==} engines: {node: '>=14.16'} - cacheable-request@13.0.18: - resolution: {integrity: sha512-rFWadDRKJs3s2eYdXlGggnBZKG7MTblkFBB0YllFds+UYnfogDp2wcR6JN97FhRkHTvq59n2vhNoHNZn29dh/Q==} - engines: {node: '>=18'} - call-bind-apply-helpers@1.0.2: resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} engines: {node: '>= 0.4'} @@ -4571,8 +4579,8 @@ packages: typescript: optional: true - crawlee@4.0.0-beta.66: - resolution: {integrity: sha512-ebhJOZKw9VDVGyt1Pslm0VwcJo382KNg0TL0KbRmej9AfobOwDq2MdVuPnIz0g2Bcm96whYaB9OLt7zW4KsI1A==} + crawlee@4.0.0-beta.71: + resolution: {integrity: sha512-I3K/RzCByRaDBeaaSDkgaTJphK+S4tLDOxV/JYGth5+K/xlIoogvbhnQ4/sTGPoR3OJwu0KbXmmNPTSFbvwkEw==} engines: {node: '>=22.0.0'} hasBin: true peerDependencies: @@ -4792,10 +4800,6 @@ packages: resolution: {integrity: sha512-+8VxcR21HhTy8nOt6jf20w0c9CADrw1O8d+VZ/YzzCt4bJ3uBjw+D1q2osAB8RnpwwaeYBxy0HyKQxD5JBMuuQ==} engines: {node: '>=14.16'} - decompress-response@10.0.0: - resolution: {integrity: sha512-oj7KWToJuuxlPr7VV0vabvxEIiqNMo+q0NueIiL3XhtwC6FVOX7Hr1c0C4eD0bmf7Zr+S/dSf2xvkH3Ad6sU3Q==} - engines: {node: '>=20'} - decompress-response@6.0.0: resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} engines: {node: '>=10'} @@ -4933,10 +4937,6 @@ packages: resolution: {integrity: sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA==} engines: {node: '>=10'} - dot-prop@7.2.0: - resolution: {integrity: sha512-Ol/IPXUARn9CSbkrdV4VJo7uCy1I3VuSiWCaFSg+8BdUOzF9n3jefIpcgAydvUZbTdEBZs2vEiTiS9m61ssiDA==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - dot-prop@8.0.2: resolution: {integrity: sha512-xaBe6ZT4DHPkg0k4Ytbvn5xoxgpG0jOS1dYxSOwAHPuNLjP3/OzN0gH55SrLqpx8cBfSaVt91lXYkApjb+nYdQ==} engines: {node: '>=16'} @@ -5292,10 +5292,6 @@ packages: resolution: {integrity: sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw==} engines: {node: '>= 14.17'} - form-data-encoder@4.1.0: - resolution: {integrity: sha512-G6NsmEW15s0Uw9XnCg+33H3ViYRyiM0hMrMhhqQOR8NFc5GhYrI+6I3u7OTw7b91J2g8rtvMBZJDbcGb2YUniw==} - engines: {node: '>= 18'} - form-data@4.0.5: resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} engines: {node: '>= 6'} @@ -5369,10 +5365,6 @@ packages: resolution: {integrity: sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==} engines: {node: '>=10'} - get-stream@9.0.1: - resolution: {integrity: sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA==} - engines: {node: '>=18'} - get-tsconfig@4.13.7: resolution: {integrity: sha512-7tN6rFgBlMgpBML5j8typ92BKFi2sFQvIdpAqLA2beia5avZDrMs0FLZiM5etShWq5irVyGcGMEA1jcDaK7A/Q==} @@ -5439,18 +5431,10 @@ packages: resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} engines: {node: '>= 0.4'} - got-scraping@4.2.1: - resolution: {integrity: sha512-rhOlO1L4H4Cm31smHJqPtAaXOUrhSKsiTrbZSHKFQW1E/mkTDopnHHpRnXJpqzE0faj+zPsVQnyifIqO+K+cLQ==} - engines: {node: '>=16'} - got@12.6.1: resolution: {integrity: sha512-mThBblvlAF1d4O5oqyvN+ZxLAYwIJK7bpMxgYqPD9okW0C3qm5FFn7k811QrcuEBwaogR3ngOFoCfs6mRv7teQ==} engines: {node: '>=14.16'} - got@14.6.6: - resolution: {integrity: sha512-QLV1qeYSo5l13mQzWgP/y0LbMr5Plr5fJilgAIwgnwseproEbtNym8xpLsDzeZ6MWXgNE6kdWGBjdh3zT/Qerg==} - engines: {node: '>=20'} - graceful-fs@4.2.10: resolution: {integrity: sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==} @@ -5731,6 +5715,62 @@ packages: engines: {node: '>=16.x'} hasBin: true + impit-darwin-arm64@0.14.3: + resolution: {integrity: sha512-kMoQB+CR+a954pnhe4kf1O2RyJCsqGRE95jIXfgqNOiIMS5O1z0cOMzG/sohJk/nodZVSJ5VdWEV4BKhRWPFSA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + impit-darwin-x64@0.14.3: + resolution: {integrity: sha512-ft9+kjz1pR8H5xbbf5U1EgwaaIea5iXHxBf2Q3NoinPpiV7KgyzYSr83pCSrAY6evWk+smG9shHgzhQtgbj1Rg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + impit-linux-arm64-gnu@0.14.3: + resolution: {integrity: sha512-+mwta93S6Ndfpca3DDblvrqV1g8Bg6gZFOlEiJHfGJZpiUIQE+A/Pjg2e+inV+WrTtlnLsvff8lgwFvkIGnRug==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + impit-linux-arm64-musl@0.14.3: + resolution: {integrity: sha512-wYhfH1J8laWkpSN2SAFCKx9npY4vXrk23ex+tOzTGf2xBBqIhpMUff2wa6dOFb0or6EhZHBssoRf0HK3h/htXQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [musl] + + impit-linux-x64-gnu@0.14.3: + resolution: {integrity: sha512-hAYnutJDGQO5bPvTPKBBFkHgbB7QL7QLDse8c4s21VSzC/EqGliSS3UqR5ZUUwaFdm3t8DJsXOr+gboAUwDR7w==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [glibc] + + impit-linux-x64-musl@0.14.3: + resolution: {integrity: sha512-wXBeHiqCjuyqxg4u5eY4OBNVzsS4Karz/ms/0mjB8aqVZLNER1NaUTeE4J3eZCdytFaJRaAeQvKTH8HfjpPsQg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [musl] + + impit-win32-arm64-msvc@0.14.3: + resolution: {integrity: sha512-XiF8MYpm4tF8TMbtsWrcEDDs6NxRNeirfiHM/AyKcr2jiN34hZwbZDM0EseD30PDwb2Pny2JWB+zTxpE1P5CZg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + + impit-win32-x64-msvc@0.14.3: + resolution: {integrity: sha512-HIaLSpU5SGVx3UDH/R3ZaGMURVgyRVjmalqXj65ABxVUqfXiBo1t5ZObDPPjqvLT3klPnrya1+xF1DQ4dfT+og==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + impit@0.14.3: + resolution: {integrity: sha512-SbCLoeW0YDRox5kQoy71jtpjZE+BwjWO411OdgfbLMbNmSX79s5Y0Y2vCE7AO6zQSq9F4Fu8SJTCKnp8rXQzpg==} + engines: {node: '>= 20'} + import-fresh@3.3.1: resolution: {integrity: sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==} engines: {node: '>=6'} @@ -5912,10 +5952,6 @@ packages: resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} engines: {node: '>=8'} - is-stream@4.0.1: - resolution: {integrity: sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==} - engines: {node: '>=18'} - is-typedarray@1.0.0: resolution: {integrity: sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==} @@ -6018,9 +6054,6 @@ packages: keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} - keyv@5.6.0: - resolution: {integrity: sha512-CYDD3SOtsHtyXeEORYRx2qBtpDJFjRTGXUtmNEMGyzYOKj1TE3tycdlho7kA1Ufx9OYWZzg52QFBGALTirzDSw==} - kind-of@6.0.3: resolution: {integrity: sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==} engines: {node: '>=0.10.0'} @@ -6708,10 +6741,6 @@ packages: resolution: {integrity: sha512-dD4UpyBh/9m4X2NVjA+73/ZPBRF+uF4zIMFvvQsabMiEK8x41L3rQ8EENOi35kyyoaJwNxEeJcP6Fj1H4U409Q==} engines: {node: '>=12'} - ow@1.1.1: - resolution: {integrity: sha512-sJBRCbS5vh1Jp9EOgwp1Ws3c16lJrUkJYlvWTYC03oyiYVwS/ns7lKRWow4w4XjDyTrA2pplQv4B2naWSR6yDA==} - engines: {node: '>=14.16'} - ow@2.0.0: resolution: {integrity: sha512-ESUigmGrdhUZ2nQSFNkeKSl6ZRPupXzprMs3yF9DYlNVpJ8XAjM/fI9RUZxA7PI1K9HQDCCvBo1jr/GEIo9joQ==} engines: {node: '>=18'} @@ -6750,10 +6779,6 @@ packages: resolution: {integrity: sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==} engines: {node: '>=12.20'} - p-cancelable@4.0.1: - resolution: {integrity: sha512-wBowNApzd45EIKdO1LaU+LrMBwAcjfPaYtVzV3lmfM3gf8Z4CHZsiIqlM8TZZ8okYvh5A1cP6gTfCRQtwUpaUg==} - engines: {node: '>=14.16'} - p-event@6.0.1: resolution: {integrity: sha512-Q6Bekk5wpzW5qIyUP4gdMEujObYstZl6DMMOSenwBvV0BlE5LkDwkjs5yHbZmdCEq2o4RJx4tE1vwxFVf2FG1w==} engines: {node: '>=16.17'} @@ -7493,9 +7518,6 @@ packages: prop-types@15.8.1: resolution: {integrity: sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==} - proper-lockfile@4.1.2: - resolution: {integrity: sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==} - property-information@7.1.0: resolution: {integrity: sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ==} @@ -7840,18 +7862,10 @@ packages: resolution: {integrity: sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg==} engines: {node: '>=14.16'} - responselike@4.0.2: - resolution: {integrity: sha512-cGk8IbWEAnaCpdAt1BHzJ3Ahz5ewDJa0KseTsE3qIRMJ3C698W8psM7byCeWVpd/Ha7FUYzuRVzXoKoM6nRUbA==} - engines: {node: '>=20'} - restore-cursor@5.1.0: resolution: {integrity: sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==} engines: {node: '>=18'} - retry@0.12.0: - resolution: {integrity: sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==} - engines: {node: '>= 4'} - retry@0.13.1: resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} engines: {node: '>= 4'} @@ -10253,28 +10267,30 @@ snapshots: optionalDependencies: conventional-commits-parser: 6.4.0 - '@crawlee/basic@4.0.0-beta.66': + '@crawlee/basic@4.0.0-beta.71': dependencies: '@apify/timeout': 0.3.2 '@apify/utilities': 2.26.0 - '@crawlee/core': 4.0.0-beta.66 - '@crawlee/got-scraping-client': 4.0.0-beta.66 - '@crawlee/types': 4.0.0-beta.66 - '@crawlee/utils': 4.0.0-beta.66 + '@crawlee/core': 4.0.0-beta.71 + '@crawlee/http-client': 4.0.0-beta.71 + '@crawlee/types': 4.0.0-beta.71 + '@crawlee/utils': 4.0.0-beta.71 csv-stringify: 6.7.0 fs-extra: 11.3.4 ow: 2.0.0 tldts: 7.0.28 tslib: 2.8.1 type-fest: 4.41.0 + optionalDependencies: + '@crawlee/impit-client': 4.0.0-beta.71 transitivePeerDependencies: - supports-color - '@crawlee/browser-pool@4.0.0-beta.66(playwright@1.60.0)(puppeteer@25.1.0)': + '@crawlee/browser-pool@4.0.0-beta.71(playwright@1.60.0)(puppeteer@25.1.0)': dependencies: '@apify/timeout': 0.3.2 - '@crawlee/core': 4.0.0-beta.66 - '@crawlee/types': 4.0.0-beta.66 + '@crawlee/core': 4.0.0-beta.71 + '@crawlee/types': 4.0.0-beta.71 fingerprint-generator: 2.1.82 fingerprint-injector: 2.1.82(playwright@1.60.0)(puppeteer@25.1.0) lodash.merge: 4.6.2 @@ -10291,13 +10307,13 @@ snapshots: transitivePeerDependencies: - supports-color - '@crawlee/browser@4.0.0-beta.66(playwright@1.60.0)(puppeteer@25.1.0)': + '@crawlee/browser@4.0.0-beta.71(playwright@1.60.0)(puppeteer@25.1.0)': dependencies: '@apify/timeout': 0.3.2 - '@crawlee/basic': 4.0.0-beta.66 - '@crawlee/browser-pool': 4.0.0-beta.66(playwright@1.60.0)(puppeteer@25.1.0) - '@crawlee/types': 4.0.0-beta.66 - '@crawlee/utils': 4.0.0-beta.66 + '@crawlee/basic': 4.0.0-beta.71 + '@crawlee/browser-pool': 4.0.0-beta.71(playwright@1.60.0)(puppeteer@25.1.0) + '@crawlee/types': 4.0.0-beta.71 + '@crawlee/utils': 4.0.0-beta.71 ow: 2.0.0 tslib: 2.8.1 type-fest: 4.41.0 @@ -10307,20 +10323,20 @@ snapshots: transitivePeerDependencies: - supports-color - '@crawlee/cheerio@4.0.0-beta.66': + '@crawlee/cheerio@4.0.0-beta.71': dependencies: - '@crawlee/http': 4.0.0-beta.66 - '@crawlee/types': 4.0.0-beta.66 - '@crawlee/utils': 4.0.0-beta.66 + '@crawlee/http': 4.0.0-beta.71 + '@crawlee/types': 4.0.0-beta.71 + '@crawlee/utils': 4.0.0-beta.71 cheerio: 1.2.0 htmlparser2: 10.1.0 tslib: 2.8.1 transitivePeerDependencies: - supports-color - '@crawlee/cli@4.0.0-beta.66(@types/node@24.12.2)': + '@crawlee/cli@4.0.0-beta.71(@types/node@24.12.2)': dependencies: - '@crawlee/templates': 4.0.0-beta.66 + '@crawlee/templates': 4.0.0-beta.71 '@inquirer/prompts': 7.10.1(@types/node@24.12.2) ansi-colors: 4.1.3 fs-extra: 11.3.4 @@ -10329,7 +10345,7 @@ snapshots: transitivePeerDependencies: - '@types/node' - '@crawlee/core@4.0.0-beta.66': + '@crawlee/core@4.0.0-beta.71': dependencies: '@apify/consts': 2.52.0 '@apify/datastructures': 2.0.3 @@ -10337,13 +10353,16 @@ snapshots: '@apify/pseudo_url': 2.0.75 '@apify/timeout': 0.3.2 '@apify/utilities': 2.26.0 - '@crawlee/memory-storage': 4.0.0-beta.66 - '@crawlee/types': 4.0.0-beta.66 - '@crawlee/utils': 4.0.0-beta.66 + '@crawlee/fs-storage': 4.0.0-beta.71 + '@crawlee/types': 4.0.0-beta.71 + '@crawlee/utils': 4.0.0-beta.71 '@sapphire/async-queue': 1.5.5 + '@sapphire/shapeshift': 4.0.0 '@vladfrangu/async_event_emitter': 2.4.7 + content-type: 1.0.5 csv-stringify: 6.7.0 json5: 2.2.3 + mime-types: 3.0.2 minimatch: 10.2.5 ow: 2.0.0 stream-json: 1.9.1 @@ -10355,25 +10374,45 @@ snapshots: transitivePeerDependencies: - supports-color - '@crawlee/got-scraping-client@4.0.0-beta.66': + '@crawlee/fs-storage-native-darwin-arm64@0.1.5-beta.18': + optional: true + + '@crawlee/fs-storage-native-darwin-x64@0.1.5-beta.18': + optional: true + + '@crawlee/fs-storage-native-linux-x64-gnu@0.1.5-beta.18': + optional: true + + '@crawlee/fs-storage-native-win32-x64-msvc@0.1.5-beta.18': + optional: true + + '@crawlee/fs-storage-native@0.1.5-beta.18': + optionalDependencies: + '@crawlee/fs-storage-native-darwin-arm64': 0.1.5-beta.18 + '@crawlee/fs-storage-native-darwin-x64': 0.1.5-beta.18 + '@crawlee/fs-storage-native-linux-x64-gnu': 0.1.5-beta.18 + '@crawlee/fs-storage-native-win32-x64-msvc': 0.1.5-beta.18 + + '@crawlee/fs-storage@4.0.0-beta.71': dependencies: - '@crawlee/http-client': 4.0.0-beta.66 - got-scraping: 4.2.1 + '@crawlee/fs-storage-native': 0.1.5-beta.18 + '@crawlee/types': 4.0.0-beta.71 + '@sapphire/shapeshift': 4.0.0 - '@crawlee/http-client@4.0.0-beta.66': + '@crawlee/http-client@4.0.0-beta.71': dependencies: - '@crawlee/types': 4.0.0-beta.66 + '@crawlee/types': 4.0.0-beta.71 tough-cookie: 6.0.1 - '@crawlee/http@4.0.0-beta.66': + '@crawlee/http@4.0.0-beta.71': dependencies: '@apify/timeout': 0.3.2 '@apify/utilities': 2.26.0 - '@crawlee/basic': 4.0.0-beta.66 - '@crawlee/core': 4.0.0-beta.66 - '@crawlee/http-client': 4.0.0-beta.66 - '@crawlee/types': 4.0.0-beta.66 - '@crawlee/utils': 4.0.0-beta.66 + '@crawlee/basic': 4.0.0-beta.71 + '@crawlee/core': 4.0.0-beta.71 + '@crawlee/http-client': 4.0.0-beta.71 + '@crawlee/types': 4.0.0-beta.71 + '@crawlee/utils': 4.0.0-beta.71 '@types/content-type': 1.1.9 cheerio: 1.2.0 content-type: 1.0.5 @@ -10385,13 +10424,21 @@ snapshots: transitivePeerDependencies: - supports-color - '@crawlee/jsdom@4.0.0-beta.66': + '@crawlee/impit-client@4.0.0-beta.71': + dependencies: + '@apify/datastructures': 2.0.3 + '@crawlee/http-client': 4.0.0-beta.71 + '@crawlee/types': 4.0.0-beta.71 + impit: 0.14.3 + tough-cookie: 6.0.1 + + '@crawlee/jsdom@4.0.0-beta.71': dependencies: '@apify/timeout': 0.3.2 '@apify/utilities': 2.26.0 - '@crawlee/http': 4.0.0-beta.66 - '@crawlee/types': 4.0.0-beta.66 - '@crawlee/utils': 4.0.0-beta.66 + '@crawlee/http': 4.0.0-beta.71 + '@crawlee/types': 4.0.0-beta.71 + '@crawlee/utils': 4.0.0-beta.71 '@types/jsdom': 21.1.7 cheerio: 1.2.0 jsdom: 26.1.0 @@ -10403,13 +10450,13 @@ snapshots: - supports-color - utf-8-validate - '@crawlee/linkedom@4.0.0-beta.66': + '@crawlee/linkedom@4.0.0-beta.71': dependencies: '@apify/timeout': 0.3.2 '@apify/utilities': 2.26.0 - '@crawlee/http': 4.0.0-beta.66 - '@crawlee/types': 4.0.0-beta.66 - '@crawlee/utils': 4.0.0-beta.66 + '@crawlee/http': 4.0.0-beta.71 + '@crawlee/types': 4.0.0-beta.71 + '@crawlee/utils': 4.0.0-beta.71 cheerio: 1.2.0 linkedom: 0.18.12 ow: 2.0.0 @@ -10418,30 +10465,17 @@ snapshots: - canvas - supports-color - '@crawlee/memory-storage@4.0.0-beta.66': - dependencies: - '@crawlee/types': 4.0.0-beta.66 - '@sapphire/async-queue': 1.5.5 - '@sapphire/shapeshift': 4.0.0 - content-type: 1.0.5 - fs-extra: 11.3.4 - json5: 2.2.3 - mime-types: 3.0.2 - p-limit: 6.2.0 - proper-lockfile: 4.1.2 - tslib: 2.8.1 - - '@crawlee/playwright@4.0.0-beta.66(playwright@1.60.0)(puppeteer@25.1.0)': + '@crawlee/playwright@4.0.0-beta.71(playwright@1.60.0)(puppeteer@25.1.0)': dependencies: '@apify/datastructures': 2.0.3 '@apify/timeout': 0.3.2 - '@crawlee/basic': 4.0.0-beta.66 - '@crawlee/browser': 4.0.0-beta.66(playwright@1.60.0)(puppeteer@25.1.0) - '@crawlee/browser-pool': 4.0.0-beta.66(playwright@1.60.0)(puppeteer@25.1.0) - '@crawlee/cheerio': 4.0.0-beta.66 - '@crawlee/core': 4.0.0-beta.66 - '@crawlee/types': 4.0.0-beta.66 - '@crawlee/utils': 4.0.0-beta.66 + '@crawlee/basic': 4.0.0-beta.71 + '@crawlee/browser': 4.0.0-beta.71(playwright@1.60.0)(puppeteer@25.1.0) + '@crawlee/browser-pool': 4.0.0-beta.71(playwright@1.60.0)(puppeteer@25.1.0) + '@crawlee/cheerio': 4.0.0-beta.71 + '@crawlee/core': 4.0.0-beta.71 + '@crawlee/types': 4.0.0-beta.71 + '@crawlee/utils': 4.0.0-beta.71 cheerio: 1.2.0 jquery: 3.7.1 ml-logistic-regression: 2.0.0 @@ -10455,14 +10489,14 @@ snapshots: - puppeteer - supports-color - '@crawlee/puppeteer@4.0.0-beta.66(playwright@1.60.0)(puppeteer@25.1.0)': + '@crawlee/puppeteer@4.0.0-beta.71(playwright@1.60.0)(puppeteer@25.1.0)': dependencies: '@apify/datastructures': 2.0.3 - '@crawlee/browser': 4.0.0-beta.66(playwright@1.60.0)(puppeteer@25.1.0) - '@crawlee/browser-pool': 4.0.0-beta.66(playwright@1.60.0)(puppeteer@25.1.0) - '@crawlee/core': 4.0.0-beta.66 - '@crawlee/types': 4.0.0-beta.66 - '@crawlee/utils': 4.0.0-beta.66 + '@crawlee/browser': 4.0.0-beta.71(playwright@1.60.0)(puppeteer@25.1.0) + '@crawlee/browser-pool': 4.0.0-beta.71(playwright@1.60.0)(puppeteer@25.1.0) + '@crawlee/core': 4.0.0-beta.71 + '@crawlee/types': 4.0.0-beta.71 + '@crawlee/utils': 4.0.0-beta.71 cheerio: 1.2.0 devtools-protocol: 0.0.1624250 jquery: 3.7.1 @@ -10474,7 +10508,7 @@ snapshots: - playwright - supports-color - '@crawlee/templates@4.0.0-beta.66': + '@crawlee/templates@4.0.0-beta.71': dependencies: tslib: 2.8.1 @@ -10482,16 +10516,16 @@ snapshots: dependencies: tslib: 2.8.1 - '@crawlee/types@4.0.0-beta.66': + '@crawlee/types@4.0.0-beta.71': dependencies: tough-cookie: 6.0.1 tslib: 2.8.1 - '@crawlee/utils@4.0.0-beta.66': + '@crawlee/utils@4.0.0-beta.71': dependencies: '@apify/ps-tree': 1.2.0 - '@crawlee/http-client': 4.0.0-beta.66 - '@crawlee/types': 4.0.0-beta.66 + '@crawlee/http-client': 4.0.0-beta.71 + '@crawlee/types': 4.0.0-beta.71 '@types/sax': 1.2.7 cheerio: 1.2.0 domhandler: 5.0.3 @@ -12446,8 +12480,6 @@ snapshots: '@jsonjoy.com/codegen': 17.67.0(tslib@2.8.1) tslib: 2.8.1 - '@keyv/serialize@1.1.1': {} - '@leichtgewicht/ip-codec@2.0.5': {} '@mdx-js/mdx@3.1.1': @@ -13049,8 +13081,6 @@ snapshots: fast-deep-equal: 3.1.3 lodash: 4.18.1 - '@sec-ant/readable-stream@0.4.1': {} - '@shikijs/core@1.29.2': dependencies: '@shikijs/engine-javascript': 1.29.2 @@ -13126,8 +13156,6 @@ snapshots: '@sindresorhus/is@6.3.1': {} - '@sindresorhus/is@7.2.0': {} - '@sindresorhus/merge-streams@4.0.0': {} '@slorber/react-helmet-async@1.3.0(react-dom@19.2.5(react@19.2.5))(react@19.2.5)': @@ -14070,8 +14098,6 @@ snapshots: dependencies: run-applescript: 7.1.0 - byte-counter@0.1.0: {} - bytes@3.0.0: {} bytes@3.1.2: {} @@ -14090,16 +14116,6 @@ snapshots: normalize-url: 8.1.1 responselike: 3.0.0 - cacheable-request@13.0.18: - dependencies: - '@types/http-cache-semantics': 4.2.0 - get-stream: 9.0.1 - http-cache-semantics: 4.2.0 - keyv: 5.6.0 - mimic-response: 4.0.0 - normalize-url: 8.1.1 - responselike: 4.0.2 - call-bind-apply-helpers@1.0.2: dependencies: es-errors: 1.3.0 @@ -14430,20 +14446,22 @@ snapshots: optionalDependencies: typescript: 6.0.3 - crawlee@4.0.0-beta.66(@types/node@24.12.2)(playwright@1.60.0)(puppeteer@25.1.0): - dependencies: - '@crawlee/basic': 4.0.0-beta.66 - '@crawlee/browser': 4.0.0-beta.66(playwright@1.60.0)(puppeteer@25.1.0) - '@crawlee/browser-pool': 4.0.0-beta.66(playwright@1.60.0)(puppeteer@25.1.0) - '@crawlee/cheerio': 4.0.0-beta.66 - '@crawlee/cli': 4.0.0-beta.66(@types/node@24.12.2) - '@crawlee/core': 4.0.0-beta.66 - '@crawlee/http': 4.0.0-beta.66 - '@crawlee/jsdom': 4.0.0-beta.66 - '@crawlee/linkedom': 4.0.0-beta.66 - '@crawlee/playwright': 4.0.0-beta.66(playwright@1.60.0)(puppeteer@25.1.0) - '@crawlee/puppeteer': 4.0.0-beta.66(playwright@1.60.0)(puppeteer@25.1.0) - '@crawlee/utils': 4.0.0-beta.66 + crawlee@4.0.0-beta.71(@types/node@24.12.2)(playwright@1.60.0)(puppeteer@25.1.0): + dependencies: + '@crawlee/basic': 4.0.0-beta.71 + '@crawlee/browser': 4.0.0-beta.71(playwright@1.60.0)(puppeteer@25.1.0) + '@crawlee/browser-pool': 4.0.0-beta.71(playwright@1.60.0)(puppeteer@25.1.0) + '@crawlee/cheerio': 4.0.0-beta.71 + '@crawlee/cli': 4.0.0-beta.71(@types/node@24.12.2) + '@crawlee/core': 4.0.0-beta.71 + '@crawlee/fs-storage': 4.0.0-beta.71 + '@crawlee/http': 4.0.0-beta.71 + '@crawlee/impit-client': 4.0.0-beta.71 + '@crawlee/jsdom': 4.0.0-beta.71 + '@crawlee/linkedom': 4.0.0-beta.71 + '@crawlee/playwright': 4.0.0-beta.71(playwright@1.60.0)(puppeteer@25.1.0) + '@crawlee/puppeteer': 4.0.0-beta.71(playwright@1.60.0)(puppeteer@25.1.0) + '@crawlee/utils': 4.0.0-beta.71 import-local: 3.2.0 tslib: 2.8.1 optionalDependencies: @@ -14670,10 +14688,6 @@ snapshots: decode-uri-component@0.4.1: {} - decompress-response@10.0.0: - dependencies: - mimic-response: 4.0.0 - decompress-response@6.0.0: dependencies: mimic-response: 3.1.0 @@ -14808,10 +14822,6 @@ snapshots: dependencies: is-obj: 2.0.0 - dot-prop@7.2.0: - dependencies: - type-fest: 2.19.0 - dot-prop@8.0.2: dependencies: type-fest: 3.13.1 @@ -15206,8 +15216,6 @@ snapshots: form-data-encoder@2.1.4: {} - form-data-encoder@4.1.0: {} - form-data@4.0.5: dependencies: asynckit: 0.4.0 @@ -15275,11 +15283,6 @@ snapshots: get-stream@6.0.1: {} - get-stream@9.0.1: - dependencies: - '@sec-ant/readable-stream': 0.4.1 - is-stream: 4.0.1 - get-tsconfig@4.13.7: dependencies: resolve-pkg-maps: 1.0.0 @@ -15362,16 +15365,6 @@ snapshots: gopd@1.2.0: {} - got-scraping@4.2.1: - dependencies: - got: 14.6.6 - header-generator: 2.1.82 - http2-wrapper: 2.2.1 - mimic-response: 4.0.0 - ow: 1.1.1 - quick-lru: 7.3.0 - tslib: 2.8.1 - got@12.6.1: dependencies: '@sindresorhus/is': 5.6.0 @@ -15386,21 +15379,6 @@ snapshots: p-cancelable: 3.0.0 responselike: 3.0.0 - got@14.6.6: - dependencies: - '@sindresorhus/is': 7.2.0 - byte-counter: 0.1.0 - cacheable-lookup: 7.0.0 - cacheable-request: 13.0.18 - decompress-response: 10.0.0 - form-data-encoder: 4.1.0 - http2-wrapper: 2.2.1 - keyv: 5.6.0 - lowercase-keys: 3.0.0 - p-cancelable: 4.0.1 - responselike: 4.0.2 - type-fest: 4.41.0 - graceful-fs@4.2.10: {} graceful-fs@4.2.11: {} @@ -15840,6 +15818,41 @@ snapshots: image-size@2.0.2: {} + impit-darwin-arm64@0.14.3: + optional: true + + impit-darwin-x64@0.14.3: + optional: true + + impit-linux-arm64-gnu@0.14.3: + optional: true + + impit-linux-arm64-musl@0.14.3: + optional: true + + impit-linux-x64-gnu@0.14.3: + optional: true + + impit-linux-x64-musl@0.14.3: + optional: true + + impit-win32-arm64-msvc@0.14.3: + optional: true + + impit-win32-x64-msvc@0.14.3: + optional: true + + impit@0.14.3: + optionalDependencies: + impit-darwin-arm64: 0.14.3 + impit-darwin-x64: 0.14.3 + impit-linux-arm64-gnu: 0.14.3 + impit-linux-arm64-musl: 0.14.3 + impit-linux-x64-gnu: 0.14.3 + impit-linux-x64-musl: 0.14.3 + impit-win32-arm64-msvc: 0.14.3 + impit-win32-x64-msvc: 0.14.3 + import-fresh@3.3.1: dependencies: parent-module: 1.0.1 @@ -15965,8 +15978,6 @@ snapshots: is-stream@2.0.1: {} - is-stream@4.0.1: {} - is-typedarray@1.0.0: {} is-wsl@2.2.0: @@ -16083,10 +16094,6 @@ snapshots: dependencies: json-buffer: 3.0.1 - keyv@5.6.0: - dependencies: - '@keyv/serialize': 1.1.1 - kind-of@6.0.3: {} kleur@3.0.3: {} @@ -17023,14 +17030,6 @@ snapshots: lodash.isequal: 4.5.0 vali-date: 1.0.0 - ow@1.1.1: - dependencies: - '@sindresorhus/is': 5.6.0 - callsites: 4.2.0 - dot-prop: 7.2.0 - lodash.isequal: 4.5.0 - vali-date: 1.0.0 - ow@2.0.0: dependencies: '@sindresorhus/is': 6.3.1 @@ -17098,8 +17097,6 @@ snapshots: p-cancelable@3.0.0: {} - p-cancelable@4.0.1: {} - p-event@6.0.1: dependencies: p-timeout: 6.1.4 @@ -17997,12 +17994,6 @@ snapshots: object-assign: 4.1.1 react-is: 16.13.1 - proper-lockfile@4.1.2: - dependencies: - graceful-fs: 4.2.11 - retry: 0.12.0 - signal-exit: 3.0.7 - property-information@7.1.0: {} proto-list@1.2.4: {} @@ -18478,17 +18469,11 @@ snapshots: dependencies: lowercase-keys: 3.0.0 - responselike@4.0.2: - dependencies: - lowercase-keys: 3.0.0 - restore-cursor@5.1.0: dependencies: onetime: 7.0.0 signal-exit: 4.1.0 - retry@0.12.0: {} - retry@0.13.1: {} reusify@1.1.0: {} diff --git a/src/actor.ts b/src/actor.ts index 98b43dbffb..e12c5da0a8 100644 --- a/src/actor.ts +++ b/src/actor.ts @@ -9,7 +9,7 @@ import type { UseStateOptions, } from '@crawlee/core'; import { Dataset, EventType, purgeDefaultStorages, RequestQueue, serviceLocator } from '@crawlee/core'; -import type { Awaitable, Constructor, Dictionary, SetStatusMessageOptions, StorageClient } from '@crawlee/types'; +import type { Awaitable, Constructor, Dictionary, StorageBackend } from '@crawlee/types'; import { sleep, snakeCaseToCamelCase } from '@crawlee/utils'; import type { ActorCallOptions, @@ -54,7 +54,7 @@ import { openStorage } from './storage.js'; import { checkCrawleeVersion, getSystemInfo, isNonEmptyObject, printOutdatedSdkWarning, validate } from './utils.js'; export interface InitOptions { - storage?: StorageClient; + storage?: StorageBackend; /** * Whether to automatically handle platform shutdown signals. * When enabled, `Actor.exit()` is called on `aborting` events and `Actor.reboot()` on `migrating` events. @@ -100,6 +100,13 @@ export interface ExitOptions { export interface MainOptions extends ExitOptions, InitOptions {} +export interface SetStatusMessageOptions { + /** If `true`, the status message is treated as final and won't be overwritten by the platform. */ + isStatusMessageTerminal?: boolean; + /** Log level used when the status message is also logged locally. Defaults to `INFO`. */ + level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR'; +} + /** * Parsed representation of the Apify environment variables. * This object is returned by the {@apilink Actor.getEnv} function. @@ -587,12 +594,12 @@ export class Actor { serviceLocator.setConfiguration(this.config); if (this.isAtHome()) { - serviceLocator.setStorageClient( + serviceLocator.setStorageBackend( new ApifyStorageClient(this.apifyClient, this.config, () => this.chargingManager), ); serviceLocator.setEventManager(this.eventManager); } else if (options.storage) { - serviceLocator.setStorageClient(options.storage); + serviceLocator.setStorageBackend(options.storage); } // Init the event manager the config uses @@ -656,7 +663,7 @@ export class Actor { this._ensureActorInit('exit'); - const client = serviceLocator.getStorageClient(); + const client = serviceLocator.getStorageBackend(); const events = serviceLocator.getEventManager(); // Remove graceful shutdown handlers to prevent them from interfering with exit @@ -1031,27 +1038,14 @@ export class Actor { break; } - const client = serviceLocator.getStorageClient(); - - // just to be sure, this should be fast - await addTimeoutToPromise( - async () => - client.setStatusMessage!(statusMessage, { - isStatusMessageTerminal, - level, - }), - 1000, - 'Setting status message timed out after 1s', - ).catch((e) => log.warning(e.message)); - const runId = this.config.actorRunId!; if (runId) { // just to be sure, this should be fast const run = await addTimeoutToPromise( - async () => this.apifyClient.run(runId).get(), + async () => this.apifyClient.run(runId).update({ statusMessage, isStatusMessageTerminal }), 1000, - 'Getting the current run timed out after 1s', + 'Setting status message timed out after 1s', ).catch((e) => log.warning(e.message)); if (run) { @@ -2206,7 +2200,7 @@ export class Actor { } private usesPushDataInterception(dataset: Dataset): boolean { - return Boolean((dataset.client as any)[USES_PUSH_DATA_INTERCEPTION]); + return Boolean((dataset.backend as any)[USES_PUSH_DATA_INTERCEPTION]); } private async pushDataViaInterceptedClient( diff --git a/src/apify_request_queue_client.ts b/src/apify_request_queue_client.ts index 501527fb39..f174bea2e9 100644 --- a/src/apify_request_queue_client.ts +++ b/src/apify_request_queue_client.ts @@ -3,36 +3,14 @@ import { createHash } from 'node:crypto'; import type { BatchAddRequestsResult, QueueOperationInfo, - RequestOptions, + RequestQueueBackend, RequestQueueInfo, + RequestQueueOperationOptions, RequestSchema, UpdateRequestSchema, } from '@crawlee/types'; import type { RequestQueueClient as ApifyRequestQueueResourceClient } from 'apify-client'; -/** - * Crawlee v4's redesigned (pull-based) `RequestQueueClient` interface. - * - * This is a local copy because the published `@crawlee/types` still ships the - * pre-redesign interface in its `.d.ts` even though `@crawlee/core`'s runtime - * already calls these methods (`addBatchOfRequests`/`fetchNextRequest`/…). Once - * crawlee publishes the regenerated types, drop this and `implements` - * `@crawlee/types`' `RequestQueueClient` directly. - */ -interface RequestQueueClientV4 { - getMetadata(): Promise; - drop(): Promise; - purge(): Promise; - addBatchOfRequests(requests: RequestSchema[], options?: RequestOptions): Promise; - getRequest(uniqueKey: string): Promise; - fetchNextRequest(): Promise; - markRequestAsHandled(request: UpdateRequestSchema): Promise; - reclaimRequest(request: UpdateRequestSchema, options?: RequestOptions): Promise; - isEmpty(): Promise; - isFinished(): Promise; - setExpectedRequestProcessingTimeSecs?(secs: number): void; -} - /** Apify request IDs are the first 15 chars of a base64url-ish SHA-256 of the unique key. */ const REQUEST_ID_LENGTH = 15; @@ -53,7 +31,7 @@ function uniqueKeyToRequestId(uniqueKey: string): string { } /** - * Implements Crawlee v4's stateful, pull-based {@link RequestQueueClient} interface + * Implements Crawlee v4's stateful, pull-based {@link RequestQueueBackend} interface * on top of `apify-client`'s REST request-queue API. * * `fetchNextRequest` locks requests server-side (`listAndLockHead`) so the queue can @@ -63,7 +41,7 @@ function uniqueKeyToRequestId(uniqueKey: string): string { * * @internal */ -export class ApifyRequestQueueClient implements RequestQueueClientV4 { +export class ApifyRequestQueueClient implements RequestQueueBackend { /** Locked request ids waiting to be handed out by {@link fetchNextRequest}, in queue order. */ private readonly queueHead: string[] = []; @@ -89,11 +67,14 @@ export class ApifyRequestQueueClient implements RequestQueueClientV4 { // already fresh, so purge is a no-op here (use drop() to remove it entirely). } - setExpectedRequestProcessingTimeSecs(secs: number): void { + async setExpectedRequestProcessingTimeSecs(secs: number): Promise { this.lockSecs = secs; } - async addBatchOfRequests(requests: RequestSchema[], options: RequestOptions = {}): Promise { + async addBatchOfRequests( + requests: RequestSchema[], + options: RequestQueueOperationOptions = {}, + ): Promise { const { forefront } = options; // The API assigns ids itself; strip any incoming id so its strict input validation passes. const apiRequests = requests.map((request) => { @@ -114,12 +95,12 @@ export class ApifyRequestQueueClient implements RequestQueueClientV4 { return result as unknown as BatchAddRequestsResult; } - async getRequest(uniqueKey: string): Promise { + async getRequest(uniqueKey: string): Promise { const request = await this.client.getRequest(uniqueKeyToRequestId(uniqueKey)); - return (request as RequestOptions | undefined) ?? undefined; + return (request as UpdateRequestSchema | undefined) ?? undefined; } - async fetchNextRequest(): Promise { + async fetchNextRequest(): Promise { await this.ensureHeadIsNonEmpty(); while (this.queueHead.length > 0) { @@ -130,12 +111,12 @@ export class ApifyRequestQueueClient implements RequestQueueClientV4 { if (!request || (request as { handledAt?: string }).handledAt) { continue; } - return request as RequestOptions; + return request as UpdateRequestSchema; } - return null; + return undefined; } - async markRequestAsHandled(request: UpdateRequestSchema): Promise { + async markRequestAsHandled(request: UpdateRequestSchema): Promise { const result = await this.client.updateRequest({ ...this.toApiRequest(request), handledAt: request.handledAt ?? new Date().toISOString(), @@ -146,8 +127,8 @@ export class ApifyRequestQueueClient implements RequestQueueClientV4 { async reclaimRequest( request: UpdateRequestSchema, - options: RequestOptions = {}, - ): Promise { + options: RequestQueueOperationOptions = {}, + ): Promise { const { forefront } = options; const result = await this.client.updateRequest(this.toApiRequest(request), { forefront }); @@ -188,7 +169,7 @@ export class ApifyRequestQueueClient implements RequestQueueClientV4 { } } - /** Crawlee's `RequestSchema`/`UpdateRequestSchema` is structurally the apify-client request shape. */ + /** Crawlee's `UpdateRequestSchema` is structurally the apify-client request shape. */ private toApiRequest( request: UpdateRequestSchema, ): Parameters[0] { diff --git a/src/apify_storage_client.ts b/src/apify_storage_client.ts index e915049b50..abb1db745e 100644 --- a/src/apify_storage_client.ts +++ b/src/apify_storage_client.ts @@ -2,14 +2,13 @@ import { AsyncLocalStorage } from 'node:async_hooks'; import type { - CreateDatasetClientOptions, - CreateKeyValueStoreClientOptions, - CreateRequestQueueClientOptions, - DatasetClient, - KeyValueStoreClient, - RequestQueueClient, - SetStatusMessageOptions, - StorageClient, + CreateDatasetBackendOptions, + CreateKeyValueStoreBackendOptions, + CreateRequestQueueBackendOptions, + DatasetBackend, + KeyValueStoreBackend, + RequestQueueBackend, + StorageBackend, } from '@crawlee/types'; import type { ApifyClient } from 'apify-client'; import { DatasetClient as ApifyDatasetClient } from 'apify-client'; @@ -155,7 +154,7 @@ function adapt( * const { items } = await dataset.getData(); * ``` */ -export class ApifyStorageClient implements StorageClient { +export class ApifyStorageClient implements StorageBackend { constructor( private readonly client: ApifyClient, private readonly config?: Configuration, @@ -169,7 +168,7 @@ export class ApifyStorageClient implements StorageClient { return info?.id === id; } - async createDatasetClient(options?: CreateDatasetClientOptions): Promise { + async createDatasetBackend(options?: CreateDatasetBackendOptions): Promise { const id = await this.resolveId(options, 'Dataset'); const client = this.chargingDatasetClient(id) ?? this.client.dataset(id); return adapt( @@ -181,10 +180,10 @@ export class ApifyStorageClient implements StorageClient { getData: 'listItems', }, noPurge, - ) as unknown as DatasetClient; + ) as unknown as DatasetBackend; } - async createKeyValueStoreClient(options?: CreateKeyValueStoreClientOptions): Promise { + async createKeyValueStoreBackend(options?: CreateKeyValueStoreBackendOptions): Promise { const id = await this.resolveId(options, 'KeyValueStore'); const client = this.client.keyValueStore(id); return adapt( @@ -202,30 +201,15 @@ export class ApifyStorageClient implements StorageClient { // crawlee expects an array; apify-client returns `{ items }`. listKeys: async (opts?: Parameters[0]) => (await client.listKeys(opts)).items, }, - ) as unknown as KeyValueStoreClient; + ) as unknown as KeyValueStoreBackend; } - /** - * Sets the run's status message (shown in the Apify console). Crawlee v4 calls - * this on the storage client to surface crawl progress. No-op when not running - * on the platform (no run id). - */ - async setStatusMessage(message: string, options?: SetStatusMessageOptions): Promise { - const runId = this.config?.actorRunId; - if (!runId) return; - await this.client.run(runId).update({ - statusMessage: message, - isStatusMessageTerminal: options?.isStatusMessageTerminal, - }); - } - - async createRequestQueueClient(options?: CreateRequestQueueClientOptions): Promise { + async createRequestQueueBackend(options?: CreateRequestQueueBackendOptions): Promise { const id = await this.resolveId(options, 'RequestQueue'); const client = this.client.requestQueue(id, options?.clientKey ? { clientKey: options.clientKey } : undefined); - // Crawlee v4's RequestQueueClient is a stateful, pull-based interface that + // Crawlee v4's RequestQueueBackend is a stateful, pull-based interface that // can't be satisfied by name-remapping; it's implemented on apify-client. - // (cast: published @crawlee/types still ships the pre-redesign interface.) - return new ApifyRequestQueueClient(client) as unknown as RequestQueueClient; + return new ApifyRequestQueueClient(client); } /** diff --git a/src/storage.ts b/src/storage.ts index 1858b618b5..fc8feffeee 100644 --- a/src/storage.ts +++ b/src/storage.ts @@ -1,5 +1,5 @@ import type { Constructor, IStorage, StorageOpenOptions } from '@crawlee/core'; -import type { StorageClient } from '@crawlee/types'; +import type { StorageBackend } from '@crawlee/types'; import { ApifyStorageClient } from './apify_storage_client.js'; import type { Configuration } from './configuration.js'; @@ -137,7 +137,7 @@ function resolveStorageIdentifier( export interface OpenStorageContext { config: Configuration; - client?: StorageClient; + client?: StorageBackend; purgedStorageAliases: Set; } @@ -174,12 +174,12 @@ export async function openStorage( ) { context.purgedStorageAliases.add(identifier.alias); const existingStorage = await storageClass.open(resolvedIdOrName ?? null, { - storageClient: context.client, + storageBackend: context.client, }); await (existingStorage as T & { drop(): Promise }).drop(); } return storageClass.open(resolvedIdOrName ?? null, { - storageClient: context.client, + storageBackend: context.client, }); } diff --git a/test/apify/actor.test.ts b/test/apify/actor.test.ts index 4f08898f83..0b1d62a9c5 100644 --- a/test/apify/actor.test.ts +++ b/test/apify/actor.test.ts @@ -1,7 +1,6 @@ import { createPublicKey } from 'node:crypto'; -import { EventType, serviceLocator } from '@crawlee/core'; -import { MemoryStorage } from '@crawlee/memory-storage'; +import { EventType, MemoryStorageBackend, serviceLocator } from '@crawlee/core'; import { sleep } from '@crawlee/utils'; import type { ApifyEnv } from 'apify'; import { Actor, Configuration, Dataset, KeyValueStore, ProxyConfiguration, RequestQueue } from 'apify'; @@ -614,7 +613,7 @@ describe('Actor', () => { beforeEach(() => { sdk = createIsolatedActor({ - storageClient: new MemoryStorage({ persistStorage: false }), + storageClient: new MemoryStorageBackend(), }).actor as Actor<{ foo: string }>; }); @@ -680,7 +679,7 @@ describe('Actor', () => { const queue = await sdk.openRequestQueue(queueId, options); expect(openSpy).toBeCalledTimes(1); expect(openSpy.mock.calls[0][0]).toBe(queueId); - expect(openSpy.mock.calls[0][1]?.storageClient).toBeDefined(); + expect(openSpy.mock.calls[0][1]?.storageBackend).toBeDefined(); expect(queue).toBe(mockRQ); }); @@ -691,7 +690,7 @@ describe('Actor', () => { await sdk.openDataset(datasetName, options); expect(openSpy).toBeCalledTimes(1); expect(openSpy.mock.calls[0][0]).toBe(datasetName); - expect(openSpy.mock.calls[0][1]?.storageClient).toBeDefined(); + expect(openSpy.mock.calls[0][1]?.storageBackend).toBeDefined(); }); describe('StorageIdentifier support', () => { diff --git a/test/createIsolatedActor.ts b/test/createIsolatedActor.ts index fed6cdea05..9a299623c3 100644 --- a/test/createIsolatedActor.ts +++ b/test/createIsolatedActor.ts @@ -1,6 +1,5 @@ -import { bindMethodsToServiceLocator, ServiceLocator } from '@crawlee/core'; -import { MemoryStorage } from '@crawlee/memory-storage'; -import type { StorageClient } from '@crawlee/types'; +import { bindMethodsToServiceLocator, MemoryStorageBackend, ServiceLocator } from '@crawlee/core'; +import type { StorageBackend } from '@crawlee/types'; import { Actor, Configuration, type PlatformEventManager } from 'apify'; import { onTestFinished } from 'vitest'; @@ -30,7 +29,7 @@ export interface IsolatedActor { * {@link initIsolatedDefaultActor}. */ export function createIsolatedActor( - options: { config?: Configuration; storageClient?: StorageClient } = {}, + options: { config?: Configuration; storageClient?: StorageBackend } = {}, ): IsolatedActor { const config = options.config ?? new Configuration(); const actor = new Actor({ configuration: config }); @@ -82,9 +81,9 @@ export async function initIsolatedDefaultActor(options: { config?: Configuration * re-setting it — harmless here because the locator is the test's own isolated * instance, not the shared global one. */ -export function useInMemoryStorage(serviceLocator: ServiceLocator): MemoryStorage { - const storage = new MemoryStorage({ persistStorage: false }); +export function useInMemoryStorage(serviceLocator: ServiceLocator): MemoryStorageBackend { + const storage = new MemoryStorageBackend(); serviceLocator.reset(); - serviceLocator.setStorageClient(storage); + serviceLocator.setStorageBackend(storage); return storage; }