diff --git a/packages/cache/README.md b/packages/cache/README.md index 5fec54ae..55185032 100644 --- a/packages/cache/README.md +++ b/packages/cache/README.md @@ -44,6 +44,8 @@ const cacheKey = await cache.restoreCache(paths, key, restoreKeys) ##### Cache segment restore timeout -A cache gets downloaded in multiple segments of fixed sizes (`1GB` for a `32-bit` runner and `2GB` for a `64-bit` runner). Sometimes, a segment download gets stuck which causes the workflow job to be stuck forever and fail. Version `v3.0.4` of cache package introduces a segment download timeout. The segment download timeout will allow the segment download to get aborted and hence allow the job to proceed with a cache miss. +A cache gets downloaded in multiple segments of fixed sizes (now `128MB` to fail-fast, previously `1GB` for a `32-bit` runner and `2GB` for a `64-bit` runner were used). Sometimes, a segment download gets stuck which causes the workflow job to be stuck forever and fail. Version `v3.0.4` of cache package introduces a segment download timeout. The segment download timeout will allow the segment download to get aborted and hence allow the job to proceed with a cache miss. + +Default value of this timeout is 10 minutes (starting `v3.2.1` and higher, previously 60 minutes in versions between `v.3.0.4` and `v3.2.0`, both included) and can be customized by specifying an [environment variable](https://docs.github.com/en/actions/learn-github-actions/environment-variables) named `SEGMENT_DOWNLOAD_TIMEOUT_MINS` with timeout value in minutes. + -Default value of this timeout is 60 minutes and can be customized by specifying an [environment variable](https://docs.github.com/en/actions/learn-github-actions/environment-variables) named `SEGMENT_DOWNLOAD_TIMEOUT_MINS` with timeout value in minutes. diff --git a/packages/cache/RELEASES.md b/packages/cache/RELEASES.md index b1d52116..7f6ae719 100644 --- a/packages/cache/RELEASES.md +++ b/packages/cache/RELEASES.md @@ -5,113 +5,158 @@ - Initial release ### 0.2.0 + - Fixes issues with the zstd compression algorithm on Windows and Ubuntu 16.04 [#469](https://github.com/actions/toolkit/pull/469) ### 0.2.1 + - Fix to await async function getCompressionMethod ### 1.0.0 + - Downloads Azure-hosted caches using the Azure SDK for speed and reliability - Displays download progress - Includes changes that break compatibility with earlier versions, including: - `retry`, `retryTypedResponse`, and `retryHttpClientResponse` moved from `cacheHttpClient` to `requestUtils` ### 1.0.1 + - Fix bug in downloading large files (> 2 GBs) with the Azure SDK ### 1.0.2 + - Use posix archive format to add support for some tools ### 1.0.3 + - Use http-client v1.0.9 - Fixes error handling so retries are not attempted on non-retryable errors (409 Conflict, for example) - Adds 5 second delay between retry attempts ### 1.0.4 + - Use @actions/core v1.2.6 - Fixes uploadChunk to throw an error if any unsuccessful response code is received ### 1.0.5 + - Fix to ensure Windows cache paths get resolved correctly ### 1.0.6 + - Make caching more verbose [#650](https://github.com/actions/toolkit/pull/650) - Use GNU tar on macOS if available [#701](https://github.com/actions/toolkit/pull/701) ### 1.0.7 + - Fixes permissions issue extracting archives with GNU tar on macOS ([issue](https://github.com/actions/cache/issues/527)) ### 1.0.8 + - Increase the allowed artifact cache size from 5GB to 10GB ([issue](https://github.com/actions/cache/discussions/497)) ### 1.0.9 - - Use @azure/ms-rest-js v2.6.0 - - Use @azure/storage-blob v12.8.0 + +- Use @azure/ms-rest-js v2.6.0 +- Use @azure/storage-blob v12.8.0 ### 1.0.10 + - Update `lockfileVersion` to `v2` in `package-lock.json [#1022](https://github.com/actions/toolkit/pull/1022) ### 1.0.11 + - Fix file downloads > 2GB([issue](https://github.com/actions/cache/issues/773)) ### 2.0.0 + - Added support to check if Actions cache service feature is available or not [#1028](https://github.com/actions/toolkit/pull/1028) ### 2.0.3 + - Update to v2.0.0 of `@actions/http-client` ### 2.0.4 + - Update to v2.0.1 of `@actions/http-client` [#1087](https://github.com/actions/toolkit/pull/1087) ### 2.0.5 + - Fix to avoid saving empty cache when no files are available for caching. ([issue](https://github.com/actions/cache/issues/624)) ### 2.0.6 + - Fix `Tar failed with error: The process '/usr/bin/tar' failed with exit code 1` issue when temp directory where tar is getting created is actually the subdirectory of the path mentioned by the user for caching. ([issue](https://github.com/actions/cache/issues/689)) ### 3.0.0 + - Updated actions/cache to suppress Actions cache server error and log warning for those error [#1122](https://github.com/actions/toolkit/pull/1122) ### 3.0.1 + - Fix [#833](https://github.com/actions/cache/issues/833) - cache doesn't work with github workspace directory. - Fix [#809](https://github.com/actions/cache/issues/809) `zstd -d: no such file or directory` error on AWS self-hosted runners. ### 3.0.2 + - Added 1 hour timeout for the download stuck issue [#810](https://github.com/actions/cache/issues/810). ### 3.0.3 + - Bug fixes for download stuck issue [#810](https://github.com/actions/cache/issues/810). ### 3.0.4 + - Fix zstd not working for windows on gnu tar in issues [#888](https://github.com/actions/cache/issues/888) and [#891](https://github.com/actions/cache/issues/891). - Allowing users to provide a custom timeout as input for aborting download of a cache segment using an environment variable `SEGMENT_DOWNLOAD_TIMEOUT_MINS`. Default is 60 minutes. ### 3.0.5 + - Update `@actions/cache` to use `@actions/core@^1.10.0` ### 3.0.6 + - Added `@azure/abort-controller` to dependencies to fix compatibility issue with ESM [#1208](https://github.com/actions/toolkit/issues/1208) ### 3.1.0-beta.1 + - Update actions/cache on windows to use gnu tar and zstd by default and fallback to bsdtar and zstd if gnu tar is not available. ([issue](https://github.com/actions/cache/issues/984)) ### 3.1.0-beta.2 + - Added support for fallback to gzip to restore old caches on windows. ### 3.1.0-beta.3 + - Bug Fixes for fallback to gzip to restore old caches on windows and bsdtar if gnutar is not available. ### 3.1.0 + - Update actions/cache on windows to use gnu tar and zstd by default - Update actions/cache on windows to fallback to bsdtar and zstd if gnu tar is not available. - Added support for fallback to gzip to restore old caches on windows. ### 3.1.1 + - Reverted changes in 3.1.0 to fix issue with symlink restoration on windows. - Added support for verbose logging about cache version during cache miss. ### 3.1.2 + - Fix issue with symlink restoration on windows. ### 3.1.3 -- Fix to prevent from setting MYSYS environement variable globally [#1329](https://github.com/actions/toolkit/pull/1329). \ No newline at end of file + +- Fix to prevent from setting MYSYS environement variable globally [#1329](https://github.com/actions/toolkit/pull/1329). + +### 3.1.4 + +- Fix zstd not being used due to `zstd --version` output change in zstd 1.5.4 release. See [#1353](https://github.com/actions/toolkit/pull/1353). + +### 3.2.0 + +- Add `lookupOnly` to cache restore `DownloadOptions`. + +### 3.2.1 + +- Updated @azure/storage-blob to `v12.13.0` diff --git a/packages/cache/__tests__/options.test.ts b/packages/cache/__tests__/options.test.ts index 83d9b227..bcb478d1 100644 --- a/packages/cache/__tests__/options.test.ts +++ b/packages/cache/__tests__/options.test.ts @@ -8,7 +8,8 @@ import { const useAzureSdk = true const downloadConcurrency = 8 const timeoutInMs = 30000 -const segmentTimeoutInMs = 3600000 +const segmentTimeoutInMs = 600000 +const lookupOnly = false const uploadConcurrency = 4 const uploadChunkSize = 32 * 1024 * 1024 @@ -19,7 +20,8 @@ test('getDownloadOptions sets defaults', async () => { useAzureSdk, downloadConcurrency, timeoutInMs, - segmentTimeoutInMs + segmentTimeoutInMs, + lookupOnly }) }) @@ -28,7 +30,8 @@ test('getDownloadOptions overrides all settings', async () => { useAzureSdk: false, downloadConcurrency: 14, timeoutInMs: 20000, - segmentTimeoutInMs: 3600000 + segmentTimeoutInMs: 3600000, + lookupOnly: true } const actualOptions = getDownloadOptions(expectedOptions) @@ -61,7 +64,8 @@ test('getDownloadOptions overrides download timeout minutes', async () => { useAzureSdk: false, downloadConcurrency: 14, timeoutInMs: 20000, - segmentTimeoutInMs: 3600000 + segmentTimeoutInMs: 3600000, + lookupOnly: true } process.env.SEGMENT_DOWNLOAD_TIMEOUT_MINS = '10' const actualOptions = getDownloadOptions(expectedOptions) @@ -72,4 +76,5 @@ test('getDownloadOptions overrides download timeout minutes', async () => { ) expect(actualOptions.timeoutInMs).toEqual(expectedOptions.timeoutInMs) expect(actualOptions.segmentTimeoutInMs).toEqual(600000) + expect(actualOptions.lookupOnly).toEqual(expectedOptions.lookupOnly) }) diff --git a/packages/cache/__tests__/restoreCache.test.ts b/packages/cache/__tests__/restoreCache.test.ts index 5318a007..7992490e 100644 --- a/packages/cache/__tests__/restoreCache.test.ts +++ b/packages/cache/__tests__/restoreCache.test.ts @@ -276,3 +276,39 @@ test('restore with cache found for restore key', async () => { expect(extractTarMock).toHaveBeenCalledWith(archivePath, compression) expect(getCompressionMock).toHaveBeenCalledTimes(1) }) + +test('restore with dry run', async () => { + const paths = ['node_modules'] + const key = 'node-test' + const options = {lookupOnly: true} + + const cacheEntry: ArtifactCacheEntry = { + cacheKey: key, + scope: 'refs/heads/main', + archiveLocation: 'www.actionscache.test/download' + } + const getCacheMock = jest.spyOn(cacheHttpClient, 'getCacheEntry') + getCacheMock.mockImplementation(async () => { + return Promise.resolve(cacheEntry) + }) + + const createTempDirectoryMock = jest.spyOn(cacheUtils, 'createTempDirectory') + const downloadCacheMock = jest.spyOn(cacheHttpClient, 'downloadCache') + + const compression = CompressionMethod.Gzip + const getCompressionMock = jest + .spyOn(cacheUtils, 'getCompressionMethod') + .mockReturnValue(Promise.resolve(compression)) + + const cacheKey = await restoreCache(paths, key, undefined, options) + + expect(cacheKey).toBe(key) + expect(getCompressionMock).toHaveBeenCalledTimes(1) + expect(getCacheMock).toHaveBeenCalledWith([key], paths, { + compressionMethod: compression, + enableCrossOsArchive: false + }) + // creating a tempDir and downloading the cache are skipped + expect(createTempDirectoryMock).toHaveBeenCalledTimes(0) + expect(downloadCacheMock).toHaveBeenCalledTimes(0) +}) diff --git a/packages/cache/package-lock.json b/packages/cache/package-lock.json index f9a20106..5e6ffac8 100644 --- a/packages/cache/package-lock.json +++ b/packages/cache/package-lock.json @@ -1,12 +1,12 @@ { "name": "@actions/cache", - "version": "3.1.3", + "version": "3.2.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@actions/cache", - "version": "3.1.3", + "version": "3.2.1", "license": "MIT", "dependencies": { "@actions/core": "^1.10.0", @@ -16,14 +16,14 @@ "@actions/io": "^1.0.1", "@azure/abort-controller": "^1.1.0", "@azure/ms-rest-js": "^2.6.0", - "@azure/storage-blob": "^12.8.0", + "@azure/storage-blob": "^12.13.0", "semver": "^6.1.0", "uuid": "^3.3.3" }, "devDependencies": { "@types/semver": "^6.0.0", "@types/uuid": "^3.4.5", - "typescript": "^3.8.3" + "typescript": "^4.8.0" } }, "node_modules/@actions/core": { @@ -112,28 +112,27 @@ "integrity": "sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw==" }, "node_modules/@azure/core-http": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/@azure/core-http/-/core-http-2.2.3.tgz", - "integrity": "sha512-xr8AeszxP418rI//W38NfJDDr0kbVAPZkURZnZ+Fle+lLWeURjDE5zNIuocA1wUPoKSP8iXc0ApW6nPtbLGswA==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@azure/core-http/-/core-http-3.0.0.tgz", + "integrity": "sha512-BxI2SlGFPPz6J1XyZNIVUf0QZLBKFX+ViFjKOkzqD18J1zOINIQ8JSBKKr+i+v8+MB6LacL6Nn/sP/TE13+s2Q==", "dependencies": { "@azure/abort-controller": "^1.0.0", - "@azure/core-asynciterator-polyfill": "^1.0.0", "@azure/core-auth": "^1.3.0", "@azure/core-tracing": "1.0.0-preview.13", + "@azure/core-util": "^1.1.1", "@azure/logger": "^1.0.0", "@types/node-fetch": "^2.5.0", "@types/tunnel": "^0.0.3", "form-data": "^4.0.0", - "node-fetch": "^2.6.6", + "node-fetch": "^2.6.7", "process": "^0.11.10", - "tough-cookie": "^4.0.0", "tslib": "^2.2.0", "tunnel": "^0.0.6", "uuid": "^8.3.0", "xml2js": "^0.4.19" }, "engines": { - "node": ">=12.0.0" + "node": ">=14.0.0" } }, "node_modules/@azure/core-http/node_modules/form-data": { @@ -149,23 +148,10 @@ "node": ">= 6" } }, - "node_modules/@azure/core-http/node_modules/tough-cookie": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.0.0.tgz", - "integrity": "sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg==", - "dependencies": { - "psl": "^1.1.33", - "punycode": "^2.1.1", - "universalify": "^0.1.2" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/@azure/core-http/node_modules/tslib": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.1.tgz", - "integrity": "sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw==" + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz", + "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==" }, "node_modules/@azure/core-http/node_modules/uuid": { "version": "8.3.2", @@ -228,6 +214,23 @@ "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.1.tgz", "integrity": "sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw==" }, + "node_modules/@azure/core-util": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.2.0.tgz", + "integrity": "sha512-ffGIw+Qs8bNKNLxz5UPkz4/VBM/EZY07mPve1ZYFqYUdPwFqRj0RPk0U7LZMOfT7GCck9YjuT1Rfp1PApNl1ng==", + "dependencies": { + "@azure/abort-controller": "^1.0.0", + "tslib": "^2.2.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@azure/core-util/node_modules/tslib": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz", + "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==" + }, "node_modules/@azure/logger": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.0.3.tgz", @@ -269,12 +272,12 @@ } }, "node_modules/@azure/storage-blob": { - "version": "12.8.0", - "resolved": "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.8.0.tgz", - "integrity": "sha512-c8+Wz19xauW0bGkTCoqZH4dYfbtBniPiGiRQOn1ca6G5jsjr4azwaTk9gwjVY8r3vY2Taf95eivLzipfIfiS4A==", + "version": "12.13.0", + "resolved": "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.13.0.tgz", + "integrity": "sha512-t3Q2lvBMJucgTjQcP5+hvEJMAsJSk0qmAnjDLie2td017IiduZbbC9BOcFfmwzR6y6cJdZOuewLCNFmEx9IrXA==", "dependencies": { "@azure/abort-controller": "^1.0.0", - "@azure/core-http": "^2.0.0", + "@azure/core-http": "^3.0.0", "@azure/core-lro": "^2.2.0", "@azure/core-paging": "^1.1.1", "@azure/core-tracing": "1.0.0-preview.13", @@ -283,7 +286,7 @@ "tslib": "^2.2.0" }, "engines": { - "node": ">=12.0.0" + "node": ">=14.0.0" } }, "node_modules/@azure/storage-blob/node_modules/tslib": { @@ -300,14 +303,14 @@ } }, "node_modules/@types/node": { - "version": "17.0.14", - "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.14.tgz", - "integrity": "sha512-SbjLmERksKOGzWzPNuW7fJM7fk3YXVTFiZWB/Hs99gwhk+/dnrQRPBQjPW9aO+fi1tAffi9PrwFvsmOKmDTyng==" + "version": "18.14.6", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.14.6.tgz", + "integrity": "sha512-93+VvleD3mXwlLI/xASjw0FzKcwzl3OdTCzm1LaRfqgS21gfFtK3zDXM5Op9TeeMsJVOaJ2VRDpT9q4Y3d0AvA==" }, "node_modules/@types/node-fetch": { - "version": "2.5.12", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.12.tgz", - "integrity": "sha512-MKgC4dlq4kKNa/mYrwpKfzQMB5X3ee5U6fSprkKpToBqBmX4nFZL9cW5jl6sWn+xpRJ7ypWh2yyqqr8UUCstSw==", + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.2.tgz", + "integrity": "sha512-DHqhlq5jeESLy19TYhLakJ07kNumXWjcDdxXsLUMJZ6ue8VZJj4kLPQVE/2mdHh3xZziNF1xppu5lwmS53HR+A==", "dependencies": { "@types/node": "*", "form-data": "^3.0.0" @@ -489,7 +492,7 @@ "node_modules/process": { "version": "0.11.10", "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", - "integrity": "sha1-czIwDoQBYb2j5podHZGn1LwW8YI=", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", "engines": { "node": ">= 0.6.0" } @@ -552,9 +555,9 @@ } }, "node_modules/typescript": { - "version": "3.8.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.8.3.tgz", - "integrity": "sha512-MYlEfn5VrLNsgudQTVJeNaQFUAI7DkhnOjdpAp4T+ku1TfQClewlbSuTVHiA+8skNBgaf02TL/kLOvig4y3G8w==", + "version": "4.8.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.4.tgz", + "integrity": "sha512-QCh+85mCy+h0IGff8r5XWzOVSbBO+KfeYrMQh7NJ58QujwcE22u+NUSmUxqF+un70P9GXKxa2HCNiTTMJknyjQ==", "dev": true, "bin": { "tsc": "bin/tsc", @@ -564,14 +567,6 @@ "node": ">=4.2.0" } }, - "node_modules/universalify": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", - "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", - "engines": { - "node": ">= 4.0.0" - } - }, "node_modules/uuid": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", @@ -700,21 +695,20 @@ } }, "@azure/core-http": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/@azure/core-http/-/core-http-2.2.3.tgz", - "integrity": "sha512-xr8AeszxP418rI//W38NfJDDr0kbVAPZkURZnZ+Fle+lLWeURjDE5zNIuocA1wUPoKSP8iXc0ApW6nPtbLGswA==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@azure/core-http/-/core-http-3.0.0.tgz", + "integrity": "sha512-BxI2SlGFPPz6J1XyZNIVUf0QZLBKFX+ViFjKOkzqD18J1zOINIQ8JSBKKr+i+v8+MB6LacL6Nn/sP/TE13+s2Q==", "requires": { "@azure/abort-controller": "^1.0.0", - "@azure/core-asynciterator-polyfill": "^1.0.0", "@azure/core-auth": "^1.3.0", "@azure/core-tracing": "1.0.0-preview.13", + "@azure/core-util": "^1.1.1", "@azure/logger": "^1.0.0", "@types/node-fetch": "^2.5.0", "@types/tunnel": "^0.0.3", "form-data": "^4.0.0", - "node-fetch": "^2.6.6", + "node-fetch": "^2.6.7", "process": "^0.11.10", - "tough-cookie": "^4.0.0", "tslib": "^2.2.0", "tunnel": "^0.0.6", "uuid": "^8.3.0", @@ -731,20 +725,10 @@ "mime-types": "^2.1.12" } }, - "tough-cookie": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.0.0.tgz", - "integrity": "sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg==", - "requires": { - "psl": "^1.1.33", - "punycode": "^2.1.1", - "universalify": "^0.1.2" - } - }, "tslib": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.1.tgz", - "integrity": "sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw==" + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz", + "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==" }, "uuid": { "version": "8.3.2", @@ -803,6 +787,22 @@ } } }, + "@azure/core-util": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.2.0.tgz", + "integrity": "sha512-ffGIw+Qs8bNKNLxz5UPkz4/VBM/EZY07mPve1ZYFqYUdPwFqRj0RPk0U7LZMOfT7GCck9YjuT1Rfp1PApNl1ng==", + "requires": { + "@azure/abort-controller": "^1.0.0", + "tslib": "^2.2.0" + }, + "dependencies": { + "tslib": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz", + "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==" + } + } + }, "@azure/logger": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.0.3.tgz", @@ -842,12 +842,12 @@ } }, "@azure/storage-blob": { - "version": "12.8.0", - "resolved": "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.8.0.tgz", - "integrity": "sha512-c8+Wz19xauW0bGkTCoqZH4dYfbtBniPiGiRQOn1ca6G5jsjr4azwaTk9gwjVY8r3vY2Taf95eivLzipfIfiS4A==", + "version": "12.13.0", + "resolved": "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.13.0.tgz", + "integrity": "sha512-t3Q2lvBMJucgTjQcP5+hvEJMAsJSk0qmAnjDLie2td017IiduZbbC9BOcFfmwzR6y6cJdZOuewLCNFmEx9IrXA==", "requires": { "@azure/abort-controller": "^1.0.0", - "@azure/core-http": "^2.0.0", + "@azure/core-http": "^3.0.0", "@azure/core-lro": "^2.2.0", "@azure/core-paging": "^1.1.1", "@azure/core-tracing": "1.0.0-preview.13", @@ -869,14 +869,14 @@ "integrity": "sha512-BuJuXRSJNQ3QoKA6GWWDyuLpOUck+9hAXNMCnrloc1aWVoy6Xq6t9PUV08aBZ4Lutqq2LEHM486bpZqoViScog==" }, "@types/node": { - "version": "17.0.14", - "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.14.tgz", - "integrity": "sha512-SbjLmERksKOGzWzPNuW7fJM7fk3YXVTFiZWB/Hs99gwhk+/dnrQRPBQjPW9aO+fi1tAffi9PrwFvsmOKmDTyng==" + "version": "18.14.6", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.14.6.tgz", + "integrity": "sha512-93+VvleD3mXwlLI/xASjw0FzKcwzl3OdTCzm1LaRfqgS21gfFtK3zDXM5Op9TeeMsJVOaJ2VRDpT9q4Y3d0AvA==" }, "@types/node-fetch": { - "version": "2.5.12", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.12.tgz", - "integrity": "sha512-MKgC4dlq4kKNa/mYrwpKfzQMB5X3ee5U6fSprkKpToBqBmX4nFZL9cW5jl6sWn+xpRJ7ypWh2yyqqr8UUCstSw==", + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.2.tgz", + "integrity": "sha512-DHqhlq5jeESLy19TYhLakJ07kNumXWjcDdxXsLUMJZ6ue8VZJj4kLPQVE/2mdHh3xZziNF1xppu5lwmS53HR+A==", "requires": { "@types/node": "*", "form-data": "^3.0.0" @@ -1016,7 +1016,7 @@ "process": { "version": "0.11.10", "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", - "integrity": "sha1-czIwDoQBYb2j5podHZGn1LwW8YI=" + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==" }, "psl": { "version": "1.8.0", @@ -1064,16 +1064,11 @@ "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==" }, "typescript": { - "version": "3.8.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.8.3.tgz", - "integrity": "sha512-MYlEfn5VrLNsgudQTVJeNaQFUAI7DkhnOjdpAp4T+ku1TfQClewlbSuTVHiA+8skNBgaf02TL/kLOvig4y3G8w==", + "version": "4.8.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.4.tgz", + "integrity": "sha512-QCh+85mCy+h0IGff8r5XWzOVSbBO+KfeYrMQh7NJ58QujwcE22u+NUSmUxqF+un70P9GXKxa2HCNiTTMJknyjQ==", "dev": true }, - "universalify": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", - "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==" - }, "uuid": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", diff --git a/packages/cache/package.json b/packages/cache/package.json index f274e00c..319e8067 100644 --- a/packages/cache/package.json +++ b/packages/cache/package.json @@ -1,6 +1,6 @@ { "name": "@actions/cache", - "version": "3.1.3", + "version": "3.2.1", "preview": true, "description": "Actions cache lib", "keywords": [ @@ -44,13 +44,13 @@ "@actions/io": "^1.0.1", "@azure/abort-controller": "^1.1.0", "@azure/ms-rest-js": "^2.6.0", - "@azure/storage-blob": "^12.8.0", + "@azure/storage-blob": "^12.13.0", "semver": "^6.1.0", "uuid": "^3.3.3" }, "devDependencies": { "@types/semver": "^6.0.0", "@types/uuid": "^3.4.5", - "typescript": "^3.8.3" + "typescript": "^4.8.0" } } diff --git a/packages/cache/src/cache.ts b/packages/cache/src/cache.ts index ffa13e78..f7fadb6f 100644 --- a/packages/cache/src/cache.ts +++ b/packages/cache/src/cache.ts @@ -100,6 +100,11 @@ export async function restoreCache( return undefined } + if (options?.lookupOnly) { + core.info('Lookup only - skipping download') + return cacheEntry.cacheKey + } + archivePath = path.join( await utils.createTempDirectory(), utils.getCacheFileName(compressionMethod) diff --git a/packages/cache/src/internal/cacheUtils.ts b/packages/cache/src/internal/cacheUtils.ts index ea1e7de6..650653ad 100644 --- a/packages/cache/src/internal/cacheUtils.ts +++ b/packages/cache/src/internal/cacheUtils.ts @@ -71,11 +71,15 @@ export async function unlinkFile(filePath: fs.PathLike): Promise { return util.promisify(fs.unlink)(filePath) } -async function getVersion(app: string): Promise { - core.debug(`Checking ${app} --version`) +async function getVersion( + app: string, + additionalArgs: string[] = [] +): Promise { let versionOutput = '' + additionalArgs.push('--version') + core.debug(`Checking ${app} ${additionalArgs.join(' ')}`) try { - await exec.exec(`${app} --version`, [], { + await exec.exec(`${app}`, additionalArgs, { ignoreReturnCode: true, silent: true, listeners: { @@ -94,18 +98,14 @@ async function getVersion(app: string): Promise { // Use zstandard if possible to maximize cache performance export async function getCompressionMethod(): Promise { - const versionOutput = await getVersion('zstd') + const versionOutput = await getVersion('zstd', ['--quiet']) const version = semver.clean(versionOutput) + core.debug(`zstd version: ${version}`) - if (!versionOutput.toLowerCase().includes('zstd command line interface')) { - // zstd is not installed + if (versionOutput === '') { return CompressionMethod.Gzip - } else if (!version || semver.lt(version, 'v1.3.2')) { - // zstd is installed but using a version earlier than v1.3.2 - // v1.3.2 is required to use the `--long` options in zstd - return CompressionMethod.ZstdWithoutLong } else { - return CompressionMethod.Zstd + return CompressionMethod.ZstdWithoutLong } } diff --git a/packages/cache/src/internal/downloadUtils.ts b/packages/cache/src/internal/downloadUtils.ts index 265c34f7..7e1b1e8a 100644 --- a/packages/cache/src/internal/downloadUtils.ts +++ b/packages/cache/src/internal/downloadUtils.ts @@ -242,7 +242,9 @@ export async function downloadCacheStorageSDK( // If the file exceeds the buffer maximum length (~1 GB on 32-bit systems and ~2 GB // on 64-bit systems), split the download into multiple segments // ~2 GB = 2147483647, beyond this, we start getting out of range error. So, capping it accordingly. - const maxSegmentSize = Math.min(2147483647, buffer.constants.MAX_LENGTH) + + // Updated segment size to 128MB = 134217728 bytes, to complete a segment faster and fail fast + const maxSegmentSize = Math.min(134217728, buffer.constants.MAX_LENGTH) const downloadProgress = new DownloadProgress(contentLength) const fd = fs.openSync(archivePath, 'w') diff --git a/packages/cache/src/options.ts b/packages/cache/src/options.ts index 652899a2..a518d207 100644 --- a/packages/cache/src/options.ts +++ b/packages/cache/src/options.ts @@ -53,6 +53,15 @@ export interface DownloadOptions { * @default 3600000 */ segmentTimeoutInMs?: number + + /** + * Weather to skip downloading the cache entry. + * If lookupOnly is set to true, the restore function will only check if + * a matching cache entry exists and return the cache key if it does. + * + * @default false + */ + lookupOnly?: boolean } /** @@ -92,7 +101,8 @@ export function getDownloadOptions(copy?: DownloadOptions): DownloadOptions { useAzureSdk: true, downloadConcurrency: 8, timeoutInMs: 30000, - segmentTimeoutInMs: 3600000 + segmentTimeoutInMs: 600000, + lookupOnly: false } if (copy) { @@ -111,6 +121,10 @@ export function getDownloadOptions(copy?: DownloadOptions): DownloadOptions { if (typeof copy.segmentTimeoutInMs === 'number') { result.segmentTimeoutInMs = copy.segmentTimeoutInMs } + + if (typeof copy.lookupOnly === 'boolean') { + result.lookupOnly = copy.lookupOnly + } } const segmentDownloadTimeoutMins = process.env['SEGMENT_DOWNLOAD_TIMEOUT_MINS'] @@ -129,6 +143,7 @@ export function getDownloadOptions(copy?: DownloadOptions): DownloadOptions { `Cache segment download timeout mins env var: ${process.env['SEGMENT_DOWNLOAD_TIMEOUT_MINS']}` ) core.debug(`Segment download timeout (ms): ${result.segmentTimeoutInMs}`) + core.debug(`Lookup only: ${result.lookupOnly}`) return result } diff --git a/packages/cache/tsconfig.json b/packages/cache/tsconfig.json index 2cddd408..1fbe66c8 100644 --- a/packages/cache/tsconfig.json +++ b/packages/cache/tsconfig.json @@ -4,7 +4,8 @@ "baseUrl": "./", "outDir": "./lib", "rootDir": "./src", - "lib": ["es6", "dom"] + "lib": ["es6", "dom"], + "useUnknownInCatchVariables": false }, "include": [ "./src" diff --git a/packages/http-client/RELEASES.md b/packages/http-client/RELEASES.md index 344555f9..d3195080 100644 --- a/packages/http-client/RELEASES.md +++ b/packages/http-client/RELEASES.md @@ -1,5 +1,9 @@ ## Releases +## 2.1.0 +- Add support for `*` and subdomains in `no_proxy` [#1355](https://github.com/actions/toolkit/pull/1355) [#1223](https://github.com/actions/toolkit/pull/1223) +- Bypass proxy for loopback IP adresses [#1360](https://github.com/actions/toolkit/pull/1360)) + ## 2.0.1 - Fix an issue with missing `tunnel` dependency [#1085](https://github.com/actions/toolkit/pull/1085) diff --git a/packages/http-client/__tests__/proxy.test.ts b/packages/http-client/__tests__/proxy.test.ts index ccbceec2..98d85c86 100644 --- a/packages/http-client/__tests__/proxy.test.ts +++ b/packages/http-client/__tests__/proxy.test.ts @@ -176,11 +176,16 @@ describe('proxy', () => { expect(bypass).toBeTruthy() }) - // Do not match wildcard ("*") as per https://github.com/actions/runner/blob/97195bad5870e2ad0915ebfef1616083aacf5818/docs/adrs/0263-proxy-support.md it('checkBypass returns true if no_proxy is "*"', () => { process.env['no_proxy'] = '*' const bypass = pm.checkBypass(new URL('https://anything.whatsoever.com')) - expect(bypass).toBeFalsy() + expect(bypass).toBeTruthy() + }) + + it('checkBypass returns true if no_proxy contains comma separated "*"', () => { + process.env['no_proxy'] = 'domain.com,* , example.com' + const bypass = pm.checkBypass(new URL('https://anything.whatsoever.com')) + expect(bypass).toBeTruthy() }) it('HttpClient does basic http get request through proxy', async () => { @@ -237,6 +242,31 @@ describe('proxy', () => { expect(_proxyConnects).toHaveLength(0) }) + it('HttpClient bypasses proxy for loopback addresses (localhost, ::1, 127.*)', async () => { + // setup a server listening on localhost:8091 + const server = http.createServer((request, response) => { + response.writeHead(200) + request.pipe(response) + }) + server.listen(8091) + try { + process.env['http_proxy'] = _proxyUrl + const httpClient = new httpm.HttpClient() + let res = await httpClient.get('http://localhost:8091') + expect(res.message.statusCode).toBe(200) + res = await httpClient.get('http://127.0.0.1:8091') + expect(res.message.statusCode).toBe(200) + + // no support for ipv6 for now + expect(httpClient.get('http://[::1]:8091')).rejects.toThrow() + + // proxy at _proxyUrl was ignored + expect(_proxyConnects).toEqual([]) + } finally { + server.close() + } + }) + it('proxyAuth not set in tunnel agent when authentication is not provided', async () => { process.env['https_proxy'] = 'http://127.0.0.1:8080' const httpClient = new httpm.HttpClient() diff --git a/packages/http-client/package.json b/packages/http-client/package.json index c1de2213..7f5c8ec3 100644 --- a/packages/http-client/package.json +++ b/packages/http-client/package.json @@ -1,6 +1,6 @@ { "name": "@actions/http-client", - "version": "2.0.1", + "version": "2.1.0", "description": "Actions Http Client", "keywords": [ "github", diff --git a/packages/http-client/src/proxy.ts b/packages/http-client/src/proxy.ts index e4e43a54..1a967e34 100644 --- a/packages/http-client/src/proxy.ts +++ b/packages/http-client/src/proxy.ts @@ -25,6 +25,11 @@ export function checkBypass(reqUrl: URL): boolean { return false } + const reqHost = reqUrl.hostname + if (isLoopbackAddress(reqHost)) { + return true + } + const noProxy = process.env['no_proxy'] || process.env['NO_PROXY'] || '' if (!noProxy) { return false @@ -52,6 +57,7 @@ export function checkBypass(reqUrl: URL): boolean { .map(x => x.trim().toUpperCase()) .filter(x => x)) { if ( + upperNoProxyItem === '*' || upperReqHosts.some( x => x === upperNoProxyItem || @@ -66,3 +72,13 @@ export function checkBypass(reqUrl: URL): boolean { return false } + +function isLoopbackAddress(host: string): boolean { + const hostLower = host.toLowerCase() + return ( + hostLower === 'localhost' || + hostLower.startsWith('127.') || + hostLower.startsWith('[::1]') || + hostLower.startsWith('[0:0:0:0:0:0:0:1]') + ) +} diff --git a/packages/io/__tests__/io.test.ts b/packages/io/__tests__/io.test.ts index 3fd1c64a..b336317e 100644 --- a/packages/io/__tests__/io.test.ts +++ b/packages/io/__tests__/io.test.ts @@ -343,13 +343,8 @@ describe('rmRF', () => { await fs.appendFile(filePath, 'some data') await assertExists(filePath) - // we need to open the file with an explicit executive lock - // otherwise node will allow the file to be deleted even though it's open - // https://github.com/nodejs/node/blob/c2e4b1fa9ad0b744616c4e4c13a5017772a630c4/deps/uv/src/win/fs.c#L499-L513 - const fd = await fs.open( - filePath, - fs.constants.O_RDONLY | ioUtil.UV_FS_O_EXLOCK - ) + const fd = await fs.open(filePath, fs.constants.O_RDONLY | 0x10000000) + await io.rmRF(testPath) // // can't remove folder with locked file on windows if (ioUtil.IS_WINDOWS) {