From a735d9bcd466cb2cb76a37653c7b978250736272 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Mon, 14 Nov 2022 05:24:09 +0000 Subject: [PATCH 01/12] Add logs for cache version on miss --- packages/cache/src/internal/cacheHttpClient.ts | 18 +++++++++++++++++- packages/cache/src/internal/contracts.d.ts | 6 ++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/packages/cache/src/internal/cacheHttpClient.ts b/packages/cache/src/internal/cacheHttpClient.ts index c66d1a73..922c2750 100644 --- a/packages/cache/src/internal/cacheHttpClient.ts +++ b/packages/cache/src/internal/cacheHttpClient.ts @@ -17,7 +17,8 @@ import { CommitCacheRequest, ReserveCacheRequest, ReserveCacheResponse, - ITypedResponseWithError + ITypedResponseWithError, + ArtifactCacheList } from './contracts' import {downloadCacheHttpClient, downloadCacheStorageSDK} from './downloadUtils' import { @@ -113,6 +114,21 @@ export async function getCacheEntry( const cacheResult = response.result const cacheDownloadUrl = cacheResult?.archiveLocation if (!cacheDownloadUrl) { + // List cache for primary key only if cache miss occurs + const resource = `caches?key=${encodeURIComponent(keys[0])}` + const response = await httpClient.getJson(getCacheApiUrl(resource)) + if(response.statusCode === 204) { + const cacheListResult = response.result + const totalCount = cacheListResult?.totalCount + if(totalCount && totalCount > 0) { + core.info(`Cache miss occurred on the cache key '${keys[0]}' and version '${version} but there is ${totalCount} existing version of the cache for this key. More info on versioning can be found here: https://github.com/actions/cache#cache-version`) + core.debug(`Other versions are as follows:`) + cacheListResult?.artifactCaches?.forEach(cacheEntry => { + core.debug(`Cache Key: ${cacheEntry?.cacheKey}, Cache Version: ${cacheEntry?.cacheVersion}, Cache Scope: ${cacheEntry?.scope}, Cache Created: ${cacheEntry?.creationTime}`) + }) + } + } + throw new Error('Cache not found.') } core.setSecret(cacheDownloadUrl) diff --git a/packages/cache/src/internal/contracts.d.ts b/packages/cache/src/internal/contracts.d.ts index 1b2a13a1..b5f53bdc 100644 --- a/packages/cache/src/internal/contracts.d.ts +++ b/packages/cache/src/internal/contracts.d.ts @@ -9,10 +9,16 @@ export interface ITypedResponseWithError extends TypedResponse { export interface ArtifactCacheEntry { cacheKey?: string scope?: string + cacheVersion?: string creationTime?: string archiveLocation?: string } +export interface ArtifactCacheList { + totalCount: number + artifactCaches?: ArtifactCacheEntry[] +} + export interface CommitCacheRequest { size: number } From aaac0e6c9895ec46d5d87b004b063b325681a35b Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Tue, 15 Nov 2022 10:32:24 +0000 Subject: [PATCH 02/12] Address review comments --- .../cache/src/internal/cacheHttpClient.ts | 41 ++++++++++++------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/packages/cache/src/internal/cacheHttpClient.ts b/packages/cache/src/internal/cacheHttpClient.ts index 922c2750..6d2cccde 100644 --- a/packages/cache/src/internal/cacheHttpClient.ts +++ b/packages/cache/src/internal/cacheHttpClient.ts @@ -115,20 +115,7 @@ export async function getCacheEntry( const cacheDownloadUrl = cacheResult?.archiveLocation if (!cacheDownloadUrl) { // List cache for primary key only if cache miss occurs - const resource = `caches?key=${encodeURIComponent(keys[0])}` - const response = await httpClient.getJson(getCacheApiUrl(resource)) - if(response.statusCode === 204) { - const cacheListResult = response.result - const totalCount = cacheListResult?.totalCount - if(totalCount && totalCount > 0) { - core.info(`Cache miss occurred on the cache key '${keys[0]}' and version '${version} but there is ${totalCount} existing version of the cache for this key. More info on versioning can be found here: https://github.com/actions/cache#cache-version`) - core.debug(`Other versions are as follows:`) - cacheListResult?.artifactCaches?.forEach(cacheEntry => { - core.debug(`Cache Key: ${cacheEntry?.cacheKey}, Cache Version: ${cacheEntry?.cacheVersion}, Cache Scope: ${cacheEntry?.scope}, Cache Created: ${cacheEntry?.creationTime}`) - }) - } - } - + await listCache(keys[0], httpClient, version) throw new Error('Cache not found.') } core.setSecret(cacheDownloadUrl) @@ -138,6 +125,32 @@ export async function getCacheEntry( return cacheResult } +async function listCache( + key: string, + httpClient: HttpClient, + version: string +): Promise { + const resource = `caches?key=${encodeURIComponent(key)}` + const response = await retryTypedResponse('listCache', async () => + httpClient.getJson(getCacheApiUrl(resource)) + ) + if (response.statusCode === 204) { + const cacheListResult = response.result + const totalCount = cacheListResult?.totalCount + if (totalCount && totalCount > 0) { + core.info( + `Cache miss occurred on the cache key '${key}' and version '${version} but there is ${totalCount} existing version of the cache for this key. More info on versioning can be found here: https://github.com/actions/cache#cache-version` + ) + core.debug(`Other versions are as follows:`) + cacheListResult?.artifactCaches?.forEach(cacheEntry => { + core.debug( + `Cache Key: ${cacheEntry?.cacheKey}, Cache Version: ${cacheEntry?.cacheVersion}, Cache Scope: ${cacheEntry?.scope}, Cache Created: ${cacheEntry?.creationTime}` + ) + }) + } + } +} + export async function downloadCache( archiveLocation: string, archivePath: string, From b9d1dd898e5a340d18ca45b95c5150213c5de7b6 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Tue, 15 Nov 2022 11:04:24 +0000 Subject: [PATCH 03/12] Address review comments --- packages/cache/src/internal/cacheHttpClient.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/cache/src/internal/cacheHttpClient.ts b/packages/cache/src/internal/cacheHttpClient.ts index 6d2cccde..b4b50f8b 100644 --- a/packages/cache/src/internal/cacheHttpClient.ts +++ b/packages/cache/src/internal/cacheHttpClient.ts @@ -115,7 +115,7 @@ export async function getCacheEntry( const cacheDownloadUrl = cacheResult?.archiveLocation if (!cacheDownloadUrl) { // List cache for primary key only if cache miss occurs - await listCache(keys[0], httpClient, version) + await listCache(keys[0], httpClient, version, cacheResult?.scope) throw new Error('Cache not found.') } core.setSecret(cacheDownloadUrl) @@ -128,7 +128,8 @@ export async function getCacheEntry( async function listCache( key: string, httpClient: HttpClient, - version: string + version: string, + scope?: string ): Promise { const resource = `caches?key=${encodeURIComponent(key)}` const response = await retryTypedResponse('listCache', async () => @@ -139,7 +140,7 @@ async function listCache( const totalCount = cacheListResult?.totalCount if (totalCount && totalCount > 0) { core.info( - `Cache miss occurred on the cache key '${key}' and version '${version} but there is ${totalCount} existing version of the cache for this key. More info on versioning can be found here: https://github.com/actions/cache#cache-version` + `No matching cache found for cache key '${key}', version '${version} and scope ${scope} but there is ${totalCount} existing version of the cache for this key. More info on versioning can be found here: https://github.com/actions/cache#cache-version` ) core.debug(`Other versions are as follows:`) cacheListResult?.artifactCaches?.forEach(cacheEntry => { From 816c1b37609e775ac7d11622568feb04065ed034 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Tue, 13 Dec 2022 11:25:40 +0000 Subject: [PATCH 04/12] Fix tests --- .../cache/src/internal/cacheHttpClient.ts | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/packages/cache/src/internal/cacheHttpClient.ts b/packages/cache/src/internal/cacheHttpClient.ts index b4b50f8b..98e9d8d2 100644 --- a/packages/cache/src/internal/cacheHttpClient.ts +++ b/packages/cache/src/internal/cacheHttpClient.ts @@ -105,6 +105,10 @@ export async function getCacheEntry( httpClient.getJson(getCacheApiUrl(resource)) ) if (response.statusCode === 204) { + // List cache for primary key only if cache miss occurs + if (core.isDebug()) { + await listCache(keys[0], httpClient, version, response) + } return null } if (!isSuccessStatusCode(response.statusCode)) { @@ -114,8 +118,6 @@ export async function getCacheEntry( const cacheResult = response.result const cacheDownloadUrl = cacheResult?.archiveLocation if (!cacheDownloadUrl) { - // List cache for primary key only if cache miss occurs - await listCache(keys[0], httpClient, version, cacheResult?.scope) throw new Error('Cache not found.') } core.setSecret(cacheDownloadUrl) @@ -129,25 +131,25 @@ async function listCache( key: string, httpClient: HttpClient, version: string, - scope?: string + getCacheEntryResponse: any ): Promise { + const scope = getCacheEntryResponse.result?.scope const resource = `caches?key=${encodeURIComponent(key)}` const response = await retryTypedResponse('listCache', async () => httpClient.getJson(getCacheApiUrl(resource)) ) - if (response.statusCode === 204) { + if (response.statusCode === 200) { const cacheListResult = response.result const totalCount = cacheListResult?.totalCount if (totalCount && totalCount > 0) { - core.info( - `No matching cache found for cache key '${key}', version '${version} and scope ${scope} but there is ${totalCount} existing version of the cache for this key. More info on versioning can be found here: https://github.com/actions/cache#cache-version` + core.debug( + `No matching cache found for cache key '${key}', version '${version} and scope ${scope} but there are ${totalCount} existing version of the cache for this key. More info on versioning can be found here: https://github.com/actions/cache#cache-version \nOther versions are as follows:` ) - core.debug(`Other versions are as follows:`) - cacheListResult?.artifactCaches?.forEach(cacheEntry => { + for (const cacheEntry of cacheListResult.artifactCaches || []) { core.debug( `Cache Key: ${cacheEntry?.cacheKey}, Cache Version: ${cacheEntry?.cacheVersion}, Cache Scope: ${cacheEntry?.scope}, Cache Created: ${cacheEntry?.creationTime}` ) - }) + } } } } From e559a15ca64cdea38a2afda1a343b25d2770274a Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Tue, 13 Dec 2022 11:36:10 +0000 Subject: [PATCH 05/12] Fix test --- packages/cache/src/internal/cacheHttpClient.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cache/src/internal/cacheHttpClient.ts b/packages/cache/src/internal/cacheHttpClient.ts index 98e9d8d2..d6ea5e4a 100644 --- a/packages/cache/src/internal/cacheHttpClient.ts +++ b/packages/cache/src/internal/cacheHttpClient.ts @@ -145,7 +145,7 @@ async function listCache( core.debug( `No matching cache found for cache key '${key}', version '${version} and scope ${scope} but there are ${totalCount} existing version of the cache for this key. More info on versioning can be found here: https://github.com/actions/cache#cache-version \nOther versions are as follows:` ) - for (const cacheEntry of cacheListResult.artifactCaches || []) { + for (const cacheEntry of cacheListResult?.artifactCaches || []) { core.debug( `Cache Key: ${cacheEntry?.cacheKey}, Cache Version: ${cacheEntry?.cacheVersion}, Cache Scope: ${cacheEntry?.scope}, Cache Created: ${cacheEntry?.creationTime}` ) From 24685611e29a439826d4525ae90c7d110ee62cbd Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Tue, 13 Dec 2022 11:48:30 +0000 Subject: [PATCH 06/12] Add current scope from github ref --- packages/cache/src/internal/cacheHttpClient.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/cache/src/internal/cacheHttpClient.ts b/packages/cache/src/internal/cacheHttpClient.ts index d6ea5e4a..532a09d6 100644 --- a/packages/cache/src/internal/cacheHttpClient.ts +++ b/packages/cache/src/internal/cacheHttpClient.ts @@ -107,7 +107,7 @@ export async function getCacheEntry( if (response.statusCode === 204) { // List cache for primary key only if cache miss occurs if (core.isDebug()) { - await listCache(keys[0], httpClient, version, response) + await listCache(keys[0], httpClient, version) } return null } @@ -130,20 +130,19 @@ export async function getCacheEntry( async function listCache( key: string, httpClient: HttpClient, - version: string, - getCacheEntryResponse: any + version: string ): Promise { - const scope = getCacheEntryResponse.result?.scope const resource = `caches?key=${encodeURIComponent(key)}` const response = await retryTypedResponse('listCache', async () => httpClient.getJson(getCacheApiUrl(resource)) ) - if (response.statusCode === 200) { + core.debug(`List Cache Response Status Code: ${response.statusCode}`) + if (response.statusCode !== 200) { const cacheListResult = response.result const totalCount = cacheListResult?.totalCount if (totalCount && totalCount > 0) { core.debug( - `No matching cache found for cache key '${key}', version '${version} and scope ${scope} but there are ${totalCount} existing version of the cache for this key. More info on versioning can be found here: https://github.com/actions/cache#cache-version \nOther versions are as follows:` + `No matching cache found for cache key '${key}', version '${version} and scope ${process.env['GITHUB_REF']} but there are ${totalCount} existing version of the cache for this key. More info on versioning can be found here: https://github.com/actions/cache#cache-version \nOther versions are as follows:` ) for (const cacheEntry of cacheListResult?.artifactCaches || []) { core.debug( From b8c50aa82d788116e1daf54f636f12d1c5f142af Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Fri, 16 Dec 2022 05:55:53 +0000 Subject: [PATCH 07/12] Fix response code --- packages/cache/src/internal/cacheHttpClient.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/cache/src/internal/cacheHttpClient.ts b/packages/cache/src/internal/cacheHttpClient.ts index 532a09d6..14a25803 100644 --- a/packages/cache/src/internal/cacheHttpClient.ts +++ b/packages/cache/src/internal/cacheHttpClient.ts @@ -136,8 +136,7 @@ async function listCache( const response = await retryTypedResponse('listCache', async () => httpClient.getJson(getCacheApiUrl(resource)) ) - core.debug(`List Cache Response Status Code: ${response.statusCode}`) - if (response.statusCode !== 200) { + if (response.statusCode === 200) { const cacheListResult = response.result const totalCount = cacheListResult?.totalCount if (totalCount && totalCount > 0) { From e96dc8a69a919789e27edf4d06e37a1293907908 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Fri, 16 Dec 2022 18:17:37 +0530 Subject: [PATCH 08/12] Update packages/cache/src/internal/cacheHttpClient.ts Co-authored-by: Bishal Prasad --- packages/cache/src/internal/cacheHttpClient.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cache/src/internal/cacheHttpClient.ts b/packages/cache/src/internal/cacheHttpClient.ts index 14a25803..0f28af3f 100644 --- a/packages/cache/src/internal/cacheHttpClient.ts +++ b/packages/cache/src/internal/cacheHttpClient.ts @@ -141,7 +141,7 @@ async function listCache( const totalCount = cacheListResult?.totalCount if (totalCount && totalCount > 0) { core.debug( - `No matching cache found for cache key '${key}', version '${version} and scope ${process.env['GITHUB_REF']} but there are ${totalCount} existing version of the cache for this key. More info on versioning can be found here: https://github.com/actions/cache#cache-version \nOther versions are as follows:` + `No matching cache found for cache key '${key}', version '${version} and scope ${process.env['GITHUB_REF']}. There exist one or more cache(s) with similar key but they have different version or scope. See more info on cache matching here: https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#matching-a-cache-key \nOther caches with similar key:` ) for (const cacheEntry of cacheListResult?.artifactCaches || []) { core.debug( From ccfa36f304056fb24583eecf03e14cdbe334bec3 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Wed, 21 Dec 2022 10:24:52 +0000 Subject: [PATCH 09/12] Address review comments --- packages/cache/src/internal/cacheHttpClient.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cache/src/internal/cacheHttpClient.ts b/packages/cache/src/internal/cacheHttpClient.ts index 14a25803..8f42c7db 100644 --- a/packages/cache/src/internal/cacheHttpClient.ts +++ b/packages/cache/src/internal/cacheHttpClient.ts @@ -107,7 +107,7 @@ export async function getCacheEntry( if (response.statusCode === 204) { // List cache for primary key only if cache miss occurs if (core.isDebug()) { - await listCache(keys[0], httpClient, version) + await printCachesListForDiagnostics(keys[0], httpClient, version) } return null } @@ -127,7 +127,7 @@ export async function getCacheEntry( return cacheResult } -async function listCache( +async function printCachesListForDiagnostics( key: string, httpClient: HttpClient, version: string @@ -143,7 +143,7 @@ async function listCache( core.debug( `No matching cache found for cache key '${key}', version '${version} and scope ${process.env['GITHUB_REF']} but there are ${totalCount} existing version of the cache for this key. More info on versioning can be found here: https://github.com/actions/cache#cache-version \nOther versions are as follows:` ) - for (const cacheEntry of cacheListResult?.artifactCaches || []) { + for (const cacheEntry of cacheListResult.artifactCaches || []) { core.debug( `Cache Key: ${cacheEntry?.cacheKey}, Cache Version: ${cacheEntry?.cacheVersion}, Cache Scope: ${cacheEntry?.scope}, Cache Created: ${cacheEntry?.creationTime}` ) From c23fe4b81fb21b14e9f64bb1b10fbe6f0baaffb0 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Wed, 21 Dec 2022 10:30:22 +0000 Subject: [PATCH 10/12] FIx test --- packages/cache/src/internal/cacheHttpClient.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cache/src/internal/cacheHttpClient.ts b/packages/cache/src/internal/cacheHttpClient.ts index f0244c43..d5ecd9a8 100644 --- a/packages/cache/src/internal/cacheHttpClient.ts +++ b/packages/cache/src/internal/cacheHttpClient.ts @@ -143,7 +143,7 @@ async function printCachesListForDiagnostics( core.debug( `No matching cache found for cache key '${key}', version '${version} and scope ${process.env['GITHUB_REF']}. There exist one or more cache(s) with similar key but they have different version or scope. See more info on cache matching here: https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#matching-a-cache-key \nOther caches with similar key:` ) - for (const cacheEntry of cacheListResult.artifactCaches || []) { + for (const cacheEntry of cacheListResult?.artifactCaches || []) { core.debug( `Cache Key: ${cacheEntry?.cacheKey}, Cache Version: ${cacheEntry?.cacheVersion}, Cache Scope: ${cacheEntry?.scope}, Cache Created: ${cacheEntry?.creationTime}` ) From b2287326447a6d61c109af734947c7608be85900 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Thu, 22 Dec 2022 20:47:35 +0530 Subject: [PATCH 11/12] Cache package release for compression change in windows (#1281) * bsd + zstd fallback implementation * bsd + zstd fallback implementation * Fix tar operations * Add -v option for testing * Fix order of args for tar * Add GNUtar as default on windows * Fix test * Fix tar tests * Fix lint issues * Fix windows gnutar test case * Temporarily remove thhe condition that prevents zstd usage on windows unless with GNUtar * Address some comments and correct compression commands * Add windows bsdtar test * Fix windows test * Fix test * Separate args * Fix old tests * Add new tests * Fix tests * Fix lint test * Refactor code * Address review comments * Fix test * Fix tar test * Add await to async function calls * Fix test * Update for beta release * Fix audit issues * Add fallback to gzip compression if cache not found * Fix test * Add test * Address review comments * Revert Address review comments * Release 3.1.0-beta.2 cache package * Fix issues * Reconfigure catch block * Add debug logging for gzip fall back * Fix test * Add end to end test for cache using bsd on windows and address review comments * Fix test * Fix test * Fix tests * Add better comments * Update packages/cache/src/internal/cacheHttpClient.ts Co-authored-by: Bishal Prasad * Address review comments * Update for new beta cache package release * Address bugbash issues * Fix tests * Release new actions/cache minor version Co-authored-by: Lovepreet Singh Co-authored-by: Bishal Prasad --- .github/workflows/cache-windows-test.yml | 90 +++++ packages/cache/RELEASES.md | 14 + packages/cache/__tests__/restoreCache.test.ts | 75 +++++ packages/cache/__tests__/tar.test.ts | 273 +++++++++++++--- packages/cache/package-lock.json | 16 +- packages/cache/package.json | 2 +- packages/cache/src/cache.ts | 30 +- .../cache/src/internal/cacheHttpClient.ts | 2 + packages/cache/src/internal/cacheUtils.ts | 18 +- packages/cache/src/internal/constants.ts | 15 + packages/cache/src/internal/contracts.d.ts | 5 + packages/cache/src/internal/tar.ts | 307 +++++++++++++----- 12 files changed, 692 insertions(+), 155 deletions(-) create mode 100644 .github/workflows/cache-windows-test.yml diff --git a/.github/workflows/cache-windows-test.yml b/.github/workflows/cache-windows-test.yml new file mode 100644 index 00000000..3868f296 --- /dev/null +++ b/.github/workflows/cache-windows-test.yml @@ -0,0 +1,90 @@ +name: cache-windows-bsd-unit-tests +on: + push: + branches: + - main + paths-ignore: + - '**.md' + pull_request: + paths-ignore: + - '**.md' + +jobs: + build: + name: Build + + runs-on: windows-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - shell: bash + run: | + rm "C:\Program Files\Git\usr\bin\tar.exe" + + - name: Set Node.js 12.x + uses: actions/setup-node@v1 + with: + node-version: 12.x + + # In order to save & restore cache from a shell script, certain env variables need to be set that are only available in the + # node context. This runs a local action that gets and sets the necessary env variables that are needed + - name: Set env variables + uses: ./packages/cache/__tests__/__fixtures__/ + + # Need root node_modules because certain npm packages like jest are configured for the entire repository and it won't be possible + # without these to just compile the cache package + - name: Install root npm packages + run: npm ci + + - name: Compile cache package + run: | + npm ci + npm run tsc + working-directory: packages/cache + + - name: Generate files in working directory + shell: bash + run: packages/cache/__tests__/create-cache-files.sh ${{ runner.os }} test-cache + + - name: Generate files outside working directory + shell: bash + run: packages/cache/__tests__/create-cache-files.sh ${{ runner.os }} ~/test-cache + + # We're using node -e to call the functions directly available in the @actions/cache package + - name: Save cache using saveCache() + run: | + node -e "Promise.resolve(require('./packages/cache/lib/cache').saveCache(['test-cache','~/test-cache'],'test-${{ runner.os }}-${{ github.run_id }}'))" + + - name: Delete cache folders before restoring + shell: bash + run: | + rm -rf test-cache + rm -rf ~/test-cache + + - name: Restore cache using restoreCache() with http-client + run: | + node -e "Promise.resolve(require('./packages/cache/lib/cache').restoreCache(['test-cache','~/test-cache'],'test-${{ runner.os }}-${{ github.run_id }}',[],{useAzureSdk: false}))" + + - name: Verify cache restored with http-client + shell: bash + run: | + packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} test-cache + packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} ~/test-cache + + - name: Delete cache folders before restoring + shell: bash + run: | + rm -rf test-cache + rm -rf ~/test-cache + + - name: Restore cache using restoreCache() with Azure SDK + run: | + node -e "Promise.resolve(require('./packages/cache/lib/cache').restoreCache(['test-cache','~/test-cache'],'test-${{ runner.os }}-${{ github.run_id }}'))" + + - name: Verify cache restored with Azure SDK + shell: bash + run: | + packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} test-cache + packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} ~/test-cache diff --git a/packages/cache/RELEASES.md b/packages/cache/RELEASES.md index 73518e1c..4a415486 100644 --- a/packages/cache/RELEASES.md +++ b/packages/cache/RELEASES.md @@ -91,3 +91,17 @@ ### 3.0.6 - Added `@azure/abort-controller` to dependencies to fix compatibility issue with ESM [#1208](https://github.com/actions/toolkit/issues/1208) + +### 3.1.0-beta.1 +- Update actions/cache on windows to use gnu tar and zstd by default and fallback to bsdtar and zstd if gnu tar is not available. ([issue](https://github.com/actions/cache/issues/984)) + +### 3.1.0-beta.2 +- Added support for fallback to gzip to restore old caches on windows. + +### 3.1.0-beta.3 +- Bug Fixes for fallback to gzip to restore old caches on windows and bsdtar if gnutar is not available. + +### 3.1.0 +- Update actions/cache on windows to use gnu tar and zstd by default +- Update actions/cache on windows to fallback to bsdtar and zstd if gnu tar is not available. +- Added support for fallback to gzip to restore old caches on windows. diff --git a/packages/cache/__tests__/restoreCache.test.ts b/packages/cache/__tests__/restoreCache.test.ts index 36ec8801..9cf45799 100644 --- a/packages/cache/__tests__/restoreCache.test.ts +++ b/packages/cache/__tests__/restoreCache.test.ts @@ -161,6 +161,81 @@ test('restore with gzip compressed cache found', async () => { expect(getCompressionMock).toHaveBeenCalledTimes(1) }) +test('restore with zstd as default but gzip compressed cache found on windows', async () => { + if (process.platform === 'win32') { + const paths = ['node_modules'] + const key = 'node-test' + + const cacheEntry: ArtifactCacheEntry = { + cacheKey: key, + scope: 'refs/heads/main', + archiveLocation: 'www.actionscache.test/download' + } + const getCacheMock = jest.spyOn(cacheHttpClient, 'getCacheEntry') + getCacheMock + .mockImplementationOnce(async () => { + return Promise.resolve(null) + }) + .mockImplementationOnce(async () => { + return Promise.resolve(cacheEntry) + }) + + const tempPath = '/foo/bar' + + const createTempDirectoryMock = jest.spyOn( + cacheUtils, + 'createTempDirectory' + ) + createTempDirectoryMock.mockImplementation(async () => { + return Promise.resolve(tempPath) + }) + + const archivePath = path.join(tempPath, CacheFilename.Gzip) + const downloadCacheMock = jest.spyOn(cacheHttpClient, 'downloadCache') + + const fileSize = 142 + const getArchiveFileSizeInBytesMock = jest + .spyOn(cacheUtils, 'getArchiveFileSizeInBytes') + .mockReturnValue(fileSize) + + const extractTarMock = jest.spyOn(tar, 'extractTar') + const unlinkFileMock = jest.spyOn(cacheUtils, 'unlinkFile') + + const compression = CompressionMethod.Zstd + const getCompressionMock = jest + .spyOn(cacheUtils, 'getCompressionMethod') + .mockReturnValue(Promise.resolve(compression)) + + const cacheKey = await restoreCache(paths, key) + + expect(cacheKey).toBe(key) + expect(getCacheMock).toHaveBeenNthCalledWith(1, [key], paths, { + compressionMethod: compression + }) + expect(getCacheMock).toHaveBeenNthCalledWith(2, [key], paths, { + compressionMethod: CompressionMethod.Gzip + }) + expect(createTempDirectoryMock).toHaveBeenCalledTimes(1) + expect(downloadCacheMock).toHaveBeenCalledWith( + cacheEntry.archiveLocation, + archivePath, + undefined + ) + expect(getArchiveFileSizeInBytesMock).toHaveBeenCalledWith(archivePath) + + expect(extractTarMock).toHaveBeenCalledTimes(1) + expect(extractTarMock).toHaveBeenCalledWith( + archivePath, + CompressionMethod.Gzip + ) + + expect(unlinkFileMock).toHaveBeenCalledTimes(1) + expect(unlinkFileMock).toHaveBeenCalledWith(archivePath) + + expect(getCompressionMock).toHaveBeenCalledTimes(1) + } +}) + test('restore with zstd compressed cache found', async () => { const paths = ['node_modules'] const key = 'node-test' diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index e4233bc9..a33f4fab 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -1,7 +1,14 @@ import * as exec from '@actions/exec' import * as io from '@actions/io' import * as path from 'path' -import {CacheFilename, CompressionMethod} from '../src/internal/constants' +import { + CacheFilename, + CompressionMethod, + GnuTarPathOnWindows, + ManifestFilename, + SystemTarPathOnWindows, + TarFilename +} from '../src/internal/constants' import * as tar from '../src/internal/tar' import * as utils from '../src/internal/cacheUtils' // eslint-disable-next-line @typescript-eslint/no-require-imports @@ -13,7 +20,7 @@ jest.mock('@actions/io') const IS_WINDOWS = process.platform === 'win32' const IS_MAC = process.platform === 'darwin' -const defaultTarPath = process.platform === 'darwin' ? 'gtar' : 'tar' +const defaultTarPath = IS_MAC ? 'gtar' : 'tar' function getTempDir(): string { return path.join(__dirname, '_temp', 'tar') @@ -28,6 +35,10 @@ beforeAll(async () => { await jest.requireActual('@actions/io').rmRF(getTempDir()) }) +beforeEach(async () => { + jest.restoreAllMocks() +}) + afterAll(async () => { delete process.env['GITHUB_WORKSPACE'] await jest.requireActual('@actions/io').rmRF(getTempDir()) @@ -41,16 +52,15 @@ test('zstd extract tar', async () => { ? `${process.env['windir']}\\fakepath\\cache.tar` : 'cache.tar' const workspace = process.env['GITHUB_WORKSPACE'] + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath await tar.extractTar(archivePath, CompressionMethod.Zstd) expect(mkdirMock).toHaveBeenCalledWith(workspace) expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${defaultTarPath}"`, [ - '--use-compress-program', - IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30', + `"${tarPath}"`, '-xf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P', @@ -58,11 +68,61 @@ test('zstd extract tar', async () => { IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace ] .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []), + .concat(IS_MAC ? ['--delay-directory-restore'] : []) + .concat([ + '--use-compress-program', + IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30' + ]) + .join(' '), + undefined, {cwd: undefined} ) }) +test('zstd extract tar with windows BSDtar', async () => { + if (IS_WINDOWS) { + const mkdirMock = jest.spyOn(io, 'mkdirP') + const execMock = jest.spyOn(exec, 'exec') + jest + .spyOn(utils, 'getGnuTarPathOnWindows') + .mockReturnValue(Promise.resolve('')) + + const archivePath = `${process.env['windir']}\\fakepath\\cache.tar` + const workspace = process.env['GITHUB_WORKSPACE'] + const tarPath = SystemTarPathOnWindows + + await tar.extractTar(archivePath, CompressionMethod.Zstd) + + expect(mkdirMock).toHaveBeenCalledWith(workspace) + expect(execMock).toHaveBeenCalledTimes(2) + + expect(execMock).toHaveBeenNthCalledWith( + 1, + [ + 'zstd -d --long=30 --force -o', + TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') + ].join(' '), + undefined, + {cwd: undefined} + ) + + expect(execMock).toHaveBeenNthCalledWith( + 2, + [ + `"${tarPath}"`, + '-xf', + TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '-P', + '-C', + workspace?.replace(/\\/g, '/') + ].join(' '), + undefined, + {cwd: undefined} + ) + } +}) + test('gzip extract tar', async () => { const mkdirMock = jest.spyOn(io, 'mkdirP') const execMock = jest.spyOn(exec, 'exec') @@ -74,50 +134,51 @@ test('gzip extract tar', async () => { await tar.extractTar(archivePath, CompressionMethod.Gzip) expect(mkdirMock).toHaveBeenCalledWith(workspace) - const tarPath = IS_WINDOWS - ? `${process.env['windir']}\\System32\\tar.exe` - : defaultTarPath + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${tarPath}"`, [ - '-z', + `"${tarPath}"`, '-xf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P', '-C', IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace - ].concat(IS_MAC ? ['--delay-directory-restore'] : []), + ] + .concat(IS_WINDOWS ? ['--force-local'] : []) + .concat(IS_MAC ? ['--delay-directory-restore'] : []) + .concat(['-z']) + .join(' '), + undefined, {cwd: undefined} ) }) -test('gzip extract GNU tar on windows', async () => { +test('gzip extract GNU tar on windows with GNUtar in path', async () => { if (IS_WINDOWS) { - jest.spyOn(fs, 'existsSync').mockReturnValueOnce(false) - - const isGnuMock = jest - .spyOn(utils, 'isGnuTarInstalled') - .mockReturnValue(Promise.resolve(true)) + // GNU tar present in path but not at default location + jest + .spyOn(utils, 'getGnuTarPathOnWindows') + .mockReturnValue(Promise.resolve('tar')) const execMock = jest.spyOn(exec, 'exec') const archivePath = `${process.env['windir']}\\fakepath\\cache.tar` const workspace = process.env['GITHUB_WORKSPACE'] await tar.extractTar(archivePath, CompressionMethod.Gzip) - expect(isGnuMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"tar"`, [ - '-z', + `"tar"`, '-xf', archivePath.replace(/\\/g, '/'), '-P', '-C', workspace?.replace(/\\/g, '/'), - '--force-local' - ], + '--force-local', + '-z' + ].join(' '), + undefined, {cwd: undefined} ) } @@ -134,13 +195,13 @@ test('zstd create tar', async () => { await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Zstd) + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath + expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${defaultTarPath}"`, [ + `"${tarPath}"`, '--posix', - '--use-compress-program', - IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30', '-cf', IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd, '--exclude', @@ -149,16 +210,81 @@ test('zstd create tar', async () => { '-C', IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, '--files-from', - 'manifest.txt' + ManifestFilename ] .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []), + .concat(IS_MAC ? ['--delay-directory-restore'] : []) + .concat([ + '--use-compress-program', + IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30' + ]) + .join(' '), + undefined, // args { cwd: archiveFolder } ) }) +test('zstd create tar with windows BSDtar', async () => { + if (IS_WINDOWS) { + const execMock = jest.spyOn(exec, 'exec') + jest + .spyOn(utils, 'getGnuTarPathOnWindows') + .mockReturnValue(Promise.resolve('')) + + const archiveFolder = getTempDir() + const workspace = process.env['GITHUB_WORKSPACE'] + const sourceDirectories = ['~/.npm/cache', `${workspace}/dist`] + + await fs.promises.mkdir(archiveFolder, {recursive: true}) + + await tar.createTar( + archiveFolder, + sourceDirectories, + CompressionMethod.Zstd + ) + + const tarPath = SystemTarPathOnWindows + + expect(execMock).toHaveBeenCalledTimes(2) + + expect(execMock).toHaveBeenNthCalledWith( + 1, + [ + `"${tarPath}"`, + '--posix', + '-cf', + TarFilename.replace(/\\/g, '/'), + '--exclude', + TarFilename.replace(/\\/g, '/'), + '-P', + '-C', + workspace?.replace(/\\/g, '/'), + '--files-from', + ManifestFilename + ].join(' '), + undefined, // args + { + cwd: archiveFolder + } + ) + + expect(execMock).toHaveBeenNthCalledWith( + 2, + [ + 'zstd -T0 --long=30 --force -o', + CacheFilename.Zstd.replace(/\\/g, '/'), + TarFilename.replace(/\\/g, '/') + ].join(' '), + undefined, // args + { + cwd: archiveFolder + } + ) + } +}) + test('gzip create tar', async () => { const execMock = jest.spyOn(exec, 'exec') @@ -170,16 +296,13 @@ test('gzip create tar', async () => { await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Gzip) - const tarPath = IS_WINDOWS - ? `${process.env['windir']}\\System32\\tar.exe` - : defaultTarPath + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${tarPath}"`, [ + `"${tarPath}"`, '--posix', - '-z', '-cf', IS_WINDOWS ? CacheFilename.Gzip.replace(/\\/g, '/') : CacheFilename.Gzip, '--exclude', @@ -188,8 +311,13 @@ test('gzip create tar', async () => { '-C', IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, '--files-from', - 'manifest.txt' - ].concat(IS_MAC ? ['--delay-directory-restore'] : []), + ManifestFilename + ] + .concat(IS_WINDOWS ? ['--force-local'] : []) + .concat(IS_MAC ? ['--delay-directory-restore'] : []) + .concat(['-z']) + .join(' '), + undefined, // args { cwd: archiveFolder } @@ -205,22 +333,65 @@ test('zstd list tar', async () => { await tar.listTar(archivePath, CompressionMethod.Zstd) + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${defaultTarPath}"`, [ - '--use-compress-program', - IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30', + `"${tarPath}"`, '-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P' ] .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []), + .concat(IS_MAC ? ['--delay-directory-restore'] : []) + .concat([ + '--use-compress-program', + IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30' + ]) + .join(' '), + undefined, {cwd: undefined} ) }) +test('zstd list tar with windows BSDtar', async () => { + if (IS_WINDOWS) { + const execMock = jest.spyOn(exec, 'exec') + jest + .spyOn(utils, 'getGnuTarPathOnWindows') + .mockReturnValue(Promise.resolve('')) + const archivePath = `${process.env['windir']}\\fakepath\\cache.tar` + + await tar.listTar(archivePath, CompressionMethod.Zstd) + + const tarPath = SystemTarPathOnWindows + expect(execMock).toHaveBeenCalledTimes(2) + + expect(execMock).toHaveBeenNthCalledWith( + 1, + [ + 'zstd -d --long=30 --force -o', + TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') + ].join(' '), + undefined, + {cwd: undefined} + ) + + expect(execMock).toHaveBeenNthCalledWith( + 2, + [ + `"${tarPath}"`, + '-tf', + TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '-P' + ].join(' '), + undefined, + {cwd: undefined} + ) + } +}) + test('zstdWithoutLong list tar', async () => { const execMock = jest.spyOn(exec, 'exec') @@ -230,18 +401,20 @@ test('zstdWithoutLong list tar', async () => { await tar.listTar(archivePath, CompressionMethod.ZstdWithoutLong) + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${defaultTarPath}"`, [ - '--use-compress-program', - IS_WINDOWS ? 'zstd -d' : 'unzstd', + `"${tarPath}"`, '-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P' ] .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []), + .concat(IS_MAC ? ['--delay-directory-restore'] : []) + .concat(['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd']) + .join(' '), + undefined, {cwd: undefined} ) }) @@ -254,18 +427,20 @@ test('gzip list tar', async () => { await tar.listTar(archivePath, CompressionMethod.Gzip) - const tarPath = IS_WINDOWS - ? `${process.env['windir']}\\System32\\tar.exe` - : defaultTarPath + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${tarPath}"`, [ - '-z', + `"${tarPath}"`, '-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P' - ].concat(IS_MAC ? ['--delay-directory-restore'] : []), + ] + .concat(IS_WINDOWS ? ['--force-local'] : []) + .concat(IS_MAC ? ['--delay-directory-restore'] : []) + .concat(['-z']) + .join(' '), + undefined, {cwd: undefined} ) }) diff --git a/packages/cache/package-lock.json b/packages/cache/package-lock.json index 48f7f7e0..a34c4e3a 100644 --- a/packages/cache/package-lock.json +++ b/packages/cache/package-lock.json @@ -1,12 +1,12 @@ { "name": "@actions/cache", - "version": "3.0.6", + "version": "3.1.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@actions/cache", - "version": "3.0.6", + "version": "3.1.0", "license": "MIT", "dependencies": { "@actions/core": "^1.10.0", @@ -457,9 +457,9 @@ } }, "node_modules/minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "dependencies": { "brace-expansion": "^1.1.7" }, @@ -998,9 +998,9 @@ } }, "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "requires": { "brace-expansion": "^1.1.7" } diff --git a/packages/cache/package.json b/packages/cache/package.json index e2d6082c..44c677f8 100644 --- a/packages/cache/package.json +++ b/packages/cache/package.json @@ -1,6 +1,6 @@ { "name": "@actions/cache", - "version": "3.0.6", + "version": "3.1.0", "preview": true, "description": "Actions cache lib", "keywords": [ diff --git a/packages/cache/src/cache.ts b/packages/cache/src/cache.ts index 609c7f94..f928a2b9 100644 --- a/packages/cache/src/cache.ts +++ b/packages/cache/src/cache.ts @@ -4,6 +4,8 @@ import * as utils from './internal/cacheUtils' import * as cacheHttpClient from './internal/cacheHttpClient' import {createTar, extractTar, listTar} from './internal/tar' import {DownloadOptions, UploadOptions} from './options' +import {CompressionMethod} from './internal/constants' +import {ArtifactCacheEntry} from './internal/contracts' export class ValidationError extends Error { constructor(message: string) { @@ -85,17 +87,35 @@ export async function restoreCache( checkKey(key) } - const compressionMethod = await utils.getCompressionMethod() + let cacheEntry: ArtifactCacheEntry | null + let compressionMethod = await utils.getCompressionMethod() let archivePath = '' try { // path are needed to compute version - const cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, { + cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, { compressionMethod }) - if (!cacheEntry?.archiveLocation) { - // Cache not found - return undefined + // This is to support the old cache entry created by gzip on windows. + if ( + process.platform === 'win32' && + compressionMethod !== CompressionMethod.Gzip + ) { + compressionMethod = CompressionMethod.Gzip + cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, { + compressionMethod + }) + if (!cacheEntry?.archiveLocation) { + return undefined + } + + core.info( + "Couldn't find cache entry with zstd compression, falling back to gzip compression." + ) + } else { + // Cache not found + return undefined + } } archivePath = path.join( diff --git a/packages/cache/src/internal/cacheHttpClient.ts b/packages/cache/src/internal/cacheHttpClient.ts index d5ecd9a8..8982b8de 100644 --- a/packages/cache/src/internal/cacheHttpClient.ts +++ b/packages/cache/src/internal/cacheHttpClient.ts @@ -104,6 +104,7 @@ export async function getCacheEntry( const response = await retryTypedResponse('getCacheEntry', async () => httpClient.getJson(getCacheApiUrl(resource)) ) + // Cache not found if (response.statusCode === 204) { // List cache for primary key only if cache miss occurs if (core.isDebug()) { @@ -118,6 +119,7 @@ export async function getCacheEntry( const cacheResult = response.result const cacheDownloadUrl = cacheResult?.archiveLocation if (!cacheDownloadUrl) { + // Cache achiveLocation not found. This should never happen, and hence bail out. throw new Error('Cache not found.') } core.setSecret(cacheDownloadUrl) diff --git a/packages/cache/src/internal/cacheUtils.ts b/packages/cache/src/internal/cacheUtils.ts index c2ace526..ea1e7de6 100644 --- a/packages/cache/src/internal/cacheUtils.ts +++ b/packages/cache/src/internal/cacheUtils.ts @@ -7,7 +7,11 @@ import * as path from 'path' import * as semver from 'semver' import * as util from 'util' import {v4 as uuidV4} from 'uuid' -import {CacheFilename, CompressionMethod} from './constants' +import { + CacheFilename, + CompressionMethod, + GnuTarPathOnWindows +} from './constants' // From https://github.com/actions/toolkit/blob/main/packages/tool-cache/src/tool-cache.ts#L23 export async function createTempDirectory(): Promise { @@ -90,11 +94,6 @@ async function getVersion(app: string): Promise { // Use zstandard if possible to maximize cache performance export async function getCompressionMethod(): Promise { - if (process.platform === 'win32' && !(await isGnuTarInstalled())) { - // Disable zstd due to bug https://github.com/actions/cache/issues/301 - return CompressionMethod.Gzip - } - const versionOutput = await getVersion('zstd') const version = semver.clean(versionOutput) @@ -116,9 +115,12 @@ export function getCacheFileName(compressionMethod: CompressionMethod): string { : CacheFilename.Zstd } -export async function isGnuTarInstalled(): Promise { +export async function getGnuTarPathOnWindows(): Promise { + if (fs.existsSync(GnuTarPathOnWindows)) { + return GnuTarPathOnWindows + } const versionOutput = await getVersion('tar') - return versionOutput.toLowerCase().includes('gnu tar') + return versionOutput.toLowerCase().includes('gnu tar') ? io.which('tar') : '' } export function assertDefined(name: string, value?: T): T { diff --git a/packages/cache/src/internal/constants.ts b/packages/cache/src/internal/constants.ts index 2f78d326..4dbff574 100644 --- a/packages/cache/src/internal/constants.ts +++ b/packages/cache/src/internal/constants.ts @@ -11,6 +11,11 @@ export enum CompressionMethod { Zstd = 'zstd' } +export enum ArchiveToolType { + GNU = 'gnu', + BSD = 'bsd' +} + // The default number of retry attempts. export const DefaultRetryAttempts = 2 @@ -21,3 +26,13 @@ export const DefaultRetryDelay = 5000 // over the socket during this period, the socket is destroyed and the download // is aborted. export const SocketTimeout = 5000 + +// The default path of GNUtar on hosted Windows runners +export const GnuTarPathOnWindows = `${process.env['PROGRAMFILES']}\\Git\\usr\\bin\\tar.exe` + +// The default path of BSDtar on hosted Windows runners +export const SystemTarPathOnWindows = `${process.env['SYSTEMDRIVE']}\\Windows\\System32\\tar.exe` + +export const TarFilename = 'cache.tar' + +export const ManifestFilename = 'manifest.txt' diff --git a/packages/cache/src/internal/contracts.d.ts b/packages/cache/src/internal/contracts.d.ts index b5f53bdc..0519ff0a 100644 --- a/packages/cache/src/internal/contracts.d.ts +++ b/packages/cache/src/internal/contracts.d.ts @@ -37,3 +37,8 @@ export interface InternalCacheOptions { compressionMethod?: CompressionMethod cacheSize?: number } + +export interface ArchiveTool { + path: string + type: string +} diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 2e28ca1a..0af6a87a 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -3,25 +3,28 @@ import * as io from '@actions/io' import {existsSync, writeFileSync} from 'fs' import * as path from 'path' import * as utils from './cacheUtils' -import {CompressionMethod} from './constants' +import {ArchiveTool} from './contracts' +import { + CompressionMethod, + SystemTarPathOnWindows, + ArchiveToolType, + TarFilename, + ManifestFilename +} from './constants' const IS_WINDOWS = process.platform === 'win32' -async function getTarPath( - args: string[], - compressionMethod: CompressionMethod -): Promise { +// Returns tar path and type: BSD or GNU +async function getTarPath(): Promise { switch (process.platform) { case 'win32': { - const systemTar = `${process.env['windir']}\\System32\\tar.exe` - if (compressionMethod !== CompressionMethod.Gzip) { - // We only use zstandard compression on windows when gnu tar is installed due to - // a bug with compressing large files with bsdtar + zstd - args.push('--force-local') + const gnuTar = await utils.getGnuTarPathOnWindows() + const systemTar = SystemTarPathOnWindows + if (gnuTar) { + // Use GNUtar as default on windows + return {path: gnuTar, type: ArchiveToolType.GNU} } else if (existsSync(systemTar)) { - return systemTar - } else if (await utils.isGnuTarInstalled()) { - args.push('--force-local') + return {path: systemTar, type: ArchiveToolType.BSD} } break } @@ -29,27 +32,133 @@ async function getTarPath( const gnuTar = await io.which('gtar', false) if (gnuTar) { // fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527 - args.push('--delay-directory-restore') - return gnuTar + return {path: gnuTar, type: ArchiveToolType.GNU} + } else { + return { + path: await io.which('tar', true), + type: ArchiveToolType.BSD + } } - break } default: break } - return await io.which('tar', true) + // Default assumption is GNU tar is present in path + return { + path: await io.which('tar', true), + type: ArchiveToolType.GNU + } } -async function execTar( - args: string[], +// Return arguments for tar as per tarPath, compressionMethod, method type and os +async function getTarArgs( + tarPath: ArchiveTool, compressionMethod: CompressionMethod, - cwd?: string -): Promise { - try { - await exec(`"${await getTarPath(args, compressionMethod)}"`, args, {cwd}) - } catch (error) { - throw new Error(`Tar failed with error: ${error?.message}`) + type: string, + archivePath = '' +): Promise { + const args = [`"${tarPath.path}"`] + const cacheFileName = utils.getCacheFileName(compressionMethod) + const tarFile = 'cache.tar' + const workingDirectory = getWorkingDirectory() + // Speficic args for BSD tar on windows for workaround + const BSD_TAR_ZSTD = + tarPath.type === ArchiveToolType.BSD && + compressionMethod !== CompressionMethod.Gzip && + IS_WINDOWS + + // Method specific args + switch (type) { + case 'create': + args.push( + '--posix', + '-cf', + BSD_TAR_ZSTD + ? tarFile + : cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '--exclude', + BSD_TAR_ZSTD + ? tarFile + : cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '-P', + '-C', + workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '--files-from', + ManifestFilename + ) + break + case 'extract': + args.push( + '-xf', + BSD_TAR_ZSTD + ? tarFile + : archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '-P', + '-C', + workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/') + ) + break + case 'list': + args.push( + '-tf', + BSD_TAR_ZSTD + ? tarFile + : archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '-P' + ) + break } + + // Platform specific args + if (tarPath.type === ArchiveToolType.GNU) { + switch (process.platform) { + case 'win32': + args.push('--force-local') + break + case 'darwin': + args.push('--delay-directory-restore') + break + } + } + + return args +} + +// Returns commands to run tar and compression program +async function getCommands( + compressionMethod: CompressionMethod, + type: string, + archivePath = '' +): Promise { + let args + + const tarPath = await getTarPath() + const tarArgs = await getTarArgs( + tarPath, + compressionMethod, + type, + archivePath + ) + const compressionArgs = + type !== 'create' + ? await getDecompressionProgram(tarPath, compressionMethod, archivePath) + : await getCompressionProgram(tarPath, compressionMethod) + const BSD_TAR_ZSTD = + tarPath.type === ArchiveToolType.BSD && + compressionMethod !== CompressionMethod.Gzip && + IS_WINDOWS + + if (BSD_TAR_ZSTD && type !== 'create') { + args = [[...compressionArgs].join(' '), [...tarArgs].join(' ')] + } else { + args = [[...tarArgs].join(' '), [...compressionArgs].join(' ')] + } + + if (BSD_TAR_ZSTD) { + return args + } + + return [args.join(' ')] } function getWorkingDirectory(): string { @@ -57,37 +166,107 @@ function getWorkingDirectory(): string { } // Common function for extractTar and listTar to get the compression method -function getCompressionProgram(compressionMethod: CompressionMethod): string[] { +async function getDecompressionProgram( + tarPath: ArchiveTool, + compressionMethod: CompressionMethod, + archivePath: string +): Promise { // -d: Decompress. // unzstd is equivalent to 'zstd -d' // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // Using 30 here because we also support 32-bit self-hosted runners. + const BSD_TAR_ZSTD = + tarPath.type === ArchiveToolType.BSD && + compressionMethod !== CompressionMethod.Gzip && + IS_WINDOWS switch (compressionMethod) { case CompressionMethod.Zstd: - return [ - '--use-compress-program', - IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30' - ] + return BSD_TAR_ZSTD + ? [ + 'zstd -d --long=30 --force -o', + TarFilename, + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') + ] + : [ + '--use-compress-program', + IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30' + ] case CompressionMethod.ZstdWithoutLong: - return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd'] + return BSD_TAR_ZSTD + ? [ + 'zstd -d --force -o', + TarFilename, + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') + ] + : ['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd'] default: return ['-z'] } } +// Used for creating the archive +// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. +// zstdmt is equivalent to 'zstd -T0' +// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. +// Using 30 here because we also support 32-bit self-hosted runners. +// Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd. +async function getCompressionProgram( + tarPath: ArchiveTool, + compressionMethod: CompressionMethod +): Promise { + const cacheFileName = utils.getCacheFileName(compressionMethod) + const BSD_TAR_ZSTD = + tarPath.type === ArchiveToolType.BSD && + compressionMethod !== CompressionMethod.Gzip && + IS_WINDOWS + switch (compressionMethod) { + case CompressionMethod.Zstd: + return BSD_TAR_ZSTD + ? [ + 'zstd -T0 --long=30 --force -o', + cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + TarFilename + ] + : [ + '--use-compress-program', + IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30' + ] + case CompressionMethod.ZstdWithoutLong: + return BSD_TAR_ZSTD + ? [ + 'zstd -T0 --force -o', + cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + TarFilename + ] + : ['--use-compress-program', IS_WINDOWS ? '"zstd -T0"' : 'zstdmt'] + default: + return ['-z'] + } +} + +// Executes all commands as separate processes +async function execCommands(commands: string[], cwd?: string): Promise { + for (const command of commands) { + try { + await exec(command, undefined, {cwd}) + } catch (error) { + throw new Error( + `${command.split(' ')[0]} failed with error: ${error?.message}` + ) + } + } +} + +// List the contents of a tar export async function listTar( archivePath: string, compressionMethod: CompressionMethod ): Promise { - const args = [ - ...getCompressionProgram(compressionMethod), - '-tf', - archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P' - ] - await execTar(args, compressionMethod) + const commands = await getCommands(compressionMethod, 'list', archivePath) + await execCommands(commands) } +// Extract a tar export async function extractTar( archivePath: string, compressionMethod: CompressionMethod @@ -95,61 +274,21 @@ export async function extractTar( // Create directory to extract tar into const workingDirectory = getWorkingDirectory() await io.mkdirP(workingDirectory) - const args = [ - ...getCompressionProgram(compressionMethod), - '-xf', - archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P', - '-C', - workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/') - ] - await execTar(args, compressionMethod) + const commands = await getCommands(compressionMethod, 'extract', archivePath) + await execCommands(commands) } +// Create a tar export async function createTar( archiveFolder: string, sourceDirectories: string[], compressionMethod: CompressionMethod ): Promise { // Write source directories to manifest.txt to avoid command length limits - const manifestFilename = 'manifest.txt' - const cacheFileName = utils.getCacheFileName(compressionMethod) writeFileSync( - path.join(archiveFolder, manifestFilename), + path.join(archiveFolder, ManifestFilename), sourceDirectories.join('\n') ) - const workingDirectory = getWorkingDirectory() - - // -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. - // zstdmt is equivalent to 'zstd -T0' - // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. - // Using 30 here because we also support 32-bit self-hosted runners. - // Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd. - function getCompressionProgram(): string[] { - switch (compressionMethod) { - case CompressionMethod.Zstd: - return [ - '--use-compress-program', - IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30' - ] - case CompressionMethod.ZstdWithoutLong: - return ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt'] - default: - return ['-z'] - } - } - const args = [ - '--posix', - ...getCompressionProgram(), - '-cf', - cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '--exclude', - cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P', - '-C', - workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '--files-from', - manifestFilename - ] - await execTar(args, compressionMethod, archiveFolder) + const commands = await getCommands(compressionMethod, 'create') + await execCommands(commands, archiveFolder) } From 86fe4abd8e89005bbd880c330af444ae4c7d3b15 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Tue, 27 Dec 2022 16:00:28 +0530 Subject: [PATCH 12/12] Revert "Cache package release for compression change in windows" (#1289) * Revert "Cache package release for compression change in windows (#1281)" This reverts commit b2287326447a6d61c109af734947c7608be85900. * Update release version to patch --- .github/workflows/cache-windows-test.yml | 90 ----- packages/cache/RELEASES.md | 14 - packages/cache/__tests__/restoreCache.test.ts | 75 ----- packages/cache/__tests__/tar.test.ts | 273 +++------------- packages/cache/package-lock.json | 4 +- packages/cache/package.json | 2 +- packages/cache/src/cache.ts | 30 +- .../cache/src/internal/cacheHttpClient.ts | 2 - packages/cache/src/internal/cacheUtils.ts | 18 +- packages/cache/src/internal/constants.ts | 15 - packages/cache/src/internal/contracts.d.ts | 5 - packages/cache/src/internal/tar.ts | 307 +++++------------- 12 files changed, 149 insertions(+), 686 deletions(-) delete mode 100644 .github/workflows/cache-windows-test.yml diff --git a/.github/workflows/cache-windows-test.yml b/.github/workflows/cache-windows-test.yml deleted file mode 100644 index 3868f296..00000000 --- a/.github/workflows/cache-windows-test.yml +++ /dev/null @@ -1,90 +0,0 @@ -name: cache-windows-bsd-unit-tests -on: - push: - branches: - - main - paths-ignore: - - '**.md' - pull_request: - paths-ignore: - - '**.md' - -jobs: - build: - name: Build - - runs-on: windows-latest - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - shell: bash - run: | - rm "C:\Program Files\Git\usr\bin\tar.exe" - - - name: Set Node.js 12.x - uses: actions/setup-node@v1 - with: - node-version: 12.x - - # In order to save & restore cache from a shell script, certain env variables need to be set that are only available in the - # node context. This runs a local action that gets and sets the necessary env variables that are needed - - name: Set env variables - uses: ./packages/cache/__tests__/__fixtures__/ - - # Need root node_modules because certain npm packages like jest are configured for the entire repository and it won't be possible - # without these to just compile the cache package - - name: Install root npm packages - run: npm ci - - - name: Compile cache package - run: | - npm ci - npm run tsc - working-directory: packages/cache - - - name: Generate files in working directory - shell: bash - run: packages/cache/__tests__/create-cache-files.sh ${{ runner.os }} test-cache - - - name: Generate files outside working directory - shell: bash - run: packages/cache/__tests__/create-cache-files.sh ${{ runner.os }} ~/test-cache - - # We're using node -e to call the functions directly available in the @actions/cache package - - name: Save cache using saveCache() - run: | - node -e "Promise.resolve(require('./packages/cache/lib/cache').saveCache(['test-cache','~/test-cache'],'test-${{ runner.os }}-${{ github.run_id }}'))" - - - name: Delete cache folders before restoring - shell: bash - run: | - rm -rf test-cache - rm -rf ~/test-cache - - - name: Restore cache using restoreCache() with http-client - run: | - node -e "Promise.resolve(require('./packages/cache/lib/cache').restoreCache(['test-cache','~/test-cache'],'test-${{ runner.os }}-${{ github.run_id }}',[],{useAzureSdk: false}))" - - - name: Verify cache restored with http-client - shell: bash - run: | - packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} test-cache - packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} ~/test-cache - - - name: Delete cache folders before restoring - shell: bash - run: | - rm -rf test-cache - rm -rf ~/test-cache - - - name: Restore cache using restoreCache() with Azure SDK - run: | - node -e "Promise.resolve(require('./packages/cache/lib/cache').restoreCache(['test-cache','~/test-cache'],'test-${{ runner.os }}-${{ github.run_id }}'))" - - - name: Verify cache restored with Azure SDK - shell: bash - run: | - packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} test-cache - packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} ~/test-cache diff --git a/packages/cache/RELEASES.md b/packages/cache/RELEASES.md index 4a415486..73518e1c 100644 --- a/packages/cache/RELEASES.md +++ b/packages/cache/RELEASES.md @@ -91,17 +91,3 @@ ### 3.0.6 - Added `@azure/abort-controller` to dependencies to fix compatibility issue with ESM [#1208](https://github.com/actions/toolkit/issues/1208) - -### 3.1.0-beta.1 -- Update actions/cache on windows to use gnu tar and zstd by default and fallback to bsdtar and zstd if gnu tar is not available. ([issue](https://github.com/actions/cache/issues/984)) - -### 3.1.0-beta.2 -- Added support for fallback to gzip to restore old caches on windows. - -### 3.1.0-beta.3 -- Bug Fixes for fallback to gzip to restore old caches on windows and bsdtar if gnutar is not available. - -### 3.1.0 -- Update actions/cache on windows to use gnu tar and zstd by default -- Update actions/cache on windows to fallback to bsdtar and zstd if gnu tar is not available. -- Added support for fallback to gzip to restore old caches on windows. diff --git a/packages/cache/__tests__/restoreCache.test.ts b/packages/cache/__tests__/restoreCache.test.ts index 9cf45799..36ec8801 100644 --- a/packages/cache/__tests__/restoreCache.test.ts +++ b/packages/cache/__tests__/restoreCache.test.ts @@ -161,81 +161,6 @@ test('restore with gzip compressed cache found', async () => { expect(getCompressionMock).toHaveBeenCalledTimes(1) }) -test('restore with zstd as default but gzip compressed cache found on windows', async () => { - if (process.platform === 'win32') { - const paths = ['node_modules'] - const key = 'node-test' - - const cacheEntry: ArtifactCacheEntry = { - cacheKey: key, - scope: 'refs/heads/main', - archiveLocation: 'www.actionscache.test/download' - } - const getCacheMock = jest.spyOn(cacheHttpClient, 'getCacheEntry') - getCacheMock - .mockImplementationOnce(async () => { - return Promise.resolve(null) - }) - .mockImplementationOnce(async () => { - return Promise.resolve(cacheEntry) - }) - - const tempPath = '/foo/bar' - - const createTempDirectoryMock = jest.spyOn( - cacheUtils, - 'createTempDirectory' - ) - createTempDirectoryMock.mockImplementation(async () => { - return Promise.resolve(tempPath) - }) - - const archivePath = path.join(tempPath, CacheFilename.Gzip) - const downloadCacheMock = jest.spyOn(cacheHttpClient, 'downloadCache') - - const fileSize = 142 - const getArchiveFileSizeInBytesMock = jest - .spyOn(cacheUtils, 'getArchiveFileSizeInBytes') - .mockReturnValue(fileSize) - - const extractTarMock = jest.spyOn(tar, 'extractTar') - const unlinkFileMock = jest.spyOn(cacheUtils, 'unlinkFile') - - const compression = CompressionMethod.Zstd - const getCompressionMock = jest - .spyOn(cacheUtils, 'getCompressionMethod') - .mockReturnValue(Promise.resolve(compression)) - - const cacheKey = await restoreCache(paths, key) - - expect(cacheKey).toBe(key) - expect(getCacheMock).toHaveBeenNthCalledWith(1, [key], paths, { - compressionMethod: compression - }) - expect(getCacheMock).toHaveBeenNthCalledWith(2, [key], paths, { - compressionMethod: CompressionMethod.Gzip - }) - expect(createTempDirectoryMock).toHaveBeenCalledTimes(1) - expect(downloadCacheMock).toHaveBeenCalledWith( - cacheEntry.archiveLocation, - archivePath, - undefined - ) - expect(getArchiveFileSizeInBytesMock).toHaveBeenCalledWith(archivePath) - - expect(extractTarMock).toHaveBeenCalledTimes(1) - expect(extractTarMock).toHaveBeenCalledWith( - archivePath, - CompressionMethod.Gzip - ) - - expect(unlinkFileMock).toHaveBeenCalledTimes(1) - expect(unlinkFileMock).toHaveBeenCalledWith(archivePath) - - expect(getCompressionMock).toHaveBeenCalledTimes(1) - } -}) - test('restore with zstd compressed cache found', async () => { const paths = ['node_modules'] const key = 'node-test' diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index a33f4fab..e4233bc9 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -1,14 +1,7 @@ import * as exec from '@actions/exec' import * as io from '@actions/io' import * as path from 'path' -import { - CacheFilename, - CompressionMethod, - GnuTarPathOnWindows, - ManifestFilename, - SystemTarPathOnWindows, - TarFilename -} from '../src/internal/constants' +import {CacheFilename, CompressionMethod} from '../src/internal/constants' import * as tar from '../src/internal/tar' import * as utils from '../src/internal/cacheUtils' // eslint-disable-next-line @typescript-eslint/no-require-imports @@ -20,7 +13,7 @@ jest.mock('@actions/io') const IS_WINDOWS = process.platform === 'win32' const IS_MAC = process.platform === 'darwin' -const defaultTarPath = IS_MAC ? 'gtar' : 'tar' +const defaultTarPath = process.platform === 'darwin' ? 'gtar' : 'tar' function getTempDir(): string { return path.join(__dirname, '_temp', 'tar') @@ -35,10 +28,6 @@ beforeAll(async () => { await jest.requireActual('@actions/io').rmRF(getTempDir()) }) -beforeEach(async () => { - jest.restoreAllMocks() -}) - afterAll(async () => { delete process.env['GITHUB_WORKSPACE'] await jest.requireActual('@actions/io').rmRF(getTempDir()) @@ -52,15 +41,16 @@ test('zstd extract tar', async () => { ? `${process.env['windir']}\\fakepath\\cache.tar` : 'cache.tar' const workspace = process.env['GITHUB_WORKSPACE'] - const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath await tar.extractTar(archivePath, CompressionMethod.Zstd) expect(mkdirMock).toHaveBeenCalledWith(workspace) expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( + `"${defaultTarPath}"`, [ - `"${tarPath}"`, + '--use-compress-program', + IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30', '-xf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P', @@ -68,61 +58,11 @@ test('zstd extract tar', async () => { IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace ] .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []) - .concat([ - '--use-compress-program', - IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30' - ]) - .join(' '), - undefined, + .concat(IS_MAC ? ['--delay-directory-restore'] : []), {cwd: undefined} ) }) -test('zstd extract tar with windows BSDtar', async () => { - if (IS_WINDOWS) { - const mkdirMock = jest.spyOn(io, 'mkdirP') - const execMock = jest.spyOn(exec, 'exec') - jest - .spyOn(utils, 'getGnuTarPathOnWindows') - .mockReturnValue(Promise.resolve('')) - - const archivePath = `${process.env['windir']}\\fakepath\\cache.tar` - const workspace = process.env['GITHUB_WORKSPACE'] - const tarPath = SystemTarPathOnWindows - - await tar.extractTar(archivePath, CompressionMethod.Zstd) - - expect(mkdirMock).toHaveBeenCalledWith(workspace) - expect(execMock).toHaveBeenCalledTimes(2) - - expect(execMock).toHaveBeenNthCalledWith( - 1, - [ - 'zstd -d --long=30 --force -o', - TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') - ].join(' '), - undefined, - {cwd: undefined} - ) - - expect(execMock).toHaveBeenNthCalledWith( - 2, - [ - `"${tarPath}"`, - '-xf', - TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P', - '-C', - workspace?.replace(/\\/g, '/') - ].join(' '), - undefined, - {cwd: undefined} - ) - } -}) - test('gzip extract tar', async () => { const mkdirMock = jest.spyOn(io, 'mkdirP') const execMock = jest.spyOn(exec, 'exec') @@ -134,51 +74,50 @@ test('gzip extract tar', async () => { await tar.extractTar(archivePath, CompressionMethod.Gzip) expect(mkdirMock).toHaveBeenCalledWith(workspace) - const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath + const tarPath = IS_WINDOWS + ? `${process.env['windir']}\\System32\\tar.exe` + : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( + `"${tarPath}"`, [ - `"${tarPath}"`, + '-z', '-xf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P', '-C', IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace - ] - .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []) - .concat(['-z']) - .join(' '), - undefined, + ].concat(IS_MAC ? ['--delay-directory-restore'] : []), {cwd: undefined} ) }) -test('gzip extract GNU tar on windows with GNUtar in path', async () => { +test('gzip extract GNU tar on windows', async () => { if (IS_WINDOWS) { - // GNU tar present in path but not at default location - jest - .spyOn(utils, 'getGnuTarPathOnWindows') - .mockReturnValue(Promise.resolve('tar')) + jest.spyOn(fs, 'existsSync').mockReturnValueOnce(false) + + const isGnuMock = jest + .spyOn(utils, 'isGnuTarInstalled') + .mockReturnValue(Promise.resolve(true)) const execMock = jest.spyOn(exec, 'exec') const archivePath = `${process.env['windir']}\\fakepath\\cache.tar` const workspace = process.env['GITHUB_WORKSPACE'] await tar.extractTar(archivePath, CompressionMethod.Gzip) + expect(isGnuMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( + `"tar"`, [ - `"tar"`, + '-z', '-xf', archivePath.replace(/\\/g, '/'), '-P', '-C', workspace?.replace(/\\/g, '/'), - '--force-local', - '-z' - ].join(' '), - undefined, + '--force-local' + ], {cwd: undefined} ) } @@ -195,13 +134,13 @@ test('zstd create tar', async () => { await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Zstd) - const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath - expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( + `"${defaultTarPath}"`, [ - `"${tarPath}"`, '--posix', + '--use-compress-program', + IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30', '-cf', IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd, '--exclude', @@ -210,81 +149,16 @@ test('zstd create tar', async () => { '-C', IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, '--files-from', - ManifestFilename + 'manifest.txt' ] .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []) - .concat([ - '--use-compress-program', - IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30' - ]) - .join(' '), - undefined, // args + .concat(IS_MAC ? ['--delay-directory-restore'] : []), { cwd: archiveFolder } ) }) -test('zstd create tar with windows BSDtar', async () => { - if (IS_WINDOWS) { - const execMock = jest.spyOn(exec, 'exec') - jest - .spyOn(utils, 'getGnuTarPathOnWindows') - .mockReturnValue(Promise.resolve('')) - - const archiveFolder = getTempDir() - const workspace = process.env['GITHUB_WORKSPACE'] - const sourceDirectories = ['~/.npm/cache', `${workspace}/dist`] - - await fs.promises.mkdir(archiveFolder, {recursive: true}) - - await tar.createTar( - archiveFolder, - sourceDirectories, - CompressionMethod.Zstd - ) - - const tarPath = SystemTarPathOnWindows - - expect(execMock).toHaveBeenCalledTimes(2) - - expect(execMock).toHaveBeenNthCalledWith( - 1, - [ - `"${tarPath}"`, - '--posix', - '-cf', - TarFilename.replace(/\\/g, '/'), - '--exclude', - TarFilename.replace(/\\/g, '/'), - '-P', - '-C', - workspace?.replace(/\\/g, '/'), - '--files-from', - ManifestFilename - ].join(' '), - undefined, // args - { - cwd: archiveFolder - } - ) - - expect(execMock).toHaveBeenNthCalledWith( - 2, - [ - 'zstd -T0 --long=30 --force -o', - CacheFilename.Zstd.replace(/\\/g, '/'), - TarFilename.replace(/\\/g, '/') - ].join(' '), - undefined, // args - { - cwd: archiveFolder - } - ) - } -}) - test('gzip create tar', async () => { const execMock = jest.spyOn(exec, 'exec') @@ -296,13 +170,16 @@ test('gzip create tar', async () => { await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Gzip) - const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath + const tarPath = IS_WINDOWS + ? `${process.env['windir']}\\System32\\tar.exe` + : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( + `"${tarPath}"`, [ - `"${tarPath}"`, '--posix', + '-z', '-cf', IS_WINDOWS ? CacheFilename.Gzip.replace(/\\/g, '/') : CacheFilename.Gzip, '--exclude', @@ -311,13 +188,8 @@ test('gzip create tar', async () => { '-C', IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, '--files-from', - ManifestFilename - ] - .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []) - .concat(['-z']) - .join(' '), - undefined, // args + 'manifest.txt' + ].concat(IS_MAC ? ['--delay-directory-restore'] : []), { cwd: archiveFolder } @@ -333,65 +205,22 @@ test('zstd list tar', async () => { await tar.listTar(archivePath, CompressionMethod.Zstd) - const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( + `"${defaultTarPath}"`, [ - `"${tarPath}"`, + '--use-compress-program', + IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30', '-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P' ] .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []) - .concat([ - '--use-compress-program', - IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30' - ]) - .join(' '), - undefined, + .concat(IS_MAC ? ['--delay-directory-restore'] : []), {cwd: undefined} ) }) -test('zstd list tar with windows BSDtar', async () => { - if (IS_WINDOWS) { - const execMock = jest.spyOn(exec, 'exec') - jest - .spyOn(utils, 'getGnuTarPathOnWindows') - .mockReturnValue(Promise.resolve('')) - const archivePath = `${process.env['windir']}\\fakepath\\cache.tar` - - await tar.listTar(archivePath, CompressionMethod.Zstd) - - const tarPath = SystemTarPathOnWindows - expect(execMock).toHaveBeenCalledTimes(2) - - expect(execMock).toHaveBeenNthCalledWith( - 1, - [ - 'zstd -d --long=30 --force -o', - TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') - ].join(' '), - undefined, - {cwd: undefined} - ) - - expect(execMock).toHaveBeenNthCalledWith( - 2, - [ - `"${tarPath}"`, - '-tf', - TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P' - ].join(' '), - undefined, - {cwd: undefined} - ) - } -}) - test('zstdWithoutLong list tar', async () => { const execMock = jest.spyOn(exec, 'exec') @@ -401,20 +230,18 @@ test('zstdWithoutLong list tar', async () => { await tar.listTar(archivePath, CompressionMethod.ZstdWithoutLong) - const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( + `"${defaultTarPath}"`, [ - `"${tarPath}"`, + '--use-compress-program', + IS_WINDOWS ? 'zstd -d' : 'unzstd', '-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P' ] .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []) - .concat(['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd']) - .join(' '), - undefined, + .concat(IS_MAC ? ['--delay-directory-restore'] : []), {cwd: undefined} ) }) @@ -427,20 +254,18 @@ test('gzip list tar', async () => { await tar.listTar(archivePath, CompressionMethod.Gzip) - const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath + const tarPath = IS_WINDOWS + ? `${process.env['windir']}\\System32\\tar.exe` + : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( + `"${tarPath}"`, [ - `"${tarPath}"`, + '-z', '-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P' - ] - .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []) - .concat(['-z']) - .join(' '), - undefined, + ].concat(IS_MAC ? ['--delay-directory-restore'] : []), {cwd: undefined} ) }) diff --git a/packages/cache/package-lock.json b/packages/cache/package-lock.json index a34c4e3a..52afe053 100644 --- a/packages/cache/package-lock.json +++ b/packages/cache/package-lock.json @@ -1,12 +1,12 @@ { "name": "@actions/cache", - "version": "3.1.0", + "version": "3.1.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@actions/cache", - "version": "3.1.0", + "version": "3.1.1", "license": "MIT", "dependencies": { "@actions/core": "^1.10.0", diff --git a/packages/cache/package.json b/packages/cache/package.json index 44c677f8..ccd37fa1 100644 --- a/packages/cache/package.json +++ b/packages/cache/package.json @@ -1,6 +1,6 @@ { "name": "@actions/cache", - "version": "3.1.0", + "version": "3.1.1", "preview": true, "description": "Actions cache lib", "keywords": [ diff --git a/packages/cache/src/cache.ts b/packages/cache/src/cache.ts index f928a2b9..609c7f94 100644 --- a/packages/cache/src/cache.ts +++ b/packages/cache/src/cache.ts @@ -4,8 +4,6 @@ import * as utils from './internal/cacheUtils' import * as cacheHttpClient from './internal/cacheHttpClient' import {createTar, extractTar, listTar} from './internal/tar' import {DownloadOptions, UploadOptions} from './options' -import {CompressionMethod} from './internal/constants' -import {ArtifactCacheEntry} from './internal/contracts' export class ValidationError extends Error { constructor(message: string) { @@ -87,35 +85,17 @@ export async function restoreCache( checkKey(key) } - let cacheEntry: ArtifactCacheEntry | null - let compressionMethod = await utils.getCompressionMethod() + const compressionMethod = await utils.getCompressionMethod() let archivePath = '' try { // path are needed to compute version - cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, { + const cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, { compressionMethod }) - if (!cacheEntry?.archiveLocation) { - // This is to support the old cache entry created by gzip on windows. - if ( - process.platform === 'win32' && - compressionMethod !== CompressionMethod.Gzip - ) { - compressionMethod = CompressionMethod.Gzip - cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, { - compressionMethod - }) - if (!cacheEntry?.archiveLocation) { - return undefined - } - core.info( - "Couldn't find cache entry with zstd compression, falling back to gzip compression." - ) - } else { - // Cache not found - return undefined - } + if (!cacheEntry?.archiveLocation) { + // Cache not found + return undefined } archivePath = path.join( diff --git a/packages/cache/src/internal/cacheHttpClient.ts b/packages/cache/src/internal/cacheHttpClient.ts index 8982b8de..d5ecd9a8 100644 --- a/packages/cache/src/internal/cacheHttpClient.ts +++ b/packages/cache/src/internal/cacheHttpClient.ts @@ -104,7 +104,6 @@ export async function getCacheEntry( const response = await retryTypedResponse('getCacheEntry', async () => httpClient.getJson(getCacheApiUrl(resource)) ) - // Cache not found if (response.statusCode === 204) { // List cache for primary key only if cache miss occurs if (core.isDebug()) { @@ -119,7 +118,6 @@ export async function getCacheEntry( const cacheResult = response.result const cacheDownloadUrl = cacheResult?.archiveLocation if (!cacheDownloadUrl) { - // Cache achiveLocation not found. This should never happen, and hence bail out. throw new Error('Cache not found.') } core.setSecret(cacheDownloadUrl) diff --git a/packages/cache/src/internal/cacheUtils.ts b/packages/cache/src/internal/cacheUtils.ts index ea1e7de6..c2ace526 100644 --- a/packages/cache/src/internal/cacheUtils.ts +++ b/packages/cache/src/internal/cacheUtils.ts @@ -7,11 +7,7 @@ import * as path from 'path' import * as semver from 'semver' import * as util from 'util' import {v4 as uuidV4} from 'uuid' -import { - CacheFilename, - CompressionMethod, - GnuTarPathOnWindows -} from './constants' +import {CacheFilename, CompressionMethod} from './constants' // From https://github.com/actions/toolkit/blob/main/packages/tool-cache/src/tool-cache.ts#L23 export async function createTempDirectory(): Promise { @@ -94,6 +90,11 @@ async function getVersion(app: string): Promise { // Use zstandard if possible to maximize cache performance export async function getCompressionMethod(): Promise { + if (process.platform === 'win32' && !(await isGnuTarInstalled())) { + // Disable zstd due to bug https://github.com/actions/cache/issues/301 + return CompressionMethod.Gzip + } + const versionOutput = await getVersion('zstd') const version = semver.clean(versionOutput) @@ -115,12 +116,9 @@ export function getCacheFileName(compressionMethod: CompressionMethod): string { : CacheFilename.Zstd } -export async function getGnuTarPathOnWindows(): Promise { - if (fs.existsSync(GnuTarPathOnWindows)) { - return GnuTarPathOnWindows - } +export async function isGnuTarInstalled(): Promise { const versionOutput = await getVersion('tar') - return versionOutput.toLowerCase().includes('gnu tar') ? io.which('tar') : '' + return versionOutput.toLowerCase().includes('gnu tar') } export function assertDefined(name: string, value?: T): T { diff --git a/packages/cache/src/internal/constants.ts b/packages/cache/src/internal/constants.ts index 4dbff574..2f78d326 100644 --- a/packages/cache/src/internal/constants.ts +++ b/packages/cache/src/internal/constants.ts @@ -11,11 +11,6 @@ export enum CompressionMethod { Zstd = 'zstd' } -export enum ArchiveToolType { - GNU = 'gnu', - BSD = 'bsd' -} - // The default number of retry attempts. export const DefaultRetryAttempts = 2 @@ -26,13 +21,3 @@ export const DefaultRetryDelay = 5000 // over the socket during this period, the socket is destroyed and the download // is aborted. export const SocketTimeout = 5000 - -// The default path of GNUtar on hosted Windows runners -export const GnuTarPathOnWindows = `${process.env['PROGRAMFILES']}\\Git\\usr\\bin\\tar.exe` - -// The default path of BSDtar on hosted Windows runners -export const SystemTarPathOnWindows = `${process.env['SYSTEMDRIVE']}\\Windows\\System32\\tar.exe` - -export const TarFilename = 'cache.tar' - -export const ManifestFilename = 'manifest.txt' diff --git a/packages/cache/src/internal/contracts.d.ts b/packages/cache/src/internal/contracts.d.ts index 0519ff0a..b5f53bdc 100644 --- a/packages/cache/src/internal/contracts.d.ts +++ b/packages/cache/src/internal/contracts.d.ts @@ -37,8 +37,3 @@ export interface InternalCacheOptions { compressionMethod?: CompressionMethod cacheSize?: number } - -export interface ArchiveTool { - path: string - type: string -} diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 0af6a87a..2e28ca1a 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -3,28 +3,25 @@ import * as io from '@actions/io' import {existsSync, writeFileSync} from 'fs' import * as path from 'path' import * as utils from './cacheUtils' -import {ArchiveTool} from './contracts' -import { - CompressionMethod, - SystemTarPathOnWindows, - ArchiveToolType, - TarFilename, - ManifestFilename -} from './constants' +import {CompressionMethod} from './constants' const IS_WINDOWS = process.platform === 'win32' -// Returns tar path and type: BSD or GNU -async function getTarPath(): Promise { +async function getTarPath( + args: string[], + compressionMethod: CompressionMethod +): Promise { switch (process.platform) { case 'win32': { - const gnuTar = await utils.getGnuTarPathOnWindows() - const systemTar = SystemTarPathOnWindows - if (gnuTar) { - // Use GNUtar as default on windows - return {path: gnuTar, type: ArchiveToolType.GNU} + const systemTar = `${process.env['windir']}\\System32\\tar.exe` + if (compressionMethod !== CompressionMethod.Gzip) { + // We only use zstandard compression on windows when gnu tar is installed due to + // a bug with compressing large files with bsdtar + zstd + args.push('--force-local') } else if (existsSync(systemTar)) { - return {path: systemTar, type: ArchiveToolType.BSD} + return systemTar + } else if (await utils.isGnuTarInstalled()) { + args.push('--force-local') } break } @@ -32,133 +29,27 @@ async function getTarPath(): Promise { const gnuTar = await io.which('gtar', false) if (gnuTar) { // fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527 - return {path: gnuTar, type: ArchiveToolType.GNU} - } else { - return { - path: await io.which('tar', true), - type: ArchiveToolType.BSD - } + args.push('--delay-directory-restore') + return gnuTar } + break } default: break } - // Default assumption is GNU tar is present in path - return { - path: await io.which('tar', true), - type: ArchiveToolType.GNU - } + return await io.which('tar', true) } -// Return arguments for tar as per tarPath, compressionMethod, method type and os -async function getTarArgs( - tarPath: ArchiveTool, +async function execTar( + args: string[], compressionMethod: CompressionMethod, - type: string, - archivePath = '' -): Promise { - const args = [`"${tarPath.path}"`] - const cacheFileName = utils.getCacheFileName(compressionMethod) - const tarFile = 'cache.tar' - const workingDirectory = getWorkingDirectory() - // Speficic args for BSD tar on windows for workaround - const BSD_TAR_ZSTD = - tarPath.type === ArchiveToolType.BSD && - compressionMethod !== CompressionMethod.Gzip && - IS_WINDOWS - - // Method specific args - switch (type) { - case 'create': - args.push( - '--posix', - '-cf', - BSD_TAR_ZSTD - ? tarFile - : cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '--exclude', - BSD_TAR_ZSTD - ? tarFile - : cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P', - '-C', - workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '--files-from', - ManifestFilename - ) - break - case 'extract': - args.push( - '-xf', - BSD_TAR_ZSTD - ? tarFile - : archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P', - '-C', - workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/') - ) - break - case 'list': - args.push( - '-tf', - BSD_TAR_ZSTD - ? tarFile - : archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P' - ) - break + cwd?: string +): Promise { + try { + await exec(`"${await getTarPath(args, compressionMethod)}"`, args, {cwd}) + } catch (error) { + throw new Error(`Tar failed with error: ${error?.message}`) } - - // Platform specific args - if (tarPath.type === ArchiveToolType.GNU) { - switch (process.platform) { - case 'win32': - args.push('--force-local') - break - case 'darwin': - args.push('--delay-directory-restore') - break - } - } - - return args -} - -// Returns commands to run tar and compression program -async function getCommands( - compressionMethod: CompressionMethod, - type: string, - archivePath = '' -): Promise { - let args - - const tarPath = await getTarPath() - const tarArgs = await getTarArgs( - tarPath, - compressionMethod, - type, - archivePath - ) - const compressionArgs = - type !== 'create' - ? await getDecompressionProgram(tarPath, compressionMethod, archivePath) - : await getCompressionProgram(tarPath, compressionMethod) - const BSD_TAR_ZSTD = - tarPath.type === ArchiveToolType.BSD && - compressionMethod !== CompressionMethod.Gzip && - IS_WINDOWS - - if (BSD_TAR_ZSTD && type !== 'create') { - args = [[...compressionArgs].join(' '), [...tarArgs].join(' ')] - } else { - args = [[...tarArgs].join(' '), [...compressionArgs].join(' ')] - } - - if (BSD_TAR_ZSTD) { - return args - } - - return [args.join(' ')] } function getWorkingDirectory(): string { @@ -166,107 +57,37 @@ function getWorkingDirectory(): string { } // Common function for extractTar and listTar to get the compression method -async function getDecompressionProgram( - tarPath: ArchiveTool, - compressionMethod: CompressionMethod, - archivePath: string -): Promise { +function getCompressionProgram(compressionMethod: CompressionMethod): string[] { // -d: Decompress. // unzstd is equivalent to 'zstd -d' // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // Using 30 here because we also support 32-bit self-hosted runners. - const BSD_TAR_ZSTD = - tarPath.type === ArchiveToolType.BSD && - compressionMethod !== CompressionMethod.Gzip && - IS_WINDOWS switch (compressionMethod) { case CompressionMethod.Zstd: - return BSD_TAR_ZSTD - ? [ - 'zstd -d --long=30 --force -o', - TarFilename, - archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') - ] - : [ - '--use-compress-program', - IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30' - ] + return [ + '--use-compress-program', + IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30' + ] case CompressionMethod.ZstdWithoutLong: - return BSD_TAR_ZSTD - ? [ - 'zstd -d --force -o', - TarFilename, - archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') - ] - : ['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd'] + return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd'] default: return ['-z'] } } -// Used for creating the archive -// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. -// zstdmt is equivalent to 'zstd -T0' -// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. -// Using 30 here because we also support 32-bit self-hosted runners. -// Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd. -async function getCompressionProgram( - tarPath: ArchiveTool, - compressionMethod: CompressionMethod -): Promise { - const cacheFileName = utils.getCacheFileName(compressionMethod) - const BSD_TAR_ZSTD = - tarPath.type === ArchiveToolType.BSD && - compressionMethod !== CompressionMethod.Gzip && - IS_WINDOWS - switch (compressionMethod) { - case CompressionMethod.Zstd: - return BSD_TAR_ZSTD - ? [ - 'zstd -T0 --long=30 --force -o', - cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - TarFilename - ] - : [ - '--use-compress-program', - IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30' - ] - case CompressionMethod.ZstdWithoutLong: - return BSD_TAR_ZSTD - ? [ - 'zstd -T0 --force -o', - cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - TarFilename - ] - : ['--use-compress-program', IS_WINDOWS ? '"zstd -T0"' : 'zstdmt'] - default: - return ['-z'] - } -} - -// Executes all commands as separate processes -async function execCommands(commands: string[], cwd?: string): Promise { - for (const command of commands) { - try { - await exec(command, undefined, {cwd}) - } catch (error) { - throw new Error( - `${command.split(' ')[0]} failed with error: ${error?.message}` - ) - } - } -} - -// List the contents of a tar export async function listTar( archivePath: string, compressionMethod: CompressionMethod ): Promise { - const commands = await getCommands(compressionMethod, 'list', archivePath) - await execCommands(commands) + const args = [ + ...getCompressionProgram(compressionMethod), + '-tf', + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '-P' + ] + await execTar(args, compressionMethod) } -// Extract a tar export async function extractTar( archivePath: string, compressionMethod: CompressionMethod @@ -274,21 +95,61 @@ export async function extractTar( // Create directory to extract tar into const workingDirectory = getWorkingDirectory() await io.mkdirP(workingDirectory) - const commands = await getCommands(compressionMethod, 'extract', archivePath) - await execCommands(commands) + const args = [ + ...getCompressionProgram(compressionMethod), + '-xf', + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '-P', + '-C', + workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/') + ] + await execTar(args, compressionMethod) } -// Create a tar export async function createTar( archiveFolder: string, sourceDirectories: string[], compressionMethod: CompressionMethod ): Promise { // Write source directories to manifest.txt to avoid command length limits + const manifestFilename = 'manifest.txt' + const cacheFileName = utils.getCacheFileName(compressionMethod) writeFileSync( - path.join(archiveFolder, ManifestFilename), + path.join(archiveFolder, manifestFilename), sourceDirectories.join('\n') ) - const commands = await getCommands(compressionMethod, 'create') - await execCommands(commands, archiveFolder) + const workingDirectory = getWorkingDirectory() + + // -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. + // zstdmt is equivalent to 'zstd -T0' + // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. + // Using 30 here because we also support 32-bit self-hosted runners. + // Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd. + function getCompressionProgram(): string[] { + switch (compressionMethod) { + case CompressionMethod.Zstd: + return [ + '--use-compress-program', + IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30' + ] + case CompressionMethod.ZstdWithoutLong: + return ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt'] + default: + return ['-z'] + } + } + const args = [ + '--posix', + ...getCompressionProgram(), + '-cf', + cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '--exclude', + cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '-P', + '-C', + workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '--files-from', + manifestFilename + ] + await execTar(args, compressionMethod, archiveFolder) }