From b2287326447a6d61c109af734947c7608be85900 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Thu, 22 Dec 2022 20:47:35 +0530 Subject: [PATCH] Cache package release for compression change in windows (#1281) * bsd + zstd fallback implementation * bsd + zstd fallback implementation * Fix tar operations * Add -v option for testing * Fix order of args for tar * Add GNUtar as default on windows * Fix test * Fix tar tests * Fix lint issues * Fix windows gnutar test case * Temporarily remove thhe condition that prevents zstd usage on windows unless with GNUtar * Address some comments and correct compression commands * Add windows bsdtar test * Fix windows test * Fix test * Separate args * Fix old tests * Add new tests * Fix tests * Fix lint test * Refactor code * Address review comments * Fix test * Fix tar test * Add await to async function calls * Fix test * Update for beta release * Fix audit issues * Add fallback to gzip compression if cache not found * Fix test * Add test * Address review comments * Revert Address review comments * Release 3.1.0-beta.2 cache package * Fix issues * Reconfigure catch block * Add debug logging for gzip fall back * Fix test * Add end to end test for cache using bsd on windows and address review comments * Fix test * Fix test * Fix tests * Add better comments * Update packages/cache/src/internal/cacheHttpClient.ts Co-authored-by: Bishal Prasad * Address review comments * Update for new beta cache package release * Address bugbash issues * Fix tests * Release new actions/cache minor version Co-authored-by: Lovepreet Singh Co-authored-by: Bishal Prasad --- .github/workflows/cache-windows-test.yml | 90 +++++ packages/cache/RELEASES.md | 14 + packages/cache/__tests__/restoreCache.test.ts | 75 +++++ packages/cache/__tests__/tar.test.ts | 273 +++++++++++++--- packages/cache/package-lock.json | 16 +- packages/cache/package.json | 2 +- packages/cache/src/cache.ts | 30 +- .../cache/src/internal/cacheHttpClient.ts | 2 + packages/cache/src/internal/cacheUtils.ts | 18 +- packages/cache/src/internal/constants.ts | 15 + packages/cache/src/internal/contracts.d.ts | 5 + packages/cache/src/internal/tar.ts | 307 +++++++++++++----- 12 files changed, 692 insertions(+), 155 deletions(-) create mode 100644 .github/workflows/cache-windows-test.yml diff --git a/.github/workflows/cache-windows-test.yml b/.github/workflows/cache-windows-test.yml new file mode 100644 index 00000000..3868f296 --- /dev/null +++ b/.github/workflows/cache-windows-test.yml @@ -0,0 +1,90 @@ +name: cache-windows-bsd-unit-tests +on: + push: + branches: + - main + paths-ignore: + - '**.md' + pull_request: + paths-ignore: + - '**.md' + +jobs: + build: + name: Build + + runs-on: windows-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - shell: bash + run: | + rm "C:\Program Files\Git\usr\bin\tar.exe" + + - name: Set Node.js 12.x + uses: actions/setup-node@v1 + with: + node-version: 12.x + + # In order to save & restore cache from a shell script, certain env variables need to be set that are only available in the + # node context. This runs a local action that gets and sets the necessary env variables that are needed + - name: Set env variables + uses: ./packages/cache/__tests__/__fixtures__/ + + # Need root node_modules because certain npm packages like jest are configured for the entire repository and it won't be possible + # without these to just compile the cache package + - name: Install root npm packages + run: npm ci + + - name: Compile cache package + run: | + npm ci + npm run tsc + working-directory: packages/cache + + - name: Generate files in working directory + shell: bash + run: packages/cache/__tests__/create-cache-files.sh ${{ runner.os }} test-cache + + - name: Generate files outside working directory + shell: bash + run: packages/cache/__tests__/create-cache-files.sh ${{ runner.os }} ~/test-cache + + # We're using node -e to call the functions directly available in the @actions/cache package + - name: Save cache using saveCache() + run: | + node -e "Promise.resolve(require('./packages/cache/lib/cache').saveCache(['test-cache','~/test-cache'],'test-${{ runner.os }}-${{ github.run_id }}'))" + + - name: Delete cache folders before restoring + shell: bash + run: | + rm -rf test-cache + rm -rf ~/test-cache + + - name: Restore cache using restoreCache() with http-client + run: | + node -e "Promise.resolve(require('./packages/cache/lib/cache').restoreCache(['test-cache','~/test-cache'],'test-${{ runner.os }}-${{ github.run_id }}',[],{useAzureSdk: false}))" + + - name: Verify cache restored with http-client + shell: bash + run: | + packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} test-cache + packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} ~/test-cache + + - name: Delete cache folders before restoring + shell: bash + run: | + rm -rf test-cache + rm -rf ~/test-cache + + - name: Restore cache using restoreCache() with Azure SDK + run: | + node -e "Promise.resolve(require('./packages/cache/lib/cache').restoreCache(['test-cache','~/test-cache'],'test-${{ runner.os }}-${{ github.run_id }}'))" + + - name: Verify cache restored with Azure SDK + shell: bash + run: | + packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} test-cache + packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} ~/test-cache diff --git a/packages/cache/RELEASES.md b/packages/cache/RELEASES.md index 73518e1c..4a415486 100644 --- a/packages/cache/RELEASES.md +++ b/packages/cache/RELEASES.md @@ -91,3 +91,17 @@ ### 3.0.6 - Added `@azure/abort-controller` to dependencies to fix compatibility issue with ESM [#1208](https://github.com/actions/toolkit/issues/1208) + +### 3.1.0-beta.1 +- Update actions/cache on windows to use gnu tar and zstd by default and fallback to bsdtar and zstd if gnu tar is not available. ([issue](https://github.com/actions/cache/issues/984)) + +### 3.1.0-beta.2 +- Added support for fallback to gzip to restore old caches on windows. + +### 3.1.0-beta.3 +- Bug Fixes for fallback to gzip to restore old caches on windows and bsdtar if gnutar is not available. + +### 3.1.0 +- Update actions/cache on windows to use gnu tar and zstd by default +- Update actions/cache on windows to fallback to bsdtar and zstd if gnu tar is not available. +- Added support for fallback to gzip to restore old caches on windows. diff --git a/packages/cache/__tests__/restoreCache.test.ts b/packages/cache/__tests__/restoreCache.test.ts index 36ec8801..9cf45799 100644 --- a/packages/cache/__tests__/restoreCache.test.ts +++ b/packages/cache/__tests__/restoreCache.test.ts @@ -161,6 +161,81 @@ test('restore with gzip compressed cache found', async () => { expect(getCompressionMock).toHaveBeenCalledTimes(1) }) +test('restore with zstd as default but gzip compressed cache found on windows', async () => { + if (process.platform === 'win32') { + const paths = ['node_modules'] + const key = 'node-test' + + const cacheEntry: ArtifactCacheEntry = { + cacheKey: key, + scope: 'refs/heads/main', + archiveLocation: 'www.actionscache.test/download' + } + const getCacheMock = jest.spyOn(cacheHttpClient, 'getCacheEntry') + getCacheMock + .mockImplementationOnce(async () => { + return Promise.resolve(null) + }) + .mockImplementationOnce(async () => { + return Promise.resolve(cacheEntry) + }) + + const tempPath = '/foo/bar' + + const createTempDirectoryMock = jest.spyOn( + cacheUtils, + 'createTempDirectory' + ) + createTempDirectoryMock.mockImplementation(async () => { + return Promise.resolve(tempPath) + }) + + const archivePath = path.join(tempPath, CacheFilename.Gzip) + const downloadCacheMock = jest.spyOn(cacheHttpClient, 'downloadCache') + + const fileSize = 142 + const getArchiveFileSizeInBytesMock = jest + .spyOn(cacheUtils, 'getArchiveFileSizeInBytes') + .mockReturnValue(fileSize) + + const extractTarMock = jest.spyOn(tar, 'extractTar') + const unlinkFileMock = jest.spyOn(cacheUtils, 'unlinkFile') + + const compression = CompressionMethod.Zstd + const getCompressionMock = jest + .spyOn(cacheUtils, 'getCompressionMethod') + .mockReturnValue(Promise.resolve(compression)) + + const cacheKey = await restoreCache(paths, key) + + expect(cacheKey).toBe(key) + expect(getCacheMock).toHaveBeenNthCalledWith(1, [key], paths, { + compressionMethod: compression + }) + expect(getCacheMock).toHaveBeenNthCalledWith(2, [key], paths, { + compressionMethod: CompressionMethod.Gzip + }) + expect(createTempDirectoryMock).toHaveBeenCalledTimes(1) + expect(downloadCacheMock).toHaveBeenCalledWith( + cacheEntry.archiveLocation, + archivePath, + undefined + ) + expect(getArchiveFileSizeInBytesMock).toHaveBeenCalledWith(archivePath) + + expect(extractTarMock).toHaveBeenCalledTimes(1) + expect(extractTarMock).toHaveBeenCalledWith( + archivePath, + CompressionMethod.Gzip + ) + + expect(unlinkFileMock).toHaveBeenCalledTimes(1) + expect(unlinkFileMock).toHaveBeenCalledWith(archivePath) + + expect(getCompressionMock).toHaveBeenCalledTimes(1) + } +}) + test('restore with zstd compressed cache found', async () => { const paths = ['node_modules'] const key = 'node-test' diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index e4233bc9..a33f4fab 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -1,7 +1,14 @@ import * as exec from '@actions/exec' import * as io from '@actions/io' import * as path from 'path' -import {CacheFilename, CompressionMethod} from '../src/internal/constants' +import { + CacheFilename, + CompressionMethod, + GnuTarPathOnWindows, + ManifestFilename, + SystemTarPathOnWindows, + TarFilename +} from '../src/internal/constants' import * as tar from '../src/internal/tar' import * as utils from '../src/internal/cacheUtils' // eslint-disable-next-line @typescript-eslint/no-require-imports @@ -13,7 +20,7 @@ jest.mock('@actions/io') const IS_WINDOWS = process.platform === 'win32' const IS_MAC = process.platform === 'darwin' -const defaultTarPath = process.platform === 'darwin' ? 'gtar' : 'tar' +const defaultTarPath = IS_MAC ? 'gtar' : 'tar' function getTempDir(): string { return path.join(__dirname, '_temp', 'tar') @@ -28,6 +35,10 @@ beforeAll(async () => { await jest.requireActual('@actions/io').rmRF(getTempDir()) }) +beforeEach(async () => { + jest.restoreAllMocks() +}) + afterAll(async () => { delete process.env['GITHUB_WORKSPACE'] await jest.requireActual('@actions/io').rmRF(getTempDir()) @@ -41,16 +52,15 @@ test('zstd extract tar', async () => { ? `${process.env['windir']}\\fakepath\\cache.tar` : 'cache.tar' const workspace = process.env['GITHUB_WORKSPACE'] + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath await tar.extractTar(archivePath, CompressionMethod.Zstd) expect(mkdirMock).toHaveBeenCalledWith(workspace) expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${defaultTarPath}"`, [ - '--use-compress-program', - IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30', + `"${tarPath}"`, '-xf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P', @@ -58,11 +68,61 @@ test('zstd extract tar', async () => { IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace ] .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []), + .concat(IS_MAC ? ['--delay-directory-restore'] : []) + .concat([ + '--use-compress-program', + IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30' + ]) + .join(' '), + undefined, {cwd: undefined} ) }) +test('zstd extract tar with windows BSDtar', async () => { + if (IS_WINDOWS) { + const mkdirMock = jest.spyOn(io, 'mkdirP') + const execMock = jest.spyOn(exec, 'exec') + jest + .spyOn(utils, 'getGnuTarPathOnWindows') + .mockReturnValue(Promise.resolve('')) + + const archivePath = `${process.env['windir']}\\fakepath\\cache.tar` + const workspace = process.env['GITHUB_WORKSPACE'] + const tarPath = SystemTarPathOnWindows + + await tar.extractTar(archivePath, CompressionMethod.Zstd) + + expect(mkdirMock).toHaveBeenCalledWith(workspace) + expect(execMock).toHaveBeenCalledTimes(2) + + expect(execMock).toHaveBeenNthCalledWith( + 1, + [ + 'zstd -d --long=30 --force -o', + TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') + ].join(' '), + undefined, + {cwd: undefined} + ) + + expect(execMock).toHaveBeenNthCalledWith( + 2, + [ + `"${tarPath}"`, + '-xf', + TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '-P', + '-C', + workspace?.replace(/\\/g, '/') + ].join(' '), + undefined, + {cwd: undefined} + ) + } +}) + test('gzip extract tar', async () => { const mkdirMock = jest.spyOn(io, 'mkdirP') const execMock = jest.spyOn(exec, 'exec') @@ -74,50 +134,51 @@ test('gzip extract tar', async () => { await tar.extractTar(archivePath, CompressionMethod.Gzip) expect(mkdirMock).toHaveBeenCalledWith(workspace) - const tarPath = IS_WINDOWS - ? `${process.env['windir']}\\System32\\tar.exe` - : defaultTarPath + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${tarPath}"`, [ - '-z', + `"${tarPath}"`, '-xf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P', '-C', IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace - ].concat(IS_MAC ? ['--delay-directory-restore'] : []), + ] + .concat(IS_WINDOWS ? ['--force-local'] : []) + .concat(IS_MAC ? ['--delay-directory-restore'] : []) + .concat(['-z']) + .join(' '), + undefined, {cwd: undefined} ) }) -test('gzip extract GNU tar on windows', async () => { +test('gzip extract GNU tar on windows with GNUtar in path', async () => { if (IS_WINDOWS) { - jest.spyOn(fs, 'existsSync').mockReturnValueOnce(false) - - const isGnuMock = jest - .spyOn(utils, 'isGnuTarInstalled') - .mockReturnValue(Promise.resolve(true)) + // GNU tar present in path but not at default location + jest + .spyOn(utils, 'getGnuTarPathOnWindows') + .mockReturnValue(Promise.resolve('tar')) const execMock = jest.spyOn(exec, 'exec') const archivePath = `${process.env['windir']}\\fakepath\\cache.tar` const workspace = process.env['GITHUB_WORKSPACE'] await tar.extractTar(archivePath, CompressionMethod.Gzip) - expect(isGnuMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"tar"`, [ - '-z', + `"tar"`, '-xf', archivePath.replace(/\\/g, '/'), '-P', '-C', workspace?.replace(/\\/g, '/'), - '--force-local' - ], + '--force-local', + '-z' + ].join(' '), + undefined, {cwd: undefined} ) } @@ -134,13 +195,13 @@ test('zstd create tar', async () => { await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Zstd) + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath + expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${defaultTarPath}"`, [ + `"${tarPath}"`, '--posix', - '--use-compress-program', - IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30', '-cf', IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd, '--exclude', @@ -149,16 +210,81 @@ test('zstd create tar', async () => { '-C', IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, '--files-from', - 'manifest.txt' + ManifestFilename ] .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []), + .concat(IS_MAC ? ['--delay-directory-restore'] : []) + .concat([ + '--use-compress-program', + IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30' + ]) + .join(' '), + undefined, // args { cwd: archiveFolder } ) }) +test('zstd create tar with windows BSDtar', async () => { + if (IS_WINDOWS) { + const execMock = jest.spyOn(exec, 'exec') + jest + .spyOn(utils, 'getGnuTarPathOnWindows') + .mockReturnValue(Promise.resolve('')) + + const archiveFolder = getTempDir() + const workspace = process.env['GITHUB_WORKSPACE'] + const sourceDirectories = ['~/.npm/cache', `${workspace}/dist`] + + await fs.promises.mkdir(archiveFolder, {recursive: true}) + + await tar.createTar( + archiveFolder, + sourceDirectories, + CompressionMethod.Zstd + ) + + const tarPath = SystemTarPathOnWindows + + expect(execMock).toHaveBeenCalledTimes(2) + + expect(execMock).toHaveBeenNthCalledWith( + 1, + [ + `"${tarPath}"`, + '--posix', + '-cf', + TarFilename.replace(/\\/g, '/'), + '--exclude', + TarFilename.replace(/\\/g, '/'), + '-P', + '-C', + workspace?.replace(/\\/g, '/'), + '--files-from', + ManifestFilename + ].join(' '), + undefined, // args + { + cwd: archiveFolder + } + ) + + expect(execMock).toHaveBeenNthCalledWith( + 2, + [ + 'zstd -T0 --long=30 --force -o', + CacheFilename.Zstd.replace(/\\/g, '/'), + TarFilename.replace(/\\/g, '/') + ].join(' '), + undefined, // args + { + cwd: archiveFolder + } + ) + } +}) + test('gzip create tar', async () => { const execMock = jest.spyOn(exec, 'exec') @@ -170,16 +296,13 @@ test('gzip create tar', async () => { await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Gzip) - const tarPath = IS_WINDOWS - ? `${process.env['windir']}\\System32\\tar.exe` - : defaultTarPath + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${tarPath}"`, [ + `"${tarPath}"`, '--posix', - '-z', '-cf', IS_WINDOWS ? CacheFilename.Gzip.replace(/\\/g, '/') : CacheFilename.Gzip, '--exclude', @@ -188,8 +311,13 @@ test('gzip create tar', async () => { '-C', IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, '--files-from', - 'manifest.txt' - ].concat(IS_MAC ? ['--delay-directory-restore'] : []), + ManifestFilename + ] + .concat(IS_WINDOWS ? ['--force-local'] : []) + .concat(IS_MAC ? ['--delay-directory-restore'] : []) + .concat(['-z']) + .join(' '), + undefined, // args { cwd: archiveFolder } @@ -205,22 +333,65 @@ test('zstd list tar', async () => { await tar.listTar(archivePath, CompressionMethod.Zstd) + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${defaultTarPath}"`, [ - '--use-compress-program', - IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30', + `"${tarPath}"`, '-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P' ] .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []), + .concat(IS_MAC ? ['--delay-directory-restore'] : []) + .concat([ + '--use-compress-program', + IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30' + ]) + .join(' '), + undefined, {cwd: undefined} ) }) +test('zstd list tar with windows BSDtar', async () => { + if (IS_WINDOWS) { + const execMock = jest.spyOn(exec, 'exec') + jest + .spyOn(utils, 'getGnuTarPathOnWindows') + .mockReturnValue(Promise.resolve('')) + const archivePath = `${process.env['windir']}\\fakepath\\cache.tar` + + await tar.listTar(archivePath, CompressionMethod.Zstd) + + const tarPath = SystemTarPathOnWindows + expect(execMock).toHaveBeenCalledTimes(2) + + expect(execMock).toHaveBeenNthCalledWith( + 1, + [ + 'zstd -d --long=30 --force -o', + TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') + ].join(' '), + undefined, + {cwd: undefined} + ) + + expect(execMock).toHaveBeenNthCalledWith( + 2, + [ + `"${tarPath}"`, + '-tf', + TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '-P' + ].join(' '), + undefined, + {cwd: undefined} + ) + } +}) + test('zstdWithoutLong list tar', async () => { const execMock = jest.spyOn(exec, 'exec') @@ -230,18 +401,20 @@ test('zstdWithoutLong list tar', async () => { await tar.listTar(archivePath, CompressionMethod.ZstdWithoutLong) + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${defaultTarPath}"`, [ - '--use-compress-program', - IS_WINDOWS ? 'zstd -d' : 'unzstd', + `"${tarPath}"`, '-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P' ] .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []), + .concat(IS_MAC ? ['--delay-directory-restore'] : []) + .concat(['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd']) + .join(' '), + undefined, {cwd: undefined} ) }) @@ -254,18 +427,20 @@ test('gzip list tar', async () => { await tar.listTar(archivePath, CompressionMethod.Gzip) - const tarPath = IS_WINDOWS - ? `${process.env['windir']}\\System32\\tar.exe` - : defaultTarPath + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${tarPath}"`, [ - '-z', + `"${tarPath}"`, '-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P' - ].concat(IS_MAC ? ['--delay-directory-restore'] : []), + ] + .concat(IS_WINDOWS ? ['--force-local'] : []) + .concat(IS_MAC ? ['--delay-directory-restore'] : []) + .concat(['-z']) + .join(' '), + undefined, {cwd: undefined} ) }) diff --git a/packages/cache/package-lock.json b/packages/cache/package-lock.json index 48f7f7e0..a34c4e3a 100644 --- a/packages/cache/package-lock.json +++ b/packages/cache/package-lock.json @@ -1,12 +1,12 @@ { "name": "@actions/cache", - "version": "3.0.6", + "version": "3.1.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@actions/cache", - "version": "3.0.6", + "version": "3.1.0", "license": "MIT", "dependencies": { "@actions/core": "^1.10.0", @@ -457,9 +457,9 @@ } }, "node_modules/minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "dependencies": { "brace-expansion": "^1.1.7" }, @@ -998,9 +998,9 @@ } }, "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "requires": { "brace-expansion": "^1.1.7" } diff --git a/packages/cache/package.json b/packages/cache/package.json index e2d6082c..44c677f8 100644 --- a/packages/cache/package.json +++ b/packages/cache/package.json @@ -1,6 +1,6 @@ { "name": "@actions/cache", - "version": "3.0.6", + "version": "3.1.0", "preview": true, "description": "Actions cache lib", "keywords": [ diff --git a/packages/cache/src/cache.ts b/packages/cache/src/cache.ts index 609c7f94..f928a2b9 100644 --- a/packages/cache/src/cache.ts +++ b/packages/cache/src/cache.ts @@ -4,6 +4,8 @@ import * as utils from './internal/cacheUtils' import * as cacheHttpClient from './internal/cacheHttpClient' import {createTar, extractTar, listTar} from './internal/tar' import {DownloadOptions, UploadOptions} from './options' +import {CompressionMethod} from './internal/constants' +import {ArtifactCacheEntry} from './internal/contracts' export class ValidationError extends Error { constructor(message: string) { @@ -85,17 +87,35 @@ export async function restoreCache( checkKey(key) } - const compressionMethod = await utils.getCompressionMethod() + let cacheEntry: ArtifactCacheEntry | null + let compressionMethod = await utils.getCompressionMethod() let archivePath = '' try { // path are needed to compute version - const cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, { + cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, { compressionMethod }) - if (!cacheEntry?.archiveLocation) { - // Cache not found - return undefined + // This is to support the old cache entry created by gzip on windows. + if ( + process.platform === 'win32' && + compressionMethod !== CompressionMethod.Gzip + ) { + compressionMethod = CompressionMethod.Gzip + cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, { + compressionMethod + }) + if (!cacheEntry?.archiveLocation) { + return undefined + } + + core.info( + "Couldn't find cache entry with zstd compression, falling back to gzip compression." + ) + } else { + // Cache not found + return undefined + } } archivePath = path.join( diff --git a/packages/cache/src/internal/cacheHttpClient.ts b/packages/cache/src/internal/cacheHttpClient.ts index d5ecd9a8..8982b8de 100644 --- a/packages/cache/src/internal/cacheHttpClient.ts +++ b/packages/cache/src/internal/cacheHttpClient.ts @@ -104,6 +104,7 @@ export async function getCacheEntry( const response = await retryTypedResponse('getCacheEntry', async () => httpClient.getJson(getCacheApiUrl(resource)) ) + // Cache not found if (response.statusCode === 204) { // List cache for primary key only if cache miss occurs if (core.isDebug()) { @@ -118,6 +119,7 @@ export async function getCacheEntry( const cacheResult = response.result const cacheDownloadUrl = cacheResult?.archiveLocation if (!cacheDownloadUrl) { + // Cache achiveLocation not found. This should never happen, and hence bail out. throw new Error('Cache not found.') } core.setSecret(cacheDownloadUrl) diff --git a/packages/cache/src/internal/cacheUtils.ts b/packages/cache/src/internal/cacheUtils.ts index c2ace526..ea1e7de6 100644 --- a/packages/cache/src/internal/cacheUtils.ts +++ b/packages/cache/src/internal/cacheUtils.ts @@ -7,7 +7,11 @@ import * as path from 'path' import * as semver from 'semver' import * as util from 'util' import {v4 as uuidV4} from 'uuid' -import {CacheFilename, CompressionMethod} from './constants' +import { + CacheFilename, + CompressionMethod, + GnuTarPathOnWindows +} from './constants' // From https://github.com/actions/toolkit/blob/main/packages/tool-cache/src/tool-cache.ts#L23 export async function createTempDirectory(): Promise { @@ -90,11 +94,6 @@ async function getVersion(app: string): Promise { // Use zstandard if possible to maximize cache performance export async function getCompressionMethod(): Promise { - if (process.platform === 'win32' && !(await isGnuTarInstalled())) { - // Disable zstd due to bug https://github.com/actions/cache/issues/301 - return CompressionMethod.Gzip - } - const versionOutput = await getVersion('zstd') const version = semver.clean(versionOutput) @@ -116,9 +115,12 @@ export function getCacheFileName(compressionMethod: CompressionMethod): string { : CacheFilename.Zstd } -export async function isGnuTarInstalled(): Promise { +export async function getGnuTarPathOnWindows(): Promise { + if (fs.existsSync(GnuTarPathOnWindows)) { + return GnuTarPathOnWindows + } const versionOutput = await getVersion('tar') - return versionOutput.toLowerCase().includes('gnu tar') + return versionOutput.toLowerCase().includes('gnu tar') ? io.which('tar') : '' } export function assertDefined(name: string, value?: T): T { diff --git a/packages/cache/src/internal/constants.ts b/packages/cache/src/internal/constants.ts index 2f78d326..4dbff574 100644 --- a/packages/cache/src/internal/constants.ts +++ b/packages/cache/src/internal/constants.ts @@ -11,6 +11,11 @@ export enum CompressionMethod { Zstd = 'zstd' } +export enum ArchiveToolType { + GNU = 'gnu', + BSD = 'bsd' +} + // The default number of retry attempts. export const DefaultRetryAttempts = 2 @@ -21,3 +26,13 @@ export const DefaultRetryDelay = 5000 // over the socket during this period, the socket is destroyed and the download // is aborted. export const SocketTimeout = 5000 + +// The default path of GNUtar on hosted Windows runners +export const GnuTarPathOnWindows = `${process.env['PROGRAMFILES']}\\Git\\usr\\bin\\tar.exe` + +// The default path of BSDtar on hosted Windows runners +export const SystemTarPathOnWindows = `${process.env['SYSTEMDRIVE']}\\Windows\\System32\\tar.exe` + +export const TarFilename = 'cache.tar' + +export const ManifestFilename = 'manifest.txt' diff --git a/packages/cache/src/internal/contracts.d.ts b/packages/cache/src/internal/contracts.d.ts index b5f53bdc..0519ff0a 100644 --- a/packages/cache/src/internal/contracts.d.ts +++ b/packages/cache/src/internal/contracts.d.ts @@ -37,3 +37,8 @@ export interface InternalCacheOptions { compressionMethod?: CompressionMethod cacheSize?: number } + +export interface ArchiveTool { + path: string + type: string +} diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 2e28ca1a..0af6a87a 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -3,25 +3,28 @@ import * as io from '@actions/io' import {existsSync, writeFileSync} from 'fs' import * as path from 'path' import * as utils from './cacheUtils' -import {CompressionMethod} from './constants' +import {ArchiveTool} from './contracts' +import { + CompressionMethod, + SystemTarPathOnWindows, + ArchiveToolType, + TarFilename, + ManifestFilename +} from './constants' const IS_WINDOWS = process.platform === 'win32' -async function getTarPath( - args: string[], - compressionMethod: CompressionMethod -): Promise { +// Returns tar path and type: BSD or GNU +async function getTarPath(): Promise { switch (process.platform) { case 'win32': { - const systemTar = `${process.env['windir']}\\System32\\tar.exe` - if (compressionMethod !== CompressionMethod.Gzip) { - // We only use zstandard compression on windows when gnu tar is installed due to - // a bug with compressing large files with bsdtar + zstd - args.push('--force-local') + const gnuTar = await utils.getGnuTarPathOnWindows() + const systemTar = SystemTarPathOnWindows + if (gnuTar) { + // Use GNUtar as default on windows + return {path: gnuTar, type: ArchiveToolType.GNU} } else if (existsSync(systemTar)) { - return systemTar - } else if (await utils.isGnuTarInstalled()) { - args.push('--force-local') + return {path: systemTar, type: ArchiveToolType.BSD} } break } @@ -29,27 +32,133 @@ async function getTarPath( const gnuTar = await io.which('gtar', false) if (gnuTar) { // fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527 - args.push('--delay-directory-restore') - return gnuTar + return {path: gnuTar, type: ArchiveToolType.GNU} + } else { + return { + path: await io.which('tar', true), + type: ArchiveToolType.BSD + } } - break } default: break } - return await io.which('tar', true) + // Default assumption is GNU tar is present in path + return { + path: await io.which('tar', true), + type: ArchiveToolType.GNU + } } -async function execTar( - args: string[], +// Return arguments for tar as per tarPath, compressionMethod, method type and os +async function getTarArgs( + tarPath: ArchiveTool, compressionMethod: CompressionMethod, - cwd?: string -): Promise { - try { - await exec(`"${await getTarPath(args, compressionMethod)}"`, args, {cwd}) - } catch (error) { - throw new Error(`Tar failed with error: ${error?.message}`) + type: string, + archivePath = '' +): Promise { + const args = [`"${tarPath.path}"`] + const cacheFileName = utils.getCacheFileName(compressionMethod) + const tarFile = 'cache.tar' + const workingDirectory = getWorkingDirectory() + // Speficic args for BSD tar on windows for workaround + const BSD_TAR_ZSTD = + tarPath.type === ArchiveToolType.BSD && + compressionMethod !== CompressionMethod.Gzip && + IS_WINDOWS + + // Method specific args + switch (type) { + case 'create': + args.push( + '--posix', + '-cf', + BSD_TAR_ZSTD + ? tarFile + : cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '--exclude', + BSD_TAR_ZSTD + ? tarFile + : cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '-P', + '-C', + workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '--files-from', + ManifestFilename + ) + break + case 'extract': + args.push( + '-xf', + BSD_TAR_ZSTD + ? tarFile + : archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '-P', + '-C', + workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/') + ) + break + case 'list': + args.push( + '-tf', + BSD_TAR_ZSTD + ? tarFile + : archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '-P' + ) + break } + + // Platform specific args + if (tarPath.type === ArchiveToolType.GNU) { + switch (process.platform) { + case 'win32': + args.push('--force-local') + break + case 'darwin': + args.push('--delay-directory-restore') + break + } + } + + return args +} + +// Returns commands to run tar and compression program +async function getCommands( + compressionMethod: CompressionMethod, + type: string, + archivePath = '' +): Promise { + let args + + const tarPath = await getTarPath() + const tarArgs = await getTarArgs( + tarPath, + compressionMethod, + type, + archivePath + ) + const compressionArgs = + type !== 'create' + ? await getDecompressionProgram(tarPath, compressionMethod, archivePath) + : await getCompressionProgram(tarPath, compressionMethod) + const BSD_TAR_ZSTD = + tarPath.type === ArchiveToolType.BSD && + compressionMethod !== CompressionMethod.Gzip && + IS_WINDOWS + + if (BSD_TAR_ZSTD && type !== 'create') { + args = [[...compressionArgs].join(' '), [...tarArgs].join(' ')] + } else { + args = [[...tarArgs].join(' '), [...compressionArgs].join(' ')] + } + + if (BSD_TAR_ZSTD) { + return args + } + + return [args.join(' ')] } function getWorkingDirectory(): string { @@ -57,37 +166,107 @@ function getWorkingDirectory(): string { } // Common function for extractTar and listTar to get the compression method -function getCompressionProgram(compressionMethod: CompressionMethod): string[] { +async function getDecompressionProgram( + tarPath: ArchiveTool, + compressionMethod: CompressionMethod, + archivePath: string +): Promise { // -d: Decompress. // unzstd is equivalent to 'zstd -d' // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // Using 30 here because we also support 32-bit self-hosted runners. + const BSD_TAR_ZSTD = + tarPath.type === ArchiveToolType.BSD && + compressionMethod !== CompressionMethod.Gzip && + IS_WINDOWS switch (compressionMethod) { case CompressionMethod.Zstd: - return [ - '--use-compress-program', - IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30' - ] + return BSD_TAR_ZSTD + ? [ + 'zstd -d --long=30 --force -o', + TarFilename, + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') + ] + : [ + '--use-compress-program', + IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30' + ] case CompressionMethod.ZstdWithoutLong: - return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd'] + return BSD_TAR_ZSTD + ? [ + 'zstd -d --force -o', + TarFilename, + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') + ] + : ['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd'] default: return ['-z'] } } +// Used for creating the archive +// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. +// zstdmt is equivalent to 'zstd -T0' +// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. +// Using 30 here because we also support 32-bit self-hosted runners. +// Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd. +async function getCompressionProgram( + tarPath: ArchiveTool, + compressionMethod: CompressionMethod +): Promise { + const cacheFileName = utils.getCacheFileName(compressionMethod) + const BSD_TAR_ZSTD = + tarPath.type === ArchiveToolType.BSD && + compressionMethod !== CompressionMethod.Gzip && + IS_WINDOWS + switch (compressionMethod) { + case CompressionMethod.Zstd: + return BSD_TAR_ZSTD + ? [ + 'zstd -T0 --long=30 --force -o', + cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + TarFilename + ] + : [ + '--use-compress-program', + IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30' + ] + case CompressionMethod.ZstdWithoutLong: + return BSD_TAR_ZSTD + ? [ + 'zstd -T0 --force -o', + cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + TarFilename + ] + : ['--use-compress-program', IS_WINDOWS ? '"zstd -T0"' : 'zstdmt'] + default: + return ['-z'] + } +} + +// Executes all commands as separate processes +async function execCommands(commands: string[], cwd?: string): Promise { + for (const command of commands) { + try { + await exec(command, undefined, {cwd}) + } catch (error) { + throw new Error( + `${command.split(' ')[0]} failed with error: ${error?.message}` + ) + } + } +} + +// List the contents of a tar export async function listTar( archivePath: string, compressionMethod: CompressionMethod ): Promise { - const args = [ - ...getCompressionProgram(compressionMethod), - '-tf', - archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P' - ] - await execTar(args, compressionMethod) + const commands = await getCommands(compressionMethod, 'list', archivePath) + await execCommands(commands) } +// Extract a tar export async function extractTar( archivePath: string, compressionMethod: CompressionMethod @@ -95,61 +274,21 @@ export async function extractTar( // Create directory to extract tar into const workingDirectory = getWorkingDirectory() await io.mkdirP(workingDirectory) - const args = [ - ...getCompressionProgram(compressionMethod), - '-xf', - archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P', - '-C', - workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/') - ] - await execTar(args, compressionMethod) + const commands = await getCommands(compressionMethod, 'extract', archivePath) + await execCommands(commands) } +// Create a tar export async function createTar( archiveFolder: string, sourceDirectories: string[], compressionMethod: CompressionMethod ): Promise { // Write source directories to manifest.txt to avoid command length limits - const manifestFilename = 'manifest.txt' - const cacheFileName = utils.getCacheFileName(compressionMethod) writeFileSync( - path.join(archiveFolder, manifestFilename), + path.join(archiveFolder, ManifestFilename), sourceDirectories.join('\n') ) - const workingDirectory = getWorkingDirectory() - - // -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. - // zstdmt is equivalent to 'zstd -T0' - // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. - // Using 30 here because we also support 32-bit self-hosted runners. - // Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd. - function getCompressionProgram(): string[] { - switch (compressionMethod) { - case CompressionMethod.Zstd: - return [ - '--use-compress-program', - IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30' - ] - case CompressionMethod.ZstdWithoutLong: - return ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt'] - default: - return ['-z'] - } - } - const args = [ - '--posix', - ...getCompressionProgram(), - '-cf', - cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '--exclude', - cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P', - '-C', - workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '--files-from', - manifestFilename - ] - await execTar(args, compressionMethod, archiveFolder) + const commands = await getCommands(compressionMethod, 'create') + await execCommands(commands, archiveFolder) }