diff --git a/.github/workflows/cache-windows-test.yml b/.github/workflows/cache-windows-test.yml deleted file mode 100644 index 3868f296..00000000 --- a/.github/workflows/cache-windows-test.yml +++ /dev/null @@ -1,90 +0,0 @@ -name: cache-windows-bsd-unit-tests -on: - push: - branches: - - main - paths-ignore: - - '**.md' - pull_request: - paths-ignore: - - '**.md' - -jobs: - build: - name: Build - - runs-on: windows-latest - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - shell: bash - run: | - rm "C:\Program Files\Git\usr\bin\tar.exe" - - - name: Set Node.js 12.x - uses: actions/setup-node@v1 - with: - node-version: 12.x - - # In order to save & restore cache from a shell script, certain env variables need to be set that are only available in the - # node context. This runs a local action that gets and sets the necessary env variables that are needed - - name: Set env variables - uses: ./packages/cache/__tests__/__fixtures__/ - - # Need root node_modules because certain npm packages like jest are configured for the entire repository and it won't be possible - # without these to just compile the cache package - - name: Install root npm packages - run: npm ci - - - name: Compile cache package - run: | - npm ci - npm run tsc - working-directory: packages/cache - - - name: Generate files in working directory - shell: bash - run: packages/cache/__tests__/create-cache-files.sh ${{ runner.os }} test-cache - - - name: Generate files outside working directory - shell: bash - run: packages/cache/__tests__/create-cache-files.sh ${{ runner.os }} ~/test-cache - - # We're using node -e to call the functions directly available in the @actions/cache package - - name: Save cache using saveCache() - run: | - node -e "Promise.resolve(require('./packages/cache/lib/cache').saveCache(['test-cache','~/test-cache'],'test-${{ runner.os }}-${{ github.run_id }}'))" - - - name: Delete cache folders before restoring - shell: bash - run: | - rm -rf test-cache - rm -rf ~/test-cache - - - name: Restore cache using restoreCache() with http-client - run: | - node -e "Promise.resolve(require('./packages/cache/lib/cache').restoreCache(['test-cache','~/test-cache'],'test-${{ runner.os }}-${{ github.run_id }}',[],{useAzureSdk: false}))" - - - name: Verify cache restored with http-client - shell: bash - run: | - packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} test-cache - packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} ~/test-cache - - - name: Delete cache folders before restoring - shell: bash - run: | - rm -rf test-cache - rm -rf ~/test-cache - - - name: Restore cache using restoreCache() with Azure SDK - run: | - node -e "Promise.resolve(require('./packages/cache/lib/cache').restoreCache(['test-cache','~/test-cache'],'test-${{ runner.os }}-${{ github.run_id }}'))" - - - name: Verify cache restored with Azure SDK - shell: bash - run: | - packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} test-cache - packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} ~/test-cache diff --git a/packages/cache/RELEASES.md b/packages/cache/RELEASES.md index 4a415486..73518e1c 100644 --- a/packages/cache/RELEASES.md +++ b/packages/cache/RELEASES.md @@ -91,17 +91,3 @@ ### 3.0.6 - Added `@azure/abort-controller` to dependencies to fix compatibility issue with ESM [#1208](https://github.com/actions/toolkit/issues/1208) - -### 3.1.0-beta.1 -- Update actions/cache on windows to use gnu tar and zstd by default and fallback to bsdtar and zstd if gnu tar is not available. ([issue](https://github.com/actions/cache/issues/984)) - -### 3.1.0-beta.2 -- Added support for fallback to gzip to restore old caches on windows. - -### 3.1.0-beta.3 -- Bug Fixes for fallback to gzip to restore old caches on windows and bsdtar if gnutar is not available. - -### 3.1.0 -- Update actions/cache on windows to use gnu tar and zstd by default -- Update actions/cache on windows to fallback to bsdtar and zstd if gnu tar is not available. -- Added support for fallback to gzip to restore old caches on windows. diff --git a/packages/cache/__tests__/restoreCache.test.ts b/packages/cache/__tests__/restoreCache.test.ts index 9cf45799..36ec8801 100644 --- a/packages/cache/__tests__/restoreCache.test.ts +++ b/packages/cache/__tests__/restoreCache.test.ts @@ -161,81 +161,6 @@ test('restore with gzip compressed cache found', async () => { expect(getCompressionMock).toHaveBeenCalledTimes(1) }) -test('restore with zstd as default but gzip compressed cache found on windows', async () => { - if (process.platform === 'win32') { - const paths = ['node_modules'] - const key = 'node-test' - - const cacheEntry: ArtifactCacheEntry = { - cacheKey: key, - scope: 'refs/heads/main', - archiveLocation: 'www.actionscache.test/download' - } - const getCacheMock = jest.spyOn(cacheHttpClient, 'getCacheEntry') - getCacheMock - .mockImplementationOnce(async () => { - return Promise.resolve(null) - }) - .mockImplementationOnce(async () => { - return Promise.resolve(cacheEntry) - }) - - const tempPath = '/foo/bar' - - const createTempDirectoryMock = jest.spyOn( - cacheUtils, - 'createTempDirectory' - ) - createTempDirectoryMock.mockImplementation(async () => { - return Promise.resolve(tempPath) - }) - - const archivePath = path.join(tempPath, CacheFilename.Gzip) - const downloadCacheMock = jest.spyOn(cacheHttpClient, 'downloadCache') - - const fileSize = 142 - const getArchiveFileSizeInBytesMock = jest - .spyOn(cacheUtils, 'getArchiveFileSizeInBytes') - .mockReturnValue(fileSize) - - const extractTarMock = jest.spyOn(tar, 'extractTar') - const unlinkFileMock = jest.spyOn(cacheUtils, 'unlinkFile') - - const compression = CompressionMethod.Zstd - const getCompressionMock = jest - .spyOn(cacheUtils, 'getCompressionMethod') - .mockReturnValue(Promise.resolve(compression)) - - const cacheKey = await restoreCache(paths, key) - - expect(cacheKey).toBe(key) - expect(getCacheMock).toHaveBeenNthCalledWith(1, [key], paths, { - compressionMethod: compression - }) - expect(getCacheMock).toHaveBeenNthCalledWith(2, [key], paths, { - compressionMethod: CompressionMethod.Gzip - }) - expect(createTempDirectoryMock).toHaveBeenCalledTimes(1) - expect(downloadCacheMock).toHaveBeenCalledWith( - cacheEntry.archiveLocation, - archivePath, - undefined - ) - expect(getArchiveFileSizeInBytesMock).toHaveBeenCalledWith(archivePath) - - expect(extractTarMock).toHaveBeenCalledTimes(1) - expect(extractTarMock).toHaveBeenCalledWith( - archivePath, - CompressionMethod.Gzip - ) - - expect(unlinkFileMock).toHaveBeenCalledTimes(1) - expect(unlinkFileMock).toHaveBeenCalledWith(archivePath) - - expect(getCompressionMock).toHaveBeenCalledTimes(1) - } -}) - test('restore with zstd compressed cache found', async () => { const paths = ['node_modules'] const key = 'node-test' diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index a33f4fab..e4233bc9 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -1,14 +1,7 @@ import * as exec from '@actions/exec' import * as io from '@actions/io' import * as path from 'path' -import { - CacheFilename, - CompressionMethod, - GnuTarPathOnWindows, - ManifestFilename, - SystemTarPathOnWindows, - TarFilename -} from '../src/internal/constants' +import {CacheFilename, CompressionMethod} from '../src/internal/constants' import * as tar from '../src/internal/tar' import * as utils from '../src/internal/cacheUtils' // eslint-disable-next-line @typescript-eslint/no-require-imports @@ -20,7 +13,7 @@ jest.mock('@actions/io') const IS_WINDOWS = process.platform === 'win32' const IS_MAC = process.platform === 'darwin' -const defaultTarPath = IS_MAC ? 'gtar' : 'tar' +const defaultTarPath = process.platform === 'darwin' ? 'gtar' : 'tar' function getTempDir(): string { return path.join(__dirname, '_temp', 'tar') @@ -35,10 +28,6 @@ beforeAll(async () => { await jest.requireActual('@actions/io').rmRF(getTempDir()) }) -beforeEach(async () => { - jest.restoreAllMocks() -}) - afterAll(async () => { delete process.env['GITHUB_WORKSPACE'] await jest.requireActual('@actions/io').rmRF(getTempDir()) @@ -52,15 +41,16 @@ test('zstd extract tar', async () => { ? `${process.env['windir']}\\fakepath\\cache.tar` : 'cache.tar' const workspace = process.env['GITHUB_WORKSPACE'] - const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath await tar.extractTar(archivePath, CompressionMethod.Zstd) expect(mkdirMock).toHaveBeenCalledWith(workspace) expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( + `"${defaultTarPath}"`, [ - `"${tarPath}"`, + '--use-compress-program', + IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30', '-xf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P', @@ -68,61 +58,11 @@ test('zstd extract tar', async () => { IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace ] .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []) - .concat([ - '--use-compress-program', - IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30' - ]) - .join(' '), - undefined, + .concat(IS_MAC ? ['--delay-directory-restore'] : []), {cwd: undefined} ) }) -test('zstd extract tar with windows BSDtar', async () => { - if (IS_WINDOWS) { - const mkdirMock = jest.spyOn(io, 'mkdirP') - const execMock = jest.spyOn(exec, 'exec') - jest - .spyOn(utils, 'getGnuTarPathOnWindows') - .mockReturnValue(Promise.resolve('')) - - const archivePath = `${process.env['windir']}\\fakepath\\cache.tar` - const workspace = process.env['GITHUB_WORKSPACE'] - const tarPath = SystemTarPathOnWindows - - await tar.extractTar(archivePath, CompressionMethod.Zstd) - - expect(mkdirMock).toHaveBeenCalledWith(workspace) - expect(execMock).toHaveBeenCalledTimes(2) - - expect(execMock).toHaveBeenNthCalledWith( - 1, - [ - 'zstd -d --long=30 --force -o', - TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') - ].join(' '), - undefined, - {cwd: undefined} - ) - - expect(execMock).toHaveBeenNthCalledWith( - 2, - [ - `"${tarPath}"`, - '-xf', - TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P', - '-C', - workspace?.replace(/\\/g, '/') - ].join(' '), - undefined, - {cwd: undefined} - ) - } -}) - test('gzip extract tar', async () => { const mkdirMock = jest.spyOn(io, 'mkdirP') const execMock = jest.spyOn(exec, 'exec') @@ -134,51 +74,50 @@ test('gzip extract tar', async () => { await tar.extractTar(archivePath, CompressionMethod.Gzip) expect(mkdirMock).toHaveBeenCalledWith(workspace) - const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath + const tarPath = IS_WINDOWS + ? `${process.env['windir']}\\System32\\tar.exe` + : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( + `"${tarPath}"`, [ - `"${tarPath}"`, + '-z', '-xf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P', '-C', IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace - ] - .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []) - .concat(['-z']) - .join(' '), - undefined, + ].concat(IS_MAC ? ['--delay-directory-restore'] : []), {cwd: undefined} ) }) -test('gzip extract GNU tar on windows with GNUtar in path', async () => { +test('gzip extract GNU tar on windows', async () => { if (IS_WINDOWS) { - // GNU tar present in path but not at default location - jest - .spyOn(utils, 'getGnuTarPathOnWindows') - .mockReturnValue(Promise.resolve('tar')) + jest.spyOn(fs, 'existsSync').mockReturnValueOnce(false) + + const isGnuMock = jest + .spyOn(utils, 'isGnuTarInstalled') + .mockReturnValue(Promise.resolve(true)) const execMock = jest.spyOn(exec, 'exec') const archivePath = `${process.env['windir']}\\fakepath\\cache.tar` const workspace = process.env['GITHUB_WORKSPACE'] await tar.extractTar(archivePath, CompressionMethod.Gzip) + expect(isGnuMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( + `"tar"`, [ - `"tar"`, + '-z', '-xf', archivePath.replace(/\\/g, '/'), '-P', '-C', workspace?.replace(/\\/g, '/'), - '--force-local', - '-z' - ].join(' '), - undefined, + '--force-local' + ], {cwd: undefined} ) } @@ -195,13 +134,13 @@ test('zstd create tar', async () => { await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Zstd) - const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath - expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( + `"${defaultTarPath}"`, [ - `"${tarPath}"`, '--posix', + '--use-compress-program', + IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30', '-cf', IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd, '--exclude', @@ -210,81 +149,16 @@ test('zstd create tar', async () => { '-C', IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, '--files-from', - ManifestFilename + 'manifest.txt' ] .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []) - .concat([ - '--use-compress-program', - IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30' - ]) - .join(' '), - undefined, // args + .concat(IS_MAC ? ['--delay-directory-restore'] : []), { cwd: archiveFolder } ) }) -test('zstd create tar with windows BSDtar', async () => { - if (IS_WINDOWS) { - const execMock = jest.spyOn(exec, 'exec') - jest - .spyOn(utils, 'getGnuTarPathOnWindows') - .mockReturnValue(Promise.resolve('')) - - const archiveFolder = getTempDir() - const workspace = process.env['GITHUB_WORKSPACE'] - const sourceDirectories = ['~/.npm/cache', `${workspace}/dist`] - - await fs.promises.mkdir(archiveFolder, {recursive: true}) - - await tar.createTar( - archiveFolder, - sourceDirectories, - CompressionMethod.Zstd - ) - - const tarPath = SystemTarPathOnWindows - - expect(execMock).toHaveBeenCalledTimes(2) - - expect(execMock).toHaveBeenNthCalledWith( - 1, - [ - `"${tarPath}"`, - '--posix', - '-cf', - TarFilename.replace(/\\/g, '/'), - '--exclude', - TarFilename.replace(/\\/g, '/'), - '-P', - '-C', - workspace?.replace(/\\/g, '/'), - '--files-from', - ManifestFilename - ].join(' '), - undefined, // args - { - cwd: archiveFolder - } - ) - - expect(execMock).toHaveBeenNthCalledWith( - 2, - [ - 'zstd -T0 --long=30 --force -o', - CacheFilename.Zstd.replace(/\\/g, '/'), - TarFilename.replace(/\\/g, '/') - ].join(' '), - undefined, // args - { - cwd: archiveFolder - } - ) - } -}) - test('gzip create tar', async () => { const execMock = jest.spyOn(exec, 'exec') @@ -296,13 +170,16 @@ test('gzip create tar', async () => { await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Gzip) - const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath + const tarPath = IS_WINDOWS + ? `${process.env['windir']}\\System32\\tar.exe` + : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( + `"${tarPath}"`, [ - `"${tarPath}"`, '--posix', + '-z', '-cf', IS_WINDOWS ? CacheFilename.Gzip.replace(/\\/g, '/') : CacheFilename.Gzip, '--exclude', @@ -311,13 +188,8 @@ test('gzip create tar', async () => { '-C', IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, '--files-from', - ManifestFilename - ] - .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []) - .concat(['-z']) - .join(' '), - undefined, // args + 'manifest.txt' + ].concat(IS_MAC ? ['--delay-directory-restore'] : []), { cwd: archiveFolder } @@ -333,65 +205,22 @@ test('zstd list tar', async () => { await tar.listTar(archivePath, CompressionMethod.Zstd) - const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( + `"${defaultTarPath}"`, [ - `"${tarPath}"`, + '--use-compress-program', + IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30', '-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P' ] .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []) - .concat([ - '--use-compress-program', - IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30' - ]) - .join(' '), - undefined, + .concat(IS_MAC ? ['--delay-directory-restore'] : []), {cwd: undefined} ) }) -test('zstd list tar with windows BSDtar', async () => { - if (IS_WINDOWS) { - const execMock = jest.spyOn(exec, 'exec') - jest - .spyOn(utils, 'getGnuTarPathOnWindows') - .mockReturnValue(Promise.resolve('')) - const archivePath = `${process.env['windir']}\\fakepath\\cache.tar` - - await tar.listTar(archivePath, CompressionMethod.Zstd) - - const tarPath = SystemTarPathOnWindows - expect(execMock).toHaveBeenCalledTimes(2) - - expect(execMock).toHaveBeenNthCalledWith( - 1, - [ - 'zstd -d --long=30 --force -o', - TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') - ].join(' '), - undefined, - {cwd: undefined} - ) - - expect(execMock).toHaveBeenNthCalledWith( - 2, - [ - `"${tarPath}"`, - '-tf', - TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P' - ].join(' '), - undefined, - {cwd: undefined} - ) - } -}) - test('zstdWithoutLong list tar', async () => { const execMock = jest.spyOn(exec, 'exec') @@ -401,20 +230,18 @@ test('zstdWithoutLong list tar', async () => { await tar.listTar(archivePath, CompressionMethod.ZstdWithoutLong) - const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( + `"${defaultTarPath}"`, [ - `"${tarPath}"`, + '--use-compress-program', + IS_WINDOWS ? 'zstd -d' : 'unzstd', '-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P' ] .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []) - .concat(['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd']) - .join(' '), - undefined, + .concat(IS_MAC ? ['--delay-directory-restore'] : []), {cwd: undefined} ) }) @@ -427,20 +254,18 @@ test('gzip list tar', async () => { await tar.listTar(archivePath, CompressionMethod.Gzip) - const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath + const tarPath = IS_WINDOWS + ? `${process.env['windir']}\\System32\\tar.exe` + : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( + `"${tarPath}"`, [ - `"${tarPath}"`, + '-z', '-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P' - ] - .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []) - .concat(['-z']) - .join(' '), - undefined, + ].concat(IS_MAC ? ['--delay-directory-restore'] : []), {cwd: undefined} ) }) diff --git a/packages/cache/package-lock.json b/packages/cache/package-lock.json index a34c4e3a..52afe053 100644 --- a/packages/cache/package-lock.json +++ b/packages/cache/package-lock.json @@ -1,12 +1,12 @@ { "name": "@actions/cache", - "version": "3.1.0", + "version": "3.1.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@actions/cache", - "version": "3.1.0", + "version": "3.1.1", "license": "MIT", "dependencies": { "@actions/core": "^1.10.0", diff --git a/packages/cache/package.json b/packages/cache/package.json index 44c677f8..ccd37fa1 100644 --- a/packages/cache/package.json +++ b/packages/cache/package.json @@ -1,6 +1,6 @@ { "name": "@actions/cache", - "version": "3.1.0", + "version": "3.1.1", "preview": true, "description": "Actions cache lib", "keywords": [ diff --git a/packages/cache/src/cache.ts b/packages/cache/src/cache.ts index f928a2b9..609c7f94 100644 --- a/packages/cache/src/cache.ts +++ b/packages/cache/src/cache.ts @@ -4,8 +4,6 @@ import * as utils from './internal/cacheUtils' import * as cacheHttpClient from './internal/cacheHttpClient' import {createTar, extractTar, listTar} from './internal/tar' import {DownloadOptions, UploadOptions} from './options' -import {CompressionMethod} from './internal/constants' -import {ArtifactCacheEntry} from './internal/contracts' export class ValidationError extends Error { constructor(message: string) { @@ -87,35 +85,17 @@ export async function restoreCache( checkKey(key) } - let cacheEntry: ArtifactCacheEntry | null - let compressionMethod = await utils.getCompressionMethod() + const compressionMethod = await utils.getCompressionMethod() let archivePath = '' try { // path are needed to compute version - cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, { + const cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, { compressionMethod }) - if (!cacheEntry?.archiveLocation) { - // This is to support the old cache entry created by gzip on windows. - if ( - process.platform === 'win32' && - compressionMethod !== CompressionMethod.Gzip - ) { - compressionMethod = CompressionMethod.Gzip - cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, { - compressionMethod - }) - if (!cacheEntry?.archiveLocation) { - return undefined - } - core.info( - "Couldn't find cache entry with zstd compression, falling back to gzip compression." - ) - } else { - // Cache not found - return undefined - } + if (!cacheEntry?.archiveLocation) { + // Cache not found + return undefined } archivePath = path.join( diff --git a/packages/cache/src/internal/cacheHttpClient.ts b/packages/cache/src/internal/cacheHttpClient.ts index 8982b8de..d5ecd9a8 100644 --- a/packages/cache/src/internal/cacheHttpClient.ts +++ b/packages/cache/src/internal/cacheHttpClient.ts @@ -104,7 +104,6 @@ export async function getCacheEntry( const response = await retryTypedResponse('getCacheEntry', async () => httpClient.getJson(getCacheApiUrl(resource)) ) - // Cache not found if (response.statusCode === 204) { // List cache for primary key only if cache miss occurs if (core.isDebug()) { @@ -119,7 +118,6 @@ export async function getCacheEntry( const cacheResult = response.result const cacheDownloadUrl = cacheResult?.archiveLocation if (!cacheDownloadUrl) { - // Cache achiveLocation not found. This should never happen, and hence bail out. throw new Error('Cache not found.') } core.setSecret(cacheDownloadUrl) diff --git a/packages/cache/src/internal/cacheUtils.ts b/packages/cache/src/internal/cacheUtils.ts index ea1e7de6..c2ace526 100644 --- a/packages/cache/src/internal/cacheUtils.ts +++ b/packages/cache/src/internal/cacheUtils.ts @@ -7,11 +7,7 @@ import * as path from 'path' import * as semver from 'semver' import * as util from 'util' import {v4 as uuidV4} from 'uuid' -import { - CacheFilename, - CompressionMethod, - GnuTarPathOnWindows -} from './constants' +import {CacheFilename, CompressionMethod} from './constants' // From https://github.com/actions/toolkit/blob/main/packages/tool-cache/src/tool-cache.ts#L23 export async function createTempDirectory(): Promise { @@ -94,6 +90,11 @@ async function getVersion(app: string): Promise { // Use zstandard if possible to maximize cache performance export async function getCompressionMethod(): Promise { + if (process.platform === 'win32' && !(await isGnuTarInstalled())) { + // Disable zstd due to bug https://github.com/actions/cache/issues/301 + return CompressionMethod.Gzip + } + const versionOutput = await getVersion('zstd') const version = semver.clean(versionOutput) @@ -115,12 +116,9 @@ export function getCacheFileName(compressionMethod: CompressionMethod): string { : CacheFilename.Zstd } -export async function getGnuTarPathOnWindows(): Promise { - if (fs.existsSync(GnuTarPathOnWindows)) { - return GnuTarPathOnWindows - } +export async function isGnuTarInstalled(): Promise { const versionOutput = await getVersion('tar') - return versionOutput.toLowerCase().includes('gnu tar') ? io.which('tar') : '' + return versionOutput.toLowerCase().includes('gnu tar') } export function assertDefined(name: string, value?: T): T { diff --git a/packages/cache/src/internal/constants.ts b/packages/cache/src/internal/constants.ts index 4dbff574..2f78d326 100644 --- a/packages/cache/src/internal/constants.ts +++ b/packages/cache/src/internal/constants.ts @@ -11,11 +11,6 @@ export enum CompressionMethod { Zstd = 'zstd' } -export enum ArchiveToolType { - GNU = 'gnu', - BSD = 'bsd' -} - // The default number of retry attempts. export const DefaultRetryAttempts = 2 @@ -26,13 +21,3 @@ export const DefaultRetryDelay = 5000 // over the socket during this period, the socket is destroyed and the download // is aborted. export const SocketTimeout = 5000 - -// The default path of GNUtar on hosted Windows runners -export const GnuTarPathOnWindows = `${process.env['PROGRAMFILES']}\\Git\\usr\\bin\\tar.exe` - -// The default path of BSDtar on hosted Windows runners -export const SystemTarPathOnWindows = `${process.env['SYSTEMDRIVE']}\\Windows\\System32\\tar.exe` - -export const TarFilename = 'cache.tar' - -export const ManifestFilename = 'manifest.txt' diff --git a/packages/cache/src/internal/contracts.d.ts b/packages/cache/src/internal/contracts.d.ts index 0519ff0a..b5f53bdc 100644 --- a/packages/cache/src/internal/contracts.d.ts +++ b/packages/cache/src/internal/contracts.d.ts @@ -37,8 +37,3 @@ export interface InternalCacheOptions { compressionMethod?: CompressionMethod cacheSize?: number } - -export interface ArchiveTool { - path: string - type: string -} diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 0af6a87a..2e28ca1a 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -3,28 +3,25 @@ import * as io from '@actions/io' import {existsSync, writeFileSync} from 'fs' import * as path from 'path' import * as utils from './cacheUtils' -import {ArchiveTool} from './contracts' -import { - CompressionMethod, - SystemTarPathOnWindows, - ArchiveToolType, - TarFilename, - ManifestFilename -} from './constants' +import {CompressionMethod} from './constants' const IS_WINDOWS = process.platform === 'win32' -// Returns tar path and type: BSD or GNU -async function getTarPath(): Promise { +async function getTarPath( + args: string[], + compressionMethod: CompressionMethod +): Promise { switch (process.platform) { case 'win32': { - const gnuTar = await utils.getGnuTarPathOnWindows() - const systemTar = SystemTarPathOnWindows - if (gnuTar) { - // Use GNUtar as default on windows - return {path: gnuTar, type: ArchiveToolType.GNU} + const systemTar = `${process.env['windir']}\\System32\\tar.exe` + if (compressionMethod !== CompressionMethod.Gzip) { + // We only use zstandard compression on windows when gnu tar is installed due to + // a bug with compressing large files with bsdtar + zstd + args.push('--force-local') } else if (existsSync(systemTar)) { - return {path: systemTar, type: ArchiveToolType.BSD} + return systemTar + } else if (await utils.isGnuTarInstalled()) { + args.push('--force-local') } break } @@ -32,133 +29,27 @@ async function getTarPath(): Promise { const gnuTar = await io.which('gtar', false) if (gnuTar) { // fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527 - return {path: gnuTar, type: ArchiveToolType.GNU} - } else { - return { - path: await io.which('tar', true), - type: ArchiveToolType.BSD - } + args.push('--delay-directory-restore') + return gnuTar } + break } default: break } - // Default assumption is GNU tar is present in path - return { - path: await io.which('tar', true), - type: ArchiveToolType.GNU - } + return await io.which('tar', true) } -// Return arguments for tar as per tarPath, compressionMethod, method type and os -async function getTarArgs( - tarPath: ArchiveTool, +async function execTar( + args: string[], compressionMethod: CompressionMethod, - type: string, - archivePath = '' -): Promise { - const args = [`"${tarPath.path}"`] - const cacheFileName = utils.getCacheFileName(compressionMethod) - const tarFile = 'cache.tar' - const workingDirectory = getWorkingDirectory() - // Speficic args for BSD tar on windows for workaround - const BSD_TAR_ZSTD = - tarPath.type === ArchiveToolType.BSD && - compressionMethod !== CompressionMethod.Gzip && - IS_WINDOWS - - // Method specific args - switch (type) { - case 'create': - args.push( - '--posix', - '-cf', - BSD_TAR_ZSTD - ? tarFile - : cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '--exclude', - BSD_TAR_ZSTD - ? tarFile - : cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P', - '-C', - workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '--files-from', - ManifestFilename - ) - break - case 'extract': - args.push( - '-xf', - BSD_TAR_ZSTD - ? tarFile - : archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P', - '-C', - workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/') - ) - break - case 'list': - args.push( - '-tf', - BSD_TAR_ZSTD - ? tarFile - : archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P' - ) - break + cwd?: string +): Promise { + try { + await exec(`"${await getTarPath(args, compressionMethod)}"`, args, {cwd}) + } catch (error) { + throw new Error(`Tar failed with error: ${error?.message}`) } - - // Platform specific args - if (tarPath.type === ArchiveToolType.GNU) { - switch (process.platform) { - case 'win32': - args.push('--force-local') - break - case 'darwin': - args.push('--delay-directory-restore') - break - } - } - - return args -} - -// Returns commands to run tar and compression program -async function getCommands( - compressionMethod: CompressionMethod, - type: string, - archivePath = '' -): Promise { - let args - - const tarPath = await getTarPath() - const tarArgs = await getTarArgs( - tarPath, - compressionMethod, - type, - archivePath - ) - const compressionArgs = - type !== 'create' - ? await getDecompressionProgram(tarPath, compressionMethod, archivePath) - : await getCompressionProgram(tarPath, compressionMethod) - const BSD_TAR_ZSTD = - tarPath.type === ArchiveToolType.BSD && - compressionMethod !== CompressionMethod.Gzip && - IS_WINDOWS - - if (BSD_TAR_ZSTD && type !== 'create') { - args = [[...compressionArgs].join(' '), [...tarArgs].join(' ')] - } else { - args = [[...tarArgs].join(' '), [...compressionArgs].join(' ')] - } - - if (BSD_TAR_ZSTD) { - return args - } - - return [args.join(' ')] } function getWorkingDirectory(): string { @@ -166,107 +57,37 @@ function getWorkingDirectory(): string { } // Common function for extractTar and listTar to get the compression method -async function getDecompressionProgram( - tarPath: ArchiveTool, - compressionMethod: CompressionMethod, - archivePath: string -): Promise { +function getCompressionProgram(compressionMethod: CompressionMethod): string[] { // -d: Decompress. // unzstd is equivalent to 'zstd -d' // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // Using 30 here because we also support 32-bit self-hosted runners. - const BSD_TAR_ZSTD = - tarPath.type === ArchiveToolType.BSD && - compressionMethod !== CompressionMethod.Gzip && - IS_WINDOWS switch (compressionMethod) { case CompressionMethod.Zstd: - return BSD_TAR_ZSTD - ? [ - 'zstd -d --long=30 --force -o', - TarFilename, - archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') - ] - : [ - '--use-compress-program', - IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30' - ] + return [ + '--use-compress-program', + IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30' + ] case CompressionMethod.ZstdWithoutLong: - return BSD_TAR_ZSTD - ? [ - 'zstd -d --force -o', - TarFilename, - archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') - ] - : ['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd'] + return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd'] default: return ['-z'] } } -// Used for creating the archive -// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. -// zstdmt is equivalent to 'zstd -T0' -// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. -// Using 30 here because we also support 32-bit self-hosted runners. -// Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd. -async function getCompressionProgram( - tarPath: ArchiveTool, - compressionMethod: CompressionMethod -): Promise { - const cacheFileName = utils.getCacheFileName(compressionMethod) - const BSD_TAR_ZSTD = - tarPath.type === ArchiveToolType.BSD && - compressionMethod !== CompressionMethod.Gzip && - IS_WINDOWS - switch (compressionMethod) { - case CompressionMethod.Zstd: - return BSD_TAR_ZSTD - ? [ - 'zstd -T0 --long=30 --force -o', - cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - TarFilename - ] - : [ - '--use-compress-program', - IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30' - ] - case CompressionMethod.ZstdWithoutLong: - return BSD_TAR_ZSTD - ? [ - 'zstd -T0 --force -o', - cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - TarFilename - ] - : ['--use-compress-program', IS_WINDOWS ? '"zstd -T0"' : 'zstdmt'] - default: - return ['-z'] - } -} - -// Executes all commands as separate processes -async function execCommands(commands: string[], cwd?: string): Promise { - for (const command of commands) { - try { - await exec(command, undefined, {cwd}) - } catch (error) { - throw new Error( - `${command.split(' ')[0]} failed with error: ${error?.message}` - ) - } - } -} - -// List the contents of a tar export async function listTar( archivePath: string, compressionMethod: CompressionMethod ): Promise { - const commands = await getCommands(compressionMethod, 'list', archivePath) - await execCommands(commands) + const args = [ + ...getCompressionProgram(compressionMethod), + '-tf', + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '-P' + ] + await execTar(args, compressionMethod) } -// Extract a tar export async function extractTar( archivePath: string, compressionMethod: CompressionMethod @@ -274,21 +95,61 @@ export async function extractTar( // Create directory to extract tar into const workingDirectory = getWorkingDirectory() await io.mkdirP(workingDirectory) - const commands = await getCommands(compressionMethod, 'extract', archivePath) - await execCommands(commands) + const args = [ + ...getCompressionProgram(compressionMethod), + '-xf', + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '-P', + '-C', + workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/') + ] + await execTar(args, compressionMethod) } -// Create a tar export async function createTar( archiveFolder: string, sourceDirectories: string[], compressionMethod: CompressionMethod ): Promise { // Write source directories to manifest.txt to avoid command length limits + const manifestFilename = 'manifest.txt' + const cacheFileName = utils.getCacheFileName(compressionMethod) writeFileSync( - path.join(archiveFolder, ManifestFilename), + path.join(archiveFolder, manifestFilename), sourceDirectories.join('\n') ) - const commands = await getCommands(compressionMethod, 'create') - await execCommands(commands, archiveFolder) + const workingDirectory = getWorkingDirectory() + + // -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. + // zstdmt is equivalent to 'zstd -T0' + // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. + // Using 30 here because we also support 32-bit self-hosted runners. + // Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd. + function getCompressionProgram(): string[] { + switch (compressionMethod) { + case CompressionMethod.Zstd: + return [ + '--use-compress-program', + IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30' + ] + case CompressionMethod.ZstdWithoutLong: + return ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt'] + default: + return ['-z'] + } + } + const args = [ + '--posix', + ...getCompressionProgram(), + '-cf', + cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '--exclude', + cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '-P', + '-C', + workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '--files-from', + manifestFilename + ] + await execTar(args, compressionMethod, archiveFolder) }