From 6f7397feb68cfc91380a4d5bf4f665d2871f4437 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Thu, 17 Nov 2022 07:26:52 +0000 Subject: [PATCH] Add GNUtar as default on windows --- packages/cache/__tests__/tar.test.ts | 58 ++++++++++++++--------- packages/cache/src/internal/cacheUtils.ts | 17 +++++-- packages/cache/src/internal/constants.ts | 3 ++ packages/cache/src/internal/tar.ts | 30 +++++------- 4 files changed, 61 insertions(+), 47 deletions(-) diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index e4233bc9..a6cd1725 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -1,7 +1,11 @@ import * as exec from '@actions/exec' import * as io from '@actions/io' import * as path from 'path' -import {CacheFilename, CompressionMethod} from '../src/internal/constants' +import { + CacheFilename, + CompressionMethod, + GnuTarPathOnWindows +} from '../src/internal/constants' import * as tar from '../src/internal/tar' import * as utils from '../src/internal/cacheUtils' // eslint-disable-next-line @typescript-eslint/no-require-imports @@ -28,6 +32,10 @@ beforeAll(async () => { await jest.requireActual('@actions/io').rmRF(getTempDir()) }) +beforeEach(async () => { + jest.restoreAllMocks() +}) + afterAll(async () => { delete process.env['GITHUB_WORKSPACE'] await jest.requireActual('@actions/io').rmRF(getTempDir()) @@ -41,13 +49,14 @@ test('zstd extract tar', async () => { ? `${process.env['windir']}\\fakepath\\cache.tar` : 'cache.tar' const workspace = process.env['GITHUB_WORKSPACE'] + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath await tar.extractTar(archivePath, CompressionMethod.Zstd) expect(mkdirMock).toHaveBeenCalledWith(workspace) expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${defaultTarPath}"`, + `"${tarPath}"`, [ '--use-compress-program', IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30', @@ -74,9 +83,7 @@ test('gzip extract tar', async () => { await tar.extractTar(archivePath, CompressionMethod.Gzip) expect(mkdirMock).toHaveBeenCalledWith(workspace) - const tarPath = IS_WINDOWS - ? `${process.env['windir']}\\System32\\tar.exe` - : defaultTarPath + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( `"${tarPath}"`, @@ -87,18 +94,19 @@ test('gzip extract tar', async () => { '-P', '-C', IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace - ].concat(IS_MAC ? ['--delay-directory-restore'] : []), + ] + .concat(IS_WINDOWS ? ['--force-local'] : []) + .concat(IS_MAC ? ['--delay-directory-restore'] : []), {cwd: undefined} ) }) -test('gzip extract GNU tar on windows', async () => { +test('gzip extract GNU tar on windows with GNUtar in path', async () => { if (IS_WINDOWS) { - jest.spyOn(fs, 'existsSync').mockReturnValueOnce(false) - + // GNU tar present in path but not at default location const isGnuMock = jest - .spyOn(utils, 'isGnuTarInstalled') - .mockReturnValue(Promise.resolve(true)) + .spyOn(utils, 'getGnuTarPathOnWindows') + .mockReturnValue(Promise.resolve('tar')) const execMock = jest.spyOn(exec, 'exec') const archivePath = `${process.env['windir']}\\fakepath\\cache.tar` const workspace = process.env['GITHUB_WORKSPACE'] @@ -134,9 +142,11 @@ test('zstd create tar', async () => { await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Zstd) + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath + expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${defaultTarPath}"`, + `"${tarPath}"`, [ '--posix', '--use-compress-program', @@ -170,9 +180,7 @@ test('gzip create tar', async () => { await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Gzip) - const tarPath = IS_WINDOWS - ? `${process.env['windir']}\\System32\\tar.exe` - : defaultTarPath + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( @@ -189,7 +197,9 @@ test('gzip create tar', async () => { IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, '--files-from', 'manifest.txt' - ].concat(IS_MAC ? ['--delay-directory-restore'] : []), + ] + .concat(IS_WINDOWS ? ['--force-local'] : []) + .concat(IS_MAC ? ['--delay-directory-restore'] : []), { cwd: archiveFolder } @@ -205,9 +215,10 @@ test('zstd list tar', async () => { await tar.listTar(archivePath, CompressionMethod.Zstd) + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${defaultTarPath}"`, + `"${tarPath}"`, [ '--use-compress-program', IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30', @@ -230,9 +241,10 @@ test('zstdWithoutLong list tar', async () => { await tar.listTar(archivePath, CompressionMethod.ZstdWithoutLong) + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${defaultTarPath}"`, + `"${tarPath}"`, [ '--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd', @@ -254,9 +266,7 @@ test('gzip list tar', async () => { await tar.listTar(archivePath, CompressionMethod.Gzip) - const tarPath = IS_WINDOWS - ? `${process.env['windir']}\\System32\\tar.exe` - : defaultTarPath + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( `"${tarPath}"`, @@ -265,7 +275,9 @@ test('gzip list tar', async () => { '-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P' - ].concat(IS_MAC ? ['--delay-directory-restore'] : []), + ] + .concat(IS_WINDOWS ? ['--force-local'] : []) + .concat(IS_MAC ? ['--delay-directory-restore'] : []), {cwd: undefined} ) -}) +}) \ No newline at end of file diff --git a/packages/cache/src/internal/cacheUtils.ts b/packages/cache/src/internal/cacheUtils.ts index c2ace526..9d4e4c3d 100644 --- a/packages/cache/src/internal/cacheUtils.ts +++ b/packages/cache/src/internal/cacheUtils.ts @@ -7,7 +7,11 @@ import * as path from 'path' import * as semver from 'semver' import * as util from 'util' import {v4 as uuidV4} from 'uuid' -import {CacheFilename, CompressionMethod} from './constants' +import { + CacheFilename, + CompressionMethod, + GnuTarPathOnWindows +} from './constants' // From https://github.com/actions/toolkit/blob/main/packages/tool-cache/src/tool-cache.ts#L23 export async function createTempDirectory(): Promise { @@ -90,7 +94,7 @@ async function getVersion(app: string): Promise { // Use zstandard if possible to maximize cache performance export async function getCompressionMethod(): Promise { - if (process.platform === 'win32' && !(await isGnuTarInstalled())) { + if (process.platform === 'win32' && !(await getGnuTarPathOnWindows())) { // Disable zstd due to bug https://github.com/actions/cache/issues/301 return CompressionMethod.Gzip } @@ -116,9 +120,12 @@ export function getCacheFileName(compressionMethod: CompressionMethod): string { : CacheFilename.Zstd } -export async function isGnuTarInstalled(): Promise { +export async function getGnuTarPathOnWindows(): Promise { + if (fs.existsSync(GnuTarPathOnWindows)) { + return GnuTarPathOnWindows + } const versionOutput = await getVersion('tar') - return versionOutput.toLowerCase().includes('gnu tar') + return versionOutput.toLowerCase().includes('gnu tar') ? io.which('tar') : '' } export function assertDefined(name: string, value?: T): T { @@ -134,4 +141,4 @@ export function isGhes(): boolean { process.env['GITHUB_SERVER_URL'] || 'https://github.com' ) return ghUrl.hostname.toUpperCase() !== 'GITHUB.COM' -} +} \ No newline at end of file diff --git a/packages/cache/src/internal/constants.ts b/packages/cache/src/internal/constants.ts index 2f78d326..35225b6c 100644 --- a/packages/cache/src/internal/constants.ts +++ b/packages/cache/src/internal/constants.ts @@ -21,3 +21,6 @@ export const DefaultRetryDelay = 5000 // over the socket during this period, the socket is destroyed and the download // is aborted. export const SocketTimeout = 5000 + +// The default path of GNUtar on hosted Windows runners +export const GnuTarPathOnWindows = `${process.env['PROGRAMFILES']}\\Git\\usr\\bin\\tar.exe` \ No newline at end of file diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 2e28ca1a..2c07dc81 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -7,21 +7,17 @@ import {CompressionMethod} from './constants' const IS_WINDOWS = process.platform === 'win32' -async function getTarPath( - args: string[], - compressionMethod: CompressionMethod -): Promise { +async function getTarPath(args: string[]): Promise { switch (process.platform) { case 'win32': { + const gnuTar = await utils.getGnuTarPathOnWindows() const systemTar = `${process.env['windir']}\\System32\\tar.exe` - if (compressionMethod !== CompressionMethod.Gzip) { - // We only use zstandard compression on windows when gnu tar is installed due to - // a bug with compressing large files with bsdtar + zstd + if (gnuTar) { + // Use GNUtar as default on windows args.push('--force-local') + return gnuTar } else if (existsSync(systemTar)) { return systemTar - } else if (await utils.isGnuTarInstalled()) { - args.push('--force-local') } break } @@ -40,13 +36,9 @@ async function getTarPath( return await io.which('tar', true) } -async function execTar( - args: string[], - compressionMethod: CompressionMethod, - cwd?: string -): Promise { +async function execTar(args: string[], cwd?: string): Promise { try { - await exec(`"${await getTarPath(args, compressionMethod)}"`, args, {cwd}) + await exec(`"${await getTarPath(args)}"`, args, {cwd}) } catch (error) { throw new Error(`Tar failed with error: ${error?.message}`) } @@ -85,7 +77,7 @@ export async function listTar( archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P' ] - await execTar(args, compressionMethod) + await execTar(args) } export async function extractTar( @@ -103,7 +95,7 @@ export async function extractTar( '-C', workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/') ] - await execTar(args, compressionMethod) + await execTar(args) } export async function createTar( @@ -151,5 +143,5 @@ export async function createTar( '--files-from', manifestFilename ] - await execTar(args, compressionMethod, archiveFolder) -} + await execTar(args, archiveFolder) +} \ No newline at end of file