1
0
Fork 0

Merge pull request #1237 from actions/pdotl/win-bsdtar-zstd

Bsdtar + zstd fallback to prevent zstd hanging issue on windows
pull/1244/head
Sampark Sharma 2022-12-01 12:14:55 +05:30 committed by GitHub
commit 785599dd2e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 383 additions and 119 deletions

View File

@ -4,7 +4,10 @@ import * as path from 'path'
import { import {
CacheFilename, CacheFilename,
CompressionMethod, CompressionMethod,
GnuTarPathOnWindows GnuTarPathOnWindows,
ManifestFilename,
SystemTarPathOnWindows,
TarFilename
} from '../src/internal/constants' } from '../src/internal/constants'
import * as tar from '../src/internal/tar' import * as tar from '../src/internal/tar'
import * as utils from '../src/internal/cacheUtils' import * as utils from '../src/internal/cacheUtils'
@ -17,7 +20,7 @@ jest.mock('@actions/io')
const IS_WINDOWS = process.platform === 'win32' const IS_WINDOWS = process.platform === 'win32'
const IS_MAC = process.platform === 'darwin' const IS_MAC = process.platform === 'darwin'
const defaultTarPath = process.platform === 'darwin' ? 'gtar' : 'tar' const defaultTarPath = IS_MAC ? 'gtar' : 'tar'
function getTempDir(): string { function getTempDir(): string {
return path.join(__dirname, '_temp', 'tar') return path.join(__dirname, '_temp', 'tar')
@ -56,10 +59,8 @@ test('zstd extract tar', async () => {
expect(mkdirMock).toHaveBeenCalledWith(workspace) expect(mkdirMock).toHaveBeenCalledWith(workspace)
expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith( expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
[ [
'--use-compress-program', `"${tarPath}"`,
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30',
'-xf', '-xf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P', '-P',
@ -67,11 +68,48 @@ test('zstd extract tar', async () => {
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace
] ]
.concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : []), .concat(IS_MAC ? ['--delay-directory-restore'] : [])
{cwd: undefined} .concat([
'--use-compress-program',
IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30'
])
.join(' ')
) )
}) })
test('zstd extract tar with windows BSDtar', async () => {
if (IS_WINDOWS) {
const mkdirMock = jest.spyOn(io, 'mkdirP')
const execMock = jest.spyOn(exec, 'exec')
jest
.spyOn(utils, 'getGnuTarPathOnWindows')
.mockReturnValue(Promise.resolve(''))
const archivePath = `${process.env['windir']}\\fakepath\\cache.tar`
const workspace = process.env['GITHUB_WORKSPACE']
const tarPath = SystemTarPathOnWindows
await tar.extractTar(archivePath, CompressionMethod.Zstd)
expect(mkdirMock).toHaveBeenCalledWith(workspace)
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
[
'zstd -d --long=30 -o',
TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'&&',
`"${tarPath}"`,
'-xf',
TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workspace?.replace(/\\/g, '/')
].join(' ')
)
}
})
test('gzip extract tar', async () => { test('gzip extract tar', async () => {
const mkdirMock = jest.spyOn(io, 'mkdirP') const mkdirMock = jest.spyOn(io, 'mkdirP')
const execMock = jest.spyOn(exec, 'exec') const execMock = jest.spyOn(exec, 'exec')
@ -86,9 +124,8 @@ test('gzip extract tar', async () => {
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith( expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
[ [
'-z', `"${tarPath}"`,
'-xf', '-xf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P', '-P',
@ -96,15 +133,16 @@ test('gzip extract tar', async () => {
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace
] ]
.concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : []), .concat(IS_MAC ? ['--delay-directory-restore'] : [])
{cwd: undefined} .concat(['-z'])
.join(' ')
) )
}) })
test('gzip extract GNU tar on windows with GNUtar in path', async () => { test('gzip extract GNU tar on windows with GNUtar in path', async () => {
if (IS_WINDOWS) { if (IS_WINDOWS) {
// GNU tar present in path but not at default location // GNU tar present in path but not at default location
const isGnuMock = jest jest
.spyOn(utils, 'getGnuTarPathOnWindows') .spyOn(utils, 'getGnuTarPathOnWindows')
.mockReturnValue(Promise.resolve('tar')) .mockReturnValue(Promise.resolve('tar'))
const execMock = jest.spyOn(exec, 'exec') const execMock = jest.spyOn(exec, 'exec')
@ -113,20 +151,18 @@ test('gzip extract GNU tar on windows with GNUtar in path', async () => {
await tar.extractTar(archivePath, CompressionMethod.Gzip) await tar.extractTar(archivePath, CompressionMethod.Gzip)
expect(isGnuMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith( expect(execMock).toHaveBeenCalledWith(
`"tar"`,
[ [
'-z', `"tar"`,
'-xf', '-xf',
archivePath.replace(/\\/g, '/'), archivePath.replace(/\\/g, '/'),
'-P', '-P',
'-C', '-C',
workspace?.replace(/\\/g, '/'), workspace?.replace(/\\/g, '/'),
'--force-local' '--force-local',
], '-z'
{cwd: undefined} ].join(' ')
) )
} }
}) })
@ -146,11 +182,9 @@ test('zstd create tar', async () => {
expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith( expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
[ [
`"${tarPath}"`,
'--posix', '--posix',
'--use-compress-program',
IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30',
'-cf', '-cf',
IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd, IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd,
'--exclude', '--exclude',
@ -159,16 +193,70 @@ test('zstd create tar', async () => {
'-C', '-C',
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace,
'--files-from', '--files-from',
'manifest.txt' ManifestFilename
] ]
.concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : []), .concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat([
'--use-compress-program',
IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30'
])
.join(' '),
undefined, // args
{ {
cwd: archiveFolder cwd: archiveFolder
} }
) )
}) })
test('zstd create tar with windows BSDtar', async () => {
if (IS_WINDOWS) {
const execMock = jest.spyOn(exec, 'exec')
jest
.spyOn(utils, 'getGnuTarPathOnWindows')
.mockReturnValue(Promise.resolve(''))
const archiveFolder = getTempDir()
const workspace = process.env['GITHUB_WORKSPACE']
const sourceDirectories = ['~/.npm/cache', `${workspace}/dist`]
await fs.promises.mkdir(archiveFolder, {recursive: true})
await tar.createTar(
archiveFolder,
sourceDirectories,
CompressionMethod.Zstd
)
const tarPath = SystemTarPathOnWindows
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
[
`"${tarPath}"`,
'--posix',
'-cf',
TarFilename.replace(/\\/g, '/'),
'--exclude',
TarFilename.replace(/\\/g, '/'),
'-P',
'-C',
workspace?.replace(/\\/g, '/'),
'--files-from',
ManifestFilename,
'&&',
'zstd -T0 --long=30 -o',
CacheFilename.Zstd.replace(/\\/g, '/'),
TarFilename.replace(/\\/g, '/')
].join(' '),
undefined, // args
{
cwd: archiveFolder
}
)
}
})
test('gzip create tar', async () => { test('gzip create tar', async () => {
const execMock = jest.spyOn(exec, 'exec') const execMock = jest.spyOn(exec, 'exec')
@ -184,10 +272,9 @@ test('gzip create tar', async () => {
expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith( expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
[ [
`"${tarPath}"`,
'--posix', '--posix',
'-z',
'-cf', '-cf',
IS_WINDOWS ? CacheFilename.Gzip.replace(/\\/g, '/') : CacheFilename.Gzip, IS_WINDOWS ? CacheFilename.Gzip.replace(/\\/g, '/') : CacheFilename.Gzip,
'--exclude', '--exclude',
@ -196,10 +283,13 @@ test('gzip create tar', async () => {
'-C', '-C',
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace,
'--files-from', '--files-from',
'manifest.txt' ManifestFilename
] ]
.concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : []), .concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat(['-z'])
.join(' '),
undefined, // args
{ {
cwd: archiveFolder cwd: archiveFolder
} }
@ -218,20 +308,49 @@ test('zstd list tar', async () => {
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith( expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
[ [
'--use-compress-program', `"${tarPath}"`,
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30',
'-tf', '-tf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P' '-P'
] ]
.concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : []), .concat(IS_MAC ? ['--delay-directory-restore'] : [])
{cwd: undefined} .concat([
'--use-compress-program',
IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30'
])
.join(' ')
) )
}) })
test('zstd list tar with windows BSDtar', async () => {
if (IS_WINDOWS) {
const execMock = jest.spyOn(exec, 'exec')
jest
.spyOn(utils, 'getGnuTarPathOnWindows')
.mockReturnValue(Promise.resolve(''))
const archivePath = `${process.env['windir']}\\fakepath\\cache.tar`
await tar.listTar(archivePath, CompressionMethod.Zstd)
const tarPath = SystemTarPathOnWindows
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
[
'zstd -d --long=30 -o',
TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'&&',
`"${tarPath}"`,
'-tf',
TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P'
].join(' ')
)
}
})
test('zstdWithoutLong list tar', async () => { test('zstdWithoutLong list tar', async () => {
const execMock = jest.spyOn(exec, 'exec') const execMock = jest.spyOn(exec, 'exec')
@ -244,17 +363,16 @@ test('zstdWithoutLong list tar', async () => {
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith( expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
[ [
'--use-compress-program', `"${tarPath}"`,
IS_WINDOWS ? 'zstd -d' : 'unzstd',
'-tf', '-tf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P' '-P'
] ]
.concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : []), .concat(IS_MAC ? ['--delay-directory-restore'] : [])
{cwd: undefined} .concat(['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd'])
.join(' ')
) )
}) })
@ -269,15 +387,15 @@ test('gzip list tar', async () => {
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith( expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
[ [
'-z', `"${tarPath}"`,
'-tf', '-tf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P' '-P'
] ]
.concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : []), .concat(IS_MAC ? ['--delay-directory-restore'] : [])
{cwd: undefined} .concat(['-z'])
.join(' ')
) )
}) })

View File

@ -94,11 +94,6 @@ async function getVersion(app: string): Promise<string> {
// Use zstandard if possible to maximize cache performance // Use zstandard if possible to maximize cache performance
export async function getCompressionMethod(): Promise<CompressionMethod> { export async function getCompressionMethod(): Promise<CompressionMethod> {
if (process.platform === 'win32' && !(await getGnuTarPathOnWindows())) {
// Disable zstd due to bug https://github.com/actions/cache/issues/301
return CompressionMethod.Gzip
}
const versionOutput = await getVersion('zstd') const versionOutput = await getVersion('zstd')
const version = semver.clean(versionOutput) const version = semver.clean(versionOutput)

View File

@ -11,6 +11,11 @@ export enum CompressionMethod {
Zstd = 'zstd' Zstd = 'zstd'
} }
export enum ArchiveToolType {
GNU = 'gnu',
BSD = 'bsd'
}
// The default number of retry attempts. // The default number of retry attempts.
export const DefaultRetryAttempts = 2 export const DefaultRetryAttempts = 2
@ -24,3 +29,10 @@ export const SocketTimeout = 5000
// The default path of GNUtar on hosted Windows runners // The default path of GNUtar on hosted Windows runners
export const GnuTarPathOnWindows = `${process.env['PROGRAMFILES']}\\Git\\usr\\bin\\tar.exe` export const GnuTarPathOnWindows = `${process.env['PROGRAMFILES']}\\Git\\usr\\bin\\tar.exe`
// The default path of BSDtar on hosted Windows runners
export const SystemTarPathOnWindows = `${process.env['SYSTEMDRIVE']}\\Windows\\System32\\tar.exe`
export const TarFilename = 'cache.tar'
export const ManifestFilename = 'manifest.txt'

View File

@ -31,3 +31,8 @@ export interface InternalCacheOptions {
compressionMethod?: CompressionMethod compressionMethod?: CompressionMethod
cacheSize?: number cacheSize?: number
} }
export interface ArchiveTool {
path: string
type: string
}

View File

@ -3,21 +3,28 @@ import * as io from '@actions/io'
import {existsSync, writeFileSync} from 'fs' import {existsSync, writeFileSync} from 'fs'
import * as path from 'path' import * as path from 'path'
import * as utils from './cacheUtils' import * as utils from './cacheUtils'
import {CompressionMethod} from './constants' import {ArchiveTool} from './contracts'
import {
CompressionMethod,
SystemTarPathOnWindows,
ArchiveToolType,
TarFilename,
ManifestFilename
} from './constants'
const IS_WINDOWS = process.platform === 'win32' const IS_WINDOWS = process.platform === 'win32'
async function getTarPath(args: string[]): Promise<string> { // Function also mutates the args array. For non-mutation call with passing an empty array.
async function getTarPath(): Promise<ArchiveTool> {
switch (process.platform) { switch (process.platform) {
case 'win32': { case 'win32': {
const gnuTar = await utils.getGnuTarPathOnWindows() const gnuTar = await utils.getGnuTarPathOnWindows()
const systemTar = `${process.env['windir']}\\System32\\tar.exe` const systemTar = SystemTarPathOnWindows
if (gnuTar) { if (gnuTar) {
// Use GNUtar as default on windows // Use GNUtar as default on windows
args.push('--force-local') return <ArchiveTool>{path: gnuTar, type: ArchiveToolType.GNU}
return gnuTar
} else if (existsSync(systemTar)) { } else if (existsSync(systemTar)) {
return systemTar return <ArchiveTool>{path: systemTar, type: ArchiveToolType.BSD}
} }
break break
} }
@ -25,22 +32,120 @@ async function getTarPath(args: string[]): Promise<string> {
const gnuTar = await io.which('gtar', false) const gnuTar = await io.which('gtar', false)
if (gnuTar) { if (gnuTar) {
// fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527 // fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527
args.push('--delay-directory-restore') return <ArchiveTool>{path: gnuTar, type: ArchiveToolType.GNU}
return gnuTar } else {
return <ArchiveTool>{
path: await io.which('tar', true),
type: ArchiveToolType.BSD
}
} }
break
} }
default: default:
break break
} }
return await io.which('tar', true) return <ArchiveTool>{
path: await io.which('tar', true),
type: ArchiveToolType.GNU
}
} }
async function execTar(args: string[], cwd?: string): Promise<void> { // Return arguments for tar as per tarPath, compressionMethod, method type and os
try { async function getTarArgs(
await exec(`"${await getTarPath(args)}"`, args, {cwd}) tarPath: ArchiveTool,
} catch (error) { compressionMethod: CompressionMethod,
throw new Error(`Tar failed with error: ${error?.message}`) type: string,
archivePath = ''
): Promise<string[]> {
const args = [`"${tarPath.path}"`]
const cacheFileName = utils.getCacheFileName(compressionMethod)
const tarFile = 'cache.tar'
const workingDirectory = getWorkingDirectory()
const BSD_TAR_ZSTD =
tarPath.type === ArchiveToolType.BSD &&
compressionMethod !== CompressionMethod.Gzip &&
IS_WINDOWS
// Method specific args
switch (type) {
case 'create':
args.push(
'--posix',
'-cf',
BSD_TAR_ZSTD
? tarFile
: cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'--exclude',
BSD_TAR_ZSTD
? tarFile
: cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'--files-from',
ManifestFilename
)
break
case 'extract':
args.push(
'-xf',
BSD_TAR_ZSTD
? tarFile
: archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
)
break
case 'list':
args.push(
'-tf',
BSD_TAR_ZSTD
? tarFile
: archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P'
)
break
}
// Platform specific args
if (tarPath.type === ArchiveToolType.GNU) {
switch (process.platform) {
case 'win32':
args.push('--force-local')
break
case 'darwin':
args.push('--delay-directory-restore')
break
}
}
return args
}
async function getArgs(
compressionMethod: CompressionMethod,
type: string,
archivePath = ''
): Promise<string> {
const tarPath = await getTarPath()
const tarArgs = await getTarArgs(
tarPath,
compressionMethod,
type,
archivePath
)
const compressionArgs =
type !== 'create'
? await getDecompressionProgram(tarPath, compressionMethod, archivePath)
: await getCompressionProgram(tarPath, compressionMethod)
const BSD_TAR_ZSTD =
tarPath.type === ArchiveToolType.BSD &&
compressionMethod !== CompressionMethod.Gzip &&
IS_WINDOWS
if (BSD_TAR_ZSTD && type !== 'create') {
return [...compressionArgs, ...tarArgs].join(' ')
} else {
return [...tarArgs, ...compressionArgs].join(' ')
} }
} }
@ -49,19 +154,82 @@ function getWorkingDirectory(): string {
} }
// Common function for extractTar and listTar to get the compression method // Common function for extractTar and listTar to get the compression method
function getCompressionProgram(compressionMethod: CompressionMethod): string[] { async function getDecompressionProgram(
tarPath: ArchiveTool,
compressionMethod: CompressionMethod,
archivePath: string
): Promise<string[]> {
// -d: Decompress. // -d: Decompress.
// unzstd is equivalent to 'zstd -d' // unzstd is equivalent to 'zstd -d'
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners. // Using 30 here because we also support 32-bit self-hosted runners.
const BSD_TAR_ZSTD =
tarPath.type === ArchiveToolType.BSD &&
compressionMethod !== CompressionMethod.Gzip &&
IS_WINDOWS
switch (compressionMethod) { switch (compressionMethod) {
case CompressionMethod.Zstd: case CompressionMethod.Zstd:
return [ return BSD_TAR_ZSTD
'--use-compress-program', ? [
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30' 'zstd -d --long=30 -o',
] TarFilename,
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'&&'
]
: [
'--use-compress-program',
IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30'
]
case CompressionMethod.ZstdWithoutLong: case CompressionMethod.ZstdWithoutLong:
return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd'] return BSD_TAR_ZSTD
? [
'zstd -d -o',
TarFilename,
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'&&'
]
: ['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd']
default:
return ['-z']
}
}
// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores.
// zstdmt is equivalent to 'zstd -T0'
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners.
// Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd.
async function getCompressionProgram(
tarPath: ArchiveTool,
compressionMethod: CompressionMethod
): Promise<string[]> {
const cacheFileName = utils.getCacheFileName(compressionMethod)
const BSD_TAR_ZSTD =
tarPath.type === ArchiveToolType.BSD &&
compressionMethod !== CompressionMethod.Gzip &&
IS_WINDOWS
switch (compressionMethod) {
case CompressionMethod.Zstd:
return BSD_TAR_ZSTD
? [
'&&',
'zstd -T0 --long=30 -o',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
TarFilename
]
: [
'--use-compress-program',
IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30'
]
case CompressionMethod.ZstdWithoutLong:
return BSD_TAR_ZSTD
? [
'&&',
'zstd -T0 -o',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
TarFilename
]
: ['--use-compress-program', IS_WINDOWS ? '"zstd -T0"' : 'zstdmt']
default: default:
return ['-z'] return ['-z']
} }
@ -71,13 +239,12 @@ export async function listTar(
archivePath: string, archivePath: string,
compressionMethod: CompressionMethod compressionMethod: CompressionMethod
): Promise<void> { ): Promise<void> {
const args = [ const args = await getArgs(compressionMethod, 'list', archivePath)
...getCompressionProgram(compressionMethod), try {
'-tf', await exec(args)
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), } catch (error) {
'-P' throw new Error(`Tar failed with error: ${error?.message}`)
] }
await execTar(args)
} }
export async function extractTar( export async function extractTar(
@ -87,15 +254,12 @@ export async function extractTar(
// Create directory to extract tar into // Create directory to extract tar into
const workingDirectory = getWorkingDirectory() const workingDirectory = getWorkingDirectory()
await io.mkdirP(workingDirectory) await io.mkdirP(workingDirectory)
const args = [ const args = await getArgs(compressionMethod, 'extract', archivePath)
...getCompressionProgram(compressionMethod), try {
'-xf', await exec(args)
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), } catch (error) {
'-P', throw new Error(`Tar failed with error: ${error?.message}`)
'-C', }
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
]
await execTar(args)
} }
export async function createTar( export async function createTar(
@ -104,44 +268,14 @@ export async function createTar(
compressionMethod: CompressionMethod compressionMethod: CompressionMethod
): Promise<void> { ): Promise<void> {
// Write source directories to manifest.txt to avoid command length limits // Write source directories to manifest.txt to avoid command length limits
const manifestFilename = 'manifest.txt'
const cacheFileName = utils.getCacheFileName(compressionMethod)
writeFileSync( writeFileSync(
path.join(archiveFolder, manifestFilename), path.join(archiveFolder, ManifestFilename),
sourceDirectories.join('\n') sourceDirectories.join('\n')
) )
const workingDirectory = getWorkingDirectory() const args = await getArgs(compressionMethod, 'create')
try {
// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. await exec(args, undefined, {cwd: archiveFolder})
// zstdmt is equivalent to 'zstd -T0' } catch (error) {
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. throw new Error(`Tar failed with error: ${error?.message}`)
// Using 30 here because we also support 32-bit self-hosted runners.
// Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd.
function getCompressionProgram(): string[] {
switch (compressionMethod) {
case CompressionMethod.Zstd:
return [
'--use-compress-program',
IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30'
]
case CompressionMethod.ZstdWithoutLong:
return ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt']
default:
return ['-z']
}
} }
const args = [
'--posix',
...getCompressionProgram(),
'-cf',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'--exclude',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'--files-from',
manifestFilename
]
await execTar(args, archiveFolder)
} }