mirror of https://github.com/actions/toolkit
Merge pull request #1237 from actions/pdotl/win-bsdtar-zstd
Bsdtar + zstd fallback to prevent zstd hanging issue on windowspull/1244/head
commit
785599dd2e
|
@ -4,7 +4,10 @@ import * as path from 'path'
|
|||
import {
|
||||
CacheFilename,
|
||||
CompressionMethod,
|
||||
GnuTarPathOnWindows
|
||||
GnuTarPathOnWindows,
|
||||
ManifestFilename,
|
||||
SystemTarPathOnWindows,
|
||||
TarFilename
|
||||
} from '../src/internal/constants'
|
||||
import * as tar from '../src/internal/tar'
|
||||
import * as utils from '../src/internal/cacheUtils'
|
||||
|
@ -17,7 +20,7 @@ jest.mock('@actions/io')
|
|||
const IS_WINDOWS = process.platform === 'win32'
|
||||
const IS_MAC = process.platform === 'darwin'
|
||||
|
||||
const defaultTarPath = process.platform === 'darwin' ? 'gtar' : 'tar'
|
||||
const defaultTarPath = IS_MAC ? 'gtar' : 'tar'
|
||||
|
||||
function getTempDir(): string {
|
||||
return path.join(__dirname, '_temp', 'tar')
|
||||
|
@ -56,10 +59,8 @@ test('zstd extract tar', async () => {
|
|||
expect(mkdirMock).toHaveBeenCalledWith(workspace)
|
||||
expect(execMock).toHaveBeenCalledTimes(1)
|
||||
expect(execMock).toHaveBeenCalledWith(
|
||||
`"${tarPath}"`,
|
||||
[
|
||||
'--use-compress-program',
|
||||
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30',
|
||||
`"${tarPath}"`,
|
||||
'-xf',
|
||||
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
|
||||
'-P',
|
||||
|
@ -67,11 +68,48 @@ test('zstd extract tar', async () => {
|
|||
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace
|
||||
]
|
||||
.concat(IS_WINDOWS ? ['--force-local'] : [])
|
||||
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
|
||||
{cwd: undefined}
|
||||
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
|
||||
.concat([
|
||||
'--use-compress-program',
|
||||
IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30'
|
||||
])
|
||||
.join(' ')
|
||||
)
|
||||
})
|
||||
|
||||
test('zstd extract tar with windows BSDtar', async () => {
|
||||
if (IS_WINDOWS) {
|
||||
const mkdirMock = jest.spyOn(io, 'mkdirP')
|
||||
const execMock = jest.spyOn(exec, 'exec')
|
||||
jest
|
||||
.spyOn(utils, 'getGnuTarPathOnWindows')
|
||||
.mockReturnValue(Promise.resolve(''))
|
||||
|
||||
const archivePath = `${process.env['windir']}\\fakepath\\cache.tar`
|
||||
const workspace = process.env['GITHUB_WORKSPACE']
|
||||
const tarPath = SystemTarPathOnWindows
|
||||
|
||||
await tar.extractTar(archivePath, CompressionMethod.Zstd)
|
||||
|
||||
expect(mkdirMock).toHaveBeenCalledWith(workspace)
|
||||
expect(execMock).toHaveBeenCalledTimes(1)
|
||||
expect(execMock).toHaveBeenCalledWith(
|
||||
[
|
||||
'zstd -d --long=30 -o',
|
||||
TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||
'&&',
|
||||
`"${tarPath}"`,
|
||||
'-xf',
|
||||
TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||
'-P',
|
||||
'-C',
|
||||
workspace?.replace(/\\/g, '/')
|
||||
].join(' ')
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
test('gzip extract tar', async () => {
|
||||
const mkdirMock = jest.spyOn(io, 'mkdirP')
|
||||
const execMock = jest.spyOn(exec, 'exec')
|
||||
|
@ -86,9 +124,8 @@ test('gzip extract tar', async () => {
|
|||
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
|
||||
expect(execMock).toHaveBeenCalledTimes(1)
|
||||
expect(execMock).toHaveBeenCalledWith(
|
||||
`"${tarPath}"`,
|
||||
[
|
||||
'-z',
|
||||
`"${tarPath}"`,
|
||||
'-xf',
|
||||
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
|
||||
'-P',
|
||||
|
@ -96,15 +133,16 @@ test('gzip extract tar', async () => {
|
|||
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace
|
||||
]
|
||||
.concat(IS_WINDOWS ? ['--force-local'] : [])
|
||||
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
|
||||
{cwd: undefined}
|
||||
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
|
||||
.concat(['-z'])
|
||||
.join(' ')
|
||||
)
|
||||
})
|
||||
|
||||
test('gzip extract GNU tar on windows with GNUtar in path', async () => {
|
||||
if (IS_WINDOWS) {
|
||||
// GNU tar present in path but not at default location
|
||||
const isGnuMock = jest
|
||||
jest
|
||||
.spyOn(utils, 'getGnuTarPathOnWindows')
|
||||
.mockReturnValue(Promise.resolve('tar'))
|
||||
const execMock = jest.spyOn(exec, 'exec')
|
||||
|
@ -113,20 +151,18 @@ test('gzip extract GNU tar on windows with GNUtar in path', async () => {
|
|||
|
||||
await tar.extractTar(archivePath, CompressionMethod.Gzip)
|
||||
|
||||
expect(isGnuMock).toHaveBeenCalledTimes(1)
|
||||
expect(execMock).toHaveBeenCalledTimes(1)
|
||||
expect(execMock).toHaveBeenCalledWith(
|
||||
`"tar"`,
|
||||
[
|
||||
'-z',
|
||||
`"tar"`,
|
||||
'-xf',
|
||||
archivePath.replace(/\\/g, '/'),
|
||||
'-P',
|
||||
'-C',
|
||||
workspace?.replace(/\\/g, '/'),
|
||||
'--force-local'
|
||||
],
|
||||
{cwd: undefined}
|
||||
'--force-local',
|
||||
'-z'
|
||||
].join(' ')
|
||||
)
|
||||
}
|
||||
})
|
||||
|
@ -146,11 +182,9 @@ test('zstd create tar', async () => {
|
|||
|
||||
expect(execMock).toHaveBeenCalledTimes(1)
|
||||
expect(execMock).toHaveBeenCalledWith(
|
||||
`"${tarPath}"`,
|
||||
[
|
||||
`"${tarPath}"`,
|
||||
'--posix',
|
||||
'--use-compress-program',
|
||||
IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30',
|
||||
'-cf',
|
||||
IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd,
|
||||
'--exclude',
|
||||
|
@ -159,16 +193,70 @@ test('zstd create tar', async () => {
|
|||
'-C',
|
||||
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace,
|
||||
'--files-from',
|
||||
'manifest.txt'
|
||||
ManifestFilename
|
||||
]
|
||||
.concat(IS_WINDOWS ? ['--force-local'] : [])
|
||||
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
|
||||
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
|
||||
.concat([
|
||||
'--use-compress-program',
|
||||
IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30'
|
||||
])
|
||||
.join(' '),
|
||||
undefined, // args
|
||||
{
|
||||
cwd: archiveFolder
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
test('zstd create tar with windows BSDtar', async () => {
|
||||
if (IS_WINDOWS) {
|
||||
const execMock = jest.spyOn(exec, 'exec')
|
||||
jest
|
||||
.spyOn(utils, 'getGnuTarPathOnWindows')
|
||||
.mockReturnValue(Promise.resolve(''))
|
||||
|
||||
const archiveFolder = getTempDir()
|
||||
const workspace = process.env['GITHUB_WORKSPACE']
|
||||
const sourceDirectories = ['~/.npm/cache', `${workspace}/dist`]
|
||||
|
||||
await fs.promises.mkdir(archiveFolder, {recursive: true})
|
||||
|
||||
await tar.createTar(
|
||||
archiveFolder,
|
||||
sourceDirectories,
|
||||
CompressionMethod.Zstd
|
||||
)
|
||||
|
||||
const tarPath = SystemTarPathOnWindows
|
||||
|
||||
expect(execMock).toHaveBeenCalledTimes(1)
|
||||
expect(execMock).toHaveBeenCalledWith(
|
||||
[
|
||||
`"${tarPath}"`,
|
||||
'--posix',
|
||||
'-cf',
|
||||
TarFilename.replace(/\\/g, '/'),
|
||||
'--exclude',
|
||||
TarFilename.replace(/\\/g, '/'),
|
||||
'-P',
|
||||
'-C',
|
||||
workspace?.replace(/\\/g, '/'),
|
||||
'--files-from',
|
||||
ManifestFilename,
|
||||
'&&',
|
||||
'zstd -T0 --long=30 -o',
|
||||
CacheFilename.Zstd.replace(/\\/g, '/'),
|
||||
TarFilename.replace(/\\/g, '/')
|
||||
].join(' '),
|
||||
undefined, // args
|
||||
{
|
||||
cwd: archiveFolder
|
||||
}
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
test('gzip create tar', async () => {
|
||||
const execMock = jest.spyOn(exec, 'exec')
|
||||
|
||||
|
@ -184,10 +272,9 @@ test('gzip create tar', async () => {
|
|||
|
||||
expect(execMock).toHaveBeenCalledTimes(1)
|
||||
expect(execMock).toHaveBeenCalledWith(
|
||||
`"${tarPath}"`,
|
||||
[
|
||||
`"${tarPath}"`,
|
||||
'--posix',
|
||||
'-z',
|
||||
'-cf',
|
||||
IS_WINDOWS ? CacheFilename.Gzip.replace(/\\/g, '/') : CacheFilename.Gzip,
|
||||
'--exclude',
|
||||
|
@ -196,10 +283,13 @@ test('gzip create tar', async () => {
|
|||
'-C',
|
||||
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace,
|
||||
'--files-from',
|
||||
'manifest.txt'
|
||||
ManifestFilename
|
||||
]
|
||||
.concat(IS_WINDOWS ? ['--force-local'] : [])
|
||||
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
|
||||
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
|
||||
.concat(['-z'])
|
||||
.join(' '),
|
||||
undefined, // args
|
||||
{
|
||||
cwd: archiveFolder
|
||||
}
|
||||
|
@ -218,20 +308,49 @@ test('zstd list tar', async () => {
|
|||
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
|
||||
expect(execMock).toHaveBeenCalledTimes(1)
|
||||
expect(execMock).toHaveBeenCalledWith(
|
||||
`"${tarPath}"`,
|
||||
[
|
||||
'--use-compress-program',
|
||||
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30',
|
||||
`"${tarPath}"`,
|
||||
'-tf',
|
||||
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
|
||||
'-P'
|
||||
]
|
||||
.concat(IS_WINDOWS ? ['--force-local'] : [])
|
||||
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
|
||||
{cwd: undefined}
|
||||
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
|
||||
.concat([
|
||||
'--use-compress-program',
|
||||
IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30'
|
||||
])
|
||||
.join(' ')
|
||||
)
|
||||
})
|
||||
|
||||
test('zstd list tar with windows BSDtar', async () => {
|
||||
if (IS_WINDOWS) {
|
||||
const execMock = jest.spyOn(exec, 'exec')
|
||||
jest
|
||||
.spyOn(utils, 'getGnuTarPathOnWindows')
|
||||
.mockReturnValue(Promise.resolve(''))
|
||||
const archivePath = `${process.env['windir']}\\fakepath\\cache.tar`
|
||||
|
||||
await tar.listTar(archivePath, CompressionMethod.Zstd)
|
||||
|
||||
const tarPath = SystemTarPathOnWindows
|
||||
expect(execMock).toHaveBeenCalledTimes(1)
|
||||
expect(execMock).toHaveBeenCalledWith(
|
||||
[
|
||||
'zstd -d --long=30 -o',
|
||||
TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||
'&&',
|
||||
`"${tarPath}"`,
|
||||
'-tf',
|
||||
TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||
'-P'
|
||||
].join(' ')
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
test('zstdWithoutLong list tar', async () => {
|
||||
const execMock = jest.spyOn(exec, 'exec')
|
||||
|
||||
|
@ -244,17 +363,16 @@ test('zstdWithoutLong list tar', async () => {
|
|||
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
|
||||
expect(execMock).toHaveBeenCalledTimes(1)
|
||||
expect(execMock).toHaveBeenCalledWith(
|
||||
`"${tarPath}"`,
|
||||
[
|
||||
'--use-compress-program',
|
||||
IS_WINDOWS ? 'zstd -d' : 'unzstd',
|
||||
`"${tarPath}"`,
|
||||
'-tf',
|
||||
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
|
||||
'-P'
|
||||
]
|
||||
.concat(IS_WINDOWS ? ['--force-local'] : [])
|
||||
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
|
||||
{cwd: undefined}
|
||||
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
|
||||
.concat(['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd'])
|
||||
.join(' ')
|
||||
)
|
||||
})
|
||||
|
||||
|
@ -269,15 +387,15 @@ test('gzip list tar', async () => {
|
|||
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
|
||||
expect(execMock).toHaveBeenCalledTimes(1)
|
||||
expect(execMock).toHaveBeenCalledWith(
|
||||
`"${tarPath}"`,
|
||||
[
|
||||
'-z',
|
||||
`"${tarPath}"`,
|
||||
'-tf',
|
||||
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
|
||||
'-P'
|
||||
]
|
||||
.concat(IS_WINDOWS ? ['--force-local'] : [])
|
||||
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
|
||||
{cwd: undefined}
|
||||
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
|
||||
.concat(['-z'])
|
||||
.join(' ')
|
||||
)
|
||||
})
|
||||
|
|
|
@ -94,11 +94,6 @@ async function getVersion(app: string): Promise<string> {
|
|||
|
||||
// Use zstandard if possible to maximize cache performance
|
||||
export async function getCompressionMethod(): Promise<CompressionMethod> {
|
||||
if (process.platform === 'win32' && !(await getGnuTarPathOnWindows())) {
|
||||
// Disable zstd due to bug https://github.com/actions/cache/issues/301
|
||||
return CompressionMethod.Gzip
|
||||
}
|
||||
|
||||
const versionOutput = await getVersion('zstd')
|
||||
const version = semver.clean(versionOutput)
|
||||
|
||||
|
|
|
@ -11,6 +11,11 @@ export enum CompressionMethod {
|
|||
Zstd = 'zstd'
|
||||
}
|
||||
|
||||
export enum ArchiveToolType {
|
||||
GNU = 'gnu',
|
||||
BSD = 'bsd'
|
||||
}
|
||||
|
||||
// The default number of retry attempts.
|
||||
export const DefaultRetryAttempts = 2
|
||||
|
||||
|
@ -24,3 +29,10 @@ export const SocketTimeout = 5000
|
|||
|
||||
// The default path of GNUtar on hosted Windows runners
|
||||
export const GnuTarPathOnWindows = `${process.env['PROGRAMFILES']}\\Git\\usr\\bin\\tar.exe`
|
||||
|
||||
// The default path of BSDtar on hosted Windows runners
|
||||
export const SystemTarPathOnWindows = `${process.env['SYSTEMDRIVE']}\\Windows\\System32\\tar.exe`
|
||||
|
||||
export const TarFilename = 'cache.tar'
|
||||
|
||||
export const ManifestFilename = 'manifest.txt'
|
||||
|
|
|
@ -31,3 +31,8 @@ export interface InternalCacheOptions {
|
|||
compressionMethod?: CompressionMethod
|
||||
cacheSize?: number
|
||||
}
|
||||
|
||||
export interface ArchiveTool {
|
||||
path: string
|
||||
type: string
|
||||
}
|
||||
|
|
|
@ -3,21 +3,28 @@ import * as io from '@actions/io'
|
|||
import {existsSync, writeFileSync} from 'fs'
|
||||
import * as path from 'path'
|
||||
import * as utils from './cacheUtils'
|
||||
import {CompressionMethod} from './constants'
|
||||
import {ArchiveTool} from './contracts'
|
||||
import {
|
||||
CompressionMethod,
|
||||
SystemTarPathOnWindows,
|
||||
ArchiveToolType,
|
||||
TarFilename,
|
||||
ManifestFilename
|
||||
} from './constants'
|
||||
|
||||
const IS_WINDOWS = process.platform === 'win32'
|
||||
|
||||
async function getTarPath(args: string[]): Promise<string> {
|
||||
// Function also mutates the args array. For non-mutation call with passing an empty array.
|
||||
async function getTarPath(): Promise<ArchiveTool> {
|
||||
switch (process.platform) {
|
||||
case 'win32': {
|
||||
const gnuTar = await utils.getGnuTarPathOnWindows()
|
||||
const systemTar = `${process.env['windir']}\\System32\\tar.exe`
|
||||
const systemTar = SystemTarPathOnWindows
|
||||
if (gnuTar) {
|
||||
// Use GNUtar as default on windows
|
||||
args.push('--force-local')
|
||||
return gnuTar
|
||||
return <ArchiveTool>{path: gnuTar, type: ArchiveToolType.GNU}
|
||||
} else if (existsSync(systemTar)) {
|
||||
return systemTar
|
||||
return <ArchiveTool>{path: systemTar, type: ArchiveToolType.BSD}
|
||||
}
|
||||
break
|
||||
}
|
||||
|
@ -25,22 +32,120 @@ async function getTarPath(args: string[]): Promise<string> {
|
|||
const gnuTar = await io.which('gtar', false)
|
||||
if (gnuTar) {
|
||||
// fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527
|
||||
args.push('--delay-directory-restore')
|
||||
return gnuTar
|
||||
return <ArchiveTool>{path: gnuTar, type: ArchiveToolType.GNU}
|
||||
} else {
|
||||
return <ArchiveTool>{
|
||||
path: await io.which('tar', true),
|
||||
type: ArchiveToolType.BSD
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
default:
|
||||
break
|
||||
}
|
||||
return await io.which('tar', true)
|
||||
return <ArchiveTool>{
|
||||
path: await io.which('tar', true),
|
||||
type: ArchiveToolType.GNU
|
||||
}
|
||||
}
|
||||
|
||||
async function execTar(args: string[], cwd?: string): Promise<void> {
|
||||
try {
|
||||
await exec(`"${await getTarPath(args)}"`, args, {cwd})
|
||||
} catch (error) {
|
||||
throw new Error(`Tar failed with error: ${error?.message}`)
|
||||
// Return arguments for tar as per tarPath, compressionMethod, method type and os
|
||||
async function getTarArgs(
|
||||
tarPath: ArchiveTool,
|
||||
compressionMethod: CompressionMethod,
|
||||
type: string,
|
||||
archivePath = ''
|
||||
): Promise<string[]> {
|
||||
const args = [`"${tarPath.path}"`]
|
||||
const cacheFileName = utils.getCacheFileName(compressionMethod)
|
||||
const tarFile = 'cache.tar'
|
||||
const workingDirectory = getWorkingDirectory()
|
||||
const BSD_TAR_ZSTD =
|
||||
tarPath.type === ArchiveToolType.BSD &&
|
||||
compressionMethod !== CompressionMethod.Gzip &&
|
||||
IS_WINDOWS
|
||||
|
||||
// Method specific args
|
||||
switch (type) {
|
||||
case 'create':
|
||||
args.push(
|
||||
'--posix',
|
||||
'-cf',
|
||||
BSD_TAR_ZSTD
|
||||
? tarFile
|
||||
: cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||
'--exclude',
|
||||
BSD_TAR_ZSTD
|
||||
? tarFile
|
||||
: cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||
'-P',
|
||||
'-C',
|
||||
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||
'--files-from',
|
||||
ManifestFilename
|
||||
)
|
||||
break
|
||||
case 'extract':
|
||||
args.push(
|
||||
'-xf',
|
||||
BSD_TAR_ZSTD
|
||||
? tarFile
|
||||
: archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||
'-P',
|
||||
'-C',
|
||||
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
|
||||
)
|
||||
break
|
||||
case 'list':
|
||||
args.push(
|
||||
'-tf',
|
||||
BSD_TAR_ZSTD
|
||||
? tarFile
|
||||
: archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||
'-P'
|
||||
)
|
||||
break
|
||||
}
|
||||
|
||||
// Platform specific args
|
||||
if (tarPath.type === ArchiveToolType.GNU) {
|
||||
switch (process.platform) {
|
||||
case 'win32':
|
||||
args.push('--force-local')
|
||||
break
|
||||
case 'darwin':
|
||||
args.push('--delay-directory-restore')
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return args
|
||||
}
|
||||
|
||||
async function getArgs(
|
||||
compressionMethod: CompressionMethod,
|
||||
type: string,
|
||||
archivePath = ''
|
||||
): Promise<string> {
|
||||
const tarPath = await getTarPath()
|
||||
const tarArgs = await getTarArgs(
|
||||
tarPath,
|
||||
compressionMethod,
|
||||
type,
|
||||
archivePath
|
||||
)
|
||||
const compressionArgs =
|
||||
type !== 'create'
|
||||
? await getDecompressionProgram(tarPath, compressionMethod, archivePath)
|
||||
: await getCompressionProgram(tarPath, compressionMethod)
|
||||
const BSD_TAR_ZSTD =
|
||||
tarPath.type === ArchiveToolType.BSD &&
|
||||
compressionMethod !== CompressionMethod.Gzip &&
|
||||
IS_WINDOWS
|
||||
if (BSD_TAR_ZSTD && type !== 'create') {
|
||||
return [...compressionArgs, ...tarArgs].join(' ')
|
||||
} else {
|
||||
return [...tarArgs, ...compressionArgs].join(' ')
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -49,19 +154,82 @@ function getWorkingDirectory(): string {
|
|||
}
|
||||
|
||||
// Common function for extractTar and listTar to get the compression method
|
||||
function getCompressionProgram(compressionMethod: CompressionMethod): string[] {
|
||||
async function getDecompressionProgram(
|
||||
tarPath: ArchiveTool,
|
||||
compressionMethod: CompressionMethod,
|
||||
archivePath: string
|
||||
): Promise<string[]> {
|
||||
// -d: Decompress.
|
||||
// unzstd is equivalent to 'zstd -d'
|
||||
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
|
||||
// Using 30 here because we also support 32-bit self-hosted runners.
|
||||
const BSD_TAR_ZSTD =
|
||||
tarPath.type === ArchiveToolType.BSD &&
|
||||
compressionMethod !== CompressionMethod.Gzip &&
|
||||
IS_WINDOWS
|
||||
switch (compressionMethod) {
|
||||
case CompressionMethod.Zstd:
|
||||
return [
|
||||
return BSD_TAR_ZSTD
|
||||
? [
|
||||
'zstd -d --long=30 -o',
|
||||
TarFilename,
|
||||
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||
'&&'
|
||||
]
|
||||
: [
|
||||
'--use-compress-program',
|
||||
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30'
|
||||
IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30'
|
||||
]
|
||||
case CompressionMethod.ZstdWithoutLong:
|
||||
return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd']
|
||||
return BSD_TAR_ZSTD
|
||||
? [
|
||||
'zstd -d -o',
|
||||
TarFilename,
|
||||
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||
'&&'
|
||||
]
|
||||
: ['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd']
|
||||
default:
|
||||
return ['-z']
|
||||
}
|
||||
}
|
||||
|
||||
// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores.
|
||||
// zstdmt is equivalent to 'zstd -T0'
|
||||
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
|
||||
// Using 30 here because we also support 32-bit self-hosted runners.
|
||||
// Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd.
|
||||
async function getCompressionProgram(
|
||||
tarPath: ArchiveTool,
|
||||
compressionMethod: CompressionMethod
|
||||
): Promise<string[]> {
|
||||
const cacheFileName = utils.getCacheFileName(compressionMethod)
|
||||
const BSD_TAR_ZSTD =
|
||||
tarPath.type === ArchiveToolType.BSD &&
|
||||
compressionMethod !== CompressionMethod.Gzip &&
|
||||
IS_WINDOWS
|
||||
switch (compressionMethod) {
|
||||
case CompressionMethod.Zstd:
|
||||
return BSD_TAR_ZSTD
|
||||
? [
|
||||
'&&',
|
||||
'zstd -T0 --long=30 -o',
|
||||
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||
TarFilename
|
||||
]
|
||||
: [
|
||||
'--use-compress-program',
|
||||
IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30'
|
||||
]
|
||||
case CompressionMethod.ZstdWithoutLong:
|
||||
return BSD_TAR_ZSTD
|
||||
? [
|
||||
'&&',
|
||||
'zstd -T0 -o',
|
||||
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||
TarFilename
|
||||
]
|
||||
: ['--use-compress-program', IS_WINDOWS ? '"zstd -T0"' : 'zstdmt']
|
||||
default:
|
||||
return ['-z']
|
||||
}
|
||||
|
@ -71,13 +239,12 @@ export async function listTar(
|
|||
archivePath: string,
|
||||
compressionMethod: CompressionMethod
|
||||
): Promise<void> {
|
||||
const args = [
|
||||
...getCompressionProgram(compressionMethod),
|
||||
'-tf',
|
||||
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||
'-P'
|
||||
]
|
||||
await execTar(args)
|
||||
const args = await getArgs(compressionMethod, 'list', archivePath)
|
||||
try {
|
||||
await exec(args)
|
||||
} catch (error) {
|
||||
throw new Error(`Tar failed with error: ${error?.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
export async function extractTar(
|
||||
|
@ -87,15 +254,12 @@ export async function extractTar(
|
|||
// Create directory to extract tar into
|
||||
const workingDirectory = getWorkingDirectory()
|
||||
await io.mkdirP(workingDirectory)
|
||||
const args = [
|
||||
...getCompressionProgram(compressionMethod),
|
||||
'-xf',
|
||||
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||
'-P',
|
||||
'-C',
|
||||
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
|
||||
]
|
||||
await execTar(args)
|
||||
const args = await getArgs(compressionMethod, 'extract', archivePath)
|
||||
try {
|
||||
await exec(args)
|
||||
} catch (error) {
|
||||
throw new Error(`Tar failed with error: ${error?.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
export async function createTar(
|
||||
|
@ -104,44 +268,14 @@ export async function createTar(
|
|||
compressionMethod: CompressionMethod
|
||||
): Promise<void> {
|
||||
// Write source directories to manifest.txt to avoid command length limits
|
||||
const manifestFilename = 'manifest.txt'
|
||||
const cacheFileName = utils.getCacheFileName(compressionMethod)
|
||||
writeFileSync(
|
||||
path.join(archiveFolder, manifestFilename),
|
||||
path.join(archiveFolder, ManifestFilename),
|
||||
sourceDirectories.join('\n')
|
||||
)
|
||||
const workingDirectory = getWorkingDirectory()
|
||||
|
||||
// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores.
|
||||
// zstdmt is equivalent to 'zstd -T0'
|
||||
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
|
||||
// Using 30 here because we also support 32-bit self-hosted runners.
|
||||
// Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd.
|
||||
function getCompressionProgram(): string[] {
|
||||
switch (compressionMethod) {
|
||||
case CompressionMethod.Zstd:
|
||||
return [
|
||||
'--use-compress-program',
|
||||
IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30'
|
||||
]
|
||||
case CompressionMethod.ZstdWithoutLong:
|
||||
return ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt']
|
||||
default:
|
||||
return ['-z']
|
||||
const args = await getArgs(compressionMethod, 'create')
|
||||
try {
|
||||
await exec(args, undefined, {cwd: archiveFolder})
|
||||
} catch (error) {
|
||||
throw new Error(`Tar failed with error: ${error?.message}`)
|
||||
}
|
||||
}
|
||||
const args = [
|
||||
'--posix',
|
||||
...getCompressionProgram(),
|
||||
'-cf',
|
||||
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||
'--exclude',
|
||||
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||
'-P',
|
||||
'-C',
|
||||
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||
'--files-from',
|
||||
manifestFilename
|
||||
]
|
||||
await execTar(args, archiveFolder)
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue