diff --git a/packages/cache/src/internal/constants.ts b/packages/cache/src/internal/constants.ts index c6d8a490..7d0eb65d 100644 --- a/packages/cache/src/internal/constants.ts +++ b/packages/cache/src/internal/constants.ts @@ -11,6 +11,11 @@ export enum CompressionMethod { Zstd = 'zstd' } +export enum ArchiveToolType { + GNU = 'gnu', + BSD = 'bsd' +} + // The default number of retry attempts. export const DefaultRetryAttempts = 2 diff --git a/packages/cache/src/internal/contracts.d.ts b/packages/cache/src/internal/contracts.d.ts index 1b2a13a1..4aa48791 100644 --- a/packages/cache/src/internal/contracts.d.ts +++ b/packages/cache/src/internal/contracts.d.ts @@ -31,3 +31,8 @@ export interface InternalCacheOptions { compressionMethod?: CompressionMethod cacheSize?: number } + +export interface ArchiveTool { + path: string + type: string +} \ No newline at end of file diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 041f2b5e..00b14fda 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -3,21 +3,26 @@ import * as io from '@actions/io' import {existsSync, writeFileSync} from 'fs' import * as path from 'path' import * as utils from './cacheUtils' -import {CompressionMethod, SystemTarPathOnWindows} from './constants' +import {ArchiveTool} from './contracts' +import { + CompressionMethod, + SystemTarPathOnWindows, + ArchiveToolType +} from './constants' const IS_WINDOWS = process.platform === 'win32' // Function also mutates the args array. For non-mutation call with passing an empty array. -async function getTarPath(): Promise { +async function getTarPath(): Promise { switch (process.platform) { case 'win32': { const gnuTar = await utils.getGnuTarPathOnWindows() const systemTar = SystemTarPathOnWindows if (gnuTar) { // Use GNUtar as default on windows - return gnuTar + return {path: gnuTar, type: ArchiveToolType.GNU} } else if (existsSync(systemTar)) { - return systemTar + return {path: systemTar, type: ArchiveToolType.BSD} } break } @@ -25,30 +30,39 @@ async function getTarPath(): Promise { const gnuTar = await io.which('gtar', false) if (gnuTar) { // fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527 - return gnuTar + return {path: gnuTar, type: ArchiveToolType.GNU} + } else { + return { + path: await io.which('tar', true), + type: ArchiveToolType.BSD + } } - break } default: break } - return await io.which('tar', true) + return { + path: await io.which('tar', true), + type: ArchiveToolType.GNU + } } +// Return arguments for tar as per tarPath, compressionMethod, method type and os async function getTarArgs( + tarPath: ArchiveTool, compressionMethod: CompressionMethod, type: string, archivePath = '' ): Promise { - const args = [] + const args = [tarPath.path] const manifestFilename = 'manifest.txt' const cacheFileName = utils.getCacheFileName(compressionMethod) const tarFile = 'cache.tar' - const tarPath = await getTarPath() const workingDirectory = getWorkingDirectory() const BSD_TAR_ZSTD = - tarPath === SystemTarPathOnWindows && - compressionMethod !== CompressionMethod.Gzip + tarPath.type === ArchiveToolType.BSD && + compressionMethod !== CompressionMethod.Gzip && + IS_WINDOWS // Method specific args switch (type) { @@ -93,45 +107,44 @@ async function getTarArgs( } // Platform specific args - switch (process.platform) { - case 'win32': { - const gnuTar = await utils.getGnuTarPathOnWindows() - if (gnuTar) { - // Use GNUtar as default on windows + if (tarPath.type === ArchiveToolType.GNU) { + switch (process.platform) { + case 'win32': args.push('--force-local') - } - break - } - case 'darwin': { - const gnuTar = await io.which('gtar', false) - if (gnuTar) { - // fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527 + break + case 'darwin': args.push('--delay-directory-restore') - } - break + break } } return args } -async function execTar(args: string[], cwd?: string): Promise { - try { - await exec(`"${await getTarPath()}"`, args, {cwd}) - } catch (error) { - throw new Error(`Tar failed with error: ${error?.message}`) - } -} - -async function execCommand( - command: string, - args: string[], - cwd?: string -): Promise { - try { - await exec(command, args, {cwd}) - } catch (error) { - throw new Error(`Tar failed with error: ${error?.message}`) +async function getArgs( + compressionMethod: CompressionMethod, + type: string, + archivePath = '' +): Promise { + const tarPath = await getTarPath() + const tarArgs = await getTarArgs( + tarPath, + compressionMethod, + type, + archivePath + ) + const compressionArgs = + type !== 'create' + ? await getDecompressionProgram(tarPath, compressionMethod, archivePath) + : await getCompressionProgram(tarPath, compressionMethod) + const BSD_TAR_ZSTD = + tarPath.type === ArchiveToolType.BSD && + compressionMethod !== CompressionMethod.Gzip && + IS_WINDOWS + if (BSD_TAR_ZSTD && type !== 'create') { + return [...compressionArgs, ...tarArgs].join(' ') + } else { + return [...tarArgs, ...compressionArgs].join(' ') } } @@ -140,7 +153,8 @@ function getWorkingDirectory(): string { } // Common function for extractTar and listTar to get the compression method -async function getCompressionProgram( +async function getDecompressionProgram( + tarPath: ArchiveTool, compressionMethod: CompressionMethod, archivePath: string ): Promise { @@ -148,11 +162,11 @@ async function getCompressionProgram( // unzstd is equivalent to 'zstd -d' // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // Using 30 here because we also support 32-bit self-hosted runners. - const tarPath = await getTarPath() const tarFile = 'cache.tar' const BSD_TAR_ZSTD = - tarPath === SystemTarPathOnWindows && - compressionMethod !== CompressionMethod.Gzip + tarPath.type === ArchiveToolType.BSD && + compressionMethod !== CompressionMethod.Gzip && + IS_WINDOWS switch (compressionMethod) { case CompressionMethod.Zstd: return BSD_TAR_ZSTD @@ -180,31 +194,54 @@ async function getCompressionProgram( } } +// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. +// zstdmt is equivalent to 'zstd -T0' +// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. +// Using 30 here because we also support 32-bit self-hosted runners. +// Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd. +async function getCompressionProgram( + tarPath: ArchiveTool, + compressionMethod: CompressionMethod +): Promise { + const cacheFileName = utils.getCacheFileName(compressionMethod) + const tarFile = 'cache.tar' + const BSD_TAR_ZSTD = + tarPath.type === ArchiveToolType.BSD && + compressionMethod !== CompressionMethod.Gzip && + IS_WINDOWS + switch (compressionMethod) { + case CompressionMethod.Zstd: + return BSD_TAR_ZSTD + ? [ + '&&', + 'zstd -T0 --long=30 -o', + cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + tarFile + ] + : [ + '--use-compress-program', + IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30' + ] + case CompressionMethod.ZstdWithoutLong: + return BSD_TAR_ZSTD + ? [ + '&&', + 'zstd -T0 -o', + cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + tarFile + ] + : ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt'] + default: + return ['-z'] + } +} + export async function listTar( archivePath: string, compressionMethod: CompressionMethod ): Promise { - const tarPath = await getTarPath() - const BSD_TAR_ZSTD = - tarPath === SystemTarPathOnWindows && - compressionMethod !== CompressionMethod.Gzip - const compressionArgs = await getCompressionProgram( - compressionMethod, - archivePath - ) - const tarArgs = await getTarArgs(compressionMethod, 'list', archivePath) - // TODO: Add a test for BSD tar on windows - if (BSD_TAR_ZSTD) { - const command = compressionArgs[0] - const args = compressionArgs - .slice(1) - .concat([tarPath]) - .concat(tarArgs) - await execCommand(command, args) - } else { - const args = tarArgs.concat(compressionArgs) - await execTar(args) - } + const args = await getArgs(compressionMethod, 'list', archivePath) + exec(args) } export async function extractTar( @@ -213,27 +250,9 @@ export async function extractTar( ): Promise { // Create directory to extract tar into const workingDirectory = getWorkingDirectory() - const tarPath = await getTarPath() - const BSD_TAR_ZSTD = - tarPath === SystemTarPathOnWindows && - compressionMethod !== CompressionMethod.Gzip await io.mkdirP(workingDirectory) - const tarArgs = await getTarArgs(compressionMethod, 'extract', archivePath) - const compressionArgs = await getCompressionProgram( - compressionMethod, - archivePath - ) - if (BSD_TAR_ZSTD) { - const command = compressionArgs[0] - const args = compressionArgs - .slice(1) - .concat([tarPath]) - .concat(tarArgs) - await execCommand(command, args) - } else { - const args = tarArgs.concat(compressionArgs) - await execTar(args) - } + const args = await getArgs(compressionMethod, 'extract', archivePath) + exec(args) } export async function createTar( @@ -243,51 +262,10 @@ export async function createTar( ): Promise { // Write source directories to manifest.txt to avoid command length limits const manifestFilename = 'manifest.txt' - const cacheFileName = utils.getCacheFileName(compressionMethod) - const tarFile = 'cache.tar' - const tarPath = await getTarPath() - const BSD_TAR_ZSTD = - tarPath === SystemTarPathOnWindows && - compressionMethod !== CompressionMethod.Gzip writeFileSync( path.join(archiveFolder, manifestFilename), sourceDirectories.join('\n') ) - - // -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. - // zstdmt is equivalent to 'zstd -T0' - // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. - // Using 30 here because we also support 32-bit self-hosted runners. - // Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd. - function getCompressionProgram(): string[] { - switch (compressionMethod) { - case CompressionMethod.Zstd: - return BSD_TAR_ZSTD - ? [ - '&&', - 'zstd -T0 --long=30 -o', - cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - tarFile - ] - : [ - '--use-compress-program', - IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30' - ] - case CompressionMethod.ZstdWithoutLong: - return BSD_TAR_ZSTD - ? [ - '&&', - 'zstd -T0 -o', - cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - tarFile - ] - : ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt'] - default: - return ['-z'] - } - } - const tarArgs = await getTarArgs(compressionMethod, 'create') - const compressionArgs = getCompressionProgram() - const args = tarArgs.concat(compressionArgs) - await execTar(args, archiveFolder) + const args = await getArgs(compressionMethod, 'create') + await exec(args) }