1
0
Fork 0

Merge pull request #469 from actions/users/aiyan/zstd-fixes

Fix two issues related to using zstd compression
pull/471/head
Aiqiao Yan 2020-05-19 13:05:21 -04:00 committed by GitHub
commit dcf5c88bb3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 184 additions and 125 deletions

View File

@ -2,4 +2,7 @@
### 0.1.0 ### 0.1.0
- Initial release - Initial release
### 0.2.0
- Fixes two issues around using zstd compression algorithm

View File

@ -38,13 +38,11 @@ test('zstd extract tar', async () => {
? `${process.env['windir']}\\fakepath\\cache.tar` ? `${process.env['windir']}\\fakepath\\cache.tar`
: 'cache.tar' : 'cache.tar'
const workspace = process.env['GITHUB_WORKSPACE'] const workspace = process.env['GITHUB_WORKSPACE']
const tarPath = 'tar'
await tar.extractTar(archivePath, CompressionMethod.Zstd) await tar.extractTar(archivePath, CompressionMethod.Zstd)
expect(mkdirMock).toHaveBeenCalledWith(workspace) expect(mkdirMock).toHaveBeenCalledWith(workspace)
const tarPath = IS_WINDOWS
? `${process.env['windir']}\\System32\\tar.exe`
: 'tar'
expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith( expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`, `"${tarPath}"`,
@ -56,7 +54,7 @@ test('zstd extract tar', async () => {
'-P', '-P',
'-C', '-C',
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace
], ].concat(IS_WINDOWS ? ['--force-local'] : []),
{cwd: undefined} {cwd: undefined}
) )
}) })
@ -95,7 +93,7 @@ test('gzip extract GNU tar on windows', async () => {
jest.spyOn(fs, 'existsSync').mockReturnValueOnce(false) jest.spyOn(fs, 'existsSync').mockReturnValueOnce(false)
const isGnuMock = jest const isGnuMock = jest
.spyOn(utils, 'useGnuTar') .spyOn(utils, 'isGnuTarInstalled')
.mockReturnValue(Promise.resolve(true)) .mockReturnValue(Promise.resolve(true))
const execMock = jest.spyOn(exec, 'exec') const execMock = jest.spyOn(exec, 'exec')
const archivePath = `${process.env['windir']}\\fakepath\\cache.tar` const archivePath = `${process.env['windir']}\\fakepath\\cache.tar`
@ -127,15 +125,12 @@ test('zstd create tar', async () => {
const archiveFolder = getTempDir() const archiveFolder = getTempDir()
const workspace = process.env['GITHUB_WORKSPACE'] const workspace = process.env['GITHUB_WORKSPACE']
const sourceDirectories = ['~/.npm/cache', `${workspace}/dist`] const sourceDirectories = ['~/.npm/cache', `${workspace}/dist`]
const tarPath = 'tar'
await fs.promises.mkdir(archiveFolder, {recursive: true}) await fs.promises.mkdir(archiveFolder, {recursive: true})
await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Zstd) await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Zstd)
const tarPath = IS_WINDOWS
? `${process.env['windir']}\\System32\\tar.exe`
: 'tar'
expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith( expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`, `"${tarPath}"`,
@ -149,7 +144,7 @@ test('zstd create tar', async () => {
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace,
'--files-from', '--files-from',
'manifest.txt' 'manifest.txt'
], ].concat(IS_WINDOWS ? ['--force-local'] : []),
{ {
cwd: archiveFolder cwd: archiveFolder
} }

191
packages/cache/package-lock.json generated vendored
View File

@ -1,92 +1,103 @@
{ {
"name": "@actions/cache", "name": "@actions/cache",
"version": "0.1.0", "version": "0.2.0",
"lockfileVersion": 1, "lockfileVersion": 1,
"requires": true, "requires": true,
"dependencies": { "dependencies": {
"@actions/core": { "@actions/core": {
"version": "1.2.4", "version": "1.2.4",
"resolved": "https://registry.npmjs.org/@actions/core/-/core-1.2.4.tgz", "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.2.4.tgz",
"integrity": "sha512-YJCEq8BE3CdN8+7HPZ/4DxJjk/OkZV2FFIf+DlZTC/4iBlzYCD5yjRR6eiOS5llO11zbRltIRuKAjMKaWTE6cg==" "integrity": "sha512-YJCEq8BE3CdN8+7HPZ/4DxJjk/OkZV2FFIf+DlZTC/4iBlzYCD5yjRR6eiOS5llO11zbRltIRuKAjMKaWTE6cg=="
}, },
"@actions/exec": { "@actions/exec": {
"version": "1.0.4", "version": "1.0.4",
"resolved": "https://registry.npmjs.org/@actions/exec/-/exec-1.0.4.tgz", "resolved": "https://registry.npmjs.org/@actions/exec/-/exec-1.0.4.tgz",
"integrity": "sha512-4DPChWow9yc9W3WqEbUj8Nr86xkpyE29ZzWjXucHItclLbEW6jr80Zx4nqv18QL6KK65+cifiQZXvnqgTV6oHw==", "integrity": "sha512-4DPChWow9yc9W3WqEbUj8Nr86xkpyE29ZzWjXucHItclLbEW6jr80Zx4nqv18QL6KK65+cifiQZXvnqgTV6oHw==",
"requires": { "requires": {
"@actions/io": "^1.0.1" "@actions/io": "^1.0.1"
} }
}, },
"@actions/glob": { "@actions/glob": {
"version": "0.1.0", "version": "0.1.0",
"resolved": "https://registry.npmjs.org/@actions/glob/-/glob-0.1.0.tgz", "resolved": "https://registry.npmjs.org/@actions/glob/-/glob-0.1.0.tgz",
"integrity": "sha512-lx8SzyQ2FE9+UUvjqY1f28QbTJv+w8qP7kHHbfQRhphrlcx0Mdmm1tZdGJzfxv1jxREa/sLW4Oy8CbGQKCJySA==", "integrity": "sha512-lx8SzyQ2FE9+UUvjqY1f28QbTJv+w8qP7kHHbfQRhphrlcx0Mdmm1tZdGJzfxv1jxREa/sLW4Oy8CbGQKCJySA==",
"requires": { "requires": {
"@actions/core": "^1.2.0", "@actions/core": "^1.2.0",
"minimatch": "^3.0.4" "minimatch": "^3.0.4"
} }
}, },
"@actions/http-client": { "@actions/http-client": {
"version": "1.0.8", "version": "1.0.8",
"resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-1.0.8.tgz", "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-1.0.8.tgz",
"integrity": "sha512-G4JjJ6f9Hb3Zvejj+ewLLKLf99ZC+9v+yCxoYf9vSyH+WkzPLB2LuUtRMGNkooMqdugGBFStIKXOuvH1W+EctA==", "integrity": "sha512-G4JjJ6f9Hb3Zvejj+ewLLKLf99ZC+9v+yCxoYf9vSyH+WkzPLB2LuUtRMGNkooMqdugGBFStIKXOuvH1W+EctA==",
"requires": { "requires": {
"tunnel": "0.0.6" "tunnel": "0.0.6"
} }
}, },
"@actions/io": { "@actions/io": {
"version": "1.0.2", "version": "1.0.2",
"resolved": "https://registry.npmjs.org/@actions/io/-/io-1.0.2.tgz", "resolved": "https://registry.npmjs.org/@actions/io/-/io-1.0.2.tgz",
"integrity": "sha512-J8KuFqVPr3p6U8W93DOXlXW6zFvrQAJANdS+vw0YhusLIq+bszW8zmK2Fh1C2kDPX8FMvwIl1OUcFgvJoXLbAg==" "integrity": "sha512-J8KuFqVPr3p6U8W93DOXlXW6zFvrQAJANdS+vw0YhusLIq+bszW8zmK2Fh1C2kDPX8FMvwIl1OUcFgvJoXLbAg=="
}, },
"@types/uuid": { "@types/semver": {
"version": "3.4.9", "version": "6.2.1",
"resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-3.4.9.tgz", "resolved": "https://registry.npmjs.org/@types/semver/-/semver-6.2.1.tgz",
"integrity": "sha512-XDwyIlt/47l2kWLTzw/mtrpLdB+GPSskR2n/PIcPn+VYhVO77rGhRncIR5GPU0KRzXuqkDO+J5qqrG0Y8P6jzQ==", "integrity": "sha512-+beqKQOh9PYxuHvijhVl+tIHvT6tuwOrE9m14zd+MT2A38KoKZhh7pYJ0SNleLtwDsiIxHDsIk9bv01oOxvSvA==",
"dev": true "dev": true
}, },
"balanced-match": { "@types/uuid": {
"version": "1.0.0", "version": "3.4.9",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-3.4.9.tgz",
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" "integrity": "sha512-XDwyIlt/47l2kWLTzw/mtrpLdB+GPSskR2n/PIcPn+VYhVO77rGhRncIR5GPU0KRzXuqkDO+J5qqrG0Y8P6jzQ==",
}, "dev": true
"brace-expansion": { },
"version": "1.1.11", "balanced-match": {
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "version": "1.0.0",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
"requires": { "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
"balanced-match": "^1.0.0", },
"concat-map": "0.0.1" "brace-expansion": {
} "version": "1.1.11",
}, "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
"concat-map": { "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"version": "0.0.1", "requires": {
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "balanced-match": "^1.0.0",
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" "concat-map": "0.0.1"
}, }
"minimatch": { },
"version": "3.0.4", "concat-map": {
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", "version": "0.0.1",
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
"requires": { "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
"brace-expansion": "^1.1.7" },
} "minimatch": {
}, "version": "3.0.4",
"tunnel": { "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
"version": "0.0.6", "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
"resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", "requires": {
"integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==" "brace-expansion": "^1.1.7"
}, }
"typescript": { },
"version": "3.8.3", "semver": {
"resolved": "https://registry.npmjs.org/typescript/-/typescript-3.8.3.tgz", "version": "6.3.0",
"integrity": "sha512-MYlEfn5VrLNsgudQTVJeNaQFUAI7DkhnOjdpAp4T+ku1TfQClewlbSuTVHiA+8skNBgaf02TL/kLOvig4y3G8w==", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
"dev": true "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw=="
}, },
"uuid": { "tunnel": {
"version": "3.4.0", "version": "0.0.6",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz",
"integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg=="
} },
} "typescript": {
"version": "3.8.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-3.8.3.tgz",
"integrity": "sha512-MYlEfn5VrLNsgudQTVJeNaQFUAI7DkhnOjdpAp4T+ku1TfQClewlbSuTVHiA+8skNBgaf02TL/kLOvig4y3G8w==",
"dev": true
},
"uuid": {
"version": "3.4.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
"integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A=="
}
}
} }

View File

@ -1,6 +1,6 @@
{ {
"name": "@actions/cache", "name": "@actions/cache",
"version": "0.1.0", "version": "0.2.0",
"preview": true, "preview": true,
"description": "Actions cache lib", "description": "Actions cache lib",
"keywords": [ "keywords": [
@ -41,10 +41,12 @@
"@actions/glob": "^0.1.0", "@actions/glob": "^0.1.0",
"@actions/http-client": "^1.0.8", "@actions/http-client": "^1.0.8",
"@actions/io": "^1.0.1", "@actions/io": "^1.0.1",
"semver": "^6.1.0",
"uuid": "^3.3.3" "uuid": "^3.3.3"
}, },
"devDependencies": { "devDependencies": {
"typescript": "^3.8.3", "typescript": "^3.8.3",
"@types/semver": "^6.0.0",
"@types/uuid": "^3.4.5" "@types/uuid": "^3.4.5"
} }
} }

View File

@ -96,7 +96,9 @@ export function getCacheVersion(
compressionMethod?: CompressionMethod compressionMethod?: CompressionMethod
): string { ): string {
const components = paths.concat( const components = paths.concat(
compressionMethod === CompressionMethod.Zstd ? [compressionMethod] : [] !compressionMethod || compressionMethod === CompressionMethod.Gzip
? []
: [compressionMethod]
) )
// Add salt to cache version to support breaking changes in cache entry // Add salt to cache version to support breaking changes in cache entry

View File

@ -4,6 +4,7 @@ import * as glob from '@actions/glob'
import * as io from '@actions/io' import * as io from '@actions/io'
import * as fs from 'fs' import * as fs from 'fs'
import * as path from 'path' import * as path from 'path'
import * as semver from 'semver'
import * as util from 'util' import * as util from 'util'
import {v4 as uuidV4} from 'uuid' import {v4 as uuidV4} from 'uuid'
import {CacheFilename, CompressionMethod} from './constants' import {CacheFilename, CompressionMethod} from './constants'
@ -82,19 +83,33 @@ async function getVersion(app: string): Promise<string> {
// Use zstandard if possible to maximize cache performance // Use zstandard if possible to maximize cache performance
export async function getCompressionMethod(): Promise<CompressionMethod> { export async function getCompressionMethod(): Promise<CompressionMethod> {
if (process.platform === 'win32' && !isGnuTarInstalled()) {
// Disable zstd due to bug https://github.com/actions/cache/issues/301
return CompressionMethod.Gzip
}
const versionOutput = await getVersion('zstd') const versionOutput = await getVersion('zstd')
return versionOutput.toLowerCase().includes('zstd command line interface') const version = semver.clean(versionOutput)
? CompressionMethod.Zstd
: CompressionMethod.Gzip if (!versionOutput.toLowerCase().includes('zstd command line interface')) {
// zstd is not installed
return CompressionMethod.Gzip
} else if (!version || semver.lt(version, 'v1.3.2')) {
// zstd is installed but using a version earlier than v1.3.2
// v1.3.2 is required to use the `--long` options in zstd
return CompressionMethod.ZstdWithoutLong
} else {
return CompressionMethod.Zstd
}
} }
export function getCacheFileName(compressionMethod: CompressionMethod): string { export function getCacheFileName(compressionMethod: CompressionMethod): string {
return compressionMethod === CompressionMethod.Zstd return compressionMethod === CompressionMethod.Gzip
? CacheFilename.Zstd ? CacheFilename.Gzip
: CacheFilename.Gzip : CacheFilename.Zstd
} }
export async function useGnuTar(): Promise<boolean> { export async function isGnuTarInstalled(): Promise<boolean> {
const versionOutput = await getVersion('tar') const versionOutput = await getVersion('tar')
return versionOutput.toLowerCase().includes('gnu tar') return versionOutput.toLowerCase().includes('gnu tar')
} }

View File

@ -5,6 +5,9 @@ export enum CacheFilename {
export enum CompressionMethod { export enum CompressionMethod {
Gzip = 'gzip', Gzip = 'gzip',
// Long range mode was added to zstd in v1.3.2.
// This enum is for earlier version of zstd that does not have --long support
ZstdWithoutLong = 'zstd-without-long',
Zstd = 'zstd' Zstd = 'zstd'
} }

View File

@ -5,23 +5,33 @@ import * as path from 'path'
import * as utils from './cacheUtils' import * as utils from './cacheUtils'
import {CompressionMethod} from './constants' import {CompressionMethod} from './constants'
async function getTarPath(args: string[]): Promise<string> { async function getTarPath(
// Explicitly use BSD Tar on Windows args: string[],
compressionMethod: CompressionMethod
): Promise<string> {
const IS_WINDOWS = process.platform === 'win32' const IS_WINDOWS = process.platform === 'win32'
if (IS_WINDOWS) { if (IS_WINDOWS) {
const systemTar = `${process.env['windir']}\\System32\\tar.exe` const systemTar = `${process.env['windir']}\\System32\\tar.exe`
if (existsSync(systemTar)) { if (compressionMethod !== CompressionMethod.Gzip) {
// We only use zstandard compression on windows when gnu tar is installed due to
// a bug with compressing large files with bsdtar + zstd
args.push('--force-local')
} else if (existsSync(systemTar)) {
return systemTar return systemTar
} else if (await utils.useGnuTar()) { } else if (await utils.isGnuTarInstalled()) {
args.push('--force-local') args.push('--force-local')
} }
} }
return await io.which('tar', true) return await io.which('tar', true)
} }
async function execTar(args: string[], cwd?: string): Promise<void> { async function execTar(
args: string[],
compressionMethod: CompressionMethod,
cwd?: string
): Promise<void> {
try { try {
await exec(`"${await getTarPath(args)}"`, args, {cwd}) await exec(`"${await getTarPath(args, compressionMethod)}"`, args, {cwd})
} catch (error) { } catch (error) {
throw new Error(`Tar failed with error: ${error?.message}`) throw new Error(`Tar failed with error: ${error?.message}`)
} }
@ -41,17 +51,25 @@ export async function extractTar(
// --d: Decompress. // --d: Decompress.
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners. // Using 30 here because we also support 32-bit self-hosted runners.
function getCompressionProgram(): string[] {
switch (compressionMethod) {
case CompressionMethod.Zstd:
return ['--use-compress-program', 'zstd -d --long=30']
case CompressionMethod.ZstdWithoutLong:
return ['--use-compress-program', 'zstd -d']
default:
return ['-z']
}
}
const args = [ const args = [
...(compressionMethod === CompressionMethod.Zstd ...getCompressionProgram(),
? ['--use-compress-program', 'zstd -d --long=30']
: ['-z']),
'-xf', '-xf',
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P', '-P',
'-C', '-C',
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/') workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
] ]
await execTar(args) await execTar(args, compressionMethod)
} }
export async function createTar( export async function createTar(
@ -66,14 +84,24 @@ export async function createTar(
path.join(archiveFolder, manifestFilename), path.join(archiveFolder, manifestFilename),
sourceDirectories.join('\n') sourceDirectories.join('\n')
) )
const workingDirectory = getWorkingDirectory()
// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. // -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores.
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners. // Using 30 here because we also support 32-bit self-hosted runners.
const workingDirectory = getWorkingDirectory() // Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd.
function getCompressionProgram(): string[] {
switch (compressionMethod) {
case CompressionMethod.Zstd:
return ['--use-compress-program', 'zstd -T0 --long=30']
case CompressionMethod.ZstdWithoutLong:
return ['--use-compress-program', 'zstd -T0']
default:
return ['-z']
}
}
const args = [ const args = [
...(compressionMethod === CompressionMethod.Zstd ...getCompressionProgram(),
? ['--use-compress-program', 'zstd -T0 --long=30']
: ['-z']),
'-cf', '-cf',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P', '-P',
@ -82,5 +110,5 @@ export async function createTar(
'--files-from', '--files-from',
manifestFilename manifestFilename
] ]
await execTar(args, archiveFolder) await execTar(args, compressionMethod, archiveFolder)
} }