actions
/
toolkit
mirror of https://github.com/actions/toolkit
1
0
Fork 0
Code

Merge pull request #469 from actions/users/aiyan/zstd-fixes 

Fix two issues related to using zstd compression
pull/471/head
Aiqiao Yan 2020-05-19 13:05:21 -04:00 committed by GitHub
parent a67b91ea15 0dea61a3a8
commit dcf5c88bb3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 184 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
packages/cache/RELEASES.md vendored
View File

@ -2,4 +2,7 @@
### 0.1.0
- Initial release
- Initial release
### 0.2.0
- Fixes two issues around using zstd compression algorithm

15
packages/cache/__tests__/tar.test.ts vendored
View File

@ -38,13 +38,11 @@ test('zstd extract tar', async () => {
? `${process.env['windir']}\\fakepath\\cache.tar`
: 'cache.tar'
const workspace = process.env['GITHUB_WORKSPACE']
const tarPath = 'tar'
await tar.extractTar(archivePath, CompressionMethod.Zstd)
expect(mkdirMock).toHaveBeenCalledWith(workspace)
const tarPath = IS_WINDOWS
? `${process.env['windir']}\\System32\\tar.exe`
: 'tar'
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
@ -56,7 +54,7 @@ test('zstd extract tar', async () => {
'-P',
'-C',
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace
],
].concat(IS_WINDOWS ? ['--force-local'] : []),
{cwd: undefined}
)
})
@ -95,7 +93,7 @@ test('gzip extract GNU tar on windows', async () => {
jest.spyOn(fs, 'existsSync').mockReturnValueOnce(false)
const isGnuMock = jest
.spyOn(utils, 'useGnuTar')
.spyOn(utils, 'isGnuTarInstalled')
.mockReturnValue(Promise.resolve(true))
const execMock = jest.spyOn(exec, 'exec')
const archivePath = `${process.env['windir']}\\fakepath\\cache.tar`
@ -127,15 +125,12 @@ test('zstd create tar', async () => {
const archiveFolder = getTempDir()
const workspace = process.env['GITHUB_WORKSPACE']
const sourceDirectories = ['~/.npm/cache', `${workspace}/dist`]
const tarPath = 'tar'
await fs.promises.mkdir(archiveFolder, {recursive: true})
await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Zstd)
const tarPath = IS_WINDOWS
? `${process.env['windir']}\\System32\\tar.exe`
: 'tar'
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
@ -149,7 +144,7 @@ test('zstd create tar', async () => {
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace,
'--files-from',
'manifest.txt'
],
].concat(IS_WINDOWS ? ['--force-local'] : []),
{
cwd: archiveFolder
}

191
packages/cache/package-lock.json generated vendored
View File

@ -1,92 +1,103 @@
{
"name": "@actions/cache",
"version": "0.1.0",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
"@actions/core": {
"version": "1.2.4",
"resolved": "https://registry.npmjs.org/@actions/core/-/core-1.2.4.tgz",
"integrity": "sha512-YJCEq8BE3CdN8+7HPZ/4DxJjk/OkZV2FFIf+DlZTC/4iBlzYCD5yjRR6eiOS5llO11zbRltIRuKAjMKaWTE6cg=="
},
"@actions/exec": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/@actions/exec/-/exec-1.0.4.tgz",
"integrity": "sha512-4DPChWow9yc9W3WqEbUj8Nr86xkpyE29ZzWjXucHItclLbEW6jr80Zx4nqv18QL6KK65+cifiQZXvnqgTV6oHw==",
"requires": {
"@actions/io": "^1.0.1"
}
},
"@actions/glob": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/@actions/glob/-/glob-0.1.0.tgz",
"integrity": "sha512-lx8SzyQ2FE9+UUvjqY1f28QbTJv+w8qP7kHHbfQRhphrlcx0Mdmm1tZdGJzfxv1jxREa/sLW4Oy8CbGQKCJySA==",
"requires": {
"@actions/core": "^1.2.0",
"minimatch": "^3.0.4"
}
},
"@actions/http-client": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-1.0.8.tgz",
"integrity": "sha512-G4JjJ6f9Hb3Zvejj+ewLLKLf99ZC+9v+yCxoYf9vSyH+WkzPLB2LuUtRMGNkooMqdugGBFStIKXOuvH1W+EctA==",
"requires": {
"tunnel": "0.0.6"
}
},
"@actions/io": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/@actions/io/-/io-1.0.2.tgz",
"integrity": "sha512-J8KuFqVPr3p6U8W93DOXlXW6zFvrQAJANdS+vw0YhusLIq+bszW8zmK2Fh1C2kDPX8FMvwIl1OUcFgvJoXLbAg=="
},
"@types/uuid": {
"version": "3.4.9",
"resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-3.4.9.tgz",
"integrity": "sha512-XDwyIlt/47l2kWLTzw/mtrpLdB+GPSskR2n/PIcPn+VYhVO77rGhRncIR5GPU0KRzXuqkDO+J5qqrG0Y8P6jzQ==",
"dev": true
},
"balanced-match": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
},
"brace-expansion": {
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"requires": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
}
},
"concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
},
"minimatch": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
"requires": {
"brace-expansion": "^1.1.7"
}
},
"tunnel": {
"version": "0.0.6",
"resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz",
"integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg=="
},
"typescript": {
"version": "3.8.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-3.8.3.tgz",
"integrity": "sha512-MYlEfn5VrLNsgudQTVJeNaQFUAI7DkhnOjdpAp4T+ku1TfQClewlbSuTVHiA+8skNBgaf02TL/kLOvig4y3G8w==",
"dev": true
},
"uuid": {
"version": "3.4.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
"integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A=="
}
}
"name": "@actions/cache",
"version": "0.2.0",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
"@actions/core": {
"version": "1.2.4",
"resolved": "https://registry.npmjs.org/@actions/core/-/core-1.2.4.tgz",
"integrity": "sha512-YJCEq8BE3CdN8+7HPZ/4DxJjk/OkZV2FFIf+DlZTC/4iBlzYCD5yjRR6eiOS5llO11zbRltIRuKAjMKaWTE6cg=="
},
"@actions/exec": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/@actions/exec/-/exec-1.0.4.tgz",
"integrity": "sha512-4DPChWow9yc9W3WqEbUj8Nr86xkpyE29ZzWjXucHItclLbEW6jr80Zx4nqv18QL6KK65+cifiQZXvnqgTV6oHw==",
"requires": {
"@actions/io": "^1.0.1"
}
},
"@actions/glob": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/@actions/glob/-/glob-0.1.0.tgz",
"integrity": "sha512-lx8SzyQ2FE9+UUvjqY1f28QbTJv+w8qP7kHHbfQRhphrlcx0Mdmm1tZdGJzfxv1jxREa/sLW4Oy8CbGQKCJySA==",
"requires": {
"@actions/core": "^1.2.0",
"minimatch": "^3.0.4"
}
},
"@actions/http-client": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-1.0.8.tgz",
"integrity": "sha512-G4JjJ6f9Hb3Zvejj+ewLLKLf99ZC+9v+yCxoYf9vSyH+WkzPLB2LuUtRMGNkooMqdugGBFStIKXOuvH1W+EctA==",
"requires": {
"tunnel": "0.0.6"
}
},
"@actions/io": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/@actions/io/-/io-1.0.2.tgz",
"integrity": "sha512-J8KuFqVPr3p6U8W93DOXlXW6zFvrQAJANdS+vw0YhusLIq+bszW8zmK2Fh1C2kDPX8FMvwIl1OUcFgvJoXLbAg=="
},
"@types/semver": {
"version": "6.2.1",
"resolved": "https://registry.npmjs.org/@types/semver/-/semver-6.2.1.tgz",
"integrity": "sha512-+beqKQOh9PYxuHvijhVl+tIHvT6tuwOrE9m14zd+MT2A38KoKZhh7pYJ0SNleLtwDsiIxHDsIk9bv01oOxvSvA==",
"dev": true
},
"@types/uuid": {
"version": "3.4.9",
"resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-3.4.9.tgz",
"integrity": "sha512-XDwyIlt/47l2kWLTzw/mtrpLdB+GPSskR2n/PIcPn+VYhVO77rGhRncIR5GPU0KRzXuqkDO+J5qqrG0Y8P6jzQ==",
"dev": true
},
"balanced-match": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
},
"brace-expansion": {
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"requires": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
}
},
"concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
},
"minimatch": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
"requires": {
"brace-expansion": "^1.1.7"
}
},
"semver": {
"version": "6.3.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
"integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw=="
},
"tunnel": {
"version": "0.0.6",
"resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz",
"integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg=="
},
"typescript": {
"version": "3.8.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-3.8.3.tgz",
"integrity": "sha512-MYlEfn5VrLNsgudQTVJeNaQFUAI7DkhnOjdpAp4T+ku1TfQClewlbSuTVHiA+8skNBgaf02TL/kLOvig4y3G8w==",
"dev": true
},
"uuid": {
"version": "3.4.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
"integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A=="
}
}
}

4
packages/cache/package.json vendored
View File

@ -1,6 +1,6 @@
{
"name": "@actions/cache",
"version": "0.1.0",
"version": "0.2.0",
"preview": true,
"description": "Actions cache lib",
"keywords": [
@ -41,10 +41,12 @@
"@actions/glob": "^0.1.0",
"@actions/http-client": "^1.0.8",
"@actions/io": "^1.0.1",
"semver": "^6.1.0",
"uuid": "^3.3.3"
},
"devDependencies": {
"typescript": "^3.8.3",
"@types/semver": "^6.0.0",
"@types/uuid": "^3.4.5"
}
}

4
packages/cache/src/internal/cacheHttpClient.ts vendored
View File

@ -96,7 +96,9 @@ export function getCacheVersion(
compressionMethod?: CompressionMethod
): string {
const components = paths.concat(
compressionMethod === CompressionMethod.Zstd ? [compressionMethod] : []
!compressionMethod || compressionMethod === CompressionMethod.Gzip
? []
: [compressionMethod]
)
// Add salt to cache version to support breaking changes in cache entry

29
packages/cache/src/internal/cacheUtils.ts vendored
View File

@ -4,6 +4,7 @@ import * as glob from '@actions/glob'
import * as io from '@actions/io'
import * as fs from 'fs'
import * as path from 'path'
import * as semver from 'semver'
import * as util from 'util'
import {v4 as uuidV4} from 'uuid'
import {CacheFilename, CompressionMethod} from './constants'
@ -82,19 +83,33 @@ async function getVersion(app: string): Promise<string> {
// Use zstandard if possible to maximize cache performance
export async function getCompressionMethod(): Promise<CompressionMethod> {
if (process.platform === 'win32' && !isGnuTarInstalled()) {
// Disable zstd due to bug https://github.com/actions/cache/issues/301
return CompressionMethod.Gzip
}
const versionOutput = await getVersion('zstd')
return versionOutput.toLowerCase().includes('zstd command line interface')
? CompressionMethod.Zstd
: CompressionMethod.Gzip
const version = semver.clean(versionOutput)
if (!versionOutput.toLowerCase().includes('zstd command line interface')) {
// zstd is not installed
return CompressionMethod.Gzip
} else if (!version || semver.lt(version, 'v1.3.2')) {
// zstd is installed but using a version earlier than v1.3.2
// v1.3.2 is required to use the `--long` options in zstd
return CompressionMethod.ZstdWithoutLong
} else {
return CompressionMethod.Zstd
}
}
export function getCacheFileName(compressionMethod: CompressionMethod): string {
return compressionMethod === CompressionMethod.Zstd
? CacheFilename.Zstd
: CacheFilename.Gzip
return compressionMethod === CompressionMethod.Gzip
? CacheFilename.Gzip
: CacheFilename.Zstd
}
export async function useGnuTar(): Promise<boolean> {
export async function isGnuTarInstalled(): Promise<boolean> {
const versionOutput = await getVersion('tar')
return versionOutput.toLowerCase().includes('gnu tar')
}

3
packages/cache/src/internal/constants.ts vendored
View File

@ -5,6 +5,9 @@ export enum CacheFilename {
export enum CompressionMethod {
Gzip = 'gzip',
// Long range mode was added to zstd in v1.3.2.
// This enum is for earlier version of zstd that does not have --long support
ZstdWithoutLong = 'zstd-without-long',
Zstd = 'zstd'
}

58
packages/cache/src/internal/tar.ts vendored
View File

@ -5,23 +5,33 @@ import * as path from 'path'
import * as utils from './cacheUtils'
import {CompressionMethod} from './constants'
async function getTarPath(args: string[]): Promise<string> {
// Explicitly use BSD Tar on Windows
async function getTarPath(
args: string[],
compressionMethod: CompressionMethod
): Promise<string> {
const IS_WINDOWS = process.platform === 'win32'
if (IS_WINDOWS) {
const systemTar = `${process.env['windir']}\\System32\\tar.exe`
if (existsSync(systemTar)) {
if (compressionMethod !== CompressionMethod.Gzip) {
// We only use zstandard compression on windows when gnu tar is installed due to
// a bug with compressing large files with bsdtar + zstd
args.push('--force-local')
} else if (existsSync(systemTar)) {
return systemTar
} else if (await utils.useGnuTar()) {
} else if (await utils.isGnuTarInstalled()) {
args.push('--force-local')
}
}
return await io.which('tar', true)
}
async function execTar(args: string[], cwd?: string): Promise<void> {
async function execTar(
args: string[],
compressionMethod: CompressionMethod,
cwd?: string
): Promise<void> {
try {
await exec(`"${await getTarPath(args)}"`, args, {cwd})
await exec(`"${await getTarPath(args, compressionMethod)}"`, args, {cwd})
} catch (error) {
throw new Error(`Tar failed with error: ${error?.message}`)
}
@ -41,17 +51,25 @@ export async function extractTar(
// --d: Decompress.
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners.
function getCompressionProgram(): string[] {
switch (compressionMethod) {
case CompressionMethod.Zstd:
return ['--use-compress-program', 'zstd -d --long=30']
case CompressionMethod.ZstdWithoutLong:
return ['--use-compress-program', 'zstd -d']
default:
return ['-z']
}
}
const args = [
...(compressionMethod === CompressionMethod.Zstd
? ['--use-compress-program', 'zstd -d --long=30']
: ['-z']),
...getCompressionProgram(),
'-xf',
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
]
await execTar(args)
await execTar(args, compressionMethod)
}
export async function createTar(
@ -66,14 +84,24 @@ export async function createTar(
path.join(archiveFolder, manifestFilename),
sourceDirectories.join('\n')
)
const workingDirectory = getWorkingDirectory()
// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores.
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners.
const workingDirectory = getWorkingDirectory()
// Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd.
function getCompressionProgram(): string[] {
switch (compressionMethod) {
case CompressionMethod.Zstd:
return ['--use-compress-program', 'zstd -T0 --long=30']
case CompressionMethod.ZstdWithoutLong:
return ['--use-compress-program', 'zstd -T0']
default:
return ['-z']
}
}
const args = [
...(compressionMethod === CompressionMethod.Zstd
? ['--use-compress-program', 'zstd -T0 --long=30']
: ['-z']),
...getCompressionProgram(),
'-cf',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
@ -82,5 +110,5 @@ export async function createTar(
'--files-from',
manifestFilename
]
await execTar(args, archiveFolder)
await execTar(args, compressionMethod, archiveFolder)
}