1
0
Fork 0

Fix issue with using zstd long range mode on ubuntu-16.04

pull/469/head
Aiqiao Yan 2020-05-18 16:33:15 -04:00
parent a67b91ea15
commit 77761a4dc9
6 changed files with 58 additions and 15 deletions

11
packages/cache/package-lock.json generated vendored
View File

@ -39,6 +39,12 @@
"resolved": "https://registry.npmjs.org/@actions/io/-/io-1.0.2.tgz", "resolved": "https://registry.npmjs.org/@actions/io/-/io-1.0.2.tgz",
"integrity": "sha512-J8KuFqVPr3p6U8W93DOXlXW6zFvrQAJANdS+vw0YhusLIq+bszW8zmK2Fh1C2kDPX8FMvwIl1OUcFgvJoXLbAg==" "integrity": "sha512-J8KuFqVPr3p6U8W93DOXlXW6zFvrQAJANdS+vw0YhusLIq+bszW8zmK2Fh1C2kDPX8FMvwIl1OUcFgvJoXLbAg=="
}, },
"@types/semver": {
"version": "6.2.1",
"resolved": "https://registry.npmjs.org/@types/semver/-/semver-6.2.1.tgz",
"integrity": "sha512-+beqKQOh9PYxuHvijhVl+tIHvT6tuwOrE9m14zd+MT2A38KoKZhh7pYJ0SNleLtwDsiIxHDsIk9bv01oOxvSvA==",
"dev": true
},
"@types/uuid": { "@types/uuid": {
"version": "3.4.9", "version": "3.4.9",
"resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-3.4.9.tgz", "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-3.4.9.tgz",
@ -72,6 +78,11 @@
"brace-expansion": "^1.1.7" "brace-expansion": "^1.1.7"
} }
}, },
"semver": {
"version": "6.3.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
"integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw=="
},
"tunnel": { "tunnel": {
"version": "0.0.6", "version": "0.0.6",
"resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz",

View File

@ -41,10 +41,12 @@
"@actions/glob": "^0.1.0", "@actions/glob": "^0.1.0",
"@actions/http-client": "^1.0.8", "@actions/http-client": "^1.0.8",
"@actions/io": "^1.0.1", "@actions/io": "^1.0.1",
"semver": "^6.1.0",
"uuid": "^3.3.3" "uuid": "^3.3.3"
}, },
"devDependencies": { "devDependencies": {
"typescript": "^3.8.3", "typescript": "^3.8.3",
"@types/semver": "^6.0.0",
"@types/uuid": "^3.4.5" "@types/uuid": "^3.4.5"
} }
} }

View File

@ -96,7 +96,9 @@ export function getCacheVersion(
compressionMethod?: CompressionMethod compressionMethod?: CompressionMethod
): string { ): string {
const components = paths.concat( const components = paths.concat(
compressionMethod === CompressionMethod.Zstd ? [compressionMethod] : [] !compressionMethod || compressionMethod === CompressionMethod.Gzip
? []
: [compressionMethod]
) )
// Add salt to cache version to support breaking changes in cache entry // Add salt to cache version to support breaking changes in cache entry

View File

@ -4,6 +4,7 @@ import * as glob from '@actions/glob'
import * as io from '@actions/io' import * as io from '@actions/io'
import * as fs from 'fs' import * as fs from 'fs'
import * as path from 'path' import * as path from 'path'
import * as semver from 'semver'
import * as util from 'util' import * as util from 'util'
import {v4 as uuidV4} from 'uuid' import {v4 as uuidV4} from 'uuid'
import {CacheFilename, CompressionMethod} from './constants' import {CacheFilename, CompressionMethod} from './constants'
@ -82,16 +83,24 @@ async function getVersion(app: string): Promise<string> {
// Use zstandard if possible to maximize cache performance // Use zstandard if possible to maximize cache performance
export async function getCompressionMethod(): Promise<CompressionMethod> { export async function getCompressionMethod(): Promise<CompressionMethod> {
const versionOutput = await getVersion('zstd') if (process.platform === 'win32') {
return versionOutput.toLowerCase().includes('zstd command line interface') // Disable zstd on windows due to bug https://github.com/actions/cache/issues/301
? CompressionMethod.Zstd return CompressionMethod.Gzip
: CompressionMethod.Gzip } else {
const versionOutput = await getVersion('zstd')
const version = semver.clean(versionOutput)
return !versionOutput.toLowerCase().includes('zstd command line interface')
? CompressionMethod.Gzip
: !version || semver.lt(version, 'v1.3.2')
? CompressionMethod.ZstdOld
: CompressionMethod.Zstd
}
} }
export function getCacheFileName(compressionMethod: CompressionMethod): string { export function getCacheFileName(compressionMethod: CompressionMethod): string {
return compressionMethod === CompressionMethod.Zstd return compressionMethod === CompressionMethod.Gzip
? CacheFilename.Zstd ? CacheFilename.Gzip
: CacheFilename.Gzip : CacheFilename.Zstd
} }
export async function useGnuTar(): Promise<boolean> { export async function useGnuTar(): Promise<boolean> {

View File

@ -5,6 +5,7 @@ export enum CacheFilename {
export enum CompressionMethod { export enum CompressionMethod {
Gzip = 'gzip', Gzip = 'gzip',
ZstdOld = 'zstd-old',
Zstd = 'zstd' Zstd = 'zstd'
} }

View File

@ -41,10 +41,18 @@ export async function extractTar(
// --d: Decompress. // --d: Decompress.
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners. // Using 30 here because we also support 32-bit self-hosted runners.
function getProg(): string[] {
switch (compressionMethod) {
case CompressionMethod.Zstd:
return ['--use-compress-program', 'zstd -d --long=30']
case CompressionMethod.ZstdOld:
return ['--use-compress-program', 'zstd -d']
default:
return ['-z']
}
}
const args = [ const args = [
...(compressionMethod === CompressionMethod.Zstd ...getProg(),
? ['--use-compress-program', 'zstd -d --long=30']
: ['-z']),
'-xf', '-xf',
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P', '-P',
@ -66,14 +74,24 @@ export async function createTar(
path.join(archiveFolder, manifestFilename), path.join(archiveFolder, manifestFilename),
sourceDirectories.join('\n') sourceDirectories.join('\n')
) )
const workingDirectory = getWorkingDirectory()
// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. // -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores.
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners. // Using 30 here because we also support 32-bit self-hosted runners.
const workingDirectory = getWorkingDirectory() // Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd.
function getProg(): string[] {
switch (compressionMethod) {
case CompressionMethod.Zstd:
return ['--use-compress-program', 'zstd -T0 --long=30']
case CompressionMethod.ZstdOld:
return ['--use-compress-program', 'zstd -T0']
default:
return ['-z']
}
}
const args = [ const args = [
...(compressionMethod === CompressionMethod.Zstd ...getProg(),
? ['--use-compress-program', 'zstd -T0 --long=30']
: ['-z']),
'-cf', '-cf',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P', '-P',