From 77761a4dc9dcbf58536d67169004a2ad8647a2c3 Mon Sep 17 00:00:00 2001 From: Aiqiao Yan Date: Mon, 18 May 2020 16:33:15 -0400 Subject: [PATCH] Fix issue with using zstd long range mode on ubuntu-16.04 --- packages/cache/package-lock.json | 11 +++++++ packages/cache/package.json | 2 ++ .../cache/src/internal/cacheHttpClient.ts | 4 ++- packages/cache/src/internal/cacheUtils.ts | 23 +++++++++---- packages/cache/src/internal/constants.ts | 1 + packages/cache/src/internal/tar.ts | 32 +++++++++++++++---- 6 files changed, 58 insertions(+), 15 deletions(-) diff --git a/packages/cache/package-lock.json b/packages/cache/package-lock.json index 08f18a7c..a6a72274 100644 --- a/packages/cache/package-lock.json +++ b/packages/cache/package-lock.json @@ -39,6 +39,12 @@ "resolved": "https://registry.npmjs.org/@actions/io/-/io-1.0.2.tgz", "integrity": "sha512-J8KuFqVPr3p6U8W93DOXlXW6zFvrQAJANdS+vw0YhusLIq+bszW8zmK2Fh1C2kDPX8FMvwIl1OUcFgvJoXLbAg==" }, + "@types/semver": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-6.2.1.tgz", + "integrity": "sha512-+beqKQOh9PYxuHvijhVl+tIHvT6tuwOrE9m14zd+MT2A38KoKZhh7pYJ0SNleLtwDsiIxHDsIk9bv01oOxvSvA==", + "dev": true + }, "@types/uuid": { "version": "3.4.9", "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-3.4.9.tgz", @@ -72,6 +78,11 @@ "brace-expansion": "^1.1.7" } }, + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + }, "tunnel": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", diff --git a/packages/cache/package.json b/packages/cache/package.json index 69e93181..25be002e 100644 --- a/packages/cache/package.json +++ b/packages/cache/package.json @@ -41,10 +41,12 @@ "@actions/glob": "^0.1.0", "@actions/http-client": "^1.0.8", "@actions/io": "^1.0.1", + "semver": "^6.1.0", "uuid": "^3.3.3" }, "devDependencies": { "typescript": "^3.8.3", + "@types/semver": "^6.0.0", "@types/uuid": "^3.4.5" } } diff --git a/packages/cache/src/internal/cacheHttpClient.ts b/packages/cache/src/internal/cacheHttpClient.ts index b7d34448..21af8031 100644 --- a/packages/cache/src/internal/cacheHttpClient.ts +++ b/packages/cache/src/internal/cacheHttpClient.ts @@ -96,7 +96,9 @@ export function getCacheVersion( compressionMethod?: CompressionMethod ): string { const components = paths.concat( - compressionMethod === CompressionMethod.Zstd ? [compressionMethod] : [] + !compressionMethod || compressionMethod === CompressionMethod.Gzip + ? [] + : [compressionMethod] ) // Add salt to cache version to support breaking changes in cache entry diff --git a/packages/cache/src/internal/cacheUtils.ts b/packages/cache/src/internal/cacheUtils.ts index f3f85006..bd698286 100644 --- a/packages/cache/src/internal/cacheUtils.ts +++ b/packages/cache/src/internal/cacheUtils.ts @@ -4,6 +4,7 @@ import * as glob from '@actions/glob' import * as io from '@actions/io' import * as fs from 'fs' import * as path from 'path' +import * as semver from 'semver' import * as util from 'util' import {v4 as uuidV4} from 'uuid' import {CacheFilename, CompressionMethod} from './constants' @@ -82,16 +83,24 @@ async function getVersion(app: string): Promise { // Use zstandard if possible to maximize cache performance export async function getCompressionMethod(): Promise { - const versionOutput = await getVersion('zstd') - return versionOutput.toLowerCase().includes('zstd command line interface') - ? CompressionMethod.Zstd - : CompressionMethod.Gzip + if (process.platform === 'win32') { + // Disable zstd on windows due to bug https://github.com/actions/cache/issues/301 + return CompressionMethod.Gzip + } else { + const versionOutput = await getVersion('zstd') + const version = semver.clean(versionOutput) + return !versionOutput.toLowerCase().includes('zstd command line interface') + ? CompressionMethod.Gzip + : !version || semver.lt(version, 'v1.3.2') + ? CompressionMethod.ZstdOld + : CompressionMethod.Zstd + } } export function getCacheFileName(compressionMethod: CompressionMethod): string { - return compressionMethod === CompressionMethod.Zstd - ? CacheFilename.Zstd - : CacheFilename.Gzip + return compressionMethod === CompressionMethod.Gzip + ? CacheFilename.Gzip + : CacheFilename.Zstd } export async function useGnuTar(): Promise { diff --git a/packages/cache/src/internal/constants.ts b/packages/cache/src/internal/constants.ts index b3d2a577..ec1d4944 100644 --- a/packages/cache/src/internal/constants.ts +++ b/packages/cache/src/internal/constants.ts @@ -5,6 +5,7 @@ export enum CacheFilename { export enum CompressionMethod { Gzip = 'gzip', + ZstdOld = 'zstd-old', Zstd = 'zstd' } diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 221c7c70..27fef40f 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -41,10 +41,18 @@ export async function extractTar( // --d: Decompress. // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // Using 30 here because we also support 32-bit self-hosted runners. + function getProg(): string[] { + switch (compressionMethod) { + case CompressionMethod.Zstd: + return ['--use-compress-program', 'zstd -d --long=30'] + case CompressionMethod.ZstdOld: + return ['--use-compress-program', 'zstd -d'] + default: + return ['-z'] + } + } const args = [ - ...(compressionMethod === CompressionMethod.Zstd - ? ['--use-compress-program', 'zstd -d --long=30'] - : ['-z']), + ...getProg(), '-xf', archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P', @@ -66,14 +74,24 @@ export async function createTar( path.join(archiveFolder, manifestFilename), sourceDirectories.join('\n') ) + const workingDirectory = getWorkingDirectory() + // -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // Using 30 here because we also support 32-bit self-hosted runners. - const workingDirectory = getWorkingDirectory() + // Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd. + function getProg(): string[] { + switch (compressionMethod) { + case CompressionMethod.Zstd: + return ['--use-compress-program', 'zstd -T0 --long=30'] + case CompressionMethod.ZstdOld: + return ['--use-compress-program', 'zstd -T0'] + default: + return ['-z'] + } + } const args = [ - ...(compressionMethod === CompressionMethod.Zstd - ? ['--use-compress-program', 'zstd -T0 --long=30'] - : ['-z']), + ...getProg(), '-cf', cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P',