Merge pull request #469 from actions/users/aiyan/zstd-fixes

Fix two issues related to using zstd compression

packages/cache/RELEASES.md

@@ -3,3 +3,6 @@
 ### 0.1.0
 
 - Initial release
+
+### 0.2.0
+- Fixes two issues around using zstd compression algorithm

packages/cache/__tests__/tar.test.ts

@@ -38,13 +38,11 @@ test('zstd extract tar', async () => {
     ? `${process.env['windir']}\\fakepath\\cache.tar`
     : 'cache.tar'
   const workspace = process.env['GITHUB_WORKSPACE']
+  const tarPath = 'tar'
 
   await tar.extractTar(archivePath, CompressionMethod.Zstd)
 
   expect(mkdirMock).toHaveBeenCalledWith(workspace)
-  const tarPath = IS_WINDOWS
-    ? `${process.env['windir']}\\System32\\tar.exe`
-    : 'tar'
   expect(execMock).toHaveBeenCalledTimes(1)
   expect(execMock).toHaveBeenCalledWith(
     `"${tarPath}"`,
@@ -56,7 +54,7 @@ test('zstd extract tar', async () => {
       '-P',
       '-C',
       IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace
-    ],
+    ].concat(IS_WINDOWS ? ['--force-local'] : []),
     {cwd: undefined}
   )
 })
@@ -95,7 +93,7 @@ test('gzip extract GNU tar on windows', async () => {
     jest.spyOn(fs, 'existsSync').mockReturnValueOnce(false)
 
     const isGnuMock = jest
-      .spyOn(utils, 'useGnuTar')
+      .spyOn(utils, 'isGnuTarInstalled')
       .mockReturnValue(Promise.resolve(true))
     const execMock = jest.spyOn(exec, 'exec')
     const archivePath = `${process.env['windir']}\\fakepath\\cache.tar`
@@ -127,15 +125,12 @@ test('zstd create tar', async () => {
   const archiveFolder = getTempDir()
   const workspace = process.env['GITHUB_WORKSPACE']
   const sourceDirectories = ['~/.npm/cache', `${workspace}/dist`]
+  const tarPath = 'tar'
 
   await fs.promises.mkdir(archiveFolder, {recursive: true})
 
   await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Zstd)
 
-  const tarPath = IS_WINDOWS
-    ? `${process.env['windir']}\\System32\\tar.exe`
-    : 'tar'
-
   expect(execMock).toHaveBeenCalledTimes(1)
   expect(execMock).toHaveBeenCalledWith(
     `"${tarPath}"`,
@@ -149,7 +144,7 @@ test('zstd create tar', async () => {
       IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace,
       '--files-from',
       'manifest.txt'
-    ],
+    ].concat(IS_WINDOWS ? ['--force-local'] : []),
     {
       cwd: archiveFolder
     }

packages/cache/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@actions/cache",
-	"version": "0.1.0",
+  "version": "0.2.0",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -39,6 +39,12 @@
       "resolved": "https://registry.npmjs.org/@actions/io/-/io-1.0.2.tgz",
       "integrity": "sha512-J8KuFqVPr3p6U8W93DOXlXW6zFvrQAJANdS+vw0YhusLIq+bszW8zmK2Fh1C2kDPX8FMvwIl1OUcFgvJoXLbAg=="
     },
+    "@types/semver": {
+      "version": "6.2.1",
+      "resolved": "https://registry.npmjs.org/@types/semver/-/semver-6.2.1.tgz",
+      "integrity": "sha512-+beqKQOh9PYxuHvijhVl+tIHvT6tuwOrE9m14zd+MT2A38KoKZhh7pYJ0SNleLtwDsiIxHDsIk9bv01oOxvSvA==",
+      "dev": true
+    },
     "@types/uuid": {
       "version": "3.4.9",
       "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-3.4.9.tgz",
@@ -72,6 +78,11 @@
         "brace-expansion": "^1.1.7"
       }
     },
+    "semver": {
+      "version": "6.3.0",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
+      "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw=="
+    },
     "tunnel": {
       "version": "0.0.6",
       "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz",

packages/cache/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@actions/cache",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "preview": true,
   "description": "Actions cache lib",
   "keywords": [
@@ -41,10 +41,12 @@
     "@actions/glob": "^0.1.0",
     "@actions/http-client": "^1.0.8",
     "@actions/io": "^1.0.1",
+    "semver": "^6.1.0",
     "uuid": "^3.3.3"
   },
   "devDependencies": {
     "typescript": "^3.8.3",
+    "@types/semver": "^6.0.0",
     "@types/uuid": "^3.4.5"
   }
 }

packages/cache/src/internal/cacheHttpClient.ts

@@ -96,7 +96,9 @@ export function getCacheVersion(
   compressionMethod?: CompressionMethod
 ): string {
   const components = paths.concat(
-    compressionMethod === CompressionMethod.Zstd ? [compressionMethod] : []
+    !compressionMethod || compressionMethod === CompressionMethod.Gzip
+      ? []
+      : [compressionMethod]
   )
 
   // Add salt to cache version to support breaking changes in cache entry

packages/cache/src/internal/cacheUtils.ts

@@ -4,6 +4,7 @@ import * as glob from '@actions/glob'
 import * as io from '@actions/io'
 import * as fs from 'fs'
 import * as path from 'path'
+import * as semver from 'semver'
 import * as util from 'util'
 import {v4 as uuidV4} from 'uuid'
 import {CacheFilename, CompressionMethod} from './constants'
@@ -82,19 +83,33 @@ async function getVersion(app: string): Promise<string> {
 
 // Use zstandard if possible to maximize cache performance
 export async function getCompressionMethod(): Promise<CompressionMethod> {
+  if (process.platform === 'win32' && !isGnuTarInstalled()) {
+    // Disable zstd due to bug https://github.com/actions/cache/issues/301
+    return CompressionMethod.Gzip
+  }
+
   const versionOutput = await getVersion('zstd')
-  return versionOutput.toLowerCase().includes('zstd command line interface')
+  const version = semver.clean(versionOutput)
-    ? CompressionMethod.Zstd
+
-    : CompressionMethod.Gzip
+  if (!versionOutput.toLowerCase().includes('zstd command line interface')) {
+    // zstd is not installed
+    return CompressionMethod.Gzip
+  } else if (!version || semver.lt(version, 'v1.3.2')) {
+    // zstd is installed but using a version earlier than v1.3.2
+    // v1.3.2 is required to use the `--long` options in zstd
+    return CompressionMethod.ZstdWithoutLong
+  } else {
+    return CompressionMethod.Zstd
+  }
 }
 
 export function getCacheFileName(compressionMethod: CompressionMethod): string {
-  return compressionMethod === CompressionMethod.Zstd
+  return compressionMethod === CompressionMethod.Gzip
-    ? CacheFilename.Zstd
+    ? CacheFilename.Gzip
-    : CacheFilename.Gzip
+    : CacheFilename.Zstd
 }
 
-export async function useGnuTar(): Promise<boolean> {
+export async function isGnuTarInstalled(): Promise<boolean> {
   const versionOutput = await getVersion('tar')
   return versionOutput.toLowerCase().includes('gnu tar')
 }

packages/cache/src/internal/constants.ts

@@ -5,6 +5,9 @@ export enum CacheFilename {
 
 export enum CompressionMethod {
   Gzip = 'gzip',
+  // Long range mode was added to zstd in v1.3.2.
+  // This enum is for earlier version of zstd that does not have --long support
+  ZstdWithoutLong = 'zstd-without-long',
   Zstd = 'zstd'
 }
 

packages/cache/src/internal/tar.ts

@@ -5,23 +5,33 @@ import * as path from 'path'
 import * as utils from './cacheUtils'
 import {CompressionMethod} from './constants'
 
-async function getTarPath(args: string[]): Promise<string> {
+async function getTarPath(
-  // Explicitly use BSD Tar on Windows
+  args: string[],
+  compressionMethod: CompressionMethod
+): Promise<string> {
   const IS_WINDOWS = process.platform === 'win32'
   if (IS_WINDOWS) {
     const systemTar = `${process.env['windir']}\\System32\\tar.exe`
-    if (existsSync(systemTar)) {
+    if (compressionMethod !== CompressionMethod.Gzip) {
+      // We only use zstandard compression on windows when gnu tar is installed due to
+      // a bug with compressing large files with bsdtar + zstd
+      args.push('--force-local')
+    } else if (existsSync(systemTar)) {
       return systemTar
-    } else if (await utils.useGnuTar()) {
+    } else if (await utils.isGnuTarInstalled()) {
       args.push('--force-local')
     }
   }
   return await io.which('tar', true)
 }
 
-async function execTar(args: string[], cwd?: string): Promise<void> {
+async function execTar(
+  args: string[],
+  compressionMethod: CompressionMethod,
+  cwd?: string
+): Promise<void> {
   try {
-    await exec(`"${await getTarPath(args)}"`, args, {cwd})
+    await exec(`"${await getTarPath(args, compressionMethod)}"`, args, {cwd})
   } catch (error) {
     throw new Error(`Tar failed with error: ${error?.message}`)
   }
@@ -41,17 +51,25 @@ export async function extractTar(
   // --d: Decompress.
   // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
   // Using 30 here because we also support 32-bit self-hosted runners.
+  function getCompressionProgram(): string[] {
+    switch (compressionMethod) {
+      case CompressionMethod.Zstd:
+        return ['--use-compress-program', 'zstd -d --long=30']
+      case CompressionMethod.ZstdWithoutLong:
+        return ['--use-compress-program', 'zstd -d']
+      default:
+        return ['-z']
+    }
+  }
   const args = [
-    ...(compressionMethod === CompressionMethod.Zstd
+    ...getCompressionProgram(),
-      ? ['--use-compress-program', 'zstd -d --long=30']
-      : ['-z']),
     '-xf',
     archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
     '-P',
     '-C',
     workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
   ]
-  await execTar(args)
+  await execTar(args, compressionMethod)
 }
 
 export async function createTar(
@@ -66,14 +84,24 @@ export async function createTar(
     path.join(archiveFolder, manifestFilename),
     sourceDirectories.join('\n')
   )
+  const workingDirectory = getWorkingDirectory()
+
   // -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores.
   // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
   // Using 30 here because we also support 32-bit self-hosted runners.
-  const workingDirectory = getWorkingDirectory()
+  // Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd.
+  function getCompressionProgram(): string[] {
+    switch (compressionMethod) {
+      case CompressionMethod.Zstd:
+        return ['--use-compress-program', 'zstd -T0 --long=30']
+      case CompressionMethod.ZstdWithoutLong:
+        return ['--use-compress-program', 'zstd -T0']
+      default:
+        return ['-z']
+    }
+  }
   const args = [
-    ...(compressionMethod === CompressionMethod.Zstd
+    ...getCompressionProgram(),
-      ? ['--use-compress-program', 'zstd -T0 --long=30']
-      : ['-z']),
     '-cf',
     cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
     '-P',
@@ -82,5 +110,5 @@ export async function createTar(
     '--files-from',
     manifestFilename
   ]
-  await execTar(args, archiveFolder)
+  await execTar(args, compressionMethod, archiveFolder)
 }