Merge pull request #1152 from actions/pdotl-zstd-win-patch

Fix zstd failing on windows when used with the gnu tar workaround

packages/cache/RELEASES.md

@@ -83,4 +83,5 @@
 - Bug fixes for download stuck issue [#810](https://github.com/actions/cache/issues/810).
 
 ### 3.0.4
+- Fix zstd not working for windows on gnu tar in issues [#888](https://github.com/actions/cache/issues/888) and [#891](https://github.com/actions/cache/issues/891).
 - Allowing users to provide a custom timeout as input for aborting download of a cache segment using an environment variable `SEGMENT_DOWNLOAD_TIMEOUT_MIN`. Default is 60 minutes.

packages/cache/__tests__/tar.test.ts

@@ -50,7 +50,7 @@ test('zstd extract tar', async () => {
     `"${defaultTarPath}"`,
     [
       '--use-compress-program',
-      'unzstd --long=30',
+      IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30',
       '-xf',
       IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
       '-P',
@@ -140,7 +140,7 @@ test('zstd create tar', async () => {
     [
       '--posix',
       '--use-compress-program',
-      'zstdmt --long=30',
+      IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30',
       '-cf',
       IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd,
       '--exclude',
@@ -210,7 +210,7 @@ test('zstd list tar', async () => {
     `"${defaultTarPath}"`,
     [
       '--use-compress-program',
-      'unzstd --long=30',
+      IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30',
       '-tf',
       IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
       '-P'
@@ -235,7 +235,7 @@ test('zstdWithoutLong list tar', async () => {
     `"${defaultTarPath}"`,
     [
       '--use-compress-program',
-      'unzstd',
+      IS_WINDOWS ? 'zstd -d' : 'unzstd',
       '-tf',
       IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
       '-P'

packages/cache/src/internal/tar.ts

@@ -5,6 +5,8 @@ import * as path from 'path'
 import * as utils from './cacheUtils'
 import {CompressionMethod} from './constants'
 
+const IS_WINDOWS = process.platform === 'win32'
+
 async function getTarPath(
   args: string[],
   compressionMethod: CompressionMethod
@@ -54,6 +56,38 @@ function getWorkingDirectory(): string {
   return process.env['GITHUB_WORKSPACE'] ?? process.cwd()
 }
 
+// Common function for extractTar and listTar to get the compression method
+function getCompressionProgram(compressionMethod: CompressionMethod): string[] {
+  // -d: Decompress.
+  // unzstd is equivalent to 'zstd -d'
+  // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
+  // Using 30 here because we also support 32-bit self-hosted runners.
+  switch (compressionMethod) {
+    case CompressionMethod.Zstd:
+      return [
+        '--use-compress-program',
+        IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30'
+      ]
+    case CompressionMethod.ZstdWithoutLong:
+      return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd']
+    default:
+      return ['-z']
+  }
+}
+
+export async function listTar(
+  archivePath: string,
+  compressionMethod: CompressionMethod
+): Promise<void> {
+  const args = [
+    ...getCompressionProgram(compressionMethod),
+    '-tf',
+    archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
+    '-P'
+  ]
+  await execTar(args, compressionMethod)
+}
+
 export async function extractTar(
   archivePath: string,
   compressionMethod: CompressionMethod
@@ -61,21 +95,8 @@ export async function extractTar(
   // Create directory to extract tar into
   const workingDirectory = getWorkingDirectory()
   await io.mkdirP(workingDirectory)
-  // --d: Decompress.
-  // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
-  // Using 30 here because we also support 32-bit self-hosted runners.
-  function getCompressionProgram(): string[] {
-    switch (compressionMethod) {
-      case CompressionMethod.Zstd:
-        return ['--use-compress-program', 'unzstd --long=30']
-      case CompressionMethod.ZstdWithoutLong:
-        return ['--use-compress-program', 'unzstd']
-      default:
-        return ['-z']
-    }
-  }
   const args = [
-    ...getCompressionProgram(),
+    ...getCompressionProgram(compressionMethod),
     '-xf',
     archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
     '-P',
@@ -100,15 +121,19 @@ export async function createTar(
   const workingDirectory = getWorkingDirectory()
 
   // -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores.
+  // zstdmt is equivalent to 'zstd -T0'
   // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
   // Using 30 here because we also support 32-bit self-hosted runners.
   // Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd.
   function getCompressionProgram(): string[] {
     switch (compressionMethod) {
       case CompressionMethod.Zstd:
-        return ['--use-compress-program', 'zstdmt --long=30']
+        return [
+          '--use-compress-program',
+          IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30'
+        ]
       case CompressionMethod.ZstdWithoutLong:
-        return ['--use-compress-program', 'zstdmt']
+        return ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt']
       default:
         return ['-z']
     }
@@ -128,30 +153,3 @@ export async function createTar(
   ]
   await execTar(args, compressionMethod, archiveFolder)
 }
-
-export async function listTar(
-  archivePath: string,
-  compressionMethod: CompressionMethod
-): Promise<void> {
-  // --d: Decompress.
-  // --long=#: Enables long distance matching with # bits.
-  // Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
-  // Using 30 here because we also support 32-bit self-hosted runners.
-  function getCompressionProgram(): string[] {
-    switch (compressionMethod) {
-      case CompressionMethod.Zstd:
-        return ['--use-compress-program', 'unzstd --long=30']
-      case CompressionMethod.ZstdWithoutLong:
-        return ['--use-compress-program', 'unzstd']
-      default:
-        return ['-z']
-    }
-  }
-  const args = [
-    ...getCompressionProgram(),
-    '-tf',
-    archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
-    '-P'
-  ]
-  await execTar(args, compressionMethod)
-}

packages/core/package-lock.json

@@ -9,7 +9,6 @@
 			"version": "1.9.1",
 			"license": "MIT",
 			"dependencies": {
-        "@actions/http-client": "^2.0.1",
 				"uuid": "^8.3.2"
 			},
 			"devDependencies": {
@@ -17,14 +16,6 @@
 				"@types/uuid": "^8.3.4"
 			}
 		},
-    "node_modules/@actions/http-client": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.0.1.tgz",
-      "integrity": "sha512-PIXiMVtz6VvyaRsGY268qvj57hXQEpsYogYOu2nrQhlf+XCGmZstmuZBbAybUl1nQGnvS1k1eEsQ69ZoD7xlSw==",
-      "dependencies": {
-        "tunnel": "^0.0.6"
-      }
-    },
 		"node_modules/@types/node": {
 			"version": "12.0.2",
 			"resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.2.tgz",
@@ -37,14 +28,6 @@
 			"integrity": "sha512-c/I8ZRb51j+pYGAu5CrFMRxqZ2ke4y2grEBO5AUjgSkSk+qT2Ea+OdWElz/OiMf5MNpn2b17kuVBwZLQJXzihw==",
 			"dev": true
 		},
-    "node_modules/tunnel": {
-      "version": "0.0.6",
-      "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz",
-      "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==",
-      "engines": {
-        "node": ">=0.6.11 <=0.7.0 || >=0.7.3"
-      }
-    },
 		"node_modules/uuid": {
 			"version": "8.3.2",
 			"resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
@@ -55,14 +38,6 @@
 		}
 	},
 	"dependencies": {
-    "@actions/http-client": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.0.1.tgz",
-      "integrity": "sha512-PIXiMVtz6VvyaRsGY268qvj57hXQEpsYogYOu2nrQhlf+XCGmZstmuZBbAybUl1nQGnvS1k1eEsQ69ZoD7xlSw==",
-      "requires": {
-        "tunnel": "^0.0.6"
-      }
-    },
 		"@types/node": {
 			"version": "12.0.2",
 			"resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.2.tgz",
@@ -75,11 +50,6 @@
 			"integrity": "sha512-c/I8ZRb51j+pYGAu5CrFMRxqZ2ke4y2grEBO5AUjgSkSk+qT2Ea+OdWElz/OiMf5MNpn2b17kuVBwZLQJXzihw==",
 			"dev": true
 		},
-    "tunnel": {
-      "version": "0.0.6",
-      "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz",
-      "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg=="
-    },
 		"uuid": {
 			"version": "8.3.2",
 			"resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",