actions
/
toolkit
mirror of https://github.com/actions/toolkit
1
0
Fork 0
Code

Merge pull request #1152 from actions/pdotl-zstd-win-patch 

Fix zstd failing on windows when used with the gnu tar workaround
pull/1163/merge^2
Lovepreet Singh 2022-08-18 19:01:00 +05:30 committed by GitHub
parent a57a4fe011 0982f1da89
commit bc4be50597
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 104 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
packages/cache/RELEASES.md vendored
View File

@ -83,4 +83,5 @@
- Bug fixes for download stuck issue [#810](https://github.com/actions/cache/issues/810). - Bug fixes for download stuck issue [#810](https://github.com/actions/cache/issues/810).
### 3.0.4 ### 3.0.4
- Allowing users to provide a custom timeout as input for aborting download of a cache segment using an environment variable `SEGMENT_DOWNLOAD_TIMEOUT_MIN`. Default is 60 minutes. - Fix zstd not working for windows on gnu tar in issues [#888](https://github.com/actions/cache/issues/888) and [#891](https://github.com/actions/cache/issues/891).
- Allowing users to provide a custom timeout as input for aborting download of a cache segment using an environment variable `SEGMENT_DOWNLOAD_TIMEOUT_MIN`. Default is 60 minutes.

8
packages/cache/__tests__/tar.test.ts vendored
View File

@ -50,7 +50,7 @@ test('zstd extract tar', async () => {
`"${defaultTarPath}"`, `"${defaultTarPath}"`,
[ [
'--use-compress-program', '--use-compress-program',
'unzstd --long=30', IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30',
'-xf', '-xf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P', '-P',
@ -140,7 +140,7 @@ test('zstd create tar', async () => {
[ [
'--posix', '--posix',
'--use-compress-program', '--use-compress-program',
'zstdmt --long=30', IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30',
'-cf', '-cf',
IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd, IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd,
'--exclude', '--exclude',
@ -210,7 +210,7 @@ test('zstd list tar', async () => {
`"${defaultTarPath}"`, `"${defaultTarPath}"`,
[ [
'--use-compress-program', '--use-compress-program',
'unzstd --long=30', IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30',
'-tf', '-tf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P' '-P'
@ -235,7 +235,7 @@ test('zstdWithoutLong list tar', async () => {
`"${defaultTarPath}"`, `"${defaultTarPath}"`,
[ [
'--use-compress-program', '--use-compress-program',
'unzstd', IS_WINDOWS ? 'zstd -d' : 'unzstd',
'-tf', '-tf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P' '-P'

84
packages/cache/src/internal/tar.ts vendored
View File

@ -5,6 +5,8 @@ import * as path from 'path'
import * as utils from './cacheUtils' import * as utils from './cacheUtils'
import {CompressionMethod} from './constants' import {CompressionMethod} from './constants'
const IS_WINDOWS = process.platform === 'win32'
async function getTarPath( async function getTarPath(
args: string[], args: string[],
compressionMethod: CompressionMethod compressionMethod: CompressionMethod
@ -54,6 +56,38 @@ function getWorkingDirectory(): string {
return process.env['GITHUB_WORKSPACE'] ?? process.cwd() return process.env['GITHUB_WORKSPACE'] ?? process.cwd()
} }
// Common function for extractTar and listTar to get the compression method
function getCompressionProgram(compressionMethod: CompressionMethod): string[] {
// -d: Decompress.
// unzstd is equivalent to 'zstd -d'
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners.
switch (compressionMethod) {
case CompressionMethod.Zstd:
return [
'--use-compress-program',
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30'
]
case CompressionMethod.ZstdWithoutLong:
return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd']
default:
return ['-z']
}
}
export async function listTar(
archivePath: string,
compressionMethod: CompressionMethod
): Promise<void> {
const args = [
...getCompressionProgram(compressionMethod),
'-tf',
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P'
]
await execTar(args, compressionMethod)
}
export async function extractTar( export async function extractTar(
archivePath: string, archivePath: string,
compressionMethod: CompressionMethod compressionMethod: CompressionMethod
@ -61,21 +95,8 @@ export async function extractTar(
// Create directory to extract tar into // Create directory to extract tar into
const workingDirectory = getWorkingDirectory() const workingDirectory = getWorkingDirectory()
await io.mkdirP(workingDirectory) await io.mkdirP(workingDirectory)
// --d: Decompress.
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners.
function getCompressionProgram(): string[] {
switch (compressionMethod) {
case CompressionMethod.Zstd:
return ['--use-compress-program', 'unzstd --long=30']
case CompressionMethod.ZstdWithoutLong:
return ['--use-compress-program', 'unzstd']
default:
return ['-z']
}
}
const args = [ const args = [
...getCompressionProgram(), ...getCompressionProgram(compressionMethod),
'-xf', '-xf',
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P', '-P',
@ -100,15 +121,19 @@ export async function createTar(
const workingDirectory = getWorkingDirectory() const workingDirectory = getWorkingDirectory()
// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. // -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores.
// zstdmt is equivalent to 'zstd -T0'
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners. // Using 30 here because we also support 32-bit self-hosted runners.
// Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd. // Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd.
function getCompressionProgram(): string[] { function getCompressionProgram(): string[] {
switch (compressionMethod) { switch (compressionMethod) {
case CompressionMethod.Zstd: case CompressionMethod.Zstd:
return ['--use-compress-program', 'zstdmt --long=30'] return [
'--use-compress-program',
IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30'
]
case CompressionMethod.ZstdWithoutLong: case CompressionMethod.ZstdWithoutLong:
return ['--use-compress-program', 'zstdmt'] return ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt']
default: default:
return ['-z'] return ['-z']
} }
@ -128,30 +153,3 @@ export async function createTar(
] ]
await execTar(args, compressionMethod, archiveFolder) await execTar(args, compressionMethod, archiveFolder)
} }
export async function listTar(
archivePath: string,
compressionMethod: CompressionMethod
): Promise<void> {
// --d: Decompress.
// --long=#: Enables long distance matching with # bits.
// Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners.
function getCompressionProgram(): string[] {
switch (compressionMethod) {
case CompressionMethod.Zstd:
return ['--use-compress-program', 'unzstd --long=30']
case CompressionMethod.ZstdWithoutLong:
return ['--use-compress-program', 'unzstd']
default:
return ['-z']
}
}
const args = [
...getCompressionProgram(),
'-tf',
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P'
]
await execTar(args, compressionMethod)
}

144
packages/core/package-lock.json generated
View File

@ -1,89 +1,59 @@
{ {
"name": "@actions/core", "name": "@actions/core",
"version": "1.9.1", "version": "1.9.1",
"lockfileVersion": 2, "lockfileVersion": 2,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "@actions/core", "name": "@actions/core",
"version": "1.9.1", "version": "1.9.1",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"@actions/http-client": "^2.0.1", "uuid": "^8.3.2"
"uuid": "^8.3.2" },
}, "devDependencies": {
"devDependencies": { "@types/node": "^12.0.2",
"@types/node": "^12.0.2", "@types/uuid": "^8.3.4"
"@types/uuid": "^8.3.4" }
} },
}, "node_modules/@types/node": {
"node_modules/@actions/http-client": { "version": "12.0.2",
"version": "2.0.1", "resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.2.tgz",
"resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.0.1.tgz", "integrity": "sha512-5tabW/i+9mhrfEOUcLDu2xBPsHJ+X5Orqy9FKpale3SjDA17j5AEpYq5vfy3oAeAHGcvANRCO3NV3d2D6q3NiA==",
"integrity": "sha512-PIXiMVtz6VvyaRsGY268qvj57hXQEpsYogYOu2nrQhlf+XCGmZstmuZBbAybUl1nQGnvS1k1eEsQ69ZoD7xlSw==", "dev": true
"dependencies": { },
"tunnel": "^0.0.6" "node_modules/@types/uuid": {
} "version": "8.3.4",
}, "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.4.tgz",
"node_modules/@types/node": { "integrity": "sha512-c/I8ZRb51j+pYGAu5CrFMRxqZ2ke4y2grEBO5AUjgSkSk+qT2Ea+OdWElz/OiMf5MNpn2b17kuVBwZLQJXzihw==",
"version": "12.0.2", "dev": true
"resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.2.tgz", },
"integrity": "sha512-5tabW/i+9mhrfEOUcLDu2xBPsHJ+X5Orqy9FKpale3SjDA17j5AEpYq5vfy3oAeAHGcvANRCO3NV3d2D6q3NiA==", "node_modules/uuid": {
"dev": true "version": "8.3.2",
}, "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
"node_modules/@types/uuid": { "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
"version": "8.3.4", "bin": {
"resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.4.tgz", "uuid": "dist/bin/uuid"
"integrity": "sha512-c/I8ZRb51j+pYGAu5CrFMRxqZ2ke4y2grEBO5AUjgSkSk+qT2Ea+OdWElz/OiMf5MNpn2b17kuVBwZLQJXzihw==", }
"dev": true }
}, },
"node_modules/tunnel": { "dependencies": {
"version": "0.0.6", "@types/node": {
"resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", "version": "12.0.2",
"integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==", "resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.2.tgz",
"engines": { "integrity": "sha512-5tabW/i+9mhrfEOUcLDu2xBPsHJ+X5Orqy9FKpale3SjDA17j5AEpYq5vfy3oAeAHGcvANRCO3NV3d2D6q3NiA==",
"node": ">=0.6.11 <=0.7.0 || >=0.7.3" "dev": true
} },
}, "@types/uuid": {
"node_modules/uuid": { "version": "8.3.4",
"version": "8.3.2", "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.4.tgz",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", "integrity": "sha512-c/I8ZRb51j+pYGAu5CrFMRxqZ2ke4y2grEBO5AUjgSkSk+qT2Ea+OdWElz/OiMf5MNpn2b17kuVBwZLQJXzihw==",
"integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", "dev": true
"bin": { },
"uuid": "dist/bin/uuid" "uuid": {
} "version": "8.3.2",
} "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
}, "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg=="
"dependencies": { }
"@actions/http-client": { }
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.0.1.tgz",
"integrity": "sha512-PIXiMVtz6VvyaRsGY268qvj57hXQEpsYogYOu2nrQhlf+XCGmZstmuZBbAybUl1nQGnvS1k1eEsQ69ZoD7xlSw==",
"requires": {
"tunnel": "^0.0.6"
}
},
"@types/node": {
"version": "12.0.2",
"resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.2.tgz",
"integrity": "sha512-5tabW/i+9mhrfEOUcLDu2xBPsHJ+X5Orqy9FKpale3SjDA17j5AEpYq5vfy3oAeAHGcvANRCO3NV3d2D6q3NiA==",
"dev": true
},
"@types/uuid": {
"version": "8.3.4",
"resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.4.tgz",
"integrity": "sha512-c/I8ZRb51j+pYGAu5CrFMRxqZ2ke4y2grEBO5AUjgSkSk+qT2Ea+OdWElz/OiMf5MNpn2b17kuVBwZLQJXzihw==",
"dev": true
},
"tunnel": {
"version": "0.0.6",
"resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz",
"integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg=="
},
"uuid": {
"version": "8.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
"integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg=="
}
}
} }