1
0
Fork 0
toolkit/packages/tool-cache/src/tool-cache.ts

778 lines
22 KiB
TypeScript

import * as core from '@actions/core'
import * as io from '@actions/io'
import * as fs from 'fs'
import * as mm from './manifest'
import * as os from 'os'
import * as path from 'path'
import * as httpm from '@actions/http-client'
import * as semver from 'semver'
import * as stream from 'stream'
import * as util from 'util'
import {ok} from 'assert'
import {OutgoingHttpHeaders} from 'http'
import uuidV4 from 'uuid/v4'
import {exec} from '@actions/exec/lib/exec'
import {ExecOptions} from '@actions/exec/lib/interfaces'
import {RetryHelper} from './retry-helper'
export class HTTPError extends Error {
constructor(readonly httpStatusCode: number | undefined) {
super(`Unexpected HTTP response: ${httpStatusCode}`)
Object.setPrototypeOf(this, new.target.prototype)
}
}
const IS_WINDOWS = process.platform === 'win32'
const IS_MAC = process.platform === 'darwin'
const userAgent = 'actions/tool-cache'
/**
* Download a tool from an url and stream it into a file
*
* @param url url of tool to download
* @param dest path to download tool
* @param auth authorization header
* @param headers other headers
* @returns path to downloaded tool
*/
export async function downloadTool(
url: string,
dest?: string,
auth?: string,
headers?: OutgoingHttpHeaders
): Promise<string> {
dest = dest || path.join(_getTempDirectory(), uuidV4())
await io.mkdirP(path.dirname(dest))
core.debug(`Downloading ${url}`)
core.debug(`Destination ${dest}`)
const maxAttempts = 3
const minSeconds = _getGlobal<number>(
'TEST_DOWNLOAD_TOOL_RETRY_MIN_SECONDS',
10
)
const maxSeconds = _getGlobal<number>(
'TEST_DOWNLOAD_TOOL_RETRY_MAX_SECONDS',
20
)
const retryHelper = new RetryHelper(maxAttempts, minSeconds, maxSeconds)
return await retryHelper.execute(
async () => {
return await downloadToolAttempt(url, dest || '', auth, headers)
},
(err: Error) => {
if (err instanceof HTTPError && err.httpStatusCode) {
// Don't retry anything less than 500, except 408 Request Timeout and 429 Too Many Requests
if (
err.httpStatusCode < 500 &&
err.httpStatusCode !== 408 &&
err.httpStatusCode !== 429
) {
return false
}
}
// Otherwise retry
return true
}
)
}
async function downloadToolAttempt(
url: string,
dest: string,
auth?: string,
headers?: OutgoingHttpHeaders
): Promise<string> {
if (fs.existsSync(dest)) {
throw new Error(`Destination file path ${dest} already exists`)
}
// Get the response headers
const http = new httpm.HttpClient(userAgent, [], {
allowRetries: false
})
if (auth) {
core.debug('set auth')
if (headers === undefined) {
headers = {}
}
headers.authorization = auth
}
const response: httpm.HttpClientResponse = await http.get(url, headers)
if (response.message.statusCode !== 200) {
const err = new HTTPError(response.message.statusCode)
core.debug(
`Failed to download from "${url}". Code(${response.message.statusCode}) Message(${response.message.statusMessage})`
)
throw err
}
// Download the response body
const pipeline = util.promisify(stream.pipeline)
const responseMessageFactory = _getGlobal<() => stream.Readable>(
'TEST_DOWNLOAD_TOOL_RESPONSE_MESSAGE_FACTORY',
() => response.message
)
const readStream = responseMessageFactory()
let succeeded = false
try {
await pipeline(readStream, fs.createWriteStream(dest))
core.debug('download complete')
succeeded = true
return dest
} finally {
// Error, delete dest before retry
if (!succeeded) {
core.debug('download failed')
try {
await io.rmRF(dest)
} catch (err) {
core.debug(`Failed to delete '${dest}'. ${err.message}`)
}
}
}
}
/**
* Extract a .7z file
*
* @param file path to the .7z file
* @param dest destination directory. Optional.
* @param _7zPath path to 7zr.exe. Optional, for long path support. Most .7z archives do not have this
* problem. If your .7z archive contains very long paths, you can pass the path to 7zr.exe which will
* gracefully handle long paths. By default 7zdec.exe is used because it is a very small program and is
* bundled with the tool lib. However it does not support long paths. 7zr.exe is the reduced command line
* interface, it is smaller than the full command line interface, and it does support long paths. At the
* time of this writing, it is freely available from the LZMA SDK that is available on the 7zip website.
* Be sure to check the current license agreement. If 7zr.exe is bundled with your action, then the path
* to 7zr.exe can be pass to this function.
* @returns path to the destination directory
*/
export async function extract7z(
file: string,
dest?: string,
_7zPath?: string
): Promise<string> {
ok(IS_WINDOWS, 'extract7z() not supported on current OS')
ok(file, 'parameter "file" is required')
dest = await _createExtractFolder(dest)
const originalCwd = process.cwd()
process.chdir(dest)
if (_7zPath) {
try {
const logLevel = core.isDebug() ? '-bb1' : '-bb0'
const args: string[] = [
'x', // eXtract files with full paths
logLevel, // -bb[0-3] : set output log level
'-bd', // disable progress indicator
'-sccUTF-8', // set charset for for console input/output
file
]
const options: ExecOptions = {
silent: true
}
await exec(`"${_7zPath}"`, args, options)
} finally {
process.chdir(originalCwd)
}
} else {
const escapedScript = path
.join(__dirname, '..', 'scripts', 'Invoke-7zdec.ps1')
.replace(/'/g, "''")
.replace(/"|\n|\r/g, '') // double-up single quotes, remove double quotes and newlines
const escapedFile = file.replace(/'/g, "''").replace(/"|\n|\r/g, '')
const escapedTarget = dest.replace(/'/g, "''").replace(/"|\n|\r/g, '')
const command = `& '${escapedScript}' -Source '${escapedFile}' -Target '${escapedTarget}'`
const args: string[] = [
'-NoLogo',
'-Sta',
'-NoProfile',
'-NonInteractive',
'-ExecutionPolicy',
'Unrestricted',
'-Command',
command
]
const options: ExecOptions = {
silent: true
}
try {
const powershellPath: string = await io.which('powershell', true)
await exec(`"${powershellPath}"`, args, options)
} finally {
process.chdir(originalCwd)
}
}
return dest
}
/**
* Extract a compressed tar archive
*
* @param file path to the tar
* @param dest destination directory. Optional.
* @param flags flags for the tar command to use for extraction. Defaults to 'xz' (extracting gzipped tars). Optional.
* @returns path to the destination directory
*/
export async function extractTar(
file: string,
dest?: string,
flags: string | string[] = 'xz'
): Promise<string> {
if (!file) {
throw new Error("parameter 'file' is required")
}
// Create dest
dest = await _createExtractFolder(dest)
// Determine whether GNU tar
core.debug('Checking tar --version')
let versionOutput = ''
await exec('tar --version', [], {
ignoreReturnCode: true,
silent: true,
listeners: {
stdout: (data: Buffer) => (versionOutput += data.toString()),
stderr: (data: Buffer) => (versionOutput += data.toString())
}
})
core.debug(versionOutput.trim())
const isGnuTar = versionOutput.toUpperCase().includes('GNU TAR')
// Initialize args
let args: string[]
if (flags instanceof Array) {
args = flags
} else {
args = [flags]
}
if (core.isDebug() && !flags.includes('v')) {
args.push('-v')
}
let destArg = dest
let fileArg = file
if (IS_WINDOWS && isGnuTar) {
args.push('--force-local')
destArg = dest.replace(/\\/g, '/')
// Technically only the dest needs to have `/` but for aesthetic consistency
// convert slashes in the file arg too.
fileArg = file.replace(/\\/g, '/')
}
if (isGnuTar) {
// Suppress warnings when using GNU tar to extract archives created by BSD tar
args.push('--warning=no-unknown-keyword')
args.push('--overwrite')
}
args.push('-C', destArg, '-f', fileArg)
await exec(`tar`, args)
return dest
}
/**
* Extract a xar compatible archive
*
* @param file path to the archive
* @param dest destination directory. Optional.
* @param flags flags for the xar. Optional.
* @returns path to the destination directory
*/
export async function extractXar(
file: string,
dest?: string,
flags: string | string[] = []
): Promise<string> {
ok(IS_MAC, 'extractXar() not supported on current OS')
ok(file, 'parameter "file" is required')
dest = await _createExtractFolder(dest)
let args: string[]
if (flags instanceof Array) {
args = flags
} else {
args = [flags]
}
args.push('-x', '-C', dest, '-f', file)
if (core.isDebug()) {
args.push('-v')
}
const xarPath: string = await io.which('xar', true)
await exec(`"${xarPath}"`, _unique(args))
return dest
}
/**
* Extract a zip
*
* @param file path to the zip
* @param dest destination directory. Optional.
* @returns path to the destination directory
*/
export async function extractZip(file: string, dest?: string): Promise<string> {
if (!file) {
throw new Error("parameter 'file' is required")
}
dest = await _createExtractFolder(dest)
if (IS_WINDOWS) {
await extractZipWin(file, dest)
} else {
await extractZipNix(file, dest)
}
return dest
}
async function extractZipWin(file: string, dest: string): Promise<void> {
// build the powershell command
const escapedFile = file.replace(/'/g, "''").replace(/"|\n|\r/g, '') // double-up single quotes, remove double quotes and newlines
const escapedDest = dest.replace(/'/g, "''").replace(/"|\n|\r/g, '')
const pwshPath = await io.which('pwsh', false)
//To match the file overwrite behavior on nix systems, we use the overwrite = true flag for ExtractToDirectory
//and the -Force flag for Expand-Archive as a fallback
if (pwshPath) {
//attempt to use pwsh with ExtractToDirectory, if this fails attempt Expand-Archive
const pwshCommand = [
`$ErrorActionPreference = 'Stop' ;`,
`try { Add-Type -AssemblyName System.IO.Compression.ZipFile } catch { } ;`,
`try { [System.IO.Compression.ZipFile]::ExtractToDirectory('${escapedFile}', '${escapedDest}', $true) }`,
`catch { if (($_.Exception.GetType().FullName -eq 'System.Management.Automation.MethodException') -or ($_.Exception.GetType().FullName -eq 'System.Management.Automation.RuntimeException') ){ Expand-Archive -LiteralPath '${escapedFile}' -DestinationPath '${escapedDest}' -Force } else { throw $_ } } ;`
].join(' ')
const args = [
'-NoLogo',
'-NoProfile',
'-NonInteractive',
'-ExecutionPolicy',
'Unrestricted',
'-Command',
pwshCommand
]
core.debug(`Using pwsh at path: ${pwshPath}`)
await exec(`"${pwshPath}"`, args)
} else {
const powershellCommand = [
`$ErrorActionPreference = 'Stop' ;`,
`try { Add-Type -AssemblyName System.IO.Compression.FileSystem } catch { } ;`,
`if ((Get-Command -Name Expand-Archive -Module Microsoft.PowerShell.Archive -ErrorAction Ignore)) { Expand-Archive -LiteralPath '${escapedFile}' -DestinationPath '${escapedDest}' -Force }`,
`else {[System.IO.Compression.ZipFile]::ExtractToDirectory('${escapedFile}', '${escapedDest}', $true) }`
].join(' ')
const args = [
'-NoLogo',
'-Sta',
'-NoProfile',
'-NonInteractive',
'-ExecutionPolicy',
'Unrestricted',
'-Command',
powershellCommand
]
const powershellPath = await io.which('powershell', true)
core.debug(`Using powershell at path: ${powershellPath}`)
await exec(`"${powershellPath}"`, args)
}
}
async function extractZipNix(file: string, dest: string): Promise<void> {
const unzipPath = await io.which('unzip', true)
const args = [file]
if (!core.isDebug()) {
args.unshift('-q')
}
args.unshift('-o') //overwrite with -o, otherwise a prompt is shown which freezes the run
await exec(`"${unzipPath}"`, args, {cwd: dest})
}
/**
* Caches a directory and installs it into the tool cacheDir
*
* @param sourceDir the directory to cache into tools
* @param tool tool name
* @param version version of the tool. semver format
* @param arch architecture of the tool. Optional. Defaults to machine architecture
*/
export async function cacheDir(
sourceDir: string,
tool: string,
version: string,
arch?: string
): Promise<string> {
version = semver.clean(version) || version
arch = arch || os.arch()
core.debug(`Caching tool ${tool} ${version} ${arch}`)
core.debug(`source dir: ${sourceDir}`)
if (!fs.statSync(sourceDir).isDirectory()) {
throw new Error('sourceDir is not a directory')
}
// Create the tool dir
const destPath: string = await _createToolPath(tool, version, arch)
// copy each child item. do not move. move can fail on Windows
// due to anti-virus software having an open handle on a file.
for (const itemName of fs.readdirSync(sourceDir)) {
const s = path.join(sourceDir, itemName)
await io.cp(s, destPath, {recursive: true})
}
// write .complete
_completeToolPath(tool, version, arch)
return destPath
}
/**
* Caches a downloaded file (GUID) and installs it
* into the tool cache with a given targetName
*
* @param sourceFile the file to cache into tools. Typically a result of downloadTool which is a guid.
* @param targetFile the name of the file name in the tools directory
* @param tool tool name
* @param version version of the tool. semver format
* @param arch architecture of the tool. Optional. Defaults to machine architecture
*/
export async function cacheFile(
sourceFile: string,
targetFile: string,
tool: string,
version: string,
arch?: string
): Promise<string> {
version = semver.clean(version) || version
arch = arch || os.arch()
core.debug(`Caching tool ${tool} ${version} ${arch}`)
core.debug(`source file: ${sourceFile}`)
if (!fs.statSync(sourceFile).isFile()) {
throw new Error('sourceFile is not a file')
}
// create the tool dir
const destFolder: string = await _createToolPath(tool, version, arch)
// copy instead of move. move can fail on Windows due to
// anti-virus software having an open handle on a file.
const destPath: string = path.join(destFolder, targetFile)
core.debug(`destination file ${destPath}`)
await io.cp(sourceFile, destPath)
// write .complete
_completeToolPath(tool, version, arch)
return destFolder
}
/**
* Finds the path to a tool version in the local installed tool cache
*
* @param toolName name of the tool
* @param versionSpec version of the tool
* @param arch optional arch. defaults to arch of computer
*/
export function find(
toolName: string,
versionSpec: string,
arch?: string
): string {
if (!toolName) {
throw new Error('toolName parameter is required')
}
if (!versionSpec) {
throw new Error('versionSpec parameter is required')
}
arch = arch || os.arch()
// attempt to resolve an explicit version
if (!isExplicitVersion(versionSpec)) {
const localVersions: string[] = findAllVersions(toolName, arch)
const match = evaluateVersions(localVersions, versionSpec)
versionSpec = match
}
// check for the explicit version in the cache
let toolPath = ''
if (versionSpec) {
versionSpec = semver.clean(versionSpec) || ''
const cachePath = path.join(
_getCacheDirectory(),
toolName,
versionSpec,
arch
)
core.debug(`checking cache: ${cachePath}`)
if (fs.existsSync(cachePath) && fs.existsSync(`${cachePath}.complete`)) {
core.debug(`Found tool in cache ${toolName} ${versionSpec} ${arch}`)
toolPath = cachePath
} else {
core.debug('not found')
}
}
return toolPath
}
/**
* Finds the paths to all versions of a tool that are installed in the local tool cache
*
* @param toolName name of the tool
* @param arch optional arch. defaults to arch of computer
*/
export function findAllVersions(toolName: string, arch?: string): string[] {
const versions: string[] = []
arch = arch || os.arch()
const toolPath = path.join(_getCacheDirectory(), toolName)
if (fs.existsSync(toolPath)) {
const children: string[] = fs.readdirSync(toolPath)
for (const child of children) {
if (isExplicitVersion(child)) {
const fullPath = path.join(toolPath, child, arch || '')
if (fs.existsSync(fullPath) && fs.existsSync(`${fullPath}.complete`)) {
versions.push(child)
}
}
}
}
return versions
}
// versions-manifest
//
// typical pattern of a setup-* action that supports JIT would be:
// 1. resolve semver against local cache
//
// 2. if no match, download
// a. query versions manifest to match
// b. if no match, fall back to source if exists (tool distribution)
// c. with download url, download, install and preprent path
export type IToolRelease = mm.IToolRelease
export type IToolReleaseFile = mm.IToolReleaseFile
interface GitHubTreeItem {
path: string
size: string
url: string
}
interface GitHubTree {
tree: GitHubTreeItem[]
truncated: boolean
}
export async function getManifestFromRepo(
owner: string,
repo: string,
auth?: string,
branch = 'master'
): Promise<IToolRelease[]> {
let releases: IToolRelease[] = []
const treeUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/${branch}`
const http: httpm.HttpClient = new httpm.HttpClient('tool-cache')
const headers: OutgoingHttpHeaders = {}
if (auth) {
core.debug('set auth')
headers.authorization = auth
}
const response = await http.getJson<GitHubTree>(treeUrl, headers)
if (!response.result) {
return releases
}
let manifestUrl = ''
for (const item of response.result.tree) {
if (item.path === 'versions-manifest.json') {
manifestUrl = item.url
break
}
}
headers['accept'] = 'application/vnd.github.VERSION.raw'
let versionsRaw = await (await http.get(manifestUrl, headers)).readBody()
if (versionsRaw) {
// shouldn't be needed but protects against invalid json saved with BOM
versionsRaw = versionsRaw.replace(/^\uFEFF/, '')
try {
releases = JSON.parse(versionsRaw)
} catch {
core.debug('Invalid json')
}
}
return releases
}
export async function findFromManifest(
versionSpec: string,
stable: boolean,
manifest: IToolRelease[],
archFilter: string = os.arch()
): Promise<IToolRelease | undefined> {
// wrap the internal impl
const match: mm.IToolRelease | undefined = await mm._findMatch(
versionSpec,
stable,
manifest,
archFilter
)
return match
}
async function _createExtractFolder(dest?: string): Promise<string> {
if (!dest) {
// create a temp dir
dest = path.join(_getTempDirectory(), uuidV4())
}
await io.mkdirP(dest)
return dest
}
async function _createToolPath(
tool: string,
version: string,
arch?: string
): Promise<string> {
const folderPath = path.join(
_getCacheDirectory(),
tool,
semver.clean(version) || version,
arch || ''
)
core.debug(`destination ${folderPath}`)
const markerPath = `${folderPath}.complete`
await io.rmRF(folderPath)
await io.rmRF(markerPath)
await io.mkdirP(folderPath)
return folderPath
}
function _completeToolPath(tool: string, version: string, arch?: string): void {
const folderPath = path.join(
_getCacheDirectory(),
tool,
semver.clean(version) || version,
arch || ''
)
const markerPath = `${folderPath}.complete`
fs.writeFileSync(markerPath, '')
core.debug('finished caching tool')
}
/**
* Check if version string is explicit
*
* @param versionSpec version string to check
*/
export function isExplicitVersion(versionSpec: string): boolean {
const c = semver.clean(versionSpec) || ''
core.debug(`isExplicit: ${c}`)
const valid = semver.valid(c) != null
core.debug(`explicit? ${valid}`)
return valid
}
/**
* Get the highest satisfiying semantic version in `versions` which satisfies `versionSpec`
*
* @param versions array of versions to evaluate
* @param versionSpec semantic version spec to satisfy
*/
export function evaluateVersions(
versions: string[],
versionSpec: string
): string {
let version = ''
core.debug(`evaluating ${versions.length} versions`)
versions = versions.sort((a, b) => {
if (semver.gt(a, b)) {
return 1
}
return -1
})
for (let i = versions.length - 1; i >= 0; i--) {
const potential: string = versions[i]
const satisfied: boolean = semver.satisfies(potential, versionSpec)
if (satisfied) {
version = potential
break
}
}
if (version) {
core.debug(`matched: ${version}`)
} else {
core.debug('match not found')
}
return version
}
/**
* Gets RUNNER_TOOL_CACHE
*/
function _getCacheDirectory(): string {
const cacheDirectory = process.env['RUNNER_TOOL_CACHE'] || ''
ok(cacheDirectory, 'Expected RUNNER_TOOL_CACHE to be defined')
return cacheDirectory
}
/**
* Gets RUNNER_TEMP
*/
function _getTempDirectory(): string {
const tempDirectory = process.env['RUNNER_TEMP'] || ''
ok(tempDirectory, 'Expected RUNNER_TEMP to be defined')
return tempDirectory
}
/**
* Gets a global variable
*/
function _getGlobal<T>(key: string, defaultValue: T): T {
/* eslint-disable @typescript-eslint/no-explicit-any */
const value = (global as any)[key] as T | undefined
/* eslint-enable @typescript-eslint/no-explicit-any */
return value !== undefined ? value : defaultValue
}
/**
* Returns an array of unique values.
* @param values Values to make unique.
*/
function _unique<T>(values: T[]): T[] {
return Array.from(new Set(values))
}