1
0
Fork 0

Tool cache install from a manifest file (#382)

pull/476/head
Bryan MacFarlane 2020-05-19 13:25:57 -04:00 committed by GitHub
parent dcf5c88bb3
commit 4e9375da09
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 633 additions and 10 deletions

View File

@ -18,7 +18,7 @@
"@typescript-eslint/await-thenable": "error",
"@typescript-eslint/ban-ts-ignore": "error",
"camelcase": "off",
"@typescript-eslint/camelcase": "error",
"@typescript-eslint/camelcase": "off",
"@typescript-eslint/class-name-casing": "error",
"@typescript-eslint/explicit-function-return-type": ["error", {"allowExpressions": true}],
"@typescript-eslint/func-call-spacing": ["error", "never"],

View File

@ -55,7 +55,6 @@ describe('@actions/context', () => {
it('works with pull_request payloads', () => {
delete process.env.GITHUB_REPOSITORY
context.payload = {
// eslint-disable-next-line @typescript-eslint/camelcase
pull_request: {number: 2},
repository: {owner: {login: 'user'}, name: 'test'}
}

View File

@ -0,0 +1,86 @@
[
{
"version": "3.0.1",
"stable": false,
"release_url": "https://github.com/actions/sometool/releases/tag/3.0.1-20200402.6",
"files": [
{
"filename": "sometool-3.0.1-linux-x64.tar.gz",
"arch": "x64",
"platform": "linux",
"download_url": "https://github.com/actions/sometool/releases/tag/3.0.1-20200402.6/sometool-1.2.3-linux-x64.tar.gz"
}
]
},
{
"version": "2.0.2",
"stable": true,
"release_url": "https://github.com/actions/sometool/releases/tag/2.0.2-20200402.6",
"files": [
{
"filename": "sometool-2.0.2-linux-x64.tar.gz",
"arch": "x64",
"platform": "linux",
"download_url": "https://github.com/actions/sometool/releases/tag/2.0.2-20200402.6/sometool-2.0.2-linux-x64.tar.gz"
},
{
"filename": "sometool-2.0.2-linux-x32.tar.gz",
"arch": "x32",
"platform": "linux",
"download_url": "https://github.com/actions/sometool/releases/tag/2.0.2-20200402.6/sometool-2.0.2-linux-x32.tar.gz"
},
{
"filename": "sometool-2.0.2-linux-x32.tar.gz",
"arch": "x64",
"platform": "windows",
"download_url": "https://github.com/actions/sometool/releases/tag/2.0.2-20200402.6/sometool-2.0.2-linux-x32.tar.gz"
}
]
},
{
"version": "1.2.4",
"stable": true,
"release_url": "https://github.com/actions/sometool/releases/tag/1.2.4-20200402.6",
"files": [
{
"filename": "sometool-1.2.4-ubuntu1804-x64.tar.gz",
"arch": "x64",
"platform": "linux",
"platform_version": "18.04",
"download_url": "https://github.com/actions/sometool/releases/tag/1.2.4-20200402.6/sometool-1.2.4-ubuntu1804-x64.tar.gz"
},
{
"filename": "sometool-1.2.4-darwin1015-x64.tar.gz",
"arch": "x64",
"platform": "darwin",
"platform_version": "10.15",
"download_url": "https://github.com/actions/sometool/releases/tag/1.2.4-20200402.6/sometool-1.2.4-darwin1015-x64.tar.gz"
}
]
},
{
"version": "1.2.3",
"stable": true,
"release_url": "https://github.com/actions/sometool/releases/tag/1.2.3-20200402.6",
"files": [
{
"filename": "sometool-1.2.3-linux-x64.tar.gz",
"arch": "x64",
"platform": "linux",
"download_url": "https://github.com/actions/sometool/releases/tag/1.2.3-20200402.6/sometool-1.2.3-linux-x64.tar.gz"
},
{
"filename": "sometool-1.2.3-linux-x32.tar.gz",
"arch": "x32",
"platform": "linux",
"download_url": "https://github.com/actions/sometool/releases/tag/1.2.3-20200402.6/sometool-1.2.3-linux-x32.tar.gz"
},
{
"filename": "sometool-1.2.3-linux-x32.zip",
"arch": "x64",
"platform": "windows",
"download_url": "https://github.com/actions/sometool/releases/tag/1.2.3-20200402.6/sometool-1.2.3-linux-x32.zip"
}
]
}
]

View File

@ -0,0 +1,272 @@
import * as tc from '../src/tool-cache'
import * as mm from '../src/manifest' // --> OFF
// needs to be require for core node modules to be mocked
// eslint-disable-next-line @typescript-eslint/no-require-imports
import osm = require('os')
// eslint-disable-next-line @typescript-eslint/no-require-imports
import cp = require('child_process')
//import {coerce} from 'semver'
// we fetch the manifest file from master of a repo
const owner = 'actions'
const repo = 'some-tool'
const fakeToken = 'notrealtoken'
// just loading data and require handles BOMs etc.
// eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires
const manifestData = require('./data/versions-manifest.json')
describe('@actions/tool-cache-manifest', () => {
let os: {platform: string; arch: string}
let getSpy: jest.SpyInstance
let platSpy: jest.SpyInstance
let archSpy: jest.SpyInstance
let execSpy: jest.SpyInstance
let readLsbSpy: jest.SpyInstance
beforeEach(() => {
// node
os = {platform: '', arch: ''}
platSpy = jest.spyOn(osm, 'platform')
platSpy.mockImplementation(() => os.platform)
archSpy = jest.spyOn(osm, 'arch')
archSpy.mockImplementation(() => os.arch)
execSpy = jest.spyOn(cp, 'execSync')
readLsbSpy = jest.spyOn(mm, '_readLinuxVersionFile')
getSpy = jest.spyOn(tc, 'getManifestFromRepo')
getSpy.mockImplementation(() => <mm.IToolRelease[]>manifestData)
})
afterEach(() => {
jest.resetAllMocks()
jest.clearAllMocks()
//jest.restoreAllMocks();
})
afterAll(async () => {}, 100000)
it('can query versions', async () => {
const manifest: mm.IToolRelease[] | null = await tc.getManifestFromRepo(
owner,
repo,
fakeToken
)
expect(manifest).toBeDefined()
const l: number = manifest ? manifest.length : 0
expect(l).toBe(4)
})
it('can match stable major version for linux x64', async () => {
os.platform = 'linux'
os.arch = 'x64'
const manifest: mm.IToolRelease[] | null = await tc.getManifestFromRepo(
owner,
repo,
fakeToken
)
const release: tc.IToolRelease | undefined = await tc.findFromManifest(
'2.x',
true,
manifest
)
expect(release).toBeDefined()
expect(release?.version).toBe('2.0.2')
expect(release?.files.length).toBe(1)
const file = release?.files[0]
expect(file).toBeDefined()
expect(file?.arch).toBe('x64')
expect(file?.platform).toBe('linux')
expect(file?.download_url).toBe(
'https://github.com/actions/sometool/releases/tag/2.0.2-20200402.6/sometool-2.0.2-linux-x64.tar.gz'
)
expect(file?.filename).toBe('sometool-2.0.2-linux-x64.tar.gz')
})
it('can match stable exact version for linux x64', async () => {
os.platform = 'linux'
os.arch = 'x64'
const manifest: mm.IToolRelease[] | null = await tc.getManifestFromRepo(
owner,
repo,
fakeToken
)
const release: tc.IToolRelease | undefined = await tc.findFromManifest(
'1.2.3',
true,
manifest
)
expect(release).toBeDefined()
expect(release?.version).toBe('1.2.3')
expect(release?.files.length).toBe(1)
const file = release?.files[0]
expect(file).toBeDefined()
expect(file?.arch).toBe('x64')
expect(file?.platform).toBe('linux')
expect(file?.download_url).toBe(
'https://github.com/actions/sometool/releases/tag/1.2.3-20200402.6/sometool-1.2.3-linux-x64.tar.gz'
)
expect(file?.filename).toBe('sometool-1.2.3-linux-x64.tar.gz')
})
it('can match with linux platform version spec', async () => {
os.platform = 'linux'
os.arch = 'x64'
readLsbSpy.mockImplementation(() => {
return `DISTRIB_ID=Ubuntu
DISTRIB_RELEASE=18.04
DISTRIB_CODENAME=bionic
DISTRIB_DESCRIPTION=Ubuntu 18.04.4 LTS`
})
const manifest: mm.IToolRelease[] | null = await tc.getManifestFromRepo(
owner,
repo,
fakeToken
)
const release: tc.IToolRelease | undefined = await tc.findFromManifest(
'1.2.4',
true,
manifest
)
expect(release).toBeDefined()
expect(release?.version).toBe('1.2.4')
expect(release?.files.length).toBe(1)
const file = release?.files[0]
expect(file).toBeDefined()
expect(file?.arch).toBe('x64')
expect(file?.platform).toBe('linux')
expect(file?.download_url).toBe(
'https://github.com/actions/sometool/releases/tag/1.2.4-20200402.6/sometool-1.2.4-ubuntu1804-x64.tar.gz'
)
expect(file?.filename).toBe('sometool-1.2.4-ubuntu1804-x64.tar.gz')
})
it('can match with darwin platform version spec', async () => {
os.platform = 'darwin'
os.arch = 'x64'
execSpy.mockImplementation(() => '10.15.1')
const manifest: mm.IToolRelease[] | null = await tc.getManifestFromRepo(
owner,
repo,
fakeToken
)
const release: tc.IToolRelease | undefined = await tc.findFromManifest(
'1.2.4',
true,
manifest
)
expect(release).toBeDefined()
expect(release?.version).toBe('1.2.4')
expect(release?.files.length).toBe(1)
const file = release?.files[0]
expect(file).toBeDefined()
expect(file?.arch).toBe('x64')
expect(file?.platform).toBe('darwin')
expect(file?.download_url).toBe(
'https://github.com/actions/sometool/releases/tag/1.2.4-20200402.6/sometool-1.2.4-darwin1015-x64.tar.gz'
)
expect(file?.filename).toBe('sometool-1.2.4-darwin1015-x64.tar.gz')
})
it('does not match with unmatched linux platform version spec', async () => {
os.platform = 'linux'
os.arch = 'x64'
readLsbSpy.mockImplementation(() => {
return `DISTRIB_ID=Ubuntu
DISTRIB_RELEASE=16.04
DISTRIB_CODENAME=xenial
DISTRIB_DESCRIPTION=Ubuntu 16.04.4 LTS`
})
const manifest: mm.IToolRelease[] | null = await tc.getManifestFromRepo(
owner,
repo,
fakeToken
)
const release: tc.IToolRelease | undefined = await tc.findFromManifest(
'1.2.4',
true,
manifest
)
expect(release).toBeUndefined()
})
it('does not match with unmatched darwin platform version spec', async () => {
os.platform = 'darwin'
os.arch = 'x64'
execSpy.mockImplementation(() => '10.14.6')
const manifest: mm.IToolRelease[] | null = await tc.getManifestFromRepo(
owner,
repo,
fakeToken
)
const release: tc.IToolRelease | undefined = await tc.findFromManifest(
'1.2.4',
true,
manifest
)
expect(release).toBeUndefined()
})
it('can get version from lsb on ubuntu-18.04', async () => {
os.platform = 'linux'
os.arch = 'x64'
//existsSpy.mockImplementation(() => true)
readLsbSpy.mockImplementation(() => {
return `DISTRIB_ID=Ubuntu
DISTRIB_RELEASE=18.04
DISTRIB_CODENAME=bionic
DISTRIB_DESCRIPTION=Ubuntu 18.04.4 LTS`
})
const version = mm._getOsVersion()
expect(osm.platform()).toBe('linux')
expect(version).toBe('18.04')
})
it('can get version from lsb on ubuntu-16.04', async () => {
os.platform = 'linux'
os.arch = 'x64'
readLsbSpy.mockImplementation(() => {
return `DISTRIB_ID=Ubuntu
DISTRIB_RELEASE=16.04
DISTRIB_CODENAME=xenial
DISTRIB_DESCRIPTION="Ubuntu 16.04.6 LTS"`
})
const version = mm._getOsVersion()
expect(osm.platform()).toBe('linux')
expect(version).toBe('16.04')
})
// sw_vers -productVersion
it('can get version on macOS', async () => {
os.platform = 'darwin'
os.arch = 'x64'
execSpy.mockImplementation(() => '10.14.6')
const version = mm._getOsVersion()
expect(osm.platform()).toBe('darwin')
expect(version).toBe('10.14.6')
})
})

View File

@ -1,6 +1,6 @@
{
"name": "@actions/tool-cache",
"version": "1.3.5",
"version": "1.5.3",
"lockfileVersion": 1,
"requires": true,
"dependencies": {

View File

@ -1,6 +1,6 @@
{
"name": "@actions/tool-cache",
"version": "1.3.5",
"version": "1.5.4",
"description": "Actions tool-cache lib",
"keywords": [
"github",

View File

@ -0,0 +1,158 @@
import * as semver from 'semver'
import {debug} from '@actions/core'
// needs to be require for core node modules to be mocked
/* eslint @typescript-eslint/no-require-imports: 0 */
import os = require('os')
import cp = require('child_process')
import fs = require('fs')
/*
NOTE: versions must be sorted descending by version in the manifest
this library short circuits on first semver spec match
platform_version is an optional filter and can be a semver spec or range
[
{
"version": "1.2.3",
"stable": true,
"release_url": "https://github.com/actions/sometool/releases/tag/1.2.3-20200402.6",
"files": [
{
"filename": "sometool-1.2.3-linux-x64.zip",
"arch": "x64",
"platform": "linux",
"platform_version": "18.04"
"download_url": "https://github.com/actions/sometool/releases/tag/1.2.3-20200402.6/sometool-1.2.3-linux-x64.zip"
},
...
]
},
...
]
*/
export interface IToolReleaseFile {
filename: string
// 'aix', 'darwin', 'freebsd', 'linux', 'openbsd',
// 'sunos', and 'win32'
// platform_version is an optional semver filter
// TODO: do we need distribution (e.g. ubuntu).
// not adding yet but might need someday.
// right now, 16.04 and 18.04 work
platform: string
platform_version?: string
// 'arm', 'arm64', 'ia32', 'mips', 'mipsel',
// 'ppc', 'ppc64', 's390', 's390x',
// 'x32', and 'x64'.
arch: string
download_url: string
}
export interface IToolRelease {
version: string
stable: boolean
release_url: string
files: IToolReleaseFile[]
}
export async function _findMatch(
versionSpec: string,
stable: boolean,
candidates: IToolRelease[],
archFilter: string
): Promise<IToolRelease | undefined> {
const platFilter = os.platform()
let result: IToolRelease | undefined
let match: IToolRelease | undefined
let file: IToolReleaseFile | undefined
for (const candidate of candidates) {
const version = candidate.version
debug(`check ${version} satisfies ${versionSpec}`)
if (
semver.satisfies(version, versionSpec) &&
(!stable || candidate.stable === stable)
) {
file = candidate.files.find(item => {
debug(
`${item.arch}===${archFilter} && ${item.platform}===${platFilter}`
)
let chk = item.arch === archFilter && item.platform === platFilter
if (chk && item.platform_version) {
const osVersion = module.exports._getOsVersion()
if (osVersion === item.platform_version) {
chk = true
} else {
chk = semver.satisfies(osVersion, item.platform_version)
}
}
return chk
})
if (file) {
debug(`matched ${candidate.version}`)
match = candidate
break
}
}
}
if (match && file) {
// clone since we're mutating the file list to be only the file that matches
result = Object.assign({}, match)
result.files = [file]
}
return result
}
export function _getOsVersion(): string {
// TODO: add windows and other linux, arm variants
// right now filtering on version is only an ubuntu and macos scenario for tools we build for hosted (python)
const plat = os.platform()
let version = ''
if (plat === 'darwin') {
version = cp.execSync('sw_vers -productVersion').toString()
} else if (plat === 'linux') {
// lsb_release process not in some containers, readfile
// Run cat /etc/lsb-release
// DISTRIB_ID=Ubuntu
// DISTRIB_RELEASE=18.04
// DISTRIB_CODENAME=bionic
// DISTRIB_DESCRIPTION="Ubuntu 18.04.4 LTS"
const lsbContents = module.exports._readLinuxVersionFile()
if (lsbContents) {
const lines = lsbContents.split('\n')
for (const line of lines) {
const parts = line.split('=')
if (parts.length === 2 && parts[0].trim() === 'DISTRIB_RELEASE') {
version = parts[1].trim()
break
}
}
}
}
return version
}
export function _readLinuxVersionFile(): string {
const lsbFile = '/etc/lsb-release'
let contents = ''
if (fs.existsSync(lsbFile)) {
contents = fs.readFileSync(lsbFile).toString()
}
return contents
}

View File

@ -1,6 +1,7 @@
import * as core from '@actions/core'
import * as io from '@actions/io'
import * as fs from 'fs'
import * as mm from './manifest'
import * as os from 'os'
import * as path from 'path'
import * as httpm from '@actions/http-client'
@ -12,6 +13,7 @@ import {exec} from '@actions/exec/lib/exec'
import {ExecOptions} from '@actions/exec/lib/interfaces'
import {ok} from 'assert'
import {RetryHelper} from './retry-helper'
import {IHeaders} from '@actions/http-client/interfaces'
export class HTTPError extends Error {
constructor(readonly httpStatusCode: number | undefined) {
@ -28,11 +30,13 @@ const userAgent = 'actions/tool-cache'
*
* @param url url of tool to download
* @param dest path to download tool
* @param auth authorization header
* @returns path to downloaded tool
*/
export async function downloadTool(
url: string,
dest?: string
dest?: string,
auth?: string
): Promise<string> {
dest = dest || path.join(_getTempDirectory(), uuidV4())
await io.mkdirP(path.dirname(dest))
@ -51,7 +55,7 @@ export async function downloadTool(
const retryHelper = new RetryHelper(maxAttempts, minSeconds, maxSeconds)
return await retryHelper.execute(
async () => {
return await downloadToolAttempt(url, dest || '')
return await downloadToolAttempt(url, dest || '', auth)
},
(err: Error) => {
if (err instanceof HTTPError && err.httpStatusCode) {
@ -71,7 +75,11 @@ export async function downloadTool(
)
}
async function downloadToolAttempt(url: string, dest: string): Promise<string> {
async function downloadToolAttempt(
url: string,
dest: string,
auth?: string
): Promise<string> {
if (fs.existsSync(dest)) {
throw new Error(`Destination file path ${dest} already exists`)
}
@ -80,7 +88,16 @@ async function downloadToolAttempt(url: string, dest: string): Promise<string> {
const http = new httpm.HttpClient(userAgent, [], {
allowRetries: false
})
const response: httpm.HttpClientResponse = await http.get(url)
let headers: IHeaders | undefined
if (auth) {
core.debug('set auth')
headers = {
authorization: auth
}
}
const response: httpm.HttpClientResponse = await http.get(url, headers)
if (response.message.statusCode !== 200) {
const err = new HTTPError(response.message.statusCode)
core.debug(
@ -202,7 +219,7 @@ export async function extract7z(
export async function extractTar(
file: string,
dest?: string,
flags: string = 'xz'
flags: string | string[] = 'xz'
): Promise<string> {
if (!file) {
throw new Error("parameter 'file' is required")
@ -226,7 +243,12 @@ export async function extractTar(
const isGnuTar = versionOutput.toUpperCase().includes('GNU TAR')
// Initialize args
const args = [flags]
let args: string[]
if (flags instanceof Array) {
args = flags
} else {
args = [flags]
}
if (core.isDebug() && !flags.includes('v')) {
args.push('-v')
@ -463,6 +485,92 @@ export function findAllVersions(toolName: string, arch?: string): string[] {
return versions
}
// versions-manifest
//
// typical pattern of a setup-* action that supports JIT would be:
// 1. resolve semver against local cache
//
// 2. if no match, download
// a. query versions manifest to match
// b. if no match, fall back to source if exists (tool distribution)
// c. with download url, download, install and preprent path
export type IToolRelease = mm.IToolRelease
export type IToolReleaseFile = mm.IToolReleaseFile
interface GitHubTreeItem {
path: string
size: string
url: string
}
interface GitHubTree {
tree: GitHubTreeItem[]
truncated: boolean
}
export async function getManifestFromRepo(
owner: string,
repo: string,
auth?: string,
branch = 'master'
): Promise<IToolRelease[]> {
let releases: IToolRelease[] = []
const treeUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/${branch}`
const http: httpm.HttpClient = new httpm.HttpClient('tool-cache')
const headers: IHeaders = {}
if (auth) {
core.debug('set auth')
headers.authorization = auth
}
const response = await http.getJson<GitHubTree>(treeUrl, headers)
if (!response.result) {
return releases
}
let manifestUrl = ''
for (const item of response.result.tree) {
if (item.path === 'versions-manifest.json') {
manifestUrl = item.url
break
}
}
headers['accept'] = 'application/vnd.github.VERSION.raw'
let versionsRaw = await (await http.get(manifestUrl, headers)).readBody()
if (versionsRaw) {
// shouldn't be needed but protects against invalid json saved with BOM
versionsRaw = versionsRaw.replace(/^\uFEFF/, '')
try {
releases = JSON.parse(versionsRaw)
} catch {
core.debug('Invalid json')
}
}
return releases
}
export async function findFromManifest(
versionSpec: string,
stable: boolean,
manifest: IToolRelease[],
archFilter: string = os.arch()
): Promise<IToolRelease | undefined> {
// wrap the internal impl
const match: mm.IToolRelease | undefined = await mm._findMatch(
versionSpec,
stable,
manifest,
archFilter
)
return match
}
async function _createExtractFolder(dest?: string): Promise<string> {
if (!dest) {
// create a temp dir