From 4e9375da095a1d1f59b3314589fab6cd0d215139 Mon Sep 17 00:00:00 2001 From: Bryan MacFarlane Date: Tue, 19 May 2020 13:25:57 -0400 Subject: [PATCH] Tool cache install from a manifest file (#382) --- .eslintrc.json | 2 +- packages/github/__tests__/lib.test.ts | 1 - .../__tests__/data/versions-manifest.json | 86 ++++++ .../tool-cache/__tests__/manifest.test.ts | 272 ++++++++++++++++++ packages/tool-cache/package-lock.json | 2 +- packages/tool-cache/package.json | 2 +- packages/tool-cache/src/manifest.ts | 158 ++++++++++ packages/tool-cache/src/tool-cache.ts | 120 +++++++- 8 files changed, 633 insertions(+), 10 deletions(-) create mode 100644 packages/tool-cache/__tests__/data/versions-manifest.json create mode 100644 packages/tool-cache/__tests__/manifest.test.ts create mode 100644 packages/tool-cache/src/manifest.ts diff --git a/.eslintrc.json b/.eslintrc.json index 2f73a3cf..5833878c 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -18,7 +18,7 @@ "@typescript-eslint/await-thenable": "error", "@typescript-eslint/ban-ts-ignore": "error", "camelcase": "off", - "@typescript-eslint/camelcase": "error", + "@typescript-eslint/camelcase": "off", "@typescript-eslint/class-name-casing": "error", "@typescript-eslint/explicit-function-return-type": ["error", {"allowExpressions": true}], "@typescript-eslint/func-call-spacing": ["error", "never"], diff --git a/packages/github/__tests__/lib.test.ts b/packages/github/__tests__/lib.test.ts index dae2a8cd..d2754729 100644 --- a/packages/github/__tests__/lib.test.ts +++ b/packages/github/__tests__/lib.test.ts @@ -55,7 +55,6 @@ describe('@actions/context', () => { it('works with pull_request payloads', () => { delete process.env.GITHUB_REPOSITORY context.payload = { - // eslint-disable-next-line @typescript-eslint/camelcase pull_request: {number: 2}, repository: {owner: {login: 'user'}, name: 'test'} } diff --git a/packages/tool-cache/__tests__/data/versions-manifest.json b/packages/tool-cache/__tests__/data/versions-manifest.json new file mode 100644 index 00000000..0c45b81a --- /dev/null +++ b/packages/tool-cache/__tests__/data/versions-manifest.json @@ -0,0 +1,86 @@ +[ + { + "version": "3.0.1", + "stable": false, + "release_url": "https://github.com/actions/sometool/releases/tag/3.0.1-20200402.6", + "files": [ + { + "filename": "sometool-3.0.1-linux-x64.tar.gz", + "arch": "x64", + "platform": "linux", + "download_url": "https://github.com/actions/sometool/releases/tag/3.0.1-20200402.6/sometool-1.2.3-linux-x64.tar.gz" + } + ] + }, + { + "version": "2.0.2", + "stable": true, + "release_url": "https://github.com/actions/sometool/releases/tag/2.0.2-20200402.6", + "files": [ + { + "filename": "sometool-2.0.2-linux-x64.tar.gz", + "arch": "x64", + "platform": "linux", + "download_url": "https://github.com/actions/sometool/releases/tag/2.0.2-20200402.6/sometool-2.0.2-linux-x64.tar.gz" + }, + { + "filename": "sometool-2.0.2-linux-x32.tar.gz", + "arch": "x32", + "platform": "linux", + "download_url": "https://github.com/actions/sometool/releases/tag/2.0.2-20200402.6/sometool-2.0.2-linux-x32.tar.gz" + }, + { + "filename": "sometool-2.0.2-linux-x32.tar.gz", + "arch": "x64", + "platform": "windows", + "download_url": "https://github.com/actions/sometool/releases/tag/2.0.2-20200402.6/sometool-2.0.2-linux-x32.tar.gz" + } + ] + }, + { + "version": "1.2.4", + "stable": true, + "release_url": "https://github.com/actions/sometool/releases/tag/1.2.4-20200402.6", + "files": [ + { + "filename": "sometool-1.2.4-ubuntu1804-x64.tar.gz", + "arch": "x64", + "platform": "linux", + "platform_version": "18.04", + "download_url": "https://github.com/actions/sometool/releases/tag/1.2.4-20200402.6/sometool-1.2.4-ubuntu1804-x64.tar.gz" + }, + { + "filename": "sometool-1.2.4-darwin1015-x64.tar.gz", + "arch": "x64", + "platform": "darwin", + "platform_version": "10.15", + "download_url": "https://github.com/actions/sometool/releases/tag/1.2.4-20200402.6/sometool-1.2.4-darwin1015-x64.tar.gz" + } + ] + }, + { + "version": "1.2.3", + "stable": true, + "release_url": "https://github.com/actions/sometool/releases/tag/1.2.3-20200402.6", + "files": [ + { + "filename": "sometool-1.2.3-linux-x64.tar.gz", + "arch": "x64", + "platform": "linux", + "download_url": "https://github.com/actions/sometool/releases/tag/1.2.3-20200402.6/sometool-1.2.3-linux-x64.tar.gz" + }, + { + "filename": "sometool-1.2.3-linux-x32.tar.gz", + "arch": "x32", + "platform": "linux", + "download_url": "https://github.com/actions/sometool/releases/tag/1.2.3-20200402.6/sometool-1.2.3-linux-x32.tar.gz" + }, + { + "filename": "sometool-1.2.3-linux-x32.zip", + "arch": "x64", + "platform": "windows", + "download_url": "https://github.com/actions/sometool/releases/tag/1.2.3-20200402.6/sometool-1.2.3-linux-x32.zip" + } + ] + } +] \ No newline at end of file diff --git a/packages/tool-cache/__tests__/manifest.test.ts b/packages/tool-cache/__tests__/manifest.test.ts new file mode 100644 index 00000000..1c93b627 --- /dev/null +++ b/packages/tool-cache/__tests__/manifest.test.ts @@ -0,0 +1,272 @@ +import * as tc from '../src/tool-cache' +import * as mm from '../src/manifest' // --> OFF + +// needs to be require for core node modules to be mocked +// eslint-disable-next-line @typescript-eslint/no-require-imports +import osm = require('os') +// eslint-disable-next-line @typescript-eslint/no-require-imports +import cp = require('child_process') +//import {coerce} from 'semver' + +// we fetch the manifest file from master of a repo +const owner = 'actions' +const repo = 'some-tool' +const fakeToken = 'notrealtoken' + +// just loading data and require handles BOMs etc. +// eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires +const manifestData = require('./data/versions-manifest.json') + +describe('@actions/tool-cache-manifest', () => { + let os: {platform: string; arch: string} + + let getSpy: jest.SpyInstance + let platSpy: jest.SpyInstance + let archSpy: jest.SpyInstance + let execSpy: jest.SpyInstance + let readLsbSpy: jest.SpyInstance + + beforeEach(() => { + // node + os = {platform: '', arch: ''} + platSpy = jest.spyOn(osm, 'platform') + + platSpy.mockImplementation(() => os.platform) + archSpy = jest.spyOn(osm, 'arch') + archSpy.mockImplementation(() => os.arch) + + execSpy = jest.spyOn(cp, 'execSync') + readLsbSpy = jest.spyOn(mm, '_readLinuxVersionFile') + + getSpy = jest.spyOn(tc, 'getManifestFromRepo') + getSpy.mockImplementation(() => manifestData) + }) + + afterEach(() => { + jest.resetAllMocks() + jest.clearAllMocks() + //jest.restoreAllMocks(); + }) + + afterAll(async () => {}, 100000) + + it('can query versions', async () => { + const manifest: mm.IToolRelease[] | null = await tc.getManifestFromRepo( + owner, + repo, + fakeToken + ) + + expect(manifest).toBeDefined() + const l: number = manifest ? manifest.length : 0 + expect(l).toBe(4) + }) + + it('can match stable major version for linux x64', async () => { + os.platform = 'linux' + os.arch = 'x64' + + const manifest: mm.IToolRelease[] | null = await tc.getManifestFromRepo( + owner, + repo, + fakeToken + ) + const release: tc.IToolRelease | undefined = await tc.findFromManifest( + '2.x', + true, + manifest + ) + expect(release).toBeDefined() + expect(release?.version).toBe('2.0.2') + expect(release?.files.length).toBe(1) + const file = release?.files[0] + expect(file).toBeDefined() + expect(file?.arch).toBe('x64') + expect(file?.platform).toBe('linux') + expect(file?.download_url).toBe( + 'https://github.com/actions/sometool/releases/tag/2.0.2-20200402.6/sometool-2.0.2-linux-x64.tar.gz' + ) + expect(file?.filename).toBe('sometool-2.0.2-linux-x64.tar.gz') + }) + + it('can match stable exact version for linux x64', async () => { + os.platform = 'linux' + os.arch = 'x64' + + const manifest: mm.IToolRelease[] | null = await tc.getManifestFromRepo( + owner, + repo, + fakeToken + ) + const release: tc.IToolRelease | undefined = await tc.findFromManifest( + '1.2.3', + true, + manifest + ) + expect(release).toBeDefined() + expect(release?.version).toBe('1.2.3') + expect(release?.files.length).toBe(1) + const file = release?.files[0] + expect(file).toBeDefined() + expect(file?.arch).toBe('x64') + expect(file?.platform).toBe('linux') + expect(file?.download_url).toBe( + 'https://github.com/actions/sometool/releases/tag/1.2.3-20200402.6/sometool-1.2.3-linux-x64.tar.gz' + ) + expect(file?.filename).toBe('sometool-1.2.3-linux-x64.tar.gz') + }) + + it('can match with linux platform version spec', async () => { + os.platform = 'linux' + os.arch = 'x64' + + readLsbSpy.mockImplementation(() => { + return `DISTRIB_ID=Ubuntu + DISTRIB_RELEASE=18.04 + DISTRIB_CODENAME=bionic + DISTRIB_DESCRIPTION=Ubuntu 18.04.4 LTS` + }) + + const manifest: mm.IToolRelease[] | null = await tc.getManifestFromRepo( + owner, + repo, + fakeToken + ) + const release: tc.IToolRelease | undefined = await tc.findFromManifest( + '1.2.4', + true, + manifest + ) + expect(release).toBeDefined() + expect(release?.version).toBe('1.2.4') + expect(release?.files.length).toBe(1) + const file = release?.files[0] + expect(file).toBeDefined() + expect(file?.arch).toBe('x64') + expect(file?.platform).toBe('linux') + expect(file?.download_url).toBe( + 'https://github.com/actions/sometool/releases/tag/1.2.4-20200402.6/sometool-1.2.4-ubuntu1804-x64.tar.gz' + ) + expect(file?.filename).toBe('sometool-1.2.4-ubuntu1804-x64.tar.gz') + }) + + it('can match with darwin platform version spec', async () => { + os.platform = 'darwin' + os.arch = 'x64' + + execSpy.mockImplementation(() => '10.15.1') + + const manifest: mm.IToolRelease[] | null = await tc.getManifestFromRepo( + owner, + repo, + fakeToken + ) + const release: tc.IToolRelease | undefined = await tc.findFromManifest( + '1.2.4', + true, + manifest + ) + expect(release).toBeDefined() + expect(release?.version).toBe('1.2.4') + expect(release?.files.length).toBe(1) + const file = release?.files[0] + expect(file).toBeDefined() + expect(file?.arch).toBe('x64') + expect(file?.platform).toBe('darwin') + expect(file?.download_url).toBe( + 'https://github.com/actions/sometool/releases/tag/1.2.4-20200402.6/sometool-1.2.4-darwin1015-x64.tar.gz' + ) + expect(file?.filename).toBe('sometool-1.2.4-darwin1015-x64.tar.gz') + }) + + it('does not match with unmatched linux platform version spec', async () => { + os.platform = 'linux' + os.arch = 'x64' + + readLsbSpy.mockImplementation(() => { + return `DISTRIB_ID=Ubuntu + DISTRIB_RELEASE=16.04 + DISTRIB_CODENAME=xenial + DISTRIB_DESCRIPTION=Ubuntu 16.04.4 LTS` + }) + + const manifest: mm.IToolRelease[] | null = await tc.getManifestFromRepo( + owner, + repo, + fakeToken + ) + const release: tc.IToolRelease | undefined = await tc.findFromManifest( + '1.2.4', + true, + manifest + ) + expect(release).toBeUndefined() + }) + + it('does not match with unmatched darwin platform version spec', async () => { + os.platform = 'darwin' + os.arch = 'x64' + + execSpy.mockImplementation(() => '10.14.6') + + const manifest: mm.IToolRelease[] | null = await tc.getManifestFromRepo( + owner, + repo, + fakeToken + ) + const release: tc.IToolRelease | undefined = await tc.findFromManifest( + '1.2.4', + true, + manifest + ) + expect(release).toBeUndefined() + }) + + it('can get version from lsb on ubuntu-18.04', async () => { + os.platform = 'linux' + os.arch = 'x64' + + //existsSpy.mockImplementation(() => true) + readLsbSpy.mockImplementation(() => { + return `DISTRIB_ID=Ubuntu + DISTRIB_RELEASE=18.04 + DISTRIB_CODENAME=bionic + DISTRIB_DESCRIPTION=Ubuntu 18.04.4 LTS` + }) + + const version = mm._getOsVersion() + + expect(osm.platform()).toBe('linux') + expect(version).toBe('18.04') + }) + + it('can get version from lsb on ubuntu-16.04', async () => { + os.platform = 'linux' + os.arch = 'x64' + + readLsbSpy.mockImplementation(() => { + return `DISTRIB_ID=Ubuntu + DISTRIB_RELEASE=16.04 + DISTRIB_CODENAME=xenial + DISTRIB_DESCRIPTION="Ubuntu 16.04.6 LTS"` + }) + + const version = mm._getOsVersion() + + expect(osm.platform()).toBe('linux') + expect(version).toBe('16.04') + }) + + // sw_vers -productVersion + it('can get version on macOS', async () => { + os.platform = 'darwin' + os.arch = 'x64' + + execSpy.mockImplementation(() => '10.14.6') + + const version = mm._getOsVersion() + + expect(osm.platform()).toBe('darwin') + expect(version).toBe('10.14.6') + }) +}) diff --git a/packages/tool-cache/package-lock.json b/packages/tool-cache/package-lock.json index 6aadb289..6be15c3d 100644 --- a/packages/tool-cache/package-lock.json +++ b/packages/tool-cache/package-lock.json @@ -1,6 +1,6 @@ { "name": "@actions/tool-cache", - "version": "1.3.5", + "version": "1.5.3", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/packages/tool-cache/package.json b/packages/tool-cache/package.json index 78b71778..e8923104 100644 --- a/packages/tool-cache/package.json +++ b/packages/tool-cache/package.json @@ -1,6 +1,6 @@ { "name": "@actions/tool-cache", - "version": "1.3.5", + "version": "1.5.4", "description": "Actions tool-cache lib", "keywords": [ "github", diff --git a/packages/tool-cache/src/manifest.ts b/packages/tool-cache/src/manifest.ts new file mode 100644 index 00000000..a85e771c --- /dev/null +++ b/packages/tool-cache/src/manifest.ts @@ -0,0 +1,158 @@ +import * as semver from 'semver' +import {debug} from '@actions/core' + +// needs to be require for core node modules to be mocked +/* eslint @typescript-eslint/no-require-imports: 0 */ + +import os = require('os') +import cp = require('child_process') +import fs = require('fs') + +/* +NOTE: versions must be sorted descending by version in the manifest + this library short circuits on first semver spec match + + platform_version is an optional filter and can be a semver spec or range +[ + { + "version": "1.2.3", + "stable": true, + "release_url": "https://github.com/actions/sometool/releases/tag/1.2.3-20200402.6", + "files": [ + { + "filename": "sometool-1.2.3-linux-x64.zip", + "arch": "x64", + "platform": "linux", + "platform_version": "18.04" + "download_url": "https://github.com/actions/sometool/releases/tag/1.2.3-20200402.6/sometool-1.2.3-linux-x64.zip" + }, + ... + ] + }, + ... +] +*/ + +export interface IToolReleaseFile { + filename: string + // 'aix', 'darwin', 'freebsd', 'linux', 'openbsd', + // 'sunos', and 'win32' + // platform_version is an optional semver filter + // TODO: do we need distribution (e.g. ubuntu). + // not adding yet but might need someday. + // right now, 16.04 and 18.04 work + platform: string + platform_version?: string + + // 'arm', 'arm64', 'ia32', 'mips', 'mipsel', + // 'ppc', 'ppc64', 's390', 's390x', + // 'x32', and 'x64'. + arch: string + + download_url: string +} + +export interface IToolRelease { + version: string + stable: boolean + release_url: string + files: IToolReleaseFile[] +} + +export async function _findMatch( + versionSpec: string, + stable: boolean, + candidates: IToolRelease[], + archFilter: string +): Promise { + const platFilter = os.platform() + + let result: IToolRelease | undefined + let match: IToolRelease | undefined + + let file: IToolReleaseFile | undefined + for (const candidate of candidates) { + const version = candidate.version + + debug(`check ${version} satisfies ${versionSpec}`) + if ( + semver.satisfies(version, versionSpec) && + (!stable || candidate.stable === stable) + ) { + file = candidate.files.find(item => { + debug( + `${item.arch}===${archFilter} && ${item.platform}===${platFilter}` + ) + + let chk = item.arch === archFilter && item.platform === platFilter + if (chk && item.platform_version) { + const osVersion = module.exports._getOsVersion() + + if (osVersion === item.platform_version) { + chk = true + } else { + chk = semver.satisfies(osVersion, item.platform_version) + } + } + + return chk + }) + + if (file) { + debug(`matched ${candidate.version}`) + match = candidate + break + } + } + } + + if (match && file) { + // clone since we're mutating the file list to be only the file that matches + result = Object.assign({}, match) + result.files = [file] + } + + return result +} + +export function _getOsVersion(): string { + // TODO: add windows and other linux, arm variants + // right now filtering on version is only an ubuntu and macos scenario for tools we build for hosted (python) + const plat = os.platform() + let version = '' + + if (plat === 'darwin') { + version = cp.execSync('sw_vers -productVersion').toString() + } else if (plat === 'linux') { + // lsb_release process not in some containers, readfile + // Run cat /etc/lsb-release + // DISTRIB_ID=Ubuntu + // DISTRIB_RELEASE=18.04 + // DISTRIB_CODENAME=bionic + // DISTRIB_DESCRIPTION="Ubuntu 18.04.4 LTS" + const lsbContents = module.exports._readLinuxVersionFile() + if (lsbContents) { + const lines = lsbContents.split('\n') + for (const line of lines) { + const parts = line.split('=') + if (parts.length === 2 && parts[0].trim() === 'DISTRIB_RELEASE') { + version = parts[1].trim() + break + } + } + } + } + + return version +} + +export function _readLinuxVersionFile(): string { + const lsbFile = '/etc/lsb-release' + let contents = '' + + if (fs.existsSync(lsbFile)) { + contents = fs.readFileSync(lsbFile).toString() + } + + return contents +} diff --git a/packages/tool-cache/src/tool-cache.ts b/packages/tool-cache/src/tool-cache.ts index 6fa2d510..385569ca 100644 --- a/packages/tool-cache/src/tool-cache.ts +++ b/packages/tool-cache/src/tool-cache.ts @@ -1,6 +1,7 @@ import * as core from '@actions/core' import * as io from '@actions/io' import * as fs from 'fs' +import * as mm from './manifest' import * as os from 'os' import * as path from 'path' import * as httpm from '@actions/http-client' @@ -12,6 +13,7 @@ import {exec} from '@actions/exec/lib/exec' import {ExecOptions} from '@actions/exec/lib/interfaces' import {ok} from 'assert' import {RetryHelper} from './retry-helper' +import {IHeaders} from '@actions/http-client/interfaces' export class HTTPError extends Error { constructor(readonly httpStatusCode: number | undefined) { @@ -28,11 +30,13 @@ const userAgent = 'actions/tool-cache' * * @param url url of tool to download * @param dest path to download tool + * @param auth authorization header * @returns path to downloaded tool */ export async function downloadTool( url: string, - dest?: string + dest?: string, + auth?: string ): Promise { dest = dest || path.join(_getTempDirectory(), uuidV4()) await io.mkdirP(path.dirname(dest)) @@ -51,7 +55,7 @@ export async function downloadTool( const retryHelper = new RetryHelper(maxAttempts, minSeconds, maxSeconds) return await retryHelper.execute( async () => { - return await downloadToolAttempt(url, dest || '') + return await downloadToolAttempt(url, dest || '', auth) }, (err: Error) => { if (err instanceof HTTPError && err.httpStatusCode) { @@ -71,7 +75,11 @@ export async function downloadTool( ) } -async function downloadToolAttempt(url: string, dest: string): Promise { +async function downloadToolAttempt( + url: string, + dest: string, + auth?: string +): Promise { if (fs.existsSync(dest)) { throw new Error(`Destination file path ${dest} already exists`) } @@ -80,7 +88,16 @@ async function downloadToolAttempt(url: string, dest: string): Promise { const http = new httpm.HttpClient(userAgent, [], { allowRetries: false }) - const response: httpm.HttpClientResponse = await http.get(url) + + let headers: IHeaders | undefined + if (auth) { + core.debug('set auth') + headers = { + authorization: auth + } + } + + const response: httpm.HttpClientResponse = await http.get(url, headers) if (response.message.statusCode !== 200) { const err = new HTTPError(response.message.statusCode) core.debug( @@ -202,7 +219,7 @@ export async function extract7z( export async function extractTar( file: string, dest?: string, - flags: string = 'xz' + flags: string | string[] = 'xz' ): Promise { if (!file) { throw new Error("parameter 'file' is required") @@ -226,7 +243,12 @@ export async function extractTar( const isGnuTar = versionOutput.toUpperCase().includes('GNU TAR') // Initialize args - const args = [flags] + let args: string[] + if (flags instanceof Array) { + args = flags + } else { + args = [flags] + } if (core.isDebug() && !flags.includes('v')) { args.push('-v') @@ -463,6 +485,92 @@ export function findAllVersions(toolName: string, arch?: string): string[] { return versions } +// versions-manifest +// +// typical pattern of a setup-* action that supports JIT would be: +// 1. resolve semver against local cache +// +// 2. if no match, download +// a. query versions manifest to match +// b. if no match, fall back to source if exists (tool distribution) +// c. with download url, download, install and preprent path + +export type IToolRelease = mm.IToolRelease +export type IToolReleaseFile = mm.IToolReleaseFile + +interface GitHubTreeItem { + path: string + size: string + url: string +} + +interface GitHubTree { + tree: GitHubTreeItem[] + truncated: boolean +} + +export async function getManifestFromRepo( + owner: string, + repo: string, + auth?: string, + branch = 'master' +): Promise { + let releases: IToolRelease[] = [] + const treeUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/${branch}` + + const http: httpm.HttpClient = new httpm.HttpClient('tool-cache') + const headers: IHeaders = {} + if (auth) { + core.debug('set auth') + headers.authorization = auth + } + + const response = await http.getJson(treeUrl, headers) + if (!response.result) { + return releases + } + + let manifestUrl = '' + for (const item of response.result.tree) { + if (item.path === 'versions-manifest.json') { + manifestUrl = item.url + break + } + } + + headers['accept'] = 'application/vnd.github.VERSION.raw' + let versionsRaw = await (await http.get(manifestUrl, headers)).readBody() + + if (versionsRaw) { + // shouldn't be needed but protects against invalid json saved with BOM + versionsRaw = versionsRaw.replace(/^\uFEFF/, '') + try { + releases = JSON.parse(versionsRaw) + } catch { + core.debug('Invalid json') + } + } + + return releases +} + +export async function findFromManifest( + versionSpec: string, + stable: boolean, + manifest: IToolRelease[], + archFilter: string = os.arch() +): Promise { + // wrap the internal impl + const match: mm.IToolRelease | undefined = await mm._findMatch( + versionSpec, + stable, + manifest, + archFilter + ) + + return match +} + async function _createExtractFolder(dest?: string): Promise { if (!dest) { // create a temp dir