From 39f7658b558ca5dca00b8f321d4aed85cd69b6e9 Mon Sep 17 00:00:00 2001 From: Nikolai Laevskii Date: Mon, 9 Oct 2023 06:35:51 +0200 Subject: [PATCH] Introduce universal archive extractor into @actions/tool-cache --- packages/tool-cache/README.md | 9 ++ packages/tool-cache/__tests__/archive.test.ts | 122 ++++++++++++++++++ .../tool-cache/__tests__/data/test.notarchive | 1 + packages/tool-cache/__tests__/data/test.zip | Bin 0 -> 182 bytes .../tool-cache/src/archive/archive-types.ts | 83 ++++++++++++ .../src/archive/get-archive-type.ts | 107 +++++++++++++++ packages/tool-cache/src/archive/index.ts | 12 ++ .../src/archive/retrieve-archive.ts | 39 ++++++ packages/tool-cache/src/archive/types.ts | 55 ++++++++ packages/tool-cache/src/tool-cache.ts | 5 + 10 files changed, 433 insertions(+) create mode 100644 packages/tool-cache/__tests__/archive.test.ts create mode 100644 packages/tool-cache/__tests__/data/test.notarchive create mode 100644 packages/tool-cache/__tests__/data/test.zip create mode 100644 packages/tool-cache/src/archive/archive-types.ts create mode 100644 packages/tool-cache/src/archive/get-archive-type.ts create mode 100644 packages/tool-cache/src/archive/index.ts create mode 100644 packages/tool-cache/src/archive/retrieve-archive.ts create mode 100644 packages/tool-cache/src/archive/types.ts diff --git a/packages/tool-cache/README.md b/packages/tool-cache/README.md index d3da0c35..619c98dd 100644 --- a/packages/tool-cache/README.md +++ b/packages/tool-cache/README.md @@ -39,6 +39,15 @@ else { } ``` +the Archive helper can be used as a shortcut for both download and extraction of an archive: + +```js +const {Archive} = require('@actions/tool-cache'); + +const node12Archive = await Archive.retrieve('https://nodejs.org/dist/v12.7.0/node-v12.7.0-linux-x64.tar.gz') +const node12ExtractedFolder = await node12Archive.extract('path/to/extract/to') +``` + #### Cache Finally, you can cache these directories in our tool-cache. This is useful if you want to switch back and forth between versions of a tool, or save a tool between runs for self-hosted runners. diff --git a/packages/tool-cache/__tests__/archive.test.ts b/packages/tool-cache/__tests__/archive.test.ts new file mode 100644 index 00000000..ac38a7b1 --- /dev/null +++ b/packages/tool-cache/__tests__/archive.test.ts @@ -0,0 +1,122 @@ +import {Archive} from '../src/tool-cache' +import * as tc from '../src/tool-cache' + +describe('archive-extractor', () => { + describe('getArchiveType', () => { + it('detects 7z', async () => { + const type = await Archive.getArchiveType( + require.resolve('./data/test.7z') + ) + expect(type).toEqual('7z') + }) + + it('detects tar', async () => { + const type1 = await Archive.getArchiveType( + require.resolve('./data/test.tar.gz') + ) + expect(type1).toEqual('tar') + + const type2 = await Archive.getArchiveType( + require.resolve('./data/test.tar.xz') + ) + + expect(type2).toEqual('tar') + }) + + it('detects zip', async () => { + const type = await Archive.getArchiveType( + require.resolve('./data/test.zip') + ) + expect(type).toEqual('zip') + }) + + it('throws on unsupported type', async () => { + await expect( + Archive.getArchiveType(require.resolve('./data/test.notarchive')) + ).rejects.toThrow('Unable to determine archive type') + }) + + it('throws on non-existent file', async () => { + await expect(Archive.getArchiveType('non-existent-file')).rejects.toThrow( + 'Unable to open non-existent-file' + ) + }) + }) + + describe('retrieveArchive', () => { + it('downloads archive', async () => { + const downloadToolSpy = jest.spyOn(tc, 'downloadTool') + + downloadToolSpy.mockImplementation(async () => + Promise.resolve('dummy-path') + ) + + await Archive.retrieve('https://test', { + type: 'zip', + downloadPath: 'dummy-download-path' + }) + + expect(downloadToolSpy).toHaveBeenLastCalledWith( + 'https://test', + 'dummy-download-path', + undefined, + undefined + ) + + downloadToolSpy.mockRestore() + }) + + it('extracts zip', async () => { + const extractZipSpy = jest.spyOn(tc, 'extractZip') + + extractZipSpy.mockImplementation(async () => + Promise.resolve('dummy-path') + ) + + const archive = new Archive.ZipArchive('dummy-path') + await archive.extract('dummy-dest') + + expect(extractZipSpy).toHaveBeenLastCalledWith('dummy-path', 'dummy-dest') + + extractZipSpy.mockRestore() + }) + + it('extracts tar', async () => { + const extractTarSpy = jest.spyOn(tc, 'extractTar') + + extractTarSpy.mockImplementation(async () => + Promise.resolve('dummy-path') + ) + + const archive = new Archive.TarArchive('dummy-path') + + await archive.extract('dummy-dest', ['flag1', 'flag2']) + + expect(extractTarSpy).toHaveBeenLastCalledWith( + 'dummy-path', + 'dummy-dest', + ['flag1', 'flag2'] + ) + + extractTarSpy.mockRestore() + }) + + it('extracts 7z', async () => { + const extract7zSpy = jest.spyOn(tc, 'extract7z') + + extract7zSpy.mockImplementation(async () => Promise.resolve('dummy-path')) + + const archive = new Archive.SevenZipArchive('dummy-path') + + await archive.extract('dummy-dest', 'dummy-7z-path') + + expect(extract7zSpy).toHaveBeenLastCalledWith( + 'dummy-path', + 'dummy-dest', + 'dummy-7z-path' + ) + + extract7zSpy.mockRestore() + }) + }) +}) diff --git a/packages/tool-cache/__tests__/data/test.notarchive b/packages/tool-cache/__tests__/data/test.notarchive new file mode 100644 index 00000000..6f4ddb6c --- /dev/null +++ b/packages/tool-cache/__tests__/data/test.notarchive @@ -0,0 +1 @@ +not an archive diff --git a/packages/tool-cache/__tests__/data/test.zip b/packages/tool-cache/__tests__/data/test.zip new file mode 100644 index 0000000000000000000000000000000000000000..330c123403908fdb46762d768b188c7f9baa9675 GIT binary patch literal 182 zcmWIWW@h1H00A})%kZ{aO~?6wY!K#TkYPwE&CRXUE2$_64dG;9-u+!5bq^4iR&X;g zvb +} + +export class ZipArchive extends ArchiveBase { + type: 'zip' = 'zip' + + constructor(public path: string) { + super(path) + } + + async extract(dest?: string): Promise { + return await extractZip(this.path, dest) + } +} + +export class TarArchive extends ArchiveBase { + type: 'tar' = 'tar' + + constructor(public path: string) { + super(path) + } + + async extract( + dest?: string, + flags?: string | string[] | undefined + ): Promise { + return await extractTar(this.path, dest, flags) + } +} + +export class SevenZipArchive extends ArchiveBase { + type: '7z' = '7z' + + constructor(public path: string) { + super(path) + } + + async extract(dest?: string, _7zPath?: string | undefined): Promise { + return await extract7z(this.path, dest, _7zPath) + } +} + +export class XarArchive extends ArchiveBase { + type: 'xar' = 'xar' + + constructor(public path: string) { + super(path) + } + + async extract( + dest?: string, + flags?: string | string[] | undefined + ): Promise { + return await extractXar(this.path, dest, flags) + } +} + +export type Archive = ZipArchive | TarArchive | SevenZipArchive | XarArchive + +// Helpers + +export const isZipArchive = (archive: Archive): archive is ZipArchive => + archive.type === 'zip' +export const isTarArchive = (archive: Archive): archive is TarArchive => + archive.type === 'tar' +export const isSevenZipArchive = ( + archive: Archive +): archive is SevenZipArchive => archive.type === '7z' +export const isXarArchive = (archive: Archive): archive is XarArchive => + archive.type === 'xar' diff --git a/packages/tool-cache/src/archive/get-archive-type.ts b/packages/tool-cache/src/archive/get-archive-type.ts new file mode 100644 index 00000000..a8b9bde6 --- /dev/null +++ b/packages/tool-cache/src/archive/get-archive-type.ts @@ -0,0 +1,107 @@ +import fs from 'fs' +import {ArchiveType} from './types' + +const MAX_READ_SIZE = 4096 +const MAX_CHUNK_SIZE = 1024 + +const SIGNATURES = { + zip: '504b0304', + gz: '1f8b08', + bz2: '425a68', + xz: 'fd377a585a00', + '7z': '377abcaf271c', + xar: '78617221', // 'xar!' in hex + tar: '7573746172' // 'ustar' in hex +} as const + +const getArchiveTypeFromBuffer = (buffer: Buffer): ArchiveType | null => { + for (const [type, signature] of Object.entries(SIGNATURES)) { + if (!buffer.toString('hex').includes(signature)) { + continue + } + + if (['bz2', 'gz', 'tar', 'xz'].includes(type)) { + return 'tar' + } + + return type as ArchiveType + } + + return null +} + +const readStreamFromDescriptor = (fd: number): fs.ReadStream => + fs.createReadStream('', { + fd, + start: 0, + end: MAX_READ_SIZE, + highWaterMark: MAX_CHUNK_SIZE + }) + +class LimitedArray { + private _array: T[] = [] + constructor(private maxLength: number) {} + push(item: T): void { + if (this._array.length >= this.maxLength) { + this._array.shift() + } + + this._array.push(item) + } + get array(): T[] { + return [...this._array] + } +} + +export const getArchiveType = async (filePath: string): Promise => + new Promise((resolve, reject) => + fs.open(filePath, 'r', (error, fd) => { + if (fd === undefined) { + reject(new Error(`Unable to open ${filePath}`)) + return + } + + if (error) { + fs.close(fd, () => reject(error)) + return + } + + const buffers = new LimitedArray(2) + const readStream = readStreamFromDescriptor(fd) + + const closeEverythingAndResolve = (result: ArchiveType): void => { + readStream.close(() => { + fs.close(fd, () => resolve(result as '7z' | 'zip' | 'xar' | 'tar')) + }) + readStream.push(null) + } + + const closeEverythingAndReject = (error?: Error): void => { + readStream.close(() => { + fs.close(fd, () => + reject( + error ?? Error(`Unable to determine archive type of ${filePath}`) + ) + ) + }) + readStream.push(null) + } + + setTimeout(closeEverythingAndReject, 100) + + readStream + .on('data', chunk => { + if (!(chunk instanceof Buffer)) return closeEverythingAndReject() + + buffers.push(chunk) + const type = getArchiveTypeFromBuffer(Buffer.concat(buffers.array)) + + if (type !== null) { + return closeEverythingAndResolve(type) + } + }) + .on('end', () => closeEverythingAndReject()) + .on('close', () => closeEverythingAndReject()) + .on('error', closeEverythingAndReject) + }) + ) diff --git a/packages/tool-cache/src/archive/index.ts b/packages/tool-cache/src/archive/index.ts new file mode 100644 index 00000000..7a8eab53 --- /dev/null +++ b/packages/tool-cache/src/archive/index.ts @@ -0,0 +1,12 @@ +export {retrieve} from './retrieve-archive' +export {getArchiveType} from './get-archive-type' +export { + ZipArchive, + TarArchive, + SevenZipArchive, + XarArchive, + isZipArchive, + isTarArchive, + isSevenZipArchive, + isXarArchive +} from './archive-types' diff --git a/packages/tool-cache/src/archive/retrieve-archive.ts b/packages/tool-cache/src/archive/retrieve-archive.ts new file mode 100644 index 00000000..2bfad930 --- /dev/null +++ b/packages/tool-cache/src/archive/retrieve-archive.ts @@ -0,0 +1,39 @@ +import { + ZipArchive, + TarArchive, + SevenZipArchive, + XarArchive +} from './archive-types' +import {downloadTool} from '../tool-cache' +import {getArchiveType} from './get-archive-type' +import {PredictTypeByOptions, RetrieveArchiveOptions} from './types' + +export const retrieve = async ( + url: string, + options?: O +): Promise> => { + const path = await downloadTool( + url, + options?.downloadPath, + options?.auth, + options?.headers + ) + + const archiveType = + options?.type === 'auto' || !options?.type + ? await getArchiveType(path) + : options.type + + switch (archiveType) { + case 'zip': + return new ZipArchive(path) as PredictTypeByOptions + case 'tar': + return new TarArchive(path) as PredictTypeByOptions + case '7z': + return new SevenZipArchive(path) as PredictTypeByOptions + case 'xar': + return new XarArchive(path) as PredictTypeByOptions + default: + throw new Error(`Unsupported archive type: ${archiveType}`) + } +} diff --git a/packages/tool-cache/src/archive/types.ts b/packages/tool-cache/src/archive/types.ts new file mode 100644 index 00000000..bb5fd3dd --- /dev/null +++ b/packages/tool-cache/src/archive/types.ts @@ -0,0 +1,55 @@ +import {OutgoingHttpHeaders} from 'http' +import { + SevenZipArchive, + TarArchive, + XarArchive, + ZipArchive +} from './archive-types' + +export type ArchiveType = 'zip' | 'tar' | '7z' | 'xar' + +interface RetrieveArchiveOptionsBase { + downloadPath?: string + type?: ArchiveType | 'auto' + auth?: string | undefined + headers?: OutgoingHttpHeaders | undefined +} + +export interface RetrieveZipArchiveOptions extends RetrieveArchiveOptionsBase { + type: 'zip' +} + +export interface RetrieveTarArchiveOptions extends RetrieveArchiveOptionsBase { + type: 'tar' +} + +export interface Retrieve7zArchiveOptions extends RetrieveArchiveOptionsBase { + type: '7z' +} + +export interface RetrieveXarArchiveOptions extends RetrieveArchiveOptionsBase { + type: 'xar' +} + +export interface RetrieveUnknownArchiveOptions + extends RetrieveArchiveOptionsBase { + type?: 'auto' +} + +export type RetrieveArchiveOptions = + | RetrieveZipArchiveOptions + | RetrieveTarArchiveOptions + | Retrieve7zArchiveOptions + | RetrieveXarArchiveOptions + | RetrieveUnknownArchiveOptions + +export type PredictTypeByOptions = + O extends RetrieveZipArchiveOptions + ? ZipArchive + : O extends Retrieve7zArchiveOptions + ? SevenZipArchive + : O extends RetrieveTarArchiveOptions + ? TarArchive + : O extends RetrieveXarArchiveOptions + ? XarArchive + : ZipArchive | TarArchive | SevenZipArchive | XarArchive diff --git a/packages/tool-cache/src/tool-cache.ts b/packages/tool-cache/src/tool-cache.ts index 694d1252..6619e4ed 100644 --- a/packages/tool-cache/src/tool-cache.ts +++ b/packages/tool-cache/src/tool-cache.ts @@ -775,3 +775,8 @@ function _getGlobal(key: string, defaultValue: T): T { function _unique(values: T[]): T[] { return Array.from(new Set(values)) } + +/** + * Archive helper exports + */ +export * as Archive from './archive'