From c5e1af5dc3cce694ff27128f2bd846c46a4557cf Mon Sep 17 00:00:00 2001 From: Thomas Boop <52323235+thboop@users.noreply.github.com> Date: Mon, 7 Jun 2021 14:26:00 -0400 Subject: [PATCH] Add HashFiles to the toolkit (#830) * add hash files to the toolkit --- packages/glob/__tests__/hash-files.test.ts | 126 ++++++++++++++++++ packages/glob/src/glob.ts | 20 +++ .../glob/src/internal-hash-file-options.ts | 12 ++ packages/glob/src/internal-hash-files.ts | 42 ++++++ 4 files changed, 200 insertions(+) create mode 100644 packages/glob/__tests__/hash-files.test.ts create mode 100644 packages/glob/src/internal-hash-file-options.ts create mode 100644 packages/glob/src/internal-hash-files.ts diff --git a/packages/glob/__tests__/hash-files.test.ts b/packages/glob/__tests__/hash-files.test.ts new file mode 100644 index 00000000..cb912ca0 --- /dev/null +++ b/packages/glob/__tests__/hash-files.test.ts @@ -0,0 +1,126 @@ +import * as io from '../../io/src/io' +import * as path from 'path' +import {hashFiles} from '../src/glob' +import {promises as fs} from 'fs' + +const IS_WINDOWS = process.platform === 'win32' + +/** + * These test focus on the ability of globber to find files + * and not on the pattern matching aspect + */ +describe('globber', () => { + beforeAll(async () => { + await io.rmRF(getTestTemp()) + }) + + it('basic hashfiles test', async () => { + const root = path.join(getTestTemp(), 'basic-hashfiles') + await fs.mkdir(path.join(root), {recursive: true}) + await fs.writeFile(path.join(root, 'test.txt'), 'test file content') + const hash = await hashFiles(`${root}/*`) + expect(hash).toEqual( + 'd8a411e8f8643821bed189e627ff57151918aa554c00c10b31c693ab2dded273' + ) + }) + + it('basic hashfiles no match should return empty string', async () => { + const root = path.join(getTestTemp(), 'empty-hashfiles') + const hash = await hashFiles(`${root}/*`) + expect(hash).toEqual('') + }) + + it('followSymbolicLinks defaults to true', async () => { + const root = path.join( + getTestTemp(), + 'defaults-to-follow-symbolic-links-true' + ) + await fs.mkdir(path.join(root, 'realdir'), {recursive: true}) + await fs.writeFile( + path.join(root, 'realdir', 'file.txt'), + 'test file content' + ) + await createSymlinkDir( + path.join(root, 'realdir'), + path.join(root, 'symDir') + ) + const testPath = path.join(root, `symDir`) + const hash = await hashFiles(testPath) + expect(hash).toEqual( + 'd8a411e8f8643821bed189e627ff57151918aa554c00c10b31c693ab2dded273' + ) + }) + + it('followSymbolicLinks set to true', async () => { + const root = path.join(getTestTemp(), 'set-to-true') + await fs.mkdir(path.join(root, 'realdir'), {recursive: true}) + await fs.writeFile(path.join(root, 'realdir', 'file'), 'test file content') + await createSymlinkDir( + path.join(root, 'realdir'), + path.join(root, 'symDir') + ) + const testPath = path.join(root, `symDir`) + const hash = await hashFiles(testPath, {followSymbolicLinks: true}) + expect(hash).toEqual( + 'd8a411e8f8643821bed189e627ff57151918aa554c00c10b31c693ab2dded273' + ) + }) + + it('followSymbolicLinks set to false', async () => { + // Create the following layout: + // + // /folder-a + // /folder-a/file + // /symDir -> /folder-a + const root = path.join(getTestTemp(), 'set-to-false') + await fs.mkdir(path.join(root, 'realdir'), {recursive: true}) + await fs.writeFile(path.join(root, 'realdir', 'file'), 'test file content') + await createSymlinkDir( + path.join(root, 'realdir'), + path.join(root, 'symDir') + ) + const testPath = path.join(root, 'symdir') + const hash = await hashFiles(testPath, {followSymbolicLinks: false}) + expect(hash).toEqual('') + }) + + it('multipath test basic', async () => { + // Create the following layout: + // + // /folder-a + // /folder-a/file + // /symDir -> /folder-a + const root = path.join(getTestTemp(), 'set-to-false') + await fs.mkdir(path.join(root, 'dir1'), {recursive: true}) + await fs.mkdir(path.join(root, 'dir2'), {recursive: true}) + await fs.writeFile( + path.join(root, 'dir1', 'testfile1.txt'), + 'test file content' + ) + await fs.writeFile( + path.join(root, 'dir2', 'testfile2.txt'), + 'test file content' + ) + const testPath = `${path.join(root, 'dir1')}\n${path.join(root, 'dir2')}` + const hash = await hashFiles(testPath) + expect(hash).toEqual( + '4e911ea5824830b6a3ec096c7833d5af8381c189ffaa825c3503a5333a73eadc' + ) + }) +}) + +function getTestTemp(): string { + return path.join(__dirname, '_temp', 'hash_files') +} + +/** + * Creates a symlink directory on OSX/Linux, and a junction point directory on Windows. + * A symlink directory is not created on Windows since it requires an elevated context. + */ +async function createSymlinkDir(real: string, link: string): Promise { + if (IS_WINDOWS) { + await fs.symlink(real, link, 'junction') + } else { + await fs.symlink(real, link) + } +} diff --git a/packages/glob/src/glob.ts b/packages/glob/src/glob.ts index 02fc9cc1..e1098885 100644 --- a/packages/glob/src/glob.ts +++ b/packages/glob/src/glob.ts @@ -1,5 +1,7 @@ import {Globber, DefaultGlobber} from './internal-globber' import {GlobOptions} from './internal-glob-options' +import {HashFileOptions} from './internal-hash-file-options' +import {hashFiles as _hashFiles} from './internal-hash-files' export {Globber, GlobOptions} @@ -15,3 +17,21 @@ export async function create( ): Promise { return await DefaultGlobber.create(patterns, options) } + +/** + * Computes the sha256 hash of a glob + * + * @param patterns Patterns separated by newlines + * @param options Glob options + */ +export async function hashFiles( + patterns: string, + options?: HashFileOptions +): Promise { + let followSymbolicLinks = true + if (options && typeof options.followSymbolicLinks === 'boolean') { + followSymbolicLinks = options.followSymbolicLinks + } + const globber = await create(patterns, {followSymbolicLinks}) + return _hashFiles(globber) +} diff --git a/packages/glob/src/internal-hash-file-options.ts b/packages/glob/src/internal-hash-file-options.ts new file mode 100644 index 00000000..00f9bb5f --- /dev/null +++ b/packages/glob/src/internal-hash-file-options.ts @@ -0,0 +1,12 @@ +/** + * Options to control globbing behavior + */ +export interface HashFileOptions { + /** + * Indicates whether to follow symbolic links. Generally should set to false + * when deleting files. + * + * @default true + */ + followSymbolicLinks?: boolean +} diff --git a/packages/glob/src/internal-hash-files.ts b/packages/glob/src/internal-hash-files.ts new file mode 100644 index 00000000..be9e3ba1 --- /dev/null +++ b/packages/glob/src/internal-hash-files.ts @@ -0,0 +1,42 @@ +import * as crypto from 'crypto' +import * as core from '@actions/core' +import * as fs from 'fs' +import * as stream from 'stream' +import * as util from 'util' +import * as path from 'path' +import {Globber} from './glob' + +export async function hashFiles(globber: Globber): Promise { + let hasMatch = false + const githubWorkspace = process.env['GITHUB_WORKSPACE'] ?? process.cwd() + const result = crypto.createHash('sha256') + let count = 0 + for await (const file of globber.globGenerator()) { + core.debug(file) + if (!file.startsWith(`${githubWorkspace}${path.sep}`)) { + core.debug(`Ignore '${file}' since it is not under GITHUB_WORKSPACE.`) + continue + } + if (fs.statSync(file).isDirectory()) { + core.debug(`Skip directory '${file}'.`) + continue + } + const hash = crypto.createHash('sha256') + const pipeline = util.promisify(stream.pipeline) + await pipeline(fs.createReadStream(file), hash) + result.write(hash.digest()) + count++ + if (!hasMatch) { + hasMatch = true + } + } + result.end() + + if (hasMatch) { + core.debug(`Found ${count} files to hash.`) + return result.digest('hex') + } else { + core.warning(`No matches found for glob`) + return '' + } +}