From 1a2c5929039617047c4b37b17b1c4cd6de0b8333 Mon Sep 17 00:00:00 2001 From: eric sciple Date: Thu, 9 Jan 2020 15:05:31 -0500 Subject: [PATCH] multiple glob patterns (#287) --- ...{glob.test.ts => internal-globber.test.ts} | 270 ++++++++++++++---- .../__tests__/internal-pattern-helper.test.ts | 124 +++----- packages/glob/src/glob.ts | 188 +----------- .../glob/src/internal-glob-options-helper.ts | 32 +++ packages/glob/src/internal-glob-options.ts | 9 +- packages/glob/src/internal-globber.ts | 242 ++++++++++++++++ packages/glob/src/internal-pattern-helper.ts | 62 ---- 7 files changed, 547 insertions(+), 380 deletions(-) rename packages/glob/__tests__/{glob.test.ts => internal-globber.test.ts} (69%) create mode 100644 packages/glob/src/internal-glob-options-helper.ts create mode 100644 packages/glob/src/internal-globber.ts diff --git a/packages/glob/__tests__/glob.test.ts b/packages/glob/__tests__/internal-globber.test.ts similarity index 69% rename from packages/glob/__tests__/glob.test.ts rename to packages/glob/__tests__/internal-globber.test.ts index 810117ae..feba4433 100644 --- a/packages/glob/__tests__/glob.test.ts +++ b/packages/glob/__tests__/internal-globber.test.ts @@ -1,36 +1,143 @@ import * as child from 'child_process' -import * as glob from '../src/glob' import * as io from '../../io/src/io' +import * as os from 'os' import * as path from 'path' +import {Globber, DefaultGlobber} from '../src/internal-globber' +import {GlobOptions} from '../src/internal-glob-options' import {promises as fs} from 'fs' const IS_WINDOWS = process.platform === 'win32' /** - * These test focus on the ability of glob to find files + * These test focus on the ability of globber to find files * and not on the pattern matching aspect */ -describe('glob', () => { +describe('globber', () => { beforeAll(async () => { await io.rmRF(getTestTemp()) }) - it('detects cycle', async () => { + it('captures cwd', async () => { + // Create the following layout: + // first-cwd + // first-cwd/the-correct-file + // second-cwd + // second-cwd/the-wrong-file + const root = path.join(getTestTemp(), 'preserves-cwd') + await fs.mkdir(path.join(root, 'first-cwd'), {recursive: true}) + await fs.writeFile( + path.join(root, 'first-cwd', 'the-correct-file.txt'), + 'test file content' + ) + await fs.mkdir(path.join(root, 'second-cwd'), {recursive: true}) + await fs.writeFile( + path.join(root, 'second-cwd', 'the-wrong-file.txt'), + 'test file content' + ) + + const originalCwd = process.cwd() + try { + process.chdir(path.join(root, 'first-cwd')) + const globber = await DefaultGlobber.create('*') + process.chdir(path.join(root, 'second-cwd')) + expect(globber.getSearchPaths()).toEqual([path.join(root, 'first-cwd')]) + const itemPaths = await globber.glob() + expect(itemPaths).toEqual([ + path.join(root, 'first-cwd', 'the-correct-file.txt') + ]) + } finally { + process.chdir(originalCwd) + } + }) + + it('defaults to followSymbolicLinks=true', async () => { + // Create the following layout: + // + // /folder-a + // /folder-a/file + // /symDir -> /folder-a + const root = path.join( + getTestTemp(), + 'defaults-to-follow-symbolic-links-true' + ) + await fs.mkdir(path.join(root, 'folder-a'), {recursive: true}) + await fs.writeFile(path.join(root, 'folder-a', 'file'), 'test file content') + await createSymlinkDir( + path.join(root, 'folder-a'), + path.join(root, 'symDir') + ) + + const itemPaths = await glob(root, {}) + expect(itemPaths).toEqual([ + root, + path.join(root, 'folder-a'), + path.join(root, 'folder-a', 'file'), + path.join(root, 'symDir'), + path.join(root, 'symDir', 'file') + ]) + }) + + it('defaults to implicitDescendants=true', async () => { + // Create the following layout: + // + // /folder-a + // /folder-a/file + const root = path.join( + getTestTemp(), + 'defaults-to-implicit-descendants-true' + ) + await fs.mkdir(path.join(root, 'folder-a'), {recursive: true}) + await fs.writeFile(path.join(root, 'folder-a', 'file'), 'test file content') + + const itemPaths = await glob(root, {}) + expect(itemPaths).toEqual([ + root, + path.join(root, 'folder-a'), + path.join(root, 'folder-a', 'file') + ]) + }) + + it('defaults to omitBrokenSymbolicLinks=true', async () => { + // Create the following layout: + // + // /folder-a + // /folder-a/file + // /symDir -> /no-such + const root = path.join( + getTestTemp(), + 'defaults-to-omit-broken-symbolic-links-true' + ) + await fs.mkdir(path.join(root, 'folder-a'), {recursive: true}) + await fs.writeFile(path.join(root, 'folder-a', 'file'), 'test file content') + await createSymlinkDir( + path.join(root, 'no-such'), + path.join(root, 'symDir') + ) + + const itemPaths = await glob(root, {}) + expect(itemPaths).toEqual([ + root, + path.join(root, 'folder-a'), + path.join(root, 'folder-a', 'file') + ]) + }) + + it('detects cycle when followSymbolicLinks=true', async () => { // Create the following layout: // // /file // /symDir -> - const root = path.join(getTestTemp(), 'detects-cycle') + const root = path.join(getTestTemp(), 'detects-cycle-when-follow-true') await fs.mkdir(root, {recursive: true}) await fs.writeFile(path.join(root, 'file'), 'test file content') await createSymlinkDir(root, path.join(root, 'symDir')) - const itemPaths = await glob.glob(root) + const itemPaths = await glob(root, {followSymbolicLinks: true}) expect(itemPaths).toEqual([root, path.join(root, 'file')]) // todo: ? expect(itemPaths[2]).toBe(path.join(root, 'symDir')) }) - it('detects deep cycle starting from middle', async () => { + it('detects deep cycle starting from middle when followSymbolicLinks=true', async () => { // Create the following layout: // // /file-under-root @@ -43,7 +150,7 @@ describe('glob', () => { // /folder-a/folder-b/folder-c/sym-folder -> const root = path.join( getTestTemp(), - 'detects-deep-cycle-starting-from-middle' + 'detects-deep-cycle-starting-from-middle-when-follow-true' ) await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-c'), { recursive: true @@ -79,7 +186,9 @@ describe('glob', () => { ) ) - const itemPaths = await glob.glob(path.join(root, 'folder-a', 'folder-b')) + const itemPaths = await glob(path.join(root, 'folder-a', 'folder-b'), { + followSymbolicLinks: true + }) expect(itemPaths).toEqual([ path.join(root, 'folder-a', 'folder-b'), path.join(root, 'folder-a', 'folder-b', 'file-under-b'), @@ -114,21 +223,23 @@ describe('glob', () => { ]) }) - it('detects cycle starting from symlink', async () => { + it('detects cycle starting from symlink when followSymbolicLinks=true', async () => { // Create the following layout: // // /file // /symDir -> const root: string = path.join( getTestTemp(), - 'detects-cycle-starting-from-symlink' + 'detects-cycle-starting-from-symlink-when-follow-true' ) await fs.mkdir(root, {recursive: true}) await fs.writeFile(path.join(root, 'file'), 'test file content') await createSymlinkDir(root, path.join(root, 'symDir')) await fs.stat(path.join(root, 'symDir')) - const itemPaths = await glob.glob(path.join(root, 'symDir')) + const itemPaths = await glob(path.join(root, 'symDir'), { + followSymbolicLinks: true + }) expect(itemPaths).toEqual([ path.join(root, 'symDir'), path.join(root, 'symDir', 'file') @@ -136,7 +247,7 @@ describe('glob', () => { // todo: ? expect(itemPaths[2]).toBe(path.join(root, 'symDir', 'symDir')); }) - it('does not follow symlink when followSymbolicLink=false', async () => { + it('does not follow symlink when followSymbolicLinks=false', async () => { // Create the following layout: // // /realDir @@ -153,7 +264,7 @@ describe('glob', () => { path.join(root, 'symDir') ) - const itemPaths = await glob.glob(root, {followSymbolicLinks: false}) + const itemPaths = await glob(root, {followSymbolicLinks: false}) expect(itemPaths).toEqual([ root, path.join(root, 'realDir'), @@ -162,7 +273,7 @@ describe('glob', () => { ]) }) - it('does not follow symlink when search path is symlink and followSymbolicLink=false', async () => { + it('does not follow symlink when search path is symlink and followSymbolicLinks=false', async () => { // Create the following layout: // realDir // realDir/file @@ -178,20 +289,23 @@ describe('glob', () => { path.join(root, 'symDir') ) - const itemPaths = await glob.glob(path.join(root, 'symDir'), { + const itemPaths = await glob(path.join(root, 'symDir'), { followSymbolicLinks: false }) expect(itemPaths).toEqual([path.join(root, 'symDir')]) }) - it('does not return broken symlink', async () => { + it('does not return broken symlink when follow-true and omit-true', async () => { // Create the following layout: // // /brokenSym -> /noSuch // /realDir // /realDir/file // /symDir -> /realDir - const root = path.join(getTestTemp(), 'does-not-return-broken-symlink') + const root = path.join( + getTestTemp(), + 'does-not-return-broken-symlink-when-follow-true-and-omit-true' + ) await fs.mkdir(root, {recursive: true}) await createSymlinkDir( path.join(root, 'noSuch'), @@ -204,7 +318,7 @@ describe('glob', () => { path.join(root, 'symDir') ) - const itemPaths = await glob.glob(root) + const itemPaths = await glob(root, {followSymbolicLinks: true}) expect(itemPaths).toEqual([ root, path.join(root, 'realDir'), @@ -214,20 +328,20 @@ describe('glob', () => { ]) }) - it('does not return broken symlink when search path is broken symlink', async () => { + it('does not return broken symlink when search path is broken symlink and followSymbolicLinks=true', async () => { // Create the following layout: // // /brokenSym -> /noSuch const root = path.join( getTestTemp(), - 'does-not-return-broken-symlink-when-search-path-is-broken-symlink' + 'does-not-return-broken-symlink-when-search-path-is-broken-symlink-and-follow-true' ) await fs.mkdir(root, {recursive: true}) const brokenSymPath = path.join(root, 'brokenSym') await createSymlinkDir(path.join(root, 'noSuch'), brokenSymPath) await fs.lstat(brokenSymPath) - const itemPaths = await glob.glob(brokenSymPath) + const itemPaths = await glob(brokenSymPath, {followSymbolicLinks: true}) expect(itemPaths).toEqual([]) }) @@ -255,29 +369,29 @@ describe('glob', () => { path.join(root, 'realDir2', 'nested2', 'symDir') ) - const options: glob.IGlobOptions = { + const options: GlobOptions = { followSymbolicLinks: true, omitBrokenSymbolicLinks: false } // Should throw try { - await glob.glob(`${root}/*Dir*/*nested*/*`, options) + await glob(`${root}/*Dir*/*nested*/*`, options) throw new Error('should not reach here') } catch (err) { expect(err.message).toMatch(/broken symbolic link/i) } // Not partial match - let itemPaths = await glob.glob(`${root}/*Dir/*nested*/*`, options) + let itemPaths = await glob(`${root}/*Dir/*nested*/*`, options) expect(itemPaths).toEqual([path.join(root, 'realDir', 'nested', 'file')]) // Not partial match - itemPaths = await glob.glob(`${root}/*Dir*/*nested/*`, options) + itemPaths = await glob(`${root}/*Dir*/*nested/*`, options) expect(itemPaths).toEqual([path.join(root, 'realDir', 'nested', 'file')]) }) - it('does not throw for broken symlinks that are not matches or partial matches', async () => { + it('does not throw for broken symlinks that are not matches or partial matches when followSymbolicLinks=true and omitBrokenSymbolicLinks=false', async () => { // Create the following layout: // // /realDir @@ -285,20 +399,20 @@ describe('glob', () => { // /symDir -> /noSuch const root = path.join( getTestTemp(), - 'does-not-throw-for-broken-symlinks-that-are-not-matches-or-partial-matches' + 'does-not-throw-for-broken-symlinks-that-are-not-matches-or-partial-matches-when-follow-true-and-omit-false' ) await fs.mkdir(path.join(root, 'realDir'), {recursive: true}) await fs.writeFile(path.join(root, 'realDir', 'file'), 'test file content') await createSymlinkDir(path.join(root, 'noSuch'), path.join(root, 'symDir')) - const options: glob.IGlobOptions = { + const options: GlobOptions = { followSymbolicLinks: true, omitBrokenSymbolicLinks: false } // Match should throw try { - await glob.glob(`${root}/*`, options) + await glob(`${root}/*`, options) throw new Error('should not reach here') } catch (err) { expect(err.message).toMatch(/broken symbolic link/i) @@ -306,18 +420,18 @@ describe('glob', () => { // Partial match should throw try { - await glob.glob(`${root}/*/*`, options) + await glob(`${root}/*/*`, options) throw new Error('should not reach here') } catch (err) { expect(err.message).toMatch(/broken symbolic link/i) } // Not match or partial match - const itemPaths = await glob.glob(`${root}/*eal*/*`, options) + const itemPaths = await glob(`${root}/*eal*/*`, options) expect(itemPaths).toEqual([path.join(root, 'realDir', 'file')]) }) - it('follows symlink', async () => { + it('follows symlink when follow-symbolic-links=true', async () => { // Create the following layout: // // /realDir @@ -331,7 +445,7 @@ describe('glob', () => { path.join(root, 'symDir') ) - const itemPaths = await glob.glob(root) + const itemPaths = await glob(root, {followSymbolicLinks: true}) expect(itemPaths).toEqual([ root, path.join(root, 'realDir'), @@ -341,14 +455,14 @@ describe('glob', () => { ]) }) - it('follows symlink when search path is symlink', async () => { + it('follows symlink when search path is symlink and follow-symbolic-links=true', async () => { // Create the following layout: // realDir // realDir/file // symDir -> realDir const root = path.join( getTestTemp(), - 'follows-symlink-when-search-path-is-symlink' + 'follows-symlink-when-search-path-is-symlink-and-follow-true' ) await fs.mkdir(path.join(root, 'realDir'), {recursive: true}) await fs.writeFile(path.join(root, 'realDir', 'file'), 'test file content') @@ -357,7 +471,9 @@ describe('glob', () => { path.join(root, 'symDir') ) - const itemPaths = await glob.glob(path.join(root, 'symDir')) + const itemPaths = await glob(path.join(root, 'symDir'), { + followSymbolicLinks: true + }) expect(itemPaths).toEqual([ path.join(root, 'symDir'), path.join(root, 'symDir', 'file') @@ -387,7 +503,7 @@ describe('glob', () => { path.join(root, 'symDir') ) - const itemPaths = await glob.glob(root, {followSymbolicLinks: false}) + const itemPaths = await glob(root, {followSymbolicLinks: false}) expect(itemPaths).toEqual([ root, path.join(root, 'brokenSym'), @@ -409,9 +525,7 @@ describe('glob', () => { const brokenSymPath = path.join(root, 'brokenSym') await createSymlinkDir(path.join(root, 'noSuch'), brokenSymPath) - const itemPaths = await glob.glob(brokenSymPath, { - followSymbolicLinks: false - }) + const itemPaths = await glob(brokenSymPath, {followSymbolicLinks: false}) expect(itemPaths).toEqual([brokenSymPath]) }) @@ -441,7 +555,7 @@ describe('glob', () => { ) await fs.writeFile(path.join(root, 'c-file'), 'test c-file content') - const itemPaths = await glob.glob(root) + const itemPaths = await glob(root) expect(itemPaths).toEqual([ root, path.join(root, 'a-file'), @@ -470,11 +584,11 @@ describe('glob', () => { // When pattern ends with `/**/` let pattern = `${root}${path.sep}**${path.sep}` expect( - await glob.glob(pattern, { + await glob(pattern, { implicitDescendants: false }) ).toHaveLength(3) // sanity check - expect(await glob.glob(pattern)).toEqual([ + expect(await glob(pattern)).toEqual([ root, path.join(root, 'dir-1'), path.join(root, 'dir-1', 'dir-2'), @@ -486,11 +600,11 @@ describe('glob', () => { // When pattern ends with something other than `/**/` pattern = `${root}${path.sep}**${path.sep}dir-?` expect( - await glob.glob(pattern, { + await glob(pattern, { implicitDescendants: false }) ).toHaveLength(2) // sanity check - expect(await glob.glob(pattern)).toEqual([ + expect(await glob(pattern)).toEqual([ path.join(root, 'dir-1'), path.join(root, 'dir-1', 'dir-2'), path.join(root, 'dir-1', 'dir-2', 'file-3'), @@ -515,9 +629,9 @@ describe('glob', () => { await fs.writeFile(path.join(root, 'dir-1', 'dir-2', 'file-3'), '') const pattern = `${root}${path.sep}**${path.sep}` - expect(await glob.glob(pattern)).toHaveLength(6) // sanity check + expect(await glob(pattern)).toHaveLength(6) // sanity check expect( - await glob.glob(pattern, { + await glob(pattern, { implicitDescendants: false }) ).toEqual([ @@ -528,7 +642,7 @@ describe('glob', () => { }) it('returns empty when search path does not exist', async () => { - const itemPaths = await glob.glob(path.join(getTestTemp(), 'nosuch')) + const itemPaths = await glob(path.join(getTestTemp(), 'nosuch')) expect(itemPaths).toEqual([]) }) @@ -548,7 +662,7 @@ describe('glob', () => { 'test .folder/file content' ) - const itemPaths = await glob.glob(root) + const itemPaths = await glob(root) expect(itemPaths).toEqual([ root, path.join(root, '.emptyFolder'), @@ -565,7 +679,7 @@ describe('glob', () => { await fs.mkdir(path.join(root, 'hello'), {recursive: true}) await fs.writeFile(path.join(root, 'hello', 'world.txt'), '') - const itemPaths = await glob.glob( + const itemPaths = await glob( `${root}${path.sep}${path.sep}${path.sep}hello` ) expect(itemPaths).toEqual([ @@ -574,13 +688,36 @@ describe('glob', () => { ]) }) - it('throws when match broken symlink and omitBrokenSymbolicLinks=false', async () => { + it('skips comments', async () => { + const searchPaths = await getSearchPaths( + `#aaa/*${os.EOL}/foo/*${os.EOL}#bbb/*${os.EOL}/bar/*` + ) + const drive = IS_WINDOWS ? process.cwd().substr(0, 2) : '' + expect(searchPaths).toEqual([ + IS_WINDOWS ? `${drive}\\foo` : '/foo', + IS_WINDOWS ? `${drive}\\bar` : '/bar' + ]) + }) + + it('skips empty lines', async () => { + const searchPaths = await getSearchPaths( + `${os.EOL}${os.EOL}/foo/*${os.EOL}${os.EOL}/bar/*${os.EOL}/baz/**${os.EOL}` + ) + const drive = IS_WINDOWS ? process.cwd().substr(0, 2) : '' + expect(searchPaths).toEqual([ + IS_WINDOWS ? `${drive}\\foo` : '/foo', + IS_WINDOWS ? `${drive}\\bar` : '/bar', + IS_WINDOWS ? `${drive}\\baz` : '/baz' + ]) + }) + + it('throws when match broken symlink and followSymbolicLinks=true and omitBrokenSymbolicLinks=false', async () => { // Create the following layout: // // /brokenSym -> /noSuch const root = path.join( getTestTemp(), - 'throws-when-match-broken-symlink-and-omit-false' + 'throws-when-match-broken-symlink-and-follow-true-and-omit-false' ) await fs.mkdir(root, {recursive: true}) await createSymlinkDir( @@ -589,20 +726,23 @@ describe('glob', () => { ) try { - await glob.glob(root, {omitBrokenSymbolicLinks: false}) + await glob(root, { + followSymbolicLinks: true, + omitBrokenSymbolicLinks: false + }) throw new Error('Expected tl.find to throw') } catch (err) { expect(err.message).toMatch(/broken symbolic link/) } }) - it('throws when search path is broken symlink and omitBrokenSymbolicLinks=false', async () => { + it('throws when search path is broken symlink and followSymbolicLinks=true and omitBrokenSymbolicLinks=false', async () => { // Create the following layout: // // /brokenSym -> /noSuch const root = path.join( getTestTemp(), - 'throws-when-search-path-is-broken-symlink-and-omit-false' + 'throws-when-search-path-is-broken-symlink-and-follow-true-and-omit-false' ) await fs.mkdir(root, {recursive: true}) const brokenSymPath = path.join(root, 'brokenSym') @@ -610,7 +750,10 @@ describe('glob', () => { await fs.lstat(brokenSymPath) try { - await glob.glob(brokenSymPath, {omitBrokenSymbolicLinks: false}) + await glob(brokenSymPath, { + followSymbolicLinks: true, + omitBrokenSymbolicLinks: false + }) throw new Error('Expected tl.find to throw') } catch (err) { expect(err.message).toMatch(/broken symbolic link/) @@ -669,3 +812,16 @@ async function createSymlinkDir(real: string, link: string): Promise { await fs.symlink(real, link) } } + +async function getSearchPaths(patterns: string): Promise { + const globber: Globber = await DefaultGlobber.create(patterns) + return globber.getSearchPaths() +} + +async function glob( + patterns: string, + options?: GlobOptions +): Promise { + const globber: Globber = await DefaultGlobber.create(patterns, options) + return await globber.glob() +} diff --git a/packages/glob/__tests__/internal-pattern-helper.test.ts b/packages/glob/__tests__/internal-pattern-helper.test.ts index dc8b0de9..1d60f6b3 100644 --- a/packages/glob/__tests__/internal-pattern-helper.test.ts +++ b/packages/glob/__tests__/internal-pattern-helper.test.ts @@ -2,18 +2,16 @@ import * as path from 'path' import * as patternHelper from '../src/internal-pattern-helper' import {MatchKind} from '../src/internal-match-kind' import {IS_WINDOWS} from '../../io/src/io-util' +import {Pattern} from '../src/internal-pattern' describe('pattern-helper', () => { it('getSearchPaths omits negate search paths', () => { const root = IS_WINDOWS ? 'C:\\' : '/' - const patterns = patternHelper.parse( - [ - `${root}search1/foo/**`, - `${root}search2/bar/**`, - `!${root}search3/baz/**` - ], - patternHelper.getOptions() - ) + const patterns = [ + `${root}search1/foo/**`, + `${root}search2/bar/**`, + `!${root}search3/baz/**` + ].map(x => new Pattern(x)) const searchPaths = patternHelper.getSearchPaths(patterns) expect(searchPaths).toEqual([ `${root}search1${path.sep}foo`, @@ -23,17 +21,14 @@ describe('pattern-helper', () => { it('getSearchPaths omits search path when ancestor is also a search path', () => { if (IS_WINDOWS) { - const patterns = patternHelper.parse( - [ - 'C:\\Search1\\Foo\\**', - 'C:\\sEARCH1\\fOO\\bar\\**', - 'C:\\sEARCH1\\foo\\bar', - 'C:\\Search2\\**', - 'C:\\Search3\\Foo\\Bar\\**', - 'C:\\sEARCH3\\fOO\\bAR\\**' - ], - patternHelper.getOptions() - ) + const patterns = [ + 'C:\\Search1\\Foo\\**', + 'C:\\sEARCH1\\fOO\\bar\\**', + 'C:\\sEARCH1\\foo\\bar', + 'C:\\Search2\\**', + 'C:\\Search3\\Foo\\Bar\\**', + 'C:\\sEARCH3\\fOO\\bAR\\**' + ].map(x => new Pattern(x)) const searchPaths = patternHelper.getSearchPaths(patterns) expect(searchPaths).toEqual([ 'C:\\Search1\\Foo', @@ -41,17 +36,15 @@ describe('pattern-helper', () => { 'C:\\Search3\\Foo\\Bar' ]) } else { - const patterns = patternHelper.parse( - [ - '/search1/foo/**', - '/search1/foo/bar/**', - '/search2/foo/bar', - '/search2/**', - '/search3/foo/bar/**', - '/search3/foo/bar/**' - ], - patternHelper.getOptions() - ) + const patterns = [ + '/search1/foo/**', + '/search1/foo/bar/**', + '/search2/foo/bar', + '/search2/**', + '/search3/foo/bar/**', + '/search3/foo/bar/**' + ].map(x => new Pattern(x)) + const searchPaths = patternHelper.getSearchPaths(patterns) expect(searchPaths).toEqual([ '/search1/foo', @@ -75,16 +68,13 @@ describe('pattern-helper', () => { `${root}solution2/proj2/README.txt`, `${root}solution2/solution2.sln` ] - const patterns = patternHelper.parse( - [ - `${root}**/*.proj`, // include all proj files - `${root}**/README.txt`, // include all README files - `!${root}**/solution2/**`, // exclude the solution 2 folder entirely - `${root}**/*.sln`, // include all sln files - `!${root}**/proj2/README.txt` // exclude proj2 README files - ], - patternHelper.getOptions({implicitDescendants: false}) - ) + const patterns = [ + `${root}**/*.proj`, // include all proj files + `${root}**/README.txt`, // include all README files + `!${root}**/solution2/**`, // exclude the solution 2 folder entirely + `${root}**/*.sln`, // include all sln files + `!${root}**/proj2/README.txt` // exclude proj2 README files + ].map(x => new Pattern(x)) const matched = itemPaths.filter( x => patternHelper.match(patterns, x) === MatchKind.All ) @@ -105,13 +95,10 @@ describe('pattern-helper', () => { `${root}foo/bar`, `${root}foo/bar/baz` ] - const patterns = patternHelper.parse( - [ - `${root}foo/**`, // include all files and directories - `!${root}foo/**/` // exclude directories - ], - patternHelper.getOptions({implicitDescendants: false}) - ) + const patterns = [ + `${root}foo/**`, // include all files and directories + `!${root}foo/**/` // exclude directories + ].map(x => new Pattern(x)) const matchKinds = itemPaths.map(x => patternHelper.match(patterns, x)) expect(matchKinds).toEqual([ MatchKind.None, @@ -129,12 +116,9 @@ describe('pattern-helper', () => { `${root}foo/bar`, `${root}foo/bar/baz` ] - const patterns = patternHelper.parse( - [ - `${root}foo/**/` // include directories only - ], - patternHelper.getOptions({implicitDescendants: false}) - ) + const patterns = [ + `${root}foo/**/` // include directories only + ].map(x => new Pattern(x)) const matchKinds = itemPaths.map(x => patternHelper.match(patterns, x)) expect(matchKinds).toEqual([ MatchKind.None, @@ -144,36 +128,14 @@ describe('pattern-helper', () => { ]) }) - it('parse skips comments', () => { - const patterns = patternHelper.parse( - ['# comment 1', ' # comment 2', '!#hello-world.txt'], - patternHelper.getOptions({implicitDescendants: false}) - ) - expect(patterns).toHaveLength(1) - expect(patterns[0].negate).toBeTruthy() - expect(patterns[0].segments.reverse()[0]).toEqual('#hello-world.txt') - }) - - it('parse skips empty patterns', () => { - const patterns = patternHelper.parse( - ['', ' ', 'hello-world.txt'], - patternHelper.getOptions({implicitDescendants: false}) - ) - expect(patterns).toHaveLength(1) - expect(patterns[0].segments.reverse()[0]).toEqual('hello-world.txt') - }) - it('partialMatch skips negate patterns', () => { const root = IS_WINDOWS ? 'C:\\' : '/' - const patterns = patternHelper.parse( - [ - `${root}search1/foo/**`, - `${root}search2/bar/**`, - `!${root}search2/bar/**`, - `!${root}search3/baz/**` - ], - patternHelper.getOptions({implicitDescendants: false}) - ) + const patterns = [ + `${root}search1/foo/**`, + `${root}search2/bar/**`, + `!${root}search2/bar/**`, + `!${root}search3/baz/**` + ].map(x => new Pattern(x)) expect(patternHelper.partialMatch(patterns, `${root}search1`)).toBeTruthy() expect( patternHelper.partialMatch(patterns, `${root}search1/foo`) diff --git a/packages/glob/src/glob.ts b/packages/glob/src/glob.ts index 861e1e8f..02fc9cc1 100644 --- a/packages/glob/src/glob.ts +++ b/packages/glob/src/glob.ts @@ -1,183 +1,17 @@ -import * as core from '@actions/core' -import * as fs from 'fs' -import * as path from 'path' -import * as patternHelper from './internal-pattern-helper' -import {IGlobOptions} from './internal-glob-options' -import {MatchKind} from './internal-match-kind' -import {Pattern} from './internal-pattern' -import {SearchState} from './internal-search-state' +import {Globber, DefaultGlobber} from './internal-globber' +import {GlobOptions} from './internal-glob-options' -export {IGlobOptions} +export {Globber, GlobOptions} /** - * Returns files and directories matching the specified glob pattern. + * Constructs a globber * - * Order of the results is not guaranteed. + * @param patterns Patterns separated by newlines + * @param options Glob options */ -export async function glob( - pattern: string, - options?: IGlobOptions -): Promise { - const result: string[] = [] - for await (const itemPath of globGenerator(pattern, options)) { - result.push(itemPath) - } - return result -} - -/** - * Returns files and directories matching the specified glob pattern. - * - * Order of the results is not guaranteed. - */ -export async function* globGenerator( - pattern: string, - options?: IGlobOptions -): AsyncGenerator { - // Set defaults options - options = patternHelper.getOptions(options) - - // Parse patterns - const patterns: Pattern[] = patternHelper.parse([pattern], options) - - // Push the search paths - const stack: SearchState[] = [] - for (const searchPath of patternHelper.getSearchPaths(patterns)) { - core.debug(`Search path '${searchPath}'`) - - // Exists? - try { - // Intentionally using lstat. Detection for broken symlink - // will be performed later (if following symlinks). - await fs.promises.lstat(searchPath) - } catch (err) { - if (err.code === 'ENOENT') { - continue - } - throw err - } - - stack.unshift(new SearchState(searchPath, 1)) - } - - // Search - const traversalChain: string[] = [] // used to detect cycles - while (stack.length) { - // Pop - const item = stack.pop() as SearchState - - // Match? - const match = patternHelper.match(patterns, item.path) - const partialMatch = - !!match || patternHelper.partialMatch(patterns, item.path) - if (!match && !partialMatch) { - continue - } - - // Stat - const stats: fs.Stats | undefined = await stat( - item, - options, - traversalChain - ) - - // Broken symlink, or symlink cycle detected, or no longer exists - if (!stats) { - continue - } - - // Directory - if (stats.isDirectory()) { - // Matched - if (match & MatchKind.Directory) { - yield item.path - } - // Descend? - else if (!partialMatch) { - continue - } - - // Push the child items in reverse - const childLevel = item.level + 1 - const childItems = (await fs.promises.readdir(item.path)).map( - x => new SearchState(path.join(item.path, x), childLevel) - ) - stack.push(...childItems.reverse()) - } - // File - else if (match & MatchKind.File) { - yield item.path - } - } -} - -/** - * Returns the search path preceeding the first segment that contains a pattern. - * - * For example, '/foo/bar*' returns '/foo'. - */ -export function getSearchPath(pattern: string): string { - const patterns: Pattern[] = patternHelper.parse( - [pattern], - patternHelper.getOptions() - ) - const searchPaths: string[] = patternHelper.getSearchPaths(patterns) - return searchPaths.length > 0 ? searchPaths[0] : '' -} - -async function stat( - item: SearchState, - options: IGlobOptions, - traversalChain: string[] -): Promise { - // Note: - // `stat` returns info about the target of a symlink (or symlink chain) - // `lstat` returns info about a symlink itself - let stats: fs.Stats - if (options.followSymbolicLinks) { - try { - // Use `stat` (following symlinks) - stats = await fs.promises.stat(item.path) - } catch (err) { - if (err.code === 'ENOENT') { - if (options.omitBrokenSymbolicLinks) { - core.debug(`Broken symlink '${item.path}'`) - return undefined - } - - throw new Error( - `No information found for the path '${item.path}'. This may indicate a broken symbolic link.` - ) - } - - throw err - } - } else { - // Use `lstat` (not following symlinks) - stats = await fs.promises.lstat(item.path) - } - - // Note, isDirectory() returns false for the lstat of a symlink - if (stats.isDirectory() && options.followSymbolicLinks) { - // Get the realpath - const realPath: string = await fs.promises.realpath(item.path) - - // Fixup the traversal chain to match the item level - while (traversalChain.length >= item.level) { - traversalChain.pop() - } - - // Test for a cycle - if (traversalChain.some((x: string) => x === realPath)) { - core.debug( - `Symlink cycle detected for path '${item.path}' and realpath '${realPath}'` - ) - return undefined - } - - // Update the traversal chain - traversalChain.push(realPath) - } - - return stats +export async function create( + patterns: string, + options?: GlobOptions +): Promise { + return await DefaultGlobber.create(patterns, options) } diff --git a/packages/glob/src/internal-glob-options-helper.ts b/packages/glob/src/internal-glob-options-helper.ts new file mode 100644 index 00000000..3c81b671 --- /dev/null +++ b/packages/glob/src/internal-glob-options-helper.ts @@ -0,0 +1,32 @@ +import * as core from '@actions/core' +import {GlobOptions} from './internal-glob-options' + +/** + * Returns a copy with defaults filled in. + */ +export function getOptions(copy?: GlobOptions): GlobOptions { + const result: GlobOptions = { + followSymbolicLinks: true, + implicitDescendants: true, + omitBrokenSymbolicLinks: true + } + + if (copy) { + if (typeof copy.followSymbolicLinks === 'boolean') { + result.followSymbolicLinks = copy.followSymbolicLinks + core.debug(`followSymbolicLinks '${result.followSymbolicLinks}'`) + } + + if (typeof copy.implicitDescendants === 'boolean') { + result.implicitDescendants = copy.implicitDescendants + core.debug(`implicitDescendants '${result.implicitDescendants}'`) + } + + if (typeof copy.omitBrokenSymbolicLinks === 'boolean') { + result.omitBrokenSymbolicLinks = copy.omitBrokenSymbolicLinks + core.debug(`omitBrokenSymbolicLinks '${result.omitBrokenSymbolicLinks}'`) + } + } + + return result +} diff --git a/packages/glob/src/internal-glob-options.ts b/packages/glob/src/internal-glob-options.ts index 8e427454..54d8b544 100644 --- a/packages/glob/src/internal-glob-options.ts +++ b/packages/glob/src/internal-glob-options.ts @@ -1,7 +1,10 @@ -export interface IGlobOptions { +/** + * Options to control globbing behavior + */ +export interface GlobOptions { /** - * Indicates whether to follow symbolic links. Generally should be true - * unless deleting files. + * Indicates whether to follow symbolic links. Generally should set to false + * when deleting files. * * @default true */ diff --git a/packages/glob/src/internal-globber.ts b/packages/glob/src/internal-globber.ts new file mode 100644 index 00000000..a84d298d --- /dev/null +++ b/packages/glob/src/internal-globber.ts @@ -0,0 +1,242 @@ +import * as core from '@actions/core' +import * as fs from 'fs' +import * as globOptionsHelper from './internal-glob-options-helper' +import * as path from 'path' +import * as patternHelper from './internal-pattern-helper' +import {GlobOptions} from './internal-glob-options' +import {MatchKind} from './internal-match-kind' +import {Pattern} from './internal-pattern' +import {SearchState} from './internal-search-state' + +const IS_WINDOWS = process.platform === 'win32' + +export {GlobOptions} + +/** + * Used to match files and directories + */ +export interface Globber { + /** + * Returns the search path preceeding the first glob segment, from each pattern. + * Duplicates and descendants of other paths are filtered out. + * + * Example 1: The patterns `/foo/*` and `/bar/*` returns `/foo` and `/bar`. + * + * Example 2: The patterns `/foo/*` and `/foo/bar/*` returns `/foo`. + */ + getSearchPaths(): string[] + + /** + * Returns files and directories matching the glob patterns. + * + * Order of the results is not guaranteed. + */ + glob(): Promise + + /** + * Returns files and directories matching the glob patterns. + * + * Order of the results is not guaranteed. + */ + globGenerator(): AsyncGenerator +} + +export class DefaultGlobber implements Globber { + private readonly options: GlobOptions + private readonly patterns: Pattern[] = [] + private readonly searchPaths: string[] = [] + + private constructor(options?: GlobOptions) { + this.options = globOptionsHelper.getOptions(options) + } + + getSearchPaths(): string[] { + // Return a copy + return this.searchPaths.slice() + } + + async glob(): Promise { + const result: string[] = [] + for await (const itemPath of this.globGenerator()) { + result.push(itemPath) + } + return result + } + + async *globGenerator(): AsyncGenerator { + // Fill in defaults options + const options = globOptionsHelper.getOptions(this.options) + + // Implicit descendants? + const patterns: Pattern[] = [] + for (const pattern of this.patterns) { + patterns.push(pattern) + if ( + options.implicitDescendants && + (pattern.trailingSeparator || + pattern.segments[pattern.segments.length - 1] !== '**') + ) { + patterns.push( + new Pattern(pattern.negate, pattern.segments.concat('**')) + ) + } + } + + // Push the search paths + const stack: SearchState[] = [] + for (const searchPath of patternHelper.getSearchPaths(patterns)) { + core.debug(`Search path '${searchPath}'`) + + // Exists? + try { + // Intentionally using lstat. Detection for broken symlink + // will be performed later (if following symlinks). + await fs.promises.lstat(searchPath) + } catch (err) { + if (err.code === 'ENOENT') { + continue + } + throw err + } + + stack.unshift(new SearchState(searchPath, 1)) + } + + // Search + const traversalChain: string[] = [] // used to detect cycles + while (stack.length) { + // Pop + const item = stack.pop() as SearchState + + // Match? + const match = patternHelper.match(patterns, item.path) + const partialMatch = + !!match || patternHelper.partialMatch(patterns, item.path) + if (!match && !partialMatch) { + continue + } + + // Stat + const stats: fs.Stats | undefined = await DefaultGlobber.stat( + item, + options, + traversalChain + ) + + // Broken symlink, or symlink cycle detected, or no longer exists + if (!stats) { + continue + } + + // Directory + if (stats.isDirectory()) { + // Matched + if (match & MatchKind.Directory) { + yield item.path + } + // Descend? + else if (!partialMatch) { + continue + } + + // Push the child items in reverse + const childLevel = item.level + 1 + const childItems = (await fs.promises.readdir(item.path)).map( + x => new SearchState(path.join(item.path, x), childLevel) + ) + stack.push(...childItems.reverse()) + } + // File + else if (match & MatchKind.File) { + yield item.path + } + } + } + + /** + * Constructs a DefaultGlobber + */ + static async create( + patterns: string, + options?: GlobOptions + ): Promise { + const result = new DefaultGlobber(options) + + if (IS_WINDOWS) { + patterns = patterns.replace(/\r\n/g, '\n') + patterns = patterns.replace(/\r/g, '\n') + } + + const lines = patterns.split('\n').map(x => x.trim()) + for (const line of lines) { + // Empty or comment + if (!line || line.startsWith('#')) { + continue + } + // Pattern + else { + result.patterns.push(new Pattern(line)) + } + } + + result.searchPaths.push(...patternHelper.getSearchPaths(result.patterns)) + return result + } + + private static async stat( + item: SearchState, + options: GlobOptions, + traversalChain: string[] + ): Promise { + // Note: + // `stat` returns info about the target of a symlink (or symlink chain) + // `lstat` returns info about a symlink itself + let stats: fs.Stats + if (options.followSymbolicLinks) { + try { + // Use `stat` (following symlinks) + stats = await fs.promises.stat(item.path) + } catch (err) { + if (err.code === 'ENOENT') { + if (options.omitBrokenSymbolicLinks) { + core.debug(`Broken symlink '${item.path}'`) + return undefined + } + + throw new Error( + `No information found for the path '${item.path}'. This may indicate a broken symbolic link.` + ) + } + + throw err + } + } else { + // Use `lstat` (not following symlinks) + stats = await fs.promises.lstat(item.path) + } + + // Note, isDirectory() returns false for the lstat of a symlink + if (stats.isDirectory() && options.followSymbolicLinks) { + // Get the realpath + const realPath: string = await fs.promises.realpath(item.path) + + // Fixup the traversal chain to match the item level + while (traversalChain.length >= item.level) { + traversalChain.pop() + } + + // Test for a cycle + if (traversalChain.some((x: string) => x === realPath)) { + core.debug( + `Symlink cycle detected for path '${item.path}' and realpath '${realPath}'` + ) + return undefined + } + + // Update the traversal chain + traversalChain.push(realPath) + } + + return stats + } +} diff --git a/packages/glob/src/internal-pattern-helper.ts b/packages/glob/src/internal-pattern-helper.ts index 9ca56b14..e0cf3a6a 100644 --- a/packages/glob/src/internal-pattern-helper.ts +++ b/packages/glob/src/internal-pattern-helper.ts @@ -1,41 +1,9 @@ -import * as core from '@actions/core' import * as pathHelper from './internal-path-helper' -import {IGlobOptions} from './internal-glob-options' import {MatchKind} from './internal-match-kind' import {Pattern} from './internal-pattern' const IS_WINDOWS = process.platform === 'win32' -/** - * Returns a copy with defaults filled in - */ -export function getOptions(copy?: IGlobOptions): IGlobOptions { - const result: IGlobOptions = { - followSymbolicLinks: true, - implicitDescendants: true, - omitBrokenSymbolicLinks: true - } - - if (copy) { - if (typeof copy.followSymbolicLinks === 'boolean') { - result.followSymbolicLinks = copy.followSymbolicLinks - core.debug(`followSymbolicLinks '${result.followSymbolicLinks}'`) - } - - if (typeof copy.implicitDescendants === 'boolean') { - result.implicitDescendants = copy.implicitDescendants - core.debug(`implicitDescendants '${result.implicitDescendants}'`) - } - - if (typeof copy.omitBrokenSymbolicLinks === 'boolean') { - result.omitBrokenSymbolicLinks = copy.omitBrokenSymbolicLinks - core.debug(`omitBrokenSymbolicLinks '${result.omitBrokenSymbolicLinks}'`) - } - } - - return result -} - /** * Given an array of patterns, returns an array of paths to search. * Duplicates and paths under other included paths are filtered out. @@ -105,36 +73,6 @@ export function match(patterns: Pattern[], itemPath: string): MatchKind { return result } -/** - * Parses the pattern strings into Pattern objects - */ -export function parse(patterns: string[], options: IGlobOptions): Pattern[] { - const result: Pattern[] = [] - - for (const patternString of patterns.map(x => x.trim())) { - // Skip empty or comment - if (!patternString || patternString.startsWith('#')) { - continue - } - - // Push - const pattern = new Pattern(patternString) - result.push(pattern) - - // Implicit descendants? - if ( - options.implicitDescendants && - (pattern.trailingSeparator || - pattern.segments[pattern.segments.length - 1] !== '**') - ) { - // Push - result.push(new Pattern(pattern.negate, pattern.segments.concat('**'))) - } - } - - return result -} - /** * Checks whether to descend further into the directory */