Multi Path Artifact Upload + Exclude Character Support (#94)

* Support for multi path upload

* Update README

* Fix tests

* Actually fix tests

* PR feedback

* Fix

* Apply suggestions from code review

Co-authored-by: Alberto Gimeno <gimenete@users.noreply.github.com>

* Fix more tests

Co-authored-by: Alberto Gimeno <gimenete@users.noreply.github.com>
pull/95/head
Konrad Pabjan 2020-07-09 20:53:45 +02:00 committed by GitHub
parent 4347a0d55a
commit f265ac5693
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 264 additions and 15 deletions

View File

@ -75,6 +75,16 @@ jobs:
name: 'GZip-Artifact' name: 'GZip-Artifact'
path: path/to/dir-3/ path: path/to/dir-3/
# Upload a directory that contains a file that will be uploaded with GZip
- name: 'Upload artifact #4'
uses: ./
with:
name: 'Multi-Path-Artifact'
path: |
path/to/dir-1/*
path/to/dir-[23]/*
!path/to/dir-3/*.txt
# Verify artifacts. Switch to download-artifact@v2 once it's out of preview # Verify artifacts. Switch to download-artifact@v2 once it's out of preview
# Download Artifact #1 and verify the correctness of the content # Download Artifact #1 and verify the correctness of the content
@ -138,3 +148,23 @@ jobs:
Write-Error "File contents of downloaded artifact is incorrect" Write-Error "File contents of downloaded artifact is incorrect"
} }
shell: pwsh shell: pwsh
- name: 'Download artifact #4'
uses: actions/download-artifact@v1
with:
name: 'Multi-Path-Artifact'
path: multi/artifact
- name: 'Verify Artifact #4'
run: |
$file1 = "multi/artifact/dir-1/file1.txt"
$file2 = "multi/artifact/dir-2/file2.txt"
if(!(Test-Path -path $file1) -or !(Test-Path -path $file2))
{
Write-Error "Expected files do not exist"
}
if(!((Get-Content $file1) -ceq "Lorem ipsum dolor sit amet") -or !((Get-Content $file2) -ceq "Hello world from file #2"))
{
Write-Error "File contents of downloaded artifacts are incorrect"
}
shell: pwsh

View File

@ -10,6 +10,9 @@ See also [download-artifact](https://github.com/actions/download-artifact).
- Specify a wildcard pattern - Specify a wildcard pattern
- Specify an individual file - Specify an individual file
- Specify a directory (previously you were limited to only this option) - Specify a directory (previously you were limited to only this option)
- Multi path upload
- Use a combination of individual files, wildcards or directories
- Support for excluding certain files
- Upload an artifact without providing a name - Upload an artifact without providing a name
- Fix for artifact uploads sometimes not working with containers - Fix for artifact uploads sometimes not working with containers
- Proxy support out of the box - Proxy support out of the box
@ -45,7 +48,7 @@ steps:
path: path/to/artifact/ # or path/to/artifact path: path/to/artifact/ # or path/to/artifact
``` ```
### Upload using a Wildcard Pattern: ### Upload using a Wildcard Pattern
```yaml ```yaml
- uses: actions/upload-artifact@v2 - uses: actions/upload-artifact@v2
with: with:
@ -53,11 +56,37 @@ steps:
path: path/**/[abc]rtifac?/* path: path/**/[abc]rtifac?/*
``` ```
### Upload using Multiple Paths and Exclusions
```yaml
- uses: actions/upload-artifact@v2
with:
name: my-artifact
path: |
path/output/bin/
path/output/test-results
!path/**/*.tmp
```
For supported wildcards along with behavior and documentation, see [@actions/glob](https://github.com/actions/toolkit/tree/master/packages/glob) which is used internally to search for files. For supported wildcards along with behavior and documentation, see [@actions/glob](https://github.com/actions/toolkit/tree/master/packages/glob) which is used internally to search for files.
If a wildcard pattern is used, the path hierarchy will be preserved after the first wildcard pattern.
```
path/to/*/directory/foo?.txt =>
∟ path/to/some/directory/foo1.txt
∟ path/to/some/directory/foo2.txt
∟ path/to/other/directory/foo1.txt
would be flattened and uploaded as =>
∟ some/directory/foo1.txt
∟ some/directory/foo2.txt
∟ other/directory/foo1.txt
```
If multiple paths are provided as input, the least common ancestor of all the search paths will be used as the root directory of the artifact. Exclude paths do not effect the directory structure.
Relative and absolute file paths are both allowed. Relative paths are rooted against the current working directory. Paths that begin with a wildcard character should be quoted to avoid being interpreted as YAML aliases. Relative and absolute file paths are both allowed. Relative paths are rooted against the current working directory. Paths that begin with a wildcard character should be quoted to avoid being interpreted as YAML aliases.
The [@actions/artifact](https://github.com/actions/toolkit/tree/master/packages/artifact) package is also used internally to handle most of the logic around uploading an artifact. There is extra documentation around upload limitations and behavior in the toolkit repo that is worth checking out. The [@actions/artifact](https://github.com/actions/toolkit/tree/master/packages/artifact) package is used internally to handle most of the logic around uploading an artifact. There is extra documentation around upload limitations and behavior in the toolkit repo that is worth checking out.
### Conditional Artifact Upload ### Conditional Artifact Upload

View File

@ -231,7 +231,7 @@ describe('Search', () => {
expect(searchResult.filesToUpload.includes(searchItem1Path)).toEqual(true) expect(searchResult.filesToUpload.includes(searchItem1Path)).toEqual(true)
expect(searchResult.filesToUpload.includes(searchItem2Path)).toEqual(true) expect(searchResult.filesToUpload.includes(searchItem2Path)).toEqual(true)
expect(searchResult.filesToUpload.includes(searchItem2Path)).toEqual(true) expect(searchResult.filesToUpload.includes(searchItem3Path)).toEqual(true)
expect(searchResult.filesToUpload.includes(searchItem4Path)).toEqual(true) expect(searchResult.filesToUpload.includes(searchItem4Path)).toEqual(true)
expect(searchResult.filesToUpload.includes(searchItem5Path)).toEqual(true) expect(searchResult.filesToUpload.includes(searchItem5Path)).toEqual(true)
expect(searchResult.filesToUpload.includes(extraSearchItem1Path)).toEqual( expect(searchResult.filesToUpload.includes(extraSearchItem1Path)).toEqual(
@ -265,7 +265,7 @@ describe('Search', () => {
expect(searchResult.filesToUpload.includes(searchItem1Path)).toEqual(true) expect(searchResult.filesToUpload.includes(searchItem1Path)).toEqual(true)
expect(searchResult.filesToUpload.includes(searchItem2Path)).toEqual(true) expect(searchResult.filesToUpload.includes(searchItem2Path)).toEqual(true)
expect(searchResult.filesToUpload.includes(searchItem2Path)).toEqual(true) expect(searchResult.filesToUpload.includes(searchItem3Path)).toEqual(true)
expect(searchResult.filesToUpload.includes(searchItem4Path)).toEqual(true) expect(searchResult.filesToUpload.includes(searchItem4Path)).toEqual(true)
expect(searchResult.filesToUpload.includes(searchItem5Path)).toEqual(true) expect(searchResult.filesToUpload.includes(searchItem5Path)).toEqual(true)
expect(searchResult.filesToUpload.includes(extraSearchItem1Path)).toEqual( expect(searchResult.filesToUpload.includes(extraSearchItem1Path)).toEqual(
@ -286,4 +286,70 @@ describe('Search', () => {
expect(searchResult.rootDirectory).toEqual(root) expect(searchResult.rootDirectory).toEqual(root)
}) })
it('Multi path search - root directory', async () => {
const searchPath1 = path.join(root, 'folder-a')
const searchPath2 = path.join(root, 'folder-d')
const searchPaths = searchPath1 + '\n' + searchPath2
const searchResult = await findFilesToUpload(searchPaths)
expect(searchResult.rootDirectory).toEqual(root)
expect(searchResult.filesToUpload.length).toEqual(7)
expect(searchResult.filesToUpload.includes(searchItem1Path)).toEqual(true)
expect(searchResult.filesToUpload.includes(searchItem2Path)).toEqual(true)
expect(searchResult.filesToUpload.includes(searchItem3Path)).toEqual(true)
expect(searchResult.filesToUpload.includes(searchItem4Path)).toEqual(true)
expect(searchResult.filesToUpload.includes(extraSearchItem1Path)).toEqual(
true
)
expect(searchResult.filesToUpload.includes(extraSearchItem2Path)).toEqual(
true
)
expect(searchResult.filesToUpload.includes(extraFileInFolderCPath)).toEqual(
true
)
})
it('Multi path search - with exclude character', async () => {
const searchPath1 = path.join(root, 'folder-a')
const searchPath2 = path.join(root, 'folder-d')
const searchPath3 = path.join(root, 'folder-a', 'folder-b', '**/extra*.txt')
// negating the third search path
const searchPaths = searchPath1 + '\n' + searchPath2 + '\n!' + searchPath3
const searchResult = await findFilesToUpload(searchPaths)
expect(searchResult.rootDirectory).toEqual(root)
expect(searchResult.filesToUpload.length).toEqual(5)
expect(searchResult.filesToUpload.includes(searchItem1Path)).toEqual(true)
expect(searchResult.filesToUpload.includes(searchItem2Path)).toEqual(true)
expect(searchResult.filesToUpload.includes(searchItem3Path)).toEqual(true)
expect(searchResult.filesToUpload.includes(searchItem4Path)).toEqual(true)
expect(searchResult.filesToUpload.includes(extraSearchItem2Path)).toEqual(
true
)
})
it('Multi path search - non root directory', async () => {
const searchPath1 = path.join(root, 'folder-h', 'folder-i')
const searchPath2 = path.join(root, 'folder-h', 'folder-j', 'folder-k')
const searchPath3 = amazingFileInFolderHPath
const searchPaths = [searchPath1, searchPath2, searchPath3].join('\n')
const searchResult = await findFilesToUpload(searchPaths)
expect(searchResult.rootDirectory).toEqual(path.join(root, 'folder-h'))
expect(searchResult.filesToUpload.length).toEqual(4)
expect(
searchResult.filesToUpload.includes(amazingFileInFolderHPath)
).toEqual(true)
expect(searchResult.filesToUpload.includes(extraSearchItem4Path)).toEqual(
true
)
expect(searchResult.filesToUpload.includes(extraSearchItem5Path)).toEqual(
true
)
expect(searchResult.filesToUpload.includes(lonelyFilePath)).toEqual(true)
})
}) })

65
dist/index.js vendored
View File

@ -6221,6 +6221,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
}; };
Object.defineProperty(exports, "__esModule", { value: true }); Object.defineProperty(exports, "__esModule", { value: true });
const glob = __importStar(__webpack_require__(281)); const glob = __importStar(__webpack_require__(281));
const path = __importStar(__webpack_require__(622));
const core_1 = __webpack_require__(470); const core_1 = __webpack_require__(470);
const fs_1 = __webpack_require__(747); const fs_1 = __webpack_require__(747);
const path_1 = __webpack_require__(622); const path_1 = __webpack_require__(622);
@ -6231,6 +6232,57 @@ function getDefaultGlobOptions() {
omitBrokenSymbolicLinks: true omitBrokenSymbolicLinks: true
}; };
} }
/**
* If multiple paths are specific, the least common ancestor (LCA) of the search paths is used as
* the delimiter to control the directory structure for the artifact. This function returns the LCA
* when given an array of search paths
*
* Example 1: The patterns `/foo/` and `/bar/` returns `/`
*
* Example 2: The patterns `~/foo/bar/*` and `~/foo/voo/two/*` and `~/foo/mo/` returns `~/foo`
*/
function getMultiPathLCA(searchPaths) {
if (searchPaths.length < 2) {
throw new Error('At least two search paths must be provided');
}
const commonPaths = new Array();
const splitPaths = new Array();
let smallestPathLength = Number.MAX_SAFE_INTEGER;
// split each of the search paths using the platform specific separator
for (const searchPath of searchPaths) {
core_1.debug(`Using search path ${searchPath}`);
const splitSearchPath = path.normalize(searchPath).split(path.sep);
// keep track of the smallest path length so that we don't accidentally later go out of bounds
smallestPathLength = Math.min(smallestPathLength, splitSearchPath.length);
splitPaths.push(splitSearchPath);
}
// on Unix-like file systems, the file separator exists at the beginning of the file path, make sure to preserve it
if (searchPaths[0].startsWith(path.sep)) {
commonPaths.push(path.sep);
}
let splitIndex = 0;
// function to check if the paths are the same at a specific index
function isPathTheSame() {
const compare = splitPaths[0][splitIndex];
for (let i = 1; i < splitPaths.length; i++) {
if (compare !== splitPaths[i][splitIndex]) {
// a non-common index has been reached
return false;
}
}
return true;
}
// Loop over all the search paths until there is a non-common ancestor or we go out of bounds
while (splitIndex < smallestPathLength) {
if (!isPathTheSame()) {
break;
}
// if all are the same, add to the end result & increment the index
commonPaths.push(splitPaths[0][splitIndex]);
splitIndex++;
}
return path.join(...commonPaths);
}
function findFilesToUpload(searchPath, globOptions) { function findFilesToUpload(searchPath, globOptions) {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
const searchResults = []; const searchResults = [];
@ -6249,13 +6301,16 @@ function findFilesToUpload(searchPath, globOptions) {
core_1.debug(`Removing ${searchResult} from rawSearchResults because it is a directory`); core_1.debug(`Removing ${searchResult} from rawSearchResults because it is a directory`);
} }
} }
/* // Calculate the root directory for the artifact using the search paths that were utilized
Only a single search pattern is being included so only 1 searchResult is expected. In the future if multiple search patterns are
simultaneously supported this will change
*/
const searchPaths = globber.getSearchPaths(); const searchPaths = globber.getSearchPaths();
if (searchPaths.length > 1) { if (searchPaths.length > 1) {
throw new Error('Only 1 search path should be returned'); core_1.info(`Multiple search paths detected. Calculating the least common ancestor of all paths`);
const lcaSearchPath = getMultiPathLCA(searchPaths);
core_1.info(`The least common ancestor is ${lcaSearchPath}. This will be the root directory of the artifact`);
return {
filesToUpload: searchResults,
rootDirectory: lcaSearchPath
};
} }
/* /*
Special case for a single file artifact that is uploaded without a directory or wildcard pattern. The directory structure is Special case for a single file artifact that is uploaded without a directory or wildcard pattern. The directory structure is

View File

@ -1,5 +1,6 @@
import * as glob from '@actions/glob' import * as glob from '@actions/glob'
import {debug} from '@actions/core' import * as path from 'path'
import {debug, info} from '@actions/core'
import {lstatSync} from 'fs' import {lstatSync} from 'fs'
import {dirname} from 'path' import {dirname} from 'path'
@ -16,6 +17,65 @@ function getDefaultGlobOptions(): glob.GlobOptions {
} }
} }
/**
* If multiple paths are specific, the least common ancestor (LCA) of the search paths is used as
* the delimiter to control the directory structure for the artifact. This function returns the LCA
* when given an array of search paths
*
* Example 1: The patterns `/foo/` and `/bar/` returns `/`
*
* Example 2: The patterns `~/foo/bar/*` and `~/foo/voo/two/*` and `~/foo/mo/` returns `~/foo`
*/
function getMultiPathLCA(searchPaths: string[]): string {
if (searchPaths.length < 2) {
throw new Error('At least two search paths must be provided')
}
const commonPaths = new Array<string>()
const splitPaths = new Array<string[]>()
let smallestPathLength = Number.MAX_SAFE_INTEGER
// split each of the search paths using the platform specific separator
for (const searchPath of searchPaths) {
debug(`Using search path ${searchPath}`)
const splitSearchPath = path.normalize(searchPath).split(path.sep)
// keep track of the smallest path length so that we don't accidentally later go out of bounds
smallestPathLength = Math.min(smallestPathLength, splitSearchPath.length)
splitPaths.push(splitSearchPath)
}
// on Unix-like file systems, the file separator exists at the beginning of the file path, make sure to preserve it
if (searchPaths[0].startsWith(path.sep)) {
commonPaths.push(path.sep)
}
let splitIndex = 0
// function to check if the paths are the same at a specific index
function isPathTheSame(): boolean {
const compare = splitPaths[0][splitIndex]
for (let i = 1; i < splitPaths.length; i++) {
if (compare !== splitPaths[i][splitIndex]) {
// a non-common index has been reached
return false
}
}
return true
}
// Loop over all the search paths until there is a non-common ancestor or we go out of bounds
while (splitIndex < smallestPathLength) {
if (!isPathTheSame()) {
break
}
// if all are the same, add to the end result & increment the index
commonPaths.push(splitPaths[0][splitIndex])
splitIndex++
}
return path.join(...commonPaths)
}
export async function findFilesToUpload( export async function findFilesToUpload(
searchPath: string, searchPath: string,
globOptions?: glob.GlobOptions globOptions?: glob.GlobOptions
@ -42,13 +102,22 @@ export async function findFilesToUpload(
} }
} }
/* // Calculate the root directory for the artifact using the search paths that were utilized
Only a single search pattern is being included so only 1 searchResult is expected. In the future if multiple search patterns are
simultaneously supported this will change
*/
const searchPaths: string[] = globber.getSearchPaths() const searchPaths: string[] = globber.getSearchPaths()
if (searchPaths.length > 1) { if (searchPaths.length > 1) {
throw new Error('Only 1 search path should be returned') info(
`Multiple search paths detected. Calculating the least common ancestor of all paths`
)
const lcaSearchPath = getMultiPathLCA(searchPaths)
info(
`The least common ancestor is ${lcaSearchPath}. This will be the root directory of the artifact`
)
return {
filesToUpload: searchResults,
rootDirectory: lcaSearchPath
}
} }
/* /*