From 9b383229c1803126c9172b29cd126b28c4c80c91 Mon Sep 17 00:00:00 2001 From: Rob Herley Date: Mon, 21 Aug 2023 21:23:54 +0000 Subject: [PATCH 01/17] add download apis to stream zip from blob storage --- packages/artifact/package-lock.json | 196 +++++++++++++++++- packages/artifact/package.json | 3 +- packages/artifact/src/artifact.ts | 4 +- .../internal/download/download-artifact.ts | 96 ++++++++- 4 files changed, 294 insertions(+), 5 deletions(-) diff --git a/packages/artifact/package-lock.json b/packages/artifact/package-lock.json index 90c4bd6f..90338dff 100644 --- a/packages/artifact/package-lock.json +++ b/packages/artifact/package-lock.json @@ -20,7 +20,8 @@ "archiver": "^5.3.1", "crypto": "^1.0.1", "jwt-decode": "^3.1.2", - "twirp-ts": "^2.5.0" + "twirp-ts": "^2.5.0", + "unzipper": "^0.10.14" }, "devDependencies": { "@types/archiver": "^5.3.2", @@ -543,6 +544,26 @@ "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz", "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==" }, + "node_modules/big-integer": { + "version": "1.6.51", + "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.51.tgz", + "integrity": "sha512-GPEid2Y9QU1Exl1rpO9B2IPJGHPSupF5GnVIP0blYvNOMer2bTvSWs1jGOUg04hTmu67nmLsQ9TBo1puaotBHg==", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/binary": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", + "integrity": "sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==", + "dependencies": { + "buffers": "~0.1.1", + "chainsaw": "~0.1.0" + }, + "engines": { + "node": "*" + } + }, "node_modules/bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", @@ -553,6 +574,11 @@ "readable-stream": "^3.4.0" } }, + "node_modules/bluebird": { + "version": "3.4.7", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz", + "integrity": "sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==" + }, "node_modules/bottleneck": { "version": "2.19.5", "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz", @@ -598,6 +624,22 @@ "node": "*" } }, + "node_modules/buffer-indexof-polyfill": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.2.tgz", + "integrity": "sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A==", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/buffers": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", + "integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==", + "engines": { + "node": ">=0.2.0" + } + }, "node_modules/camel-case": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/camel-case/-/camel-case-4.1.2.tgz", @@ -607,6 +649,17 @@ "tslib": "^2.0.3" } }, + "node_modules/chainsaw": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", + "integrity": "sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==", + "dependencies": { + "traverse": ">=0.3.0 <0.4" + }, + "engines": { + "node": "*" + } + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -704,6 +757,41 @@ "dot-object": "bin/dot-object" } }, + "node_modules/duplexer2": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", + "integrity": "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==", + "dependencies": { + "readable-stream": "^2.0.2" + } + }, + "node_modules/duplexer2/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/duplexer2/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "node_modules/duplexer2/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/end-of-stream": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", @@ -743,6 +831,20 @@ "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" }, + "node_modules/fstream": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz", + "integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==", + "dependencies": { + "graceful-fs": "^4.1.2", + "inherits": "~2.0.0", + "mkdirp": ">=0.5 0", + "rimraf": "2" + }, + "engines": { + "node": ">=0.6" + } + }, "node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", @@ -856,6 +958,11 @@ "safe-buffer": "~5.1.0" } }, + "node_modules/listenercount": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/listenercount/-/listenercount-1.0.1.tgz", + "integrity": "sha512-3mk/Zag0+IJxeDrxSgaDPy4zZ3w05PRZeJNnlWhzFz5OkX49J4krc+A8X2d2M69vGMBEX0uyl8M+W+8gH+kBqQ==" + }, "node_modules/lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", @@ -924,6 +1031,25 @@ "node": "*" } }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/mkdirp": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "dependencies": { + "minimist": "^1.2.6" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, "node_modules/no-case": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/no-case/-/no-case-3.0.4.tgz", @@ -1057,6 +1183,17 @@ "node": ">=10" } }, + "node_modules/rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -1081,6 +1218,11 @@ "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" }, + "node_modules/setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==" + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -1109,6 +1251,14 @@ "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" }, + "node_modules/traverse": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", + "integrity": "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==", + "engines": { + "node": "*" + } + }, "node_modules/ts-poet": { "version": "4.15.0", "resolved": "https://registry.npmjs.org/ts-poet/-/ts-poet-4.15.0.tgz", @@ -1177,6 +1327,50 @@ "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz", "integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w==" }, + "node_modules/unzipper": { + "version": "0.10.14", + "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.10.14.tgz", + "integrity": "sha512-ti4wZj+0bQTiX2KmKWuwj7lhV+2n//uXEotUmGuQqrbVZSEGFMbI68+c6JCQ8aAmUWYvtHEz2A8K6wXvueR/6g==", + "dependencies": { + "big-integer": "^1.6.17", + "binary": "~0.3.0", + "bluebird": "~3.4.1", + "buffer-indexof-polyfill": "~1.0.0", + "duplexer2": "~0.1.4", + "fstream": "^1.0.12", + "graceful-fs": "^4.2.2", + "listenercount": "~1.0.1", + "readable-stream": "~2.3.6", + "setimmediate": "~1.0.4" + } + }, + "node_modules/unzipper/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/unzipper/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "node_modules/unzipper/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", diff --git a/packages/artifact/package.json b/packages/artifact/package.json index b9e86639..b6d0ae1c 100644 --- a/packages/artifact/package.json +++ b/packages/artifact/package.json @@ -50,7 +50,8 @@ "archiver": "^5.3.1", "crypto": "^1.0.1", "jwt-decode": "^3.1.2", - "twirp-ts": "^2.5.0" + "twirp-ts": "^2.5.0", + "unzipper": "^0.10.14" }, "devDependencies": { "@types/archiver": "^5.3.2", diff --git a/packages/artifact/src/artifact.ts b/packages/artifact/src/artifact.ts index b566df8e..5742b32d 100644 --- a/packages/artifact/src/artifact.ts +++ b/packages/artifact/src/artifact.ts @@ -1,10 +1,10 @@ import {ArtifactClient, Client} from './internal/client' -import {UploadOptions, UploadResponse} from './internal/shared/interfaces' /** * Exported functionality that we want to expose for any users of @actions/artifact */ -export {ArtifactClient, UploadOptions, UploadResponse} +export * from './internal/shared/interfaces' +export {ArtifactClient} export function create(): ArtifactClient { return Client.create() diff --git a/packages/artifact/src/internal/download/download-artifact.ts b/packages/artifact/src/internal/download/download-artifact.ts index 7e9dfbfd..0c6d0306 100644 --- a/packages/artifact/src/internal/download/download-artifact.ts +++ b/packages/artifact/src/internal/download/download-artifact.ts @@ -1,7 +1,52 @@ +import path from 'path' +import fs from 'fs/promises' +import {PathLike} from 'fs' +import github from '@actions/github' +import core from '@actions/core' +import httpClient from '@actions/http-client' +import unzipper from 'unzipper' import { DownloadArtifactOptions, DownloadArtifactResponse } from '../shared/interfaces' +import {getUserAgentString} from '../shared/user-agent' + +const scrubQueryParameters = (url: string): string => { + const parsed = new URL(url) + parsed.search = '' + return parsed.toString() +} + +async function exists(path: string): Promise { + try { + await fs.access(path) + return true + } catch (error) { + if (error.code === 'ENOENT') { + return false + } else { + throw error + } + } +} + +async function streamExtract(url: string, directory: PathLike): Promise { + const client = new httpClient.HttpClient(getUserAgentString()) + const response = await client.get(url) + + if (response.message.statusCode !== 200) { + throw new Error( + `Unexpected HTTP response from blob storage: ${response.message.statusCode} ${response.message.statusMessage}` + ) + } + + return new Promise((resolve, reject) => { + response.message + .pipe(unzipper.Extract({path: directory})) + .on('finish', resolve) + .on('error', reject) + }) +} export async function downloadArtifact( artifactId: number, @@ -10,5 +55,54 @@ export async function downloadArtifact( token: string, options?: DownloadArtifactOptions ): Promise { - throw new Error('Not implemented') + let downloadPath = options?.path || process.cwd() // TODO: make this align with GITHUB_WORKSPACE + if (options?.createArtifactFolder) { + downloadPath = path.join(downloadPath, 'my-artifact') // TODO: need to pass artifact name + } + + if (!(await exists(downloadPath))) { + core.debug(`Creating artifact folder: ${downloadPath}`) + await fs.mkdir(downloadPath, {recursive: true}) + } else { + core.warning(`Artifact folder already exists: ${downloadPath}`) + } + + const api = github.getOctokit(token) + + core.info( + `Downloading artifact ${artifactId} from ${repositoryOwner}/${repositoryName}` + ) + + const {headers, status} = await api.rest.actions.downloadArtifact({ + owner: repositoryOwner, + repo: repositoryName, + artifact_id: artifactId, + archive_format: 'zip', + request: { + redirect: 'manual' + } + }) + + if (status !== 302) { + throw new Error(`Unable to download artifact. Unexpected status: ${status}`) + } + + const {location} = headers + if (!location) { + throw new Error(`Unable to redirect to artifact download url`) + } + + core.info( + `Redirecting to blob download url: ${scrubQueryParameters(location)}` + ) + + try { + core.info(`Starting download of artifact to: ${downloadPath}`) + await streamExtract(location, downloadPath) + core.info(`Artifact download completed successfully.`) + } catch (error) { + throw new Error(`Unable to download and extract artifact: ${error.message}`) + } + + return {success: true} } From 3aaff6685bad835f3ea399d50cf01ca62845a532 Mon Sep 17 00:00:00 2001 From: Rob Herley Date: Mon, 21 Aug 2023 17:47:17 -0400 Subject: [PATCH 02/17] cleanup --- .../src/internal/download/download-artifact.ts | 14 +++++++------- .../artifact/src/internal/shared/interfaces.ts | 4 ++++ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/packages/artifact/src/internal/download/download-artifact.ts b/packages/artifact/src/internal/download/download-artifact.ts index 0c6d0306..17699dec 100644 --- a/packages/artifact/src/internal/download/download-artifact.ts +++ b/packages/artifact/src/internal/download/download-artifact.ts @@ -1,9 +1,9 @@ import path from 'path' import fs from 'fs/promises' import {PathLike} from 'fs' -import github from '@actions/github' -import core from '@actions/core' -import httpClient from '@actions/http-client' +import * as github from '@actions/github' +import * as core from '@actions/core' +import * as httpClient from '@actions/http-client' import unzipper from 'unzipper' import { DownloadArtifactOptions, @@ -61,16 +61,16 @@ export async function downloadArtifact( } if (!(await exists(downloadPath))) { - core.debug(`Creating artifact folder: ${downloadPath}`) + core.debug(`Artifact destination folder does not exist, creating: ${downloadPath}`) await fs.mkdir(downloadPath, {recursive: true}) } else { - core.warning(`Artifact folder already exists: ${downloadPath}`) + core.debug(`Artifact destination folder already exists: ${downloadPath}`) } const api = github.getOctokit(token) core.info( - `Downloading artifact ${artifactId} from ${repositoryOwner}/${repositoryName}` + `Downloading artifact '${artifactId}' from '${repositoryOwner}/${repositoryName}'` ) const {headers, status} = await api.rest.actions.downloadArtifact({ @@ -104,5 +104,5 @@ export async function downloadArtifact( throw new Error(`Unable to download and extract artifact: ${error.message}`) } - return {success: true} + return {success: true, downloadPath} } diff --git a/packages/artifact/src/internal/shared/interfaces.ts b/packages/artifact/src/internal/shared/interfaces.ts index d1f7a7aa..4f9a5c43 100644 --- a/packages/artifact/src/internal/shared/interfaces.ts +++ b/packages/artifact/src/internal/shared/interfaces.ts @@ -80,6 +80,10 @@ export interface DownloadArtifactResponse { * If the artifact download was successful */ success: boolean + /** + * The path where the artifact was downloaded to + */ + downloadPath?: string } export interface DownloadArtifactOptions { From 0555a5f45852d405b1093aebc39b6485ec6bc1c5 Mon Sep 17 00:00:00 2001 From: Bethany Date: Tue, 22 Aug 2023 09:17:43 -0700 Subject: [PATCH 03/17] add get-artifact logic --- packages/artifact/package-lock.json | 9 +++ packages/artifact/package.json | 1 + .../internal/download/download-artifact.ts | 2 +- .../src/internal/find/get-artifact.ts | 66 ++++++++++++++++++- 4 files changed, 75 insertions(+), 3 deletions(-) diff --git a/packages/artifact/package-lock.json b/packages/artifact/package-lock.json index 90338dff..d9e415c3 100644 --- a/packages/artifact/package-lock.json +++ b/packages/artifact/package-lock.json @@ -17,6 +17,7 @@ "@octokit/plugin-request-log": "^1.0.4", "@octokit/plugin-retry": "^3.0.9", "@protobuf-ts/plugin": "^2.2.3-alpha.1", + "@types/unzipper": "^0.10.6", "archiver": "^5.3.1", "crypto": "^1.0.1", "jwt-decode": "^3.1.2", @@ -441,6 +442,14 @@ "@types/node": "*" } }, + "node_modules/@types/unzipper": { + "version": "0.10.6", + "resolved": "https://registry.npmjs.org/@types/unzipper/-/unzipper-0.10.6.tgz", + "integrity": "sha512-zcBj329AHgKLQyz209N/S9R0GZqXSkUQO4tJSYE3x02qg4JuDFpgKMj50r82Erk1natCWQDIvSccDddt7jPzjA==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/archiver": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/archiver/-/archiver-5.3.1.tgz", diff --git a/packages/artifact/package.json b/packages/artifact/package.json index b6d0ae1c..e2ebc939 100644 --- a/packages/artifact/package.json +++ b/packages/artifact/package.json @@ -47,6 +47,7 @@ "@octokit/plugin-request-log": "^1.0.4", "@octokit/plugin-retry": "^3.0.9", "@protobuf-ts/plugin": "^2.2.3-alpha.1", + "@types/unzipper": "^0.10.6", "archiver": "^5.3.1", "crypto": "^1.0.1", "jwt-decode": "^3.1.2", diff --git a/packages/artifact/src/internal/download/download-artifact.ts b/packages/artifact/src/internal/download/download-artifact.ts index 17699dec..ecbac621 100644 --- a/packages/artifact/src/internal/download/download-artifact.ts +++ b/packages/artifact/src/internal/download/download-artifact.ts @@ -30,7 +30,7 @@ async function exists(path: string): Promise { } } -async function streamExtract(url: string, directory: PathLike): Promise { +async function streamExtract(url: string, directory: string): Promise { const client = new httpClient.HttpClient(getUserAgentString()) const response = await client.get(url) diff --git a/packages/artifact/src/internal/find/get-artifact.ts b/packages/artifact/src/internal/find/get-artifact.ts index 3b28ff91..9b40da32 100644 --- a/packages/artifact/src/internal/find/get-artifact.ts +++ b/packages/artifact/src/internal/find/get-artifact.ts @@ -1,4 +1,22 @@ -import {GetArtifactResponse} from '../shared/interfaces' +import {Artifact, GetArtifactResponse} from '../shared/interfaces' +import {getOctokit} from '@actions/github' +import {getUserAgentString} from '../shared/user-agent' +import {defaults as defaultGitHubOptions} from '@actions/github/lib/utils' +import { RetryOptions, getRetryOptions } from './retry-options' +import {RequestRequestOptions} from '@octokit/types' +import {requestLog} from '@octokit/plugin-request-log' +import {retry} from '@octokit/plugin-retry' + +type Options = { + log?: Console + userAgent?: string + previews?: string[] + retry?: RetryOptions + request?: RequestRequestOptions +} + +const maxRetryNumber = 5 +const exemptStatusCodes = [400, 401, 403, 404, 422] // https://github.com/octokit/plugin-retry.js/blob/9a2443746c350b3beedec35cf26e197ea318a261/src/index.ts#L14 export async function getArtifact( artifactName: string, @@ -7,5 +25,49 @@ export async function getArtifact( repositoryName: string, token: string ): Promise { - throw new Error('Not implemented') + + const [retryOpts, requestOpts] = getRetryOptions( + maxRetryNumber, + exemptStatusCodes, + defaultGitHubOptions + ) + + const opts: Options = { + log: undefined, + userAgent: getUserAgentString(), + previews: undefined, + retry: retryOpts, + request: requestOpts + } + + const github = getOctokit(token, opts, retry, requestLog) + + const getArtifactResp = await github.request('GET /repos/{owner}/{repo}/actions/runs/{run_id}/artifacts{?name}', { + owner: repositoryOwner, + repo: repositoryName, + run_id: workflowRunId, + name: artifactName, + }) + + if (getArtifactResp.status !== 200) { + return { + success: false, + } + } + + if (getArtifactResp.data.artifacts.length === 0) { + return { + success: false, + } + } + + return { + success: true, + artifact: { + name: getArtifactResp.data.artifacts[0].name, + id: getArtifactResp.data.artifacts[0].id, + url: getArtifactResp.data.artifacts[0].url, + size: getArtifactResp.data.artifacts[0].size_in_bytes, + }, + } } From 4214a1ff24ee8178826063c6b3d27fe16bb5d719 Mon Sep 17 00:00:00 2001 From: Bethany Date: Tue, 22 Aug 2023 09:57:14 -0700 Subject: [PATCH 04/17] update dependencies and prettier --- packages/artifact/package-lock.json | 27 +++++++++++++++++++ packages/artifact/package.json | 1 + .../internal/download/download-artifact.ts | 4 ++- .../src/internal/find/get-artifact.ts | 26 +++++++++--------- .../src/internal/find/list-artifacts.ts | 4 ++- 5 files changed, 48 insertions(+), 14 deletions(-) diff --git a/packages/artifact/package-lock.json b/packages/artifact/package-lock.json index d9e415c3..83971346 100644 --- a/packages/artifact/package-lock.json +++ b/packages/artifact/package-lock.json @@ -16,6 +16,7 @@ "@octokit/core": "^3.5.1", "@octokit/plugin-request-log": "^1.0.4", "@octokit/plugin-retry": "^3.0.9", + "@octokit/request-error": "^5.0.0", "@protobuf-ts/plugin": "^2.2.3-alpha.1", "@types/unzipper": "^0.10.6", "archiver": "^5.3.1", @@ -293,6 +294,32 @@ "universal-user-agent": "^6.0.0" } }, + "node_modules/@octokit/request-error": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.0.0.tgz", + "integrity": "sha512-1ue0DH0Lif5iEqT52+Rf/hf0RmGO9NWFjrzmrkArpG9trFfDM/efx00BJHdLGuro4BR/gECxCU2Twf5OKrRFsQ==", + "dependencies": { + "@octokit/types": "^11.0.0", + "deprecation": "^2.0.0", + "once": "^1.4.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/request-error/node_modules/@octokit/openapi-types": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-18.0.0.tgz", + "integrity": "sha512-V8GImKs3TeQRxRtXFpG2wl19V7444NIOTDF24AWuIbmNaNYOQMWRbjcGDXV5B+0n887fgDcuMNOmlul+k+oJtw==" + }, + "node_modules/@octokit/request-error/node_modules/@octokit/types": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-11.1.0.tgz", + "integrity": "sha512-Fz0+7GyLm/bHt8fwEqgvRBWwIV1S6wRRyq+V6exRKLVWaKGsuy6H9QFYeBVDV7rK6fO3XwHgQOPxv+cLj2zpXQ==", + "dependencies": { + "@octokit/openapi-types": "^18.0.0" + } + }, "node_modules/@octokit/request/node_modules/@octokit/request-error": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-2.1.0.tgz", diff --git a/packages/artifact/package.json b/packages/artifact/package.json index e2ebc939..dc776716 100644 --- a/packages/artifact/package.json +++ b/packages/artifact/package.json @@ -46,6 +46,7 @@ "@octokit/core": "^3.5.1", "@octokit/plugin-request-log": "^1.0.4", "@octokit/plugin-retry": "^3.0.9", + "@octokit/request-error": "^5.0.0", "@protobuf-ts/plugin": "^2.2.3-alpha.1", "@types/unzipper": "^0.10.6", "archiver": "^5.3.1", diff --git a/packages/artifact/src/internal/download/download-artifact.ts b/packages/artifact/src/internal/download/download-artifact.ts index ecbac621..6a90364d 100644 --- a/packages/artifact/src/internal/download/download-artifact.ts +++ b/packages/artifact/src/internal/download/download-artifact.ts @@ -61,7 +61,9 @@ export async function downloadArtifact( } if (!(await exists(downloadPath))) { - core.debug(`Artifact destination folder does not exist, creating: ${downloadPath}`) + core.debug( + `Artifact destination folder does not exist, creating: ${downloadPath}` + ) await fs.mkdir(downloadPath, {recursive: true}) } else { core.debug(`Artifact destination folder already exists: ${downloadPath}`) diff --git a/packages/artifact/src/internal/find/get-artifact.ts b/packages/artifact/src/internal/find/get-artifact.ts index 9b40da32..b911e77a 100644 --- a/packages/artifact/src/internal/find/get-artifact.ts +++ b/packages/artifact/src/internal/find/get-artifact.ts @@ -2,7 +2,7 @@ import {Artifact, GetArtifactResponse} from '../shared/interfaces' import {getOctokit} from '@actions/github' import {getUserAgentString} from '../shared/user-agent' import {defaults as defaultGitHubOptions} from '@actions/github/lib/utils' -import { RetryOptions, getRetryOptions } from './retry-options' +import {RetryOptions, getRetryOptions} from './retry-options' import {RequestRequestOptions} from '@octokit/types' import {requestLog} from '@octokit/plugin-request-log' import {retry} from '@octokit/plugin-retry' @@ -25,7 +25,6 @@ export async function getArtifact( repositoryName: string, token: string ): Promise { - const [retryOpts, requestOpts] = getRetryOptions( maxRetryNumber, exemptStatusCodes, @@ -42,22 +41,25 @@ export async function getArtifact( const github = getOctokit(token, opts, retry, requestLog) - const getArtifactResp = await github.request('GET /repos/{owner}/{repo}/actions/runs/{run_id}/artifacts{?name}', { - owner: repositoryOwner, - repo: repositoryName, - run_id: workflowRunId, - name: artifactName, - }) + const getArtifactResp = await github.request( + 'GET /repos/{owner}/{repo}/actions/runs/{run_id}/artifacts{?name}', + { + owner: repositoryOwner, + repo: repositoryName, + run_id: workflowRunId, + name: artifactName + } + ) if (getArtifactResp.status !== 200) { return { - success: false, + success: false } } if (getArtifactResp.data.artifacts.length === 0) { return { - success: false, + success: false } } @@ -67,7 +69,7 @@ export async function getArtifact( name: getArtifactResp.data.artifacts[0].name, id: getArtifactResp.data.artifacts[0].id, url: getArtifactResp.data.artifacts[0].url, - size: getArtifactResp.data.artifacts[0].size_in_bytes, - }, + size: getArtifactResp.data.artifacts[0].size_in_bytes + } } } diff --git a/packages/artifact/src/internal/find/list-artifacts.ts b/packages/artifact/src/internal/find/list-artifacts.ts index 7fd96838..2b357ac8 100644 --- a/packages/artifact/src/internal/find/list-artifacts.ts +++ b/packages/artifact/src/internal/find/list-artifacts.ts @@ -29,7 +29,9 @@ export async function listArtifacts( repositoryName: string, token: string ): Promise { - info(`Fetching artifact list for workflow run ${workflowRunId} in repository ${repositoryOwner}/${repositoryName}`) + info( + `Fetching artifact list for workflow run ${workflowRunId} in repository ${repositoryOwner}/${repositoryName}` + ) const artifacts: Artifact[] = [] const [retryOpts, requestOpts] = getRetryOptions( From 81a802e7e0235cfaa92616d9f2fa66b7a282cafb Mon Sep 17 00:00:00 2001 From: Bethany Date: Tue, 22 Aug 2023 10:06:40 -0700 Subject: [PATCH 05/17] lint --- .../src/internal/download/download-artifact.ts | 1 - packages/artifact/src/internal/find/get-artifact.ts | 2 +- packages/artifact/src/internal/find/list-artifacts.ts | 10 +++++----- packages/artifact/src/internal/shared/user-agent.ts | 2 +- packages/artifact/src/internal/upload/blob-upload.ts | 2 +- .../artifact/src/internal/upload/upload-artifact.ts | 6 +++--- packages/artifact/tsconfig.json | 3 ++- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/packages/artifact/src/internal/download/download-artifact.ts b/packages/artifact/src/internal/download/download-artifact.ts index 6a90364d..afd80d9b 100644 --- a/packages/artifact/src/internal/download/download-artifact.ts +++ b/packages/artifact/src/internal/download/download-artifact.ts @@ -1,6 +1,5 @@ import path from 'path' import fs from 'fs/promises' -import {PathLike} from 'fs' import * as github from '@actions/github' import * as core from '@actions/core' import * as httpClient from '@actions/http-client' diff --git a/packages/artifact/src/internal/find/get-artifact.ts b/packages/artifact/src/internal/find/get-artifact.ts index b911e77a..60f11319 100644 --- a/packages/artifact/src/internal/find/get-artifact.ts +++ b/packages/artifact/src/internal/find/get-artifact.ts @@ -1,4 +1,4 @@ -import {Artifact, GetArtifactResponse} from '../shared/interfaces' +import {GetArtifactResponse} from '../shared/interfaces' import {getOctokit} from '@actions/github' import {getUserAgentString} from '../shared/user-agent' import {defaults as defaultGitHubOptions} from '@actions/github/lib/utils' diff --git a/packages/artifact/src/internal/find/list-artifacts.ts b/packages/artifact/src/internal/find/list-artifacts.ts index 2b357ac8..40f5738b 100644 --- a/packages/artifact/src/internal/find/list-artifacts.ts +++ b/packages/artifact/src/internal/find/list-artifacts.ts @@ -72,14 +72,14 @@ export async function listArtifacts( } // Iterate over the first page - listArtifactResponse.artifacts.forEach(artifact => { + for (const artifact of listArtifactResponse.artifacts) { artifacts.push({ name: artifact.name, id: artifact.id, url: artifact.url, size: artifact.size_in_bytes }) - }) + } // Iterate over any remaining pages for ( @@ -99,19 +99,19 @@ export async function listArtifacts( page: currentPageNumber }) - listArtifactResponse.artifacts.forEach(artifact => { + for (const artifact of listArtifactResponse.artifacts) { artifacts.push({ name: artifact.name, id: artifact.id, url: artifact.url, size: artifact.size_in_bytes }) - }) + } } info(`Finished fetching artifact list`) return { - artifacts: artifacts + artifacts } } diff --git a/packages/artifact/src/internal/shared/user-agent.ts b/packages/artifact/src/internal/shared/user-agent.ts index eee01446..ee47a96c 100644 --- a/packages/artifact/src/internal/shared/user-agent.ts +++ b/packages/artifact/src/internal/shared/user-agent.ts @@ -1,4 +1,4 @@ -var packageJson = require('../../../package.json') +import * as packageJson from '../../../package.json' /** * Ensure that this User Agent String is used in all HTTP calls so that we can monitor telemetry between different versions of this package diff --git a/packages/artifact/src/internal/upload/blob-upload.ts b/packages/artifact/src/internal/upload/blob-upload.ts index 82af468a..42a3fbd5 100644 --- a/packages/artifact/src/internal/upload/blob-upload.ts +++ b/packages/artifact/src/internal/upload/blob-upload.ts @@ -91,6 +91,6 @@ export async function uploadZipToBlobStorage( return { isSuccess: true, uploadSize: uploadByteCount, - md5Hash: md5Hash + md5Hash } } diff --git a/packages/artifact/src/internal/upload/upload-artifact.ts b/packages/artifact/src/internal/upload/upload-artifact.ts index 35cd9ac3..01b37626 100644 --- a/packages/artifact/src/internal/upload/upload-artifact.ts +++ b/packages/artifact/src/internal/upload/upload-artifact.ts @@ -61,7 +61,7 @@ export async function uploadArtifact( const createArtifactReq: CreateArtifactRequest = { workflowRunBackendId: backendIds.workflowRunBackendId, workflowJobRunBackendId: backendIds.workflowJobRunBackendId, - name: name, + name, version: 4 } @@ -96,13 +96,13 @@ export async function uploadArtifact( const finalizeArtifactReq: FinalizeArtifactRequest = { workflowRunBackendId: backendIds.workflowRunBackendId, workflowJobRunBackendId: backendIds.workflowJobRunBackendId, - name: name, + name, size: uploadResult.uploadSize!.toString() } if (uploadResult.md5Hash) { finalizeArtifactReq.hash = StringValue.create({ - value: `md5:${uploadResult.md5Hash!}` + value: `md5:${uploadResult.md5Hash}` }) } diff --git a/packages/artifact/tsconfig.json b/packages/artifact/tsconfig.json index cee05147..048d5b76 100644 --- a/packages/artifact/tsconfig.json +++ b/packages/artifact/tsconfig.json @@ -16,5 +16,6 @@ }, "include": [ "./src" - ] + ], + "resolveJsonModule": true } From dd26bb1149d1db07fab41a3074e0592be610180d Mon Sep 17 00:00:00 2001 From: Bethany Date: Tue, 22 Aug 2023 11:33:00 -0700 Subject: [PATCH 06/17] use require --- .eslintignore | 1 + packages/artifact/src/internal/find/get-artifact.ts | 7 +++++++ packages/artifact/src/internal/shared/user-agent.ts | 2 +- packages/artifact/tsconfig.json | 3 +-- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.eslintignore b/.eslintignore index 3ac58072..a0478cef 100644 --- a/.eslintignore +++ b/.eslintignore @@ -3,3 +3,4 @@ packages/*/node_modules/ packages/*/lib/ packages/glob/__tests__/_temp packages/*/src/generated/*/ +packages/artifact/src/internal/shared/user-agent.ts diff --git a/packages/artifact/src/internal/find/get-artifact.ts b/packages/artifact/src/internal/find/get-artifact.ts index 60f11319..16cd163b 100644 --- a/packages/artifact/src/internal/find/get-artifact.ts +++ b/packages/artifact/src/internal/find/get-artifact.ts @@ -6,6 +6,7 @@ import {RetryOptions, getRetryOptions} from './retry-options' import {RequestRequestOptions} from '@octokit/types' import {requestLog} from '@octokit/plugin-request-log' import {retry} from '@octokit/plugin-retry' +import * as core from '@actions/core' type Options = { log?: Console @@ -52,17 +53,23 @@ export async function getArtifact( ) if (getArtifactResp.status !== 200) { + core.warning('non-200 response from GitHub API: ${getArtifactResp.status}') return { success: false } } if (getArtifactResp.data.artifacts.length === 0) { + core.warning('no artifacts found') return { success: false } } + if (getArtifactResp.data.artifacts.length > 1) { + core.warning('more than one artifact found, returning first') + } + return { success: true, artifact: { diff --git a/packages/artifact/src/internal/shared/user-agent.ts b/packages/artifact/src/internal/shared/user-agent.ts index ee47a96c..ec8ab1c7 100644 --- a/packages/artifact/src/internal/shared/user-agent.ts +++ b/packages/artifact/src/internal/shared/user-agent.ts @@ -1,4 +1,4 @@ -import * as packageJson from '../../../package.json' +const packageJson = require('../../../package.json') /** * Ensure that this User Agent String is used in all HTTP calls so that we can monitor telemetry between different versions of this package diff --git a/packages/artifact/tsconfig.json b/packages/artifact/tsconfig.json index 048d5b76..cee05147 100644 --- a/packages/artifact/tsconfig.json +++ b/packages/artifact/tsconfig.json @@ -16,6 +16,5 @@ }, "include": [ "./src" - ], - "resolveJsonModule": true + ] } From 671bf1ebd55da086e8dd1d842d2f75142a24df24 Mon Sep 17 00:00:00 2001 From: Bethany Date: Tue, 22 Aug 2023 11:44:38 -0700 Subject: [PATCH 07/17] use GITHUB_WORKSPACE as default download dir --- .../artifact/src/internal/download/download-artifact.ts | 3 ++- packages/artifact/src/internal/shared/config.ts | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/artifact/src/internal/download/download-artifact.ts b/packages/artifact/src/internal/download/download-artifact.ts index afd80d9b..9f9b47a6 100644 --- a/packages/artifact/src/internal/download/download-artifact.ts +++ b/packages/artifact/src/internal/download/download-artifact.ts @@ -9,6 +9,7 @@ import { DownloadArtifactResponse } from '../shared/interfaces' import {getUserAgentString} from '../shared/user-agent' +import { getGitHubWorkspaceDir } from '../shared/config' const scrubQueryParameters = (url: string): string => { const parsed = new URL(url) @@ -54,7 +55,7 @@ export async function downloadArtifact( token: string, options?: DownloadArtifactOptions ): Promise { - let downloadPath = options?.path || process.cwd() // TODO: make this align with GITHUB_WORKSPACE + let downloadPath = options?.path || getGitHubWorkspaceDir() if (options?.createArtifactFolder) { downloadPath = path.join(downloadPath, 'my-artifact') // TODO: need to pass artifact name } diff --git a/packages/artifact/src/internal/shared/config.ts b/packages/artifact/src/internal/shared/config.ts index dabb853b..8d8a1668 100644 --- a/packages/artifact/src/internal/shared/config.ts +++ b/packages/artifact/src/internal/shared/config.ts @@ -26,3 +26,11 @@ export function isGhes(): boolean { ) return ghUrl.hostname.toUpperCase() !== 'GITHUB.COM' } + +export function getGitHubWorkspaceDir(): string { + const ghWorkspaceDir = process.env['GITHUB_WORKSPACE'] + if (!ghWorkspaceDir) { + throw new Error('Unable to get the GITHUB_WORKSPACE env variable') + } + return ghWorkspaceDir +} From 6adf053d36890f4d0afc88a11fb634e40a6737c0 Mon Sep 17 00:00:00 2001 From: Bethany Date: Tue, 22 Aug 2023 11:47:14 -0700 Subject: [PATCH 08/17] prettier --- packages/artifact/src/internal/download/download-artifact.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/artifact/src/internal/download/download-artifact.ts b/packages/artifact/src/internal/download/download-artifact.ts index 9f9b47a6..a799539e 100644 --- a/packages/artifact/src/internal/download/download-artifact.ts +++ b/packages/artifact/src/internal/download/download-artifact.ts @@ -9,7 +9,7 @@ import { DownloadArtifactResponse } from '../shared/interfaces' import {getUserAgentString} from '../shared/user-agent' -import { getGitHubWorkspaceDir } from '../shared/config' +import {getGitHubWorkspaceDir} from '../shared/config' const scrubQueryParameters = (url: string): string => { const parsed = new URL(url) From ced07aa89c5da3cec1848b9424dcce8c85607ba0 Mon Sep 17 00:00:00 2001 From: Bethany Date: Wed, 23 Aug 2023 06:47:51 -0700 Subject: [PATCH 09/17] Use options to specify download folder --- .../artifact/src/internal/download/download-artifact.ts | 4 ++-- packages/artifact/src/internal/shared/interfaces.ts | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/artifact/src/internal/download/download-artifact.ts b/packages/artifact/src/internal/download/download-artifact.ts index a799539e..c8bf2cd3 100644 --- a/packages/artifact/src/internal/download/download-artifact.ts +++ b/packages/artifact/src/internal/download/download-artifact.ts @@ -56,8 +56,8 @@ export async function downloadArtifact( options?: DownloadArtifactOptions ): Promise { let downloadPath = options?.path || getGitHubWorkspaceDir() - if (options?.createArtifactFolder) { - downloadPath = path.join(downloadPath, 'my-artifact') // TODO: need to pass artifact name + if (options?.createArtifactFolderName) { + downloadPath = path.join(downloadPath, options?.createArtifactFolderName) } if (!(await exists(downloadPath))) { diff --git a/packages/artifact/src/internal/shared/interfaces.ts b/packages/artifact/src/internal/shared/interfaces.ts index 4f9a5c43..d8132d7b 100644 --- a/packages/artifact/src/internal/shared/interfaces.ts +++ b/packages/artifact/src/internal/shared/interfaces.ts @@ -93,10 +93,10 @@ export interface DownloadArtifactOptions { path?: string /** - * Specifies if a root folder with the artifact name is created for the artifact that is downloaded - * Zip contents are expanded into this folder. Defaults to false if not specified + * Specifies if a root folder with the given name is created for the artifact that is downloaded + * Zip contents are expanded into this folder. A folder will not be created if not specified. * */ - createArtifactFolder?: boolean + createArtifactFolderName?: string } /***************************************************************************** From b4f8e602b2aee7ed1d141afda970e07225317d05 Mon Sep 17 00:00:00 2001 From: Bethany Date: Wed, 23 Aug 2023 07:21:01 -0700 Subject: [PATCH 10/17] remove folder option in favor of path --- .../artifact/src/internal/download/download-artifact.ts | 3 --- packages/artifact/src/internal/shared/interfaces.ts | 6 ------ 2 files changed, 9 deletions(-) diff --git a/packages/artifact/src/internal/download/download-artifact.ts b/packages/artifact/src/internal/download/download-artifact.ts index c8bf2cd3..64abe289 100644 --- a/packages/artifact/src/internal/download/download-artifact.ts +++ b/packages/artifact/src/internal/download/download-artifact.ts @@ -56,9 +56,6 @@ export async function downloadArtifact( options?: DownloadArtifactOptions ): Promise { let downloadPath = options?.path || getGitHubWorkspaceDir() - if (options?.createArtifactFolderName) { - downloadPath = path.join(downloadPath, options?.createArtifactFolderName) - } if (!(await exists(downloadPath))) { core.debug( diff --git a/packages/artifact/src/internal/shared/interfaces.ts b/packages/artifact/src/internal/shared/interfaces.ts index d8132d7b..7e3e862f 100644 --- a/packages/artifact/src/internal/shared/interfaces.ts +++ b/packages/artifact/src/internal/shared/interfaces.ts @@ -91,12 +91,6 @@ export interface DownloadArtifactOptions { * Denotes where the artifact will be downloaded to. If not specified then the artifact is download to GITHUB_WORKSPACE */ path?: string - - /** - * Specifies if a root folder with the given name is created for the artifact that is downloaded - * Zip contents are expanded into this folder. A folder will not be created if not specified. - * */ - createArtifactFolderName?: string } /***************************************************************************** From 88f749f68653a31ae33525e39fe25165b511d6ae Mon Sep 17 00:00:00 2001 From: Bethany Date: Wed, 23 Aug 2023 07:28:17 -0700 Subject: [PATCH 11/17] lint --- packages/artifact/src/internal/download/download-artifact.ts | 3 +-- packages/artifact/src/internal/upload/upload-artifact.ts | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/artifact/src/internal/download/download-artifact.ts b/packages/artifact/src/internal/download/download-artifact.ts index 64abe289..32ac56ad 100644 --- a/packages/artifact/src/internal/download/download-artifact.ts +++ b/packages/artifact/src/internal/download/download-artifact.ts @@ -1,4 +1,3 @@ -import path from 'path' import fs from 'fs/promises' import * as github from '@actions/github' import * as core from '@actions/core' @@ -55,7 +54,7 @@ export async function downloadArtifact( token: string, options?: DownloadArtifactOptions ): Promise { - let downloadPath = options?.path || getGitHubWorkspaceDir() + const downloadPath = options?.path || getGitHubWorkspaceDir() if (!(await exists(downloadPath))) { core.debug( diff --git a/packages/artifact/src/internal/upload/upload-artifact.ts b/packages/artifact/src/internal/upload/upload-artifact.ts index 01b37626..55911476 100644 --- a/packages/artifact/src/internal/upload/upload-artifact.ts +++ b/packages/artifact/src/internal/upload/upload-artifact.ts @@ -97,7 +97,7 @@ export async function uploadArtifact( workflowRunBackendId: backendIds.workflowRunBackendId, workflowJobRunBackendId: backendIds.workflowJobRunBackendId, name, - size: uploadResult.uploadSize!.toString() + size: uploadResult.uploadSize ? uploadResult.uploadSize.toString() : '0' } if (uploadResult.md5Hash) { From b2da9aa12c1cea5fccc411c1bc0723a3cbd43957 Mon Sep 17 00:00:00 2001 From: Bethany Date: Wed, 23 Aug 2023 07:35:23 -0700 Subject: [PATCH 12/17] use string interpolation --- packages/artifact/src/internal/find/get-artifact.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/artifact/src/internal/find/get-artifact.ts b/packages/artifact/src/internal/find/get-artifact.ts index 16cd163b..87f65773 100644 --- a/packages/artifact/src/internal/find/get-artifact.ts +++ b/packages/artifact/src/internal/find/get-artifact.ts @@ -53,7 +53,7 @@ export async function getArtifact( ) if (getArtifactResp.status !== 200) { - core.warning('non-200 response from GitHub API: ${getArtifactResp.status}') + core.warning(`non-200 response from GitHub API: ${getArtifactResp.status}`) return { success: false } @@ -67,7 +67,9 @@ export async function getArtifact( } if (getArtifactResp.data.artifacts.length > 1) { - core.warning('more than one artifact found, returning first') + core.warning( + 'more than one artifact found for a single name, returning first' + ) } return { From 4b6a4d80e13a01ecbf41e39098883d69cb54c0d3 Mon Sep 17 00:00:00 2001 From: Bethany Date: Wed, 23 Aug 2023 10:12:06 -0700 Subject: [PATCH 13/17] use inline eslint disable --- .eslintignore | 1 - packages/artifact/src/internal/shared/user-agent.ts | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.eslintignore b/.eslintignore index a0478cef..3ac58072 100644 --- a/.eslintignore +++ b/.eslintignore @@ -3,4 +3,3 @@ packages/*/node_modules/ packages/*/lib/ packages/glob/__tests__/_temp packages/*/src/generated/*/ -packages/artifact/src/internal/shared/user-agent.ts diff --git a/packages/artifact/src/internal/shared/user-agent.ts b/packages/artifact/src/internal/shared/user-agent.ts index ec8ab1c7..94c07e89 100644 --- a/packages/artifact/src/internal/shared/user-agent.ts +++ b/packages/artifact/src/internal/shared/user-agent.ts @@ -1,3 +1,4 @@ +// eslint-disable-next-line @typescript-eslint/no-var-requires, @typescript-eslint/no-require-imports const packageJson = require('../../../package.json') /** From 06e751600e2c016997ae8b1393bff0c4c79fc52f Mon Sep 17 00:00:00 2001 From: Bethany Date: Wed, 23 Aug 2023 10:36:33 -0700 Subject: [PATCH 14/17] move constants to retry-options --- .../src/internal/find/get-artifact.ts | 14 +++--------- .../src/internal/find/list-artifacts.ts | 22 ++++--------------- .../src/internal/find/retry-options.ts | 10 ++++++--- 3 files changed, 14 insertions(+), 32 deletions(-) diff --git a/packages/artifact/src/internal/find/get-artifact.ts b/packages/artifact/src/internal/find/get-artifact.ts index 87f65773..3e585b33 100644 --- a/packages/artifact/src/internal/find/get-artifact.ts +++ b/packages/artifact/src/internal/find/get-artifact.ts @@ -2,19 +2,11 @@ import {GetArtifactResponse} from '../shared/interfaces' import {getOctokit} from '@actions/github' import {getUserAgentString} from '../shared/user-agent' import {defaults as defaultGitHubOptions} from '@actions/github/lib/utils' -import {RetryOptions, getRetryOptions} from './retry-options' -import {RequestRequestOptions} from '@octokit/types' +import {getRetryOptions} from './retry-options' import {requestLog} from '@octokit/plugin-request-log' import {retry} from '@octokit/plugin-retry' import * as core from '@actions/core' - -type Options = { - log?: Console - userAgent?: string - previews?: string[] - retry?: RetryOptions - request?: RequestRequestOptions -} +import {OctokitOptions} from '@octokit/core/dist-types/types' const maxRetryNumber = 5 const exemptStatusCodes = [400, 401, 403, 404, 422] // https://github.com/octokit/plugin-retry.js/blob/9a2443746c350b3beedec35cf26e197ea318a261/src/index.ts#L14 @@ -32,7 +24,7 @@ export async function getArtifact( defaultGitHubOptions ) - const opts: Options = { + const opts: OctokitOptions = { log: undefined, userAgent: getUserAgentString(), previews: undefined, diff --git a/packages/artifact/src/internal/find/list-artifacts.ts b/packages/artifact/src/internal/find/list-artifacts.ts index 40f5738b..ea15fb36 100644 --- a/packages/artifact/src/internal/find/list-artifacts.ts +++ b/packages/artifact/src/internal/find/list-artifacts.ts @@ -2,26 +2,16 @@ import {info, warning, debug} from '@actions/core' import {getOctokit} from '@actions/github' import {ListArtifactsResponse, Artifact} from '../shared/interfaces' import {getUserAgentString} from '../shared/user-agent' -import {RetryOptions, getRetryOptions} from './retry-options' +import {getRetryOptions} from './retry-options' import {defaults as defaultGitHubOptions} from '@actions/github/lib/utils' import {requestLog} from '@octokit/plugin-request-log' import {retry} from '@octokit/plugin-retry' -import {RequestRequestOptions} from '@octokit/types' - -type Options = { - log?: Console - userAgent?: string - previews?: string[] - retry?: RetryOptions - request?: RequestRequestOptions -} +import {OctokitOptions} from '@octokit/core/dist-types/types' // Limiting to 1000 for perf reasons const maximumArtifactCount = 1000 const paginationCount = 100 const maxNumberOfPages = maximumArtifactCount / paginationCount -const maxRetryNumber = 5 -const exemptStatusCodes = [400, 401, 403, 404, 422] // https://github.com/octokit/plugin-retry.js/blob/9a2443746c350b3beedec35cf26e197ea318a261/src/index.ts#L14 export async function listArtifacts( workflowRunId: number, @@ -34,13 +24,9 @@ export async function listArtifacts( ) const artifacts: Artifact[] = [] - const [retryOpts, requestOpts] = getRetryOptions( - maxRetryNumber, - exemptStatusCodes, - defaultGitHubOptions - ) + const [retryOpts, requestOpts] = getRetryOptions(defaultGitHubOptions) - const opts: Options = { + const opts: OctokitOptions = { log: undefined, userAgent: getUserAgentString(), previews: undefined, diff --git a/packages/artifact/src/internal/find/retry-options.ts b/packages/artifact/src/internal/find/retry-options.ts index 3fe51b5a..ab10309b 100644 --- a/packages/artifact/src/internal/find/retry-options.ts +++ b/packages/artifact/src/internal/find/retry-options.ts @@ -7,10 +7,14 @@ export type RetryOptions = { enabled?: boolean } +// Defaults for fetching artifacts +const defaultMaxRetryNumber = 5 +const defaultExemptStatusCodes = [400, 401, 403, 404, 422] // https://github.com/octokit/plugin-retry.js/blob/9a2443746c350b3beedec35cf26e197ea318a261/src/index.ts#L14 + export function getRetryOptions( - retries: number, - exemptStatusCodes: number[], - defaultOptions: OctokitOptions + defaultOptions: OctokitOptions, + retries: number = defaultMaxRetryNumber, + exemptStatusCodes: number[] = defaultExemptStatusCodes ): [RetryOptions, RequestRequestOptions | undefined] { if (retries <= 0) { return [{enabled: false}, defaultOptions.request] From 291200d54fe224ac22ceff54211a0a083f5c4495 Mon Sep 17 00:00:00 2001 From: Bethany Date: Wed, 23 Aug 2023 10:40:25 -0700 Subject: [PATCH 15/17] include get artifact changes --- packages/artifact/src/internal/find/get-artifact.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/packages/artifact/src/internal/find/get-artifact.ts b/packages/artifact/src/internal/find/get-artifact.ts index 3e585b33..24c38a99 100644 --- a/packages/artifact/src/internal/find/get-artifact.ts +++ b/packages/artifact/src/internal/find/get-artifact.ts @@ -8,9 +8,6 @@ import {retry} from '@octokit/plugin-retry' import * as core from '@actions/core' import {OctokitOptions} from '@octokit/core/dist-types/types' -const maxRetryNumber = 5 -const exemptStatusCodes = [400, 401, 403, 404, 422] // https://github.com/octokit/plugin-retry.js/blob/9a2443746c350b3beedec35cf26e197ea318a261/src/index.ts#L14 - export async function getArtifact( artifactName: string, workflowRunId: number, @@ -18,11 +15,7 @@ export async function getArtifact( repositoryName: string, token: string ): Promise { - const [retryOpts, requestOpts] = getRetryOptions( - maxRetryNumber, - exemptStatusCodes, - defaultGitHubOptions - ) + const [retryOpts, requestOpts] = getRetryOptions(defaultGitHubOptions) const opts: OctokitOptions = { log: undefined, From 67c3b7a45c0383556d59f3f287dd47dfaf0ff970 Mon Sep 17 00:00:00 2001 From: Rob Herley Date: Wed, 23 Aug 2023 23:18:03 -0400 Subject: [PATCH 16/17] add tests for download artifact --- .../__tests__/download-artifact.test.ts | 279 ++++++++++++++++++ .../internal/download/download-artifact.ts | 7 +- 2 files changed, 280 insertions(+), 6 deletions(-) create mode 100644 packages/artifact/__tests__/download-artifact.test.ts diff --git a/packages/artifact/__tests__/download-artifact.test.ts b/packages/artifact/__tests__/download-artifact.test.ts new file mode 100644 index 00000000..bdd128d2 --- /dev/null +++ b/packages/artifact/__tests__/download-artifact.test.ts @@ -0,0 +1,279 @@ +import fs from 'fs' +import * as http from 'http' +import * as net from 'net' +import * as path from 'path' +import * as core from '@actions/core' +import * as github from '@actions/github' +import {HttpClient} from '@actions/http-client' +import type {RestEndpointMethods} from '@octokit/plugin-rest-endpoint-methods/dist-types/generated/method-types' +import archiver from 'archiver' + +import {downloadArtifact} from '../src/internal/download/download-artifact' +import {getUserAgentString} from '../src/internal/shared/user-agent' + +type MockedDownloadArtifact = jest.MockedFunction< + RestEndpointMethods['actions']['downloadArtifact'] +> + +const testDir = path.join(__dirname, '_temp', 'download-artifact') +const fixtures = { + workspaceDir: path.join(testDir, 'workspace'), + exampleArtifact: { + path: path.join(testDir, 'artifact.zip'), + files: [ + { + path: 'hello.txt', + content: 'Hello World!' + }, + { + path: 'goodbye.txt', + content: 'Goodbye World!' + } + ] + }, + artifactID: 1234, + repositoryOwner: 'actions', + repositoryName: 'toolkit', + token: 'ghp_1234567890', + blobStorageUrl: 'https://blob-storage.local?signed=true', +} + +jest.mock('@actions/github', () => ({ + getOctokit: jest.fn().mockReturnValue({ + rest: { + actions: { + downloadArtifact: jest.fn() + } + } + }) +})) + +jest.mock('@actions/http-client') + +// Create a zip archive with the contents of the example artifact +const createTestArchive = async (): Promise => { + const archive = archiver('zip', { + zlib: {level: 9} + }) + for (const file of fixtures.exampleArtifact.files) { + archive.append(file.content, {name: file.path}) + } + archive.finalize() + + return new Promise((resolve, reject) => { + archive.pipe(fs.createWriteStream(fixtures.exampleArtifact.path)) + archive.on('error', reject) + archive.on('finish', resolve) + }) +} + +const expectExtractedArchive = async (dir: string): Promise => { + for (const file of fixtures.exampleArtifact.files) { + const filePath = path.join(dir, file.path) + expect(fs.readFileSync(filePath, 'utf8')).toEqual(file.content) + } +} + +describe('download-artifact', () => { + beforeEach(async () => { + jest.spyOn(core, 'debug').mockImplementation(() => {}) + jest.spyOn(core, 'info').mockImplementation(() => {}) + jest.spyOn(core, 'warning').mockImplementation(() => {}) + + await fs.promises.mkdir(testDir, {recursive: true}) + await createTestArchive() + + process.env['GITHUB_WORKSPACE'] = fixtures.workspaceDir + }) + + afterEach(async () => { + jest.restoreAllMocks() + await fs.promises.rm(testDir, {recursive: true}) + delete process.env['GITHUB_WORKSPACE'] + }) + + it('should successfully download an artifact to $GITHUB_WORKSPACE', async () => { + const downloadArtifactMock = github.getOctokit(fixtures.token).rest.actions + .downloadArtifact as MockedDownloadArtifact + downloadArtifactMock.mockResolvedValueOnce({ + headers: { + location: fixtures.blobStorageUrl + }, + status: 302, + url: '', + data: Buffer.from('') + }) + + const getMock = jest.fn(() => { + const message = new http.IncomingMessage(new net.Socket()) + message.statusCode = 200 + message.push(fs.readFileSync(fixtures.exampleArtifact.path)) + return { + message + } + }) + const httpClientMock = (HttpClient as jest.Mock).mockImplementation(() => { + return { + get: getMock + } + }) + + const response = await downloadArtifact( + fixtures.artifactID, + fixtures.repositoryOwner, + fixtures.repositoryName, + fixtures.token + ) + + expect(downloadArtifactMock).toHaveBeenCalledWith({ + owner: fixtures.repositoryOwner, + repo: fixtures.repositoryName, + artifact_id: fixtures.artifactID, + archive_format: 'zip', + request: { + redirect: 'manual' + } + }) + expect(httpClientMock).toHaveBeenCalledWith(getUserAgentString()) + expect(getMock).toHaveBeenCalledWith(fixtures.blobStorageUrl) + + expectExtractedArchive(fixtures.workspaceDir) + + expect(response.success).toBe(true) + expect(response.downloadPath).toBe(fixtures.workspaceDir) + }) + + it('should successfully download an artifact to user defined path', async () => { + const customPath = path.join(testDir, 'custom') + + const downloadArtifactMock = github.getOctokit(fixtures.token).rest.actions + .downloadArtifact as MockedDownloadArtifact + downloadArtifactMock.mockResolvedValueOnce({ + headers: { + location: fixtures.blobStorageUrl + }, + status: 302, + url: '', + data: Buffer.from('') + }) + + const getMock = jest.fn(() => { + const message = new http.IncomingMessage(new net.Socket()) + message.statusCode = 200 + message.push(fs.readFileSync(fixtures.exampleArtifact.path)) + return { + message + } + }) + const httpClientMock = (HttpClient as jest.Mock).mockImplementation(() => { + return { + get: getMock + } + }) + + const response = await downloadArtifact( + fixtures.artifactID, + fixtures.repositoryOwner, + fixtures.repositoryName, + fixtures.token, + { + path: customPath + } + ) + + expect(downloadArtifactMock).toHaveBeenCalledWith({ + owner: fixtures.repositoryOwner, + repo: fixtures.repositoryName, + artifact_id: fixtures.artifactID, + archive_format: 'zip', + request: { + redirect: 'manual' + } + }) + expect(httpClientMock).toHaveBeenCalledWith(getUserAgentString()) + expect(getMock).toHaveBeenCalledWith(fixtures.blobStorageUrl) + + expectExtractedArchive(customPath) + + expect(response.success).toBe(true) + expect(response.downloadPath).toBe(customPath) + }) + + it('should fail if download artifact API does not respond with location', async () => { + const downloadArtifactMock = github.getOctokit(fixtures.token).rest.actions + .downloadArtifact as MockedDownloadArtifact + downloadArtifactMock.mockResolvedValueOnce({ + headers: {}, + status: 302, + url: '', + data: Buffer.from('') + }) + + await expect( + downloadArtifact( + fixtures.artifactID, + fixtures.repositoryOwner, + fixtures.repositoryName, + fixtures.token + ) + ).rejects.toBeInstanceOf(Error) + + expect(downloadArtifactMock).toHaveBeenCalledWith({ + owner: fixtures.repositoryOwner, + repo: fixtures.repositoryName, + artifact_id: fixtures.artifactID, + archive_format: 'zip', + request: { + redirect: 'manual' + } + }) + }) + + it('should fail if blob storage response is non-200', async () => { + const downloadArtifactMock = github.getOctokit(fixtures.token).rest.actions + .downloadArtifact as MockedDownloadArtifact + downloadArtifactMock.mockResolvedValueOnce({ + headers: { + location: fixtures.blobStorageUrl + }, + status: 302, + url: '', + data: Buffer.from('') + }) + + const getMock = jest.fn(() => { + const message = new http.IncomingMessage(new net.Socket()) + message.statusCode = 500 + message.push('Internal Server Error') + return { + message + } + }) + const httpClientMock = (HttpClient as jest.Mock).mockImplementation(() => { + return { + get: getMock + } + }) + + await expect( + downloadArtifact( + fixtures.artifactID, + fixtures.repositoryOwner, + fixtures.repositoryName, + fixtures.token + ) + ).rejects.toBeInstanceOf(Error) + + expect(downloadArtifactMock).toHaveBeenCalledWith({ + owner: fixtures.repositoryOwner, + repo: fixtures.repositoryName, + artifact_id: fixtures.artifactID, + archive_format: 'zip', + request: { + redirect: 'manual' + } + }) + expect(httpClientMock).toHaveBeenCalledWith(getUserAgentString()) + expect(getMock).toHaveBeenCalledWith(fixtures.blobStorageUrl) + }) +}) diff --git a/packages/artifact/src/internal/download/download-artifact.ts b/packages/artifact/src/internal/download/download-artifact.ts index 32ac56ad..6301bfb5 100644 --- a/packages/artifact/src/internal/download/download-artifact.ts +++ b/packages/artifact/src/internal/download/download-artifact.ts @@ -39,12 +39,7 @@ async function streamExtract(url: string, directory: string): Promise { ) } - return new Promise((resolve, reject) => { - response.message - .pipe(unzipper.Extract({path: directory})) - .on('finish', resolve) - .on('error', reject) - }) + return response.message.pipe(unzipper.Extract({path: directory})).promise() } export async function downloadArtifact( From 9d756b2bc9811c8cc80d362636497b6c52697c51 Mon Sep 17 00:00:00 2001 From: Rob Herley Date: Thu, 24 Aug 2023 09:16:35 -0400 Subject: [PATCH 17/17] linter --- packages/artifact/__tests__/download-artifact.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/artifact/__tests__/download-artifact.test.ts b/packages/artifact/__tests__/download-artifact.test.ts index bdd128d2..e86cb3a7 100644 --- a/packages/artifact/__tests__/download-artifact.test.ts +++ b/packages/artifact/__tests__/download-artifact.test.ts @@ -35,7 +35,7 @@ const fixtures = { repositoryOwner: 'actions', repositoryName: 'toolkit', token: 'ghp_1234567890', - blobStorageUrl: 'https://blob-storage.local?signed=true', + blobStorageUrl: 'https://blob-storage.local?signed=true' } jest.mock('@actions/github', () => ({