From 20bcd3b8cbc225042125c1d6601abae27b08c655 Mon Sep 17 00:00:00 2001 From: Rob Herley Date: Mon, 20 Nov 2023 11:08:50 -0500 Subject: [PATCH 1/3] add compression level input --- action.yml | 14 ++++++++++- dist/index.js | 54 ++++++++++++++++++++++++++++++++---------- src/constants.ts | 3 ++- src/input-helper.ts | 12 ++++++++++ src/upload-artifact.ts | 9 ++++++- src/upload-inputs.ts | 5 ++++ 6 files changed, 81 insertions(+), 16 deletions(-) diff --git a/action.yml b/action.yml index 163b233..09e3026 100644 --- a/action.yml +++ b/action.yml @@ -1,7 +1,7 @@ name: 'Upload a Build Artifact' description: 'Upload a build artifact that can be used by subsequent workflow steps' author: 'GitHub' -inputs: +inputs: name: description: 'Artifact name' default: 'artifact' @@ -23,6 +23,18 @@ inputs: Minimum 1 day. Maximum 90 days unless changed from the repository settings page. + compression-level: + description: > + The level of compression for Zlib to be applied to the artifact archive. + The value can range from 0 to 9: + - 0: No compression + - 1: Best speed + - 6: Default compression (same as GNU Gzip) + - 9: Best compression + Higher levels will result in better compression, but will take longer to complete. + For large files that are not easily compressed, a value of 0 is recommended for significantly faster uploads. + default: '6' + outputs: artifact-id: description: > diff --git a/dist/index.js b/dist/index.js index ebe96c0..60c3f83 100644 --- a/dist/index.js +++ b/dist/index.js @@ -5126,12 +5126,16 @@ exports.createArtifactTwirpClient = createArtifactTwirpClient; /***/ }), /***/ 95042: -/***/ ((__unused_webpack_module, exports) => { +/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) { "use strict"; +var __importDefault = (this && this.__importDefault) || function (mod) { + return (mod && mod.__esModule) ? mod : { "default": mod }; +}; Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.getGitHubWorkspaceDir = exports.isGhes = exports.getResultsServiceUrl = exports.getRuntimeToken = exports.getUploadChunkSize = void 0; +exports.getConcurrency = exports.getGitHubWorkspaceDir = exports.isGhes = exports.getResultsServiceUrl = exports.getRuntimeToken = exports.getUploadChunkSize = void 0; +const os_1 = __importDefault(__nccwpck_require__(22037)); // Used for controlling the highWaterMark value of the zip that is being streamed // The same value is used as the chunk size that is use during upload to blob storage function getUploadChunkSize() { @@ -5167,6 +5171,18 @@ function getGitHubWorkspaceDir() { return ghWorkspaceDir; } exports.getGitHubWorkspaceDir = getGitHubWorkspaceDir; +// Mimics behavior of azcopy: https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azcopy-optimize +// If your machine has fewer than 5 CPUs, then the value of this variable is set to 32. +// Otherwise, the default value is equal to 16 multiplied by the number of CPUs. The maximum value of this variable is 300. +function getConcurrency() { + const numCPUs = os_1.default.cpus().length; + if (numCPUs <= 4) { + return 32; + } + const concurrency = 16 * numCPUs; + return concurrency > 300 ? 300 : concurrency; +} +exports.getConcurrency = getConcurrency; //# sourceMappingURL=config.js.map /***/ }), @@ -5309,11 +5325,11 @@ const stream = __importStar(__nccwpck_require__(12781)); function uploadZipToBlobStorage(authenticatedUploadURL, zipUploadStream) { return __awaiter(this, void 0, void 0, function* () { let uploadByteCount = 0; - const maxBuffers = 5; + const maxConcurrency = (0, config_1.getConcurrency)(); const bufferSize = (0, config_1.getUploadChunkSize)(); const blobClient = new storage_blob_1.BlobClient(authenticatedUploadURL); const blockBlobClient = blobClient.getBlockBlobClient(); - core.debug(`Uploading artifact zip to blob storage with maxBuffers: ${maxBuffers}, bufferSize: ${bufferSize}`); + core.debug(`Uploading artifact zip to blob storage with maxConcurrency: ${maxConcurrency}, bufferSize: ${bufferSize}`); const uploadCallback = (progress) => { core.info(`Uploaded bytes ${progress.loadedBytes}`); uploadByteCount = progress.loadedBytes; @@ -5329,7 +5345,7 @@ function uploadZipToBlobStorage(authenticatedUploadURL, zipUploadStream) { zipUploadStream.pipe(hashStream).setEncoding('hex'); // This stream is used to compute a hash of the zip content that gets used. Integrity check try { core.info('Beginning upload of artifact content to blob storage'); - yield blockBlobClient.uploadStream(uploadStream, bufferSize, maxBuffers, options); + yield blockBlobClient.uploadStream(uploadStream, bufferSize, maxConcurrency, options); core.info('Finished uploading artifact content to blob storage!'); hashStream.end(); sha256Hash = hashStream.read(); @@ -5553,7 +5569,7 @@ function uploadArtifact(name, files, rootDirectory, options) { success: false }; } - const zipUploadStream = yield (0, zip_1.createZipUploadStream)(zipSpecification); + const zipUploadStream = yield (0, zip_1.createZipUploadStream)(zipSpecification, options === null || options === void 0 ? void 0 : options.compressionLevel); // get the IDs needed for the artifact creation const backendIds = (0, util_1.getBackendIdsFromToken)(); if (!backendIds.workflowRunBackendId || !backendIds.workflowJobRunBackendId) { @@ -5784,12 +5800,13 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge }); }; Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.createZipUploadStream = exports.ZipUploadStream = void 0; +exports.createZipUploadStream = exports.ZipUploadStream = exports.DEFAULT_COMPRESSION_LEVEL = void 0; const stream = __importStar(__nccwpck_require__(12781)); const archiver = __importStar(__nccwpck_require__(71160)); const core = __importStar(__nccwpck_require__(66526)); const fs_1 = __nccwpck_require__(57147); const config_1 = __nccwpck_require__(95042); +exports.DEFAULT_COMPRESSION_LEVEL = 6; // Custom stream transformer so we can set the highWaterMark property // See https://github.com/nodejs/node/issues/8855 class ZipUploadStream extends stream.Transform { @@ -5804,14 +5821,11 @@ class ZipUploadStream extends stream.Transform { } } exports.ZipUploadStream = ZipUploadStream; -function createZipUploadStream(uploadSpecification) { +function createZipUploadStream(uploadSpecification, compressionLevel = exports.DEFAULT_COMPRESSION_LEVEL) { return __awaiter(this, void 0, void 0, function* () { const zip = archiver.create('zip', { - zlib: { level: 9 } // Sets the compression level. - // Available options are 0-9 - // 0 => no compression - // 1 => fastest with low compression - // 9 => highest compression ratio but the slowest + highWaterMark: (0, config_1.getUploadChunkSize)(), + zlib: { level: compressionLevel } }); // register callbacks for various events during the zip lifecycle zip.on('error', zipErrorCallback); @@ -121087,6 +121101,7 @@ var Inputs; Inputs["Path"] = "path"; Inputs["IfNoFilesFound"] = "if-no-files-found"; Inputs["RetentionDays"] = "retention-days"; + Inputs["CompressionLevel"] = "compression-level"; })(Inputs = exports.Inputs || (exports.Inputs = {})); var NoFileOptions; (function (NoFileOptions) { @@ -121162,6 +121177,16 @@ function getInputs() { core.setFailed('Invalid retention-days'); } } + const compressionLevelStr = core.getInput(constants_1.Inputs.CompressionLevel); + if (compressionLevelStr) { + inputs.compressionLevel = parseInt(compressionLevelStr); + if (isNaN(inputs.compressionLevel)) { + core.setFailed('Invalid compression-level'); + } + if (inputs.compressionLevel < 0 || inputs.compressionLevel > 9) { + core.setFailed('Invalid compression-level. Valid values are 0-9'); + } + } return inputs; } exports.getInputs = getInputs; @@ -121411,6 +121436,9 @@ function run() { if (inputs.retentionDays) { options.retentionDays = inputs.retentionDays; } + if (typeof inputs.compressionLevel !== 'undefined') { + options.compressionLevel = inputs.compressionLevel; + } const uploadResponse = yield artifactClient.uploadArtifact(inputs.artifactName, searchResult.filesToUpload, searchResult.rootDirectory, options); if (uploadResponse.success === false) { core.setFailed(`An error was encountered when uploading ${inputs.artifactName}.`); diff --git a/src/constants.ts b/src/constants.ts index 9d64a61..746dd02 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -3,7 +3,8 @@ export enum Inputs { Name = 'name', Path = 'path', IfNoFilesFound = 'if-no-files-found', - RetentionDays = 'retention-days' + RetentionDays = 'retention-days', + CompressionLevel = 'compression-level' } export enum NoFileOptions { diff --git a/src/input-helper.ts b/src/input-helper.ts index 8344823..f1c1238 100644 --- a/src/input-helper.ts +++ b/src/input-helper.ts @@ -36,5 +36,17 @@ export function getInputs(): UploadInputs { } } + const compressionLevelStr = core.getInput(Inputs.CompressionLevel) + if (compressionLevelStr) { + inputs.compressionLevel = parseInt(compressionLevelStr) + if (isNaN(inputs.compressionLevel)) { + core.setFailed('Invalid compression-level') + } + + if (inputs.compressionLevel < 0 || inputs.compressionLevel > 9) { + core.setFailed('Invalid compression-level. Valid values are 0-9') + } + } + return inputs } diff --git a/src/upload-artifact.ts b/src/upload-artifact.ts index 8059f54..896929d 100644 --- a/src/upload-artifact.ts +++ b/src/upload-artifact.ts @@ -1,5 +1,8 @@ import * as core from '../node_modules/@actions/core/' -import {UploadOptions, create} from '../node_modules/@actions/artifact/lib/artifact' +import { + UploadOptions, + create +} from '../node_modules/@actions/artifact/lib/artifact' import {findFilesToUpload} from './search' import {getInputs} from './input-helper' import {NoFileOptions} from './constants' @@ -43,6 +46,10 @@ async function run(): Promise { options.retentionDays = inputs.retentionDays } + if (typeof inputs.compressionLevel !== 'undefined') { + options.compressionLevel = inputs.compressionLevel + } + const uploadResponse = await artifactClient.uploadArtifact( inputs.artifactName, searchResult.filesToUpload, diff --git a/src/upload-inputs.ts b/src/upload-inputs.ts index 37325df..0f5c17e 100644 --- a/src/upload-inputs.ts +++ b/src/upload-inputs.ts @@ -20,4 +20,9 @@ export interface UploadInputs { * Duration after which artifact will expire in days */ retentionDays: number + + /** + * The level of compression for Zlib to be applied to the artifact archive. + */ + compressionLevel?: number } From 59ef34ab243724e7ad4c9963f232c9535cd88880 Mon Sep 17 00:00:00 2001 From: Rob Herley Date: Mon, 20 Nov 2023 11:28:41 -0500 Subject: [PATCH 2/3] bump for more logs --- dist/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/dist/index.js b/dist/index.js index 60c3f83..32bce2a 100644 --- a/dist/index.js +++ b/dist/index.js @@ -5823,6 +5823,7 @@ class ZipUploadStream extends stream.Transform { exports.ZipUploadStream = ZipUploadStream; function createZipUploadStream(uploadSpecification, compressionLevel = exports.DEFAULT_COMPRESSION_LEVEL) { return __awaiter(this, void 0, void 0, function* () { + core.debug(`Creating Artifact archive with compressionLevel: ${compressionLevel}`); const zip = archiver.create('zip', { highWaterMark: (0, config_1.getUploadChunkSize)(), zlib: { level: compressionLevel } From fa39c94ed2fb3448e5e6789776c205e43dbc3af5 Mon Sep 17 00:00:00 2001 From: Rob Herley Date: Mon, 20 Nov 2023 20:56:30 -0500 Subject: [PATCH 3/3] consume latest @actions/artifact from toolkit --- dist/index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dist/index.js b/dist/index.js index 32bce2a..0906d58 100644 --- a/dist/index.js +++ b/dist/index.js @@ -5032,8 +5032,8 @@ class ArtifactHttpClient { // JSON generated client. request(service, method, contentType, data) { return __awaiter(this, void 0, void 0, function* () { - const url = `${this.baseUrl}/twirp/${service}/${method}`; - (0, core_1.debug)(`Requesting ${url}`); + const url = new URL(`/twirp/${service}/${method}`, this.baseUrl).href; + (0, core_1.debug)(`Requesting: ${url}`); const headers = { 'Content-Type': contentType }; @@ -5155,7 +5155,7 @@ function getResultsServiceUrl() { if (!resultsUrl) { throw new Error('Unable to get the ACTIONS_RESULTS_URL env variable'); } - return resultsUrl; + return new URL(resultsUrl).origin; } exports.getResultsServiceUrl = getResultsServiceUrl; function isGhes() {