diff --git a/packages/artifact/__tests__/config.test.ts b/packages/artifact/__tests__/config.test.ts index 11bbe396..4057ec3e 100644 --- a/packages/artifact/__tests__/config.test.ts +++ b/packages/artifact/__tests__/config.test.ts @@ -1,4 +1,10 @@ import * as config from '../src/internal/shared/config' +import os from 'os' + +// Mock the 'os' module +jest.mock('os', () => ({ + cpus: jest.fn() +})) beforeEach(() => { jest.resetModules() @@ -35,10 +41,12 @@ describe('uploadChunkTimeoutEnv', () => { it('should return default 300000 when no env set', () => { expect(config.getUploadChunkTimeout()).toBe(300000) }) + it('should return value set in ACTIONS_UPLOAD_TIMEOUT_MS', () => { process.env.ACTIONS_UPLOAD_TIMEOUT_MS = '150000' expect(config.getUploadChunkTimeout()).toBe(150000) }) + it('should throw if value set in ACTIONS_UPLOAD_TIMEOUT_MS is invalid', () => { process.env.ACTIONS_UPLOAD_TIMEOUT_MS = 'abc' expect(() => { @@ -46,3 +54,40 @@ describe('uploadChunkTimeoutEnv', () => { }).toThrow() }) }) + +describe('uploadConcurrencyEnv', () => { + it('should return default 32 when cpu num is <= 4', () => { + ;(os.cpus as jest.Mock).mockReturnValue(new Array(4)) + expect(config.getConcurrency()).toBe(32) + }) + + it('should return 16 * num of cpu when cpu num is > 4', () => { + ;(os.cpus as jest.Mock).mockReturnValue(new Array(6)) + expect(config.getConcurrency()).toBe(96) + }) + + it('should return up to 300 max value', () => { + ;(os.cpus as jest.Mock).mockReturnValue(new Array(32)) + expect(config.getConcurrency()).toBe(300) + }) + + it('should return override value when ACTIONS_UPLOAD_CONCURRENCY is set', () => { + ;(os.cpus as jest.Mock).mockReturnValue(new Array(4)) + process.env.ACTIONS_UPLOAD_CONCURRENCY = '10' + expect(config.getConcurrency()).toBe(10) + }) + + it('should throw with invalid value of ACTIONS_UPLOAD_CONCURRENCY', () => { + ;(os.cpus as jest.Mock).mockReturnValue(new Array(4)) + process.env.ACTIONS_UPLOAD_CONCURRENCY = 'abc' + expect(() => { + config.getConcurrency() + }).toThrow() + }) + + it('cannot go over currency cap when override value is greater', () => { + ;(os.cpus as jest.Mock).mockReturnValue(new Array(4)) + process.env.ACTIONS_UPLOAD_CONCURRENCY = '40' + expect(config.getConcurrency()).toBe(32) + }) +}) diff --git a/packages/artifact/src/internal/shared/config.ts b/packages/artifact/src/internal/shared/config.ts index 75bbf8b5..d9d9ae35 100644 --- a/packages/artifact/src/internal/shared/config.ts +++ b/packages/artifact/src/internal/shared/config.ts @@ -1,4 +1,5 @@ import os from 'os' +import {info} from '@actions/core' // Used for controlling the highWaterMark value of the zip that is being streamed // The same value is used as the chunk size that is use during upload to blob storage @@ -44,20 +45,51 @@ export function getGitHubWorkspaceDir(): string { return ghWorkspaceDir } -// From testing, setting this value to 10 yielded best results in terms of reliability and there are no impact on performance either +// Mimics behavior of azcopy: https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azcopy-optimize +// If your machine has fewer than 5 CPUs, then the value of this variable is set to 32. +// Otherwise, the default value is equal to 16 multiplied by the number of CPUs. The maximum value of this variable is 300. +// This value can be lowered with ACTIONS_UPLOAD_CONCURRENCY variable. export function getConcurrency(): number { - return 10 + const numCPUs = os.cpus().length + let concurrencyCap = 32 + + if (numCPUs > 4) { + const concurrency = 16 * numCPUs + concurrencyCap = concurrency > 300 ? 300 : concurrency + } + + const concurrencyOverride = process.env['ACTIONS_UPLOAD_CONCURRENCY'] + if (concurrencyOverride) { + const concurrency = parseInt(concurrencyOverride) + if (isNaN(concurrency)) { + throw new Error( + 'Invalid value set for ACTIONS_UPLOAD_CONCURRENCY env variable' + ) + } + + if (concurrency < concurrencyCap) { + return concurrency + } + + info( + `ACTIONS_UPLOAD_CONCURRENCY is higher than the cap of ${concurrencyCap} based on the number of cpus. Lowering it to the cap.` + ) + } + + return concurrencyCap } export function getUploadChunkTimeout(): number { - const timeoutVar = process.env['ACTIONS_UPLOAD_TIMEOUT_MS'] + const timeoutVar = process.env['ACTIONS_UPLOAD_TIMEOUT_MS'] if (!timeoutVar) { return 300000 // 5 minutes } const timeout = parseInt(timeoutVar) if (isNaN(timeout)) { - throw new Error('Invalid value set for ACTIONS_UPLOAD_TIMEOUT_MS env variable') + throw new Error( + 'Invalid value set for ACTIONS_UPLOAD_TIMEOUT_MS env variable' + ) } return timeout