Tune upload options

packages/cache/__tests__/options.test.ts

@@ -11,8 +11,6 @@ const downloadConcurrency = 8
 const timeoutInMs = 30000
 const segmentTimeoutInMs = 600000
 const lookupOnly = false
-const uploadConcurrency = 4
-const uploadChunkSize = 32 * 1024 * 1024
 
 test('getDownloadOptions sets defaults', async () => {
   const actualOptions = getDownloadOptions()
@@ -43,13 +41,14 @@ test('getDownloadOptions overrides all settings', async () => {
 })
 
 test('getUploadOptions sets defaults', async () => {
+  const expectedOptions: UploadOptions = {
+    uploadConcurrency: 4,
+    uploadChunkSize: 32 * 1024 * 1024,
+    useAzureSdk: false
+  }
   const actualOptions = getUploadOptions()
 
-  expect(actualOptions).toEqual({
-    uploadConcurrency,
-    uploadChunkSize,
-    useAzureSdk
-  })
+  expect(actualOptions).toEqual(expectedOptions)
 })
 
 test('getUploadOptions overrides all settings', async () => {
@@ -64,6 +63,34 @@ test('getUploadOptions overrides all settings', async () => {
   expect(actualOptions).toEqual(expectedOptions)
 })
 
+test('env variables override all getUploadOptions settings', async () => {
+  const expectedOptions: UploadOptions = {
+    uploadConcurrency: 16,
+    uploadChunkSize: 64 * 1024 * 1024,
+    useAzureSdk: true
+  }
+
+  process.env.CACHE_UPLOAD_CONCURRENCY = '16'
+  process.env.CACHE_UPLOAD_CHUNK_SIZE = '64'
+
+  const actualOptions = getUploadOptions(expectedOptions)
+  expect(actualOptions).toEqual(expectedOptions)
+})
+
+test('env variables override all getUploadOptions settings but do not exceed caps', async () => {
+  const expectedOptions: UploadOptions = {
+    uploadConcurrency: 32,
+    uploadChunkSize: 128 * 1024 * 1024,
+    useAzureSdk: true
+  }
+
+  process.env.CACHE_UPLOAD_CONCURRENCY = '64'
+  process.env.CACHE_UPLOAD_CHUNK_SIZE = '256'
+
+  const actualOptions = getUploadOptions(expectedOptions)
+  expect(actualOptions).toEqual(expectedOptions)
+})
+
 test('getDownloadOptions overrides download timeout minutes', async () => {
   const expectedOptions: DownloadOptions = {
     useAzureSdk: false,

packages/cache/src/options.ts

@@ -88,6 +88,7 @@ export interface DownloadOptions {
  * @param copy the original upload options
  */
 export function getUploadOptions(copy?: UploadOptions): UploadOptions {
+  // Defaults if not overriden
   const result: UploadOptions = {
     useAzureSdk: false,
     uploadConcurrency: 4,
@@ -108,6 +109,25 @@ export function getUploadOptions(copy?: UploadOptions): UploadOptions {
     }
   }
 
+  /**
+   * Add env var overrides
+   */
+  // Cap the uploadConcurrency at 32
+  result.uploadConcurrency = !isNaN(
+    Number(process.env['CACHE_UPLOAD_CONCURRENCY'])
+  )
+    ? Math.min(32, Number(process.env['CACHE_UPLOAD_CONCURRENCY']))
+    : result.uploadConcurrency
+  // Cap the uploadChunkSize at 128MiB
+  result.uploadChunkSize = !isNaN(
+    Number(process.env['CACHE_UPLOAD_CHUNK_SIZE'])
+  )
+    ? Math.min(
+        128 * 1024 * 1024,
+        Number(process.env['CACHE_UPLOAD_CHUNK_SIZE']) * 1024 * 1024
+      )
+    : result.uploadChunkSize
+
   core.debug(`Use Azure SDK: ${result.useAzureSdk}`)
   core.debug(`Upload concurrency: ${result.uploadConcurrency}`)
   core.debug(`Upload chunk size: ${result.uploadChunkSize}`)