Merge pull request #6 from WarpBuilds/fix-change-to-multipart-s3

changes to multipart download for s3

packages/warp-cache/package.json

@@ -1,6 +1,6 @@
 {
   "name": "github-actions.warp-cache",
-  "version": "1.1.15",
+  "version": "1.1.16",
   "preview": true,
   "description": "Github action to use WarpBuild's in-house cache offering",
   "keywords": [

packages/warp-cache/src/cache.ts

@@ -136,25 +136,6 @@ export async function restoreCache(
           return cacheKey
         }
 
-        try {
-          let readStream: NodeJS.ReadableStream | undefined
-          let downloadCommandPipe = getDownloadCommandPipeForWget(
-            cacheEntry?.s3?.pre_signed_url
-          )
-          await extractStreamingTar(
-            readStream,
-            archivePath,
-            compressionMethod,
-            downloadCommandPipe
-          )
-        } catch (error) {
-          core.debug(`Failed to download cache: ${error}`)
-          core.info(
-            `Streaming download failed. Likely a cloud provider issue. Retrying with multipart download`
-          )
-          // Wait 1 second
-          await new Promise(resolve => setTimeout(resolve, 1000))
-
         try {
           await cacheHttpClient.downloadCache(
             cacheEntry.provider,
@@ -178,7 +159,6 @@ export async function restoreCache(
         )
 
         await extractTar(archivePath, compressionMethod)
-        }
 
         core.info('Cache restored successfully')
         break

packages/warp-cache/src/internal/cacheHttpClient.ts

@@ -9,6 +9,7 @@ import * as crypto from 'crypto'
 
 import * as utils from './cacheUtils'
 import {CompressionMethod} from './constants'
+import os from 'os'
 import {
   InternalCacheOptions,
   ITypedResponseWithError,
@@ -211,7 +212,14 @@ export async function downloadCache(
 ): Promise<void> {
   switch (provider) {
     case 's3':
-      await downloadCacheMultiConnection(archiveLocation, archivePath, 8)
+      {
+        const numberOfConnections = 2 + os.cpus().length
+        await downloadCacheMultiConnection(
+          archiveLocation,
+          archivePath,
+          Math.min(numberOfConnections, 30)
+        )
+      }
       break
     case 'gcs': {
       if (!gcsToken) {