From 86e2ed1cf5bcdda6771820a827919a2464c6180a Mon Sep 17 00:00:00 2001 From: Prajjwal Date: Tue, 4 Jun 2024 11:04:28 +0530 Subject: [PATCH] changes to multipart download for s3 --- packages/warp-cache/package.json | 2 +- packages/warp-cache/src/cache.ts | 58 ++++++------------- .../src/internal/cacheHttpClient.ts | 10 +++- 3 files changed, 29 insertions(+), 41 deletions(-) diff --git a/packages/warp-cache/package.json b/packages/warp-cache/package.json index 391ec711..f106d4e2 100644 --- a/packages/warp-cache/package.json +++ b/packages/warp-cache/package.json @@ -1,6 +1,6 @@ { "name": "github-actions.warp-cache", - "version": "1.1.15", + "version": "1.1.16", "preview": true, "description": "Github action to use WarpBuild's in-house cache offering", "keywords": [ diff --git a/packages/warp-cache/src/cache.ts b/packages/warp-cache/src/cache.ts index 4bf08219..1a009a94 100644 --- a/packages/warp-cache/src/cache.ts +++ b/packages/warp-cache/src/cache.ts @@ -137,49 +137,29 @@ export async function restoreCache( } try { - let readStream: NodeJS.ReadableStream | undefined - let downloadCommandPipe = getDownloadCommandPipeForWget( - cacheEntry?.s3?.pre_signed_url - ) - await extractStreamingTar( - readStream, - archivePath, - compressionMethod, - downloadCommandPipe + await cacheHttpClient.downloadCache( + cacheEntry.provider, + cacheEntry.s3?.pre_signed_url, + archivePath ) } catch (error) { - core.debug(`Failed to download cache: ${error}`) - core.info( - `Streaming download failed. Likely a cloud provider issue. Retrying with multipart download` - ) - // Wait 1 second - await new Promise(resolve => setTimeout(resolve, 1000)) - - try { - await cacheHttpClient.downloadCache( - cacheEntry.provider, - cacheEntry.s3?.pre_signed_url, - archivePath - ) - } catch (error) { - core.info('Cache Miss. Failed to download cache.') - return undefined - } - - if (core.isDebug()) { - await listTar(archivePath, compressionMethod) - } - - const archiveFileSize = utils.getArchiveFileSizeInBytes(archivePath) - core.info( - `Cache Size: ~${Math.round( - archiveFileSize / (1024 * 1024) - )} MB (${archiveFileSize} B)` - ) - - await extractTar(archivePath, compressionMethod) + core.info('Cache Miss. Failed to download cache.') + return undefined } + if (core.isDebug()) { + await listTar(archivePath, compressionMethod) + } + + const archiveFileSize = utils.getArchiveFileSizeInBytes(archivePath) + core.info( + `Cache Size: ~${Math.round( + archiveFileSize / (1024 * 1024) + )} MB (${archiveFileSize} B)` + ) + + await extractTar(archivePath, compressionMethod) + core.info('Cache restored successfully') break } diff --git a/packages/warp-cache/src/internal/cacheHttpClient.ts b/packages/warp-cache/src/internal/cacheHttpClient.ts index f0f20f32..8154f1c2 100644 --- a/packages/warp-cache/src/internal/cacheHttpClient.ts +++ b/packages/warp-cache/src/internal/cacheHttpClient.ts @@ -9,6 +9,7 @@ import * as crypto from 'crypto' import * as utils from './cacheUtils' import {CompressionMethod} from './constants' +import os from 'os' import { InternalCacheOptions, ITypedResponseWithError, @@ -211,7 +212,14 @@ export async function downloadCache( ): Promise { switch (provider) { case 's3': - await downloadCacheMultiConnection(archiveLocation, archivePath, 8) + { + const numberOfConnections = 2 + os.cpus().length + await downloadCacheMultiConnection( + archiveLocation, + archivePath, + Math.min(numberOfConnections, 30) + ) + } break case 'gcs': { if (!gcsToken) {