From 5670c35d065111ec595cda88908c54dfcefa69ad Mon Sep 17 00:00:00 2001 From: Prajjwal Date: Mon, 3 Jun 2024 16:24:00 +0530 Subject: [PATCH 1/4] adds streaming download for s3 --- packages/warp-cache/src/cache.ts | 40 +++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/packages/warp-cache/src/cache.ts b/packages/warp-cache/src/cache.ts index d27d1329..33b98d07 100644 --- a/packages/warp-cache/src/cache.ts +++ b/packages/warp-cache/src/cache.ts @@ -138,11 +138,36 @@ export async function restoreCache( return cacheKey } - await cacheHttpClient.downloadCache( - cacheEntry.provider, - cacheEntry.s3?.pre_signed_url, - archivePath - ) + try { + let readStream: NodeJS.ReadableStream | undefined + let downloadCommandPipe = getDownloadCommandPipeForWget( + cacheEntry?.s3?.pre_signed_url + ) + await extractStreamingTar( + readStream, + archivePath, + compressionMethod, + downloadCommandPipe + ) + } catch (error) { + core.debug(`Failed to download cache: ${error}`) + core.info( + `Streaming download failed. Likely a cloud provider issue. Retrying with multipart download` + ) + // Wait 1 second + await new Promise(resolve => setTimeout(resolve, 1000)) + + try { + await cacheHttpClient.downloadCache( + cacheEntry.provider, + cacheEntry.s3?.pre_signed_url, + archivePath + ) + } catch (error) { + core.info('Cache Miss. Failed to download cache.') + return undefined + } + } if (core.isDebug()) { await listTar(archivePath, compressionMethod) @@ -343,7 +368,10 @@ export async function saveCache( // Calculate number of chunks required. This is only required if backend is S3 as Google Cloud SDK will do it for us const uploadOptions = getUploadOptions() const maxChunkSize = uploadOptions?.uploadChunkSize ?? 32 * 1024 * 1024 // Default 32MB - const numberOfChunks = Math.floor(archiveFileSize / maxChunkSize) + const numberOfChunks = Math.min( + Math.floor(archiveFileSize / maxChunkSize), + 1 + ) const reserveCacheResponse = await cacheHttpClient.reserveCache( key, numberOfChunks, From 57e2581e9dcc9dd1095f3c1facef98bdd688359d Mon Sep 17 00:00:00 2001 From: Prajjwal Date: Mon, 3 Jun 2024 16:24:40 +0530 Subject: [PATCH 2/4] bumps to 1.1.13 --- packages/warp-cache/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/warp-cache/package.json b/packages/warp-cache/package.json index 31093db6..0ee528d8 100644 --- a/packages/warp-cache/package.json +++ b/packages/warp-cache/package.json @@ -1,6 +1,6 @@ { "name": "github-actions.warp-cache", - "version": "1.1.12", + "version": "1.1.13", "preview": true, "description": "Github action to use WarpBuild's in-house cache offering", "keywords": [ From b719dac88bd979032c0a07295602241da069dbff Mon Sep 17 00:00:00 2001 From: Prajjwal Date: Mon, 3 Jun 2024 16:40:16 +0530 Subject: [PATCH 3/4] adds better error handling --- packages/warp-cache/package.json | 2 +- packages/warp-cache/src/cache.ts | 2 +- packages/warp-cache/src/internal/cacheHttpClient.ts | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/warp-cache/package.json b/packages/warp-cache/package.json index 0ee528d8..fffed1ed 100644 --- a/packages/warp-cache/package.json +++ b/packages/warp-cache/package.json @@ -1,6 +1,6 @@ { "name": "github-actions.warp-cache", - "version": "1.1.13", + "version": "1.1.14", "preview": true, "description": "Github action to use WarpBuild's in-house cache offering", "keywords": [ diff --git a/packages/warp-cache/src/cache.ts b/packages/warp-cache/src/cache.ts index 33b98d07..b21bef5a 100644 --- a/packages/warp-cache/src/cache.ts +++ b/packages/warp-cache/src/cache.ts @@ -368,7 +368,7 @@ export async function saveCache( // Calculate number of chunks required. This is only required if backend is S3 as Google Cloud SDK will do it for us const uploadOptions = getUploadOptions() const maxChunkSize = uploadOptions?.uploadChunkSize ?? 32 * 1024 * 1024 // Default 32MB - const numberOfChunks = Math.min( + const numberOfChunks = Math.max( Math.floor(archiveFileSize / maxChunkSize), 1 ) diff --git a/packages/warp-cache/src/internal/cacheHttpClient.ts b/packages/warp-cache/src/internal/cacheHttpClient.ts index c83f999d..f0f20f32 100644 --- a/packages/warp-cache/src/internal/cacheHttpClient.ts +++ b/packages/warp-cache/src/internal/cacheHttpClient.ts @@ -379,6 +379,9 @@ export async function saveCache( !S3UploadId || !S3UploadKey ) { + core.debug( + `S3 params are not set. Number of Chunks: ${S3NumberOfChunks}, PreSigned URLs: ${S3PreSignedURLs}, Upload ID: ${S3UploadId}, Upload Key: ${S3UploadKey}` + ) throw new Error( 'Unable to upload cache to S3. One of the following required parameters is missing: numberOfChunks, preSignedURLs, uploadId, uploadKey.' ) From d78ac4e8518f3eb8464be803d4112ffbaf05bf40 Mon Sep 17 00:00:00 2001 From: Prajjwal Date: Mon, 3 Jun 2024 17:01:03 +0530 Subject: [PATCH 4/4] removes duplicate extraction --- packages/warp-cache/package.json | 2 +- packages/warp-cache/src/cache.ts | 32 +++++++++++++++----------------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/packages/warp-cache/package.json b/packages/warp-cache/package.json index fffed1ed..391ec711 100644 --- a/packages/warp-cache/package.json +++ b/packages/warp-cache/package.json @@ -1,6 +1,6 @@ { "name": "github-actions.warp-cache", - "version": "1.1.14", + "version": "1.1.15", "preview": true, "description": "Github action to use WarpBuild's in-house cache offering", "keywords": [ diff --git a/packages/warp-cache/src/cache.ts b/packages/warp-cache/src/cache.ts index b21bef5a..4bf08219 100644 --- a/packages/warp-cache/src/cache.ts +++ b/packages/warp-cache/src/cache.ts @@ -123,7 +123,7 @@ export async function restoreCache( ) core.debug(`Archive Path: ${archivePath}`) - let cacheKey: string = '' + let cacheKey = cacheEntry?.cache_entry?.cache_user_given_key ?? primaryKey switch (cacheEntry.provider) { case 's3': { @@ -131,8 +131,6 @@ export async function restoreCache( return undefined } - cacheKey = cacheEntry.s3.pre_signed_url - if (options?.lookupOnly) { core.info('Lookup only - skipping download') return cacheKey @@ -167,20 +165,21 @@ export async function restoreCache( core.info('Cache Miss. Failed to download cache.') return undefined } + + if (core.isDebug()) { + await listTar(archivePath, compressionMethod) + } + + const archiveFileSize = utils.getArchiveFileSizeInBytes(archivePath) + core.info( + `Cache Size: ~${Math.round( + archiveFileSize / (1024 * 1024) + )} MB (${archiveFileSize} B)` + ) + + await extractTar(archivePath, compressionMethod) } - if (core.isDebug()) { - await listTar(archivePath, compressionMethod) - } - - const archiveFileSize = utils.getArchiveFileSizeInBytes(archivePath) - core.info( - `Cache Size: ~${Math.round( - archiveFileSize / (1024 * 1024) - )} MB (${archiveFileSize} B)` - ) - - await extractTar(archivePath, compressionMethod) core.info('Cache restored successfully') break } @@ -189,7 +188,6 @@ export async function restoreCache( if (!cacheEntry.gcs?.cache_key) { return undefined } - cacheKey = cacheEntry.gcs?.cache_key if (options?.lookupOnly) { core.info('Lookup only - skipping download') @@ -280,7 +278,7 @@ export async function restoreCache( } } - return cacheEntry?.cache_entry?.cache_user_given_key ?? cacheKey + return cacheKey } catch (error) { const typedError = error as Error if (typedError.name === ValidationError.name) {