1
0
Fork 0

adds basic GCP download as fallback to multipart

pull/1935/head
Prajjwal 2024-05-17 17:45:06 +05:30
parent ef01eae26e
commit ac477ea9c4
4 changed files with 73 additions and 8 deletions

View File

@ -1,6 +1,6 @@
{
"name": "github-actions.warp-cache",
"version": "1.1.5",
"version": "1.1.6",
"preview": true,
"description": "Github action to use WarpBuild's in-house cache offering",
"keywords": [

View File

@ -225,14 +225,31 @@ export async function restoreCache(
downloadCommandPipe
)
} catch (error) {
core.info(`Streaming download failed. Retrying: ${error}`)
// Try to download the cache using the non-streaming method
await cacheHttpClient.downloadCache(
cacheEntry.provider,
archiveLocation,
archivePath,
cacheEntry.gcs?.short_lived_token?.access_token ?? ''
core.info(
`Streaming download failed. Retrying with multipart: ${error}`
)
// Wait 1 second
await new Promise(resolve => setTimeout(resolve, 1000))
// Try to download the cache using the non-streaming method
try {
await cacheHttpClient.downloadCache(
cacheEntry.provider,
archiveLocation,
archivePath,
cacheEntry.gcs?.short_lived_token?.access_token ?? ''
)
} catch (error) {
core.info(
`Multipart Download failed. Retrying with basic download: ${error}`
)
await new Promise(resolve => setTimeout(resolve, 1000))
await cacheHttpClient.downloadCacheSingleThread(
cacheEntry.provider,
archiveLocation,
archivePath,
cacheEntry.gcs?.short_lived_token?.access_token ?? ''
)
}
if (core.isDebug()) {
await listTar(archivePath, compressionMethod)

View File

@ -15,6 +15,7 @@ import {
InternalS3CompletedPart
} from './contracts'
import {
downloadCacheGCP,
downloadCacheMultiConnection,
downloadCacheMultipartGCP,
downloadCacheStreamingGCP
@ -230,6 +231,33 @@ export async function downloadCache(
}
}
export async function downloadCacheSingleThread(
provider: string,
archiveLocation: string,
archivePath: string,
gcsToken?: string
): Promise<void> {
switch (provider) {
case 's3':
break
case 'gcs': {
if (!gcsToken) {
throw new Error(
'Unable to download cache from GCS. GCP token is not provided.'
)
}
const oauth2Client = new OAuth2Client()
oauth2Client.setCredentials({access_token: gcsToken})
const storage = new Storage({
authClient: oauth2Client
})
await downloadCacheGCP(storage, archiveLocation, archivePath)
break
}
}
}
export function downloadCacheStreaming(
provider: string,
archiveLocation: string,

View File

@ -322,6 +322,26 @@ export async function downloadCacheMultipartGCP(
}
}
export async function downloadCacheGCP(
storage: Storage,
archiveLocation: string,
archivePath: string
) {
try {
const {bucketName, objectName} =
utils.retrieveGCSBucketAndObjectName(archiveLocation)
await storage.bucket(bucketName).file(objectName).download({
destination: archivePath,
validation: 'crc32c'
})
} catch (error) {
core.debug(`Failed to download cache: ${error}`)
core.error(`Failed to download cache.`)
throw error
}
}
/**
* Download the cache to a provider writable stream using GCloud SDK
*