1
0
Fork 0

Add download cache v2

Link-/blobcache-spike
Bassem Dghaidi 2024-06-17 01:17:10 -07:00 committed by GitHub
parent 5e5faf73fc
commit 5afc042a74
2 changed files with 139 additions and 1 deletions

View File

@ -6,8 +6,14 @@ import * as cacheHttpClient from './internal/cacheHttpClient'
import * as cacheTwirpClient from './internal/cacheTwirpClient' import * as cacheTwirpClient from './internal/cacheTwirpClient'
import {createTar, extractTar, listTar} from './internal/tar' import {createTar, extractTar, listTar} from './internal/tar'
import {DownloadOptions, UploadOptions} from './options' import {DownloadOptions, UploadOptions} from './options'
import {GetCacheBlobUploadURLRequest, GetCacheBlobUploadURLResponse} from './generated/results/api/v1/blobcache' import {
GetCacheBlobUploadURLRequest,
GetCacheBlobUploadURLResponse,
GetCachedBlobRequest,
GetCachedBlobResponse
} from './generated/results/api/v1/blobcache'
import {UploadCacheStream} from './internal/v2/upload-cache' import {UploadCacheStream} from './internal/v2/upload-cache'
import {StreamExtract} from './internal/v2/download-cache'
import { import {
UploadZipSpecification, UploadZipSpecification,
getUploadZipSpecification getUploadZipSpecification
@ -81,6 +87,23 @@ export async function restoreCache(
): Promise<string | undefined> { ): Promise<string | undefined> {
checkPaths(paths) checkPaths(paths)
console.debug(`Cache Service Version: ${CacheServiceVersion}`)
switch (CacheServiceVersion) {
case "v2":
return await restoreCachev2(paths, primaryKey, restoreKeys, options, enableCrossOsArchive)
case "v1":
default:
return await restoreCachev1(paths, primaryKey, restoreKeys, options, enableCrossOsArchive)
}
}
async function restoreCachev1(
paths: string[],
primaryKey: string,
restoreKeys?: string[],
options?: DownloadOptions,
enableCrossOsArchive = false
) {
restoreKeys = restoreKeys || [] restoreKeys = restoreKeys || []
const keys = [primaryKey, ...restoreKeys] const keys = [primaryKey, ...restoreKeys]
@ -162,6 +185,54 @@ export async function restoreCache(
return undefined return undefined
} }
async function restoreCachev2(
paths: string[],
primaryKey: string,
restoreKeys?: string[],
options?: DownloadOptions,
enableCrossOsArchive = false
) {
restoreKeys = restoreKeys || []
const keys = [primaryKey, ...restoreKeys]
core.debug('Resolved Keys:')
core.debug(JSON.stringify(keys))
if (keys.length > 10) {
throw new ValidationError(
`Key Validation Error: Keys are limited to a maximum of 10.`
)
}
for (const key of keys) {
checkKey(key)
}
try {
const twirpClient = cacheTwirpClient.internalBlobCacheTwirpClient()
const getSignedDownloadURLRequest: GetCachedBlobRequest = {
owner: "github",
keys: keys,
}
const signedDownloadURL: GetCachedBlobResponse = await twirpClient.GetCachedBlob(getSignedDownloadURLRequest)
core.info(`GetCachedBlobResponse: ${JSON.stringify(signedDownloadURL)}`)
if (signedDownloadURL.blobs.length === 0) {
// Cache not found
core.warning(`Cache not found for keys: ${keys.join(', ')}`)
return undefined
}
core.info(`Starting download of artifact to: ${paths[0]}`)
await StreamExtract(signedDownloadURL.blobs[0].signedUrl, paths[0])
core.info(`Artifact download completed successfully.`)
} catch (error) {
throw new Error(`Unable to download and extract cache: ${error.message}`)
}
return undefined
}
/** /**
* Saves a list of files with the specified key * Saves a list of files with the specified key
* *

View File

@ -0,0 +1,67 @@
import * as core from '@actions/core'
import * as httpClient from '@actions/http-client'
import unzip from 'unzip-stream'
const packageJson = require('../../../package.json')
export async function StreamExtract(url: string, directory: string): Promise<void> {
let retryCount = 0
while (retryCount < 5) {
try {
await streamExtractExternal(url, directory)
return
} catch (error) {
retryCount++
core.debug(
`Failed to download cache after ${retryCount} retries due to ${error.message}. Retrying in 5 seconds...`
)
// wait 5 seconds before retrying
await new Promise(resolve => setTimeout(resolve, 5000))
}
}
throw new Error(`Cache download failed after ${retryCount} retries.`)
}
export async function streamExtractExternal(
url: string,
directory: string
): Promise<void> {
const client = new httpClient.HttpClient(`@actions/cache-${packageJson.version}`)
const response = await client.get(url)
if (response.message.statusCode !== 200) {
throw new Error(
`Unexpected HTTP response from blob storage: ${response.message.statusCode} ${response.message.statusMessage}`
)
}
const timeout = 30 * 1000 // 30 seconds
return new Promise((resolve, reject) => {
const timerFn = (): void => {
response.message.destroy(
new Error(`Blob storage chunk did not respond in ${timeout}ms`)
)
}
const timer = setTimeout(timerFn, timeout)
response.message
.on('data', () => {
timer.refresh()
})
.on('error', (error: Error) => {
core.debug(
`response.message: Cache download failed: ${error.message}`
)
clearTimeout(timer)
reject(error)
})
.pipe(unzip.Extract({path: directory}))
.on('close', () => {
clearTimeout(timer)
resolve()
})
.on('error', (error: Error) => {
reject(error)
})
})
}