1
0
Fork 0

Refactor cache upload functionality and improve test cases

pull/1882/head
Bassem Dghaidi 2024-11-28 03:52:09 -08:00 committed by GitHub
parent c5a5de05f6
commit df166709a3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 44 additions and 43 deletions

View File

@ -1,14 +1,14 @@
import * as core from '@actions/core' import * as core from '@actions/core'
import * as path from 'path' import * as path from 'path'
import {saveCache} from '../src/cache' import { saveCache } from '../src/cache'
import * as cacheUtils from '../src/internal/cacheUtils' import * as cacheUtils from '../src/internal/cacheUtils'
import {CacheFilename, CompressionMethod} from '../src/internal/constants' import { CacheFilename, CompressionMethod } from '../src/internal/constants'
import * as config from '../src/internal/config' import * as config from '../src/internal/config'
import * as tar from '../src/internal/tar' import * as tar from '../src/internal/tar'
import {CacheServiceClientJSON} from '../src/generated/results/api/v1/cache.twirp' import { CacheServiceClientJSON } from '../src/generated/results/api/v1/cache.twirp'
import * as uploadCacheModule from '../src/internal/blob/upload-cache' import * as uploadCacheModule from '../src/internal/uploadUtils'
import {BlobUploadCommonResponse} from '@azure/storage-blob' import { BlobUploadCommonResponse } from '@azure/storage-blob'
import {InvalidResponseError} from '../src/internal/shared/errors' import { InvalidResponseError } from '../src/internal/shared/errors'
let logDebugMock: jest.SpyInstance let logDebugMock: jest.SpyInstance
@ -28,11 +28,11 @@ jest.mock('@azure/storage-blob', () => ({
beforeAll(() => { beforeAll(() => {
process.env['ACTIONS_RUNTIME_TOKEN'] = 'token' process.env['ACTIONS_RUNTIME_TOKEN'] = 'token'
jest.spyOn(console, 'log').mockImplementation(() => {}) jest.spyOn(console, 'log').mockImplementation(() => { })
jest.spyOn(core, 'debug').mockImplementation(() => {}) jest.spyOn(core, 'debug').mockImplementation(() => { })
jest.spyOn(core, 'info').mockImplementation(() => {}) jest.spyOn(core, 'info').mockImplementation(() => { })
jest.spyOn(core, 'warning').mockImplementation(() => {}) jest.spyOn(core, 'warning').mockImplementation(() => { })
jest.spyOn(core, 'error').mockImplementation(() => {}) jest.spyOn(core, 'error').mockImplementation(() => { })
jest.spyOn(cacheUtils, 'resolvePaths').mockImplementation(async filePaths => { jest.spyOn(cacheUtils, 'resolvePaths').mockImplementation(async filePaths => {
return filePaths.map(x => path.resolve(x)) return filePaths.map(x => path.resolve(x))
}) })
@ -100,7 +100,7 @@ test('create cache entry failure', async () => {
const createCacheEntryMock = jest const createCacheEntryMock = jest
.spyOn(CacheServiceClientJSON.prototype, 'CreateCacheEntry') .spyOn(CacheServiceClientJSON.prototype, 'CreateCacheEntry')
.mockReturnValue(Promise.resolve({ok: false, signedUploadUrl: ''})) .mockReturnValue(Promise.resolve({ ok: false, signedUploadUrl: '' }))
const createTarMock = jest.spyOn(tar, 'createTar') const createTarMock = jest.spyOn(tar, 'createTar')
const finalizeCacheEntryMock = jest.spyOn( const finalizeCacheEntryMock = jest.spyOn(
@ -116,8 +116,8 @@ test('create cache entry failure', async () => {
.spyOn(cacheUtils, 'getArchiveFileSizeInBytes') .spyOn(cacheUtils, 'getArchiveFileSizeInBytes')
.mockReturnValueOnce(archiveFileSize) .mockReturnValueOnce(archiveFileSize)
const cacheVersion = cacheUtils.getCacheVersion(paths, compression) const cacheVersion = cacheUtils.getCacheVersion(paths, compression)
const uploadCacheFileMock = jest const uploadCacheArchiveSDKMock = jest
.spyOn(uploadCacheModule, 'uploadCacheFile') .spyOn(uploadCacheModule, 'uploadCacheArchiveSDK')
.mockReturnValueOnce( .mockReturnValueOnce(
Promise.resolve({ Promise.resolve({
_response: { _response: {
@ -139,7 +139,7 @@ test('create cache entry failure', async () => {
expect(createTarMock).toHaveBeenCalledTimes(1) expect(createTarMock).toHaveBeenCalledTimes(1)
expect(getCompressionMock).toHaveBeenCalledTimes(1) expect(getCompressionMock).toHaveBeenCalledTimes(1)
expect(finalizeCacheEntryMock).toHaveBeenCalledTimes(0) expect(finalizeCacheEntryMock).toHaveBeenCalledTimes(0)
expect(uploadCacheFileMock).toHaveBeenCalledTimes(0) expect(uploadCacheArchiveSDKMock).toHaveBeenCalledTimes(0)
}) })
test('finalize save cache failure', async () => { test('finalize save cache failure', async () => {
@ -152,12 +152,12 @@ test('finalize save cache failure', async () => {
const createCacheEntryMock = jest const createCacheEntryMock = jest
.spyOn(CacheServiceClientJSON.prototype, 'CreateCacheEntry') .spyOn(CacheServiceClientJSON.prototype, 'CreateCacheEntry')
.mockReturnValue( .mockReturnValue(
Promise.resolve({ok: true, signedUploadUrl: signedUploadURL}) Promise.resolve({ ok: true, signedUploadUrl: signedUploadURL })
) )
const createTarMock = jest.spyOn(tar, 'createTar') const createTarMock = jest.spyOn(tar, 'createTar')
const uploadCacheMock = jest.spyOn(uploadCacheModule, 'uploadCacheFile') const uploadCacheMock = jest.spyOn(uploadCacheModule, 'uploadCacheArchiveSDK')
uploadCacheMock.mockReturnValueOnce( uploadCacheMock.mockReturnValueOnce(
Promise.resolve({ Promise.resolve({
_response: { _response: {
@ -179,7 +179,7 @@ test('finalize save cache failure', async () => {
const finalizeCacheEntryMock = jest const finalizeCacheEntryMock = jest
.spyOn(CacheServiceClientJSON.prototype, 'FinalizeCacheEntryUpload') .spyOn(CacheServiceClientJSON.prototype, 'FinalizeCacheEntryUpload')
.mockReturnValue(Promise.resolve({ok: false, entryId: ''})) .mockReturnValue(Promise.resolve({ ok: false, entryId: '' }))
const cacheId = await saveCache([paths], key) const cacheId = await saveCache([paths], key)
@ -218,7 +218,7 @@ test('save with uploadCache Server error will fail', async () => {
jest jest
.spyOn(CacheServiceClientJSON.prototype, 'CreateCacheEntry') .spyOn(CacheServiceClientJSON.prototype, 'CreateCacheEntry')
.mockReturnValue( .mockReturnValue(
Promise.resolve({ok: true, signedUploadUrl: signedUploadURL}) Promise.resolve({ ok: true, signedUploadUrl: signedUploadURL })
) )
const archiveFileSize = 1024 const archiveFileSize = 1024
@ -226,7 +226,7 @@ test('save with uploadCache Server error will fail', async () => {
.spyOn(cacheUtils, 'getArchiveFileSizeInBytes') .spyOn(cacheUtils, 'getArchiveFileSizeInBytes')
.mockReturnValueOnce(archiveFileSize) .mockReturnValueOnce(archiveFileSize)
jest jest
.spyOn(uploadCacheModule, 'uploadCacheFile') .spyOn(uploadCacheModule, 'uploadCacheArchiveSDK')
.mockRejectedValueOnce(new InvalidResponseError('boom')) .mockRejectedValueOnce(new InvalidResponseError('boom'))
const cacheId = await saveCache([paths], key) const cacheId = await saveCache([paths], key)
@ -241,14 +241,14 @@ test('uploadFile returns 500', async () => {
jest jest
.spyOn(CacheServiceClientJSON.prototype, 'CreateCacheEntry') .spyOn(CacheServiceClientJSON.prototype, 'CreateCacheEntry')
.mockReturnValue( .mockReturnValue(
Promise.resolve({ok: true, signedUploadUrl: signedUploadURL}) Promise.resolve({ ok: true, signedUploadUrl: signedUploadURL })
) )
const archiveFileSize = 1024 const archiveFileSize = 1024
jest jest
.spyOn(cacheUtils, 'getArchiveFileSizeInBytes') .spyOn(cacheUtils, 'getArchiveFileSizeInBytes')
.mockReturnValueOnce(archiveFileSize) .mockReturnValueOnce(archiveFileSize)
jest.spyOn(uploadCacheModule, 'uploadCacheFile').mockRestore() jest.spyOn(uploadCacheModule, 'uploadCacheArchiveSDK').mockRestore()
uploadFileMock = jest.fn().mockResolvedValueOnce({ uploadFileMock = jest.fn().mockResolvedValueOnce({
_response: { _response: {
@ -279,11 +279,11 @@ test('save with valid inputs uploads a cache', async () => {
jest jest
.spyOn(CacheServiceClientJSON.prototype, 'CreateCacheEntry') .spyOn(CacheServiceClientJSON.prototype, 'CreateCacheEntry')
.mockReturnValue( .mockReturnValue(
Promise.resolve({ok: true, signedUploadUrl: signedUploadURL}) Promise.resolve({ ok: true, signedUploadUrl: signedUploadURL })
) )
const uploadCacheMock = jest const uploadCacheMock = jest
.spyOn(uploadCacheModule, 'uploadCacheFile') .spyOn(uploadCacheModule, 'uploadCacheArchiveSDK')
.mockReturnValueOnce( .mockReturnValueOnce(
Promise.resolve({ Promise.resolve({
_response: { _response: {
@ -300,7 +300,7 @@ test('save with valid inputs uploads a cache', async () => {
const finalizeCacheEntryMock = jest const finalizeCacheEntryMock = jest
.spyOn(CacheServiceClientJSON.prototype, 'FinalizeCacheEntryUpload') .spyOn(CacheServiceClientJSON.prototype, 'FinalizeCacheEntryUpload')
.mockReturnValue(Promise.resolve({ok: true, entryId: cacheId.toString()})) .mockReturnValue(Promise.resolve({ ok: true, entryId: cacheId.toString() }))
const expectedCacheId = await saveCache([paths], key) const expectedCacheId = await saveCache([paths], key)

View File

@ -3,18 +3,18 @@ import * as path from 'path'
import * as utils from './internal/cacheUtils' import * as utils from './internal/cacheUtils'
import * as cacheHttpClient from './internal/cacheHttpClient' import * as cacheHttpClient from './internal/cacheHttpClient'
import * as cacheTwirpClient from './internal/shared/cacheTwirpClient' import * as cacheTwirpClient from './internal/shared/cacheTwirpClient'
import {downloadCacheStorageSDK} from './internal/downloadUtils' import { downloadCacheStorageSDK } from './internal/downloadUtils'
import {getCacheServiceVersion, isGhes} from './internal/config' import { getCacheServiceVersion, isGhes } from './internal/config'
import {DownloadOptions, UploadOptions} from './options' import { DownloadOptions, UploadOptions } from './options'
import {createTar, extractTar, listTar} from './internal/tar' import { createTar, extractTar, listTar } from './internal/tar'
import { import {
CreateCacheEntryRequest, CreateCacheEntryRequest,
FinalizeCacheEntryUploadRequest, FinalizeCacheEntryUploadRequest,
FinalizeCacheEntryUploadResponse, FinalizeCacheEntryUploadResponse,
GetCacheEntryDownloadURLRequest GetCacheEntryDownloadURLRequest
} from './generated/results/api/v1/cache' } from './generated/results/api/v1/cache'
import {CacheFileSizeLimit} from './internal/constants' import { CacheFileSizeLimit } from './internal/constants'
import {uploadCacheFile} from './internal/blob/upload-cache' import { uploadCacheArchiveSDK } from './internal/uploadUtils'
export class ValidationError extends Error { export class ValidationError extends Error {
constructor(message: string) { constructor(message: string) {
super(message) super(message)
@ -275,9 +275,9 @@ async function restoreCacheV2(
response.signedDownloadUrl, response.signedDownloadUrl,
archivePath, archivePath,
options || options ||
({ ({
timeoutInMs: 30000 timeoutInMs: 30000
} as DownloadOptions) } as DownloadOptions)
) )
const archiveFileSize = utils.getArchiveFileSizeInBytes(archivePath) const archiveFileSize = utils.getArchiveFileSizeInBytes(archivePath)
@ -414,9 +414,9 @@ async function saveCacheV1(
} else if (reserveCacheResponse?.statusCode === 400) { } else if (reserveCacheResponse?.statusCode === 400) {
throw new Error( throw new Error(
reserveCacheResponse?.error?.message ?? reserveCacheResponse?.error?.message ??
`Cache size of ~${Math.round( `Cache size of ~${Math.round(
archiveFileSize / (1024 * 1024) archiveFileSize / (1024 * 1024)
)} MB (${archiveFileSize} B) is over the data cap limit, not saving cache.` )} MB (${archiveFileSize} B) is over the data cap limit, not saving cache.`
) )
} else { } else {
throw new ReserveCacheError( throw new ReserveCacheError(
@ -521,7 +521,7 @@ async function saveCacheV2(
} }
core.debug(`Attempting to upload cache located at: ${archivePath}`) core.debug(`Attempting to upload cache located at: ${archivePath}`)
const uploadResponse = await uploadCacheFile( const uploadResponse = await uploadCacheArchiveSDK(
response.signedUploadUrl, response.signedUploadUrl,
archivePath archivePath
) )

View File

@ -5,12 +5,13 @@ import {
BlockBlobClient, BlockBlobClient,
BlockBlobParallelUploadOptions BlockBlobParallelUploadOptions
} from '@azure/storage-blob' } from '@azure/storage-blob'
import {InvalidResponseError} from '../shared/errors' import { InvalidResponseError } from './shared/errors'
export async function uploadCacheFile( export async function uploadCacheArchiveSDK
signedUploadURL: string, (
archivePath: string signedUploadURL: string,
): Promise<BlobUploadCommonResponse> { archivePath: string
): Promise<BlobUploadCommonResponse> {
// Specify data transfer options // Specify data transfer options
const uploadOptions: BlockBlobParallelUploadOptions = { const uploadOptions: BlockBlobParallelUploadOptions = {
blockSize: 4 * 1024 * 1024, // 4 MiB max block size blockSize: 4 * 1024 * 1024, // 4 MiB max block size