From bfdba95ecee3430fb9010832a0bdda9f1b0aaf06 Mon Sep 17 00:00:00 2001 From: Yaroslav Dynnikov Date: Thu, 26 Nov 2020 01:56:57 +0300 Subject: [PATCH] Make caching more verbose - Print cache size when saving cache similarly to restoring - Print restore success similarly to saving - Print cached file list if debug logging is enabled See also: https://github.com/actions/cache/issues/471 --- packages/cache/__tests__/tar.test.ts | 72 +++++++++++++++++++ packages/cache/src/cache.ts | 10 ++- .../cache/src/internal/cacheHttpClient.ts | 4 ++ packages/cache/src/internal/tar.ts | 27 +++++++ 4 files changed, 112 insertions(+), 1 deletion(-) diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index b891d85b..cafbb543 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -186,3 +186,75 @@ test('gzip create tar', async () => { } ) }) + +test('zstd list tar', async () => { + const execMock = jest.spyOn(exec, 'exec') + + const archivePath = IS_WINDOWS + ? `${process.env['windir']}\\fakepath\\cache.tar` + : 'cache.tar' + const tarPath = 'tar' + + await tar.listTar(archivePath, CompressionMethod.Zstd) + + expect(execMock).toHaveBeenCalledTimes(1) + expect(execMock).toHaveBeenCalledWith( + `"${tarPath}"`, + [ + '--use-compress-program', + 'zstd -d --long=30', + '-tf', + IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, + '-P' + ].concat(IS_WINDOWS ? ['--force-local'] : []), + {cwd: undefined} + ) +}) + +test('zstdWithoutLong list tar', async () => { + const execMock = jest.spyOn(exec, 'exec') + + const archivePath = IS_WINDOWS + ? `${process.env['windir']}\\fakepath\\cache.tar` + : 'cache.tar' + const tarPath = 'tar' + + await tar.listTar(archivePath, CompressionMethod.ZstdWithoutLong) + + expect(execMock).toHaveBeenCalledTimes(1) + expect(execMock).toHaveBeenCalledWith( + `"${tarPath}"`, + [ + '--use-compress-program', + 'zstd -d', + '-tf', + IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, + '-P' + ].concat(IS_WINDOWS ? ['--force-local'] : []), + {cwd: undefined} + ) +}) + +test('gzip list tar', async () => { + const execMock = jest.spyOn(exec, 'exec') + const archivePath = IS_WINDOWS + ? `${process.env['windir']}\\fakepath\\cache.tar` + : 'cache.tar' + + await tar.listTar(archivePath, CompressionMethod.Gzip) + + const tarPath = IS_WINDOWS + ? `${process.env['windir']}\\System32\\tar.exe` + : 'tar' + expect(execMock).toHaveBeenCalledTimes(1) + expect(execMock).toHaveBeenCalledWith( + `"${tarPath}"`, + [ + '-z', + '-tf', + IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, + '-P' + ], + {cwd: undefined} + ) +}) diff --git a/packages/cache/src/cache.ts b/packages/cache/src/cache.ts index 57ea1527..a6870f8f 100644 --- a/packages/cache/src/cache.ts +++ b/packages/cache/src/cache.ts @@ -2,7 +2,7 @@ import * as core from '@actions/core' import * as path from 'path' import * as utils from './internal/cacheUtils' import * as cacheHttpClient from './internal/cacheHttpClient' -import {createTar, extractTar} from './internal/tar' +import {createTar, extractTar, listTar} from './internal/tar' import {DownloadOptions, UploadOptions} from './options' export class ValidationError extends Error { @@ -100,6 +100,10 @@ export async function restoreCache( options ) + if (core.isDebug()) { + await listTar(archivePath, compressionMethod) + } + const archiveFileSize = utils.getArchiveFileSizeIsBytes(archivePath) core.info( `Cache Size: ~${Math.round( @@ -108,6 +112,7 @@ export async function restoreCache( ) await extractTar(archivePath, compressionMethod) + core.info('Cache restored successfully') } finally { // Try to delete the archive to save space try { @@ -162,6 +167,9 @@ export async function saveCache( core.debug(`Archive Path: ${archivePath}`) await createTar(archiveFolder, cachePaths, compressionMethod) + if (core.isDebug()) { + await listTar(archivePath, compressionMethod) + } const fileSizeLimit = 5 * 1024 * 1024 * 1024 // 5GB per repo limit const archiveFileSize = utils.getArchiveFileSizeIsBytes(archivePath) diff --git a/packages/cache/src/internal/cacheHttpClient.ts b/packages/cache/src/internal/cacheHttpClient.ts index dab39115..000257a5 100644 --- a/packages/cache/src/internal/cacheHttpClient.ts +++ b/packages/cache/src/internal/cacheHttpClient.ts @@ -301,6 +301,10 @@ export async function saveCache( // Commit Cache core.debug('Commiting cache') const cacheSize = utils.getArchiveFileSizeIsBytes(archivePath) + core.info( + `Cache Size: ~${Math.round(cacheSize / (1024 * 1024))} MB (${cacheSize} B)` + ) + const commitCacheResponse = await commitCache(httpClient, cacheId, cacheSize) if (!isSuccessStatusCode(commitCacheResponse.statusCode)) { throw new Error( diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 680433e7..4c1ee6db 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -113,3 +113,30 @@ export async function createTar( ] await execTar(args, compressionMethod, archiveFolder) } + +export async function listTar( + archivePath: string, + compressionMethod: CompressionMethod +): Promise { + // --d: Decompress. + // --long=#: Enables long distance matching with # bits. + // Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. + // Using 30 here because we also support 32-bit self-hosted runners. + function getCompressionProgram(): string[] { + switch (compressionMethod) { + case CompressionMethod.Zstd: + return ['--use-compress-program', 'zstd -d --long=30'] + case CompressionMethod.ZstdWithoutLong: + return ['--use-compress-program', 'zstd -d'] + default: + return ['-z'] + } + } + const args = [ + ...getCompressionProgram(), + '-tf', + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '-P' + ] + await execTar(args, compressionMethod) +}