diff --git a/packages/artifact/src/internal/upload/upload-artifact.ts b/packages/artifact/src/internal/upload/upload-artifact.ts index e810ccc6..6e174d8e 100644 --- a/packages/artifact/src/internal/upload/upload-artifact.ts +++ b/packages/artifact/src/internal/upload/upload-artifact.ts @@ -1,119 +1,124 @@ +import * as stream from 'stream' +import * as ZipStream from 'zip-stream' import * as core from '@actions/core' -import { - UploadArtifactOptions, - UploadArtifactResponse -} from '../shared/interfaces' -import {getExpiration} from './retention' -import {validateArtifactName} from './path-and-artifact-name-validation' -import {internalArtifactTwirpClient} from '../shared/artifact-twirp-client' -import { - UploadZipSpecification, - getUploadZipSpecification, - validateRootDirectory -} from './upload-zip-specification' -import {getBackendIdsFromToken} from '../shared/util' -import {uploadZipToBlobStorage} from './blob-upload' -import {createZipUploadStream} from './zip' -import { - CreateArtifactRequest, - FinalizeArtifactRequest, - StringValue -} from '../../generated' -import {FilesNotFoundError, InvalidResponseError} from '../shared/errors' +import async from 'async' +import {createReadStream} from 'fs' +import {UploadZipSpecification} from './upload-zip-specification' +import {getUploadChunkSize} from '../shared/config' -export async function uploadArtifact( - name: string, - files: string[], - rootDirectory: string, - options?: UploadArtifactOptions | undefined -): Promise { - validateArtifactName(name) - validateRootDirectory(rootDirectory) +export const DEFAULT_COMPRESSION_LEVEL = 6 - const zipSpecification: UploadZipSpecification[] = getUploadZipSpecification( - files, - rootDirectory - ) - if (zipSpecification.length === 0) { - throw new FilesNotFoundError( - zipSpecification.flatMap(s => (s.sourcePath ? [s.sourcePath] : [])) - ) - } - - // get the IDs needed for the artifact creation - const backendIds = getBackendIdsFromToken() - - // create the artifact client - const artifactClient = internalArtifactTwirpClient() - - // create the artifact - const createArtifactReq: CreateArtifactRequest = { - workflowRunBackendId: backendIds.workflowRunBackendId, - workflowJobRunBackendId: backendIds.workflowJobRunBackendId, - name, - version: 4 - } - - // if there is a retention period, add it to the request - const expiresAt = getExpiration(options?.retentionDays) - if (expiresAt) { - createArtifactReq.expiresAt = expiresAt - } - - const createArtifactResp = - await artifactClient.CreateArtifact(createArtifactReq) - if (!createArtifactResp.ok) { - throw new InvalidResponseError( - 'CreateArtifact: response from backend was not ok' - ) - } - // Create the zipupload stream for use in blob upload - const zipUploadStream = await createZipUploadStream( - zipSpecification, - options?.compressionLevel - ).catch(err => { - throw new InvalidResponseError( - `createZipUploadStream: response from backend was not ok: ${err}` - ) - }) - - // Upload zip to blob storage - const uploadResult = await uploadZipToBlobStorage( - createArtifactResp.signedUploadUrl, - zipUploadStream - ).catch(err => { - throw new InvalidResponseError( - `uploadZipToBlobStorage: response blob was not ok: ${err}` - ) - }) - // finalize the artifact - const finalizeArtifactReq: FinalizeArtifactRequest = { - workflowRunBackendId: backendIds.workflowRunBackendId, - workflowJobRunBackendId: backendIds.workflowJobRunBackendId, - name, - size: uploadResult.uploadSize ? uploadResult.uploadSize.toString() : '0' - } - if (uploadResult.sha256Hash) { - finalizeArtifactReq.hash = StringValue.create({ - value: `sha256:${uploadResult.sha256Hash}` +// Custom stream transformer so we can set the highWaterMark property +// See https://github.com/nodejs/node/issues/8855 +export class ZipUploadStream extends stream.Transform { + constructor(bufferSize: number) { + super({ + highWaterMark: bufferSize }) } - core.info(`Finalizing artifact upload`) - const finalizeArtifactResp = - await artifactClient.FinalizeArtifact(finalizeArtifactReq) - if (!finalizeArtifactResp.ok) { - throw new InvalidResponseError( - 'FinalizeArtifact: response from backend was not ok' - ) - } - const artifactId = BigInt(finalizeArtifactResp.artifactId) - core.info( - `Artifact ${name}.zip successfully finalized. Artifact ID ${artifactId}` - ) - - return { - size: uploadResult.uploadSize, - id: Number(artifactId) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + _transform(chunk: any, enc: any, cb: any): void { + cb(null, chunk) } } + +export async function createZipUploadStream( + uploadSpecification: UploadZipSpecification[], + compressionLevel: number = DEFAULT_COMPRESSION_LEVEL +): Promise { + core.debug( + `Creating Artifact archive with compressionLevel: ${compressionLevel}` + ) + + const zlibOptions = { + zlib: { + level: compressionLevel, + bufferSize: getUploadChunkSize() + } + } + const zip = new ZipStream.default(zlibOptions) + + const bufferSize = getUploadChunkSize() + const zipUploadStream = new ZipUploadStream(bufferSize) + zip.pipe(zipUploadStream) + // register callbacks for various events during the zip lifecycle + zip.on('error', zipErrorCallback) + zip.on('warning', zipWarningCallback) + zip.on('finish', zipFinishCallback) + zip.on('end', zipEndCallback) + const addFileToZip = ( + file: UploadZipSpecification, + callback: (error?: Error) => void + ) => { + if (file.sourcePath !== null) { + zip.entry( + createReadStream(file.sourcePath), + {name: file.destinationPath}, + (error: any) => { + if (error) { + callback(error) + return + } + callback() + } + ) + } else { + zip.entry('', {name: file.destinationPath}, (error: any) => { + if (error) { + callback(error) + return + } + callback() + }) + } + } + + async.eachSeries(uploadSpecification, addFileToZip, (error: any) => { + if (error) { + core.error('Failed to add a file to the zip:') + core.info(error) + return + } + zip.finalize() // Finalize the archive once all files have been added + }) + + core.debug( + `Zip write high watermark value ${zipUploadStream.writableHighWaterMark}` + ) + core.debug( + `Zip read high watermark value ${zipUploadStream.readableHighWaterMark}` + ) + + return zipUploadStream +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const zipErrorCallback = (error: any): void => { + core.error('An error has occurred while creating the zip file for upload') + core.info(error) + + throw new Error('An error has occurred during zip creation for the artifact') +} +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const zipWarningCallback = (error: any): void => { + if (error.code === 'ENOENT') { + core.warning( + 'ENOENT warning during artifact zip creation. No such file or directory' + ) + core.info(error) + } else { + core.warning( + `A non-blocking warning has occurred during artifact zip creation: ${error.code}` + ) + core.info(error) + } +} + +const zipFinishCallback = (): void => { + core.debug('Zip stream for upload has finished.') +} + +const zipEndCallback = (): void => { + core.debug('Zip stream for upload has ended.') +} diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 579757c3..3f717694 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -22,9 +22,7 @@ export class ZipUploadStream extends stream.Transform { cb(null, chunk) } } -interface NodeJSError extends Error { - code?: string -} + export async function createZipUploadStream( uploadSpecification: UploadZipSpecification[], compressionLevel: number = DEFAULT_COMPRESSION_LEVEL @@ -32,6 +30,7 @@ export async function createZipUploadStream( core.debug( `Creating Artifact archive with compressionLevel: ${compressionLevel}` ) + const zlibOptions = { zlib: { level: compressionLevel, @@ -39,14 +38,16 @@ export async function createZipUploadStream( } } const zip = new ZipStream.default(zlibOptions) + const bufferSize = getUploadChunkSize() const zipUploadStream = new ZipUploadStream(bufferSize) - + zip.pipe(zipUploadStream) // register callbacks for various events during the zip lifecycle zip.on('error', zipErrorCallback) zip.on('warning', zipWarningCallback) zip.on('finish', zipFinishCallback) zip.on('end', zipEndCallback) + const addFileToZip = ( file: UploadZipSpecification, callback: (error?: Error) => void @@ -55,18 +56,18 @@ export async function createZipUploadStream( zip.entry( createReadStream(file.sourcePath), {name: file.destinationPath}, - (error: NodeJSError) => { + (error: unknown) => { if (error) { - callback(error) + callback(error as Error) // Cast the error object to the Error type return } callback() } ) } else { - zip.entry('', {name: file.destinationPath}, (error: NodeJSError) => { + zip.entry('', {name: file.destinationPath}, (error: unknown) => { if (error) { - callback(error) + callback(error as Error) return } callback() @@ -74,10 +75,10 @@ export async function createZipUploadStream( } } - async.eachSeries(uploadSpecification, addFileToZip, (error: NodeJSError) => { + async.eachSeries(uploadSpecification, addFileToZip, (error: unknown) => { if (error) { core.error('Failed to add a file to the zip:') - core.info(error.toString()) + core.info(error.toString()) // Convert error to string return } zip.finalize() // Finalize the archive once all files have been added @@ -93,23 +94,25 @@ export async function createZipUploadStream( return zipUploadStream } -const zipErrorCallback = (error: Error): void => { +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const zipErrorCallback = (error: any): void => { core.error('An error has occurred while creating the zip file for upload') - core.info(error.message) + core.info(error) throw new Error('An error has occurred during zip creation for the artifact') } -const zipWarningCallback = (error: NodeJSError): void => { +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const zipWarningCallback = (error: any): void => { if (error.code === 'ENOENT') { core.warning( 'ENOENT warning during artifact zip creation. No such file or directory' ) - core.info(error.toString()) // Convert error object to string + core.info(error) } else { core.warning( `A non-blocking warning has occurred during artifact zip creation: ${error.code}` ) - core.info(error.toString()) // Convert error object to string + core.info(error) } }