1
0
Fork 0

adding asysnc zip entry logic handling

pull/1690/head
Vallie Joseph 2024-04-08 15:27:00 +00:00
parent 38f54e9d73
commit ce603525f8
1 changed files with 37 additions and 60 deletions

View File

@ -22,7 +22,9 @@ export class ZipUploadStream extends stream.Transform {
cb(null, chunk) cb(null, chunk)
} }
} }
interface NodeJSError extends Error {
code?: string
}
export async function createZipUploadStream( export async function createZipUploadStream(
uploadSpecification: UploadZipSpecification[], uploadSpecification: UploadZipSpecification[],
compressionLevel: number = DEFAULT_COMPRESSION_LEVEL compressionLevel: number = DEFAULT_COMPRESSION_LEVEL
@ -37,73 +39,49 @@ export async function createZipUploadStream(
} }
} }
const zip = new ZipStream.default(zlibOptions) const zip = new ZipStream.default(zlibOptions)
const bufferSize = getUploadChunkSize()
const zipUploadStream = new ZipUploadStream(bufferSize)
// register callbacks for various events during the zip lifecycle // register callbacks for various events during the zip lifecycle
zip.on('error', zipErrorCallback) zip.on('error', zipErrorCallback)
zip.on('warning', zipWarningCallback) zip.on('warning', zipWarningCallback)
zip.on('finish', zipFinishCallback) zip.on('finish', zipFinishCallback)
zip.on('end', zipEndCallback) zip.on('end', zipEndCallback)
// see https://caolan.github.io/async/v3/docs.html#queue for options const addFileToZip = (
const fileUploadQueue = async.queue(function (fileItem, callback) { file: UploadZipSpecification,
try { callback: (error?: Error) => void
core.debug(`adding ${fileItem} to the file queue`) ): void => {
callback()
} catch (err) {
core.error(`task experienced an error: ${fileItem} ${err}`)
callback(err)
}
}) // concurrency for uploads automatically set to 1
fileUploadQueue.error(function (err, task) {
core.error(`task experienced an error: ${task} ${err}`)
})
for (const file of uploadSpecification) {
if (file.sourcePath !== null) { if (file.sourcePath !== null) {
const readStream = createReadStream(file.sourcePath)
readStream.on('end', () => {
core.debug('The upload read stream is ending')
})
readStream.on('error', function (err) {
core.debug(`${err}`)
}) // Catch any errors from createReadStream
const fileEntry = zip.entry(
readStream,
{name: file.destinationPath},
function (err, entry) {
if (err) {
core.error('A file entry error occurred:', err)
}
core.debug(`File entry was succesfull: ${entry}`)
}
)
fileUploadQueue.push(fileEntry)
} else {
fileUploadQueue.push(
zip.entry( zip.entry(
null, createReadStream(file.sourcePath),
{name: `${file.destinationPath}/`}, {name: file.destinationPath},
function (err, entry) { (error: NodeJSError) => {
if (err) { if (error) {
core.error('A directory entry error occurred:', err) callback(error)
return
} }
core.debug(`File entry was succesfull: ${entry}`) callback()
} }
) )
) } else {
zip.entry('', {name: file.destinationPath}, (error: NodeJSError) => {
if (error) {
callback(error)
return
} }
} callback()
})
core.debug(`Starting the finalizing of all entries`) }
}
fileUploadQueue.drain(() => {
core.debug('all items have been processed') async.eachSeries(uploadSpecification, addFileToZip, (error: NodeJSError) => {
if (error) {
core.error('Failed to add a file to the zip:')
core.info(error.toString())
return
}
zip.finalize() // Finalize the archive once all files have been added
}) })
zip.finalize()
core.debug(`Finalizing entries`)
const bufferSize = getUploadChunkSize()
const zipUploadStream = new ZipUploadStream(bufferSize)
zip.pipe(zipUploadStream) // Pipe the zip stream into zipUploadStream
core.debug( core.debug(
`Zip write high watermark value ${zipUploadStream.writableHighWaterMark}` `Zip write high watermark value ${zipUploadStream.writableHighWaterMark}`
@ -111,28 +89,27 @@ export async function createZipUploadStream(
core.debug( core.debug(
`Zip read high watermark value ${zipUploadStream.readableHighWaterMark}` `Zip read high watermark value ${zipUploadStream.readableHighWaterMark}`
) )
return zipUploadStream return zipUploadStream
} }
// eslint-disable-next-line @typescript-eslint/no-explicit-any const zipErrorCallback = (error: Error): void => {
const zipErrorCallback = (error: any): void => {
core.error('An error has occurred while creating the zip file for upload') core.error('An error has occurred while creating the zip file for upload')
core.info(error) core.info(error.message)
throw new Error('An error has occurred during zip creation for the artifact') throw new Error('An error has occurred during zip creation for the artifact')
} }
// eslint-disable-next-line @typescript-eslint/no-explicit-any const zipWarningCallback = (error: NodeJSError): void => {
const zipWarningCallback = (error: any): void => {
if (error.code === 'ENOENT') { if (error.code === 'ENOENT') {
core.warning( core.warning(
'ENOENT warning during artifact zip creation. No such file or directory' 'ENOENT warning during artifact zip creation. No such file or directory'
) )
core.info(error) core.info(error.toString()) // Convert error object to string
} else { } else {
core.warning( core.warning(
`A non-blocking warning has occurred during artifact zip creation: ${error.code}` `A non-blocking warning has occurred during artifact zip creation: ${error.code}`
) )
core.info(error) core.info(error.toString()) // Convert error object to string
} }
} }