From 12fa0be194e662ab1dea8f3fd579ad36190bf481 Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Wed, 2 Aug 2023 17:04:48 -0400 Subject: [PATCH] This should work E2E... --- .../src/internal/upload/upload-artifact.ts | 168 +++++++++++++++++- .../internal/upload/upload-specification.ts | 94 ++++++++++ 2 files changed, 256 insertions(+), 6 deletions(-) create mode 100644 packages/artifact/src/internal/upload/upload-specification.ts diff --git a/packages/artifact/src/internal/upload/upload-artifact.ts b/packages/artifact/src/internal/upload/upload-artifact.ts index 63911b37..94d74de4 100644 --- a/packages/artifact/src/internal/upload/upload-artifact.ts +++ b/packages/artifact/src/internal/upload/upload-artifact.ts @@ -1,7 +1,34 @@ - +import * as core from '@actions/core' import {checkArtifactName} from './path-and-artifact-name-validation' import {UploadOptions} from './upload-options' import {UploadResponse} from './upload-response' +import { UploadSpecification, getUploadSpecification } from './upload-specification' +import { ArtifactHttpClient } from '../artifact-http-client' +import { ArtifactServiceClientJSON } from 'src/generated' + +import {BlobClient, BlockBlobUploadStreamOptions} from '@azure/storage-blob' +import { TransferProgressEvent } from '@azure/core-http'; +import * as a from 'archiver' +import * as fs from 'fs' +import * as stream from 'stream' + +import {getBackendIds, BackendIds} from '../util' + +const bufferSize = 1024 * 1024 * 8 // 8 MB + +// Custom stream transformer so we can set the highWaterMark property +// See https://github.com/nodejs/node/issues/8855 +export class ZipUploadStream extends stream.Transform { + constructor(bufferSize: number) { + super({ + highWaterMark: bufferSize + }) + } + + _transform(chunk:any, enc:any, cb:any) { + cb(null, chunk) + } +} export async function uploadArtifact( name: string, @@ -10,19 +37,148 @@ export async function uploadArtifact( options?: UploadOptions | undefined ): Promise { + let uploadByteCount = 0 + // Need to keep checking the artifact name checkArtifactName(name) - // TODO Twirp call to create new artifact + // Get specification for the files being uploaded + const uploadSpecification: UploadSpecification[] = getUploadSpecification( + name, + rootDirectory, + files + ) - // TODO Upload to blob storage using SAS URL - // testing.ts is being used to prototype this functionality + if (uploadSpecification.length === 0) { + core.warning(`No files found that can be uploaded`) + } else { + const artifactClient = new ArtifactHttpClient('@actions/artifact-upload') + const jsonClient = new ArtifactServiceClientJSON(artifactClient) - // TODO Twirp call to finalize the new artifact upload + const backendIDs: BackendIds = getBackendIds() + + try { + + + + + const createResp = await jsonClient.CreateArtifact({workflowRunBackendId: backendIDs.workflowRunBackendId, workflowJobRunBackendId: backendIDs.workflowJobRunBackendId, name: name, version: 4}) + + if (!createResp.ok) { + core.error("CreateArtifact failed") + } + + console.log(createResp.signedUploadUrl) + + // Blob upload start + + const blobClient = new BlobClient(createResp.signedUploadUrl); + const zip = a.create('zip', { + zlib: { level: 9 } // Sets the compression level. + // Available options are 0-9 + // 0 => no compression + // 1 => fastest with low compression + // 9 => highest compression ratio but the slowest + }); + + console.log("file specification") + for (const file of uploadSpecification) { + console.log("uploadPath:" + file.uploadFilePath + " absolute:" + file.absoluteFilePath) + zip.append(fs.createReadStream(file.absoluteFilePath), {name: file.uploadFilePath}) + } + + const zipUploadStream = new ZipUploadStream(bufferSize) + zip.pipe(zipUploadStream) + zip.finalize(); + + console.log("Write high watermark value " + zipUploadStream.writableHighWaterMark) + console.log("Read high watermark value " + zipUploadStream.readableHighWaterMark) + + // good practice to catch warnings (ie stat failures and other non-blocking errors) + zip.on('warning', function(err) { + if (err.code === 'ENOENT') { + console.log("zip error ENOENT") + } else { + console.log("some other warning ") + console.log(err) + } + }); + + // good practice to catch this error explicitly + zip.on('error', function(err) { + console.log("some error with zip ") + console.log(err) + }); + + zip.on("progress", function(progress: a.ProgressData) { + console.log(progress) + + /* This outputs data like this, we could potentially do something with this for even more logging to show the status of the zip creation + { + entries: { total: 7, processed: 1 }, + fs: { totalBytes: 0, processedBytes: 0 } + } + { + entries: { total: 7, processed: 2 }, + fs: { totalBytes: 0, processedBytes: 0 } + } + */ + }) + + + // We can add these to debug logging + zip.on('end', function() { + console.log("zip ending") + }); + zip.on('finish', function() { + console.log("zip finished") + }); + + // Upload options + const maxBuffers = 5 + const blockBlobClient = blobClient.getBlockBlobClient() + + var myCallback = function(progress: TransferProgressEvent) { + console.log("Byte upload count " + progress.loadedBytes) + uploadByteCount = progress.loadedBytes + }; + + const options: BlockBlobUploadStreamOptions = { + blobHTTPHeaders: { "blobContentType": "zip" }, + onProgress: myCallback + } + + // Upload! + try { + await blockBlobClient.uploadStream( + zipUploadStream, + bufferSize, + maxBuffers, + options + ); + } catch (error){ + console.log(error) + } + console.log("final upload size in bytes is " + uploadByteCount) + + console.log("we are done with the blob upload!") + // Blob upload end + + const finalizeResp = await jsonClient.FinalizeArtifact({workflowRunBackendId: backendIDs.workflowRunBackendId, workflowJobRunBackendId: backendIDs.workflowJobRunBackendId, name: name, size: BigInt(5)}) + + if (!finalizeResp.ok) { + core.error("FinalizeArtifact failed") + } + } catch (e) { + console.log(e) + } + + console.log("FinalizeArtifact succeeded") + } const uploadResponse: UploadResponse = { artifactName: name, - size: 0 + size: uploadByteCount } return uploadResponse diff --git a/packages/artifact/src/internal/upload/upload-specification.ts b/packages/artifact/src/internal/upload/upload-specification.ts new file mode 100644 index 00000000..5ec1e3fa --- /dev/null +++ b/packages/artifact/src/internal/upload/upload-specification.ts @@ -0,0 +1,94 @@ +import * as fs from 'fs' +import {debug} from '@actions/core' +import {join, normalize, resolve} from 'path' +import {checkArtifactFilePath} from './path-and-artifact-name-validation' + +export interface UploadSpecification { + absoluteFilePath: string + uploadFilePath: string +} + +/** + * Creates a specification that describes how each file that is part of the artifact will be uploaded + * @param artifactName the name of the artifact being uploaded. Used during upload to denote where the artifact is stored on the server + * @param rootDirectory an absolute file path that denotes the path that should be removed from the beginning of each artifact file + * @param artifactFiles a list of absolute file paths that denote what should be uploaded as part of the artifact + */ +export function getUploadSpecification( + artifactName: string, + rootDirectory: string, + artifactFiles: string[] +): UploadSpecification[] { + // artifact name was checked earlier on, no need to check again + const specifications: UploadSpecification[] = [] + + if (!fs.existsSync(rootDirectory)) { + throw new Error(`Provided rootDirectory ${rootDirectory} does not exist`) + } + if (!fs.statSync(rootDirectory).isDirectory()) { + throw new Error( + `Provided rootDirectory ${rootDirectory} is not a valid directory` + ) + } + // Normalize and resolve, this allows for either absolute or relative paths to be used + rootDirectory = normalize(rootDirectory) + rootDirectory = resolve(rootDirectory) + + /* + Example to demonstrate behavior + + Input: + artifactName: my-artifact + rootDirectory: '/home/user/files/plz-upload' + artifactFiles: [ + '/home/user/files/plz-upload/file1.txt', + '/home/user/files/plz-upload/file2.txt', + '/home/user/files/plz-upload/dir/file3.txt' + ] + + Output: + specifications: [ + ['/home/user/files/plz-upload/file1.txt', 'my-artifact/file1.txt'], + ['/home/user/files/plz-upload/file1.txt', 'my-artifact/file2.txt'], + ['/home/user/files/plz-upload/file1.txt', 'my-artifact/dir/file3.txt'] + ] + */ + for (let file of artifactFiles) { + if (!fs.existsSync(file)) { + throw new Error(`File ${file} does not exist`) + } + if (!fs.statSync(file).isDirectory()) { + // Normalize and resolve, this allows for either absolute or relative paths to be used + file = normalize(file) + file = resolve(file) + if (!file.startsWith(rootDirectory)) { + throw new Error( + `The rootDirectory: ${rootDirectory} is not a parent directory of the file: ${file}` + ) + } + + // Check for forbidden characters in file paths that will be rejected during upload + const uploadPath = file.replace(rootDirectory, '') + checkArtifactFilePath(uploadPath) + + /* + uploadFilePath denotes where the file will be uploaded in the file container on the server. During a run, if multiple artifacts are uploaded, they will all + be saved in the same container. The artifact name is used as the root directory in the container to separate and distinguish uploaded artifacts + + path.join handles all the following cases and would return 'artifact-name/file-to-upload.txt + join('artifact-name/', 'file-to-upload.txt') + join('artifact-name/', '/file-to-upload.txt') + join('artifact-name', 'file-to-upload.txt') + join('artifact-name', '/file-to-upload.txt') + */ + specifications.push({ + absoluteFilePath: file, + uploadFilePath: join(artifactName, uploadPath) + }) + } else { + // Directories are rejected by the server during upload + debug(`Removing ${file} from rawSearchResults because it is a directory`) + } + } + return specifications +} \ No newline at end of file