toolkit/packages/artifact/__tests__/upload-artifact.test.ts

import * as uploadZipSpecification from '../src/internal/upload/upload-zip-specification'
import * as zip from '../src/internal/upload/zip'
import * as util from '../src/internal/shared/util'
import * as config from '../src/internal/shared/config'
import {ArtifactServiceClientJSON} from '../src/generated'
import * as blobUpload from '../src/internal/upload/blob-upload'
import {uploadArtifact} from '../src/internal/upload/upload-artifact'
import {noopLogs} from './common'
import {FilesNotFoundError} from '../src/internal/shared/errors'
import {BlockBlobUploadStreamOptions} from '@azure/storage-blob'
import * as fs from 'fs'
import * as path from 'path'
import unzip from 'unzip-stream'

const uploadStreamMock = jest.fn()
const blockBlobClientMock = jest.fn().mockImplementation(() => ({
  uploadStream: uploadStreamMock
}))

jest.mock('@azure/storage-blob', () => ({
  BlobClient: jest.fn().mockImplementation(() => {
    return {
      getBlockBlobClient: blockBlobClientMock
    }
  })
}))

const fixtures = {
  uploadDirectory: path.join(__dirname, '_temp', 'plz-upload'),
  files: [
    {name: 'file1.txt', content: 'test 1 file content'},
    {name: 'file2.txt', content: 'test 2 file content'},
    {name: 'file3.txt', content: 'test 3 file content'},
    {
      name: 'real.txt',
      content: 'from a symlink'
    },
    {
      name: 'relative.txt',
      content: 'from a symlink',
      symlink: 'real.txt',
      relative: true
    },
    {
      name: 'absolute.txt',
      content: 'from a symlink',
      symlink: 'real.txt',
      relative: false
    }
  ],
  backendIDs: {
    workflowRunBackendId: '67dbcc20-e851-4452-a7c3-2cc0d2e0ec67',
    workflowJobRunBackendId: '5f49179d-3386-4c38-85f7-00f8138facd0'
  },
  runtimeToken: 'test-token',
  resultsServiceURL: 'http://results.local',
  inputs: {
    artifactName: 'test-artifact',
    files: [
      '/home/user/files/plz-upload/file1.txt',
      '/home/user/files/plz-upload/file2.txt',
      '/home/user/files/plz-upload/dir/file3.txt'
    ],
    rootDirectory: '/home/user/files/plz-upload'
  }
}

describe('upload-artifact', () => {
  beforeAll(() => {
    fs.mkdirSync(fixtures.uploadDirectory, {
      recursive: true
    })

    for (const file of fixtures.files) {
      if (file.symlink) {
        let symlinkPath = file.symlink
        if (!file.relative) {
          symlinkPath = path.join(fixtures.uploadDirectory, file.symlink)
        }

        if (!fs.existsSync(path.join(fixtures.uploadDirectory, file.name))) {
          fs.symlinkSync(
            symlinkPath,
            path.join(fixtures.uploadDirectory, file.name),
            'file'
          )
        }
      } else {
        fs.writeFileSync(
          path.join(fixtures.uploadDirectory, file.name),
          file.content
        )
      }
    }
  })

  beforeEach(() => {
    noopLogs()
    jest
      .spyOn(uploadZipSpecification, 'validateRootDirectory')
      .mockReturnValue()
    jest
      .spyOn(util, 'getBackendIdsFromToken')
      .mockReturnValue(fixtures.backendIDs)
    jest
      .spyOn(uploadZipSpecification, 'getUploadZipSpecification')
      .mockReturnValue(
        fixtures.files.map(file => ({
          sourcePath: path.join(fixtures.uploadDirectory, file.name),
          destinationPath: file.name,
          stats: new fs.Stats()
        }))
      )
    jest.spyOn(config, 'getRuntimeToken').mockReturnValue(fixtures.runtimeToken)
    jest
      .spyOn(config, 'getResultsServiceUrl')
      .mockReturnValue(fixtures.resultsServiceURL)
  })

  afterEach(() => {
    jest.restoreAllMocks()
  })

  it('should reject if there are no files to upload', async () => {
    jest
      .spyOn(uploadZipSpecification, 'getUploadZipSpecification')
      .mockClear()
      .mockReturnValue([])

    const uploadResp = uploadArtifact(
      fixtures.inputs.artifactName,
      fixtures.inputs.files,
      fixtures.inputs.rootDirectory
    )
    await expect(uploadResp).rejects.toThrowError(FilesNotFoundError)
  })

  it('should reject if no backend IDs are found', async () => {
    jest.spyOn(util, 'getBackendIdsFromToken').mockRestore()

    const uploadResp = uploadArtifact(
      fixtures.inputs.artifactName,
      fixtures.inputs.files,
      fixtures.inputs.rootDirectory
    )

    await expect(uploadResp).rejects.toThrow()
  })

  it('should return false if the creation request fails', async () => {
    jest
      .spyOn(zip, 'createZipUploadStream')
      .mockReturnValue(Promise.resolve(new zip.ZipUploadStream(1)))
    jest
      .spyOn(ArtifactServiceClientJSON.prototype, 'CreateArtifact')
      .mockReturnValue(Promise.resolve({ok: false, signedUploadUrl: ''}))

    const uploadResp = uploadArtifact(
      fixtures.inputs.artifactName,
      fixtures.inputs.files,
      fixtures.inputs.rootDirectory
    )

    await expect(uploadResp).rejects.toThrow()
  })

  it('should return false if blob storage upload is unsuccessful', async () => {
    jest
      .spyOn(zip, 'createZipUploadStream')
      .mockReturnValue(Promise.resolve(new zip.ZipUploadStream(1)))
    jest
      .spyOn(ArtifactServiceClientJSON.prototype, 'CreateArtifact')
      .mockReturnValue(
        Promise.resolve({
          ok: true,
          signedUploadUrl: 'https://signed-upload-url.com'
        })
      )
    jest
      .spyOn(blobUpload, 'uploadZipToBlobStorage')
      .mockReturnValue(Promise.reject(new Error('boom')))

    const uploadResp = uploadArtifact(
      fixtures.inputs.artifactName,
      fixtures.inputs.files,
      fixtures.inputs.rootDirectory
    )

    await expect(uploadResp).rejects.toThrow()
  })

  it('should reject if finalize artifact fails', async () => {
    jest
      .spyOn(zip, 'createZipUploadStream')
      .mockReturnValue(Promise.resolve(new zip.ZipUploadStream(1)))
    jest
      .spyOn(ArtifactServiceClientJSON.prototype, 'CreateArtifact')
      .mockReturnValue(
        Promise.resolve({
          ok: true,
          signedUploadUrl: 'https://signed-upload-url.com'
        })
      )
    jest.spyOn(blobUpload, 'uploadZipToBlobStorage').mockReturnValue(
      Promise.resolve({
        uploadSize: 1234,
        sha256Hash: 'test-sha256-hash'
      })
    )
    jest
      .spyOn(ArtifactServiceClientJSON.prototype, 'FinalizeArtifact')
      .mockReturnValue(Promise.resolve({ok: false, artifactId: ''}))

    const uploadResp = uploadArtifact(
      fixtures.inputs.artifactName,
      fixtures.inputs.files,
      fixtures.inputs.rootDirectory
    )

    await expect(uploadResp).rejects.toThrow()
  })

  it('should successfully upload an artifact', async () => {
    jest
      .spyOn(uploadZipSpecification, 'getUploadZipSpecification')
      .mockRestore()

    jest
      .spyOn(ArtifactServiceClientJSON.prototype, 'CreateArtifact')
      .mockReturnValue(
        Promise.resolve({
          ok: true,
          signedUploadUrl: 'https://signed-upload-url.local'
        })
      )
    jest
      .spyOn(ArtifactServiceClientJSON.prototype, 'FinalizeArtifact')
      .mockReturnValue(
        Promise.resolve({
          ok: true,
          artifactId: '1'
        })
      )

    let loadedBytes = 0
    const uploadedZip = path.join(
      fixtures.uploadDirectory,
      '..',
      'uploaded.zip'
    )
    uploadStreamMock.mockImplementation(
      async (
        stream: NodeJS.ReadableStream,
        bufferSize?: number,
        maxConcurrency?: number,
        options?: BlockBlobUploadStreamOptions
      ) => {
        const {onProgress} = options || {}

        if (fs.existsSync(uploadedZip)) {
          fs.unlinkSync(uploadedZip)
        }
        const uploadedZipStream = fs.createWriteStream(uploadedZip)

        onProgress?.({loadedBytes: 0})
        return new Promise((resolve, reject) => {
          stream.on('data', chunk => {
            loadedBytes += chunk.length
            uploadedZipStream.write(chunk)
            onProgress?.({loadedBytes})
          })
          stream.on('end', () => {
            onProgress?.({loadedBytes})
            uploadedZipStream.end()
            resolve({})
          })
          stream.on('error', err => {
            reject(err)
          })
        })
      }
    )

    const {id, size} = await uploadArtifact(
      fixtures.inputs.artifactName,
      fixtures.files.map(file =>
        path.join(fixtures.uploadDirectory, file.name)
      ),
      fixtures.uploadDirectory
    )

    expect(id).toBe(1)
    expect(size).toBe(loadedBytes)

    const extractedDirectory = path.join(
      fixtures.uploadDirectory,
      '..',
      'extracted'
    )
    if (fs.existsSync(extractedDirectory)) {
      fs.rmdirSync(extractedDirectory, {recursive: true})
    }

    const extract = new Promise((resolve, reject) => {
      fs.createReadStream(uploadedZip)
        .pipe(unzip.Extract({path: extractedDirectory}))
        .on('close', () => {
          resolve(true)
        })
        .on('error', err => {
          reject(err)
        })
    })

    await expect(extract).resolves.toBe(true)
    for (const file of fixtures.files) {
      const filePath = path.join(extractedDirectory, file.name)
      expect(fs.existsSync(filePath)).toBe(true)
      expect(fs.readFileSync(filePath, 'utf8')).toBe(file.content)
    }
  })

  it('should throw an error uploading blob chunks get delayed', async () => {
    jest
      .spyOn(ArtifactServiceClientJSON.prototype, 'CreateArtifact')
      .mockReturnValue(
        Promise.resolve({
          ok: true,
          signedUploadUrl: 'https://signed-upload-url.local'
        })
      )
    jest
      .spyOn(ArtifactServiceClientJSON.prototype, 'FinalizeArtifact')
      .mockReturnValue(
        Promise.resolve({
          ok: true,
          artifactId: '1'
        })
      )
    jest
      .spyOn(config, 'getResultsServiceUrl')
      .mockReturnValue('https://results.local')

    jest.spyOn(config, 'getUploadChunkTimeout').mockReturnValue(2_000)

    uploadStreamMock.mockImplementation(
      async (
        stream: NodeJS.ReadableStream,
        bufferSize?: number,
        maxConcurrency?: number,
        options?: BlockBlobUploadStreamOptions
      ) => {
        const {onProgress, abortSignal} = options || {}
        onProgress?.({loadedBytes: 0})
        return new Promise(resolve => {
          abortSignal?.addEventListener('abort', () => {
            resolve({})
          })
        })
      }
    )

    const uploadResp = uploadArtifact(
      fixtures.inputs.artifactName,
      fixtures.inputs.files,
      fixtures.inputs.rootDirectory
    )

    await expect(uploadResp).rejects.toThrow('Upload progress stalled.')
  })
})