From 8cf7e029dec484137caa02e79907e5892ba91df9 Mon Sep 17 00:00:00 2001 From: Cedric Wille Date: Mon, 19 Jun 2023 16:21:37 -0400 Subject: [PATCH] Add abort of stalled download There is a known issue with Azure Blob Storage which sometimes causes downloads to stall. This commit adds logic to abort any download that is hanging for more than 2 minutes --- packages/cache/src/internal/downloadUtils.ts | 36 +++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/packages/cache/src/internal/downloadUtils.ts b/packages/cache/src/internal/downloadUtils.ts index 7e1b1e8a..2c8d13fd 100644 --- a/packages/cache/src/internal/downloadUtils.ts +++ b/packages/cache/src/internal/downloadUtils.ts @@ -37,9 +37,12 @@ export class DownloadProgress { segmentSize: number segmentOffset: number receivedBytes: number + previouslyReceivedBytes: number + lastTimeOfNewBytes?: number startTime: number displayedComplete: boolean timeoutHandle?: ReturnType + abortController?: AbortController constructor(contentLength: number) { this.contentLength = contentLength @@ -47,8 +50,11 @@ export class DownloadProgress { this.segmentSize = 0 this.segmentOffset = 0 this.receivedBytes = 0 + this.previouslyReceivedBytes = 0 + this.lastTimeOfNewBytes = undefined this.displayedComplete = false this.startTime = Date.now() + this.abortController = undefined } /** @@ -62,6 +68,7 @@ export class DownloadProgress { this.segmentIndex = this.segmentIndex + 1 this.segmentSize = segmentSize this.receivedBytes = 0 + this.previouslyReceivedBytes = 0 core.debug( `Downloading segment at offset ${this.segmentOffset} with length ${this.segmentSize}...` @@ -84,6 +91,22 @@ export class DownloadProgress { return this.segmentOffset + this.receivedBytes } + setLastTimeOfNewBytes(): void { + this.lastTimeOfNewBytes = Date.now(); + } + + getLastTimeOfNewBytes(): number | undefined { + return this.lastTimeOfNewBytes; + } + + setAbortController(abortReference: AbortController): void { + this.abortController = abortReference; + } + + triggerAbortController(): void { + this.abortController?.abort(); + } + /** * Returns true if the download is complete. */ @@ -125,7 +148,17 @@ export class DownloadProgress { */ onProgress(): (progress: TransferProgressEvent) => void { return (progress: TransferProgressEvent) => { - this.setReceivedBytes(progress.loadedBytes) + if (progress.loadedBytes > this.getTransferredBytes()) { + this.setReceivedBytes(progress.loadedBytes); + this.setLastTimeOfNewBytes(); + } else { + if (this.getLastTimeOfNewBytes !== undefined && Date.now() - this.getLastTimeOfNewBytes()! > 120000) { // if download hanging for more than 2 minutes + this.triggerAbortController(); + throw new Error( + 'Aborting cache download as the download has stalled for more than 2 minutes.' + ) + } + } } } @@ -252,6 +285,7 @@ export async function downloadCacheStorageSDK( try { downloadProgress.startDisplayTimer() const controller = new AbortController() + downloadProgress.setAbortController(controller) const abortSignal = controller.signal while (!downloadProgress.isDone()) { const segmentStart =