diff --git a/packages/cache/src/internal/downloadUtils.ts b/packages/cache/src/internal/downloadUtils.ts
index e4f421f8..adaf739a 100644
--- a/packages/cache/src/internal/downloadUtils.ts
+++ b/packages/cache/src/internal/downloadUtils.ts
@@ -249,7 +249,12 @@ export async function downloadCacheStorageSDK(
 
     try {
       downloadProgress.startDisplayTimer()
-      const controller = new AbortController()
+      const abortTimeInMs =
+        options.abortTimeInMs === undefined ? 3600000 : options.abortTimeInMs
+      const abortSignal = AbortController.timeout(abortTimeInMs)
+      abortSignal.addEventListener('abort', () => {
+        core.warning("Cache download aborted, segment download timed out.")
+      })
       while (!downloadProgress.isDone()) {
         const segmentStart =
           downloadProgress.segmentOffset + downloadProgress.segmentSize
@@ -260,22 +265,18 @@ export async function downloadCacheStorageSDK(
         )
 
         downloadProgress.nextSegment(segmentSize)
-        const abortTimeInMs =
-          options.abortTimeInMs === undefined ? 3600000 : options.abortTimeInMs
-        const result = await promiseWithTimeout(
-          abortTimeInMs,
-          client.downloadToBuffer(segmentStart, segmentSize, {
-            abortSignal: controller.signal,
+
+        const result = await client.downloadToBuffer(
+          segmentStart,
+          segmentSize,
+          {
+            abortSignal: abortSignal,
             concurrency: options.downloadConcurrency,
             onProgress: downloadProgress.onProgress()
-          })
+          }
         )
-        if (result === 'timeout') {
-          controller.abort()
-          throw new Error('Download aborted, segment download timed out.')
-        } else if (Buffer.isBuffer(result)) {
-          fs.writeFileSync(fd, result)
-        }
+
+        fs.writeFileSync(fd, result)
       }
     } finally {
       downloadProgress.stopDisplayTimer()
@@ -283,18 +284,3 @@ export async function downloadCacheStorageSDK(
     }
   }
 }
-
-const promiseWithTimeout = async (
-  timeoutMs: number,
-  promise: Promise<Buffer>
-): Promise<unknown> => {
-  let timeoutHandle: NodeJS.Timeout
-  const timeoutPromise = new Promise(resolve => {
-    timeoutHandle = setTimeout(() => resolve('timeout'), timeoutMs)
-  })
-
-  return Promise.race([promise, timeoutPromise]).then(result => {
-    clearTimeout(timeoutHandle)
-    return result
-  })
-}