adds wget for streaming download

packages/warp-cache/package.json

@@ -1,6 +1,6 @@
 {
   "name": "github-actions.warp-cache",
-  "version": "1.1.0",
+  "version": "1.1.1",
   "preview": true,
   "description": "Github action to use WarpBuild's in-house cache offering",
   "keywords": [

packages/warp-cache/src/cache.ts

@@ -10,6 +10,8 @@ import {
 } from './internal/tar'
 import {DownloadOptions, getUploadOptions} from './options'
 import {isSuccessStatusCode} from './internal/requestUtils'
+import {getDownloadCommandPipeForWget} from './internal/downloadUtils'
+import {ChildProcessWithoutNullStreams} from 'child_process'
 
 export class ValidationError extends Error {
   constructor(message: string) {
@@ -171,39 +173,56 @@ export async function restoreCache(
 
         const archiveLocation = `gs://${cacheEntry.gcs?.bucket_name}/${cacheEntry.gcs?.cache_key}`
 
-        // await cacheHttpClient.downloadCache(
+        /*
-        //   cacheEntry.provider,
+        * Alternate, Multipart download method for GCS
-        //   archiveLocation,
+        await cacheHttpClient.downloadCache(
-        //   archivePath,
+          cacheEntry.provider,
-        //   cacheEntry.gcs?.short_lived_token?.access_token ?? ''
+          archiveLocation,
-        // )
+          archivePath,
+          cacheEntry.gcs?.short_lived_token?.access_token ?? ''
+        )
 
-        // if (core.isDebug()) {
+        if (core.isDebug()) {
-        //   await listTar(archivePath, compressionMethod)
+          await listTar(archivePath, compressionMethod)
-        // }
+        }
 
-        // const archiveFileSize = utils.getArchiveFileSizeInBytes(archivePath)
+        const archiveFileSize = utils.getArchiveFileSizeInBytes(archivePath)
-        // core.info(
+        core.info(
-        //   `Cache Size: ~${Math.round(
+          `Cache Size: ~${Math.round(
-        //     archiveFileSize / (1024 * 1024)
+            archiveFileSize / (1024 * 1024)
-        //   )} MB (${archiveFileSize} B)`
+          )} MB (${archiveFileSize} B)`
-        // )
+        )
 
-        // await extractTar(archivePath, compressionMethod)
+        await extractTar(archivePath, compressionMethod)
+        */
 
         // For GCS, we do a streaming download which means that we extract the archive while we are downloading it.
 
-        const readStream = cacheHttpClient.downloadCacheStreaming(
+        let readStream: NodeJS.ReadableStream | undefined
-          'gcs',
+        let downloadCommandPipe: ChildProcessWithoutNullStreams | undefined
-          archiveLocation,
-          cacheEntry?.gcs?.short_lived_token?.access_token ?? ''
-        )
 
-        if (!readStream) {
+        if (cacheEntry?.gcs?.pre_signed_url) {
-          return undefined
+          downloadCommandPipe = getDownloadCommandPipeForWget(
+            cacheEntry?.gcs?.pre_signed_url
+          )
+        } else {
+          readStream = cacheHttpClient.downloadCacheStreaming(
+            'gcs',
+            archiveLocation,
+            cacheEntry?.gcs?.short_lived_token?.access_token ?? ''
+          )
+
+          if (!readStream) {
+            return undefined
+          }
         }
 
-        await extractStreamingTar(readStream, archivePath, compressionMethod)
+        await extractStreamingTar(
+          readStream,
+          archivePath,
+          compressionMethod,
+          downloadCommandPipe
+        )
         core.info('Cache restored successfully')
         break
       }

packages/warp-cache/src/internal/downloadUtils.ts

@@ -14,6 +14,7 @@ import {retryHttpClientResponse} from './requestUtils'
 
 import {AbortController} from '@azure/abort-controller'
 import {Storage, TransferManager} from '@google-cloud/storage'
+import {ChildProcessWithoutNullStreams, spawn} from 'child_process'
 
 /**
  * Pipes the body of a HTTP response to a stream
@@ -350,3 +351,9 @@ export function downloadCacheStreamingGCP(
     throw error
   }
 }
+
+export function getDownloadCommandPipeForWget(
+  url: string
+): ChildProcessWithoutNullStreams {
+  return spawn('wget', ['-qO', '-', url])
+}

packages/warp-cache/src/internal/tar.ts

@@ -417,9 +417,10 @@ export async function extractTar(
  * NOTE: Currently tested only on archives created using tar and zstd
  */
 export async function extractStreamingTar(
-  stream: NodeJS.ReadableStream,
+  stream: NodeJS.ReadableStream | undefined,
   archivePath: string,
-  compressionMethod: CompressionMethod
+  compressionMethod: CompressionMethod,
+  downloadCommandPipe?: ChildProcessWithoutNullStreams
 ): Promise<void> {
   const workingDirectory = getWorkingDirectory()
   await io.mkdirP(workingDirectory)
@@ -429,6 +430,10 @@ export async function extractStreamingTar(
     archivePath
   )
 
+  if (downloadCommandPipe) {
+    commandPipes.unshift(downloadCommandPipe)
+  }
+
   if (commandPipes.length < 2) {
     throw new Error(
       'At least two processes should be present as the archive is compressed at least twice.'
@@ -436,7 +441,9 @@ export async function extractStreamingTar(
   }
 
   return new Promise((resolve, reject) => {
-    stream.pipe(commandPipes[0].stdin)
+    if (stream) {
+      stream.pipe(commandPipes[0].stdin)
+    }
     for (let i = 0; i < commandPipes.length - 1; i++) {
       commandPipes[i].stdout.pipe(commandPipes[i + 1].stdin)
 

packages/warp-cache/src/internal/warpcache-ts-sdk/.openapi-generator/FILES

@@ -9,6 +9,7 @@ common.ts
 configuration.ts
 git_push.sh
 index.ts
+models/commons-append-operation-input.ts
 models/commons-cache-entry.ts
 models/commons-commit-cache-request.ts
 models/commons-commit-cache-response.ts
@@ -20,6 +21,7 @@ models/commons-gcsget-cache-reponse.ts
 models/commons-gcsreserve-cache-response.ts
 models/commons-get-cache-request.ts
 models/commons-get-cache-response.ts
+models/commons-operation.ts
 models/commons-reserve-cache-request.ts
 models/commons-reserve-cache-response.ts
 models/commons-s3-commit-cache-response.ts

packages/warp-cache/src/internal/warpcache-ts-sdk/api/default-api.ts

@@ -22,6 +22,8 @@ import { DUMMY_BASE_URL, assertParamExists, setApiKeyToObject, setBasicAuthToObj
 // @ts-ignore
 import { BASE_PATH, COLLECTION_FORMATS, RequestArgs, BaseAPI, RequiredError, operationServerMap } from '../base';
 // @ts-ignore
+import { CommonsAppendOperationInput } from '../models';
+// @ts-ignore
 import { CommonsCommitCacheRequest } from '../models';
 // @ts-ignore
 import { CommonsCommitCacheResponse } from '../models';
@@ -34,6 +36,8 @@ import { CommonsGetCacheRequest } from '../models';
 // @ts-ignore
 import { CommonsGetCacheResponse } from '../models';
 // @ts-ignore
+import { CommonsOperation } from '../models';
+// @ts-ignore
 import { CommonsReserveCacheRequest } from '../models';
 // @ts-ignore
 import { CommonsReserveCacheResponse } from '../models';
@@ -207,6 +211,42 @@ export const DefaultApiAxiosParamCreator = function (configuration?: Configurati
 
 
     
+            localVarHeaderParameter['Content-Type'] = 'application/json';
+
+            setSearchParams(localVarUrlObj, localVarQueryParameter);
+            let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {};
+            localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers};
+            localVarRequestOptions.data = serializeDataIfNeeded(body, localVarRequestOptions, configuration)
+
+            return {
+                url: toPathString(localVarUrlObj),
+                options: localVarRequestOptions,
+            };
+        },
+        /**
+         * record operation
+         * @summary record operation
+         * @param {CommonsAppendOperationInput} body Record Operation details Request Body
+         * @param {*} [options] Override http request option.
+         * @throws {RequiredError}
+         */
+        v1InstrumentationOperationPost: async (body: CommonsAppendOperationInput, options: RawAxiosRequestConfig = {}): Promise<RequestArgs> => {
+            // verify required parameter 'body' is not null or undefined
+            assertParamExists('v1InstrumentationOperationPost', 'body', body)
+            const localVarPath = `/v1/instrumentation/operation`;
+            // use dummy base URL string because the URL constructor only accepts absolute URLs.
+            const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL);
+            let baseOptions;
+            if (configuration) {
+                baseOptions = configuration.baseOptions;
+            }
+
+            const localVarRequestOptions = { method: 'POST', ...baseOptions, ...options};
+            const localVarHeaderParameter = {} as any;
+            const localVarQueryParameter = {} as any;
+
+
+    
             localVarHeaderParameter['Content-Type'] = 'application/json';
 
             setSearchParams(localVarUrlObj, localVarQueryParameter);
@@ -293,6 +333,19 @@ export const DefaultApiFp = function(configuration?: Configuration) {
             const localVarOperationServerBasePath = operationServerMap['DefaultApi.v1CacheReservePost']?.[localVarOperationServerIndex]?.url;
             return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath);
         },
+        /**
+         * record operation
+         * @summary record operation
+         * @param {CommonsAppendOperationInput} body Record Operation details Request Body
+         * @param {*} [options] Override http request option.
+         * @throws {RequiredError}
+         */
+        async v1InstrumentationOperationPost(body: CommonsAppendOperationInput, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<CommonsOperation>> {
+            const localVarAxiosArgs = await localVarAxiosParamCreator.v1InstrumentationOperationPost(body, options);
+            const localVarOperationServerIndex = configuration?.serverIndex ?? 0;
+            const localVarOperationServerBasePath = operationServerMap['DefaultApi.v1InstrumentationOperationPost']?.[localVarOperationServerIndex]?.url;
+            return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath);
+        },
     }
 };
 
@@ -352,6 +405,16 @@ export const DefaultApiFactory = function (configuration?: Configuration, basePa
         v1CacheReservePost(requestParameters: DefaultApiV1CacheReservePostRequest, options?: RawAxiosRequestConfig): AxiosPromise<CommonsReserveCacheResponse> {
             return localVarFp.v1CacheReservePost(requestParameters.body, options).then((request) => request(axios, basePath));
         },
+        /**
+         * record operation
+         * @summary record operation
+         * @param {DefaultApiV1InstrumentationOperationPostRequest} requestParameters Request parameters.
+         * @param {*} [options] Override http request option.
+         * @throws {RequiredError}
+         */
+        v1InstrumentationOperationPost(requestParameters: DefaultApiV1InstrumentationOperationPostRequest, options?: RawAxiosRequestConfig): AxiosPromise<CommonsOperation> {
+            return localVarFp.v1InstrumentationOperationPost(requestParameters.body, options).then((request) => request(axios, basePath));
+        },
     };
 };
 
@@ -411,6 +474,20 @@ export interface DefaultApiV1CacheReservePostRequest {
     readonly body: CommonsReserveCacheRequest
 }
 
+/**
+ * Request parameters for v1InstrumentationOperationPost operation in DefaultApi.
+ * @export
+ * @interface DefaultApiV1InstrumentationOperationPostRequest
+ */
+export interface DefaultApiV1InstrumentationOperationPostRequest {
+    /**
+     * Record Operation details Request Body
+     * @type {CommonsAppendOperationInput}
+     * @memberof DefaultApiV1InstrumentationOperationPost
+     */
+    readonly body: CommonsAppendOperationInput
+}
+
 /**
  * DefaultApi - object-oriented interface
  * @export
@@ -476,5 +553,17 @@ export class DefaultApi extends BaseAPI {
     public v1CacheReservePost(requestParameters: DefaultApiV1CacheReservePostRequest, options?: RawAxiosRequestConfig) {
         return DefaultApiFp(this.configuration).v1CacheReservePost(requestParameters.body, options).then((request) => request(this.axios, this.basePath));
     }
+
+    /**
+     * record operation
+     * @summary record operation
+     * @param {DefaultApiV1InstrumentationOperationPostRequest} requestParameters Request parameters.
+     * @param {*} [options] Override http request option.
+     * @throws {RequiredError}
+     * @memberof DefaultApi
+     */
+    public v1InstrumentationOperationPost(requestParameters: DefaultApiV1InstrumentationOperationPostRequest, options?: RawAxiosRequestConfig) {
+        return DefaultApiFp(this.configuration).v1InstrumentationOperationPost(requestParameters.body, options).then((request) => request(this.axios, this.basePath));
+    }
 }
 

packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-append-operation-input.ts

@@ -0,0 +1,60 @@
+/* tslint:disable */
+/* eslint-disable */
+/**
+ * 
+ * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+ *
+ * The version of the OpenAPI document: 1.0.0
+ * 
+ *
+ * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+ * https://openapi-generator.tech
+ * Do not edit the class manually.
+ */
+
+
+
+/**
+ * 
+ * @export
+ * @interface CommonsAppendOperationInput
+ */
+export interface CommonsAppendOperationInput {
+    /**
+     * 
+     * @type {string}
+     * @memberof CommonsAppendOperationInput
+     */
+    'cache_id'?: string;
+    /**
+     * 
+     * @type {string}
+     * @memberof CommonsAppendOperationInput
+     */
+    'external_id'?: string;
+    /**
+     * 
+     * @type {{ [key: string]: any; }}
+     * @memberof CommonsAppendOperationInput
+     */
+    'meta'?: { [key: string]: any; };
+    /**
+     * 
+     * @type {string}
+     * @memberof CommonsAppendOperationInput
+     */
+    'operation_type'?: string;
+    /**
+     * 
+     * @type {number}
+     * @memberof CommonsAppendOperationInput
+     */
+    'size'?: number;
+    /**
+     * 
+     * @type {string}
+     * @memberof CommonsAppendOperationInput
+     */
+    'time'?: string;
+}
+

packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-cache-entry.ts

@@ -50,6 +50,12 @@ export interface CommonsCacheEntry {
      * @memberof CommonsCacheEntry
      */
     'organization_id'?: string;
+    /**
+     * 
+     * @type {string}
+     * @memberof CommonsCacheEntry
+     */
+    'storage_backend_id'?: string;
     /**
      * 
      * @type {string}
@@ -62,6 +68,12 @@ export interface CommonsCacheEntry {
      * @memberof CommonsCacheEntry
      */
     'vcs_organization_name'?: string;
+    /**
+     * 
+     * @type {string}
+     * @memberof CommonsCacheEntry
+     */
+    'vcs_ref'?: string;
     /**
      * 
      * @type {string}

packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-gcsget-cache-reponse.ts

@@ -47,6 +47,12 @@ export interface CommonsGCSGetCacheReponse {
      * @memberof CommonsGCSGetCacheReponse
      */
     'method'?: string;
+    /**
+     * 
+     * @type {string}
+     * @memberof CommonsGCSGetCacheReponse
+     */
+    'pre_signed_url'?: string;
     /**
      * 
      * @type {string}

packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-operation.ts

@@ -0,0 +1,66 @@
+/* tslint:disable */
+/* eslint-disable */
+/**
+ * 
+ * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+ *
+ * The version of the OpenAPI document: 1.0.0
+ * 
+ *
+ * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+ * https://openapi-generator.tech
+ * Do not edit the class manually.
+ */
+
+
+
+/**
+ * 
+ * @export
+ * @interface CommonsOperation
+ */
+export interface CommonsOperation {
+    /**
+     * 
+     * @type {string}
+     * @memberof CommonsOperation
+     */
+    'cache_id'?: string;
+    /**
+     * 
+     * @type {string}
+     * @memberof CommonsOperation
+     */
+    'created-at'?: string;
+    /**
+     * 
+     * @type {string}
+     * @memberof CommonsOperation
+     */
+    'id'?: string;
+    /**
+     * 
+     * @type {{ [key: string]: any; }}
+     * @memberof CommonsOperation
+     */
+    'meta'?: { [key: string]: any; };
+    /**
+     * 
+     * @type {string}
+     * @memberof CommonsOperation
+     */
+    'operation_type'?: string;
+    /**
+     * 
+     * @type {string}
+     * @memberof CommonsOperation
+     */
+    'organization_id'?: string;
+    /**
+     * 
+     * @type {string}
+     * @memberof CommonsOperation
+     */
+    'updated_at'?: string;
+}
+

packages/warp-cache/src/internal/warpcache-ts-sdk/models/index.ts

@@ -1,3 +1,4 @@
+export * from './commons-append-operation-input';
 export * from './commons-cache-entry';
 export * from './commons-commit-cache-request';
 export * from './commons-commit-cache-response';
@@ -9,6 +10,7 @@ export * from './commons-gcsget-cache-reponse';
 export * from './commons-gcsreserve-cache-response';
 export * from './commons-get-cache-request';
 export * from './commons-get-cache-response';
+export * from './commons-operation';
 export * from './commons-reserve-cache-request';
 export * from './commons-reserve-cache-response';
 export * from './commons-s3-commit-cache-response';

packages/warp-cache/src/internal/warpcache-ts-sdk/schema.ts

@@ -1,6 +1,7 @@
 /* istanbul ignore file */
 /* tslint:disable */
 /* eslint-disable */
+export { $commons_AppendOperationInput } from './schemas/$commons_AppendOperationInput';
 export { $commons_CacheAnnotationsMap } from './schemas/$commons_CacheAnnotationsMap';
 export { $commons_CacheEntry } from './schemas/$commons_CacheEntry';
 export { $commons_CommitCacheRequest } from './schemas/$commons_CommitCacheRequest';
@@ -13,6 +14,7 @@ export { $commons_GCSGetCacheReponse } from './schemas/$commons_GCSGetCacheRepon
 export { $commons_GCSReserveCacheResponse } from './schemas/$commons_GCSReserveCacheResponse';
 export { $commons_GetCacheRequest } from './schemas/$commons_GetCacheRequest';
 export { $commons_GetCacheResponse } from './schemas/$commons_GetCacheResponse';
+export { $commons_Operation } from './schemas/$commons_Operation';
 export { $commons_ReserveCacheRequest } from './schemas/$commons_ReserveCacheRequest';
 export { $commons_ReserveCacheResponse } from './schemas/$commons_ReserveCacheResponse';
 export { $commons_S3CommitCacheResponse } from './schemas/$commons_S3CommitCacheResponse';

packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_AppendOperationInput.ts

@@ -0,0 +1,26 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+export const $commons_AppendOperationInput = {
+    properties: {
+        cache_id: {
+            type: 'string',
+        },
+        external_id: {
+            type: 'string',
+        },
+        meta: {
+            properties: {
+            },
+        },
+        operation_type: {
+            type: 'string',
+        },
+        size: {
+            type: 'number',
+        },
+        time: {
+            type: 'string',
+        },
+    },
+} as const;

packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CacheEntry.ts

@@ -18,12 +18,18 @@ export const $commons_CacheEntry = {
         organization_id: {
             type: 'string',
         },
+        storage_backend_id: {
+            type: 'string',
+        },
         updated_at: {
             type: 'string',
         },
         vcs_organization_name: {
             type: 'string',
         },
+        vcs_ref: {
+            type: 'string',
+        },
         vcs_repository_name: {
             type: 'string',
         },

packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GCSGetCacheReponse.ts

@@ -15,6 +15,9 @@ export const $commons_GCSGetCacheReponse = {
         method: {
             type: 'string',
         },
+        pre_signed_url: {
+            type: 'string',
+        },
         project_id: {
             type: 'string',
         },

packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_Operation.ts

@@ -0,0 +1,29 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+export const $commons_Operation = {
+    properties: {
+        cache_id: {
+            type: 'string',
+        },
+        'created-at': {
+            type: 'string',
+        },
+        id: {
+            type: 'string',
+        },
+        meta: {
+            properties: {
+            },
+        },
+        operation_type: {
+            type: 'string',
+        },
+        organization_id: {
+            type: 'string',
+        },
+        updated_at: {
+            type: 'string',
+        },
+    },
+} as const;

packages/warp-cache/src/test.ts

@@ -4,6 +4,8 @@ import {getCompressionMethod} from './internal/cacheUtils'
 
 process.env['WARPBUILD_CACHE_URL'] = 'https://cache.dev.warpbuild.dev'
 // process.env['WARPBUILD_CACHE_URL'] = 'http://localhost:8000'
+// process.env['WARPBUILD_CACHE_URL'] =
+//   'https://6134-36-255-234-176.ngrok-free.app'
 process.env['RUNNER_TEMP'] = '/Users/prajjwal/Repos/warpbuild/playground/tmp_fs'
 process.env['NODE_DEBUG'] = 'http'
 process.env['RUNNER_DEBUG'] = '1'