actions
/
toolkit
mirror of https://github.com/actions/toolkit
1
0
Fork 0
Code

adds wget for streaming download

pull/1935/head
Prajjwal 2024-04-15 17:42:41 +05:30
parent 63c6089651
commit d989701710
17 changed files with 366 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/warp-cache/package.json
View File

@ -1,6 +1,6 @@
{
"name": "github-actions.warp-cache",
"version": "1.1.0",
"version": "1.1.1",
"preview": true,
"description": "Github action to use WarpBuild's in-house cache offering",
"keywords": [

55
packages/warp-cache/src/cache.ts
View File

@ -10,6 +10,8 @@ import {
} from './internal/tar'
import {DownloadOptions, getUploadOptions} from './options'
import {isSuccessStatusCode} from './internal/requestUtils'
import {getDownloadCommandPipeForWget} from './internal/downloadUtils'
import {ChildProcessWithoutNullStreams} from 'child_process'
export class ValidationError extends Error {
constructor(message: string) {
@ -171,29 +173,40 @@ export async function restoreCache(
const archiveLocation = `gs://${cacheEntry.gcs?.bucket_name}/${cacheEntry.gcs?.cache_key}`
// await cacheHttpClient.downloadCache(
// cacheEntry.provider,
// archiveLocation,
// archivePath,
// cacheEntry.gcs?.short_lived_token?.access_token ?? ''
// )
/*
* Alternate, Multipart download method for GCS
await cacheHttpClient.downloadCache(
cacheEntry.provider,
archiveLocation,
archivePath,
cacheEntry.gcs?.short_lived_token?.access_token ?? ''
)
// if (core.isDebug()) {
// await listTar(archivePath, compressionMethod)
// }
if (core.isDebug()) {
await listTar(archivePath, compressionMethod)
}
// const archiveFileSize = utils.getArchiveFileSizeInBytes(archivePath)
// core.info(
// `Cache Size: ~${Math.round(
// archiveFileSize / (1024 * 1024)
// )} MB (${archiveFileSize} B)`
// )
const archiveFileSize = utils.getArchiveFileSizeInBytes(archivePath)
core.info(
`Cache Size: ~${Math.round(
archiveFileSize / (1024 * 1024)
)} MB (${archiveFileSize} B)`
)
// await extractTar(archivePath, compressionMethod)
await extractTar(archivePath, compressionMethod)
*/
// For GCS, we do a streaming download which means that we extract the archive while we are downloading it.
const readStream = cacheHttpClient.downloadCacheStreaming(
let readStream: NodeJS.ReadableStream | undefined
let downloadCommandPipe: ChildProcessWithoutNullStreams | undefined
if (cacheEntry?.gcs?.pre_signed_url) {
downloadCommandPipe = getDownloadCommandPipeForWget(
cacheEntry?.gcs?.pre_signed_url
)
} else {
readStream = cacheHttpClient.downloadCacheStreaming(
'gcs',
archiveLocation,
cacheEntry?.gcs?.short_lived_token?.access_token ?? ''
@ -202,8 +215,14 @@ export async function restoreCache(
if (!readStream) {
return undefined
}
}
await extractStreamingTar(readStream, archivePath, compressionMethod)
await extractStreamingTar(
readStream,
archivePath,
compressionMethod,
downloadCommandPipe
)
core.info('Cache restored successfully')
break
}

7
packages/warp-cache/src/internal/downloadUtils.ts
View File

@ -14,6 +14,7 @@ import {retryHttpClientResponse} from './requestUtils'
import {AbortController} from '@azure/abort-controller'
import {Storage, TransferManager} from '@google-cloud/storage'
import {ChildProcessWithoutNullStreams, spawn} from 'child_process'
/**
* Pipes the body of a HTTP response to a stream
@ -350,3 +351,9 @@ export function downloadCacheStreamingGCP(
throw error
}
}
export function getDownloadCommandPipeForWget(
url: string
): ChildProcessWithoutNullStreams {
return spawn('wget', ['-qO', '-', url])
}

11
packages/warp-cache/src/internal/tar.ts
View File

@ -417,9 +417,10 @@ export async function extractTar(
* NOTE: Currently tested only on archives created using tar and zstd
*/
export async function extractStreamingTar(
stream: NodeJS.ReadableStream,
stream: NodeJS.ReadableStream | undefined,
archivePath: string,
compressionMethod: CompressionMethod
compressionMethod: CompressionMethod,
downloadCommandPipe?: ChildProcessWithoutNullStreams
): Promise<void> {
const workingDirectory = getWorkingDirectory()
await io.mkdirP(workingDirectory)
@ -429,6 +430,10 @@ export async function extractStreamingTar(
archivePath
)
if (downloadCommandPipe) {
commandPipes.unshift(downloadCommandPipe)
}
if (commandPipes.length < 2) {
throw new Error(
'At least two processes should be present as the archive is compressed at least twice.'
@ -436,7 +441,9 @@ export async function extractStreamingTar(
}
return new Promise((resolve, reject) => {
if (stream) {
stream.pipe(commandPipes[0].stdin)
}
for (let i = 0; i < commandPipes.length - 1; i++) {
commandPipes[i].stdout.pipe(commandPipes[i + 1].stdin)

2
packages/warp-cache/src/internal/warpcache-ts-sdk/.openapi-generator/FILES
View File

@ -9,6 +9,7 @@ common.ts
configuration.ts
git_push.sh
index.ts
models/commons-append-operation-input.ts
models/commons-cache-entry.ts
models/commons-commit-cache-request.ts
models/commons-commit-cache-response.ts
@ -20,6 +21,7 @@ models/commons-gcsget-cache-reponse.ts
models/commons-gcsreserve-cache-response.ts
models/commons-get-cache-request.ts
models/commons-get-cache-response.ts
models/commons-operation.ts
models/commons-reserve-cache-request.ts
models/commons-reserve-cache-response.ts
models/commons-s3-commit-cache-response.ts

89
packages/warp-cache/src/internal/warpcache-ts-sdk/api/default-api.ts
View File

@ -22,6 +22,8 @@ import { DUMMY_BASE_URL, assertParamExists, setApiKeyToObject, setBasicAuthToObj
// @ts-ignore
import { BASE_PATH, COLLECTION_FORMATS, RequestArgs, BaseAPI, RequiredError, operationServerMap } from '../base';
// @ts-ignore
import { CommonsAppendOperationInput } from '../models';
// @ts-ignore
import { CommonsCommitCacheRequest } from '../models';
// @ts-ignore
import { CommonsCommitCacheResponse } from '../models';
@ -34,6 +36,8 @@ import { CommonsGetCacheRequest } from '../models';
// @ts-ignore
import { CommonsGetCacheResponse } from '../models';
// @ts-ignore
import { CommonsOperation } from '../models';
// @ts-ignore
import { CommonsReserveCacheRequest } from '../models';
// @ts-ignore
import { CommonsReserveCacheResponse } from '../models';
@ -207,6 +211,42 @@ export const DefaultApiAxiosParamCreator = function (configuration?: Configurati
localVarHeaderParameter['Content-Type'] = 'application/json';
setSearchParams(localVarUrlObj, localVarQueryParameter);
let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {};
localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers};
localVarRequestOptions.data = serializeDataIfNeeded(body, localVarRequestOptions, configuration)
return {
url: toPathString(localVarUrlObj),
options: localVarRequestOptions,
};
},
/**
* record operation
* @summary record operation
* @param {CommonsAppendOperationInput} body Record Operation details Request Body
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
v1InstrumentationOperationPost: async (body: CommonsAppendOperationInput, options: RawAxiosRequestConfig = {}): Promise<RequestArgs> => {
// verify required parameter 'body' is not null or undefined
assertParamExists('v1InstrumentationOperationPost', 'body', body)
const localVarPath = `/v1/instrumentation/operation`;
// use dummy base URL string because the URL constructor only accepts absolute URLs.
const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL);
let baseOptions;
if (configuration) {
baseOptions = configuration.baseOptions;
}
const localVarRequestOptions = { method: 'POST', ...baseOptions, ...options};
const localVarHeaderParameter = {} as any;
const localVarQueryParameter = {} as any;
localVarHeaderParameter['Content-Type'] = 'application/json';
setSearchParams(localVarUrlObj, localVarQueryParameter);
@ -293,6 +333,19 @@ export const DefaultApiFp = function(configuration?: Configuration) {
const localVarOperationServerBasePath = operationServerMap['DefaultApi.v1CacheReservePost']?.[localVarOperationServerIndex]?.url;
return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath);
},
/**
* record operation
* @summary record operation
* @param {CommonsAppendOperationInput} body Record Operation details Request Body
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
async v1InstrumentationOperationPost(body: CommonsAppendOperationInput, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<CommonsOperation>> {
const localVarAxiosArgs = await localVarAxiosParamCreator.v1InstrumentationOperationPost(body, options);
const localVarOperationServerIndex = configuration?.serverIndex ?? 0;
const localVarOperationServerBasePath = operationServerMap['DefaultApi.v1InstrumentationOperationPost']?.[localVarOperationServerIndex]?.url;
return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath);
},
}
};
@ -352,6 +405,16 @@ export const DefaultApiFactory = function (configuration?: Configuration, basePa
v1CacheReservePost(requestParameters: DefaultApiV1CacheReservePostRequest, options?: RawAxiosRequestConfig): AxiosPromise<CommonsReserveCacheResponse> {
return localVarFp.v1CacheReservePost(requestParameters.body, options).then((request) => request(axios, basePath));
},
/**
* record operation
* @summary record operation
* @param {DefaultApiV1InstrumentationOperationPostRequest} requestParameters Request parameters.
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
v1InstrumentationOperationPost(requestParameters: DefaultApiV1InstrumentationOperationPostRequest, options?: RawAxiosRequestConfig): AxiosPromise<CommonsOperation> {
return localVarFp.v1InstrumentationOperationPost(requestParameters.body, options).then((request) => request(axios, basePath));
},
};
};
@ -411,6 +474,20 @@ export interface DefaultApiV1CacheReservePostRequest {
readonly body: CommonsReserveCacheRequest
}
/**
* Request parameters for v1InstrumentationOperationPost operation in DefaultApi.
* @export
* @interface DefaultApiV1InstrumentationOperationPostRequest
*/
export interface DefaultApiV1InstrumentationOperationPostRequest {
/**
* Record Operation details Request Body
* @type {CommonsAppendOperationInput}
* @memberof DefaultApiV1InstrumentationOperationPost
*/
readonly body: CommonsAppendOperationInput
}
/**
* DefaultApi - object-oriented interface
* @export
@ -476,5 +553,17 @@ export class DefaultApi extends BaseAPI {
public v1CacheReservePost(requestParameters: DefaultApiV1CacheReservePostRequest, options?: RawAxiosRequestConfig) {
return DefaultApiFp(this.configuration).v1CacheReservePost(requestParameters.body, options).then((request) => request(this.axios, this.basePath));
}
/**
* record operation
* @summary record operation
* @param {DefaultApiV1InstrumentationOperationPostRequest} requestParameters Request parameters.
* @param {*} [options] Override http request option.
* @throws {RequiredError}
* @memberof DefaultApi
*/
public v1InstrumentationOperationPost(requestParameters: DefaultApiV1InstrumentationOperationPostRequest, options?: RawAxiosRequestConfig) {
return DefaultApiFp(this.configuration).v1InstrumentationOperationPost(requestParameters.body, options).then((request) => request(this.axios, this.basePath));
}
}

60
packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-append-operation-input.ts Normal file
View File

@ -0,0 +1,60 @@
/* tslint:disable */
/* eslint-disable */
/**
*
* No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
*
* The version of the OpenAPI document: 1.0.0
*
*
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
* https://openapi-generator.tech
* Do not edit the class manually.
*/
/**
*
* @export
* @interface CommonsAppendOperationInput
*/
export interface CommonsAppendOperationInput {
/**
*
* @type {string}
* @memberof CommonsAppendOperationInput
*/
'cache_id'?: string;
/**
*
* @type {string}
* @memberof CommonsAppendOperationInput
*/
'external_id'?: string;
/**
*
* @type {{ [key: string]: any; }}
* @memberof CommonsAppendOperationInput
*/
'meta'?: { [key: string]: any; };
/**
*
* @type {string}
* @memberof CommonsAppendOperationInput
*/
'operation_type'?: string;
/**
*
* @type {number}
* @memberof CommonsAppendOperationInput
*/
'size'?: number;
/**
*
* @type {string}
* @memberof CommonsAppendOperationInput
*/
'time'?: string;
}

12
packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-cache-entry.ts
View File

@ -50,6 +50,12 @@ export interface CommonsCacheEntry {
* @memberof CommonsCacheEntry
*/
'organization_id'?: string;
/**
*
* @type {string}
* @memberof CommonsCacheEntry
*/
'storage_backend_id'?: string;
/**
*
* @type {string}
@ -62,6 +68,12 @@ export interface CommonsCacheEntry {
* @memberof CommonsCacheEntry
*/
'vcs_organization_name'?: string;
/**
*
* @type {string}
* @memberof CommonsCacheEntry
*/
'vcs_ref'?: string;
/**
*
* @type {string}

6
packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-gcsget-cache-reponse.ts
View File

@ -47,6 +47,12 @@ export interface CommonsGCSGetCacheReponse {
* @memberof CommonsGCSGetCacheReponse
*/
'method'?: string;
/**
*
* @type {string}
* @memberof CommonsGCSGetCacheReponse
*/
'pre_signed_url'?: string;
/**
*
* @type {string}

66
packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-operation.ts Normal file
View File

@ -0,0 +1,66 @@
/* tslint:disable */
/* eslint-disable */
/**
*
* No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
*
* The version of the OpenAPI document: 1.0.0
*
*
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
* https://openapi-generator.tech
* Do not edit the class manually.
*/
/**
*
* @export
* @interface CommonsOperation
*/
export interface CommonsOperation {
/**
*
* @type {string}
* @memberof CommonsOperation
*/
'cache_id'?: string;
/**
*
* @type {string}
* @memberof CommonsOperation
*/
'created-at'?: string;
/**
*
* @type {string}
* @memberof CommonsOperation
*/
'id'?: string;
/**
*
* @type {{ [key: string]: any; }}
* @memberof CommonsOperation
*/
'meta'?: { [key: string]: any; };
/**
*
* @type {string}
* @memberof CommonsOperation
*/
'operation_type'?: string;
/**
*
* @type {string}
* @memberof CommonsOperation
*/
'organization_id'?: string;
/**
*
* @type {string}
* @memberof CommonsOperation
*/
'updated_at'?: string;
}

2
packages/warp-cache/src/internal/warpcache-ts-sdk/models/index.ts
View File

@ -1,3 +1,4 @@
export * from './commons-append-operation-input';
export * from './commons-cache-entry';
export * from './commons-commit-cache-request';
export * from './commons-commit-cache-response';
@ -9,6 +10,7 @@ export * from './commons-gcsget-cache-reponse';
export * from './commons-gcsreserve-cache-response';
export * from './commons-get-cache-request';
export * from './commons-get-cache-response';
export * from './commons-operation';
export * from './commons-reserve-cache-request';
export * from './commons-reserve-cache-response';
export * from './commons-s3-commit-cache-response';

2
packages/warp-cache/src/internal/warpcache-ts-sdk/schema.ts
View File

@ -1,6 +1,7 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export { $commons_AppendOperationInput } from './schemas/$commons_AppendOperationInput';
export { $commons_CacheAnnotationsMap } from './schemas/$commons_CacheAnnotationsMap';
export { $commons_CacheEntry } from './schemas/$commons_CacheEntry';
export { $commons_CommitCacheRequest } from './schemas/$commons_CommitCacheRequest';
@ -13,6 +14,7 @@ export { $commons_GCSGetCacheReponse } from './schemas/$commons_GCSGetCacheRepon
export { $commons_GCSReserveCacheResponse } from './schemas/$commons_GCSReserveCacheResponse';
export { $commons_GetCacheRequest } from './schemas/$commons_GetCacheRequest';
export { $commons_GetCacheResponse } from './schemas/$commons_GetCacheResponse';
export { $commons_Operation } from './schemas/$commons_Operation';
export { $commons_ReserveCacheRequest } from './schemas/$commons_ReserveCacheRequest';
export { $commons_ReserveCacheResponse } from './schemas/$commons_ReserveCacheResponse';
export { $commons_S3CommitCacheResponse } from './schemas/$commons_S3CommitCacheResponse';

26
packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_AppendOperationInput.ts Normal file
View File

@ -0,0 +1,26 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export const $commons_AppendOperationInput = {
properties: {
cache_id: {
type: 'string',
},
external_id: {
type: 'string',
},
meta: {
properties: {
},
},
operation_type: {
type: 'string',
},
size: {
type: 'number',
},
time: {
type: 'string',
},
},
} as const;

6
packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CacheEntry.ts
View File

@ -18,12 +18,18 @@ export const $commons_CacheEntry = {
organization_id: {
type: 'string',
},
storage_backend_id: {
type: 'string',
},
updated_at: {
type: 'string',
},
vcs_organization_name: {
type: 'string',
},
vcs_ref: {
type: 'string',
},
vcs_repository_name: {
type: 'string',
},

3
packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GCSGetCacheReponse.ts
View File

@ -15,6 +15,9 @@ export const $commons_GCSGetCacheReponse = {
method: {
type: 'string',
},
pre_signed_url: {
type: 'string',
},
project_id: {
type: 'string',
},

29
packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_Operation.ts Normal file
View File

@ -0,0 +1,29 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export const $commons_Operation = {
properties: {
cache_id: {
type: 'string',
},
'created-at': {
type: 'string',
},
id: {
type: 'string',
},
meta: {
properties: {
},
},
operation_type: {
type: 'string',
},
organization_id: {
type: 'string',
},
updated_at: {
type: 'string',
},
},
} as const;

2
packages/warp-cache/src/test.ts
View File

@ -4,6 +4,8 @@ import {getCompressionMethod} from './internal/cacheUtils'
process.env['WARPBUILD_CACHE_URL'] = 'https://cache.dev.warpbuild.dev'
// process.env['WARPBUILD_CACHE_URL'] = 'http://localhost:8000'
// process.env['WARPBUILD_CACHE_URL'] =
// 'https://6134-36-255-234-176.ngrok-free.app'
process.env['RUNNER_TEMP'] = '/Users/prajjwal/Repos/warpbuild/playground/tmp_fs'
process.env['NODE_DEBUG'] = 'http'
process.env['RUNNER_DEBUG'] = '1'