From 1fc87f92d986971813ee5c38af1e06f2e94fceae Mon Sep 17 00:00:00 2001 From: Prajjwal Date: Wed, 10 Apr 2024 10:14:06 +0530 Subject: [PATCH 01/11] adds streaming download support --- packages/warp-cache/__tests__/cache.test.ts | 4 +- packages/warp-cache/package-lock.json | 1007 ++++++++++++++++- packages/warp-cache/package.json | 9 +- packages/warp-cache/src/cache.ts | 76 +- .../src/internal/cacheHttpClient.ts | 38 +- .../warp-cache/src/internal/contracts.d.ts | 3 + .../warp-cache/src/internal/downloadUtils.ts | 45 + packages/warp-cache/src/internal/tar.ts | 207 +++- packages/warp-cache/src/test.ts | 27 +- 9 files changed, 1365 insertions(+), 51 deletions(-) diff --git a/packages/warp-cache/__tests__/cache.test.ts b/packages/warp-cache/__tests__/cache.test.ts index f8420fd5..915a6987 100644 --- a/packages/warp-cache/__tests__/cache.test.ts +++ b/packages/warp-cache/__tests__/cache.test.ts @@ -2,10 +2,10 @@ import * as cache from '../src/cache' test('isFeatureAvailable returns true if server url is set', () => { try { - process.env['WARP_CACHE_URL'] = 'http://cache.com' + process.env['WARPBUILD_CACHE_URL'] = 'http://cache.com' expect(cache.isFeatureAvailable()).toBe(true) } finally { - delete process.env['WARP_CACHE_URL'] + delete process.env['WARPBUILD_CACHE_URL'] } }) diff --git a/packages/warp-cache/package-lock.json b/packages/warp-cache/package-lock.json index a9e84023..d824dd26 100644 --- a/packages/warp-cache/package-lock.json +++ b/packages/warp-cache/package-lock.json @@ -1,12 +1,12 @@ { "name": "github-actions.warp-cache", - "version": "0.1.0", + "version": "0.3.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "github-actions.warp-cache", - "version": "0.1.0", + "version": "0.3.0", "license": "MIT", "dependencies": { "@actions/core": "^1.10.0", @@ -17,6 +17,7 @@ "@azure/abort-controller": "^1.1.0", "@azure/ms-rest-js": "^2.6.0", "@azure/storage-blob": "^12.13.0", + "@google-cloud/storage": "^7.9.0", "axios": "^1.6.2", "semver": "^6.3.1", "uuid": "^3.3.3" @@ -248,6 +249,69 @@ "node": ">=14.0.0" } }, + "node_modules/@google-cloud/paginator": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-5.0.0.tgz", + "integrity": "sha512-87aeg6QQcEPxGCOthnpUjvw4xAZ57G7pL8FS0C4e/81fr3FjkpUpibf1s2v5XGyGhUVGF4Jfg7yEcxqn2iUw1w==", + "dependencies": { + "arrify": "^2.0.0", + "extend": "^3.0.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/projectify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-4.0.0.tgz", + "integrity": "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/promisify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.0.0.tgz", + "integrity": "sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==", + "engines": { + "node": ">=14" + } + }, + "node_modules/@google-cloud/storage": { + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-7.9.0.tgz", + "integrity": "sha512-PlFl7g3r91NmXtZHXsSEfTZES5ysD3SSBWmX4iBdQ2TFH7tN/Vn/IhnVELCHtgh1vc+uYPZ7XvRYaqtDCdghIA==", + "dependencies": { + "@google-cloud/paginator": "^5.0.0", + "@google-cloud/projectify": "^4.0.0", + "@google-cloud/promisify": "^4.0.0", + "abort-controller": "^3.0.0", + "async-retry": "^1.3.3", + "compressible": "^2.0.12", + "duplexify": "^4.1.3", + "ent": "^2.2.0", + "fast-xml-parser": "^4.3.0", + "gaxios": "^6.0.2", + "google-auth-library": "^9.6.3", + "mime": "^3.0.0", + "mime-types": "^2.0.8", + "p-limit": "^3.0.1", + "retry-request": "^7.0.0", + "teeny-request": "^9.0.0", + "uuid": "^8.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@google-cloud/storage/node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/@opentelemetry/api": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.4.1.tgz", @@ -256,6 +320,19 @@ "node": ">=8.0.0" } }, + "node_modules/@tootallnate/once": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", + "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", + "engines": { + "node": ">= 10" + } + }, + "node_modules/@types/caseless": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.5.tgz", + "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==" + }, "node_modules/@types/node": { "version": "20.4.6", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.4.6.tgz", @@ -283,12 +360,28 @@ "node": ">= 6" } }, + "node_modules/@types/request": { + "version": "2.48.12", + "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.12.tgz", + "integrity": "sha512-G3sY+NpsA9jnwm0ixhAFQSJ3Q9JkpLZpJbI3GMv0mIAT0y3mRabYeINzal5WOChIiaTEGQYlHOKgkaM9EisWHw==", + "dependencies": { + "@types/caseless": "*", + "@types/node": "*", + "@types/tough-cookie": "*", + "form-data": "^2.5.0" + } + }, "node_modules/@types/semver": { "version": "6.2.3", "resolved": "https://registry.npmjs.org/@types/semver/-/semver-6.2.3.tgz", "integrity": "sha512-KQf+QAMWKMrtBMsB8/24w53tEsxllMj6TuA80TT/5igJalLI/zm0L3oXRbIAl4Ohfc85gyHX/jhMwsVkmhLU4A==", "dev": true }, + "node_modules/@types/tough-cookie": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", + "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==" + }, "node_modules/@types/tunnel": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/@types/tunnel/-/tunnel-0.0.3.tgz", @@ -314,6 +407,33 @@ "node": ">=6.5" } }, + "node_modules/agent-base": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/arrify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", + "engines": { + "node": ">=8" + } + }, + "node_modules/async-retry": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", + "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", + "dependencies": { + "retry": "0.13.1" + } + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -347,6 +467,33 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/bignumber.js": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.2.tgz", + "integrity": "sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==", + "engines": { + "node": "*" + } + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -356,6 +503,11 @@ "concat-map": "0.0.1" } }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==" + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -367,11 +519,38 @@ "node": ">= 0.8" } }, + "node_modules/compressible": { + "version": "2.0.18", + "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", + "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", + "dependencies": { + "mime-db": ">= 1.43.0 < 2" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" }, + "node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -380,6 +559,38 @@ "node": ">=0.4.0" } }, + "node_modules/duplexify": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz", + "integrity": "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==", + "dependencies": { + "end-of-stream": "^1.4.1", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1", + "stream-shift": "^1.0.2" + } + }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/ent": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz", + "integrity": "sha512-GHrMyVZQWvTIdDtpiEXdHZnFQKzeO09apj8Cbl4pKWy4i0Oprcq17usfDt5aO63swf0JOeMWjWQE/LzgSRuWpA==" + }, "node_modules/event-target-shim": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", @@ -396,6 +607,32 @@ "node": ">=0.8.x" } }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, + "node_modules/fast-xml-parser": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.3.6.tgz", + "integrity": "sha512-M2SovcRxD4+vC493Uc2GZVcZaj66CCJhWurC4viynVSTvrpErCShNcDz1lAho6n9REQKvL/ll4A4/fw6Y9z8nw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/follow-redirects": { "version": "1.15.3", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz", @@ -428,6 +665,163 @@ "node": ">= 0.12" } }, + "node_modules/gaxios": { + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.4.0.tgz", + "integrity": "sha512-apAloYrY4dlBGlhauDAYSZveafb5U6+L9titing1wox6BvWM0TSXBp603zTrLpyLMGkrcFgohnUN150dFN/zOA==", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/gaxios/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/gcp-metadata": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.0.tgz", + "integrity": "sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==", + "dependencies": { + "gaxios": "^6.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/google-auth-library": { + "version": "9.7.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.7.0.tgz", + "integrity": "sha512-I/AvzBiUXDzLOy4iIZ2W+Zq33W4lcukQv1nl7C8WUA6SQwyQwUwu3waNmWNAvzds//FG8SZ+DnKnW/2k6mQS8A==", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^6.1.1", + "gcp-metadata": "^6.1.0", + "gtoken": "^7.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/gtoken": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", + "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", + "dependencies": { + "gaxios": "^6.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/http-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", + "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", + "dependencies": { + "@tootallnate/once": "2", + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/http-proxy-agent/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz", + "integrity": "sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg==", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, + "node_modules/jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "dependencies": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "dependencies": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/mime": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", + "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", @@ -458,6 +852,11 @@ "node": "*" } }, + "node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, "node_modules/node-fetch": { "version": "2.6.12", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.12.tgz", @@ -477,6 +876,28 @@ } } }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/process": { "version": "0.11.10", "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", @@ -490,6 +911,59 @@ "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "engines": { + "node": ">= 4" + } + }, + "node_modules/retry-request": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-7.0.2.tgz", + "integrity": "sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==", + "dependencies": { + "@types/request": "^2.48.8", + "extend": "^3.0.2", + "teeny-request": "^9.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/sax": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", @@ -503,6 +977,87 @@ "semver": "bin/semver.js" } }, + "node_modules/stream-events": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", + "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", + "dependencies": { + "stubs": "^3.0.0" + } + }, + "node_modules/stream-shift": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", + "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==" + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/strnum": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" + }, + "node_modules/stubs": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", + "integrity": "sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==" + }, + "node_modules/teeny-request": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-9.0.0.tgz", + "integrity": "sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==", + "dependencies": { + "http-proxy-agent": "^5.0.0", + "https-proxy-agent": "^5.0.0", + "node-fetch": "^2.6.9", + "stream-events": "^1.0.5", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/teeny-request/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/teeny-request/node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/teeny-request/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", @@ -534,6 +1089,11 @@ "node": ">=14.17" } }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, "node_modules/uuid": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", @@ -557,6 +1117,11 @@ "webidl-conversions": "^3.0.0" } }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + }, "node_modules/xml2js": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz", @@ -576,6 +1141,17 @@ "engines": { "node": ">=4.0" } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } } }, "dependencies": { @@ -767,11 +1343,71 @@ "tslib": "^2.2.0" } }, + "@google-cloud/paginator": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-5.0.0.tgz", + "integrity": "sha512-87aeg6QQcEPxGCOthnpUjvw4xAZ57G7pL8FS0C4e/81fr3FjkpUpibf1s2v5XGyGhUVGF4Jfg7yEcxqn2iUw1w==", + "requires": { + "arrify": "^2.0.0", + "extend": "^3.0.2" + } + }, + "@google-cloud/projectify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-4.0.0.tgz", + "integrity": "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==" + }, + "@google-cloud/promisify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.0.0.tgz", + "integrity": "sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==" + }, + "@google-cloud/storage": { + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-7.9.0.tgz", + "integrity": "sha512-PlFl7g3r91NmXtZHXsSEfTZES5ysD3SSBWmX4iBdQ2TFH7tN/Vn/IhnVELCHtgh1vc+uYPZ7XvRYaqtDCdghIA==", + "requires": { + "@google-cloud/paginator": "^5.0.0", + "@google-cloud/projectify": "^4.0.0", + "@google-cloud/promisify": "^4.0.0", + "abort-controller": "^3.0.0", + "async-retry": "^1.3.3", + "compressible": "^2.0.12", + "duplexify": "^4.1.3", + "ent": "^2.2.0", + "fast-xml-parser": "^4.3.0", + "gaxios": "^6.0.2", + "google-auth-library": "^9.6.3", + "mime": "^3.0.0", + "mime-types": "^2.0.8", + "p-limit": "^3.0.1", + "retry-request": "^7.0.0", + "teeny-request": "^9.0.0", + "uuid": "^8.0.0" + }, + "dependencies": { + "uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" + } + } + }, "@opentelemetry/api": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.4.1.tgz", "integrity": "sha512-O2yRJce1GOc6PAy3QxFM4NzFiWzvScDC1/5ihYBL6BUEVdq0XMWN01sppE+H6bBXbaFYipjwFLEWLg5PaSOThA==" }, + "@tootallnate/once": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", + "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==" + }, + "@types/caseless": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.5.tgz", + "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==" + }, "@types/node": { "version": "20.4.6", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.4.6.tgz", @@ -798,12 +1434,28 @@ } } }, + "@types/request": { + "version": "2.48.12", + "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.12.tgz", + "integrity": "sha512-G3sY+NpsA9jnwm0ixhAFQSJ3Q9JkpLZpJbI3GMv0mIAT0y3mRabYeINzal5WOChIiaTEGQYlHOKgkaM9EisWHw==", + "requires": { + "@types/caseless": "*", + "@types/node": "*", + "@types/tough-cookie": "*", + "form-data": "^2.5.0" + } + }, "@types/semver": { "version": "6.2.3", "resolved": "https://registry.npmjs.org/@types/semver/-/semver-6.2.3.tgz", "integrity": "sha512-KQf+QAMWKMrtBMsB8/24w53tEsxllMj6TuA80TT/5igJalLI/zm0L3oXRbIAl4Ohfc85gyHX/jhMwsVkmhLU4A==", "dev": true }, + "@types/tough-cookie": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", + "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==" + }, "@types/tunnel": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/@types/tunnel/-/tunnel-0.0.3.tgz", @@ -826,6 +1478,27 @@ "event-target-shim": "^5.0.0" } }, + "agent-base": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", + "requires": { + "debug": "^4.3.4" + } + }, + "arrify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==" + }, + "async-retry": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", + "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", + "requires": { + "retry": "0.13.1" + } + }, "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -858,6 +1531,16 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, + "base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, + "bignumber.js": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.2.tgz", + "integrity": "sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==" + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -867,6 +1550,11 @@ "concat-map": "0.0.1" } }, + "buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==" + }, "combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -875,16 +1563,64 @@ "delayed-stream": "~1.0.0" } }, + "compressible": { + "version": "2.0.18", + "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", + "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", + "requires": { + "mime-db": ">= 1.43.0 < 2" + } + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" }, + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "requires": { + "ms": "2.1.2" + } + }, "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" }, + "duplexify": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz", + "integrity": "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==", + "requires": { + "end-of-stream": "^1.4.1", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1", + "stream-shift": "^1.0.2" + } + }, + "ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "requires": { + "safe-buffer": "^5.0.1" + } + }, + "end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "requires": { + "once": "^1.4.0" + } + }, + "ent": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz", + "integrity": "sha512-GHrMyVZQWvTIdDtpiEXdHZnFQKzeO09apj8Cbl4pKWy4i0Oprcq17usfDt5aO63swf0JOeMWjWQE/LzgSRuWpA==" + }, "event-target-shim": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", @@ -895,6 +1631,19 @@ "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==" }, + "extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, + "fast-xml-parser": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.3.6.tgz", + "integrity": "sha512-M2SovcRxD4+vC493Uc2GZVcZaj66CCJhWurC4viynVSTvrpErCShNcDz1lAho6n9REQKvL/ll4A4/fw6Y9z8nw==", + "requires": { + "strnum": "^1.0.5" + } + }, "follow-redirects": { "version": "1.15.3", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz", @@ -910,6 +1659,127 @@ "mime-types": "^2.1.12" } }, + "gaxios": { + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.4.0.tgz", + "integrity": "sha512-apAloYrY4dlBGlhauDAYSZveafb5U6+L9titing1wox6BvWM0TSXBp603zTrLpyLMGkrcFgohnUN150dFN/zOA==", + "requires": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9", + "uuid": "^9.0.1" + }, + "dependencies": { + "uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==" + } + } + }, + "gcp-metadata": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.0.tgz", + "integrity": "sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==", + "requires": { + "gaxios": "^6.0.0", + "json-bigint": "^1.0.0" + } + }, + "google-auth-library": { + "version": "9.7.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.7.0.tgz", + "integrity": "sha512-I/AvzBiUXDzLOy4iIZ2W+Zq33W4lcukQv1nl7C8WUA6SQwyQwUwu3waNmWNAvzds//FG8SZ+DnKnW/2k6mQS8A==", + "requires": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^6.1.1", + "gcp-metadata": "^6.1.0", + "gtoken": "^7.0.0", + "jws": "^4.0.0" + } + }, + "gtoken": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", + "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", + "requires": { + "gaxios": "^6.0.0", + "jws": "^4.0.0" + } + }, + "http-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", + "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", + "requires": { + "@tootallnate/once": "2", + "agent-base": "6", + "debug": "4" + }, + "dependencies": { + "agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "requires": { + "debug": "4" + } + } + } + }, + "https-proxy-agent": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz", + "integrity": "sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg==", + "requires": { + "agent-base": "^7.0.2", + "debug": "4" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==" + }, + "json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "requires": { + "bignumber.js": "^9.0.0" + } + }, + "jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "requires": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "requires": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, + "mime": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", + "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==" + }, "mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", @@ -931,6 +1801,11 @@ "brace-expansion": "^1.1.7" } }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, "node-fetch": { "version": "2.6.12", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.12.tgz", @@ -939,6 +1814,22 @@ "whatwg-url": "^5.0.0" } }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "requires": { + "wrappy": "1" + } + }, + "p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "requires": { + "yocto-queue": "^0.1.0" + } + }, "process": { "version": "0.11.10", "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", @@ -949,6 +1840,36 @@ "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, + "readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==" + }, + "retry-request": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-7.0.2.tgz", + "integrity": "sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==", + "requires": { + "@types/request": "^2.48.8", + "extend": "^3.0.2", + "teeny-request": "^9.0.0" + } + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, "sax": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", @@ -959,6 +1880,73 @@ "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==" }, + "stream-events": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", + "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", + "requires": { + "stubs": "^3.0.0" + } + }, + "stream-shift": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", + "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==" + }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "requires": { + "safe-buffer": "~5.2.0" + } + }, + "strnum": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" + }, + "stubs": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", + "integrity": "sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==" + }, + "teeny-request": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-9.0.0.tgz", + "integrity": "sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==", + "requires": { + "http-proxy-agent": "^5.0.0", + "https-proxy-agent": "^5.0.0", + "node-fetch": "^2.6.9", + "stream-events": "^1.0.5", + "uuid": "^9.0.0" + }, + "dependencies": { + "agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "requires": { + "debug": "4" + } + }, + "https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "requires": { + "agent-base": "6", + "debug": "4" + } + }, + "uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==" + } + } + }, "tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", @@ -980,6 +1968,11 @@ "integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==", "dev": true }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, "uuid": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", @@ -999,6 +1992,11 @@ "webidl-conversions": "^3.0.0" } }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + }, "xml2js": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz", @@ -1012,6 +2010,11 @@ "version": "11.0.1", "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==" + }, + "yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==" } } } diff --git a/packages/warp-cache/package.json b/packages/warp-cache/package.json index c00661c6..f9da2e19 100644 --- a/packages/warp-cache/package.json +++ b/packages/warp-cache/package.json @@ -1,6 +1,6 @@ { "name": "github-actions.warp-cache", - "version": "0.2.0", + "version": "0.3.0", "preview": true, "description": "Github action to use WarpBuild's in-house cache offering", "keywords": [ @@ -9,7 +9,7 @@ "cache", "warpbuild" ], - "homepage": "https://github.com/actions/toolkit/tree/main/packages/cache", + "homepage": "https://github.com/WarpBuilds/toolkit/tree/main/packages/warp-cache", "license": "MIT", "main": "lib/cache.js", "types": "lib/cache.d.ts", @@ -26,7 +26,7 @@ }, "repository": { "type": "git", - "url": "git+https://github.com/actions/toolkit.git", + "url": "git+https://github.com/WarpBuilds/toolkit.git", "directory": "packages/cache" }, "scripts": { @@ -35,7 +35,7 @@ "tsc": "tsc" }, "bugs": { - "url": "https://github.com/actions/toolkit/issues" + "url": "https://github.com/WarpBuilds/toolkit/issues" }, "dependencies": { "@actions/core": "^1.10.0", @@ -46,6 +46,7 @@ "@azure/abort-controller": "^1.1.0", "@azure/ms-rest-js": "^2.6.0", "@azure/storage-blob": "^12.13.0", + "@google-cloud/storage": "^7.9.0", "axios": "^1.6.2", "semver": "^6.3.1", "uuid": "^3.3.3" diff --git a/packages/warp-cache/src/cache.ts b/packages/warp-cache/src/cache.ts index 51c048d1..94d2bc8c 100644 --- a/packages/warp-cache/src/cache.ts +++ b/packages/warp-cache/src/cache.ts @@ -2,7 +2,12 @@ import * as core from '@actions/core' import * as path from 'path' import * as utils from './internal/cacheUtils' import * as cacheHttpClient from './internal/cacheHttpClient' -import {createTar, extractTar, listTar} from './internal/tar' +import { + createTar, + extractStreamingTar, + extractTar, + listTar +} from './internal/tar' import {DownloadOptions, getUploadOptions} from './options' export class ValidationError extends Error { @@ -50,7 +55,7 @@ function checkKey(key: string): void { */ export function isFeatureAvailable(): boolean { - return !!process.env['WARP_CACHE_URL'] + return !!process.env['WARPBUILD_CACHE_URL'] } /** @@ -95,14 +100,15 @@ export async function restoreCache( compressionMethod, enableCrossOsArchive }) - if (!cacheEntry?.pre_signed_url) { - // Cache not found + + if (!cacheEntry) { + // Internal Error return undefined } if (options?.lookupOnly) { core.info('Lookup only - skipping download') - return cacheEntry.cache_key + return cacheEntry?.cache_key } archivePath = path.join( @@ -111,30 +117,60 @@ export async function restoreCache( ) core.debug(`Archive Path: ${archivePath}`) - // Download the cache from the cache entry - await cacheHttpClient.downloadCache(cacheEntry.pre_signed_url, archivePath) + switch (cacheEntry.provider) { + case 's3': { + if (!cacheEntry.pre_signed_url) { + // Cache not found + return undefined + } - if (core.isDebug()) { - await listTar(archivePath, compressionMethod) + await cacheHttpClient.downloadCache( + cacheEntry.pre_signed_url, + archivePath + ) + + if (core.isDebug()) { + await listTar(archivePath, compressionMethod) + } + + const archiveFileSize = utils.getArchiveFileSizeInBytes(archivePath) + core.info( + `Cache Size: ~${Math.round( + archiveFileSize / (1024 * 1024) + )} MB (${archiveFileSize} B)` + ) + + await extractTar(archivePath, compressionMethod) + core.info('Cache restored successfully') + break + } + + case 'gcs': { + // For GCS, we do a streaming download which means that we extract the archive while we are downloading it. + const archiveLocation = cacheEntry.archive_location ?? '' + + const readStream = cacheHttpClient.downloadCacheStreaming( + 'gcs', + archiveLocation + ) + + if (!readStream) { + return undefined + } + + await extractStreamingTar(readStream, archivePath, compressionMethod) + core.info('Cache restored successfully') + break + } } - const archiveFileSize = utils.getArchiveFileSizeInBytes(archivePath) - core.info( - `Cache Size: ~${Math.round( - archiveFileSize / (1024 * 1024) - )} MB (${archiveFileSize} B)` - ) - - await extractTar(archivePath, compressionMethod) - core.info('Cache restored successfully') - return cacheEntry.cache_key } catch (error) { const typedError = error as Error if (typedError.name === ValidationError.name) { throw error } else { - // Supress all non-validation cache related errors because caching should be optional + // Suppress all non-validation cache related errors because caching should be optional core.warning(`Failed to restore: ${(error as Error).message}`) } } finally { diff --git a/packages/warp-cache/src/internal/cacheHttpClient.ts b/packages/warp-cache/src/internal/cacheHttpClient.ts index 4f9fd115..31122df7 100644 --- a/packages/warp-cache/src/internal/cacheHttpClient.ts +++ b/packages/warp-cache/src/internal/cacheHttpClient.ts @@ -21,7 +21,10 @@ import { InternalS3CompletedPart, CommitCacheResponse } from './contracts' -import {downloadCacheMultiConnection} from './downloadUtils' +import { + downloadCacheMultiConnection, + downloadCacheStreamingGCP +} from './downloadUtils' import {isSuccessStatusCode, retryTypedResponse} from './requestUtils' import axios, {AxiosError} from 'axios' @@ -29,12 +32,14 @@ const versionSalt = '1.0' function getCacheApiUrl(resource: string): string { const baseUrl: string = - process.env['WARP_CACHE_URL'] ?? 'https://cache.warpbuild.com' + process.env['WARPBUILD_CACHE_URL'] ?? 'https://cache.warpbuild.com' if (!baseUrl) { throw new Error('Cache Service Url not found, unable to restore cache.') } - const url = `${baseUrl}/v1/${resource}` + const provider: string = process.env['STORAGE_PROVIDER'] ?? 'gcs' + + const url = `${baseUrl}/v1/${resource}/${provider}` core.debug(`Resource Url: ${url}`) return url } @@ -54,11 +59,11 @@ function getRequestOptions(): RequestOptions { } function createHttpClient(): HttpClient { - const token = process.env['WARP_RUNNER_VERIFICATION_TOKEN'] ?? '' + const token = process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] ?? '' const bearerCredentialHandler = new BearerCredentialHandler(token) return new HttpClient( - 'actions/cache', + 'warp/cache', [bearerCredentialHandler], getRequestOptions() ) @@ -106,7 +111,7 @@ export async function getCacheEntry( const response = await retryTypedResponse('getCacheEntry', async () => httpClient.getJson(getCacheApiUrl(resource)) ) - // Cache not found + if (response.statusCode === 204) { // List cache for primary key only if cache miss occurs if (core.isDebug()) { @@ -119,9 +124,9 @@ export async function getCacheEntry( } const cacheResult = response.result - const cacheDownloadUrl = cacheResult?.pre_signed_url + const cacheDownloadUrl = cacheResult?.archive_location if (!cacheDownloadUrl) { - // Cache achiveLocation not found. This should never happen, and hence bail out. + // Cache archiveLocation not found. This should never happen, and hence bail out. throw new Error('Cache not found.') } core.setSecret(cacheDownloadUrl) @@ -163,7 +168,20 @@ export async function downloadCache( await downloadCacheMultiConnection(archiveLocation, archivePath, 8) } -// Reserve Cache +export function downloadCacheStreaming( + provider: string, + archiveLocation: string +): NodeJS.ReadableStream | undefined { + switch (provider) { + case 's3': + return undefined + case 'gcs': + return downloadCacheStreamingGCP(archiveLocation) + default: + return undefined + } +} + export async function reserveCache( cacheKey: string, numberOfChunks: number, @@ -240,7 +258,6 @@ async function uploadFileToS3( preSignedURLs: string[], archivePath: string ): Promise { - // Upload Chunks const fileSize = utils.getArchiveFileSizeInBytes(archivePath) const numberOfChunks = preSignedURLs.length @@ -334,7 +351,6 @@ export async function saveCache( // Sort parts in ascending order by partNumber completedParts.sort((a, b) => a.PartNumber - b.PartNumber) - // Commit Cache core.debug('Committing cache') const cacheSize = utils.getArchiveFileSizeInBytes(archivePath) core.info( diff --git a/packages/warp-cache/src/internal/contracts.d.ts b/packages/warp-cache/src/internal/contracts.d.ts index 39058e82..c29f1161 100644 --- a/packages/warp-cache/src/internal/contracts.d.ts +++ b/packages/warp-cache/src/internal/contracts.d.ts @@ -7,7 +7,10 @@ export interface ITypedResponseWithError extends TypedResponse { } export interface ArtifactCacheEntry { + provider: string + auth_method: string cache_key?: string + archive_location?: string pre_signed_url?: string cache_version?: string } diff --git a/packages/warp-cache/src/internal/downloadUtils.ts b/packages/warp-cache/src/internal/downloadUtils.ts index 269a1ff8..aa5ab9b2 100644 --- a/packages/warp-cache/src/internal/downloadUtils.ts +++ b/packages/warp-cache/src/internal/downloadUtils.ts @@ -13,6 +13,7 @@ import {DownloadOptions} from '../options' import {retryHttpClientResponse} from './requestUtils' import {AbortController} from '@azure/abort-controller' +import {Storage} from '@google-cloud/storage' /** * Pipes the body of a HTTP response to a stream @@ -292,3 +293,47 @@ export async function downloadCacheMultiConnection( await fileHandle?.close() } } + +/** + * Download the cache to a provider writable stream using GCloud SDK + * + * @param archiveLocation the URL for the cache + */ +export function downloadCacheStreamingGCP( + archiveLocation: string +): NodeJS.ReadableStream | undefined { + try { + const storage = new Storage({ + token: process.env['GCP_ACCESS_TOKEN'] + }) + + // The archiveLocation for GCP will be in the format of gs:/// + const bucketName = archiveLocation.split('/')[2] + if (!bucketName || bucketName.length < 2) { + throw new Error( + `Invalid GCS URL: ${archiveLocation}. Should be in the format gs:///` + ) + } + + const fileName = archiveLocation.split('/').slice(3).join('/') + if (!fileName || fileName.length < 1) { + throw new Error( + `Invalid GCS URL: ${archiveLocation}. Should be in the format gs:///` + ) + } + + storage + .bucket(bucketName) + .file(fileName) + .getMetadata() + .then(data => { + core.info(`File size: ${data[0]?.size} bytes`) + }) + + return storage.bucket(bucketName).file(fileName).createReadStream() + } catch (error) { + core.debug(`Failed to download cache: ${error}`) + core.error(`Failed to download cache.`) + throw error + } +} diff --git a/packages/warp-cache/src/internal/tar.ts b/packages/warp-cache/src/internal/tar.ts index adf61069..29ebebf9 100644 --- a/packages/warp-cache/src/internal/tar.ts +++ b/packages/warp-cache/src/internal/tar.ts @@ -11,9 +11,17 @@ import { TarFilename, ManifestFilename } from './constants' +import {ChildProcessWithoutNullStreams, spawn} from 'child_process' const IS_WINDOWS = process.platform === 'win32' +enum TAR_MODE { + CREATE = 'create', + EXTRACT = 'extract', + EXTRACT_STREAM = 'extractStream', + LIST = 'list' +} + // Returns tar path and type: BSD or GNU async function getTarPath(): Promise { switch (process.platform) { @@ -54,7 +62,7 @@ async function getTarPath(): Promise { async function getTarArgs( tarPath: ArchiveTool, compressionMethod: CompressionMethod, - type: string, + type: TAR_MODE, archivePath = '' ): Promise { const args = [`"${tarPath.path}"`] @@ -69,7 +77,7 @@ async function getTarArgs( // Method specific args switch (type) { - case 'create': + case TAR_MODE.CREATE: args.push( '--posix', '-cf', @@ -87,7 +95,7 @@ async function getTarArgs( ManifestFilename ) break - case 'extract': + case TAR_MODE.EXTRACT: args.push( '-xf', BSD_TAR_ZSTD @@ -98,7 +106,16 @@ async function getTarArgs( workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/') ) break - case 'list': + case TAR_MODE.EXTRACT_STREAM: + args.push( + '-xf', + '-', + '-P', + '-C', + workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/') + ) + break + case TAR_MODE.LIST: args.push( '-tf', BSD_TAR_ZSTD @@ -127,7 +144,7 @@ async function getTarArgs( // Returns commands to run tar and compression program async function getCommands( compressionMethod: CompressionMethod, - type: string, + type: TAR_MODE, archivePath = '' ): Promise { let args @@ -139,8 +156,9 @@ async function getCommands( type, archivePath ) + const compressionArgs = - type !== 'create' + type !== TAR_MODE.CREATE ? await getDecompressionProgram(tarPath, compressionMethod, archivePath) : await getCompressionProgram(tarPath, compressionMethod) const BSD_TAR_ZSTD = @@ -148,7 +166,7 @@ async function getCommands( compressionMethod !== CompressionMethod.Gzip && IS_WINDOWS - if (BSD_TAR_ZSTD && type !== 'create') { + if (BSD_TAR_ZSTD && type !== TAR_MODE.CREATE) { args = [[...compressionArgs].join(' '), [...tarArgs].join(' ')] } else { args = [[...tarArgs].join(' '), [...compressionArgs].join(' ')] @@ -161,6 +179,42 @@ async function getCommands( return [args.join(' ')] } +/* + * Returns command pipes to stream data to tar and compression program. + * Only supports tar and zstd at the moment + * @returns Array of ChildProcessWithoutNullStreams. Pipe to the processes in the order they are returned + */ +async function getCommandPipes( + compressionMethod: CompressionMethod, + type: TAR_MODE, + archivePath = '' +): Promise { + const spawnedProcesses: ChildProcessWithoutNullStreams[] = [] + + const tarPath = await getTarPath() + const tarArgs = await getTarArgs( + tarPath, + compressionMethod, + type, + archivePath + ) + // Remove tar executable from tarArgs + tarArgs.shift() + + let zstdInfo = + type !== TAR_MODE.CREATE + ? await getDecompressionProgramStream(tarPath, compressionMethod) + : await getCompressionProgramStream(tarPath, compressionMethod) + + const zstdProcess = spawn(zstdInfo.command, zstdInfo.args) + spawnedProcesses.push(zstdProcess) + + const tarProcess = spawn(tarPath.path, tarArgs) + spawnedProcesses.push(tarProcess) + + return spawnedProcesses +} + function getWorkingDirectory(): string { return process.env['GITHUB_WORKSPACE'] ?? process.cwd() } @@ -204,6 +258,39 @@ async function getDecompressionProgram( } } +// Alternative to getDecompressionProgram which returns zstd that command that can be piped into +async function getDecompressionProgramStream( + tarPath: ArchiveTool, + compressionMethod: CompressionMethod +): Promise<{command: string; args: string[]}> { + const BSD_TAR_ZSTD = + tarPath.type === ArchiveToolType.BSD && + compressionMethod !== CompressionMethod.Gzip && + IS_WINDOWS + + switch (compressionMethod) { + case CompressionMethod.Zstd: + return BSD_TAR_ZSTD + ? {command: 'zstd', args: ['-d', '--long=30', '--force', '--stdout']} + : { + command: IS_WINDOWS ? 'zstd' : 'unzstd', + args: IS_WINDOWS + ? ['-d', '--long=30', '--stdout', '-T0'] + : ['--long=30', '--stdout', '-T0'] + } + case CompressionMethod.ZstdWithoutLong: + return BSD_TAR_ZSTD + ? {command: 'zstd', args: ['-d', '--force', '--stdout']} + : { + command: IS_WINDOWS ? 'zstd' : 'unzstd', + args: ['-d', '--stdout', '-T0'] + } + default: + // Assuming gzip is the default method if none specified + return {command: 'gzip', args: ['-d']} + } +} + // Used for creating the archive // -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. // zstdmt is equivalent to 'zstd -T0' @@ -244,6 +331,44 @@ async function getCompressionProgram( } } +async function getCompressionProgramStream( + tarPath: ArchiveTool, + compressionMethod: CompressionMethod +): Promise<{command: string; args: string[]}> { + const BSD_TAR_ZSTD = + tarPath.type === ArchiveToolType.BSD && + compressionMethod !== CompressionMethod.Gzip && + IS_WINDOWS + + switch (compressionMethod) { + case CompressionMethod.Zstd: + return BSD_TAR_ZSTD + ? { + command: 'zstd', + args: ['-T0', '--long=30', '--force', '--stdout'] + } + : { + command: IS_WINDOWS ? 'zstd' : 'zstdmt', + args: IS_WINDOWS + ? ['-T0', '--long=30', '--stdout', '-T0'] + : ['--long=30', '--stdout', '-T0'] + } + case CompressionMethod.ZstdWithoutLong: + return BSD_TAR_ZSTD + ? { + command: 'zstd', + args: ['-T0', '--force', '--stdout'] + } + : { + command: IS_WINDOWS ? 'zstd' : 'zstdmt', + args: ['-T0', '--stdout'] + } + default: + // Assuming gzip is the default method if none specified + return {command: 'gzip', args: []} + } +} + // Executes all commands as separate processes async function execCommands(commands: string[], cwd?: string): Promise { for (const command of commands) { @@ -265,11 +390,14 @@ export async function listTar( archivePath: string, compressionMethod: CompressionMethod ): Promise { - const commands = await getCommands(compressionMethod, 'list', archivePath) + const commands = await getCommands( + compressionMethod, + TAR_MODE.LIST, + archivePath + ) await execCommands(commands) } -// Extract a tar export async function extractTar( archivePath: string, compressionMethod: CompressionMethod @@ -277,10 +405,67 @@ export async function extractTar( // Create directory to extract tar into const workingDirectory = getWorkingDirectory() await io.mkdirP(workingDirectory) - const commands = await getCommands(compressionMethod, 'extract', archivePath) + const commands = await getCommands( + compressionMethod, + TAR_MODE.EXTRACT, + archivePath + ) await execCommands(commands) } +// Supports only archives created using tar and zstd +export async function extractStreamingTar( + stream: NodeJS.ReadableStream, + archivePath: string, + compressionMethod: CompressionMethod +): Promise { + const workingDirectory = getWorkingDirectory() + await io.mkdirP(workingDirectory) + const commandPipes = await getCommandPipes( + compressionMethod, + TAR_MODE.EXTRACT_STREAM, + archivePath + ) + + if (commandPipes.length < 2) { + throw new Error( + 'At least two processes should be present as the archive is compressed at least twice.' + ) + } + + return new Promise((resolve, reject) => { + stream.pipe(commandPipes[0].stdin) + for (let i = 0; i < commandPipes.length - 1; i++) { + commandPipes[i].stdout.pipe(commandPipes[i + 1].stdin) + + commandPipes[i].stderr.on('data', data => { + reject( + new Error(`Error in ${commandPipes[i].spawnfile}: ${data.toString()}`) + ) + }) + + commandPipes[i].on('error', error => { + reject( + new Error(`Error in ${commandPipes[i].spawnfile}: ${error.message}`) + ) + }) + } + + const lastCommand = commandPipes[commandPipes.length - 1] + lastCommand.stderr.on('data', data => { + console.error(`Error in ${lastCommand.spawnfile}:`, data.toString()) + reject(new Error(`Error in ${lastCommand.spawnfile}: ${data.toString()}`)) + }) + lastCommand.on('close', code => { + if (code === 0) { + resolve() + } else { + reject(new Error(`Last command exited with code ${code}`)) + } + }) + }) +} + // Create a tar export async function createTar( archiveFolder: string, @@ -292,6 +477,6 @@ export async function createTar( path.join(archiveFolder, ManifestFilename), sourceDirectories.join('\n') ) - const commands = await getCommands(compressionMethod, 'create') + const commands = await getCommands(compressionMethod, TAR_MODE.CREATE) await execCommands(commands, archiveFolder) } diff --git a/packages/warp-cache/src/test.ts b/packages/warp-cache/src/test.ts index 9e4f3532..973ca0eb 100644 --- a/packages/warp-cache/src/test.ts +++ b/packages/warp-cache/src/test.ts @@ -1,6 +1,12 @@ +import {exec, spawn} from 'child_process' import {deleteCache, restoreCache, saveCache} from './cache' +import {downloadCacheStreamingGCP} from './internal/downloadUtils' -process.env['WARP_CACHE_URL'] = 'http://localhost:8002' +import fs, {write} from 'fs' +import {extractStreamingTar} from './internal/tar' +import {CompressionMethod} from './internal/constants' + +process.env['WARPBUILD_CACHE_URL'] = 'http://localhost:8002' process.env['RUNNER_TEMP'] = '/Users/prajjwal/Repos/warpbuild/playground/tmp_fs' process.env['NODE_DEBUG'] = 'http' @@ -19,3 +25,22 @@ process.env['NODE_DEBUG'] = 'http' // ) // deleteCache(['test-fs-local-key']) + +process.env['GCP_ACCESS_TOKEN'] = + 'ya29.c.c0AY_VpZgcQopWxkSf9wIIo9NED0YFh3VIgZ1wx1ulvSCrq5iTiZWbrRGPej2vA835U2HkNdrLwaVKFLeL57v1s-guzSvihNnHMMJ4wUPJHZPQd-CJ90i6F0NYcjQuv7SC2EBkaKciM-Act0IDygPwzwwixCe-4iCxcUv3YUysZcee9Qknxq5UBPfGjqQArVKifC2fScJ7HnBmbbSc8t1mDp9mLiIpax9V31anOQ-4QK1kqSgi4gh0m-Cd7v24S7Kfc5IEcQLrVyI62W4Y4HywRJ2V_qBx3ZKFMmO1lV5Tl3wHX40XyD1J2Cc6kXbF4LHHPcMnRf85ylaXaUGMwDNlkDPFHRJmOkWnZF8-v_Y4868-Mmektdl8khWvCQwGSLHo_jCKehCJZl1qK1gzNfie7Rgm9qbooMAEg1KkPPiDBmMY_WUsBo1-a0vuHrE90IhtvKI_TNTeH-pUDjSFMsbgrhnbGu5oN6DXk--WyjHy9slW6r8TDjB8UjPE2uiaGbYrQZsRPoaKVAxVylc9tFONyPwJ10MUerPq3ESq49QUASdasuYCef0CZ_R3kJyIMQe7p6WBfOZ0L11ZTz_tnFn1Oa8JGHvYl1xvx79EbHjo4mvyr5WTAXa42g-gCnPnJFLaN649DRZbdRzbbc3-bQbqFuictuoSQmOjhrqW6_0_44wVhlga9Ok9kZ4_lx6Oqvq9SiI6IxIJSBVnXet3MgzoRdJur8Ws766sinJ_iFkZdsQdj2IQ_hj74vh61v1i84xIZY-bp-IrvQQf_vZm6bbBZXxaXhiVphpij7nY5Rz3qS2d0e3byc1iUW63jXlY1iIhlvsd1i2Zd4YVyQrfgSy_zpuXJOqhS1MwBrkddb4F-r3wQtRJ1ttmbpSJOpeYzewzSeVopk8pmOaUSd0rS4qQkY1UdhQoavyn54VMj5U8BiOkjo-wV2MUXl0FlVF7u3-c3vUhlZ1JrMj6xiWFXys_QBMtU55jMe31UV-saSFxM7f1-xk1_2xoou8' + +const readStream = downloadCacheStreamingGCP( + 'gs://cache-bench-test/custom_modules.tar.zst' +) + +extractStreamingTar( + readStream!, + '/tmp/custom_modules', + CompressionMethod.ZstdWithoutLong +) + .then(() => { + console.log('done') + }) + .catch(err => { + console.log(err) + }) From 917853e23b11a6feb84747c4792805b2a8b7d63d Mon Sep 17 00:00:00 2001 From: Prajjwal Date: Thu, 11 Apr 2024 11:12:08 +0530 Subject: [PATCH 02/11] feat: adds streaming download and upload for gcs provider --- packages/warp-cache/__tests__/tar.test.ts | 2 +- packages/warp-cache/src/cache.ts | 97 ++-- .../src/internal/cacheHttpClient.ts | 297 +++++------ .../warp-cache/src/internal/cacheUtils.ts | 25 + .../warp-cache/src/internal/contracts.d.ts | 41 -- .../warp-cache/src/internal/downloadUtils.ts | 24 +- packages/warp-cache/src/internal/tar.ts | 5 +- .../warp-cache/src/internal/uploadUtils.ts | 131 +++++ .../.openapi-generator-ignore | 23 + .../warpcache-ts-sdk/.openapi-generator/FILES | 31 ++ .../.openapi-generator/VERSION | 1 + .../src/internal/warpcache-ts-sdk/api.ts | 18 + .../warpcache-ts-sdk/api/default-api.ts | 493 ++++++++++++++++++ .../src/internal/warpcache-ts-sdk/base.ts | 86 +++ .../src/internal/warpcache-ts-sdk/common.ts | 150 ++++++ .../warpcache-ts-sdk/configuration.ts | 110 ++++ .../src/internal/warpcache-ts-sdk/index.ts | 21 + .../models/commons-commit-cache-request.ts | 69 +++ .../models/commons-commit-cache-response.ts | 48 ++ .../models/commons-delete-cache-response.ts | 45 ++ .../commons-gcscommit-cache-response.ts | 57 ++ .../models/commons-gcsget-cache-reponse.ts | 63 +++ .../commons-gcsreserve-cache-response.ts | 57 ++ .../models/commons-get-cache-response.ts | 48 ++ .../models/commons-reserve-cache-request.ts | 42 ++ .../models/commons-reserve-cache-response.ts | 48 ++ .../commons-s3-commit-cache-response.ts | 36 ++ .../commons-s3-delete-cache-response.ts | 36 ++ .../models/commons-s3-get-cache-response.ts | 42 ++ .../commons-s3-reserve-cache-response.ts | 42 ++ .../models/commons-short-lived-token.ts | 36 ++ .../internal/warpcache-ts-sdk/models/index.ts | 16 + .../models/types-completed-part.ts | 60 +++ .../models/warp-build-apierror.ts | 54 ++ .../src/internal/warpcache-ts-sdk/schema.ts | 20 + .../schemas/$WarpBuildAPIError.ts | 22 + .../schemas/$commons_CommitCacheRequest.ts | 36 ++ .../schemas/$commons_CommitCacheResponse.ts | 16 + .../schemas/$commons_DeleteCacheResponse.ts | 16 + .../$commons_GCSCommitCacheResponse.ts | 23 + .../$commons_GCSDeleteCacheResponse.ts | 7 + .../schemas/$commons_GCSGetCacheReponse.ts | 25 + .../$commons_GCSReserveCacheResponse.ts | 23 + .../schemas/$commons_GetCacheResponse.ts | 16 + .../schemas/$commons_ReserveCacheRequest.ts | 17 + .../schemas/$commons_ReserveCacheResponse.ts | 16 + .../schemas/$commons_S3CommitCacheResponse.ts | 15 + .../schemas/$commons_S3DeleteCacheResponse.ts | 15 + .../schemas/$commons_S3GetCacheResponse.ts | 16 + .../$commons_S3ReserveCacheResponse.ts | 19 + .../schemas/$commons_ShortLivedToken.ts | 13 + .../schemas/$types_CompletedPart.ts | 25 + .../warpcache-ts-sdk/tsconfig.esm.json | 7 + packages/warp-cache/src/test.ts | 51 +- 54 files changed, 2457 insertions(+), 295 deletions(-) create mode 100644 packages/warp-cache/src/internal/uploadUtils.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/.openapi-generator-ignore create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/.openapi-generator/FILES create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/.openapi-generator/VERSION create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/api.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/api/default-api.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/base.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/common.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/configuration.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/index.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-commit-cache-request.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-commit-cache-response.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-delete-cache-response.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-gcscommit-cache-response.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-gcsget-cache-reponse.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-gcsreserve-cache-response.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-get-cache-response.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-reserve-cache-request.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-reserve-cache-response.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-s3-commit-cache-response.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-s3-delete-cache-response.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-s3-get-cache-response.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-s3-reserve-cache-response.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-short-lived-token.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/models/index.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/models/types-completed-part.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/models/warp-build-apierror.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/schema.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$WarpBuildAPIError.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CommitCacheRequest.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CommitCacheResponse.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_DeleteCacheResponse.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GCSCommitCacheResponse.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GCSDeleteCacheResponse.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GCSGetCacheReponse.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GCSReserveCacheResponse.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GetCacheResponse.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_ReserveCacheRequest.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_ReserveCacheResponse.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_S3CommitCacheResponse.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_S3DeleteCacheResponse.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_S3GetCacheResponse.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_S3ReserveCacheResponse.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_ShortLivedToken.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$types_CompletedPart.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/tsconfig.esm.json diff --git a/packages/warp-cache/__tests__/tar.test.ts b/packages/warp-cache/__tests__/tar.test.ts index 4145d9a9..70a62562 100644 --- a/packages/warp-cache/__tests__/tar.test.ts +++ b/packages/warp-cache/__tests__/tar.test.ts @@ -12,7 +12,7 @@ import { import * as tar from '../src/internal/tar' import * as utils from '../src/internal/cacheUtils' // eslint-disable-next-line @typescript-eslint/no-require-imports -import fs = require('fs') +import fs from 'fs' jest.mock('@actions/exec') jest.mock('@actions/io') diff --git a/packages/warp-cache/src/cache.ts b/packages/warp-cache/src/cache.ts index 94d2bc8c..3a537bf7 100644 --- a/packages/warp-cache/src/cache.ts +++ b/packages/warp-cache/src/cache.ts @@ -106,26 +106,29 @@ export async function restoreCache( return undefined } - if (options?.lookupOnly) { - core.info('Lookup only - skipping download') - return cacheEntry?.cache_key - } - archivePath = path.join( await utils.createTempDirectory(), utils.getCacheFileName(compressionMethod) ) core.debug(`Archive Path: ${archivePath}`) + let cacheKey: string = '' + switch (cacheEntry.provider) { case 's3': { - if (!cacheEntry.pre_signed_url) { - // Cache not found + if (!cacheEntry.s3?.pre_signed_url) { return undefined } + cacheKey = cacheEntry.s3.pre_signed_url + + if (options?.lookupOnly) { + core.info('Lookup only - skipping download') + return cacheKey + } + await cacheHttpClient.downloadCache( - cacheEntry.pre_signed_url, + cacheEntry.s3?.pre_signed_url, archivePath ) @@ -146,12 +149,23 @@ export async function restoreCache( } case 'gcs': { + if (!cacheEntry.gcs?.cache_key) { + return undefined + } + cacheKey = cacheEntry.gcs?.cache_key + + if (options?.lookupOnly) { + core.info('Lookup only - skipping download') + return cacheKey + } + // For GCS, we do a streaming download which means that we extract the archive while we are downloading it. - const archiveLocation = cacheEntry.archive_location ?? '' + const archiveLocation = `gs://${cacheEntry.gcs?.bucket_name}/${cacheEntry.gcs?.cache_key}` const readStream = cacheHttpClient.downloadCacheStreaming( 'gcs', - archiveLocation + archiveLocation, + cacheEntry?.gcs?.short_lived_token?.access_token ?? '' ) if (!readStream) { @@ -164,7 +178,7 @@ export async function restoreCache( } } - return cacheEntry.cache_key + return cacheKey } catch (error) { const typedError = error as Error if (typedError.name === ValidationError.name) { @@ -228,21 +242,20 @@ export async function saveCache( if (core.isDebug()) { await listTar(archivePath, compressionMethod) } - const fileSizeLimit = 20 * 1024 * 1024 * 1024 // 20GB per repo limit + const fileSizeLimit = 1000 * 1024 * 1024 * 1024 // 1000GB per repo limit const archiveFileSize = utils.getArchiveFileSizeInBytes(archivePath) core.debug(`File Size: ${archiveFileSize}`) - // For GHES, this check will take place in ReserveCache API with enterprise file size limit - if (archiveFileSize > fileSizeLimit && !utils.isGhes()) { + if (archiveFileSize > fileSizeLimit) { throw new Error( `Cache size of ~${Math.round( archiveFileSize / (1024 * 1024) - )} MB (${archiveFileSize} B) is over the 10GB limit, not saving cache.` + )} MB (${archiveFileSize} B) is over the 1000GB limit, not saving cache.` ) } core.debug('Reserving Cache') - // Calculate number of chunks required + // Calculate number of chunks required. This is only required if backend is S3 as Google Cloud SDK will do it for us const uploadOptions = getUploadOptions() const maxChunkSize = uploadOptions?.uploadChunkSize ?? 32 * 1024 * 1024 // Default 32MB const numberOfChunks = Math.floor(archiveFileSize / maxChunkSize) @@ -265,20 +278,46 @@ export async function saveCache( ) } - core.debug(`Saving Cache`) - cacheKey = await cacheHttpClient.saveCache( - key, - cacheHttpClient.getCacheVersion( - paths, - compressionMethod, - enableCrossOsArchive - ), - reserveCacheResponse?.result?.upload_id ?? '', - reserveCacheResponse?.result?.upload_key ?? '', - numberOfChunks, - reserveCacheResponse?.result?.pre_signed_urls ?? [], - archivePath + const cacheVersion = cacheHttpClient.getCacheVersion( + paths, + compressionMethod, + enableCrossOsArchive ) + + switch (reserveCacheResponse.result?.provider) { + case 's3': + core.debug(`Saving Cache to S3`) + cacheKey = await cacheHttpClient.saveCache( + 's3', + key, + cacheVersion, + archivePath, + reserveCacheResponse?.result?.s3?.upload_id ?? '', + reserveCacheResponse?.result?.s3?.upload_key ?? '', + numberOfChunks, + reserveCacheResponse?.result?.s3?.pre_signed_urls ?? [] + ) + break + + case 'gcs': + core.debug(`Saving Cache to GCS`) + cacheKey = await cacheHttpClient.saveCache( + 'gcs', + key, + cacheVersion, + archivePath, + // S3 Params are undefined for GCS + undefined, + undefined, + undefined, + undefined, + reserveCacheResponse?.result?.gcs?.short_lived_token?.access_token ?? + '', + reserveCacheResponse?.result?.gcs?.bucket_name ?? '', + reserveCacheResponse?.result?.gcs?.cache_key ?? '' + ) + break + } } catch (error) { const typedError = error as Error if (typedError.name === ValidationError.name) { diff --git a/packages/warp-cache/src/internal/cacheHttpClient.ts b/packages/warp-cache/src/internal/cacheHttpClient.ts index 31122df7..77140e58 100644 --- a/packages/warp-cache/src/internal/cacheHttpClient.ts +++ b/packages/warp-cache/src/internal/cacheHttpClient.ts @@ -6,27 +6,28 @@ import { TypedResponse } from '@actions/http-client/lib/interfaces' import * as crypto from 'crypto' -import * as fs from 'fs' import * as utils from './cacheUtils' import {CompressionMethod} from './constants' import { - ArtifactCacheEntry, InternalCacheOptions, - CommitCacheRequest, - ReserveCacheRequest, - ReserveCacheResponse, ITypedResponseWithError, - ArtifactCacheList, - InternalS3CompletedPart, - CommitCacheResponse + InternalS3CompletedPart } from './contracts' import { downloadCacheMultiConnection, downloadCacheStreamingGCP } from './downloadUtils' import {isSuccessStatusCode, retryTypedResponse} from './requestUtils' -import axios, {AxiosError} from 'axios' +import {Storage} from '@google-cloud/storage' +import { + CommonsCommitCacheRequest, + CommonsCommitCacheResponse, + CommonsGetCacheResponse, + CommonsReserveCacheRequest, + CommonsReserveCacheResponse +} from './warpcache-ts-sdk' +import {multiPartUploadToGCS, uploadFileToS3} from './uploadUtils' const versionSalt = '1.0' @@ -37,9 +38,7 @@ function getCacheApiUrl(resource: string): string { throw new Error('Cache Service Url not found, unable to restore cache.') } - const provider: string = process.env['STORAGE_PROVIDER'] ?? 'gcs' - - const url = `${baseUrl}/v1/${resource}/${provider}` + const url = `${baseUrl}/v1/${resource}` core.debug(`Resource Url: ${url}`) return url } @@ -97,7 +96,7 @@ export async function getCacheEntry( keys: string[], paths: string[], options?: InternalCacheOptions -): Promise { +): Promise { const httpClient = createHttpClient() const version = getCacheVersion( paths, @@ -109,14 +108,14 @@ export async function getCacheEntry( )}&version=${version}` const response = await retryTypedResponse('getCacheEntry', async () => - httpClient.getJson(getCacheApiUrl(resource)) + httpClient.getJson(getCacheApiUrl(resource)) ) if (response.statusCode === 204) { - // List cache for primary key only if cache miss occurs - if (core.isDebug()) { - await printCachesListForDiagnostics(keys[0], httpClient, version) - } + // TODO: List cache for primary key only if cache miss occurs + // if (core.isDebug()) { + // await printCachesListForDiagnostics(keys[0], httpClient, version) + // } return null } if (!isSuccessStatusCode(response.statusCode)) { @@ -124,18 +123,13 @@ export async function getCacheEntry( } const cacheResult = response.result - const cacheDownloadUrl = cacheResult?.archive_location - if (!cacheDownloadUrl) { - // Cache archiveLocation not found. This should never happen, and hence bail out. - throw new Error('Cache not found.') - } - core.setSecret(cacheDownloadUrl) core.debug(`Cache Result:`) core.debug(JSON.stringify(cacheResult)) return cacheResult } +/* async function printCachesListForDiagnostics( key: string, httpClient: HttpClient, @@ -160,6 +154,7 @@ async function printCachesListForDiagnostics( } } } +*/ export async function downloadCache( archiveLocation: string, @@ -170,13 +165,23 @@ export async function downloadCache( export function downloadCacheStreaming( provider: string, - archiveLocation: string + archiveLocation: string, + gcsToken?: string ): NodeJS.ReadableStream | undefined { switch (provider) { case 's3': return undefined - case 'gcs': - return downloadCacheStreamingGCP(archiveLocation) + case 'gcs': { + if (!gcsToken) { + throw new Error( + 'Unable to download cache from GCS. GCP token is not provided.' + ) + } + const storage = new Storage({ + token: gcsToken + }) + return downloadCacheStreamingGCP(storage, archiveLocation) + } default: return undefined } @@ -186,16 +191,16 @@ export async function reserveCache( cacheKey: string, numberOfChunks: number, options?: InternalCacheOptions -): Promise> { +): Promise> { const httpClient = createHttpClient() - const reserveCacheRequest: ReserveCacheRequest = { + const reserveCacheRequest: CommonsReserveCacheRequest = { cache_key: cacheKey, number_of_chunks: numberOfChunks, content_type: 'application/zstd' } const response = await retryTypedResponse('reserveCache', async () => - httpClient.postJson( + httpClient.postJson( getCacheApiUrl('cache/reserve'), reserveCacheRequest ) @@ -203,114 +208,20 @@ export async function reserveCache( return response } -function getContentRange(start: number, end: number): string { - // Format: `bytes start-end/filesize - // start and end are inclusive - // filesize can be * - // For a 200 byte chunk starting at byte 0: - // Content-Range: bytes 0-199/* - return `bytes ${start}-${end}/*` -} - -async function uploadChunk( - resourceUrl: string, - openStream: () => NodeJS.ReadableStream, - partNumber: number, - start: number, - end: number -): Promise { - core.debug( - `Uploading chunk of size ${ - end - start + 1 - } bytes at offset ${start} with content range: ${getContentRange( - start, - end - )}` - ) - - // Manually convert the readable stream to a buffer. S3 doesn't allow stream as input - const chunks = await utils.streamToBuffer(openStream()) - - try { - // HACK: Using axios here as S3 API doesn't allow readable stream as input and Github's HTTP client is not able to send buffer as body - const response = await axios.request({ - method: 'PUT', - url: resourceUrl, - headers: { - 'Content-Type': 'application/octet-stream' - }, - data: chunks - }) - return { - ETag: response.headers.etag ?? '', - PartNumber: partNumber - } - } catch (error) { - throw new Error( - `Cache service responded with ${ - (error as AxiosError).status - } during upload chunk.` - ) - } -} - -async function uploadFileToS3( - preSignedURLs: string[], - archivePath: string -): Promise { - const fileSize = utils.getArchiveFileSizeInBytes(archivePath) - const numberOfChunks = preSignedURLs.length - - const fd = fs.openSync(archivePath, 'r') - - core.debug('Awaiting all uploads') - let offset = 0 - - try { - const completedParts = await Promise.all( - preSignedURLs.map(async (presignedURL, index) => { - const chunkSize = Math.ceil(fileSize / numberOfChunks) - const start = offset - const end = offset + chunkSize - 1 - offset += chunkSize - - return await uploadChunk( - presignedURL, - () => - fs - .createReadStream(archivePath, { - fd, - start, - end, - autoClose: false - }) - .on('error', error => { - throw new Error( - `Cache upload failed because file read failed with ${error.message}` - ) - }), - index + 1, - start, - end - ) - }) - ) - - return completedParts - } finally { - fs.closeSync(fd) - } -} - async function commitCache( - httpClient: HttpClient, cacheKey: string, cacheVersion: string, - uploadKey: string, - uploadID: string, - parts: InternalS3CompletedPart[] -): Promise> { - const commitCacheRequest: CommitCacheRequest = { + uploadKey?: string, + uploadID?: string, + parts?: InternalS3CompletedPart[] +): Promise> { + const httpClient = createHttpClient() + + if (!parts) { + parts = [] + } + + const commitCacheRequest: CommonsCommitCacheRequest = { cache_key: cacheKey, cache_version: cacheVersion, upload_key: uploadKey, @@ -320,7 +231,7 @@ async function commitCache( vcs_type: 'github' } return await retryTypedResponse('commitCache', async () => - httpClient.postJson( + httpClient.postJson( getCacheApiUrl(`cache/commit`), commitCacheRequest ) @@ -328,43 +239,97 @@ async function commitCache( } export async function saveCache( + provider: string, cacheKey: string, cacheVersion: string, - uploadId: string, - uploadKey: string, - numberOfChunks: number, - preSignedURLs: string[], - archivePath: string + archivePath: string, + S3UploadId?: string, + S3UploadKey?: string, + S3NumberOfChunks?: number, + S3PreSignedURLs?: string[], + GCSAuthToken?: string, + GCSBucketName?: string, + GCSObjectName?: string ): Promise { - // Number of chunks should match the number of pre-signed URLs - if (numberOfChunks !== preSignedURLs.length) { - throw new Error( - `Number of chunks (${numberOfChunks}) should match the number of pre-signed URLs (${preSignedURLs.length}).` - ) - } - - const httpClient = createHttpClient() - - core.debug('Upload cache') - const completedParts = await uploadFileToS3(preSignedURLs, archivePath) - - // Sort parts in ascending order by partNumber - completedParts.sort((a, b) => a.PartNumber - b.PartNumber) - - core.debug('Committing cache') const cacheSize = utils.getArchiveFileSizeInBytes(archivePath) core.info( `Cache Size: ~${Math.round(cacheSize / (1024 * 1024))} MB (${cacheSize} B)` ) - const commitCacheResponse = await commitCache( - httpClient, - cacheKey, - cacheVersion, - uploadKey, - uploadId, - completedParts - ) + let commitCacheResponse: TypedResponse = { + headers: {}, + statusCode: 0, + result: null + } + + let cacheKeyResponse = '' + + switch (provider) { + case 's3': { + if ( + !S3NumberOfChunks || + !S3PreSignedURLs || + !S3UploadId || + !S3UploadKey + ) { + throw new Error( + 'Unable to upload cache to S3. One of the following required parameters is missing: numberOfChunks, preSignedURLs, uploadId, uploadKey.' + ) + } + + // Number of chunks should match the number of pre-signed URLs + if (S3NumberOfChunks !== S3PreSignedURLs.length) { + throw new Error( + `Number of chunks (${S3NumberOfChunks}) should match the number of pre-signed URLs (${S3PreSignedURLs.length}).` + ) + } + + core.debug('Uploading cache') + const completedParts = await uploadFileToS3(S3PreSignedURLs, archivePath) + + // Sort parts in ascending order by partNumber + completedParts.sort((a, b) => a.PartNumber - b.PartNumber) + + core.debug('Committing cache') + commitCacheResponse = await commitCache( + cacheKey, + cacheVersion, + S3UploadKey, + S3UploadId, + completedParts + ) + + cacheKeyResponse = commitCacheResponse.result?.s3?.cache_key ?? '' + + break + } + + case 'gcs': { + if (!GCSBucketName || !GCSObjectName || !GCSAuthToken) { + throw new Error( + 'Unable to upload cache to GCS. One of the following required parameters is missing: GCSBucketName, GCSObjectName, GCSAuthToken.' + ) + } + + core.debug('Uploading cache') + const storage = new Storage({ + token: GCSAuthToken + }) + await multiPartUploadToGCS( + storage, + archivePath, + GCSBucketName, + GCSObjectName + ) + + core.debug('Committing cache') + commitCacheResponse = await commitCache(cacheKey, cacheVersion) + + cacheKeyResponse = commitCacheResponse.result?.gcs?.cache_key ?? '' + break + } + } + if (!isSuccessStatusCode(commitCacheResponse.statusCode)) { throw new Error( `Cache service responded with ${commitCacheResponse.statusCode} during commit cache.` @@ -372,7 +337,7 @@ export async function saveCache( } core.info('Cache saved successfully') - return commitCacheResponse.result?.cache_key ?? '' + return cacheKeyResponse } export async function deleteCache(keys: string[]) { diff --git a/packages/warp-cache/src/internal/cacheUtils.ts b/packages/warp-cache/src/internal/cacheUtils.ts index 8513d1c3..571c5391 100644 --- a/packages/warp-cache/src/internal/cacheUtils.ts +++ b/packages/warp-cache/src/internal/cacheUtils.ts @@ -146,3 +146,28 @@ export function streamToBuffer(stream: NodeJS.ReadableStream): Promise { stream.on('end', () => resolve(Buffer.concat(buffer))) }) } + +/* + * Retrieve the bucket name and object name from the GCS URL + * @param gcsURL - The URL for the cache in the format gs:/// + */ +export function retrieveGCSBucketAndObjectName(gcsURL: string): { + bucketName: string + objectName: string +} { + const bucketName = gcsURL.split('/')[2] + if (!bucketName || bucketName.length < 2) { + throw new Error( + `Invalid GCS URL: ${gcsURL}. Should be in the format gs:///` + ) + } + + const objectName = gcsURL.split('/').slice(3).join('/') + if (!objectName || objectName.length < 1) { + throw new Error( + `Invalid GCS URL: ${gcsURL}. Should be in the format gs:///` + ) + } + + return {bucketName, objectName} +} diff --git a/packages/warp-cache/src/internal/contracts.d.ts b/packages/warp-cache/src/internal/contracts.d.ts index c29f1161..e4c3f035 100644 --- a/packages/warp-cache/src/internal/contracts.d.ts +++ b/packages/warp-cache/src/internal/contracts.d.ts @@ -6,47 +6,6 @@ export interface ITypedResponseWithError extends TypedResponse { error?: HttpClientError } -export interface ArtifactCacheEntry { - provider: string - auth_method: string - cache_key?: string - archive_location?: string - pre_signed_url?: string - cache_version?: string -} - -export interface ArtifactCacheList { - totalCount: number - artifactCaches?: ArtifactCacheEntry[] -} - -export interface CommitCacheRequest { - cache_key: string - cache_version: string - upload_key: string - upload_id: string - parts: InternalS3CompletedPart[] - os: string - vcs_type: string -} - -export interface CommitCacheResponse { - cache_key: string - cache_version: string -} - -export interface ReserveCacheRequest { - cache_key: string - content_type: string - number_of_chunks: number -} - -export interface ReserveCacheResponse { - pre_signed_urls: string[] - upload_key: string - upload_id: string -} - export interface InternalCacheOptions { compressionMethod?: CompressionMethod enableCrossOsArchive?: boolean diff --git a/packages/warp-cache/src/internal/downloadUtils.ts b/packages/warp-cache/src/internal/downloadUtils.ts index aa5ab9b2..9019f2d4 100644 --- a/packages/warp-cache/src/internal/downloadUtils.ts +++ b/packages/warp-cache/src/internal/downloadUtils.ts @@ -300,37 +300,23 @@ export async function downloadCacheMultiConnection( * @param archiveLocation the URL for the cache */ export function downloadCacheStreamingGCP( + storage: Storage, archiveLocation: string ): NodeJS.ReadableStream | undefined { try { - const storage = new Storage({ - token: process.env['GCP_ACCESS_TOKEN'] - }) - // The archiveLocation for GCP will be in the format of gs:/// - const bucketName = archiveLocation.split('/')[2] - if (!bucketName || bucketName.length < 2) { - throw new Error( - `Invalid GCS URL: ${archiveLocation}. Should be in the format gs:///` - ) - } - - const fileName = archiveLocation.split('/').slice(3).join('/') - if (!fileName || fileName.length < 1) { - throw new Error( - `Invalid GCS URL: ${archiveLocation}. Should be in the format gs:///` - ) - } + const {bucketName, objectName} = + utils.retrieveGCSBucketAndObjectName(archiveLocation) storage .bucket(bucketName) - .file(fileName) + .file(objectName) .getMetadata() .then(data => { core.info(`File size: ${data[0]?.size} bytes`) }) - return storage.bucket(bucketName).file(fileName).createReadStream() + return storage.bucket(bucketName).file(objectName).createReadStream() } catch (error) { core.debug(`Failed to download cache: ${error}`) core.error(`Failed to download cache.`) diff --git a/packages/warp-cache/src/internal/tar.ts b/packages/warp-cache/src/internal/tar.ts index 29ebebf9..74a6d18f 100644 --- a/packages/warp-cache/src/internal/tar.ts +++ b/packages/warp-cache/src/internal/tar.ts @@ -413,7 +413,9 @@ export async function extractTar( await execCommands(commands) } -// Supports only archives created using tar and zstd +/* + * NOTE: Currently tested only on archives created using tar and zstd + */ export async function extractStreamingTar( stream: NodeJS.ReadableStream, archivePath: string, @@ -466,7 +468,6 @@ export async function extractStreamingTar( }) } -// Create a tar export async function createTar( archiveFolder: string, sourceDirectories: string[], diff --git a/packages/warp-cache/src/internal/uploadUtils.ts b/packages/warp-cache/src/internal/uploadUtils.ts new file mode 100644 index 00000000..c368e9b1 --- /dev/null +++ b/packages/warp-cache/src/internal/uploadUtils.ts @@ -0,0 +1,131 @@ +import * as core from '@actions/core' +import * as utils from './cacheUtils' + +import fs from 'fs' + +import axios, {AxiosError} from 'axios' +import {InternalS3CompletedPart} from './contracts' + +import {Storage, TransferManager} from '@google-cloud/storage' + +function getContentRange(start: number, end: number): string { + // Format: `bytes start-end/filesize + // start and end are inclusive + // filesize can be * + // For a 200 byte chunk starting at byte 0: + // Content-Range: bytes 0-199/* + return `bytes ${start}-${end}/*` +} + +async function uploadChunk( + resourceUrl: string, + openStream: () => NodeJS.ReadableStream, + partNumber: number, + start: number, + end: number +): Promise { + core.debug( + `Uploading chunk of size ${ + end - start + 1 + } bytes at offset ${start} with content range: ${getContentRange( + start, + end + )}` + ) + + // Manually convert the readable stream to a buffer. S3 doesn't allow stream as input + const chunks = await utils.streamToBuffer(openStream()) + + try { + // HACK: Using axios here as S3 API doesn't allow readable stream as input and Github's HTTP client is not able to send buffer as body + const response = await axios.request({ + method: 'PUT', + url: resourceUrl, + headers: { + 'Content-Type': 'application/octet-stream' + }, + data: chunks + }) + return { + ETag: response.headers.etag ?? '', + PartNumber: partNumber + } + } catch (error) { + throw new Error( + `Cache service responded with ${ + (error as AxiosError).status + } during upload chunk.` + ) + } +} + +export async function uploadFileToS3( + preSignedURLs: string[], + archivePath: string +): Promise { + const fileSize = utils.getArchiveFileSizeInBytes(archivePath) + const numberOfChunks = preSignedURLs.length + + const fd = fs.openSync(archivePath, 'r') + + core.debug('Awaiting all uploads') + let offset = 0 + + try { + const completedParts = await Promise.all( + preSignedURLs.map(async (presignedURL, index) => { + const chunkSize = Math.ceil(fileSize / numberOfChunks) + const start = offset + const end = offset + chunkSize - 1 + offset += chunkSize + + return await uploadChunk( + presignedURL, + () => + fs + .createReadStream(archivePath, { + fd, + start, + end, + autoClose: false + }) + .on('error', error => { + throw new Error( + `Cache upload failed because file read failed with ${error.message}` + ) + }), + index + 1, + start, + end + ) + }) + ) + + return completedParts + } finally { + fs.closeSync(fd) + } +} + +/* + * Uploads the cache to GCS + * @param localArchivePath - The path to the cache archive + * @param bucketName - The name of the bucket in GCS + * @param objectName - The name of the object in GCS + */ +export async function multiPartUploadToGCS( + storage: Storage, + localArchivePath: string, + bucketName: string, + objectName: string +) { + try { + const transferManager = new TransferManager(storage.bucket(bucketName)) + + await transferManager.uploadFileInChunks(localArchivePath, { + uploadName: objectName + }) + } catch (error) { + throw new Error(`Failed to upload to GCS: ${error}`) + } +} diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/.openapi-generator-ignore b/packages/warp-cache/src/internal/warpcache-ts-sdk/.openapi-generator-ignore new file mode 100644 index 00000000..7484ee59 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/.openapi-generator-ignore @@ -0,0 +1,23 @@ +# OpenAPI Generator Ignore +# Generated by openapi-generator https://github.com/openapitools/openapi-generator + +# Use this file to prevent files from being overwritten by the generator. +# The patterns follow closely to .gitignore or .dockerignore. + +# As an example, the C# client generator defines ApiClient.cs. +# You can make changes and tell OpenAPI Generator to ignore just this file by uncommenting the following line: +#ApiClient.cs + +# You can match any string of characters against a directory, file or extension with a single asterisk (*): +#foo/*/qux +# The above matches foo/bar/qux and foo/baz/qux, but not foo/bar/baz/qux + +# You can recursively match patterns against a directory, file or extension with a double asterisk (**): +#foo/**/qux +# This matches foo/bar/qux, foo/baz/qux, and foo/bar/baz/qux + +# You can also negate patterns with an exclamation (!). +# For example, you can ignore all files in a docs folder with the file extension .md: +#docs/*.md +# Then explicitly reverse the ignore rule for a single file: +#!docs/README.md diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/.openapi-generator/FILES b/packages/warp-cache/src/internal/warpcache-ts-sdk/.openapi-generator/FILES new file mode 100644 index 00000000..c77d09e9 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/.openapi-generator/FILES @@ -0,0 +1,31 @@ +.gitignore +.npmignore +.openapi-generator-ignore +README.md +api.ts +api/default-api.ts +base.ts +common.ts +configuration.ts +git_push.sh +index.ts +models/commons-commit-cache-request.ts +models/commons-commit-cache-response.ts +models/commons-delete-cache-response.ts +models/commons-gcscommit-cache-response.ts +models/commons-gcsget-cache-reponse.ts +models/commons-gcsreserve-cache-response.ts +models/commons-get-cache-response.ts +models/commons-reserve-cache-request.ts +models/commons-reserve-cache-response.ts +models/commons-s3-commit-cache-response.ts +models/commons-s3-delete-cache-response.ts +models/commons-s3-get-cache-response.ts +models/commons-s3-reserve-cache-response.ts +models/commons-short-lived-token.ts +models/index.ts +models/types-completed-part.ts +models/warp-build-apierror.ts +package.json +tsconfig.esm.json +tsconfig.json diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/.openapi-generator/VERSION b/packages/warp-cache/src/internal/warpcache-ts-sdk/.openapi-generator/VERSION new file mode 100644 index 00000000..ba7f754d --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/.openapi-generator/VERSION @@ -0,0 +1 @@ +7.4.0 diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/api.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/api.ts new file mode 100644 index 00000000..7e2a1dbc --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/api.ts @@ -0,0 +1,18 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + + +export * from './api/default-api'; + diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/api/default-api.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/api/default-api.ts new file mode 100644 index 00000000..7a233cb2 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/api/default-api.ts @@ -0,0 +1,493 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +import type { Configuration } from '../configuration'; +import type { AxiosPromise, AxiosInstance, RawAxiosRequestConfig } from 'axios'; +import globalAxios from 'axios'; +// Some imports not used depending on template conditions +// @ts-ignore +import { DUMMY_BASE_URL, assertParamExists, setApiKeyToObject, setBasicAuthToObject, setBearerAuthToObject, setOAuthToObject, setSearchParams, serializeDataIfNeeded, toPathString, createRequestFunction } from '../common'; +// @ts-ignore +import { BASE_PATH, COLLECTION_FORMATS, RequestArgs, BaseAPI, RequiredError, operationServerMap } from '../base'; +// @ts-ignore +import { CommonsCommitCacheRequest } from '../models'; +// @ts-ignore +import { CommonsCommitCacheResponse } from '../models'; +// @ts-ignore +import { CommonsDeleteCacheResponse } from '../models'; +// @ts-ignore +import { CommonsGetCacheResponse } from '../models'; +// @ts-ignore +import { CommonsReserveCacheRequest } from '../models'; +// @ts-ignore +import { CommonsReserveCacheResponse } from '../models'; +// @ts-ignore +import { WarpBuildAPIError } from '../models'; +/** + * DefaultApi - axios parameter creator + * @export + */ +export const DefaultApiAxiosParamCreator = function (configuration?: Configuration) { + return { + /** + * do ping + * @summary pings the api + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + pingGet: async (options: RawAxiosRequestConfig = {}): Promise => { + const localVarPath = `/ping`; + // use dummy base URL string because the URL constructor only accepts absolute URLs. + const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); + let baseOptions; + if (configuration) { + baseOptions = configuration.baseOptions; + } + + const localVarRequestOptions = { method: 'GET', ...baseOptions, ...options}; + const localVarHeaderParameter = {} as any; + const localVarQueryParameter = {} as any; + + + + setSearchParams(localVarUrlObj, localVarQueryParameter); + let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; + localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; + + return { + url: toPathString(localVarUrlObj), + options: localVarRequestOptions, + }; + }, + /** + * commit cache + * @summary commit cache + * @param {CommonsCommitCacheRequest} body Commit Cache Request Body + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + v1CacheCommitPost: async (body: CommonsCommitCacheRequest, options: RawAxiosRequestConfig = {}): Promise => { + // verify required parameter 'body' is not null or undefined + assertParamExists('v1CacheCommitPost', 'body', body) + const localVarPath = `/v1/cache/commit`; + // use dummy base URL string because the URL constructor only accepts absolute URLs. + const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); + let baseOptions; + if (configuration) { + baseOptions = configuration.baseOptions; + } + + const localVarRequestOptions = { method: 'POST', ...baseOptions, ...options}; + const localVarHeaderParameter = {} as any; + const localVarQueryParameter = {} as any; + + + + localVarHeaderParameter['Content-Type'] = 'application/json'; + + setSearchParams(localVarUrlObj, localVarQueryParameter); + let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; + localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; + localVarRequestOptions.data = serializeDataIfNeeded(body, localVarRequestOptions, configuration) + + return { + url: toPathString(localVarUrlObj), + options: localVarRequestOptions, + }; + }, + /** + * delete cache + * @summary delete cache + * @param {string} keys cache keys + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + v1CacheDeleteDelete: async (keys: string, options: RawAxiosRequestConfig = {}): Promise => { + // verify required parameter 'keys' is not null or undefined + assertParamExists('v1CacheDeleteDelete', 'keys', keys) + const localVarPath = `/v1/cache/delete`; + // use dummy base URL string because the URL constructor only accepts absolute URLs. + const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); + let baseOptions; + if (configuration) { + baseOptions = configuration.baseOptions; + } + + const localVarRequestOptions = { method: 'DELETE', ...baseOptions, ...options}; + const localVarHeaderParameter = {} as any; + const localVarQueryParameter = {} as any; + + if (keys !== undefined) { + localVarQueryParameter['keys'] = keys; + } + + + + setSearchParams(localVarUrlObj, localVarQueryParameter); + let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; + localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; + + return { + url: toPathString(localVarUrlObj), + options: localVarRequestOptions, + }; + }, + /** + * get cache + * @summary get cache + * @param {string} keys cache keys + * @param {string} version cache version + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + v1CacheGet: async (keys: string, version: string, options: RawAxiosRequestConfig = {}): Promise => { + // verify required parameter 'keys' is not null or undefined + assertParamExists('v1CacheGet', 'keys', keys) + // verify required parameter 'version' is not null or undefined + assertParamExists('v1CacheGet', 'version', version) + const localVarPath = `/v1/cache`; + // use dummy base URL string because the URL constructor only accepts absolute URLs. + const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); + let baseOptions; + if (configuration) { + baseOptions = configuration.baseOptions; + } + + const localVarRequestOptions = { method: 'GET', ...baseOptions, ...options}; + const localVarHeaderParameter = {} as any; + const localVarQueryParameter = {} as any; + + if (keys !== undefined) { + localVarQueryParameter['keys'] = keys; + } + + if (version !== undefined) { + localVarQueryParameter['version'] = version; + } + + + + setSearchParams(localVarUrlObj, localVarQueryParameter); + let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; + localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; + + return { + url: toPathString(localVarUrlObj), + options: localVarRequestOptions, + }; + }, + /** + * reserve cache + * @summary reserve cache + * @param {CommonsReserveCacheRequest} body Reserve Cache Request Body + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + v1CacheReservePost: async (body: CommonsReserveCacheRequest, options: RawAxiosRequestConfig = {}): Promise => { + // verify required parameter 'body' is not null or undefined + assertParamExists('v1CacheReservePost', 'body', body) + const localVarPath = `/v1/cache/reserve`; + // use dummy base URL string because the URL constructor only accepts absolute URLs. + const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); + let baseOptions; + if (configuration) { + baseOptions = configuration.baseOptions; + } + + const localVarRequestOptions = { method: 'POST', ...baseOptions, ...options}; + const localVarHeaderParameter = {} as any; + const localVarQueryParameter = {} as any; + + + + localVarHeaderParameter['Content-Type'] = 'application/json'; + + setSearchParams(localVarUrlObj, localVarQueryParameter); + let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; + localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; + localVarRequestOptions.data = serializeDataIfNeeded(body, localVarRequestOptions, configuration) + + return { + url: toPathString(localVarUrlObj), + options: localVarRequestOptions, + }; + }, + } +}; + +/** + * DefaultApi - functional programming interface + * @export + */ +export const DefaultApiFp = function(configuration?: Configuration) { + const localVarAxiosParamCreator = DefaultApiAxiosParamCreator(configuration) + return { + /** + * do ping + * @summary pings the api + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + async pingGet(options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.pingGet(options); + const localVarOperationServerIndex = configuration?.serverIndex ?? 0; + const localVarOperationServerBasePath = operationServerMap['DefaultApi.pingGet']?.[localVarOperationServerIndex]?.url; + return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); + }, + /** + * commit cache + * @summary commit cache + * @param {CommonsCommitCacheRequest} body Commit Cache Request Body + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + async v1CacheCommitPost(body: CommonsCommitCacheRequest, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.v1CacheCommitPost(body, options); + const localVarOperationServerIndex = configuration?.serverIndex ?? 0; + const localVarOperationServerBasePath = operationServerMap['DefaultApi.v1CacheCommitPost']?.[localVarOperationServerIndex]?.url; + return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); + }, + /** + * delete cache + * @summary delete cache + * @param {string} keys cache keys + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + async v1CacheDeleteDelete(keys: string, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.v1CacheDeleteDelete(keys, options); + const localVarOperationServerIndex = configuration?.serverIndex ?? 0; + const localVarOperationServerBasePath = operationServerMap['DefaultApi.v1CacheDeleteDelete']?.[localVarOperationServerIndex]?.url; + return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); + }, + /** + * get cache + * @summary get cache + * @param {string} keys cache keys + * @param {string} version cache version + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + async v1CacheGet(keys: string, version: string, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.v1CacheGet(keys, version, options); + const localVarOperationServerIndex = configuration?.serverIndex ?? 0; + const localVarOperationServerBasePath = operationServerMap['DefaultApi.v1CacheGet']?.[localVarOperationServerIndex]?.url; + return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); + }, + /** + * reserve cache + * @summary reserve cache + * @param {CommonsReserveCacheRequest} body Reserve Cache Request Body + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + async v1CacheReservePost(body: CommonsReserveCacheRequest, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.v1CacheReservePost(body, options); + const localVarOperationServerIndex = configuration?.serverIndex ?? 0; + const localVarOperationServerBasePath = operationServerMap['DefaultApi.v1CacheReservePost']?.[localVarOperationServerIndex]?.url; + return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); + }, + } +}; + +/** + * DefaultApi - factory interface + * @export + */ +export const DefaultApiFactory = function (configuration?: Configuration, basePath?: string, axios?: AxiosInstance) { + const localVarFp = DefaultApiFp(configuration) + return { + /** + * do ping + * @summary pings the api + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + pingGet(options?: RawAxiosRequestConfig): AxiosPromise { + return localVarFp.pingGet(options).then((request) => request(axios, basePath)); + }, + /** + * commit cache + * @summary commit cache + * @param {DefaultApiV1CacheCommitPostRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + v1CacheCommitPost(requestParameters: DefaultApiV1CacheCommitPostRequest, options?: RawAxiosRequestConfig): AxiosPromise { + return localVarFp.v1CacheCommitPost(requestParameters.body, options).then((request) => request(axios, basePath)); + }, + /** + * delete cache + * @summary delete cache + * @param {DefaultApiV1CacheDeleteDeleteRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + v1CacheDeleteDelete(requestParameters: DefaultApiV1CacheDeleteDeleteRequest, options?: RawAxiosRequestConfig): AxiosPromise { + return localVarFp.v1CacheDeleteDelete(requestParameters.keys, options).then((request) => request(axios, basePath)); + }, + /** + * get cache + * @summary get cache + * @param {DefaultApiV1CacheGetRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + v1CacheGet(requestParameters: DefaultApiV1CacheGetRequest, options?: RawAxiosRequestConfig): AxiosPromise { + return localVarFp.v1CacheGet(requestParameters.keys, requestParameters.version, options).then((request) => request(axios, basePath)); + }, + /** + * reserve cache + * @summary reserve cache + * @param {DefaultApiV1CacheReservePostRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + v1CacheReservePost(requestParameters: DefaultApiV1CacheReservePostRequest, options?: RawAxiosRequestConfig): AxiosPromise { + return localVarFp.v1CacheReservePost(requestParameters.body, options).then((request) => request(axios, basePath)); + }, + }; +}; + +/** + * Request parameters for v1CacheCommitPost operation in DefaultApi. + * @export + * @interface DefaultApiV1CacheCommitPostRequest + */ +export interface DefaultApiV1CacheCommitPostRequest { + /** + * Commit Cache Request Body + * @type {CommonsCommitCacheRequest} + * @memberof DefaultApiV1CacheCommitPost + */ + readonly body: CommonsCommitCacheRequest +} + +/** + * Request parameters for v1CacheDeleteDelete operation in DefaultApi. + * @export + * @interface DefaultApiV1CacheDeleteDeleteRequest + */ +export interface DefaultApiV1CacheDeleteDeleteRequest { + /** + * cache keys + * @type {string} + * @memberof DefaultApiV1CacheDeleteDelete + */ + readonly keys: string +} + +/** + * Request parameters for v1CacheGet operation in DefaultApi. + * @export + * @interface DefaultApiV1CacheGetRequest + */ +export interface DefaultApiV1CacheGetRequest { + /** + * cache keys + * @type {string} + * @memberof DefaultApiV1CacheGet + */ + readonly keys: string + + /** + * cache version + * @type {string} + * @memberof DefaultApiV1CacheGet + */ + readonly version: string +} + +/** + * Request parameters for v1CacheReservePost operation in DefaultApi. + * @export + * @interface DefaultApiV1CacheReservePostRequest + */ +export interface DefaultApiV1CacheReservePostRequest { + /** + * Reserve Cache Request Body + * @type {CommonsReserveCacheRequest} + * @memberof DefaultApiV1CacheReservePost + */ + readonly body: CommonsReserveCacheRequest +} + +/** + * DefaultApi - object-oriented interface + * @export + * @class DefaultApi + * @extends {BaseAPI} + */ +export class DefaultApi extends BaseAPI { + /** + * do ping + * @summary pings the api + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof DefaultApi + */ + public pingGet(options?: RawAxiosRequestConfig) { + return DefaultApiFp(this.configuration).pingGet(options).then((request) => request(this.axios, this.basePath)); + } + + /** + * commit cache + * @summary commit cache + * @param {DefaultApiV1CacheCommitPostRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof DefaultApi + */ + public v1CacheCommitPost(requestParameters: DefaultApiV1CacheCommitPostRequest, options?: RawAxiosRequestConfig) { + return DefaultApiFp(this.configuration).v1CacheCommitPost(requestParameters.body, options).then((request) => request(this.axios, this.basePath)); + } + + /** + * delete cache + * @summary delete cache + * @param {DefaultApiV1CacheDeleteDeleteRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof DefaultApi + */ + public v1CacheDeleteDelete(requestParameters: DefaultApiV1CacheDeleteDeleteRequest, options?: RawAxiosRequestConfig) { + return DefaultApiFp(this.configuration).v1CacheDeleteDelete(requestParameters.keys, options).then((request) => request(this.axios, this.basePath)); + } + + /** + * get cache + * @summary get cache + * @param {DefaultApiV1CacheGetRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof DefaultApi + */ + public v1CacheGet(requestParameters: DefaultApiV1CacheGetRequest, options?: RawAxiosRequestConfig) { + return DefaultApiFp(this.configuration).v1CacheGet(requestParameters.keys, requestParameters.version, options).then((request) => request(this.axios, this.basePath)); + } + + /** + * reserve cache + * @summary reserve cache + * @param {DefaultApiV1CacheReservePostRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof DefaultApi + */ + public v1CacheReservePost(requestParameters: DefaultApiV1CacheReservePostRequest, options?: RawAxiosRequestConfig) { + return DefaultApiFp(this.configuration).v1CacheReservePost(requestParameters.body, options).then((request) => request(this.axios, this.basePath)); + } +} + diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/base.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/base.ts new file mode 100644 index 00000000..794123f1 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/base.ts @@ -0,0 +1,86 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +import type { Configuration } from './configuration'; +// Some imports not used depending on template conditions +// @ts-ignore +import type { AxiosPromise, AxiosInstance, RawAxiosRequestConfig } from 'axios'; +import globalAxios from 'axios'; + +export const BASE_PATH = "http://localhost".replace(/\/+$/, ""); + +/** + * + * @export + */ +export const COLLECTION_FORMATS = { + csv: ",", + ssv: " ", + tsv: "\t", + pipes: "|", +}; + +/** + * + * @export + * @interface RequestArgs + */ +export interface RequestArgs { + url: string; + options: RawAxiosRequestConfig; +} + +/** + * + * @export + * @class BaseAPI + */ +export class BaseAPI { + protected configuration: Configuration | undefined; + + constructor(configuration?: Configuration, protected basePath: string = BASE_PATH, protected axios: AxiosInstance = globalAxios) { + if (configuration) { + this.configuration = configuration; + this.basePath = configuration.basePath ?? basePath; + } + } +}; + +/** + * + * @export + * @class RequiredError + * @extends {Error} + */ +export class RequiredError extends Error { + constructor(public field: string, msg?: string) { + super(msg); + this.name = "RequiredError" + } +} + +interface ServerMap { + [key: string]: { + url: string, + description: string, + }[]; +} + +/** + * + * @export + */ +export const operationServerMap: ServerMap = { +} diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/common.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/common.ts new file mode 100644 index 00000000..674fad9c --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/common.ts @@ -0,0 +1,150 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +import type { Configuration } from "./configuration"; +import type { RequestArgs } from "./base"; +import type { AxiosInstance, AxiosResponse } from 'axios'; +import { RequiredError } from "./base"; + +/** + * + * @export + */ +export const DUMMY_BASE_URL = 'https://example.com' + +/** + * + * @throws {RequiredError} + * @export + */ +export const assertParamExists = function (functionName: string, paramName: string, paramValue: unknown) { + if (paramValue === null || paramValue === undefined) { + throw new RequiredError(paramName, `Required parameter ${paramName} was null or undefined when calling ${functionName}.`); + } +} + +/** + * + * @export + */ +export const setApiKeyToObject = async function (object: any, keyParamName: string, configuration?: Configuration) { + if (configuration && configuration.apiKey) { + const localVarApiKeyValue = typeof configuration.apiKey === 'function' + ? await configuration.apiKey(keyParamName) + : await configuration.apiKey; + object[keyParamName] = localVarApiKeyValue; + } +} + +/** + * + * @export + */ +export const setBasicAuthToObject = function (object: any, configuration?: Configuration) { + if (configuration && (configuration.username || configuration.password)) { + object["auth"] = { username: configuration.username, password: configuration.password }; + } +} + +/** + * + * @export + */ +export const setBearerAuthToObject = async function (object: any, configuration?: Configuration) { + if (configuration && configuration.accessToken) { + const accessToken = typeof configuration.accessToken === 'function' + ? await configuration.accessToken() + : await configuration.accessToken; + object["Authorization"] = "Bearer " + accessToken; + } +} + +/** + * + * @export + */ +export const setOAuthToObject = async function (object: any, name: string, scopes: string[], configuration?: Configuration) { + if (configuration && configuration.accessToken) { + const localVarAccessTokenValue = typeof configuration.accessToken === 'function' + ? await configuration.accessToken(name, scopes) + : await configuration.accessToken; + object["Authorization"] = "Bearer " + localVarAccessTokenValue; + } +} + +function setFlattenedQueryParams(urlSearchParams: URLSearchParams, parameter: any, key: string = ""): void { + if (parameter == null) return; + if (typeof parameter === "object") { + if (Array.isArray(parameter)) { + (parameter as any[]).forEach(item => setFlattenedQueryParams(urlSearchParams, item, key)); + } + else { + Object.keys(parameter).forEach(currentKey => + setFlattenedQueryParams(urlSearchParams, parameter[currentKey], `${key}${key !== '' ? '.' : ''}${currentKey}`) + ); + } + } + else { + if (urlSearchParams.has(key)) { + urlSearchParams.append(key, parameter); + } + else { + urlSearchParams.set(key, parameter); + } + } +} + +/** + * + * @export + */ +export const setSearchParams = function (url: URL, ...objects: any[]) { + const searchParams = new URLSearchParams(url.search); + setFlattenedQueryParams(searchParams, objects); + url.search = searchParams.toString(); +} + +/** + * + * @export + */ +export const serializeDataIfNeeded = function (value: any, requestOptions: any, configuration?: Configuration) { + const nonString = typeof value !== 'string'; + const needsSerialization = nonString && configuration && configuration.isJsonMime + ? configuration.isJsonMime(requestOptions.headers['Content-Type']) + : nonString; + return needsSerialization + ? JSON.stringify(value !== undefined ? value : {}) + : (value || ""); +} + +/** + * + * @export + */ +export const toPathString = function (url: URL) { + return url.pathname + url.search + url.hash +} + +/** + * + * @export + */ +export const createRequestFunction = function (axiosArgs: RequestArgs, globalAxios: AxiosInstance, BASE_PATH: string, configuration?: Configuration) { + return >(axios: AxiosInstance = globalAxios, basePath: string = BASE_PATH) => { + const axiosRequestArgs = {...axiosArgs.options, url: (axios.defaults.baseURL ? '' : configuration?.basePath ?? basePath) + axiosArgs.url}; + return axios.request(axiosRequestArgs); + }; +} diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/configuration.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/configuration.ts new file mode 100644 index 00000000..a33851c5 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/configuration.ts @@ -0,0 +1,110 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +export interface ConfigurationParameters { + apiKey?: string | Promise | ((name: string) => string) | ((name: string) => Promise); + username?: string; + password?: string; + accessToken?: string | Promise | ((name?: string, scopes?: string[]) => string) | ((name?: string, scopes?: string[]) => Promise); + basePath?: string; + serverIndex?: number; + baseOptions?: any; + formDataCtor?: new () => any; +} + +export class Configuration { + /** + * parameter for apiKey security + * @param name security name + * @memberof Configuration + */ + apiKey?: string | Promise | ((name: string) => string) | ((name: string) => Promise); + /** + * parameter for basic security + * + * @type {string} + * @memberof Configuration + */ + username?: string; + /** + * parameter for basic security + * + * @type {string} + * @memberof Configuration + */ + password?: string; + /** + * parameter for oauth2 security + * @param name security name + * @param scopes oauth2 scope + * @memberof Configuration + */ + accessToken?: string | Promise | ((name?: string, scopes?: string[]) => string) | ((name?: string, scopes?: string[]) => Promise); + /** + * override base path + * + * @type {string} + * @memberof Configuration + */ + basePath?: string; + /** + * override server index + * + * @type {number} + * @memberof Configuration + */ + serverIndex?: number; + /** + * base options for axios calls + * + * @type {any} + * @memberof Configuration + */ + baseOptions?: any; + /** + * The FormData constructor that will be used to create multipart form data + * requests. You can inject this here so that execution environments that + * do not support the FormData class can still run the generated client. + * + * @type {new () => FormData} + */ + formDataCtor?: new () => any; + + constructor(param: ConfigurationParameters = {}) { + this.apiKey = param.apiKey; + this.username = param.username; + this.password = param.password; + this.accessToken = param.accessToken; + this.basePath = param.basePath; + this.serverIndex = param.serverIndex; + this.baseOptions = param.baseOptions; + this.formDataCtor = param.formDataCtor; + } + + /** + * Check if the given MIME is a JSON MIME. + * JSON MIME examples: + * application/json + * application/json; charset=UTF8 + * APPLICATION/JSON + * application/vnd.company+json + * @param mime - MIME (Multipurpose Internet Mail Extensions) + * @return True if the given MIME is JSON, false otherwise. + */ + public isJsonMime(mime: string): boolean { + const jsonMime: RegExp = new RegExp('^(application\/json|[^;/ \t]+\/[^;/ \t]+[+]json)[ \t]*(;.*)?$', 'i'); + return mime !== null && (jsonMime.test(mime) || mime.toLowerCase() === 'application/json-patch+json'); + } +} diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/index.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/index.ts new file mode 100644 index 00000000..ef3098fe --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/index.ts @@ -0,0 +1,21 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +export * from "./api"; +export * from "./configuration"; +export * from "./models"; +import * as Schema from "./schema"; + +export {Schema}; diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-commit-cache-request.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-commit-cache-request.ts new file mode 100644 index 00000000..789c1274 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-commit-cache-request.ts @@ -0,0 +1,69 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +// May contain unused imports in some cases +// @ts-ignore +import { TypesCompletedPart } from './types-completed-part'; + +/** + * + * @export + * @interface CommonsCommitCacheRequest + */ +export interface CommonsCommitCacheRequest { + /** + * + * @type {string} + * @memberof CommonsCommitCacheRequest + */ + 'cache_key': string; + /** + * + * @type {string} + * @memberof CommonsCommitCacheRequest + */ + 'cache_version': string; + /** + * + * @type {string} + * @memberof CommonsCommitCacheRequest + */ + 'os': string; + /** + * + * @type {Array} + * @memberof CommonsCommitCacheRequest + */ + 'parts': Array; + /** + * UploadID * This is not supported for GCS cache. When passed this will be ignored. * + * @type {string} + * @memberof CommonsCommitCacheRequest + */ + 'upload_id'?: string; + /** + * UploadKey * This is not supported for GCS cache. When passed this will be ignored. * + * @type {string} + * @memberof CommonsCommitCacheRequest + */ + 'upload_key'?: string; + /** + * + * @type {string} + * @memberof CommonsCommitCacheRequest + */ + 'vcs_type': string; +} + diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-commit-cache-response.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-commit-cache-response.ts new file mode 100644 index 00000000..b3969eed --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-commit-cache-response.ts @@ -0,0 +1,48 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +// May contain unused imports in some cases +// @ts-ignore +import { CommonsGCSCommitCacheResponse } from './commons-gcscommit-cache-response'; +// May contain unused imports in some cases +// @ts-ignore +import { CommonsS3CommitCacheResponse } from './commons-s3-commit-cache-response'; + +/** + * + * @export + * @interface CommonsCommitCacheResponse + */ +export interface CommonsCommitCacheResponse { + /** + * + * @type {CommonsGCSCommitCacheResponse} + * @memberof CommonsCommitCacheResponse + */ + 'gcs'?: CommonsGCSCommitCacheResponse; + /** + * + * @type {string} + * @memberof CommonsCommitCacheResponse + */ + 'provider'?: string; + /** + * + * @type {CommonsS3CommitCacheResponse} + * @memberof CommonsCommitCacheResponse + */ + 's3'?: CommonsS3CommitCacheResponse; +} + diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-delete-cache-response.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-delete-cache-response.ts new file mode 100644 index 00000000..8d03a281 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-delete-cache-response.ts @@ -0,0 +1,45 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +// May contain unused imports in some cases +// @ts-ignore +import { CommonsS3DeleteCacheResponse } from './commons-s3-delete-cache-response'; + +/** + * + * @export + * @interface CommonsDeleteCacheResponse + */ +export interface CommonsDeleteCacheResponse { + /** + * + * @type {object} + * @memberof CommonsDeleteCacheResponse + */ + 'gcs'?: object; + /** + * + * @type {string} + * @memberof CommonsDeleteCacheResponse + */ + 'provider'?: string; + /** + * + * @type {CommonsS3DeleteCacheResponse} + * @memberof CommonsDeleteCacheResponse + */ + 's3'?: CommonsS3DeleteCacheResponse; +} + diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-gcscommit-cache-response.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-gcscommit-cache-response.ts new file mode 100644 index 00000000..74449fc7 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-gcscommit-cache-response.ts @@ -0,0 +1,57 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +// May contain unused imports in some cases +// @ts-ignore +import { CommonsShortLivedToken } from './commons-short-lived-token'; + +/** + * + * @export + * @interface CommonsGCSCommitCacheResponse + */ +export interface CommonsGCSCommitCacheResponse { + /** + * + * @type {string} + * @memberof CommonsGCSCommitCacheResponse + */ + 'bucket_name'?: string; + /** + * CacheKey is the resolved cache key which might contain some prefix or suffix in addition to the cache key provided by the user. This is the actual storage location in gcs. + * @type {string} + * @memberof CommonsGCSCommitCacheResponse + */ + 'cache_key': string; + /** + * Method contains the auth method to be used to connect to the GCP storage backend + * @type {string} + * @memberof CommonsGCSCommitCacheResponse + */ + 'method'?: string; + /** + * + * @type {string} + * @memberof CommonsGCSCommitCacheResponse + */ + 'project_id'?: string; + /** + * + * @type {CommonsShortLivedToken} + * @memberof CommonsGCSCommitCacheResponse + */ + 'short_lived_token'?: CommonsShortLivedToken; +} + diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-gcsget-cache-reponse.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-gcsget-cache-reponse.ts new file mode 100644 index 00000000..aa7ab6f5 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-gcsget-cache-reponse.ts @@ -0,0 +1,63 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +// May contain unused imports in some cases +// @ts-ignore +import { CommonsShortLivedToken } from './commons-short-lived-token'; + +/** + * + * @export + * @interface CommonsGCSGetCacheReponse + */ +export interface CommonsGCSGetCacheReponse { + /** + * + * @type {string} + * @memberof CommonsGCSGetCacheReponse + */ + 'bucket_name'?: string; + /** + * + * @type {string} + * @memberof CommonsGCSGetCacheReponse + */ + 'cache_key'?: string; + /** + * + * @type {string} + * @memberof CommonsGCSGetCacheReponse + */ + 'cache_version'?: string; + /** + * Method contains the auth method to be used to connect to the GCP storage backend + * @type {string} + * @memberof CommonsGCSGetCacheReponse + */ + 'method'?: string; + /** + * + * @type {string} + * @memberof CommonsGCSGetCacheReponse + */ + 'project_id'?: string; + /** + * + * @type {CommonsShortLivedToken} + * @memberof CommonsGCSGetCacheReponse + */ + 'short_lived_token'?: CommonsShortLivedToken; +} + diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-gcsreserve-cache-response.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-gcsreserve-cache-response.ts new file mode 100644 index 00000000..776491ef --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-gcsreserve-cache-response.ts @@ -0,0 +1,57 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +// May contain unused imports in some cases +// @ts-ignore +import { CommonsShortLivedToken } from './commons-short-lived-token'; + +/** + * + * @export + * @interface CommonsGCSReserveCacheResponse + */ +export interface CommonsGCSReserveCacheResponse { + /** + * + * @type {string} + * @memberof CommonsGCSReserveCacheResponse + */ + 'bucket_name'?: string; + /** + * CacheKey is the resolved cache key which might contain some prefix or suffix in addition to the cache key provided by the user. This is the actual storage location in gcs. + * @type {string} + * @memberof CommonsGCSReserveCacheResponse + */ + 'cache_key': string; + /** + * Method contains the auth method to be used to connect to the GCP storage backend + * @type {string} + * @memberof CommonsGCSReserveCacheResponse + */ + 'method'?: string; + /** + * + * @type {string} + * @memberof CommonsGCSReserveCacheResponse + */ + 'project_id'?: string; + /** + * + * @type {CommonsShortLivedToken} + * @memberof CommonsGCSReserveCacheResponse + */ + 'short_lived_token'?: CommonsShortLivedToken; +} + diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-get-cache-response.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-get-cache-response.ts new file mode 100644 index 00000000..4ced0787 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-get-cache-response.ts @@ -0,0 +1,48 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +// May contain unused imports in some cases +// @ts-ignore +import { CommonsGCSGetCacheReponse } from './commons-gcsget-cache-reponse'; +// May contain unused imports in some cases +// @ts-ignore +import { CommonsS3GetCacheResponse } from './commons-s3-get-cache-response'; + +/** + * + * @export + * @interface CommonsGetCacheResponse + */ +export interface CommonsGetCacheResponse { + /** + * + * @type {CommonsGCSGetCacheReponse} + * @memberof CommonsGetCacheResponse + */ + 'gcs'?: CommonsGCSGetCacheReponse; + /** + * + * @type {string} + * @memberof CommonsGetCacheResponse + */ + 'provider'?: string; + /** + * + * @type {CommonsS3GetCacheResponse} + * @memberof CommonsGetCacheResponse + */ + 's3'?: CommonsS3GetCacheResponse; +} + diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-reserve-cache-request.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-reserve-cache-request.ts new file mode 100644 index 00000000..19be9872 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-reserve-cache-request.ts @@ -0,0 +1,42 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + + +/** + * + * @export + * @interface CommonsReserveCacheRequest + */ +export interface CommonsReserveCacheRequest { + /** + * + * @type {string} + * @memberof CommonsReserveCacheRequest + */ + 'cache_key': string; + /** + * ContentType contains the content type of the cache. * This is not supported for GCS cache. When passed this will be ignored. * + * @type {string} + * @memberof CommonsReserveCacheRequest + */ + 'content_type'?: string; + /** + * NumberOfChunks contains the number of chunks the cache will be split into. Minimum value: 1. Maximum value: 10000. * This is not supported for GCS cache. When passed this will be ignored. * + * @type {number} + * @memberof CommonsReserveCacheRequest + */ + 'number_of_chunks'?: number; +} + diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-reserve-cache-response.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-reserve-cache-response.ts new file mode 100644 index 00000000..ed1ae9c5 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-reserve-cache-response.ts @@ -0,0 +1,48 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +// May contain unused imports in some cases +// @ts-ignore +import { CommonsGCSReserveCacheResponse } from './commons-gcsreserve-cache-response'; +// May contain unused imports in some cases +// @ts-ignore +import { CommonsS3ReserveCacheResponse } from './commons-s3-reserve-cache-response'; + +/** + * + * @export + * @interface CommonsReserveCacheResponse + */ +export interface CommonsReserveCacheResponse { + /** + * + * @type {CommonsGCSReserveCacheResponse} + * @memberof CommonsReserveCacheResponse + */ + 'gcs'?: CommonsGCSReserveCacheResponse; + /** + * + * @type {string} + * @memberof CommonsReserveCacheResponse + */ + 'provider'?: string; + /** + * + * @type {CommonsS3ReserveCacheResponse} + * @memberof CommonsReserveCacheResponse + */ + 's3'?: CommonsS3ReserveCacheResponse; +} + diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-s3-commit-cache-response.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-s3-commit-cache-response.ts new file mode 100644 index 00000000..f7afafde --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-s3-commit-cache-response.ts @@ -0,0 +1,36 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + + +/** + * + * @export + * @interface CommonsS3CommitCacheResponse + */ +export interface CommonsS3CommitCacheResponse { + /** + * + * @type {string} + * @memberof CommonsS3CommitCacheResponse + */ + 'cache_key': string; + /** + * + * @type {string} + * @memberof CommonsS3CommitCacheResponse + */ + 'cache_version': string; +} + diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-s3-delete-cache-response.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-s3-delete-cache-response.ts new file mode 100644 index 00000000..0a7069eb --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-s3-delete-cache-response.ts @@ -0,0 +1,36 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + + +/** + * + * @export + * @interface CommonsS3DeleteCacheResponse + */ +export interface CommonsS3DeleteCacheResponse { + /** + * + * @type {string} + * @memberof CommonsS3DeleteCacheResponse + */ + 'cache_key': string; + /** + * + * @type {string} + * @memberof CommonsS3DeleteCacheResponse + */ + 'cache_version': string; +} + diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-s3-get-cache-response.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-s3-get-cache-response.ts new file mode 100644 index 00000000..7f8c0385 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-s3-get-cache-response.ts @@ -0,0 +1,42 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + + +/** + * + * @export + * @interface CommonsS3GetCacheResponse + */ +export interface CommonsS3GetCacheResponse { + /** + * + * @type {string} + * @memberof CommonsS3GetCacheResponse + */ + 'cache_key'?: string; + /** + * + * @type {string} + * @memberof CommonsS3GetCacheResponse + */ + 'cache_version'?: string; + /** + * + * @type {string} + * @memberof CommonsS3GetCacheResponse + */ + 'pre_signed_url'?: string; +} + diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-s3-reserve-cache-response.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-s3-reserve-cache-response.ts new file mode 100644 index 00000000..7257cd3e --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-s3-reserve-cache-response.ts @@ -0,0 +1,42 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + + +/** + * + * @export + * @interface CommonsS3ReserveCacheResponse + */ +export interface CommonsS3ReserveCacheResponse { + /** + * + * @type {Array} + * @memberof CommonsS3ReserveCacheResponse + */ + 'pre_signed_urls'?: Array; + /** + * + * @type {string} + * @memberof CommonsS3ReserveCacheResponse + */ + 'upload_id'?: string; + /** + * + * @type {string} + * @memberof CommonsS3ReserveCacheResponse + */ + 'upload_key'?: string; +} + diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-short-lived-token.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-short-lived-token.ts new file mode 100644 index 00000000..82cc84f2 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-short-lived-token.ts @@ -0,0 +1,36 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + + +/** + * + * @export + * @interface CommonsShortLivedToken + */ +export interface CommonsShortLivedToken { + /** + * AccessToken contains the short lived access token to be used to connect to the GCP storage backend + * @type {string} + * @memberof CommonsShortLivedToken + */ + 'access_token'?: string; + /** + * ExpiresAt contains the expiry time of the short lived access token format: date-time + * @type {string} + * @memberof CommonsShortLivedToken + */ + 'expires_at'?: string; +} + diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/index.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/index.ts new file mode 100644 index 00000000..10725161 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/index.ts @@ -0,0 +1,16 @@ +export * from './commons-commit-cache-request'; +export * from './commons-commit-cache-response'; +export * from './commons-delete-cache-response'; +export * from './commons-gcscommit-cache-response'; +export * from './commons-gcsget-cache-reponse'; +export * from './commons-gcsreserve-cache-response'; +export * from './commons-get-cache-response'; +export * from './commons-reserve-cache-request'; +export * from './commons-reserve-cache-response'; +export * from './commons-s3-commit-cache-response'; +export * from './commons-s3-delete-cache-response'; +export * from './commons-s3-get-cache-response'; +export * from './commons-s3-reserve-cache-response'; +export * from './commons-short-lived-token'; +export * from './types-completed-part'; +export * from './warp-build-apierror'; diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/types-completed-part.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/types-completed-part.ts new file mode 100644 index 00000000..2c9e8d2c --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/types-completed-part.ts @@ -0,0 +1,60 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + + +/** + * + * @export + * @interface TypesCompletedPart + */ +export interface TypesCompletedPart { + /** + * The base64-encoded, 32-bit CRC32 checksum of the object. This will only be present if it was uploaded with the object. With multipart uploads, this may not be a checksum value of the object. For more information about how checksums are calculated with multipart uploads, see Checking object integrity (https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html#large-object-checksums) in the Amazon S3 User Guide. + * @type {string} + * @memberof TypesCompletedPart + */ + 'ChecksumCRC32'?: string; + /** + * The base64-encoded, 32-bit CRC32C checksum of the object. This will only be present if it was uploaded with the object. With multipart uploads, this may not be a checksum value of the object. For more information about how checksums are calculated with multipart uploads, see Checking object integrity (https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html#large-object-checksums) in the Amazon S3 User Guide. + * @type {string} + * @memberof TypesCompletedPart + */ + 'ChecksumCRC32C'?: string; + /** + * The base64-encoded, 160-bit SHA-1 digest of the object. This will only be present if it was uploaded with the object. With multipart uploads, this may not be a checksum value of the object. For more information about how checksums are calculated with multipart uploads, see Checking object integrity (https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html#large-object-checksums) in the Amazon S3 User Guide. + * @type {string} + * @memberof TypesCompletedPart + */ + 'ChecksumSHA1'?: string; + /** + * The base64-encoded, 256-bit SHA-256 digest of the object. This will only be present if it was uploaded with the object. With multipart uploads, this may not be a checksum value of the object. For more information about how checksums are calculated with multipart uploads, see Checking object integrity (https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html#large-object-checksums) in the Amazon S3 User Guide. + * @type {string} + * @memberof TypesCompletedPart + */ + 'ChecksumSHA256'?: string; + /** + * Entity tag returned when the part was uploaded. + * @type {string} + * @memberof TypesCompletedPart + */ + 'ETag'?: string; + /** + * Part number that identifies the part. This is a positive integer between 1 and 10,000. + * @type {number} + * @memberof TypesCompletedPart + */ + 'PartNumber'?: number; +} + diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/warp-build-apierror.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/warp-build-apierror.ts new file mode 100644 index 00000000..b2f0d6b9 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/warp-build-apierror.ts @@ -0,0 +1,54 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + + +/** + * + * @export + * @interface WarpBuildAPIError + */ +export interface WarpBuildAPIError { + /** + * + * @type {string} + * @memberof WarpBuildAPIError + */ + 'code'?: string; + /** + * + * @type {string} + * @memberof WarpBuildAPIError + */ + 'description'?: string; + /** + * + * @type {string} + * @memberof WarpBuildAPIError + */ + 'message'?: string; + /** + * + * @type {string} + * @memberof WarpBuildAPIError + */ + 'sub_code'?: string; + /** + * + * @type {string} + * @memberof WarpBuildAPIError + */ + 'sub_message'?: string; +} + diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schema.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schema.ts new file mode 100644 index 00000000..a03165f1 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schema.ts @@ -0,0 +1,20 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export { $commons_CommitCacheRequest } from './schemas/$commons_CommitCacheRequest'; +export { $commons_CommitCacheResponse } from './schemas/$commons_CommitCacheResponse'; +export { $commons_DeleteCacheResponse } from './schemas/$commons_DeleteCacheResponse'; +export { $commons_GCSCommitCacheResponse } from './schemas/$commons_GCSCommitCacheResponse'; +export { $commons_GCSDeleteCacheResponse } from './schemas/$commons_GCSDeleteCacheResponse'; +export { $commons_GCSGetCacheReponse } from './schemas/$commons_GCSGetCacheReponse'; +export { $commons_GCSReserveCacheResponse } from './schemas/$commons_GCSReserveCacheResponse'; +export { $commons_GetCacheResponse } from './schemas/$commons_GetCacheResponse'; +export { $commons_ReserveCacheRequest } from './schemas/$commons_ReserveCacheRequest'; +export { $commons_ReserveCacheResponse } from './schemas/$commons_ReserveCacheResponse'; +export { $commons_S3CommitCacheResponse } from './schemas/$commons_S3CommitCacheResponse'; +export { $commons_S3DeleteCacheResponse } from './schemas/$commons_S3DeleteCacheResponse'; +export { $commons_S3GetCacheResponse } from './schemas/$commons_S3GetCacheResponse'; +export { $commons_S3ReserveCacheResponse } from './schemas/$commons_S3ReserveCacheResponse'; +export { $commons_ShortLivedToken } from './schemas/$commons_ShortLivedToken'; +export { $types_CompletedPart } from './schemas/$types_CompletedPart'; +export { $WarpBuildAPIError } from './schemas/$WarpBuildAPIError'; diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$WarpBuildAPIError.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$WarpBuildAPIError.ts new file mode 100644 index 00000000..0cc97b15 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$WarpBuildAPIError.ts @@ -0,0 +1,22 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export const $WarpBuildAPIError = { + properties: { + code: { + type: 'string', + }, + description: { + type: 'string', + }, + message: { + type: 'string', + }, + sub_code: { + type: 'string', + }, + sub_message: { + type: 'string', + }, + }, +} as const; \ No newline at end of file diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CommitCacheRequest.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CommitCacheRequest.ts new file mode 100644 index 00000000..cb58a343 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CommitCacheRequest.ts @@ -0,0 +1,36 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export const $commons_CommitCacheRequest = { + properties: { + cache_key: { + type: 'string', + isRequired: true, + }, + cache_version: { + type: 'string', + isRequired: true, + }, + os: { + type: 'string', + isRequired: true, + }, + parts: { + type: 'array', + contains: { + type: 'types_CompletedPart', + }, + isRequired: true, + }, + upload_id: { + type: 'string', + }, + upload_key: { + type: 'string', + }, + vcs_type: { + type: 'string', + isRequired: true, + }, + }, +} as const; \ No newline at end of file diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CommitCacheResponse.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CommitCacheResponse.ts new file mode 100644 index 00000000..b46f134a --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CommitCacheResponse.ts @@ -0,0 +1,16 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export const $commons_CommitCacheResponse = { + properties: { + gcs: { + type: 'commons_GCSCommitCacheResponse', + }, + provider: { + type: 'string', + }, + s3: { + type: 'commons_S3CommitCacheResponse', + }, + }, +} as const; \ No newline at end of file diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_DeleteCacheResponse.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_DeleteCacheResponse.ts new file mode 100644 index 00000000..d599057e --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_DeleteCacheResponse.ts @@ -0,0 +1,16 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export const $commons_DeleteCacheResponse = { + properties: { + gcs: { + type: 'commons_GCSDeleteCacheResponse', + }, + provider: { + type: 'string', + }, + s3: { + type: 'commons_S3DeleteCacheResponse', + }, + }, +} as const; \ No newline at end of file diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GCSCommitCacheResponse.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GCSCommitCacheResponse.ts new file mode 100644 index 00000000..02f330ce --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GCSCommitCacheResponse.ts @@ -0,0 +1,23 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export const $commons_GCSCommitCacheResponse = { + properties: { + bucket_name: { + type: 'string', + }, + cache_key: { + type: 'string', + isRequired: true, + }, + method: { + type: 'string', + }, + project_id: { + type: 'string', + }, + short_lived_token: { + type: 'commons_ShortLivedToken', + }, + }, +} as const; \ No newline at end of file diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GCSDeleteCacheResponse.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GCSDeleteCacheResponse.ts new file mode 100644 index 00000000..92377d71 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GCSDeleteCacheResponse.ts @@ -0,0 +1,7 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export const $commons_GCSDeleteCacheResponse = { + properties: { + }, +} as const; \ No newline at end of file diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GCSGetCacheReponse.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GCSGetCacheReponse.ts new file mode 100644 index 00000000..1b2fb812 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GCSGetCacheReponse.ts @@ -0,0 +1,25 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export const $commons_GCSGetCacheReponse = { + properties: { + bucket_name: { + type: 'string', + }, + cache_key: { + type: 'string', + }, + cache_version: { + type: 'string', + }, + method: { + type: 'string', + }, + project_id: { + type: 'string', + }, + short_lived_token: { + type: 'commons_ShortLivedToken', + }, + }, +} as const; \ No newline at end of file diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GCSReserveCacheResponse.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GCSReserveCacheResponse.ts new file mode 100644 index 00000000..fb154ccf --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GCSReserveCacheResponse.ts @@ -0,0 +1,23 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export const $commons_GCSReserveCacheResponse = { + properties: { + bucket_name: { + type: 'string', + }, + cache_key: { + type: 'string', + isRequired: true, + }, + method: { + type: 'string', + }, + project_id: { + type: 'string', + }, + short_lived_token: { + type: 'commons_ShortLivedToken', + }, + }, +} as const; \ No newline at end of file diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GetCacheResponse.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GetCacheResponse.ts new file mode 100644 index 00000000..c4482770 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GetCacheResponse.ts @@ -0,0 +1,16 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export const $commons_GetCacheResponse = { + properties: { + gcs: { + type: 'commons_GCSGetCacheReponse', + }, + provider: { + type: 'string', + }, + s3: { + type: 'commons_S3GetCacheResponse', + }, + }, +} as const; \ No newline at end of file diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_ReserveCacheRequest.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_ReserveCacheRequest.ts new file mode 100644 index 00000000..6b02ef6e --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_ReserveCacheRequest.ts @@ -0,0 +1,17 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export const $commons_ReserveCacheRequest = { + properties: { + cache_key: { + type: 'string', + isRequired: true, + }, + content_type: { + type: 'string', + }, + number_of_chunks: { + type: 'number', + }, + }, +} as const; \ No newline at end of file diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_ReserveCacheResponse.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_ReserveCacheResponse.ts new file mode 100644 index 00000000..f1b04e41 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_ReserveCacheResponse.ts @@ -0,0 +1,16 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export const $commons_ReserveCacheResponse = { + properties: { + gcs: { + type: 'commons_GCSReserveCacheResponse', + }, + provider: { + type: 'string', + }, + s3: { + type: 'commons_S3ReserveCacheResponse', + }, + }, +} as const; \ No newline at end of file diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_S3CommitCacheResponse.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_S3CommitCacheResponse.ts new file mode 100644 index 00000000..43eb0727 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_S3CommitCacheResponse.ts @@ -0,0 +1,15 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export const $commons_S3CommitCacheResponse = { + properties: { + cache_key: { + type: 'string', + isRequired: true, + }, + cache_version: { + type: 'string', + isRequired: true, + }, + }, +} as const; \ No newline at end of file diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_S3DeleteCacheResponse.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_S3DeleteCacheResponse.ts new file mode 100644 index 00000000..1a14a355 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_S3DeleteCacheResponse.ts @@ -0,0 +1,15 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export const $commons_S3DeleteCacheResponse = { + properties: { + cache_key: { + type: 'string', + isRequired: true, + }, + cache_version: { + type: 'string', + isRequired: true, + }, + }, +} as const; \ No newline at end of file diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_S3GetCacheResponse.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_S3GetCacheResponse.ts new file mode 100644 index 00000000..a23e1d99 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_S3GetCacheResponse.ts @@ -0,0 +1,16 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export const $commons_S3GetCacheResponse = { + properties: { + cache_key: { + type: 'string', + }, + cache_version: { + type: 'string', + }, + pre_signed_url: { + type: 'string', + }, + }, +} as const; \ No newline at end of file diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_S3ReserveCacheResponse.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_S3ReserveCacheResponse.ts new file mode 100644 index 00000000..46948818 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_S3ReserveCacheResponse.ts @@ -0,0 +1,19 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export const $commons_S3ReserveCacheResponse = { + properties: { + pre_signed_urls: { + type: 'array', + contains: { + type: 'string', + }, + }, + upload_id: { + type: 'string', + }, + upload_key: { + type: 'string', + }, + }, +} as const; \ No newline at end of file diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_ShortLivedToken.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_ShortLivedToken.ts new file mode 100644 index 00000000..228a2a29 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_ShortLivedToken.ts @@ -0,0 +1,13 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export const $commons_ShortLivedToken = { + properties: { + access_token: { + type: 'string', + }, + expires_at: { + type: 'string', + }, + }, +} as const; \ No newline at end of file diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$types_CompletedPart.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$types_CompletedPart.ts new file mode 100644 index 00000000..924f56cf --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$types_CompletedPart.ts @@ -0,0 +1,25 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export const $types_CompletedPart = { + properties: { + ChecksumCRC32: { + type: 'string', + }, + ChecksumCRC32C: { + type: 'string', + }, + ChecksumSHA1: { + type: 'string', + }, + ChecksumSHA256: { + type: 'string', + }, + ETag: { + type: 'string', + }, + PartNumber: { + type: 'number', + }, + }, +} as const; \ No newline at end of file diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/tsconfig.esm.json b/packages/warp-cache/src/internal/warpcache-ts-sdk/tsconfig.esm.json new file mode 100644 index 00000000..2c0331cc --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/tsconfig.esm.json @@ -0,0 +1,7 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "module": "esnext", + "outDir": "dist/esm" + } +} diff --git a/packages/warp-cache/src/test.ts b/packages/warp-cache/src/test.ts index 973ca0eb..4d06bc72 100644 --- a/packages/warp-cache/src/test.ts +++ b/packages/warp-cache/src/test.ts @@ -1,46 +1,23 @@ -import {exec, spawn} from 'child_process' import {deleteCache, restoreCache, saveCache} from './cache' -import {downloadCacheStreamingGCP} from './internal/downloadUtils' - -import fs, {write} from 'fs' -import {extractStreamingTar} from './internal/tar' -import {CompressionMethod} from './internal/constants' process.env['WARPBUILD_CACHE_URL'] = 'http://localhost:8002' process.env['RUNNER_TEMP'] = '/Users/prajjwal/Repos/warpbuild/playground/tmp_fs' process.env['NODE_DEBUG'] = 'http' +process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] = + 'eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3MTMwNzE5ODgsInJlcG8iOiJiZW5jaG1hcmtzIiwicmVwb093bmVyIjoiV2FycEJ1aWxkcyIsIngtd2FycGJ1aWxkLW9yZ2FuaXphdGlvbi1pZCI6IndmbW4wODBlaWY4cm5pd3EifQ.Wat-RATKl_KB39SF6egch3nF3_dD8hDE3lbl9wm7AyBUs9pUNEDejJtgHO0xQfGvSN-qRTPbJ_glHKPUIRHE3w' -// saveCache( -// ['/Users/prajjwal/Repos/warpbuild/playground/test_fs'], -// 'test-fs-local-key', -// true -// ) - -// restoreCache( -// ['/Users/prajjwal/Repos/warpbuild/playground/test_fs'], -// 'test-fs-local-key', -// [], -// {}, -// true -// ) - -// deleteCache(['test-fs-local-key']) - -process.env['GCP_ACCESS_TOKEN'] = - 'ya29.c.c0AY_VpZgcQopWxkSf9wIIo9NED0YFh3VIgZ1wx1ulvSCrq5iTiZWbrRGPej2vA835U2HkNdrLwaVKFLeL57v1s-guzSvihNnHMMJ4wUPJHZPQd-CJ90i6F0NYcjQuv7SC2EBkaKciM-Act0IDygPwzwwixCe-4iCxcUv3YUysZcee9Qknxq5UBPfGjqQArVKifC2fScJ7HnBmbbSc8t1mDp9mLiIpax9V31anOQ-4QK1kqSgi4gh0m-Cd7v24S7Kfc5IEcQLrVyI62W4Y4HywRJ2V_qBx3ZKFMmO1lV5Tl3wHX40XyD1J2Cc6kXbF4LHHPcMnRf85ylaXaUGMwDNlkDPFHRJmOkWnZF8-v_Y4868-Mmektdl8khWvCQwGSLHo_jCKehCJZl1qK1gzNfie7Rgm9qbooMAEg1KkPPiDBmMY_WUsBo1-a0vuHrE90IhtvKI_TNTeH-pUDjSFMsbgrhnbGu5oN6DXk--WyjHy9slW6r8TDjB8UjPE2uiaGbYrQZsRPoaKVAxVylc9tFONyPwJ10MUerPq3ESq49QUASdasuYCef0CZ_R3kJyIMQe7p6WBfOZ0L11ZTz_tnFn1Oa8JGHvYl1xvx79EbHjo4mvyr5WTAXa42g-gCnPnJFLaN649DRZbdRzbbc3-bQbqFuictuoSQmOjhrqW6_0_44wVhlga9Ok9kZ4_lx6Oqvq9SiI6IxIJSBVnXet3MgzoRdJur8Ws766sinJ_iFkZdsQdj2IQ_hj74vh61v1i84xIZY-bp-IrvQQf_vZm6bbBZXxaXhiVphpij7nY5Rz3qS2d0e3byc1iUW63jXlY1iIhlvsd1i2Zd4YVyQrfgSy_zpuXJOqhS1MwBrkddb4F-r3wQtRJ1ttmbpSJOpeYzewzSeVopk8pmOaUSd0rS4qQkY1UdhQoavyn54VMj5U8BiOkjo-wV2MUXl0FlVF7u3-c3vUhlZ1JrMj6xiWFXys_QBMtU55jMe31UV-saSFxM7f1-xk1_2xoou8' - -const readStream = downloadCacheStreamingGCP( - 'gs://cache-bench-test/custom_modules.tar.zst' +saveCache( + ['/Users/prajjwal/Repos/warpbuild/playground/test_fs'], + 'test-fs-local-key', + true ) -extractStreamingTar( - readStream!, - '/tmp/custom_modules', - CompressionMethod.ZstdWithoutLong +restoreCache( + ['/Users/prajjwal/Repos/warpbuild/playground/test_fs'], + 'test-fs-local-key', + [], + {}, + true ) - .then(() => { - console.log('done') - }) - .catch(err => { - console.log(err) - }) + +deleteCache(['test-fs-local-key']) From 9a8c6b26ce867c3b00f5ff81d69405af3d732773 Mon Sep 17 00:00:00 2001 From: Prajjwal Date: Mon, 15 Apr 2024 10:13:42 +0530 Subject: [PATCH 03/11] modifies for warpcache --- packages/warp-cache/package.json | 2 +- packages/warp-cache/src/cache.ts | 80 +++++++----- .../src/internal/cacheHttpClient.ts | 78 ++++++++--- .../warp-cache/src/internal/contracts.d.ts | 1 + .../warp-cache/src/internal/requestUtils.ts | 2 +- .../warpcache-ts-sdk/.openapi-generator/FILES | 4 + .../warpcache-ts-sdk/api/default-api.ts | 123 ++++++++---------- .../models/commons-cache-entry.ts | 72 ++++++++++ .../models/commons-commit-cache-request.ts | 24 +++- .../models/commons-commit-cache-response.ts | 21 +++ .../models/commons-delete-cache-request.ts | 54 ++++++++ .../models/commons-delete-cache-response.ts | 22 +++- .../commons-gcsdelete-cache-response.ts | 36 +++++ .../models/commons-get-cache-request.ts | 60 +++++++++ .../models/commons-get-cache-response.ts | 15 +++ .../models/commons-reserve-cache-request.ts | 24 ++++ .../models/commons-reserve-cache-response.ts | 6 + .../models/commons-s3-get-cache-response.ts | 6 + .../internal/warpcache-ts-sdk/models/index.ts | 4 + .../src/internal/warpcache-ts-sdk/schema.ts | 4 + .../schemas/$commons_CacheAnnotationsMap.ts | 9 ++ .../schemas/$commons_CacheEntry.ts | 31 +++++ .../schemas/$commons_CommitCacheRequest.ts | 13 +- .../schemas/$commons_CommitCacheResponse.ts | 9 ++ .../schemas/$commons_DeleteCacheRequest.ts | 24 ++++ .../schemas/$commons_DeleteCacheResponse.ts | 6 + .../$commons_GCSDeleteCacheResponse.ts | 8 ++ .../schemas/$commons_GetCacheRequest.ts | 30 +++++ .../schemas/$commons_GetCacheResponse.ts | 6 + .../schemas/$commons_ReserveCacheRequest.ts | 13 ++ .../schemas/$commons_ReserveCacheResponse.ts | 3 + .../schemas/$commons_S3GetCacheResponse.ts | 3 + packages/warp-cache/src/test.ts | 54 +++++--- 33 files changed, 701 insertions(+), 146 deletions(-) create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-cache-entry.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-delete-cache-request.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-gcsdelete-cache-response.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-get-cache-request.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CacheAnnotationsMap.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CacheEntry.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_DeleteCacheRequest.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GetCacheRequest.ts diff --git a/packages/warp-cache/package.json b/packages/warp-cache/package.json index f9da2e19..bd498fbe 100644 --- a/packages/warp-cache/package.json +++ b/packages/warp-cache/package.json @@ -1,6 +1,6 @@ { "name": "github-actions.warp-cache", - "version": "0.3.0", + "version": "1.0.0", "preview": true, "description": "Github action to use WarpBuild's in-house cache offering", "keywords": [ diff --git a/packages/warp-cache/src/cache.ts b/packages/warp-cache/src/cache.ts index 3a537bf7..562c592e 100644 --- a/packages/warp-cache/src/cache.ts +++ b/packages/warp-cache/src/cache.ts @@ -66,6 +66,7 @@ export function isFeatureAvailable(): boolean { * @param restoreKeys an optional ordered list of keys to use for restoring the cache if no cache hit occurred for key * @param downloadOptions cache download options * @param enableCrossOsArchive an optional boolean enabled to restore on windows any cache created on any platform + * @param enableCrossArchArchive an optional boolean enabled to restore cache created on any arch * @returns string returns the key for the cache hit, otherwise returns undefined */ export async function restoreCache( @@ -73,22 +74,23 @@ export async function restoreCache( primaryKey: string, restoreKeys?: string[], options?: DownloadOptions, - enableCrossOsArchive = false + enableCrossOsArchive = false, + enableCrossArchArchive = false ): Promise { checkPaths(paths) + checkKey(primaryKey) restoreKeys = restoreKeys ?? [] - const keys = [primaryKey, ...restoreKeys] - core.debug('Resolved Keys:') - core.debug(JSON.stringify(keys)) + core.debug('Resolved Restore Keys:') + core.debug(JSON.stringify(restoreKeys)) - if (keys.length > 10) { + if (restoreKeys.length > 9) { throw new ValidationError( `Key Validation Error: Keys are limited to a maximum of 10.` ) } - for (const key of keys) { + for (const key of restoreKeys) { checkKey(key) } @@ -96,10 +98,16 @@ export async function restoreCache( let archivePath = '' try { // path are needed to compute version - const cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, { - compressionMethod, - enableCrossOsArchive - }) + const cacheEntry = await cacheHttpClient.getCacheEntry( + primaryKey, + restoreKeys, + paths, + { + compressionMethod, + enableCrossOsArchive, + enableCrossArchArchive + } + ) if (!cacheEntry) { // Internal Error @@ -205,13 +213,14 @@ export async function restoreCache( * @param paths a list of file paths to be cached * @param key an explicit key for restoring the cache * @param enableCrossOsArchive an optional boolean enabled to save cache on windows which could be restored on any platform - * @param options cache upload options - * @returns number returns cacheId if the cache was saved successfully and throws an error if save fails + * @param enableCrossArchArchive an optional boolean enabled to save cache on any arch which could be restored on any arch + * @returns string returns cacheId if the cache was saved successfully and throws an error if save fails */ export async function saveCache( paths: string[], key: string, - enableCrossOsArchive = false + enableCrossOsArchive = false, + enableCrossArchArchive = false ): Promise { checkPaths(paths) checkKey(key) @@ -254,6 +263,13 @@ export async function saveCache( ) } + const cacheVersion = cacheHttpClient.getCacheVersion( + paths, + compressionMethod, + enableCrossOsArchive, + enableCrossArchArchive + ) + core.debug('Reserving Cache') // Calculate number of chunks required. This is only required if backend is S3 as Google Cloud SDK will do it for us const uploadOptions = getUploadOptions() @@ -262,11 +278,7 @@ export async function saveCache( const reserveCacheResponse = await cacheHttpClient.reserveCache( key, numberOfChunks, - { - compressionMethod, - enableCrossOsArchive, - cacheSize: archiveFileSize - } + cacheVersion ) if (reserveCacheResponse?.statusCode === 400) { @@ -278,12 +290,6 @@ export async function saveCache( ) } - const cacheVersion = cacheHttpClient.getCacheVersion( - paths, - compressionMethod, - enableCrossOsArchive - ) - switch (reserveCacheResponse.result?.provider) { case 's3': core.debug(`Saving Cache to S3`) @@ -341,18 +347,30 @@ export async function saveCache( /** * Deletes an entire cache by cache key. - * @param keys The cache keys + * @param key The cache keys */ -export async function deleteCache(keys: string[]): Promise { - for (const key of keys) { - checkKey(key) - } +export async function deleteCache( + paths: string[], + key: string, + enableCrossOsArchive = false, + enableCrossArchArchive = false +): Promise { + checkKey(key) core.debug('Deleting Cache') - core.debug(`Cache Keys: ${keys}`) + core.debug(`Cache Key: ${key}`) + + const compressionMethod = await utils.getCompressionMethod() + + const cacheVersion = cacheHttpClient.getCacheVersion( + paths, + compressionMethod, + enableCrossOsArchive, + enableCrossArchArchive + ) try { - await cacheHttpClient.deleteCache(keys) + await cacheHttpClient.deleteCache(key, cacheVersion) } catch (error) { core.warning(`Failed to delete cache: ${error}`) } diff --git a/packages/warp-cache/src/internal/cacheHttpClient.ts b/packages/warp-cache/src/internal/cacheHttpClient.ts index 77140e58..2b545ece 100644 --- a/packages/warp-cache/src/internal/cacheHttpClient.ts +++ b/packages/warp-cache/src/internal/cacheHttpClient.ts @@ -23,11 +23,14 @@ import {Storage} from '@google-cloud/storage' import { CommonsCommitCacheRequest, CommonsCommitCacheResponse, + CommonsDeleteCacheResponse, CommonsGetCacheResponse, CommonsReserveCacheRequest, CommonsReserveCacheResponse } from './warpcache-ts-sdk' import {multiPartUploadToGCS, uploadFileToS3} from './uploadUtils' +import {CommonsGetCacheRequest} from './warpcache-ts-sdk/models/commons-get-cache-request' +import {CommonsDeleteCacheRequest} from './warpcache-ts-sdk/models/commons-delete-cache-request' const versionSalt = '1.0' @@ -47,6 +50,16 @@ function createAcceptHeader(type: string, apiVersion: string): string { return `${type};api-version=${apiVersion}` } +function getVCSRepository(): string { + const vcsRepository = process.env['GITHUB_REPOSITORY'] ?? '' + return vcsRepository +} + +function getVCSRef(): string { + const vcsBranch = process.env['GITHUB_REF'] ?? '' + return vcsBranch +} + function getRequestOptions(): RequestOptions { const requestOptions: RequestOptions = { headers: { @@ -71,7 +84,8 @@ function createHttpClient(): HttpClient { export function getCacheVersion( paths: string[], compressionMethod?: CompressionMethod, - enableCrossOsArchive = false + enableCrossOsArchive = false, + enableCrossArchArchive = false ): string { const components = paths @@ -86,6 +100,11 @@ export function getCacheVersion( components.push('windows-only') } + // Add architecture to cache version + if (!enableCrossArchArchive) { + components.push(process.arch) + } + // Add salt to cache version to support breaking changes in cache entry components.push(versionSalt) @@ -93,7 +112,8 @@ export function getCacheVersion( } export async function getCacheEntry( - keys: string[], + key: string, + restoreKeys: string[], paths: string[], options?: InternalCacheOptions ): Promise { @@ -101,14 +121,23 @@ export async function getCacheEntry( const version = getCacheVersion( paths, options?.compressionMethod, - options?.enableCrossOsArchive + options?.enableCrossOsArchive, + options?.enableCrossArchArchive ) - const resource = `cache?keys=${encodeURIComponent( - keys.join(',') - )}&version=${version}` + + const getCacheRequest: CommonsGetCacheRequest = { + cache_key: key, + restore_keys: restoreKeys, + cache_version: version, + vcs_repository: getVCSRepository(), + vcs_ref: getVCSRef() + } const response = await retryTypedResponse('getCacheEntry', async () => - httpClient.getJson(getCacheApiUrl(resource)) + httpClient.postJson( + getCacheApiUrl('cache/get'), + getCacheRequest + ) ) if (response.statusCode === 204) { @@ -190,14 +219,17 @@ export function downloadCacheStreaming( export async function reserveCache( cacheKey: string, numberOfChunks: number, - options?: InternalCacheOptions + cacheVersion: string ): Promise> { const httpClient = createHttpClient() const reserveCacheRequest: CommonsReserveCacheRequest = { cache_key: cacheKey, + cache_version: cacheVersion, number_of_chunks: numberOfChunks, - content_type: 'application/zstd' + content_type: 'application/zstd', + vcs_repository: getVCSRepository(), + vcs_ref: getVCSRef() } const response = await retryTypedResponse('reserveCache', async () => httpClient.postJson( @@ -227,8 +259,9 @@ async function commitCache( upload_key: uploadKey, upload_id: uploadID, parts: parts, - os: process.env['RUNNER_OS'] ?? 'Linux', - vcs_type: 'github' + vcs_type: 'github', + vcs_repository: getVCSRepository(), + vcs_ref: getVCSRef() } return await retryTypedResponse('commitCache', async () => httpClient.postJson( @@ -340,13 +373,24 @@ export async function saveCache( return cacheKeyResponse } -export async function deleteCache(keys: string[]) { +export async function deleteCache(cacheKey: string, cacheVersion: string) { const httpClient = createHttpClient() - const resource = `cache?keys=${encodeURIComponent(keys.join(','))}` - const response = await httpClient.del(getCacheApiUrl(resource)) - if (!isSuccessStatusCode(response.message.statusCode)) { - throw new Error( - `Cache service responded with ${response.message.statusCode}` + + const deleteCacheRequest: CommonsDeleteCacheRequest = { + cache_key: cacheKey, + cache_version: cacheVersion, + vcs_repository: getVCSRepository(), + vcs_ref: getVCSRef() + } + + const response = await retryTypedResponse('deleteCacheEntry', async () => + httpClient.postJson( + getCacheApiUrl('cache/delete'), + deleteCacheRequest ) + ) + + if (!isSuccessStatusCode(response.statusCode)) { + throw new Error(`Cache service responded with ${response.statusCode}`) } } diff --git a/packages/warp-cache/src/internal/contracts.d.ts b/packages/warp-cache/src/internal/contracts.d.ts index e4c3f035..1a098a9d 100644 --- a/packages/warp-cache/src/internal/contracts.d.ts +++ b/packages/warp-cache/src/internal/contracts.d.ts @@ -9,6 +9,7 @@ export interface ITypedResponseWithError extends TypedResponse { export interface InternalCacheOptions { compressionMethod?: CompressionMethod enableCrossOsArchive?: boolean + enableCrossArchArchive?: boolean cacheSize?: number } diff --git a/packages/warp-cache/src/internal/requestUtils.ts b/packages/warp-cache/src/internal/requestUtils.ts index 043c8a7c..4a1dd782 100644 --- a/packages/warp-cache/src/internal/requestUtils.ts +++ b/packages/warp-cache/src/internal/requestUtils.ts @@ -111,7 +111,7 @@ export async function retryTypedResponse( if (error instanceof HttpClientError) { return { statusCode: error.statusCode, - result: null, + result: error.result ?? null, headers: {}, error } diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/.openapi-generator/FILES b/packages/warp-cache/src/internal/warpcache-ts-sdk/.openapi-generator/FILES index c77d09e9..b3e971af 100644 --- a/packages/warp-cache/src/internal/warpcache-ts-sdk/.openapi-generator/FILES +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/.openapi-generator/FILES @@ -9,12 +9,16 @@ common.ts configuration.ts git_push.sh index.ts +models/commons-cache-entry.ts models/commons-commit-cache-request.ts models/commons-commit-cache-response.ts +models/commons-delete-cache-request.ts models/commons-delete-cache-response.ts models/commons-gcscommit-cache-response.ts +models/commons-gcsdelete-cache-response.ts models/commons-gcsget-cache-reponse.ts models/commons-gcsreserve-cache-response.ts +models/commons-get-cache-request.ts models/commons-get-cache-response.ts models/commons-reserve-cache-request.ts models/commons-reserve-cache-response.ts diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/api/default-api.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/api/default-api.ts index 7a233cb2..7c11bc4e 100644 --- a/packages/warp-cache/src/internal/warpcache-ts-sdk/api/default-api.ts +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/api/default-api.ts @@ -26,8 +26,12 @@ import { CommonsCommitCacheRequest } from '../models'; // @ts-ignore import { CommonsCommitCacheResponse } from '../models'; // @ts-ignore +import { CommonsDeleteCacheRequest } from '../models'; +// @ts-ignore import { CommonsDeleteCacheResponse } from '../models'; // @ts-ignore +import { CommonsGetCacheRequest } from '../models'; +// @ts-ignore import { CommonsGetCacheResponse } from '../models'; // @ts-ignore import { CommonsReserveCacheRequest } from '../models'; @@ -110,13 +114,13 @@ export const DefaultApiAxiosParamCreator = function (configuration?: Configurati /** * delete cache * @summary delete cache - * @param {string} keys cache keys + * @param {CommonsDeleteCacheRequest} body Delete Cache Request Body * @param {*} [options] Override http request option. * @throws {RequiredError} */ - v1CacheDeleteDelete: async (keys: string, options: RawAxiosRequestConfig = {}): Promise => { - // verify required parameter 'keys' is not null or undefined - assertParamExists('v1CacheDeleteDelete', 'keys', keys) + v1CacheDeletePost: async (body: CommonsDeleteCacheRequest, options: RawAxiosRequestConfig = {}): Promise => { + // verify required parameter 'body' is not null or undefined + assertParamExists('v1CacheDeletePost', 'body', body) const localVarPath = `/v1/cache/delete`; // use dummy base URL string because the URL constructor only accepts absolute URLs. const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); @@ -125,19 +129,18 @@ export const DefaultApiAxiosParamCreator = function (configuration?: Configurati baseOptions = configuration.baseOptions; } - const localVarRequestOptions = { method: 'DELETE', ...baseOptions, ...options}; + const localVarRequestOptions = { method: 'POST', ...baseOptions, ...options}; const localVarHeaderParameter = {} as any; const localVarQueryParameter = {} as any; - if (keys !== undefined) { - localVarQueryParameter['keys'] = keys; - } - + localVarHeaderParameter['Content-Type'] = 'application/json'; + setSearchParams(localVarUrlObj, localVarQueryParameter); let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; + localVarRequestOptions.data = serializeDataIfNeeded(body, localVarRequestOptions, configuration) return { url: toPathString(localVarUrlObj), @@ -147,17 +150,14 @@ export const DefaultApiAxiosParamCreator = function (configuration?: Configurati /** * get cache * @summary get cache - * @param {string} keys cache keys - * @param {string} version cache version + * @param {CommonsGetCacheRequest} body Get Cache Request Body * @param {*} [options] Override http request option. * @throws {RequiredError} */ - v1CacheGet: async (keys: string, version: string, options: RawAxiosRequestConfig = {}): Promise => { - // verify required parameter 'keys' is not null or undefined - assertParamExists('v1CacheGet', 'keys', keys) - // verify required parameter 'version' is not null or undefined - assertParamExists('v1CacheGet', 'version', version) - const localVarPath = `/v1/cache`; + v1CacheGetPost: async (body: CommonsGetCacheRequest, options: RawAxiosRequestConfig = {}): Promise => { + // verify required parameter 'body' is not null or undefined + assertParamExists('v1CacheGetPost', 'body', body) + const localVarPath = `/v1/cache/get`; // use dummy base URL string because the URL constructor only accepts absolute URLs. const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); let baseOptions; @@ -165,23 +165,18 @@ export const DefaultApiAxiosParamCreator = function (configuration?: Configurati baseOptions = configuration.baseOptions; } - const localVarRequestOptions = { method: 'GET', ...baseOptions, ...options}; + const localVarRequestOptions = { method: 'POST', ...baseOptions, ...options}; const localVarHeaderParameter = {} as any; const localVarQueryParameter = {} as any; - if (keys !== undefined) { - localVarQueryParameter['keys'] = keys; - } - - if (version !== undefined) { - localVarQueryParameter['version'] = version; - } - + localVarHeaderParameter['Content-Type'] = 'application/json'; + setSearchParams(localVarUrlObj, localVarQueryParameter); let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; + localVarRequestOptions.data = serializeDataIfNeeded(body, localVarRequestOptions, configuration) return { url: toPathString(localVarUrlObj), @@ -262,28 +257,27 @@ export const DefaultApiFp = function(configuration?: Configuration) { /** * delete cache * @summary delete cache - * @param {string} keys cache keys + * @param {CommonsDeleteCacheRequest} body Delete Cache Request Body * @param {*} [options] Override http request option. * @throws {RequiredError} */ - async v1CacheDeleteDelete(keys: string, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { - const localVarAxiosArgs = await localVarAxiosParamCreator.v1CacheDeleteDelete(keys, options); + async v1CacheDeletePost(body: CommonsDeleteCacheRequest, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.v1CacheDeletePost(body, options); const localVarOperationServerIndex = configuration?.serverIndex ?? 0; - const localVarOperationServerBasePath = operationServerMap['DefaultApi.v1CacheDeleteDelete']?.[localVarOperationServerIndex]?.url; + const localVarOperationServerBasePath = operationServerMap['DefaultApi.v1CacheDeletePost']?.[localVarOperationServerIndex]?.url; return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); }, /** * get cache * @summary get cache - * @param {string} keys cache keys - * @param {string} version cache version + * @param {CommonsGetCacheRequest} body Get Cache Request Body * @param {*} [options] Override http request option. * @throws {RequiredError} */ - async v1CacheGet(keys: string, version: string, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { - const localVarAxiosArgs = await localVarAxiosParamCreator.v1CacheGet(keys, version, options); + async v1CacheGetPost(body: CommonsGetCacheRequest, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.v1CacheGetPost(body, options); const localVarOperationServerIndex = configuration?.serverIndex ?? 0; - const localVarOperationServerBasePath = operationServerMap['DefaultApi.v1CacheGet']?.[localVarOperationServerIndex]?.url; + const localVarOperationServerBasePath = operationServerMap['DefaultApi.v1CacheGetPost']?.[localVarOperationServerIndex]?.url; return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); }, /** @@ -331,22 +325,22 @@ export const DefaultApiFactory = function (configuration?: Configuration, basePa /** * delete cache * @summary delete cache - * @param {DefaultApiV1CacheDeleteDeleteRequest} requestParameters Request parameters. + * @param {DefaultApiV1CacheDeletePostRequest} requestParameters Request parameters. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - v1CacheDeleteDelete(requestParameters: DefaultApiV1CacheDeleteDeleteRequest, options?: RawAxiosRequestConfig): AxiosPromise { - return localVarFp.v1CacheDeleteDelete(requestParameters.keys, options).then((request) => request(axios, basePath)); + v1CacheDeletePost(requestParameters: DefaultApiV1CacheDeletePostRequest, options?: RawAxiosRequestConfig): AxiosPromise { + return localVarFp.v1CacheDeletePost(requestParameters.body, options).then((request) => request(axios, basePath)); }, /** * get cache * @summary get cache - * @param {DefaultApiV1CacheGetRequest} requestParameters Request parameters. + * @param {DefaultApiV1CacheGetPostRequest} requestParameters Request parameters. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - v1CacheGet(requestParameters: DefaultApiV1CacheGetRequest, options?: RawAxiosRequestConfig): AxiosPromise { - return localVarFp.v1CacheGet(requestParameters.keys, requestParameters.version, options).then((request) => request(axios, basePath)); + v1CacheGetPost(requestParameters: DefaultApiV1CacheGetPostRequest, options?: RawAxiosRequestConfig): AxiosPromise { + return localVarFp.v1CacheGetPost(requestParameters.body, options).then((request) => request(axios, basePath)); }, /** * reserve cache @@ -376,38 +370,31 @@ export interface DefaultApiV1CacheCommitPostRequest { } /** - * Request parameters for v1CacheDeleteDelete operation in DefaultApi. + * Request parameters for v1CacheDeletePost operation in DefaultApi. * @export - * @interface DefaultApiV1CacheDeleteDeleteRequest + * @interface DefaultApiV1CacheDeletePostRequest */ -export interface DefaultApiV1CacheDeleteDeleteRequest { +export interface DefaultApiV1CacheDeletePostRequest { /** - * cache keys - * @type {string} - * @memberof DefaultApiV1CacheDeleteDelete + * Delete Cache Request Body + * @type {CommonsDeleteCacheRequest} + * @memberof DefaultApiV1CacheDeletePost */ - readonly keys: string + readonly body: CommonsDeleteCacheRequest } /** - * Request parameters for v1CacheGet operation in DefaultApi. + * Request parameters for v1CacheGetPost operation in DefaultApi. * @export - * @interface DefaultApiV1CacheGetRequest + * @interface DefaultApiV1CacheGetPostRequest */ -export interface DefaultApiV1CacheGetRequest { +export interface DefaultApiV1CacheGetPostRequest { /** - * cache keys - * @type {string} - * @memberof DefaultApiV1CacheGet + * Get Cache Request Body + * @type {CommonsGetCacheRequest} + * @memberof DefaultApiV1CacheGetPost */ - readonly keys: string - - /** - * cache version - * @type {string} - * @memberof DefaultApiV1CacheGet - */ - readonly version: string + readonly body: CommonsGetCacheRequest } /** @@ -457,25 +444,25 @@ export class DefaultApi extends BaseAPI { /** * delete cache * @summary delete cache - * @param {DefaultApiV1CacheDeleteDeleteRequest} requestParameters Request parameters. + * @param {DefaultApiV1CacheDeletePostRequest} requestParameters Request parameters. * @param {*} [options] Override http request option. * @throws {RequiredError} * @memberof DefaultApi */ - public v1CacheDeleteDelete(requestParameters: DefaultApiV1CacheDeleteDeleteRequest, options?: RawAxiosRequestConfig) { - return DefaultApiFp(this.configuration).v1CacheDeleteDelete(requestParameters.keys, options).then((request) => request(this.axios, this.basePath)); + public v1CacheDeletePost(requestParameters: DefaultApiV1CacheDeletePostRequest, options?: RawAxiosRequestConfig) { + return DefaultApiFp(this.configuration).v1CacheDeletePost(requestParameters.body, options).then((request) => request(this.axios, this.basePath)); } /** * get cache * @summary get cache - * @param {DefaultApiV1CacheGetRequest} requestParameters Request parameters. + * @param {DefaultApiV1CacheGetPostRequest} requestParameters Request parameters. * @param {*} [options] Override http request option. * @throws {RequiredError} * @memberof DefaultApi */ - public v1CacheGet(requestParameters: DefaultApiV1CacheGetRequest, options?: RawAxiosRequestConfig) { - return DefaultApiFp(this.configuration).v1CacheGet(requestParameters.keys, requestParameters.version, options).then((request) => request(this.axios, this.basePath)); + public v1CacheGetPost(requestParameters: DefaultApiV1CacheGetPostRequest, options?: RawAxiosRequestConfig) { + return DefaultApiFp(this.configuration).v1CacheGetPost(requestParameters.body, options).then((request) => request(this.axios, this.basePath)); } /** diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-cache-entry.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-cache-entry.ts new file mode 100644 index 00000000..520b23d2 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-cache-entry.ts @@ -0,0 +1,72 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + + +/** + * + * @export + * @interface CommonsCacheEntry + */ +export interface CommonsCacheEntry { + /** + * + * @type {string} + * @memberof CommonsCacheEntry + */ + 'cache_key'?: string; + /** + * + * @type {string} + * @memberof CommonsCacheEntry + */ + 'cache_version'?: string; + /** + * + * @type {string} + * @memberof CommonsCacheEntry + */ + 'created_at'?: string; + /** + * + * @type {string} + * @memberof CommonsCacheEntry + */ + 'id'?: string; + /** + * + * @type {string} + * @memberof CommonsCacheEntry + */ + 'organization_id'?: string; + /** + * + * @type {string} + * @memberof CommonsCacheEntry + */ + 'updated_at'?: string; + /** + * + * @type {string} + * @memberof CommonsCacheEntry + */ + 'vcs_organization_name'?: string; + /** + * + * @type {string} + * @memberof CommonsCacheEntry + */ + 'vcs_repository_name'?: string; +} + diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-commit-cache-request.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-commit-cache-request.ts index 789c1274..1be9a8d0 100644 --- a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-commit-cache-request.ts +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-commit-cache-request.ts @@ -23,6 +23,12 @@ import { TypesCompletedPart } from './types-completed-part'; * @interface CommonsCommitCacheRequest */ export interface CommonsCommitCacheRequest { + /** + * + * @type {{ [key: string]: string; }} + * @memberof CommonsCommitCacheRequest + */ + 'annotations'?: { [key: string]: string; }; /** * * @type {string} @@ -35,12 +41,6 @@ export interface CommonsCommitCacheRequest { * @memberof CommonsCommitCacheRequest */ 'cache_version': string; - /** - * - * @type {string} - * @memberof CommonsCommitCacheRequest - */ - 'os': string; /** * * @type {Array} @@ -59,6 +59,18 @@ export interface CommonsCommitCacheRequest { * @memberof CommonsCommitCacheRequest */ 'upload_key'?: string; + /** + * VCSRef is the ref of the repository in vcs for which cache is being used. This can be a branch, git tag, or pull request ref. + * @type {string} + * @memberof CommonsCommitCacheRequest + */ + 'vcs_ref'?: string; + /** + * VCSRepository is the repository name in vcs. It can be of the format / or . While saving the entry, / will be trimmed if passed. + * @type {string} + * @memberof CommonsCommitCacheRequest + */ + 'vcs_repository'?: string; /** * * @type {string} diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-commit-cache-response.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-commit-cache-response.ts index b3969eed..29e295c1 100644 --- a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-commit-cache-response.ts +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-commit-cache-response.ts @@ -13,6 +13,9 @@ */ +// May contain unused imports in some cases +// @ts-ignore +import { CommonsCacheEntry } from './commons-cache-entry'; // May contain unused imports in some cases // @ts-ignore import { CommonsGCSCommitCacheResponse } from './commons-gcscommit-cache-response'; @@ -26,6 +29,18 @@ import { CommonsS3CommitCacheResponse } from './commons-s3-commit-cache-response * @interface CommonsCommitCacheResponse */ export interface CommonsCommitCacheResponse { + /** + * + * @type {{ [key: string]: string; }} + * @memberof CommonsCommitCacheResponse + */ + 'annotations'?: { [key: string]: string; }; + /** + * + * @type {CommonsCacheEntry} + * @memberof CommonsCommitCacheResponse + */ + 'cache_entry'?: CommonsCacheEntry; /** * * @type {CommonsGCSCommitCacheResponse} @@ -44,5 +59,11 @@ export interface CommonsCommitCacheResponse { * @memberof CommonsCommitCacheResponse */ 's3'?: CommonsS3CommitCacheResponse; + /** + * VCSRepository is the repository name in vcs. It can be of the format / or . While saving the entry, / will be trimmed if passed. + * @type {string} + * @memberof CommonsCommitCacheResponse + */ + 'vcs_repository'?: string; } diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-delete-cache-request.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-delete-cache-request.ts new file mode 100644 index 00000000..c736959c --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-delete-cache-request.ts @@ -0,0 +1,54 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + + +/** + * + * @export + * @interface CommonsDeleteCacheRequest + */ +export interface CommonsDeleteCacheRequest { + /** + * + * @type {{ [key: string]: string; }} + * @memberof CommonsDeleteCacheRequest + */ + 'annotations'?: { [key: string]: string; }; + /** + * + * @type {string} + * @memberof CommonsDeleteCacheRequest + */ + 'cache_key': string; + /** + * + * @type {string} + * @memberof CommonsDeleteCacheRequest + */ + 'cache_version': string; + /** + * VCSRef is the ref of the repository in vcs for which cache is being used. This can be a branch, git tag, or pull request ref. + * @type {string} + * @memberof CommonsDeleteCacheRequest + */ + 'vcs_ref'?: string; + /** + * VCSRepository is the repository name in vcs. It can be of the format / or . While saving the entry, / will be trimmed if passed. + * @type {string} + * @memberof CommonsDeleteCacheRequest + */ + 'vcs_repository'?: string; +} + diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-delete-cache-response.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-delete-cache-response.ts index 8d03a281..2c090c55 100644 --- a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-delete-cache-response.ts +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-delete-cache-response.ts @@ -13,6 +13,12 @@ */ +// May contain unused imports in some cases +// @ts-ignore +import { CommonsCacheEntry } from './commons-cache-entry'; +// May contain unused imports in some cases +// @ts-ignore +import { CommonsGCSDeleteCacheResponse } from './commons-gcsdelete-cache-response'; // May contain unused imports in some cases // @ts-ignore import { CommonsS3DeleteCacheResponse } from './commons-s3-delete-cache-response'; @@ -25,10 +31,22 @@ import { CommonsS3DeleteCacheResponse } from './commons-s3-delete-cache-response export interface CommonsDeleteCacheResponse { /** * - * @type {object} + * @type {{ [key: string]: string; }} * @memberof CommonsDeleteCacheResponse */ - 'gcs'?: object; + 'annotations'?: { [key: string]: string; }; + /** + * + * @type {CommonsCacheEntry} + * @memberof CommonsDeleteCacheResponse + */ + 'cache_entry'?: CommonsCacheEntry; + /** + * + * @type {CommonsGCSDeleteCacheResponse} + * @memberof CommonsDeleteCacheResponse + */ + 'gcs'?: CommonsGCSDeleteCacheResponse; /** * * @type {string} diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-gcsdelete-cache-response.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-gcsdelete-cache-response.ts new file mode 100644 index 00000000..36dd7938 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-gcsdelete-cache-response.ts @@ -0,0 +1,36 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + + +/** + * + * @export + * @interface CommonsGCSDeleteCacheResponse + */ +export interface CommonsGCSDeleteCacheResponse { + /** + * + * @type {string} + * @memberof CommonsGCSDeleteCacheResponse + */ + 'cache_key': string; + /** + * + * @type {string} + * @memberof CommonsGCSDeleteCacheResponse + */ + 'cache_version': string; +} + diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-get-cache-request.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-get-cache-request.ts new file mode 100644 index 00000000..978de0be --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-get-cache-request.ts @@ -0,0 +1,60 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + + +/** + * + * @export + * @interface CommonsGetCacheRequest + */ +export interface CommonsGetCacheRequest { + /** + * + * @type {{ [key: string]: string; }} + * @memberof CommonsGetCacheRequest + */ + 'annotations'?: { [key: string]: string; }; + /** + * + * @type {string} + * @memberof CommonsGetCacheRequest + */ + 'cache_key': string; + /** + * + * @type {string} + * @memberof CommonsGetCacheRequest + */ + 'cache_version': string; + /** + * + * @type {Array} + * @memberof CommonsGetCacheRequest + */ + 'restore_keys'?: Array; + /** + * VCSRef is the ref of the repository in vcs for which cache is being used. This can be a branch, git tag, or pull request ref. + * @type {string} + * @memberof CommonsGetCacheRequest + */ + 'vcs_ref'?: string; + /** + * VCSRepository is the repository name in vcs. It can be of the format / or . While saving the entry, / will be trimmed if passed. + * @type {string} + * @memberof CommonsGetCacheRequest + */ + 'vcs_repository'?: string; +} + diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-get-cache-response.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-get-cache-response.ts index 4ced0787..af8de777 100644 --- a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-get-cache-response.ts +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-get-cache-response.ts @@ -13,6 +13,9 @@ */ +// May contain unused imports in some cases +// @ts-ignore +import { CommonsCacheEntry } from './commons-cache-entry'; // May contain unused imports in some cases // @ts-ignore import { CommonsGCSGetCacheReponse } from './commons-gcsget-cache-reponse'; @@ -26,6 +29,18 @@ import { CommonsS3GetCacheResponse } from './commons-s3-get-cache-response'; * @interface CommonsGetCacheResponse */ export interface CommonsGetCacheResponse { + /** + * + * @type {{ [key: string]: string; }} + * @memberof CommonsGetCacheResponse + */ + 'annotations'?: { [key: string]: string; }; + /** + * + * @type {CommonsCacheEntry} + * @memberof CommonsGetCacheResponse + */ + 'cache_entry'?: CommonsCacheEntry; /** * * @type {CommonsGCSGetCacheReponse} diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-reserve-cache-request.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-reserve-cache-request.ts index 19be9872..3ef82167 100644 --- a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-reserve-cache-request.ts +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-reserve-cache-request.ts @@ -20,12 +20,24 @@ * @interface CommonsReserveCacheRequest */ export interface CommonsReserveCacheRequest { + /** + * + * @type {{ [key: string]: string; }} + * @memberof CommonsReserveCacheRequest + */ + 'annotations'?: { [key: string]: string; }; /** * * @type {string} * @memberof CommonsReserveCacheRequest */ 'cache_key': string; + /** + * + * @type {string} + * @memberof CommonsReserveCacheRequest + */ + 'cache_version': string; /** * ContentType contains the content type of the cache. * This is not supported for GCS cache. When passed this will be ignored. * * @type {string} @@ -38,5 +50,17 @@ export interface CommonsReserveCacheRequest { * @memberof CommonsReserveCacheRequest */ 'number_of_chunks'?: number; + /** + * VCSRef is the ref of the repository in vcs for which cache is being used. This can be a branch, git tag, or pull request ref. + * @type {string} + * @memberof CommonsReserveCacheRequest + */ + 'vcs_ref'?: string; + /** + * VCSRepository is the repository name in vcs. It can be of the format / or . While saving the entry, / will be trimmed if passed. + * @type {string} + * @memberof CommonsReserveCacheRequest + */ + 'vcs_repository'?: string; } diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-reserve-cache-response.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-reserve-cache-response.ts index ed1ae9c5..d24499a9 100644 --- a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-reserve-cache-response.ts +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-reserve-cache-response.ts @@ -26,6 +26,12 @@ import { CommonsS3ReserveCacheResponse } from './commons-s3-reserve-cache-respon * @interface CommonsReserveCacheResponse */ export interface CommonsReserveCacheResponse { + /** + * + * @type {{ [key: string]: string; }} + * @memberof CommonsReserveCacheResponse + */ + 'annotations'?: { [key: string]: string; }; /** * * @type {CommonsGCSReserveCacheResponse} diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-s3-get-cache-response.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-s3-get-cache-response.ts index 7f8c0385..9dabfcec 100644 --- a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-s3-get-cache-response.ts +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-s3-get-cache-response.ts @@ -20,6 +20,12 @@ * @interface CommonsS3GetCacheResponse */ export interface CommonsS3GetCacheResponse { + /** + * + * @type {{ [key: string]: string; }} + * @memberof CommonsS3GetCacheResponse + */ + 'annotations'?: { [key: string]: string; }; /** * * @type {string} diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/index.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/index.ts index 10725161..dc9cc139 100644 --- a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/index.ts +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/index.ts @@ -1,9 +1,13 @@ +export * from './commons-cache-entry'; export * from './commons-commit-cache-request'; export * from './commons-commit-cache-response'; +export * from './commons-delete-cache-request'; export * from './commons-delete-cache-response'; export * from './commons-gcscommit-cache-response'; +export * from './commons-gcsdelete-cache-response'; export * from './commons-gcsget-cache-reponse'; export * from './commons-gcsreserve-cache-response'; +export * from './commons-get-cache-request'; export * from './commons-get-cache-response'; export * from './commons-reserve-cache-request'; export * from './commons-reserve-cache-response'; diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schema.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schema.ts index a03165f1..0c653785 100644 --- a/packages/warp-cache/src/internal/warpcache-ts-sdk/schema.ts +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schema.ts @@ -1,13 +1,17 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ +export { $commons_CacheAnnotationsMap } from './schemas/$commons_CacheAnnotationsMap'; +export { $commons_CacheEntry } from './schemas/$commons_CacheEntry'; export { $commons_CommitCacheRequest } from './schemas/$commons_CommitCacheRequest'; export { $commons_CommitCacheResponse } from './schemas/$commons_CommitCacheResponse'; +export { $commons_DeleteCacheRequest } from './schemas/$commons_DeleteCacheRequest'; export { $commons_DeleteCacheResponse } from './schemas/$commons_DeleteCacheResponse'; export { $commons_GCSCommitCacheResponse } from './schemas/$commons_GCSCommitCacheResponse'; export { $commons_GCSDeleteCacheResponse } from './schemas/$commons_GCSDeleteCacheResponse'; export { $commons_GCSGetCacheReponse } from './schemas/$commons_GCSGetCacheReponse'; export { $commons_GCSReserveCacheResponse } from './schemas/$commons_GCSReserveCacheResponse'; +export { $commons_GetCacheRequest } from './schemas/$commons_GetCacheRequest'; export { $commons_GetCacheResponse } from './schemas/$commons_GetCacheResponse'; export { $commons_ReserveCacheRequest } from './schemas/$commons_ReserveCacheRequest'; export { $commons_ReserveCacheResponse } from './schemas/$commons_ReserveCacheResponse'; diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CacheAnnotationsMap.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CacheAnnotationsMap.ts new file mode 100644 index 00000000..938fe3ae --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CacheAnnotationsMap.ts @@ -0,0 +1,9 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export const $commons_CacheAnnotationsMap = { + type: 'dictionary', + contains: { + type: 'string', + }, +} as const; \ No newline at end of file diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CacheEntry.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CacheEntry.ts new file mode 100644 index 00000000..3a15516a --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CacheEntry.ts @@ -0,0 +1,31 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export const $commons_CacheEntry = { + properties: { + cache_key: { + type: 'string', + }, + cache_version: { + type: 'string', + }, + created_at: { + type: 'string', + }, + id: { + type: 'string', + }, + organization_id: { + type: 'string', + }, + updated_at: { + type: 'string', + }, + vcs_organization_name: { + type: 'string', + }, + vcs_repository_name: { + type: 'string', + }, + }, +} as const; \ No newline at end of file diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CommitCacheRequest.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CommitCacheRequest.ts index cb58a343..13b89768 100644 --- a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CommitCacheRequest.ts +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CommitCacheRequest.ts @@ -3,6 +3,9 @@ /* eslint-disable */ export const $commons_CommitCacheRequest = { properties: { + annotations: { + type: 'commons_CacheAnnotationsMap', + }, cache_key: { type: 'string', isRequired: true, @@ -11,10 +14,6 @@ export const $commons_CommitCacheRequest = { type: 'string', isRequired: true, }, - os: { - type: 'string', - isRequired: true, - }, parts: { type: 'array', contains: { @@ -28,6 +27,12 @@ export const $commons_CommitCacheRequest = { upload_key: { type: 'string', }, + vcs_ref: { + type: 'string', + }, + vcs_repository: { + type: 'string', + }, vcs_type: { type: 'string', isRequired: true, diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CommitCacheResponse.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CommitCacheResponse.ts index b46f134a..602d4364 100644 --- a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CommitCacheResponse.ts +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CommitCacheResponse.ts @@ -3,6 +3,12 @@ /* eslint-disable */ export const $commons_CommitCacheResponse = { properties: { + annotations: { + type: 'commons_CacheAnnotationsMap', + }, + cache_entry: { + type: 'commons_CacheEntry', + }, gcs: { type: 'commons_GCSCommitCacheResponse', }, @@ -12,5 +18,8 @@ export const $commons_CommitCacheResponse = { s3: { type: 'commons_S3CommitCacheResponse', }, + vcs_repository: { + type: 'string', + }, }, } as const; \ No newline at end of file diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_DeleteCacheRequest.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_DeleteCacheRequest.ts new file mode 100644 index 00000000..0be5db31 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_DeleteCacheRequest.ts @@ -0,0 +1,24 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export const $commons_DeleteCacheRequest = { + properties: { + annotations: { + type: 'commons_CacheAnnotationsMap', + }, + cache_key: { + type: 'string', + isRequired: true, + }, + cache_version: { + type: 'string', + isRequired: true, + }, + vcs_ref: { + type: 'string', + }, + vcs_repository: { + type: 'string', + }, + }, +} as const; \ No newline at end of file diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_DeleteCacheResponse.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_DeleteCacheResponse.ts index d599057e..f2907e5c 100644 --- a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_DeleteCacheResponse.ts +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_DeleteCacheResponse.ts @@ -3,6 +3,12 @@ /* eslint-disable */ export const $commons_DeleteCacheResponse = { properties: { + annotations: { + type: 'commons_CacheAnnotationsMap', + }, + cache_entry: { + type: 'commons_CacheEntry', + }, gcs: { type: 'commons_GCSDeleteCacheResponse', }, diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GCSDeleteCacheResponse.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GCSDeleteCacheResponse.ts index 92377d71..5dd4e43c 100644 --- a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GCSDeleteCacheResponse.ts +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GCSDeleteCacheResponse.ts @@ -3,5 +3,13 @@ /* eslint-disable */ export const $commons_GCSDeleteCacheResponse = { properties: { + cache_key: { + type: 'string', + isRequired: true, + }, + cache_version: { + type: 'string', + isRequired: true, + }, }, } as const; \ No newline at end of file diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GetCacheRequest.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GetCacheRequest.ts new file mode 100644 index 00000000..a5b5fcc4 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GetCacheRequest.ts @@ -0,0 +1,30 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export const $commons_GetCacheRequest = { + properties: { + annotations: { + type: 'commons_CacheAnnotationsMap', + }, + cache_key: { + type: 'string', + isRequired: true, + }, + cache_version: { + type: 'string', + isRequired: true, + }, + restore_keys: { + type: 'array', + contains: { + type: 'string', + }, + }, + vcs_ref: { + type: 'string', + }, + vcs_repository: { + type: 'string', + }, + }, +} as const; \ No newline at end of file diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GetCacheResponse.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GetCacheResponse.ts index c4482770..9b54b333 100644 --- a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GetCacheResponse.ts +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GetCacheResponse.ts @@ -3,6 +3,12 @@ /* eslint-disable */ export const $commons_GetCacheResponse = { properties: { + annotations: { + type: 'commons_CacheAnnotationsMap', + }, + cache_entry: { + type: 'commons_CacheEntry', + }, gcs: { type: 'commons_GCSGetCacheReponse', }, diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_ReserveCacheRequest.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_ReserveCacheRequest.ts index 6b02ef6e..b9f1b463 100644 --- a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_ReserveCacheRequest.ts +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_ReserveCacheRequest.ts @@ -3,15 +3,28 @@ /* eslint-disable */ export const $commons_ReserveCacheRequest = { properties: { + annotations: { + type: 'commons_CacheAnnotationsMap', + }, cache_key: { type: 'string', isRequired: true, }, + cache_version: { + type: 'string', + isRequired: true, + }, content_type: { type: 'string', }, number_of_chunks: { type: 'number', }, + vcs_ref: { + type: 'string', + }, + vcs_repository: { + type: 'string', + }, }, } as const; \ No newline at end of file diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_ReserveCacheResponse.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_ReserveCacheResponse.ts index f1b04e41..57d72176 100644 --- a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_ReserveCacheResponse.ts +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_ReserveCacheResponse.ts @@ -3,6 +3,9 @@ /* eslint-disable */ export const $commons_ReserveCacheResponse = { properties: { + annotations: { + type: 'commons_CacheAnnotationsMap', + }, gcs: { type: 'commons_GCSReserveCacheResponse', }, diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_S3GetCacheResponse.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_S3GetCacheResponse.ts index a23e1d99..0f3d85d2 100644 --- a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_S3GetCacheResponse.ts +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_S3GetCacheResponse.ts @@ -3,6 +3,9 @@ /* eslint-disable */ export const $commons_S3GetCacheResponse = { properties: { + annotations: { + type: 'commons_CacheAnnotationsMap', + }, cache_key: { type: 'string', }, diff --git a/packages/warp-cache/src/test.ts b/packages/warp-cache/src/test.ts index 4d06bc72..53be237b 100644 --- a/packages/warp-cache/src/test.ts +++ b/packages/warp-cache/src/test.ts @@ -1,23 +1,45 @@ import {deleteCache, restoreCache, saveCache} from './cache' +import {getCacheVersion} from './internal/cacheHttpClient' +import {getCompressionMethod} from './internal/cacheUtils' -process.env['WARPBUILD_CACHE_URL'] = 'http://localhost:8002' +process.env['WARPBUILD_CACHE_URL'] = 'https://cache.dev.warpbuild.dev' +// process.env['WARPBUILD_CACHE_URL'] = 'http://localhost:8000' process.env['RUNNER_TEMP'] = '/Users/prajjwal/Repos/warpbuild/playground/tmp_fs' process.env['NODE_DEBUG'] = 'http' -process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] = - 'eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3MTMwNzE5ODgsInJlcG8iOiJiZW5jaG1hcmtzIiwicmVwb093bmVyIjoiV2FycEJ1aWxkcyIsIngtd2FycGJ1aWxkLW9yZ2FuaXphdGlvbi1pZCI6IndmbW4wODBlaWY4cm5pd3EifQ.Wat-RATKl_KB39SF6egch3nF3_dD8hDE3lbl9wm7AyBUs9pUNEDejJtgHO0xQfGvSN-qRTPbJ_glHKPUIRHE3w' +process.env['RUNNER_DEBUG'] = '1' +process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] = '' +process.env['GITHUB_REPOSITORY'] = 'Warpbuilds/backend-cache' +process.env['GITHUB_REF'] = 'refs/heads/main' -saveCache( - ['/Users/prajjwal/Repos/warpbuild/playground/test_fs'], - 'test-fs-local-key', - true -) +// saveCache( +// ['/Users/prajjwal/Repos/warpbuild/playground/test_fs'], +// 'test-fs-local-key', +// true +// ) -restoreCache( - ['/Users/prajjwal/Repos/warpbuild/playground/test_fs'], - 'test-fs-local-key', - [], - {}, - true -) +// saveCache( +// ['/Users/prajjwal/Repos/warpbuild/playground/test_fs'], +// 'test-fs-local-key-2', +// true +// ) -deleteCache(['test-fs-local-key']) +// saveCache( +// ['/Users/prajjwal/Repos/warpbuild/playground/test_fs'], +// 'test-fs-local-key', +// true, +// true +// ) + +// restoreCache( +// ['/Users/prajjwal/Repos/warpbuild/playground/test_fs'], +// 'test-fs-local-key-3', +// ['test-fs'], +// {}, +// true, +// false +// ) + +// deleteCache( +// ['/Users/prajjwal/Repos/warpbuild/playground/test_fs'], +// 'test-fs-local-key' +// ) From a09029945fc25baa9352ebdd1a2563c8553dc392 Mon Sep 17 00:00:00 2001 From: Prajjwal Date: Mon, 15 Apr 2024 11:58:21 +0530 Subject: [PATCH 04/11] adds explicit token auth mechanism --- packages/warp-cache/package-lock.json | 5 +++-- packages/warp-cache/package.json | 3 ++- packages/warp-cache/src/cache.ts | 3 ++- packages/warp-cache/src/internal/cacheHttpClient.ts | 9 +++++++-- packages/warp-cache/src/test.ts | 13 +++++++------ 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/packages/warp-cache/package-lock.json b/packages/warp-cache/package-lock.json index d824dd26..5b357845 100644 --- a/packages/warp-cache/package-lock.json +++ b/packages/warp-cache/package-lock.json @@ -1,12 +1,12 @@ { "name": "github-actions.warp-cache", - "version": "0.3.0", + "version": "1.0.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "github-actions.warp-cache", - "version": "0.3.0", + "version": "1.0.0", "license": "MIT", "dependencies": { "@actions/core": "^1.10.0", @@ -19,6 +19,7 @@ "@azure/storage-blob": "^12.13.0", "@google-cloud/storage": "^7.9.0", "axios": "^1.6.2", + "google-auth-library": "^9.7.0", "semver": "^6.3.1", "uuid": "^3.3.3" }, diff --git a/packages/warp-cache/package.json b/packages/warp-cache/package.json index bd498fbe..092faa98 100644 --- a/packages/warp-cache/package.json +++ b/packages/warp-cache/package.json @@ -1,6 +1,6 @@ { "name": "github-actions.warp-cache", - "version": "1.0.0", + "version": "1.0.1", "preview": true, "description": "Github action to use WarpBuild's in-house cache offering", "keywords": [ @@ -48,6 +48,7 @@ "@azure/storage-blob": "^12.13.0", "@google-cloud/storage": "^7.9.0", "axios": "^1.6.2", + "google-auth-library": "^9.7.0", "semver": "^6.3.1", "uuid": "^3.3.3" }, diff --git a/packages/warp-cache/src/cache.ts b/packages/warp-cache/src/cache.ts index 562c592e..8eee08fb 100644 --- a/packages/warp-cache/src/cache.ts +++ b/packages/warp-cache/src/cache.ts @@ -9,6 +9,7 @@ import { listTar } from './internal/tar' import {DownloadOptions, getUploadOptions} from './options' +import {isSuccessStatusCode} from './internal/requestUtils' export class ValidationError extends Error { constructor(message: string) { @@ -281,7 +282,7 @@ export async function saveCache( cacheVersion ) - if (reserveCacheResponse?.statusCode === 400) { + if (!isSuccessStatusCode(reserveCacheResponse?.statusCode)) { throw new Error( reserveCacheResponse?.error?.message ?? `Cache size of ~${Math.round( diff --git a/packages/warp-cache/src/internal/cacheHttpClient.ts b/packages/warp-cache/src/internal/cacheHttpClient.ts index 2b545ece..8fab972f 100644 --- a/packages/warp-cache/src/internal/cacheHttpClient.ts +++ b/packages/warp-cache/src/internal/cacheHttpClient.ts @@ -31,6 +31,7 @@ import { import {multiPartUploadToGCS, uploadFileToS3} from './uploadUtils' import {CommonsGetCacheRequest} from './warpcache-ts-sdk/models/commons-get-cache-request' import {CommonsDeleteCacheRequest} from './warpcache-ts-sdk/models/commons-delete-cache-request' +import {OAuth2Client} from 'google-auth-library' const versionSalt = '1.0' @@ -206,8 +207,10 @@ export function downloadCacheStreaming( 'Unable to download cache from GCS. GCP token is not provided.' ) } + const oauth2Client = new OAuth2Client() + oauth2Client.setCredentials({access_token: gcsToken}) const storage = new Storage({ - token: gcsToken + authClient: oauth2Client }) return downloadCacheStreamingGCP(storage, archiveLocation) } @@ -345,8 +348,10 @@ export async function saveCache( } core.debug('Uploading cache') + const oauth2Client = new OAuth2Client() + oauth2Client.setCredentials({access_token: GCSAuthToken}) const storage = new Storage({ - token: GCSAuthToken + authClient: oauth2Client }) await multiPartUploadToGCS( storage, diff --git a/packages/warp-cache/src/test.ts b/packages/warp-cache/src/test.ts index 53be237b..65340121 100644 --- a/packages/warp-cache/src/test.ts +++ b/packages/warp-cache/src/test.ts @@ -7,15 +7,16 @@ process.env['WARPBUILD_CACHE_URL'] = 'https://cache.dev.warpbuild.dev' process.env['RUNNER_TEMP'] = '/Users/prajjwal/Repos/warpbuild/playground/tmp_fs' process.env['NODE_DEBUG'] = 'http' process.env['RUNNER_DEBUG'] = '1' -process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] = '' +process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] = + 'eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3MTM0MTg3MzMsInJlcG8iOiJiZW5jaG1hcmtzIiwicmVwb093bmVyIjoiV2FycEJ1aWxkcyIsIngtd2FycGJ1aWxkLW9yZ2FuaXphdGlvbi1pZCI6IndmbW4wODBlaWY4cm5pd3EifQ.a435J9ccjs9V_FzQMdbwTvXOYU8hvRieYkXM7yumlWAJyxDTsq4mi3CP1Ob9y6nLEKr35TYqGwxKFSTOW1oxYQ' process.env['GITHUB_REPOSITORY'] = 'Warpbuilds/backend-cache' process.env['GITHUB_REF'] = 'refs/heads/main' -// saveCache( -// ['/Users/prajjwal/Repos/warpbuild/playground/test_fs'], -// 'test-fs-local-key', -// true -// ) +saveCache( + ['/Users/prajjwal/Repos/warpbuild/playground/test_fs'], + 'test-fs-local-key', + true +) // saveCache( // ['/Users/prajjwal/Repos/warpbuild/playground/test_fs'], From a73dbad5a756133dd757bcda34b18777ddba01a9 Mon Sep 17 00:00:00 2001 From: Prajjwal Date: Mon, 15 Apr 2024 14:05:22 +0530 Subject: [PATCH 05/11] changes to multipart download --- packages/warp-cache/package.json | 2 +- packages/warp-cache/src/cache.ts | 44 ++++++++++++++++--- .../src/internal/cacheHttpClient.ts | 26 ++++++++++- .../warp-cache/src/internal/downloadUtils.ts | 27 +++++++++++- packages/warp-cache/src/test.ts | 26 +++++------ 5 files changed, 101 insertions(+), 24 deletions(-) diff --git a/packages/warp-cache/package.json b/packages/warp-cache/package.json index 092faa98..85024a22 100644 --- a/packages/warp-cache/package.json +++ b/packages/warp-cache/package.json @@ -1,6 +1,6 @@ { "name": "github-actions.warp-cache", - "version": "1.0.1", + "version": "1.0.2", "preview": true, "description": "Github action to use WarpBuild's in-house cache offering", "keywords": [ diff --git a/packages/warp-cache/src/cache.ts b/packages/warp-cache/src/cache.ts index 8eee08fb..71cc60dd 100644 --- a/packages/warp-cache/src/cache.ts +++ b/packages/warp-cache/src/cache.ts @@ -137,6 +137,7 @@ export async function restoreCache( } await cacheHttpClient.downloadCache( + cacheEntry.provider, cacheEntry.s3?.pre_signed_url, archivePath ) @@ -168,20 +169,41 @@ export async function restoreCache( return cacheKey } - // For GCS, we do a streaming download which means that we extract the archive while we are downloading it. const archiveLocation = `gs://${cacheEntry.gcs?.bucket_name}/${cacheEntry.gcs?.cache_key}` - const readStream = cacheHttpClient.downloadCacheStreaming( - 'gcs', + await cacheHttpClient.downloadCache( + cacheEntry.provider, archiveLocation, - cacheEntry?.gcs?.short_lived_token?.access_token ?? '' + archivePath, + cacheEntry.gcs?.short_lived_token?.access_token ?? '' ) - if (!readStream) { - return undefined + if (core.isDebug()) { + await listTar(archivePath, compressionMethod) } - await extractStreamingTar(readStream, archivePath, compressionMethod) + const archiveFileSize = utils.getArchiveFileSizeInBytes(archivePath) + core.info( + `Cache Size: ~${Math.round( + archiveFileSize / (1024 * 1024) + )} MB (${archiveFileSize} B)` + ) + + await extractTar(archivePath, compressionMethod) + + // For GCS, we do a streaming download which means that we extract the archive while we are downloading it. + + // const readStream = cacheHttpClient.downloadCacheStreaming( + // 'gcs', + // archiveLocation, + // cacheEntry?.gcs?.short_lived_token?.access_token ?? '' + // ) + + // if (!readStream) { + // return undefined + // } + + // await extractStreamingTar(readStream, archivePath, compressionMethod) core.info('Cache restored successfully') break } @@ -283,6 +305,14 @@ export async function saveCache( ) if (!isSuccessStatusCode(reserveCacheResponse?.statusCode)) { + core.debug(`Failed to reserve cache: ${reserveCacheResponse?.statusCode}`) + core.debug( + `Reserve Cache Request: ${JSON.stringify({ + key, + numberOfChunks, + cacheVersion + })}` + ) throw new Error( reserveCacheResponse?.error?.message ?? `Cache size of ~${Math.round( diff --git a/packages/warp-cache/src/internal/cacheHttpClient.ts b/packages/warp-cache/src/internal/cacheHttpClient.ts index 8fab972f..c41e179a 100644 --- a/packages/warp-cache/src/internal/cacheHttpClient.ts +++ b/packages/warp-cache/src/internal/cacheHttpClient.ts @@ -16,6 +16,7 @@ import { } from './contracts' import { downloadCacheMultiConnection, + downloadCacheMultipartGCP, downloadCacheStreamingGCP } from './downloadUtils' import {isSuccessStatusCode, retryTypedResponse} from './requestUtils' @@ -187,10 +188,31 @@ async function printCachesListForDiagnostics( */ export async function downloadCache( + provider: string, archiveLocation: string, - archivePath: string + archivePath: string, + gcsToken?: string ): Promise { - await downloadCacheMultiConnection(archiveLocation, archivePath, 8) + switch (provider) { + case 's3': + await downloadCacheMultiConnection(archiveLocation, archivePath, 8) + break + case 'gcs': { + if (!gcsToken) { + throw new Error( + 'Unable to download cache from GCS. GCP token is not provided.' + ) + } + + const oauth2Client = new OAuth2Client() + oauth2Client.setCredentials({access_token: gcsToken}) + const storage = new Storage({ + authClient: oauth2Client + }) + await downloadCacheMultipartGCP(storage, archiveLocation, archivePath) + break + } + } } export function downloadCacheStreaming( diff --git a/packages/warp-cache/src/internal/downloadUtils.ts b/packages/warp-cache/src/internal/downloadUtils.ts index 9019f2d4..9faeb3aa 100644 --- a/packages/warp-cache/src/internal/downloadUtils.ts +++ b/packages/warp-cache/src/internal/downloadUtils.ts @@ -13,7 +13,7 @@ import {DownloadOptions} from '../options' import {retryHttpClientResponse} from './requestUtils' import {AbortController} from '@azure/abort-controller' -import {Storage} from '@google-cloud/storage' +import {Storage, TransferManager} from '@google-cloud/storage' /** * Pipes the body of a HTTP response to a stream @@ -294,6 +294,31 @@ export async function downloadCacheMultiConnection( } } +/** + * Download cache in multipart using the Gcloud SDK + * + * @param archiveLocation the URL for the cache + */ +export async function downloadCacheMultipartGCP( + storage: Storage, + archiveLocation: string, + archivePath: string +) { + try { + const {bucketName, objectName} = + utils.retrieveGCSBucketAndObjectName(archiveLocation) + + const transferManager = new TransferManager(storage.bucket(bucketName)) + await transferManager.downloadFileInChunks(objectName, { + destination: archivePath + }) + } catch (error) { + core.debug(`Failed to download cache: ${error}`) + core.error(`Failed to download cache.`) + throw error + } +} + /** * Download the cache to a provider writable stream using GCloud SDK * diff --git a/packages/warp-cache/src/test.ts b/packages/warp-cache/src/test.ts index 65340121..9620df75 100644 --- a/packages/warp-cache/src/test.ts +++ b/packages/warp-cache/src/test.ts @@ -12,11 +12,11 @@ process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] = process.env['GITHUB_REPOSITORY'] = 'Warpbuilds/backend-cache' process.env['GITHUB_REF'] = 'refs/heads/main' -saveCache( - ['/Users/prajjwal/Repos/warpbuild/playground/test_fs'], - 'test-fs-local-key', - true -) +// saveCache( +// ['/Users/prajjwal/Repos/warpbuild/playground/test_fs'], +// 'test-fs-local-key', +// true +// ) // saveCache( // ['/Users/prajjwal/Repos/warpbuild/playground/test_fs'], @@ -31,14 +31,14 @@ saveCache( // true // ) -// restoreCache( -// ['/Users/prajjwal/Repos/warpbuild/playground/test_fs'], -// 'test-fs-local-key-3', -// ['test-fs'], -// {}, -// true, -// false -// ) +restoreCache( + ['/Users/prajjwal/Repos/warpbuild/playground/test_fs'], + 'test-fs-local-key', + ['test-fs'], + {}, + true, + false +) // deleteCache( // ['/Users/prajjwal/Repos/warpbuild/playground/test_fs'], From 5e3d03b01d60c81f9284bc3268dadae2a407af24 Mon Sep 17 00:00:00 2001 From: Prajjwal Date: Mon, 15 Apr 2024 14:05:31 +0530 Subject: [PATCH 06/11] comments test --- packages/warp-cache/src/test.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/warp-cache/src/test.ts b/packages/warp-cache/src/test.ts index 9620df75..3ae42c69 100644 --- a/packages/warp-cache/src/test.ts +++ b/packages/warp-cache/src/test.ts @@ -31,14 +31,14 @@ process.env['GITHUB_REF'] = 'refs/heads/main' // true // ) -restoreCache( - ['/Users/prajjwal/Repos/warpbuild/playground/test_fs'], - 'test-fs-local-key', - ['test-fs'], - {}, - true, - false -) +// restoreCache( +// ['/Users/prajjwal/Repos/warpbuild/playground/test_fs'], +// 'test-fs-local-key', +// ['test-fs'], +// {}, +// true, +// false +// ) // deleteCache( // ['/Users/prajjwal/Repos/warpbuild/playground/test_fs'], From 84009488746ab5ba019d433ddd6b70121a9b09ec Mon Sep 17 00:00:00 2001 From: Prajjwal Date: Mon, 15 Apr 2024 14:05:45 +0530 Subject: [PATCH 07/11] bumps version --- packages/warp-cache/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/warp-cache/package.json b/packages/warp-cache/package.json index 85024a22..3dbf04f4 100644 --- a/packages/warp-cache/package.json +++ b/packages/warp-cache/package.json @@ -1,6 +1,6 @@ { "name": "github-actions.warp-cache", - "version": "1.0.2", + "version": "1.0.3", "preview": true, "description": "Github action to use WarpBuild's in-house cache offering", "keywords": [ From d0787ac29d795ab7e512a0c4205b0f383fe39956 Mon Sep 17 00:00:00 2001 From: Prajjwal Date: Mon, 15 Apr 2024 14:32:00 +0530 Subject: [PATCH 08/11] adjusts chunk size --- packages/warp-cache/package.json | 2 +- packages/warp-cache/src/internal/downloadUtils.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/warp-cache/package.json b/packages/warp-cache/package.json index 3dbf04f4..fd5e8b96 100644 --- a/packages/warp-cache/package.json +++ b/packages/warp-cache/package.json @@ -1,6 +1,6 @@ { "name": "github-actions.warp-cache", - "version": "1.0.3", + "version": "1.0.6", "preview": true, "description": "Github action to use WarpBuild's in-house cache offering", "keywords": [ diff --git a/packages/warp-cache/src/internal/downloadUtils.ts b/packages/warp-cache/src/internal/downloadUtils.ts index 9faeb3aa..f01734b3 100644 --- a/packages/warp-cache/src/internal/downloadUtils.ts +++ b/packages/warp-cache/src/internal/downloadUtils.ts @@ -310,7 +310,8 @@ export async function downloadCacheMultipartGCP( const transferManager = new TransferManager(storage.bucket(bucketName)) await transferManager.downloadFileInChunks(objectName, { - destination: archivePath + destination: archivePath, + chunkSizeBytes: 8 * 1024 * 1024 // 8MB chunk size }) } catch (error) { core.debug(`Failed to download cache: ${error}`) From 63c608965147aa2e0a93c2025f5a83b2fe90b2dd Mon Sep 17 00:00:00 2001 From: Prajjwal Date: Mon, 15 Apr 2024 16:18:16 +0530 Subject: [PATCH 09/11] moves to streaming for gcs --- packages/warp-cache/package.json | 2 +- packages/warp-cache/src/cache.ts | 50 +++++++++---------- .../warp-cache/src/internal/downloadUtils.ts | 3 +- packages/warp-cache/src/test.ts | 16 +++--- 4 files changed, 36 insertions(+), 35 deletions(-) diff --git a/packages/warp-cache/package.json b/packages/warp-cache/package.json index fd5e8b96..55330def 100644 --- a/packages/warp-cache/package.json +++ b/packages/warp-cache/package.json @@ -1,6 +1,6 @@ { "name": "github-actions.warp-cache", - "version": "1.0.6", + "version": "1.1.0", "preview": true, "description": "Github action to use WarpBuild's in-house cache offering", "keywords": [ diff --git a/packages/warp-cache/src/cache.ts b/packages/warp-cache/src/cache.ts index 71cc60dd..b648cf0c 100644 --- a/packages/warp-cache/src/cache.ts +++ b/packages/warp-cache/src/cache.ts @@ -171,39 +171,39 @@ export async function restoreCache( const archiveLocation = `gs://${cacheEntry.gcs?.bucket_name}/${cacheEntry.gcs?.cache_key}` - await cacheHttpClient.downloadCache( - cacheEntry.provider, - archiveLocation, - archivePath, - cacheEntry.gcs?.short_lived_token?.access_token ?? '' - ) + // await cacheHttpClient.downloadCache( + // cacheEntry.provider, + // archiveLocation, + // archivePath, + // cacheEntry.gcs?.short_lived_token?.access_token ?? '' + // ) - if (core.isDebug()) { - await listTar(archivePath, compressionMethod) - } + // if (core.isDebug()) { + // await listTar(archivePath, compressionMethod) + // } - const archiveFileSize = utils.getArchiveFileSizeInBytes(archivePath) - core.info( - `Cache Size: ~${Math.round( - archiveFileSize / (1024 * 1024) - )} MB (${archiveFileSize} B)` - ) + // const archiveFileSize = utils.getArchiveFileSizeInBytes(archivePath) + // core.info( + // `Cache Size: ~${Math.round( + // archiveFileSize / (1024 * 1024) + // )} MB (${archiveFileSize} B)` + // ) - await extractTar(archivePath, compressionMethod) + // await extractTar(archivePath, compressionMethod) // For GCS, we do a streaming download which means that we extract the archive while we are downloading it. - // const readStream = cacheHttpClient.downloadCacheStreaming( - // 'gcs', - // archiveLocation, - // cacheEntry?.gcs?.short_lived_token?.access_token ?? '' - // ) + const readStream = cacheHttpClient.downloadCacheStreaming( + 'gcs', + archiveLocation, + cacheEntry?.gcs?.short_lived_token?.access_token ?? '' + ) - // if (!readStream) { - // return undefined - // } + if (!readStream) { + return undefined + } - // await extractStreamingTar(readStream, archivePath, compressionMethod) + await extractStreamingTar(readStream, archivePath, compressionMethod) core.info('Cache restored successfully') break } diff --git a/packages/warp-cache/src/internal/downloadUtils.ts b/packages/warp-cache/src/internal/downloadUtils.ts index f01734b3..6cebefaf 100644 --- a/packages/warp-cache/src/internal/downloadUtils.ts +++ b/packages/warp-cache/src/internal/downloadUtils.ts @@ -311,7 +311,8 @@ export async function downloadCacheMultipartGCP( const transferManager = new TransferManager(storage.bucket(bucketName)) await transferManager.downloadFileInChunks(objectName, { destination: archivePath, - chunkSizeBytes: 8 * 1024 * 1024 // 8MB chunk size + noReturnData: true, + chunkSizeBytes: 1024 * 1024 * 8 }) } catch (error) { core.debug(`Failed to download cache: ${error}`) diff --git a/packages/warp-cache/src/test.ts b/packages/warp-cache/src/test.ts index 3ae42c69..9620df75 100644 --- a/packages/warp-cache/src/test.ts +++ b/packages/warp-cache/src/test.ts @@ -31,14 +31,14 @@ process.env['GITHUB_REF'] = 'refs/heads/main' // true // ) -// restoreCache( -// ['/Users/prajjwal/Repos/warpbuild/playground/test_fs'], -// 'test-fs-local-key', -// ['test-fs'], -// {}, -// true, -// false -// ) +restoreCache( + ['/Users/prajjwal/Repos/warpbuild/playground/test_fs'], + 'test-fs-local-key', + ['test-fs'], + {}, + true, + false +) // deleteCache( // ['/Users/prajjwal/Repos/warpbuild/playground/test_fs'], From d989701710f36e402eafabd4605b432a4820a329 Mon Sep 17 00:00:00 2001 From: Prajjwal Date: Mon, 15 Apr 2024 17:42:41 +0530 Subject: [PATCH 10/11] adds wget for streaming download --- packages/warp-cache/package.json | 2 +- packages/warp-cache/src/cache.ts | 67 +++++++++----- .../warp-cache/src/internal/downloadUtils.ts | 7 ++ packages/warp-cache/src/internal/tar.ts | 13 ++- .../warpcache-ts-sdk/.openapi-generator/FILES | 2 + .../warpcache-ts-sdk/api/default-api.ts | 89 +++++++++++++++++++ .../models/commons-append-operation-input.ts | 60 +++++++++++++ .../models/commons-cache-entry.ts | 12 +++ .../models/commons-gcsget-cache-reponse.ts | 6 ++ .../models/commons-operation.ts | 66 ++++++++++++++ .../internal/warpcache-ts-sdk/models/index.ts | 2 + .../src/internal/warpcache-ts-sdk/schema.ts | 2 + .../schemas/$commons_AppendOperationInput.ts | 26 ++++++ .../schemas/$commons_CacheEntry.ts | 6 ++ .../schemas/$commons_GCSGetCacheReponse.ts | 3 + .../schemas/$commons_Operation.ts | 29 ++++++ packages/warp-cache/src/test.ts | 2 + 17 files changed, 366 insertions(+), 28 deletions(-) create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-append-operation-input.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-operation.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_AppendOperationInput.ts create mode 100644 packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_Operation.ts diff --git a/packages/warp-cache/package.json b/packages/warp-cache/package.json index 55330def..f2c393ed 100644 --- a/packages/warp-cache/package.json +++ b/packages/warp-cache/package.json @@ -1,6 +1,6 @@ { "name": "github-actions.warp-cache", - "version": "1.1.0", + "version": "1.1.1", "preview": true, "description": "Github action to use WarpBuild's in-house cache offering", "keywords": [ diff --git a/packages/warp-cache/src/cache.ts b/packages/warp-cache/src/cache.ts index b648cf0c..7dd9839e 100644 --- a/packages/warp-cache/src/cache.ts +++ b/packages/warp-cache/src/cache.ts @@ -10,6 +10,8 @@ import { } from './internal/tar' import {DownloadOptions, getUploadOptions} from './options' import {isSuccessStatusCode} from './internal/requestUtils' +import {getDownloadCommandPipeForWget} from './internal/downloadUtils' +import {ChildProcessWithoutNullStreams} from 'child_process' export class ValidationError extends Error { constructor(message: string) { @@ -171,39 +173,56 @@ export async function restoreCache( const archiveLocation = `gs://${cacheEntry.gcs?.bucket_name}/${cacheEntry.gcs?.cache_key}` - // await cacheHttpClient.downloadCache( - // cacheEntry.provider, - // archiveLocation, - // archivePath, - // cacheEntry.gcs?.short_lived_token?.access_token ?? '' - // ) + /* + * Alternate, Multipart download method for GCS + await cacheHttpClient.downloadCache( + cacheEntry.provider, + archiveLocation, + archivePath, + cacheEntry.gcs?.short_lived_token?.access_token ?? '' + ) - // if (core.isDebug()) { - // await listTar(archivePath, compressionMethod) - // } + if (core.isDebug()) { + await listTar(archivePath, compressionMethod) + } - // const archiveFileSize = utils.getArchiveFileSizeInBytes(archivePath) - // core.info( - // `Cache Size: ~${Math.round( - // archiveFileSize / (1024 * 1024) - // )} MB (${archiveFileSize} B)` - // ) + const archiveFileSize = utils.getArchiveFileSizeInBytes(archivePath) + core.info( + `Cache Size: ~${Math.round( + archiveFileSize / (1024 * 1024) + )} MB (${archiveFileSize} B)` + ) - // await extractTar(archivePath, compressionMethod) + await extractTar(archivePath, compressionMethod) + */ // For GCS, we do a streaming download which means that we extract the archive while we are downloading it. - const readStream = cacheHttpClient.downloadCacheStreaming( - 'gcs', - archiveLocation, - cacheEntry?.gcs?.short_lived_token?.access_token ?? '' - ) + let readStream: NodeJS.ReadableStream | undefined + let downloadCommandPipe: ChildProcessWithoutNullStreams | undefined - if (!readStream) { - return undefined + if (cacheEntry?.gcs?.pre_signed_url) { + downloadCommandPipe = getDownloadCommandPipeForWget( + cacheEntry?.gcs?.pre_signed_url + ) + } else { + readStream = cacheHttpClient.downloadCacheStreaming( + 'gcs', + archiveLocation, + cacheEntry?.gcs?.short_lived_token?.access_token ?? '' + ) + + if (!readStream) { + return undefined + } } - await extractStreamingTar(readStream, archivePath, compressionMethod) + await extractStreamingTar( + readStream, + archivePath, + compressionMethod, + downloadCommandPipe + ) core.info('Cache restored successfully') break } diff --git a/packages/warp-cache/src/internal/downloadUtils.ts b/packages/warp-cache/src/internal/downloadUtils.ts index 6cebefaf..1be7ef91 100644 --- a/packages/warp-cache/src/internal/downloadUtils.ts +++ b/packages/warp-cache/src/internal/downloadUtils.ts @@ -14,6 +14,7 @@ import {retryHttpClientResponse} from './requestUtils' import {AbortController} from '@azure/abort-controller' import {Storage, TransferManager} from '@google-cloud/storage' +import {ChildProcessWithoutNullStreams, spawn} from 'child_process' /** * Pipes the body of a HTTP response to a stream @@ -350,3 +351,9 @@ export function downloadCacheStreamingGCP( throw error } } + +export function getDownloadCommandPipeForWget( + url: string +): ChildProcessWithoutNullStreams { + return spawn('wget', ['-qO', '-', url]) +} diff --git a/packages/warp-cache/src/internal/tar.ts b/packages/warp-cache/src/internal/tar.ts index 74a6d18f..0926cd24 100644 --- a/packages/warp-cache/src/internal/tar.ts +++ b/packages/warp-cache/src/internal/tar.ts @@ -417,9 +417,10 @@ export async function extractTar( * NOTE: Currently tested only on archives created using tar and zstd */ export async function extractStreamingTar( - stream: NodeJS.ReadableStream, + stream: NodeJS.ReadableStream | undefined, archivePath: string, - compressionMethod: CompressionMethod + compressionMethod: CompressionMethod, + downloadCommandPipe?: ChildProcessWithoutNullStreams ): Promise { const workingDirectory = getWorkingDirectory() await io.mkdirP(workingDirectory) @@ -429,6 +430,10 @@ export async function extractStreamingTar( archivePath ) + if (downloadCommandPipe) { + commandPipes.unshift(downloadCommandPipe) + } + if (commandPipes.length < 2) { throw new Error( 'At least two processes should be present as the archive is compressed at least twice.' @@ -436,7 +441,9 @@ export async function extractStreamingTar( } return new Promise((resolve, reject) => { - stream.pipe(commandPipes[0].stdin) + if (stream) { + stream.pipe(commandPipes[0].stdin) + } for (let i = 0; i < commandPipes.length - 1; i++) { commandPipes[i].stdout.pipe(commandPipes[i + 1].stdin) diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/.openapi-generator/FILES b/packages/warp-cache/src/internal/warpcache-ts-sdk/.openapi-generator/FILES index b3e971af..1181d96a 100644 --- a/packages/warp-cache/src/internal/warpcache-ts-sdk/.openapi-generator/FILES +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/.openapi-generator/FILES @@ -9,6 +9,7 @@ common.ts configuration.ts git_push.sh index.ts +models/commons-append-operation-input.ts models/commons-cache-entry.ts models/commons-commit-cache-request.ts models/commons-commit-cache-response.ts @@ -20,6 +21,7 @@ models/commons-gcsget-cache-reponse.ts models/commons-gcsreserve-cache-response.ts models/commons-get-cache-request.ts models/commons-get-cache-response.ts +models/commons-operation.ts models/commons-reserve-cache-request.ts models/commons-reserve-cache-response.ts models/commons-s3-commit-cache-response.ts diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/api/default-api.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/api/default-api.ts index 7c11bc4e..f5e134e3 100644 --- a/packages/warp-cache/src/internal/warpcache-ts-sdk/api/default-api.ts +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/api/default-api.ts @@ -22,6 +22,8 @@ import { DUMMY_BASE_URL, assertParamExists, setApiKeyToObject, setBasicAuthToObj // @ts-ignore import { BASE_PATH, COLLECTION_FORMATS, RequestArgs, BaseAPI, RequiredError, operationServerMap } from '../base'; // @ts-ignore +import { CommonsAppendOperationInput } from '../models'; +// @ts-ignore import { CommonsCommitCacheRequest } from '../models'; // @ts-ignore import { CommonsCommitCacheResponse } from '../models'; @@ -34,6 +36,8 @@ import { CommonsGetCacheRequest } from '../models'; // @ts-ignore import { CommonsGetCacheResponse } from '../models'; // @ts-ignore +import { CommonsOperation } from '../models'; +// @ts-ignore import { CommonsReserveCacheRequest } from '../models'; // @ts-ignore import { CommonsReserveCacheResponse } from '../models'; @@ -207,6 +211,42 @@ export const DefaultApiAxiosParamCreator = function (configuration?: Configurati + localVarHeaderParameter['Content-Type'] = 'application/json'; + + setSearchParams(localVarUrlObj, localVarQueryParameter); + let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; + localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; + localVarRequestOptions.data = serializeDataIfNeeded(body, localVarRequestOptions, configuration) + + return { + url: toPathString(localVarUrlObj), + options: localVarRequestOptions, + }; + }, + /** + * record operation + * @summary record operation + * @param {CommonsAppendOperationInput} body Record Operation details Request Body + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + v1InstrumentationOperationPost: async (body: CommonsAppendOperationInput, options: RawAxiosRequestConfig = {}): Promise => { + // verify required parameter 'body' is not null or undefined + assertParamExists('v1InstrumentationOperationPost', 'body', body) + const localVarPath = `/v1/instrumentation/operation`; + // use dummy base URL string because the URL constructor only accepts absolute URLs. + const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); + let baseOptions; + if (configuration) { + baseOptions = configuration.baseOptions; + } + + const localVarRequestOptions = { method: 'POST', ...baseOptions, ...options}; + const localVarHeaderParameter = {} as any; + const localVarQueryParameter = {} as any; + + + localVarHeaderParameter['Content-Type'] = 'application/json'; setSearchParams(localVarUrlObj, localVarQueryParameter); @@ -293,6 +333,19 @@ export const DefaultApiFp = function(configuration?: Configuration) { const localVarOperationServerBasePath = operationServerMap['DefaultApi.v1CacheReservePost']?.[localVarOperationServerIndex]?.url; return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); }, + /** + * record operation + * @summary record operation + * @param {CommonsAppendOperationInput} body Record Operation details Request Body + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + async v1InstrumentationOperationPost(body: CommonsAppendOperationInput, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.v1InstrumentationOperationPost(body, options); + const localVarOperationServerIndex = configuration?.serverIndex ?? 0; + const localVarOperationServerBasePath = operationServerMap['DefaultApi.v1InstrumentationOperationPost']?.[localVarOperationServerIndex]?.url; + return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); + }, } }; @@ -352,6 +405,16 @@ export const DefaultApiFactory = function (configuration?: Configuration, basePa v1CacheReservePost(requestParameters: DefaultApiV1CacheReservePostRequest, options?: RawAxiosRequestConfig): AxiosPromise { return localVarFp.v1CacheReservePost(requestParameters.body, options).then((request) => request(axios, basePath)); }, + /** + * record operation + * @summary record operation + * @param {DefaultApiV1InstrumentationOperationPostRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + v1InstrumentationOperationPost(requestParameters: DefaultApiV1InstrumentationOperationPostRequest, options?: RawAxiosRequestConfig): AxiosPromise { + return localVarFp.v1InstrumentationOperationPost(requestParameters.body, options).then((request) => request(axios, basePath)); + }, }; }; @@ -411,6 +474,20 @@ export interface DefaultApiV1CacheReservePostRequest { readonly body: CommonsReserveCacheRequest } +/** + * Request parameters for v1InstrumentationOperationPost operation in DefaultApi. + * @export + * @interface DefaultApiV1InstrumentationOperationPostRequest + */ +export interface DefaultApiV1InstrumentationOperationPostRequest { + /** + * Record Operation details Request Body + * @type {CommonsAppendOperationInput} + * @memberof DefaultApiV1InstrumentationOperationPost + */ + readonly body: CommonsAppendOperationInput +} + /** * DefaultApi - object-oriented interface * @export @@ -476,5 +553,17 @@ export class DefaultApi extends BaseAPI { public v1CacheReservePost(requestParameters: DefaultApiV1CacheReservePostRequest, options?: RawAxiosRequestConfig) { return DefaultApiFp(this.configuration).v1CacheReservePost(requestParameters.body, options).then((request) => request(this.axios, this.basePath)); } + + /** + * record operation + * @summary record operation + * @param {DefaultApiV1InstrumentationOperationPostRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof DefaultApi + */ + public v1InstrumentationOperationPost(requestParameters: DefaultApiV1InstrumentationOperationPostRequest, options?: RawAxiosRequestConfig) { + return DefaultApiFp(this.configuration).v1InstrumentationOperationPost(requestParameters.body, options).then((request) => request(this.axios, this.basePath)); + } } diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-append-operation-input.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-append-operation-input.ts new file mode 100644 index 00000000..53b1c901 --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-append-operation-input.ts @@ -0,0 +1,60 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + + +/** + * + * @export + * @interface CommonsAppendOperationInput + */ +export interface CommonsAppendOperationInput { + /** + * + * @type {string} + * @memberof CommonsAppendOperationInput + */ + 'cache_id'?: string; + /** + * + * @type {string} + * @memberof CommonsAppendOperationInput + */ + 'external_id'?: string; + /** + * + * @type {{ [key: string]: any; }} + * @memberof CommonsAppendOperationInput + */ + 'meta'?: { [key: string]: any; }; + /** + * + * @type {string} + * @memberof CommonsAppendOperationInput + */ + 'operation_type'?: string; + /** + * + * @type {number} + * @memberof CommonsAppendOperationInput + */ + 'size'?: number; + /** + * + * @type {string} + * @memberof CommonsAppendOperationInput + */ + 'time'?: string; +} + diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-cache-entry.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-cache-entry.ts index 520b23d2..c50acc0f 100644 --- a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-cache-entry.ts +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-cache-entry.ts @@ -50,6 +50,12 @@ export interface CommonsCacheEntry { * @memberof CommonsCacheEntry */ 'organization_id'?: string; + /** + * + * @type {string} + * @memberof CommonsCacheEntry + */ + 'storage_backend_id'?: string; /** * * @type {string} @@ -62,6 +68,12 @@ export interface CommonsCacheEntry { * @memberof CommonsCacheEntry */ 'vcs_organization_name'?: string; + /** + * + * @type {string} + * @memberof CommonsCacheEntry + */ + 'vcs_ref'?: string; /** * * @type {string} diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-gcsget-cache-reponse.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-gcsget-cache-reponse.ts index aa7ab6f5..c97823fd 100644 --- a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-gcsget-cache-reponse.ts +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-gcsget-cache-reponse.ts @@ -47,6 +47,12 @@ export interface CommonsGCSGetCacheReponse { * @memberof CommonsGCSGetCacheReponse */ 'method'?: string; + /** + * + * @type {string} + * @memberof CommonsGCSGetCacheReponse + */ + 'pre_signed_url'?: string; /** * * @type {string} diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-operation.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-operation.ts new file mode 100644 index 00000000..e8bcb0ac --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/commons-operation.ts @@ -0,0 +1,66 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + + +/** + * + * @export + * @interface CommonsOperation + */ +export interface CommonsOperation { + /** + * + * @type {string} + * @memberof CommonsOperation + */ + 'cache_id'?: string; + /** + * + * @type {string} + * @memberof CommonsOperation + */ + 'created-at'?: string; + /** + * + * @type {string} + * @memberof CommonsOperation + */ + 'id'?: string; + /** + * + * @type {{ [key: string]: any; }} + * @memberof CommonsOperation + */ + 'meta'?: { [key: string]: any; }; + /** + * + * @type {string} + * @memberof CommonsOperation + */ + 'operation_type'?: string; + /** + * + * @type {string} + * @memberof CommonsOperation + */ + 'organization_id'?: string; + /** + * + * @type {string} + * @memberof CommonsOperation + */ + 'updated_at'?: string; +} + diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/index.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/index.ts index dc9cc139..ebce51f6 100644 --- a/packages/warp-cache/src/internal/warpcache-ts-sdk/models/index.ts +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/models/index.ts @@ -1,3 +1,4 @@ +export * from './commons-append-operation-input'; export * from './commons-cache-entry'; export * from './commons-commit-cache-request'; export * from './commons-commit-cache-response'; @@ -9,6 +10,7 @@ export * from './commons-gcsget-cache-reponse'; export * from './commons-gcsreserve-cache-response'; export * from './commons-get-cache-request'; export * from './commons-get-cache-response'; +export * from './commons-operation'; export * from './commons-reserve-cache-request'; export * from './commons-reserve-cache-response'; export * from './commons-s3-commit-cache-response'; diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schema.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schema.ts index 0c653785..7981371a 100644 --- a/packages/warp-cache/src/internal/warpcache-ts-sdk/schema.ts +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schema.ts @@ -1,6 +1,7 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ +export { $commons_AppendOperationInput } from './schemas/$commons_AppendOperationInput'; export { $commons_CacheAnnotationsMap } from './schemas/$commons_CacheAnnotationsMap'; export { $commons_CacheEntry } from './schemas/$commons_CacheEntry'; export { $commons_CommitCacheRequest } from './schemas/$commons_CommitCacheRequest'; @@ -13,6 +14,7 @@ export { $commons_GCSGetCacheReponse } from './schemas/$commons_GCSGetCacheRepon export { $commons_GCSReserveCacheResponse } from './schemas/$commons_GCSReserveCacheResponse'; export { $commons_GetCacheRequest } from './schemas/$commons_GetCacheRequest'; export { $commons_GetCacheResponse } from './schemas/$commons_GetCacheResponse'; +export { $commons_Operation } from './schemas/$commons_Operation'; export { $commons_ReserveCacheRequest } from './schemas/$commons_ReserveCacheRequest'; export { $commons_ReserveCacheResponse } from './schemas/$commons_ReserveCacheResponse'; export { $commons_S3CommitCacheResponse } from './schemas/$commons_S3CommitCacheResponse'; diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_AppendOperationInput.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_AppendOperationInput.ts new file mode 100644 index 00000000..276eb8ea --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_AppendOperationInput.ts @@ -0,0 +1,26 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export const $commons_AppendOperationInput = { + properties: { + cache_id: { + type: 'string', + }, + external_id: { + type: 'string', + }, + meta: { + properties: { + }, + }, + operation_type: { + type: 'string', + }, + size: { + type: 'number', + }, + time: { + type: 'string', + }, + }, +} as const; \ No newline at end of file diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CacheEntry.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CacheEntry.ts index 3a15516a..a376859e 100644 --- a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CacheEntry.ts +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_CacheEntry.ts @@ -18,12 +18,18 @@ export const $commons_CacheEntry = { organization_id: { type: 'string', }, + storage_backend_id: { + type: 'string', + }, updated_at: { type: 'string', }, vcs_organization_name: { type: 'string', }, + vcs_ref: { + type: 'string', + }, vcs_repository_name: { type: 'string', }, diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GCSGetCacheReponse.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GCSGetCacheReponse.ts index 1b2fb812..196d3192 100644 --- a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GCSGetCacheReponse.ts +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_GCSGetCacheReponse.ts @@ -15,6 +15,9 @@ export const $commons_GCSGetCacheReponse = { method: { type: 'string', }, + pre_signed_url: { + type: 'string', + }, project_id: { type: 'string', }, diff --git a/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_Operation.ts b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_Operation.ts new file mode 100644 index 00000000..9b32d1ed --- /dev/null +++ b/packages/warp-cache/src/internal/warpcache-ts-sdk/schemas/$commons_Operation.ts @@ -0,0 +1,29 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export const $commons_Operation = { + properties: { + cache_id: { + type: 'string', + }, + 'created-at': { + type: 'string', + }, + id: { + type: 'string', + }, + meta: { + properties: { + }, + }, + operation_type: { + type: 'string', + }, + organization_id: { + type: 'string', + }, + updated_at: { + type: 'string', + }, + }, +} as const; \ No newline at end of file diff --git a/packages/warp-cache/src/test.ts b/packages/warp-cache/src/test.ts index 9620df75..6da40584 100644 --- a/packages/warp-cache/src/test.ts +++ b/packages/warp-cache/src/test.ts @@ -4,6 +4,8 @@ import {getCompressionMethod} from './internal/cacheUtils' process.env['WARPBUILD_CACHE_URL'] = 'https://cache.dev.warpbuild.dev' // process.env['WARPBUILD_CACHE_URL'] = 'http://localhost:8000' +// process.env['WARPBUILD_CACHE_URL'] = +// 'https://6134-36-255-234-176.ngrok-free.app' process.env['RUNNER_TEMP'] = '/Users/prajjwal/Repos/warpbuild/playground/tmp_fs' process.env['NODE_DEBUG'] = 'http' process.env['RUNNER_DEBUG'] = '1' From 560aa2ff07274848dd376bc487cf40e52f14f619 Mon Sep 17 00:00:00 2001 From: Prajjwal Date: Mon, 15 Apr 2024 17:52:36 +0530 Subject: [PATCH 11/11] adds annotations to api requests --- packages/warp-cache/package.json | 2 +- .../src/internal/cacheHttpClient.ts | 26 ++++++++++++++++--- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/packages/warp-cache/package.json b/packages/warp-cache/package.json index f2c393ed..2a4895df 100644 --- a/packages/warp-cache/package.json +++ b/packages/warp-cache/package.json @@ -1,6 +1,6 @@ { "name": "github-actions.warp-cache", - "version": "1.1.1", + "version": "1.1.2", "preview": true, "description": "Github action to use WarpBuild's in-house cache offering", "keywords": [ diff --git a/packages/warp-cache/src/internal/cacheHttpClient.ts b/packages/warp-cache/src/internal/cacheHttpClient.ts index c41e179a..566f8c94 100644 --- a/packages/warp-cache/src/internal/cacheHttpClient.ts +++ b/packages/warp-cache/src/internal/cacheHttpClient.ts @@ -62,6 +62,20 @@ function getVCSRef(): string { return vcsBranch } +function getAnnotations(): {[key: string]: string} { + const annotations: {[key: string]: string} = { + GITHUB_WORKFLOW: process.env['GITHUB_WORKFLOW'] ?? '', + GITHUB_RUN_ID: process.env['GITHUB_RUN_ID'] ?? '', + GITHUB_RUN_ATTEMPT: process.env['GITHUB_RUN_ATTEMPT'] ?? '', + GITHUB_JOB: process.env['GITHUB_JOB'] ?? '', + GITHUB_REPOSITORY: process.env['GITHUB_REPOSITORY'] ?? '', + GITHUB_REF: process.env['GITHUB_REF'] ?? '', + GITHUB_ACTION: process.env['GITHUB_ACTION'] ?? '', + RUNNER_NAME: process.env['RUNNER_NAME'] ?? '' + } + return annotations +} + function getRequestOptions(): RequestOptions { const requestOptions: RequestOptions = { headers: { @@ -132,7 +146,8 @@ export async function getCacheEntry( restore_keys: restoreKeys, cache_version: version, vcs_repository: getVCSRepository(), - vcs_ref: getVCSRef() + vcs_ref: getVCSRef(), + annotations: getAnnotations() } const response = await retryTypedResponse('getCacheEntry', async () => @@ -254,7 +269,8 @@ export async function reserveCache( number_of_chunks: numberOfChunks, content_type: 'application/zstd', vcs_repository: getVCSRepository(), - vcs_ref: getVCSRef() + vcs_ref: getVCSRef(), + annotations: getAnnotations() } const response = await retryTypedResponse('reserveCache', async () => httpClient.postJson( @@ -286,7 +302,8 @@ async function commitCache( parts: parts, vcs_type: 'github', vcs_repository: getVCSRepository(), - vcs_ref: getVCSRef() + vcs_ref: getVCSRef(), + annotations: getAnnotations() } return await retryTypedResponse('commitCache', async () => httpClient.postJson( @@ -407,7 +424,8 @@ export async function deleteCache(cacheKey: string, cacheVersion: string) { cache_key: cacheKey, cache_version: cacheVersion, vcs_repository: getVCSRepository(), - vcs_ref: getVCSRef() + vcs_ref: getVCSRef(), + annotations: getAnnotations() } const response = await retryTypedResponse('deleteCacheEntry', async () =>