Format and update tests

joshmgross/chunked-upload-testing
Josh Gross 2019-12-18 10:59:51 -05:00
parent 2ce22df8c4
commit cdec5dec0d
4 changed files with 122 additions and 81 deletions

View File

@ -212,14 +212,14 @@ test("save with large cache outputs warning", async () => {
const IS_WINDOWS = process.platform === "win32"; const IS_WINDOWS = process.platform === "win32";
const args = IS_WINDOWS const args = IS_WINDOWS
? [ ? [
"-cz", "-cz",
"--force-local", "--force-local",
"-f", "-f",
archivePath.replace(/\\/g, "/"), archivePath.replace(/\\/g, "/"),
"-C", "-C",
cachePath.replace(/\\/g, "/"), cachePath.replace(/\\/g, "/"),
"." "."
] ]
: ["-cz", "-f", archivePath, "-C", cachePath, "."]; : ["-cz", "-f", archivePath, "-C", cachePath, "."];
expect(execMock).toHaveBeenCalledTimes(1); expect(execMock).toHaveBeenCalledTimes(1);
@ -259,6 +259,11 @@ test("save with server error outputs warning", async () => {
const cachePath = path.resolve(inputPath); const cachePath = path.resolve(inputPath);
testUtils.setInput(Inputs.Path, inputPath); testUtils.setInput(Inputs.Path, inputPath);
const cacheId = 4;
const reserveCacheMock = jest.spyOn(cacheHttpClient, "reserveCache").mockImplementationOnce(() => {
return Promise.resolve(cacheId);
});
const execMock = jest.spyOn(exec, "exec"); const execMock = jest.spyOn(exec, "exec");
const saveCacheMock = jest const saveCacheMock = jest
@ -269,26 +274,29 @@ test("save with server error outputs warning", async () => {
await run(); await run();
expect(reserveCacheMock).toHaveBeenCalledTimes(1);
expect(reserveCacheMock).toHaveBeenCalledWith(primaryKey);
const archivePath = path.join("/foo/bar", "cache.tgz"); const archivePath = path.join("/foo/bar", "cache.tgz");
const IS_WINDOWS = process.platform === "win32"; const IS_WINDOWS = process.platform === "win32";
const args = IS_WINDOWS const args = IS_WINDOWS
? [ ? [
"-cz", "-cz",
"--force-local", "--force-local",
"-f", "-f",
archivePath.replace(/\\/g, "/"), archivePath.replace(/\\/g, "/"),
"-C", "-C",
cachePath.replace(/\\/g, "/"), cachePath.replace(/\\/g, "/"),
"." "."
] ]
: ["-cz", "-f", archivePath, "-C", cachePath, "."]; : ["-cz", "-f", archivePath, "-C", cachePath, "."];
expect(execMock).toHaveBeenCalledTimes(1); expect(execMock).toHaveBeenCalledTimes(1);
expect(execMock).toHaveBeenCalledWith(`"tar"`, args); expect(execMock).toHaveBeenCalledWith(`"tar"`, args);
expect(saveCacheMock).toHaveBeenCalledTimes(1); expect(saveCacheMock).toHaveBeenCalledTimes(1);
expect(saveCacheMock).toHaveBeenCalledWith(primaryKey, archivePath); expect(saveCacheMock).toHaveBeenCalledWith(cacheId, archivePath);
expect(logWarningMock).toHaveBeenCalledTimes(1); expect(logWarningMock).toHaveBeenCalledTimes(1);
expect(logWarningMock).toHaveBeenCalledWith("HTTP Error Occurred"); expect(logWarningMock).toHaveBeenCalledWith("HTTP Error Occurred");
@ -321,32 +329,40 @@ test("save with valid inputs uploads a cache", async () => {
const cachePath = path.resolve(inputPath); const cachePath = path.resolve(inputPath);
testUtils.setInput(Inputs.Path, inputPath); testUtils.setInput(Inputs.Path, inputPath);
const cacheId = 4;
const reserveCacheMock = jest.spyOn(cacheHttpClient, "reserveCache").mockImplementationOnce(() => {
return Promise.resolve(cacheId);
});
const execMock = jest.spyOn(exec, "exec"); const execMock = jest.spyOn(exec, "exec");
const saveCacheMock = jest.spyOn(cacheHttpClient, "saveCache"); const saveCacheMock = jest.spyOn(cacheHttpClient, "saveCache");
await run(); await run();
expect(reserveCacheMock).toHaveBeenCalledTimes(1);
expect(reserveCacheMock).toHaveBeenCalledWith(primaryKey);
const archivePath = path.join("/foo/bar", "cache.tgz"); const archivePath = path.join("/foo/bar", "cache.tgz");
const IS_WINDOWS = process.platform === "win32"; const IS_WINDOWS = process.platform === "win32";
const args = IS_WINDOWS const args = IS_WINDOWS
? [ ? [
"-cz", "-cz",
"--force-local", "--force-local",
"-f", "-f",
archivePath.replace(/\\/g, "/"), archivePath.replace(/\\/g, "/"),
"-C", "-C",
cachePath.replace(/\\/g, "/"), cachePath.replace(/\\/g, "/"),
"." "."
] ]
: ["-cz", "-f", archivePath, "-C", cachePath, "."]; : ["-cz", "-f", archivePath, "-C", cachePath, "."];
expect(execMock).toHaveBeenCalledTimes(1); expect(execMock).toHaveBeenCalledTimes(1);
expect(execMock).toHaveBeenCalledWith(`"tar"`, args); expect(execMock).toHaveBeenCalledWith(`"tar"`, args);
expect(saveCacheMock).toHaveBeenCalledTimes(1); expect(saveCacheMock).toHaveBeenCalledTimes(1);
expect(saveCacheMock).toHaveBeenCalledWith(primaryKey, archivePath); expect(saveCacheMock).toHaveBeenCalledWith(cacheId, archivePath);
expect(failedMock).toHaveBeenCalledTimes(0); expect(failedMock).toHaveBeenCalledTimes(0);
}); });

View File

@ -107,9 +107,7 @@ export async function downloadCache(
} }
// Reserve Cache // Reserve Cache
export async function reserveCache( export async function reserveCache(key: string): Promise<number> {
key: string
): Promise<number> {
const restClient = createRestClient(); const restClient = createRestClient();
const reserveCacheRequest: ReserveCacheRequest = { const reserveCacheRequest: ReserveCacheRequest = {
@ -133,14 +131,6 @@ function getContentRange(start: number, end: number): string {
return `bytes ${start}-${end}/*`; return `bytes ${start}-${end}/*`;
} }
// function bufferToStream(buffer: Buffer): NodeJS.ReadableStream {
// const stream = new Duplex();
// stream.push(buffer);
// stream.push(null);
// return stream;
// }
async function uploadChunk( async function uploadChunk(
restClient: RestClient, restClient: RestClient,
resourceUrl: string, resourceUrl: string,
@ -148,14 +138,87 @@ async function uploadChunk(
start: number, start: number,
end: number end: number
): Promise<IRestResponse<void>> { ): Promise<IRestResponse<void>> {
core.debug(`Uploading chunk of size ${end - start + 1} bytes at offset ${start} with content range: ${getContentRange(start, end)}`); core.debug(
`Uploading chunk of size ${end -
start +
1} bytes at offset ${start} with content range: ${getContentRange(
start,
end
)}`
);
const requestOptions = getRequestOptions(); const requestOptions = getRequestOptions();
requestOptions.additionalHeaders = { requestOptions.additionalHeaders = {
"Content-Type": "application/octet-stream", "Content-Type": "application/octet-stream",
"Content-Range": getContentRange(start, end) "Content-Range": getContentRange(start, end)
}; };
return await restClient.uploadStream<void>("PATCH", resourceUrl, data, requestOptions); return await restClient.uploadStream<void>(
"PATCH",
resourceUrl,
data,
requestOptions
);
}
async function uploadFile(
restClient: RestClient,
cacheId: number,
archivePath: string
): Promise<void> {
// Upload Chunks
const fileSize = fs.statSync(archivePath).size;
const resourceUrl = getCacheApiUrl() + "caches/" + cacheId.toString();
const responses: IRestResponse<void>[] = [];
const fd = fs.openSync(archivePath, "r");
const concurrency = 4; // # of HTTP requests in parallel
const MAX_CHUNK_SIZE = 32000000; // 32 MB Chunks
core.debug(`Concurrency: ${concurrency} and Chunk Size: ${MAX_CHUNK_SIZE}`);
const parallelUploads = [...new Array(concurrency).keys()];
core.debug("Awaiting all uploads");
let offset = 0;
await Promise.all(
parallelUploads.map(async () => {
while (offset < fileSize) {
const chunkSize =
offset + MAX_CHUNK_SIZE > fileSize
? fileSize - offset
: MAX_CHUNK_SIZE;
const start = offset;
const end = offset + chunkSize - 1;
offset += MAX_CHUNK_SIZE;
const chunk = fs.createReadStream(archivePath, {
fd,
start,
end,
autoClose: false
});
responses.push(
await uploadChunk(
restClient,
resourceUrl,
chunk,
start,
end
)
);
}
})
);
fs.closeSync(fd);
const failedResponse = responses.find(
x => !isSuccessStatusCode(x.statusCode)
);
if (failedResponse) {
throw new Error(
`Cache service responded with ${failedResponse.statusCode} during chunk upload.`
);
}
return;
} }
async function commitCache( async function commitCache(
@ -172,44 +235,6 @@ async function commitCache(
); );
} }
async function uploadFile(restClient: RestClient, cacheId: number, archivePath: string): Promise<void> {
// Upload Chunks
const fileSize = fs.statSync(archivePath).size;
const resourceUrl = getCacheApiUrl() + "caches/" + cacheId.toString();
const responses: IRestResponse<void>[] = [];
const fd = fs.openSync(archivePath, "r");
const concurrency = 16; // # of HTTP requests in parallel
const MAX_CHUNK_SIZE = 32000000; // 32 MB Chunks
core.debug(`Concurrency: ${concurrency} and Chunk Size: ${MAX_CHUNK_SIZE}`);
const parallelUploads = [...new Array(concurrency).keys()];
core.debug("Awaiting all uploads");
let offset = 0;
await Promise.all(parallelUploads.map(async () => {
while (offset < fileSize) {
const chunkSize = offset + MAX_CHUNK_SIZE > fileSize ? fileSize - offset : MAX_CHUNK_SIZE;
const start = offset;
const end = offset + chunkSize - 1;
offset += MAX_CHUNK_SIZE;
const chunk = fs.createReadStream(archivePath, { fd, start, end, autoClose: false });
responses.push(await uploadChunk(restClient, resourceUrl, chunk, start, end));
}
}));
fs.closeSync(fd);
const failedResponse = responses.find(
x => !isSuccessStatusCode(x.statusCode)
);
if (failedResponse) {
throw new Error(
`Cache service responded with ${failedResponse.statusCode} during chunk upload.`
);
}
return;
}
export async function saveCache( export async function saveCache(
cacheId: number, cacheId: number,
archivePath: string archivePath: string
@ -219,8 +244,8 @@ export async function saveCache(
core.debug("Upload cache"); core.debug("Upload cache");
await uploadFile(restClient, cacheId, archivePath); await uploadFile(restClient, cacheId, archivePath);
core.debug("Commiting cache");
// Commit Cache // Commit Cache
core.debug("Commiting cache");
const cacheSize = utils.getArchiveFileSize(archivePath); const cacheSize = utils.getArchiveFileSize(archivePath);
const commitCacheResponse = await commitCache( const commitCacheResponse = await commitCache(
restClient, restClient,

View File

@ -82,7 +82,6 @@ async function run(): Promise<void> {
cacheEntry?.archiveLocation, cacheEntry?.archiveLocation,
archivePath archivePath
); );
await exec(`md5sum`, [archivePath]);
const archiveFileSize = utils.getArchiveFileSize(archivePath); const archiveFileSize = utils.getArchiveFileSize(archivePath);
core.info( core.info(

View File

@ -38,7 +38,9 @@ async function run(): Promise<void> {
core.debug("Reserving Cache"); core.debug("Reserving Cache");
const cacheId = await cacheHttpClient.reserveCache(primaryKey); const cacheId = await cacheHttpClient.reserveCache(primaryKey);
if (cacheId < 0) { if (cacheId < 0) {
core.info(`Unable to reserve cache with key ${primaryKey}, another job may be creating this cache.`); core.info(
`Unable to reserve cache with key ${primaryKey}, another job may be creating this cache.`
);
return; return;
} }
core.debug(`Cache ID: ${cacheId}`); core.debug(`Cache ID: ${cacheId}`);
@ -84,7 +86,6 @@ async function run(): Promise<void> {
return; return;
} }
await exec(`md5sum`, [archivePath]);
core.debug("Saving Cache"); core.debug("Saving Cache");
await cacheHttpClient.saveCache(cacheId, archivePath); await cacheHttpClient.saveCache(cacheId, archivePath);
} catch (error) { } catch (error) {