Skip to content

Commit a3ce58a

Browse files
kormidekormide
authored
refactor: model release artifacts (#240)
1 parent 026d8c3 commit a3ce58a

File tree

8 files changed

+506
-313
lines changed

8 files changed

+506
-313
lines changed

‎dist/cli/index.js‎

Lines changed: 72 additions & 72 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/domain/BUILD.bazel‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ load("//bazel/ts:defs.bzl", "ts_project")
44
ts_project(
55
name = "domain",
66
srcs = [
7+
"artifact.ts",
78
"configuration.ts",
89
"create-entry.ts",
910
"error.ts",
@@ -41,6 +42,7 @@ ts_project(
4142
name = "domain_tests",
4243
testonly = True,
4344
srcs = [
45+
"artifact.spec.ts",
4446
"configuration.spec.ts",
4547
"create-entry.spec.ts",
4648
"find-registry-fork.spec.ts",

‎src/domain/artifact.spec.ts‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
import { randomUUID } from 'node:crypto';
2+
import fs, { WriteStream } from 'node:fs';
3+
import os from 'node:os';
4+
import path from 'node:path';
5+
6+
import axios from 'axios';
7+
import axiosRetry from 'axios-retry';
8+
import { mocked } from 'jest-mock';
9+
10+
import { expectThrownError } from '../test/util';
11+
import { Artifact, ArtifactDownloadError } from './artifact';
12+
import { computeIntegrityHash } from './integrity-hash';
13+
14+
jest.mock('node:fs');
15+
jest.mock('node:os');
16+
jest.mock('axios');
17+
jest.mock('axios-retry');
18+
jest.mock('./integrity-hash');
19+
20+
const ARTIFACT_URL = 'https://foo.bar/artifact.baz';
21+
const TEMP_DIR = '/tmp';
22+
const TEMP_FOLDER = 'artifact-1234';
23+
24+
beforeEach(() => {
25+
mocked(axios.get).mockReturnValue(
26+
Promise.resolve({
27+
data: {
28+
pipe: jest.fn(),
29+
},
30+
status: 200,
31+
})
32+
);
33+
34+
mocked(fs.createWriteStream).mockReturnValue({
35+
on: jest.fn((event: string, func: (...args: any[]) => unknown) => {
36+
if (event === 'finish') {
37+
func();
38+
}
39+
}),
40+
} as any);
41+
42+
mocked(os.tmpdir).mockReturnValue(TEMP_DIR);
43+
mocked(fs.mkdtempSync).mockReturnValue(path.join(TEMP_DIR, TEMP_FOLDER));
44+
mocked(computeIntegrityHash).mockReturnValue(`sha256-${randomUUID()}`);
45+
});
46+
47+
describe('Artifact', () => {
48+
describe('download', () => {
49+
test('downloads the artifact', async () => {
50+
const artifact = new Artifact(ARTIFACT_URL);
51+
await artifact.download();
52+
53+
expect(axios.get).toHaveBeenCalledWith(ARTIFACT_URL, {
54+
responseType: 'stream',
55+
});
56+
});
57+
58+
test('retries the request if it fails', async () => {
59+
const artifact = new Artifact(ARTIFACT_URL);
60+
61+
// Restore the original behavior of exponentialDelay.
62+
mocked(axiosRetry.exponentialDelay).mockImplementation(
63+
jest.requireActual('axios-retry').exponentialDelay
64+
);
65+
66+
await artifact.download();
67+
68+
expect(axiosRetry).toHaveBeenCalledWith(axios, {
69+
retries: 3,
70+
71+
retryCondition: expect.matchesPredicate(
72+
// eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
73+
(retryConditionFn: Function) => {
74+
// Make sure HTTP 404 errors are retried.
75+
const notFoundError = { response: { status: 404 } };
76+
return retryConditionFn.call(this, notFoundError);
77+
}
78+
),
79+
// eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
80+
retryDelay: expect.matchesPredicate((retryDelayFn: Function) => {
81+
// Make sure the retry delays follow exponential backoff
82+
// and the final retry happens after at least 1 minute total
83+
// (in this case, at least 70 seconds).
84+
// Axios randomly adds an extra 0-20% of jitter to each delay.
85+
// Test upper bounds as well to ensure the workflow completes reasonably quickly
86+
// (in this case, no more than 84 seconds total).
87+
const firstRetryDelay = retryDelayFn.call(this, 0);
88+
const secondRetryDelay = retryDelayFn.call(this, 1);
89+
const thirdRetryDelay = retryDelayFn.call(this, 2);
90+
return (
91+
10000 <= firstRetryDelay &&
92+
firstRetryDelay <= 12000 &&
93+
20000 <= secondRetryDelay &&
94+
secondRetryDelay <= 24000 &&
95+
40000 <= thirdRetryDelay &&
96+
thirdRetryDelay <= 48000
97+
);
98+
}),
99+
shouldResetTimeout: true,
100+
});
101+
});
102+
103+
test('saves the artifact to disk', async () => {
104+
const artifact = new Artifact(ARTIFACT_URL);
105+
106+
await artifact.download();
107+
108+
const expectedPath = path.join(TEMP_DIR, TEMP_FOLDER, 'artifact.baz');
109+
expect(fs.createWriteStream).toHaveBeenCalledWith(expectedPath, {
110+
flags: 'w',
111+
});
112+
113+
const mockedAxiosResponse = await (mocked(axios.get).mock.results[0]
114+
.value as Promise<{ data: { pipe: Function } }>); // eslint-disable-line @typescript-eslint/no-unsafe-function-type
115+
const mockedWriteStream = mocked(fs.createWriteStream).mock.results[0]
116+
.value as WriteStream;
117+
118+
expect(mockedAxiosResponse.data.pipe).toHaveBeenCalledWith(
119+
mockedWriteStream
120+
);
121+
});
122+
123+
test('sets the diskPath', async () => {
124+
const artifact = new Artifact(ARTIFACT_URL);
125+
126+
await artifact.download();
127+
128+
const expectedPath = path.join(TEMP_DIR, TEMP_FOLDER, 'artifact.baz');
129+
expect(artifact.diskPath).toEqual(expectedPath);
130+
});
131+
132+
test('throws on a non 200 status', async () => {
133+
const artifact = new Artifact(ARTIFACT_URL);
134+
135+
mocked(axios.get).mockRejectedValue({
136+
response: {
137+
status: 401,
138+
},
139+
});
140+
141+
const thrownError = await expectThrownError(
142+
() => artifact.download(),
143+
ArtifactDownloadError
144+
);
145+
146+
expect(thrownError.message.includes(ARTIFACT_URL)).toEqual(true);
147+
expect(thrownError.message.includes('401')).toEqual(true);
148+
});
149+
});
150+
151+
describe('computeIntegrityHash', () => {
152+
test('throws when artifact has not yet been downloaded', () => {
153+
const artifact = new Artifact(ARTIFACT_URL);
154+
155+
expect(() => artifact.computeIntegrityHash()).toThrowWithMessage(
156+
Error,
157+
`The artifact ${ARTIFACT_URL} must be downloaded before an integrity hash can be calculated`
158+
);
159+
});
160+
161+
test('computes the integrity of the file', async () => {
162+
const artifact = new Artifact(ARTIFACT_URL);
163+
await artifact.download();
164+
165+
const expected = `sha256-${randomUUID()}`;
166+
mocked(computeIntegrityHash).mockReturnValue(expected);
167+
168+
const actual = await artifact.computeIntegrityHash();
169+
170+
expect(expected).toEqual(actual);
171+
expect(computeIntegrityHash).toHaveBeenCalledWith(artifact.diskPath);
172+
});
173+
});
174+
175+
describe('cleanup', () => {
176+
test('removed the stored file', async () => {
177+
const artifact = new Artifact(ARTIFACT_URL);
178+
await artifact.download();
179+
const diskPath = artifact.diskPath;
180+
artifact.cleanup();
181+
182+
expect(fs.rmSync).toHaveBeenCalledWith(diskPath, { force: true });
183+
});
184+
185+
test('removes the diskPath', async () => {
186+
const artifact = new Artifact(ARTIFACT_URL);
187+
await artifact.download();
188+
artifact.cleanup();
189+
190+
expect(() => artifact.diskPath).toThrowWithMessage(
191+
Error,
192+
`The artifact ${ARTIFACT_URL} has not been downloaded yet`
193+
);
194+
});
195+
});
196+
});

‎src/domain/artifact.ts‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
import fs from 'node:fs';
2+
import os from 'node:os';
3+
import path from 'node:path';
4+
import { parse as parseUrl } from 'node:url';
5+
6+
import axios, { AxiosError, AxiosResponse } from 'axios';
7+
import axiosRetry from 'axios-retry';
8+
9+
import { computeIntegrityHash } from './integrity-hash.js';
10+
11+
export class ArtifactDownloadError extends Error {
12+
constructor(
13+
public readonly url: string,
14+
public readonly statusCode: number
15+
) {
16+
super(
17+
`Failed to download artifact from ${url}. Received status ${statusCode}`
18+
);
19+
}
20+
}
21+
22+
/**
23+
* An artifact that can be downloaded and have its integrity hash computed.
24+
*/
25+
export class Artifact {
26+
private _diskPath: string | null = null;
27+
public constructor(public readonly url: string) {}
28+
29+
public async download(): Promise<void> {
30+
let url = this.url;
31+
if (this._diskPath !== null) {
32+
throw new Error(
33+
`Artifact ${url} already downloaded to ${this._diskPath}`
34+
);
35+
}
36+
37+
const parsed = parseUrl(url);
38+
39+
if (process.env.INTEGRATION_TESTING) {
40+
// Point downloads to the standin github server
41+
// during integration testing.
42+
const [host, port] =
43+
process.env.GITHUB_API_ENDPOINT.split('://')[1].split(':');
44+
45+
parsed.host = host;
46+
parsed.port = port;
47+
48+
url = `http://${host}:${port}${parsed.path}`;
49+
}
50+
51+
const filename = path.basename(parsed.pathname);
52+
53+
const dest = path.join(
54+
fs.mkdtempSync(path.join(os.tmpdir(), 'artifact-')),
55+
filename
56+
);
57+
58+
// Support downloading from another location on disk.
59+
// Useful for swapping in a local path for e2e tests.
60+
if (url.startsWith('file://')) {
61+
fs.copyFileSync(url.substring('file://'.length), dest);
62+
this._diskPath = dest;
63+
return;
64+
}
65+
66+
const writer = fs.createWriteStream(dest, { flags: 'w' });
67+
68+
// Retry the request in case the artifact is still being uploaded.
69+
// Exponential backoff with 3 retries and a delay factor of 10 seconds
70+
// gives you at least 70 seconds to upload a release archive.
71+
axiosRetry(axios, {
72+
retries: 3,
73+
retryDelay: exponentialDelay,
74+
shouldResetTimeout: true,
75+
retryCondition: defaultRetryPlus404,
76+
});
77+
78+
let response: AxiosResponse;
79+
80+
try {
81+
response = await axios.get(url, {
82+
responseType: 'stream',
83+
});
84+
} catch (e: any) {
85+
// https://axios-http.com/docs/handling_errors
86+
if (e.response) {
87+
throw new ArtifactDownloadError(url, e.response.status);
88+
} else if (e.request) {
89+
throw new Error(`GET ${url} failed; no response received`);
90+
} else {
91+
throw new Error(`Failed to GET ${url} failed: ${e.message}`);
92+
}
93+
}
94+
95+
response.data.pipe(writer);
96+
97+
await new Promise((resolve, reject) => {
98+
writer.on('finish', () => {
99+
this._diskPath = dest;
100+
resolve(null);
101+
});
102+
writer.on('error', reject);
103+
});
104+
}
105+
106+
public get diskPath(): string {
107+
if (this._diskPath === null) {
108+
throw new Error(`The artifact ${this.url} has not been downloaded yet`);
109+
}
110+
111+
return this._diskPath;
112+
}
113+
114+
public computeIntegrityHash(): string {
115+
if (this._diskPath === null) {
116+
throw new Error(
117+
`The artifact ${this.url} must be downloaded before an integrity hash can be calculated`
118+
);
119+
}
120+
return computeIntegrityHash(this._diskPath);
121+
}
122+
123+
public cleanup(): void {
124+
fs.rmSync(this._diskPath, { force: true });
125+
this._diskPath = null;
126+
}
127+
}
128+
129+
function exponentialDelay(
130+
retryCount: number,
131+
error: AxiosError | undefined
132+
): number {
133+
// Default delay factor is 10 seconds, but can be overridden for testing.
134+
const delayFactor = Number(process.env.BACKOFF_DELAY_FACTOR) || 10_000;
135+
return axiosRetry.exponentialDelay(retryCount, error, delayFactor);
136+
}
137+
138+
function defaultRetryPlus404(error: AxiosError): boolean {
139+
// Publish-to-BCR needs to support retrying when GitHub returns 404
140+
// in order to support automated release workflows that upload artifacts
141+
// within a minute or so of publishing a release.
142+
// Apart from this case, use the default retry condition.
143+
return (
144+
error.response.status === 404 ||
145+
axiosRetry.isNetworkOrIdempotentRequestError(error)
146+
);
147+
}

0 commit comments

Comments
 (0)