Skip to content

Commit 3531018

Browse files
nirinchevgagik
nirinchev
and
gagik
authored
chore(releasing): validate tarball sha1 before publishing to homebrew MONGOSH-2059 (#2407)
* chore(releasing): validate tarball sha1 before publishing to homebrew * set encoding for json responses * Apply suggestions from code review Co-authored-by: Gagik Amaryan <[email protected]> --------- Co-authored-by: Gagik Amaryan <[email protected]>
1 parent 23d51db commit 3531018

File tree

5 files changed

+179
-34
lines changed

5 files changed

+179
-34
lines changed

‎.evergreen/verify-packaged-artifact.sh

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,13 @@ verify_using_gpg() {
3131
verify_using_powershell() {
3232
echo "Verifying $1 using powershell"
3333
powershell Get-AuthenticodeSignature -FilePath $ARTIFACTS_DIR/$1 > "$TMP_FILE" 2>&1
34+
35+
# Get-AuthenticodeSignature just outputs text, it doesn't exit with a non-zero
36+
# code if the file is not signed
37+
if grep -q NotSigned "$TMP_FILE"; then
38+
echo "File $1 is not signed"
39+
exit 1
40+
fi
3441
}
3542

3643
verify_using_codesign() {
@@ -91,4 +98,4 @@ else
9198
(cd "$ARTIFACTS_DIR" && bash "$BASEDIR/retry-with-backoff.sh" curl -sSfLO --url "$(cat "$ARTIFACT_URL_FILE").sig")
9299
verify_using_gpg $ARTIFACT_FILE_NAME
93100
fi
94-
fi
101+
fi

‎packages/build/src/homebrew/publish-to-homebrew.spec.ts

Lines changed: 11 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ describe('HomebrewPublisher', function () {
1010
let homebrewCore: GithubRepo;
1111
let homebrewCoreFork: GithubRepo;
1212
let createPullRequest: sinon.SinonStub;
13-
let httpsSha256: sinon.SinonStub;
13+
let npmPackageSha256: sinon.SinonStub;
1414
let generateFormula: sinon.SinonStub;
1515
let updateHomebrewFork: sinon.SinonStub;
1616

@@ -41,7 +41,7 @@ describe('HomebrewPublisher', function () {
4141
homebrewCoreFork,
4242
});
4343

44-
httpsSha256 = sinon.stub(testPublisher, 'httpsSha256');
44+
npmPackageSha256 = sinon.stub(testPublisher, 'npmPackageSha256');
4545
generateFormula = sinon.stub(testPublisher, 'generateFormula');
4646
updateHomebrewFork = sinon.stub(testPublisher, 'updateHomebrewFork');
4747
};
@@ -61,11 +61,9 @@ describe('HomebrewPublisher', function () {
6161
isDryRun: false,
6262
});
6363

64-
httpsSha256
64+
npmPackageSha256
6565
.rejects()
66-
.withArgs(
67-
'https://registry.npmjs.org/@mongosh/cli-repl/-/cli-repl-1.0.0.tgz'
68-
)
66+
.withArgs('https://registry.npmjs.org/@mongosh/cli-repl/1.0.0')
6967
.resolves('sha');
7068

7169
generateFormula
@@ -97,7 +95,7 @@ describe('HomebrewPublisher', function () {
9795

9896
await testPublisher.publish();
9997

100-
expect(httpsSha256).to.have.been.called;
98+
expect(npmPackageSha256).to.have.been.called;
10199
expect(generateFormula).to.have.been.called;
102100
expect(updateHomebrewFork).to.have.been.called;
103101
expect(createPullRequest).to.have.been.called;
@@ -110,11 +108,9 @@ describe('HomebrewPublisher', function () {
110108
isDryRun: false,
111109
});
112110

113-
httpsSha256
111+
npmPackageSha256
114112
.rejects()
115-
.withArgs(
116-
'https://registry.npmjs.org/@mongosh/cli-repl/-/cli-repl-1.0.0.tgz'
117-
)
113+
.withArgs('https://registry.npmjs.org/@mongosh/cli-repl/1.0.0')
118114
.resolves('sha');
119115

120116
generateFormula
@@ -136,18 +132,16 @@ describe('HomebrewPublisher', function () {
136132

137133
await testPublisher.publish();
138134

139-
expect(httpsSha256).to.have.been.called;
135+
expect(npmPackageSha256).to.have.been.called;
140136
expect(generateFormula).to.have.been.called;
141137
expect(updateHomebrewFork).to.have.been.called;
142138
expect(createPullRequest).to.not.have.been.called;
143139
});
144140

145141
it('silently ignores an error while deleting the PR branch', async function () {
146-
httpsSha256
142+
npmPackageSha256
147143
.rejects()
148-
.withArgs(
149-
'https://registry.npmjs.org/@mongosh/cli-repl/-/cli-repl-1.0.0.tgz'
150-
)
144+
.withArgs('https://registry.npmjs.org/@mongosh/cli-repl/1.0.0')
151145
.resolves('sha');
152146

153147
generateFormula
@@ -179,7 +173,7 @@ describe('HomebrewPublisher', function () {
179173

180174
await testPublisher.publish();
181175

182-
expect(httpsSha256).to.have.been.called;
176+
expect(npmPackageSha256).to.have.been.called;
183177
expect(generateFormula).to.have.been.called;
184178
expect(updateHomebrewFork).to.have.been.called;
185179
expect(createPullRequest).to.have.been.called;

‎packages/build/src/homebrew/publish-to-homebrew.ts

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import type { GithubRepo } from '@mongodb-js/devtools-github-repo';
22
import { generateUpdatedFormula as generateUpdatedFormulaFn } from './generate-formula';
33
import { updateHomebrewFork as updateHomebrewForkFn } from './update-homebrew-fork';
4-
import { httpsSha256 as httpsSha256Fn } from './utils';
4+
import { npmPackageSha256 as npmPackageSha256Fn } from './utils';
55

66
export type HomebrewPublisherConfig = {
77
homebrewCore: GithubRepo;
@@ -12,19 +12,19 @@ export type HomebrewPublisherConfig = {
1212
};
1313

1414
export class HomebrewPublisher {
15-
readonly httpsSha256: typeof httpsSha256Fn;
15+
readonly npmPackageSha256: typeof npmPackageSha256Fn;
1616
readonly generateFormula: typeof generateUpdatedFormulaFn;
1717
readonly updateHomebrewFork: typeof updateHomebrewForkFn;
1818

1919
constructor(
2020
public config: HomebrewPublisherConfig,
2121
{
22-
httpsSha256 = httpsSha256Fn,
22+
npmPackageSha256 = npmPackageSha256Fn,
2323
generateFormula = generateUpdatedFormulaFn,
2424
updateHomebrewFork = updateHomebrewForkFn,
2525
} = {}
2626
) {
27-
this.httpsSha256 = httpsSha256;
27+
this.npmPackageSha256 = npmPackageSha256;
2828
this.generateFormula = generateFormula;
2929
this.updateHomebrewFork = updateHomebrewFork;
3030
}
@@ -38,10 +38,10 @@ export class HomebrewPublisher {
3838
githubReleaseLink,
3939
} = this.config;
4040

41-
const cliReplPackageUrl = `https://registry.npmjs.org/@mongosh/cli-repl/-/cli-repl-${packageVersion}.tgz`;
41+
const cliReplPackageUrl = `https://registry.npmjs.org/@mongosh/cli-repl/${packageVersion}`;
4242
const packageSha = isDryRun
4343
? `dryRun-fakesha256-${Date.now()}`
44-
: await this.httpsSha256(cliReplPackageUrl);
44+
: await this.npmPackageSha256(cliReplPackageUrl);
4545

4646
const homebrewFormula = await this.generateFormula(
4747
{ version: packageVersion, sha: packageSha },

‎packages/build/src/homebrew/utils.spec.ts

Lines changed: 86 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,96 @@
11
import { expect } from 'chai';
2-
import { httpsSha256 } from './utils';
2+
import { npmPackageSha256 } from './utils';
3+
import sinon from 'sinon';
4+
import crypto from 'crypto';
35

46
describe('Homebrew utils', function () {
5-
describe('httpsSha256', function () {
7+
describe('npmPackageSha256', function () {
68
it('computes the correct sha', async function () {
7-
const url =
8-
'https://registry.npmjs.org/@mongosh/cli-repl/-/cli-repl-0.6.1.tgz';
9+
const url = 'https://registry.npmjs.org/@mongosh/cli-repl/0.6.1';
910
const expectedSha =
1011
'3721ea662cd3775373d4d70f7593993564563d9379704896478db1d63f6c8470';
1112

12-
expect(await httpsSha256(url)).to.equal(expectedSha);
13+
expect(await npmPackageSha256(url)).to.equal(expectedSha);
14+
});
15+
16+
describe('when response sha mismatches', function () {
17+
const fakeTarball = Buffer.from('mongosh-2.4.2.tgz');
18+
const fakeTarballShasum = crypto
19+
.createHash('sha1')
20+
.update(fakeTarball)
21+
.digest('hex');
22+
23+
it('retries', async function () {
24+
const httpGet = sinon.stub();
25+
httpGet
26+
.withArgs(
27+
'https://registry.npmjs.org/@mongosh/cli-repl/2.4.2',
28+
'json'
29+
)
30+
.resolves({
31+
dist: {
32+
tarball:
33+
'https://registry.npmjs.org/@mongosh/cli-repl/-/cli-repl-2.4.2.tgz',
34+
shasum: fakeTarballShasum,
35+
},
36+
});
37+
38+
httpGet
39+
.withArgs(
40+
'https://registry.npmjs.org/@mongosh/cli-repl/-/cli-repl-2.4.2.tgz',
41+
'binary'
42+
)
43+
.onFirstCall()
44+
.resolves(Buffer.from('mongosh-2.4.2-incomplete.tgz')) // Simulate incomplete/wrong binary download
45+
.onSecondCall()
46+
.resolves(fakeTarball);
47+
48+
const sha = await npmPackageSha256(
49+
'https://registry.npmjs.org/@mongosh/cli-repl/2.4.2',
50+
httpGet
51+
);
52+
53+
expect(sha).to.equal(
54+
crypto.createHash('sha256').update(fakeTarball).digest('hex')
55+
);
56+
});
57+
58+
it('throws if retries are exhausted', async function () {
59+
const httpGet = sinon.stub();
60+
httpGet
61+
.withArgs(
62+
'https://registry.npmjs.org/@mongosh/cli-repl/2.4.2',
63+
'json'
64+
)
65+
.resolves({
66+
dist: {
67+
tarball:
68+
'https://registry.npmjs.org/@mongosh/cli-repl/-/cli-repl-2.4.2.tgz',
69+
shasum: fakeTarballShasum,
70+
},
71+
});
72+
73+
httpGet
74+
.withArgs(
75+
'https://registry.npmjs.org/@mongosh/cli-repl/-/cli-repl-2.4.2.tgz',
76+
'binary'
77+
)
78+
.resolves(Buffer.from('mongosh-2.4.2-incomplete.tgz')); // Simulate incomplete/wrong binary download
79+
80+
const incompleteTarballShasum = crypto
81+
.createHash('sha1')
82+
.update(Buffer.from('mongosh-2.4.2-incomplete.tgz'))
83+
.digest('hex');
84+
85+
const err = await npmPackageSha256(
86+
'https://registry.npmjs.org/@mongosh/cli-repl/2.4.2',
87+
httpGet
88+
).catch((e) => e);
89+
90+
expect(err.message).to.equal(
91+
`shasum mismatch: expected '${fakeTarballShasum}', got '${incompleteTarballShasum}'`
92+
);
93+
});
1394
});
1495
});
1596
});

‎packages/build/src/homebrew/utils.ts

Lines changed: 68 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,76 @@
11
import crypto from 'crypto';
22
import https from 'https';
33

4-
export function httpsSha256(url: string): Promise<string> {
5-
return new Promise((resolve, reject) => {
4+
export async function npmPackageSha256(
5+
packageUrl: string,
6+
httpGetFn: typeof httpGet = httpGet
7+
): Promise<string> {
8+
const json = await httpGetFn<{
9+
dist: {
10+
tarball: string;
11+
shasum: string;
12+
}
13+
}>(packageUrl, 'json');
14+
const tarballUrl = json.dist.tarball;
15+
const shasum = json.dist.shasum;
16+
17+
const tarball = await getTarballWithRetries(tarballUrl, shasum, httpGetFn);
18+
const hash = crypto.createHash('sha256');
19+
hash.update(tarball);
20+
return hash.digest('hex');
21+
}
22+
23+
async function getTarballWithRetries(
24+
url: string,
25+
shasum: string,
26+
httpGetFn: typeof httpGet,
27+
attempts = 3
28+
): Promise<Buffer> {
29+
try {
30+
const tarball = await httpGetFn(url, 'binary');
31+
const hash = crypto.createHash('sha1').update(tarball).digest('hex');
32+
if (hash !== shasum) {
33+
throw new Error(`shasum mismatch: expected '${shasum}', got '${hash}'`);
34+
}
35+
36+
return tarball;
37+
} catch (err) {
38+
if (attempts === 0) {
39+
throw err;
40+
}
41+
42+
return getTarballWithRetries(url, shasum, httpGetFn, attempts - 1);
43+
}
44+
}
45+
46+
export function httpGet<T>(url: string, response: 'json'): Promise<T>;
47+
export function httpGet(url: string, response: 'binary'): Promise<Buffer>;
48+
export async function httpGet<T>(
49+
url: string,
50+
responseType: 'json' | 'binary'
51+
): Promise<T | Buffer> {
52+
const response = await new Promise<string | Buffer[]>((resolve, reject) => {
653
https.get(url, (stream) => {
7-
const hash = crypto.createHash('sha256');
54+
if (responseType === 'json') {
55+
stream.setEncoding('utf8');
56+
}
57+
58+
let data: string | Buffer[] = responseType === 'json' ? '' : [];
859
stream.on('error', (err) => reject(err));
9-
stream.on('data', (chunk) => hash.update(chunk));
10-
stream.on('end', () => resolve(hash.digest('hex')));
60+
stream.on('data', (chunk) => {
61+
if (typeof data === 'string') {
62+
data += chunk;
63+
} else {
64+
data.push(chunk);
65+
}
66+
});
67+
stream.on('end', () => resolve(data));
1168
});
1269
});
70+
71+
if (typeof response === 'string') {
72+
return JSON.parse(response);
73+
}
74+
75+
return Buffer.concat(response);
1376
}

0 commit comments

Comments
 (0)