Skip to content

Commit 7bce109

Browse files
rose-mrose-m
authored
chore(build): wait until packages are really available in PPAs MONGOSH-619 (#685)
1 parent 2990153 commit 7bce109

File tree

4 files changed

+180
-36
lines changed

4 files changed

+180
-36
lines changed

‎packages/build/src/barque.spec.ts‎

Lines changed: 91 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -49,17 +49,38 @@ describe('Barque', () => {
4949

5050
describe('releaseToBarque', () => {
5151
context('platform is linux', () => {
52-
it('execCurator function succeeds', async() => {
53-
barque.execCurator = sinon.stub().returns(Promise.resolve(true));
54-
barque.createCuratorDir = sinon.stub().returns(Promise.resolve('./'));
55-
barque.extractLatestCurator = sinon.stub().returns(Promise.resolve(true));
52+
context('execCurator function succeeds', () => {
53+
[
54+
{
55+
variant: BuildVariant.Debian,
56+
url: 'https://s3.amazonaws.com/mciuploads/mongosh/5ed7ee5d8683818eb28d9d3b5c65837cde4a08f5/mongosh_0.1.0_amd64.deb',
57+
publishedUrls: [
58+
`${Barque.PPA_REPO_BASE_URL}/apt/debian/dists/buster/mongodb-org/4.4/main/binary-amd64/mongosh_0.1.0_amd64.deb`,
59+
`${Barque.PPA_REPO_BASE_URL}/apt/ubuntu/dists/bionic/mongodb-org/4.4/multiverse/binary-amd64/mongosh_0.1.0_amd64.deb`,
60+
`${Barque.PPA_REPO_BASE_URL}/apt/ubuntu/dists/focal/mongodb-org/4.4/multiverse/binary-amd64/mongosh_0.1.0_amd64.deb`,
61+
]
62+
},
63+
{
64+
variant: BuildVariant.Redhat,
65+
url: 'https://s3.amazonaws.com/mciuploads/mongosh/5ed7ee5d8683818eb28d9d3b5c65837cde4a08f5/mongosh-0.1.0-x86_64.rpm',
66+
publishedUrls: [
67+
`${Barque.PPA_REPO_BASE_URL}/yum/redhat/8/mongodb-org/4.4/x86_64/RPMS/mongosh-0.1.0-x86_64.rpm`,
68+
]
69+
}
70+
].forEach(({ variant, url, publishedUrls }) => {
71+
it(`publishes ${variant} packages`, async() => {
72+
barque.execCurator = sinon.stub().resolves(true);
73+
barque.createCuratorDir = sinon.stub().resolves('./');
74+
barque.extractLatestCurator = sinon.stub().resolves(true);
5675

57-
const debUrl = 'https://s3.amazonaws.com/mciuploads/mongosh/5ed7ee5d8683818eb28d9d3b5c65837cde4a08f5/mongosh_0.1.0_amd64.deb';
76+
const releasedUrls = await barque.releaseToBarque(variant, url);
5877

59-
await barque.releaseToBarque(BuildVariant.Debian, debUrl);
60-
expect(barque.createCuratorDir).to.have.been.called;
61-
expect(barque.extractLatestCurator).to.have.been.called;
62-
expect(barque.execCurator).to.have.been.called;
78+
expect(releasedUrls).to.deep.equal(publishedUrls);
79+
expect(barque.createCuratorDir).to.have.been.called;
80+
expect(barque.extractLatestCurator).to.have.been.called;
81+
expect(barque.execCurator).to.have.been.called;
82+
});
83+
});
6384
});
6485

6586
it('execCurator function fails', async() => {
@@ -84,18 +105,13 @@ describe('Barque', () => {
84105

85106
it('platform is not linux', async() => {
86107
config.platform = 'macos';
87-
barque = new Barque(config);
88-
89-
barque.execCurator = sinon.stub().returns(Promise.resolve(true));
90-
barque.createCuratorDir = sinon.stub().returns(Promise.resolve('./'));
91-
barque.extractLatestCurator = sinon.stub().returns(Promise.resolve(true));
92-
93-
const debUrl = 'https://s3.amazonaws.com/mciuploads/mongosh/5ed7ee5d8683818eb28d9d3b5c65837cde4a08f5/mongosh_0.1.0_linux.deb';
94-
95-
await barque.releaseToBarque(BuildVariant.Debian, debUrl);
96-
expect(barque.createCuratorDir).to.not.have.been.called;
97-
expect(barque.extractLatestCurator).to.not.have.been.called;
98-
expect(barque.execCurator).to.not.have.been.called;
108+
try {
109+
barque = new Barque(config);
110+
} catch (e) {
111+
expect(e.message).to.contain('only supported on linux');
112+
return;
113+
}
114+
expect.fail('Expected error');
99115
});
100116
});
101117

@@ -165,6 +181,60 @@ describe('Barque', () => {
165181
});
166182
});
167183

184+
describe('waitUntilPackagesAreAvailable', () => {
185+
beforeEach(() => {
186+
nock.cleanAll();
187+
});
188+
189+
context('with packages published one after the other', () => {
190+
let nockRepo: nock.Scope;
191+
192+
beforeEach(() => {
193+
nockRepo = nock(Barque.PPA_REPO_BASE_URL);
194+
195+
nockRepo.head('/apt/dist/package1.deb').reply(200);
196+
197+
nockRepo.head('/apt/dist/package2.deb').twice().reply(404);
198+
nockRepo.head('/apt/dist/package2.deb').reply(200);
199+
200+
nockRepo.head('/apt/dist/package3.deb').reply(404);
201+
nockRepo.head('/apt/dist/package3.deb').reply(200);
202+
});
203+
204+
it('waits until all packages are available', async() => {
205+
await barque.waitUntilPackagesAreAvailable([
206+
`${Barque.PPA_REPO_BASE_URL}/apt/dist/package1.deb`,
207+
`${Barque.PPA_REPO_BASE_URL}/apt/dist/package2.deb`,
208+
`${Barque.PPA_REPO_BASE_URL}/apt/dist/package3.deb`
209+
], 300, 1);
210+
211+
expect(nock.isDone()).to.be.true;
212+
});
213+
});
214+
215+
context('with really slow packages', () => {
216+
let nockRepo: nock.Scope;
217+
218+
beforeEach(() => {
219+
nockRepo = nock(Barque.PPA_REPO_BASE_URL);
220+
nockRepo.head('/apt/dist/package1.deb').reply(200);
221+
nockRepo.head('/apt/dist/package2.deb').reply(404).persist();
222+
});
223+
224+
it('fails when the timeout is hit', async() => {
225+
try {
226+
await barque.waitUntilPackagesAreAvailable([
227+
`${Barque.PPA_REPO_BASE_URL}/apt/dist/package1.deb`,
228+
`${Barque.PPA_REPO_BASE_URL}/apt/dist/package2.deb`,
229+
], 5, 1);
230+
} catch (e) {
231+
expect(e.message).to.contain('the following packages are still not available');
232+
expect(e.message).to.contain('package2.deb');
233+
}
234+
});
235+
});
236+
});
237+
168238
describe('LATEST_CURATOR', () => {
169239
it('can be downloaded', async() => {
170240
const response = await fetch(LATEST_CURATOR, {

‎packages/build/src/barque.ts‎

Lines changed: 79 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import path from 'path';
66
import stream from 'stream';
77
import tar from 'tar-fs';
88
import tmp from 'tmp-promise';
9-
import util from 'util';
9+
import util, { promisify } from 'util';
1010
import { BuildVariant, Config, Platform } from './config';
1111

1212
const pipeline = util.promisify(stream.pipeline);
@@ -42,11 +42,17 @@ enum Arch {
4242
}
4343

4444
export class Barque {
45+
public static readonly PPA_REPO_BASE_URL = 'https://repo.mongodb.org' as const;
46+
4547
private config: Config;
4648
private mongodbEdition: string;
4749
private mongodbVersion: string;
4850

4951
constructor(config: Config) {
52+
if (config.platform !== Platform.Linux) {
53+
throw new Error('Barque publishing is only supported on linux platforms');
54+
}
55+
5056
this.config = config;
5157
// hard code mongodb edition to 'org' for now
5258
this.mongodbEdition = 'org';
@@ -56,29 +62,31 @@ export class Barque {
5662
}
5763

5864
/**
59-
* Upload current package to barque, MongoDB's PPA for linux distros.
65+
* Upload a distributable package to barque, MongoDB's PPA for linux distros.
66+
*
67+
* Note that this method returns the URLs where the packages _will_ be available.
68+
* This method does not wait for the packages to really be available.
69+
* Use `waitUntilPackagesAreAvailable` for this purpose.
6070
*
61-
* @param {string} tarballURL- The uploaded to Evergreen tarball URL.
62-
* @param {Config} config - Config object.
71+
* @param buildVariant - The distributable package build variant to publish.
72+
* @param packageUrl - The Evergreen URL of the distributable package.
6373
*
64-
* @returns {Promise} The promise.
74+
* @returns The URLs where the packages will be available.
6575
*/
66-
async releaseToBarque(buildVariant: BuildVariant, tarballURL: string): Promise<any> {
67-
if (this.config.platform !== Platform.Linux) {
68-
return;
69-
}
70-
76+
async releaseToBarque(buildVariant: BuildVariant, packageUrl: string): Promise<string[]> {
7177
const repoConfig = path.join(this.config.rootDir, 'config', 'repo-config.yml');
7278
const curatorDirPath = await this.createCuratorDir();
7379
await this.extractLatestCurator(curatorDirPath);
7480

7581
const targetDistros = this.getTargetDistros(buildVariant);
7682
const targetArchitecture = this.getTargetArchitecture(buildVariant);
83+
84+
const publishedPackageUrls: string[] = [];
7785
for (const distro of targetDistros) {
7886
try {
7987
await this.execCurator(
8088
curatorDirPath,
81-
tarballURL,
89+
packageUrl,
8290
repoConfig,
8391
distro,
8492
targetArchitecture
@@ -87,12 +95,15 @@ export class Barque {
8795
console.error('Curator failed', error);
8896
throw new Error(`Curator is unable to upload to barque ${error}`);
8997
}
98+
99+
publishedPackageUrls.push(this.computePublishedPackageUrl(distro, targetArchitecture, packageUrl));
90100
}
101+
return publishedPackageUrls;
91102
}
92103

93104
async execCurator(
94105
curatorDirPath: string,
95-
tarballURL: string,
106+
packageUrl: string,
96107
repoConfig: string,
97108
distro: Distro,
98109
architecture: Arch
@@ -107,7 +118,7 @@ export class Barque {
107118
'--arch', architecture,
108119
'--edition', this.mongodbEdition,
109120
'--version', this.mongodbVersion,
110-
'--packages', tarballURL
121+
'--packages', packageUrl
111122
], {
112123
// curator looks for these options in env
113124
env: {
@@ -147,6 +158,61 @@ export class Barque {
147158
}
148159
}
149160

161+
computePublishedPackageUrl(distro: Distro, targetArchitecture: Arch, packageUrl: string): string {
162+
const packageFileName = packageUrl.split('/').slice(-1);
163+
const packageFolderVersion = this.mongodbVersion.split('.').slice(0, 2).join('.');
164+
switch (distro) {
165+
case Distro.Debian10:
166+
return `${Barque.PPA_REPO_BASE_URL}/apt/debian/dists/buster/mongodb-org/${packageFolderVersion}/main/binary-${targetArchitecture}/${packageFileName}`;
167+
case Distro.Ubuntu1804:
168+
return `${Barque.PPA_REPO_BASE_URL}/apt/ubuntu/dists/bionic/mongodb-org/${packageFolderVersion}/multiverse/binary-${targetArchitecture}/${packageFileName}`;
169+
case Distro.Ubuntu2004:
170+
return `${Barque.PPA_REPO_BASE_URL}/apt/ubuntu/dists/focal/mongodb-org/${packageFolderVersion}/multiverse/binary-${targetArchitecture}/${packageFileName}`;
171+
case Distro.Redhat80:
172+
return `${Barque.PPA_REPO_BASE_URL}/yum/redhat/8/mongodb-org/${packageFolderVersion}/${targetArchitecture}/RPMS/${packageFileName}`;
173+
default:
174+
throw new Error(`Unsupported distro: ${distro}`);
175+
}
176+
}
177+
178+
/**
179+
* Waits until the given packages are available under the specified URLs or throws an error if there
180+
* are still remaining packages after the timeout.
181+
*
182+
* Note that the method will try all URLs at least once after an initial delay of `sleepTimeSeconds`.
183+
*/
184+
async waitUntilPackagesAreAvailable(publishedPackageUrls: string[], timeoutSeconds: number, sleepTimeSeconds = 10): Promise<void> {
185+
let remainingPackages = [...publishedPackageUrls];
186+
const sleep = promisify(setTimeout);
187+
188+
const startMs = new Date().getTime();
189+
const failOnTimeout = () => {
190+
if (new Date().getTime() - startMs > timeoutSeconds * 1000) {
191+
throw new Error(`Barque timed out - the following packages are still not available: ${remainingPackages.join(', ')}`);
192+
}
193+
};
194+
195+
while (remainingPackages.length) {
196+
console.info(`Waiting for availability of:\n - ${remainingPackages.join('\n - ')}`);
197+
await sleep(sleepTimeSeconds * 1000);
198+
199+
const promises = remainingPackages.map(async url => await fetch(url, {
200+
method: 'HEAD'
201+
}));
202+
const responses = await Promise.all(promises);
203+
204+
const newRemainingPackages: string[] = [];
205+
for (let i = 0; i < remainingPackages.length; i++) {
206+
if (responses[i].status !== 200) {
207+
newRemainingPackages.push(remainingPackages[i]);
208+
}
209+
}
210+
remainingPackages = newRemainingPackages;
211+
212+
failOnTimeout();
213+
}
214+
}
215+
150216
/**
151217
* Create a staging dir in /tmp to download the latest version of curator.
152218
*

‎packages/build/src/run-publish.spec.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,8 @@ describe('publish', () => {
8181
githubRepo = createStubRepo();
8282
mongoHomebrewRepo = createStubRepo();
8383
barque = createStubBarque({
84-
releaseToBarque: sinon.stub().resolves(true)
84+
releaseToBarque: sinon.stub().resolves(['package-url']),
85+
waitUntilPackagesAreAvailable: sinon.stub().resolves()
8586
});
8687
});
8788

@@ -188,6 +189,7 @@ describe('publish', () => {
188189
BuildVariant.Debian,
189190
'https://s3.amazonaws.com/mciuploads/project/v0.7.0-draft.42/mongosh_0.7.0_amd64.deb'
190191
);
192+
expect(barque.waitUntilPackagesAreAvailable).to.have.been.called;
191193
});
192194

193195
it('updates the download center config', async() => {

‎packages/build/src/run-publish.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,11 +92,17 @@ async function publishArtifactsToBarque(
9292
BuildVariant.Debian,
9393
BuildVariant.Redhat
9494
];
95+
96+
const publishedPackages: string[] = [];
9597
for await (const variant of variantsForBarque) {
9698
const tarballName = getTarballFile(variant, releaseVersion, packageName);
9799
const tarballUrl = getEvergreenArtifactUrl(project, mostRecentDraftTag, tarballName.path);
98100
console.info(`mongosh: Publishing ${variant} artifact to barque ${tarballUrl}`);
99-
await barque.releaseToBarque(variant, tarballUrl);
101+
const packageUrls = await barque.releaseToBarque(variant, tarballUrl);
102+
publishedPackages.push(...packageUrls);
100103
}
104+
105+
await barque.waitUntilPackagesAreAvailable(publishedPackages, 300);
106+
101107
console.info('mongosh: Submitting to barque complete');
102108
}

0 commit comments

Comments
 (0)