Skip to content
This repository was archived by the owner on Aug 6, 2025. It is now read-only.

Commit d6ad55e

Browse files
branberrybranberry
authored
[DOP-3856]: Point enhanced app to existing buckets by default (#858)
* [DOP-3856]: Point enhanced app to existing buckets by default * [DOP-3856]: Update buckets * [DOP-3856]: Update mut to latest version, add additional env vars, update timeout for handleJobs, remove unnecessary message, and add deploy command for SQS * [DOP-3856]: Add types * [DOP-3856]: Address review comments * [DOP-3856]: Address review comments * [DOP-3856]: Address review comments and add additional error handling * [DOP-3856]: Refactor workflows * [DOP-3856]: Fix typo * [DOP-3856]: Update package.json * [DOP-3856]: Update package.json * [DOP-3856]: Update workflow names * [DOP-3856]: Update workflow names
1 parent e80d6d1 commit d6ad55e

24 files changed

+285
-428
lines changed

.github/workflows/deploy-stg-enhanced-webhooks.yml renamed to .github/workflows/deploy-prd-enhanced-webhooks.yml

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,15 @@
11
on:
22
push:
33
paths: ["api/v2/**", "cdk-infra/lib/constructs/api/**"]
4-
branches:
5-
- "master"
6-
- "integration"
4+
tags:
5+
- 'v*'
76

87
concurrency:
9-
group: environment-stg-enhanced-webhooks-${{ github.ref }}
8+
group: environment-prd-enhanced-webhooks-${{ github.ref }}
109
cancel-in-progress: true
11-
name: Deploy Staging ECS Enhanced Webhooks
10+
name: Deploy Production ECS Enhanced Webhooks
1211
jobs:
13-
deploy-stg:
12+
deploy-prd:
1413
runs-on: ubuntu-latest
1514
steps:
1615
- uses: actions/checkout@v2
@@ -27,5 +26,5 @@ jobs:
2726
run: |
2827
cd cdk-infra/
2928
npm ci
30-
npm run deploy:enhanced:webhooks
29+
npm run deploy:enhanced:webhooks -- -c env=prd
3130

.github/workflows/deploy-stg-enhanced-worker.yml renamed to .github/workflows/deploy-prd-enhanced-worker.yml

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,15 @@
11
on:
22
push:
33
paths: ["src/**", "cdk-infra/lib/constructs/worker/**"]
4-
branches:
5-
- "master"
6-
- "integration"
4+
tags:
5+
- 'v*'
76

87
concurrency:
9-
group: environment-stg-enhanced-worker-${{ github.ref }}
8+
group: environment-prd-enhanced-worker-${{ github.ref }}
109
cancel-in-progress: true
11-
name: Deploy Staging ECS Enhanced Worker
10+
name: Deploy Production ECS Enhanced Worker
1211
jobs:
13-
deploy-stg:
12+
deploy-prd:
1413
runs-on: ubuntu-latest
1514
steps:
1615
- uses: actions/checkout@v2
@@ -27,5 +26,5 @@ jobs:
2726
run: |
2827
cd cdk-infra/
2928
npm ci
30-
npm run deploy:enhanced:worker
29+
npm run deploy:enhanced:worker -- -c env=prd
3130

.github/workflows/deploy-stg-enhanced.yml renamed to .github/workflows/deploy-prd-enhanced.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
on: [workflow_dispatch]
22
concurrency:
3-
group: environment-stg-enhanced-${{ github.ref }}
3+
group: environment-prd-enhanced-${{ github.ref }}
44
cancel-in-progress: true
5-
name: Deploy Staging ECS Enhanced
5+
name: Deploy Production ECS Enhanced
66
jobs:
7-
deploy-stg:
7+
deploy-prd:
88
runs-on: ubuntu-latest
99
steps:
1010
- uses: actions/checkout@v2
@@ -22,5 +22,5 @@ jobs:
2222
npm ci
2323
cd cdk-infra/
2424
npm ci
25-
npm run deploy:enhanced
25+
npm run deploy:enhanced -- -c env=prd
2626

‎Dockerfile.enhanced‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ FROM ubuntu:20.04
2424
ARG WORK_DIRECTORY=/home/docsworker-xlarge
2525
ARG SNOOTY_PARSER_VERSION=0.14.0
2626
ARG SNOOTY_FRONTEND_VERSION=0.14.6
27-
ARG MUT_VERSION=0.10.3
27+
ARG MUT_VERSION=0.10.5
2828
ARG REDOC_CLI_VERSION=1.2.0
2929
ARG NPM_BASE_64_AUTH
3030
ARG NPM_EMAIL

‎api/controllers/v2/github.ts‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,8 +91,17 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise<APIGatewayPr
9191
body: errMsg,
9292
};
9393
}
94-
95-
const body = JSON.parse(event.body);
94+
let body;
95+
try {
96+
body = JSON.parse(event.body);
97+
} catch (e) {
98+
console.log('[TriggerBuild]: ERROR! Could not parse event.body', e);
99+
return {
100+
statusCode: 502,
101+
headers: { 'Content-Type': 'text/plain' },
102+
body: ' ERROR! Could not parse event.body',
103+
};
104+
}
96105

97106
if (body.deleted) {
98107
return {

‎api/controllers/v2/jobs.ts‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,14 +58,13 @@ export const HandleJobs = async (event: SQSEvent): Promise<void> => {
5858
await Promise.all(
5959
messages.map(async (message: SQSRecord) => {
6060
const consoleLogger = new ConsoleLogger();
61+
console.log('SQS RECORD: ', message);
6162
const body = JSON.parse(message.body);
6263
const jobId = body['jobId'];
6364
const jobStatus = body['jobStatus'];
65+
6466
try {
6567
switch (jobStatus) {
66-
case JobStatus[JobStatus.inQueue]:
67-
await NotifyBuildProgress(jobId);
68-
break;
6968
case JobStatus[JobStatus.inProgress]:
7069
await NotifyBuildProgress(jobId);
7170
break;

‎cdk-infra/lib/constructs/api/webhook-api-construct.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ export class WebhookApiConstruct extends Construct {
8989
handler: 'HandleJobs',
9090
environment,
9191
bundling,
92+
timeout,
9293
});
9394

9495
// generic handler for the root endpoint

‎cdk-infra/lib/constructs/api/webhook-env-construct.ts‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ import { IQueue } from 'aws-cdk-lib/aws-sqs';
22
import { StringParameter } from 'aws-cdk-lib/aws-ssm';
33
import { Construct } from 'constructs';
44
import { getSsmPathPrefix } from '../../../utils/ssm';
5+
import { getDashboardUrl } from '../../../utils/slack';
6+
import { getEnv } from '../../../utils/env';
57

68
interface WebhookEnvConstructProps {
79
jobsQueue: IQueue;
@@ -15,24 +17,31 @@ export class WebhookEnvConstruct extends Construct {
1517
super(scope, id);
1618

1719
const ssmPrefix = getSsmPathPrefix();
20+
const env = getEnv();
1821

1922
const dbName = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/dbname`);
2023
const dbUsername = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/username`);
2124
const dbHost = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/host`);
2225
const jobCollection = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/collections/job/queue`);
23-
26+
const entitlementCollection = StringParameter.valueFromLookup(
27+
this,
28+
`${ssmPrefix}/atlas/collections/user/entitlements`
29+
);
2430
const dbPassword = secureStrings['MONGO_ATLAS_PASSWORD'];
2531
this.environment = {
2632
...secureStrings,
2733
MONGO_ATLAS_USERNAME: dbUsername,
2834
MONGO_ATLAS_PASSWORD: dbPassword,
35+
MONGO_ATLAS_HOST: dbHost,
2936
MONGO_ATLAS_URL: `mongodb+srv://${dbUsername}:${dbPassword}@${dbHost}/admin?retryWrites=true`,
3037
DB_NAME: dbName,
3138
JOB_QUEUE_COL_NAME: jobCollection,
3239
NODE_CONFIG_DIR: './config',
3340
JOBS_QUEUE_URL: jobsQueue.queueUrl,
3441
JOB_UPDATES_QUEUE_URL: jobUpdatesQueue.queueUrl,
3542
NODE_OPTIONS: '--enable-source-maps',
43+
USER_ENTITLEMENT_COL_NAME: entitlementCollection,
44+
DASHBOARD_URL: getDashboardUrl(env, jobCollection),
3645
};
3746
}
3847
}

‎cdk-infra/lib/constructs/queue/queues-construct.ts‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,25 @@ export class AutoBuilderQueuesConstruct extends Construct {
88
constructor(scope: Construct, id: string) {
99
super(scope, id);
1010

11-
const jobsQueue = new Queue(this, 'JobsQueue');
12-
const jobUpdatesQueue = new Queue(this, 'JobUpdatesQueue');
11+
const maxReceiveCount = 3;
12+
13+
const jobsQueueDlq = new Queue(this, 'jobsQueueDlq');
14+
15+
const jobsQueue = new Queue(this, 'JobsQueue', {
16+
deadLetterQueue: {
17+
queue: jobsQueueDlq,
18+
maxReceiveCount,
19+
},
20+
});
21+
22+
const jobUpdatesQueueDlq = new Queue(this, 'jobUpdatesQueueDlq');
23+
24+
const jobUpdatesQueue = new Queue(this, 'JobUpdatesQueue', {
25+
deadLetterQueue: {
26+
queue: jobUpdatesQueueDlq,
27+
maxReceiveCount,
28+
},
29+
});
1330

1431
this.jobsQueue = jobsQueue;
1532
this.jobUpdatesQueue = jobUpdatesQueue;

‎cdk-infra/lib/constructs/worker/buckets-construct.ts‎

Lines changed: 14 additions & 144 deletions
Original file line numberDiff line numberDiff line change
@@ -1,159 +1,29 @@
11
import { Construct } from 'constructs';
2-
import { BlockPublicAccess, Bucket, RedirectProtocol, RoutingRule } from 'aws-cdk-lib/aws-s3';
3-
import { RemovalPolicy } from 'aws-cdk-lib';
4-
import { docsBucketNames } from '../../../utils/buckets';
5-
import { getEnv, getFeatureName } from '../../../utils/env';
6-
import { getHostUrl, getPrefixUrl } from '../../../utils/url';
2+
import { Bucket, IBucket } from 'aws-cdk-lib/aws-s3';
3+
import { createCustomBucket, docsBucketNames } from '../../../utils/buckets';
4+
import { getEnv, getFeatureName, getUseCustomBuckets } from '../../../utils/env';
75

86
export class WorkerBucketsConstruct extends Construct {
9-
readonly buckets: Bucket[];
7+
readonly buckets: IBucket[];
108
constructor(scope: Construct, id: string) {
119
super(scope, id);
1210

1311
const env = getEnv();
1412

15-
const buckets: Bucket[] = [];
16-
17-
docsBucketNames.forEach((bucketName) => {
18-
let websiteRoutingRules: RoutingRule[] | undefined;
19-
20-
if (bucketName === 'docs-mongodb-org') {
21-
const hostName = getHostUrl(env);
22-
const prefixUrl = getPrefixUrl(env);
23-
// docs-mongodb-org has specific routing roles that the rest of the buckets do not have
24-
websiteRoutingRules = [
25-
{
26-
condition: {
27-
keyPrefixEquals: `${prefixUrl}/master`,
28-
},
29-
protocol: RedirectProtocol.HTTPS,
30-
hostName,
31-
replaceKey: {
32-
prefixWithKey: `${prefixUrl}/upcoming`,
33-
},
34-
},
35-
{
36-
condition: {
37-
keyPrefixEquals: `${prefixUrl}/atlas/cli/v1.3.0`,
38-
},
39-
protocol: RedirectProtocol.HTTPS,
40-
hostName,
41-
replaceKey: {
42-
prefixWithKey: `${prefixUrl}/atlas/cli/v1.3`,
43-
},
44-
},
45-
{
46-
condition: {
47-
keyPrefixEquals: `${prefixUrl}/atlas/cli/v1.2.1`,
48-
},
49-
protocol: RedirectProtocol.HTTPS,
50-
hostName,
51-
replaceKey: {
52-
prefixWithKey: `${prefixUrl}/atlas/cli/v1.2`,
53-
},
54-
},
55-
{
56-
condition: {
57-
keyPrefixEquals: `${prefixUrl}/atlas/cli/v1.2.0`,
58-
},
59-
protocol: RedirectProtocol.HTTPS,
60-
hostName,
61-
replaceKey: {
62-
prefixWithKey: `${prefixUrl}/atlas/cli/v1.2`,
63-
},
64-
},
65-
{
66-
condition: {
67-
keyPrefixEquals: `${prefixUrl}/atlas/cli/v1.1.7`,
68-
},
69-
protocol: RedirectProtocol.HTTPS,
70-
hostName,
71-
replaceKey: {
72-
prefixWithKey: `${prefixUrl}/atlas/cli/v1.1`,
73-
},
74-
},
75-
{
76-
condition: {
77-
keyPrefixEquals: `${prefixUrl}/atlas/cli/v1.1.6`,
78-
},
79-
protocol: RedirectProtocol.HTTPS,
80-
hostName,
81-
replaceKey: {
82-
prefixWithKey: `${prefixUrl}/atlas/cli/v1.1`,
83-
},
84-
},
85-
{
86-
condition: {
87-
keyPrefixEquals: `${prefixUrl}/atlas/cli/v1.1.5`,
88-
},
89-
protocol: RedirectProtocol.HTTPS,
90-
hostName,
91-
replaceKey: {
92-
prefixWithKey: `${prefixUrl}/atlas/cli/v1.1`,
93-
},
94-
},
95-
{
96-
condition: {
97-
keyPrefixEquals: `${prefixUrl}/atlas/cli/v1.1.4`,
98-
},
99-
protocol: RedirectProtocol.HTTPS,
100-
hostName,
101-
replaceKey: {
102-
prefixWithKey: `${prefixUrl}/atlas/cli/v1.1`,
103-
},
104-
},
105-
{
106-
condition: {
107-
keyPrefixEquals: `${prefixUrl}/atlas/cli/v1.1.3`,
108-
},
109-
protocol: RedirectProtocol.HTTPS,
110-
hostName,
111-
replaceKey: {
112-
prefixWithKey: `${prefixUrl}/atlas/cli/v1.1`,
113-
},
114-
},
115-
{
116-
condition: {
117-
keyPrefixEquals: `${prefixUrl}/atlas/cli/v1.1.2`,
118-
},
119-
protocol: RedirectProtocol.HTTPS,
120-
hostName,
121-
replaceKey: {
122-
prefixWithKey: `${prefixUrl}/atlas/cli/v1.1`,
123-
},
124-
},
125-
{
126-
condition: {
127-
keyPrefixEquals: `${prefixUrl}/atlas/cli/v1.1.0`,
128-
},
129-
protocol: RedirectProtocol.HTTPS,
130-
hostName,
131-
replaceKey: {
132-
prefixWithKey: `${prefixUrl}/atlas/cli/v1.1`,
133-
},
134-
},
135-
];
136-
}
137-
13+
const buckets: IBucket[] = docsBucketNames.map((bucketName) => {
13814
const featureName = getFeatureName();
15+
const useCustomBuckets = getUseCustomBuckets();
16+
17+
// If we want to use buckets that don't currently exist, we can call this method to create
18+
// them for individual testing purposes
19+
if (useCustomBuckets) return createCustomBucket({ scope: this, featureName, env, bucketName });
13920

140-
const stackBucketName = `${featureName}-${bucketName}-${env}`.toLowerCase();
21+
const bucketEnv = env === 'prd' ? 'prd-staging' : env;
22+
const bucketConstructId = `${featureName}-${bucketName}-${bucketEnv}`.toLowerCase();
14123

142-
const bucket = new Bucket(this, stackBucketName, {
143-
removalPolicy: RemovalPolicy.DESTROY,
144-
websiteRoutingRules,
145-
bucketName: stackBucketName,
146-
websiteIndexDocument: 'index.html',
147-
websiteErrorDocument: 'docs-qa/404/index.html',
148-
blockPublicAccess: new BlockPublicAccess({
149-
blockPublicAcls: false,
150-
blockPublicPolicy: false,
151-
ignorePublicAcls: false,
152-
restrictPublicBuckets: false,
153-
}),
154-
});
24+
const bucket = Bucket.fromBucketName(this, bucketConstructId, `${bucketName}-${bucketEnv}`);
15525

156-
buckets.push(bucket);
26+
return bucket;
15727
});
15828

15929
this.buckets = buckets;

0 commit comments

Comments
 (0)