Skip to content
This repository was archived by the owner on Aug 6, 2025. It is now read-only.

Commit 9422df1

Browse files
authored
DOP-3818: Create webhook to mark build data for deletion (#859)
1 parent d6ad55e commit 9422df1

19 files changed

+404
-70
lines changed

api/config/custom-environment-variables.json

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,11 @@
66
"dbUsername": "MONGO_ATLAS_USERNAME",
77
"dbPassword": "MONGO_ATLAS_PASSWORD",
88
"dbName": "DB_NAME",
9+
"snootyDbName": "SNOOTY_DB_NAME",
910
"dbUrl": "MONGO_ATLAS_URL",
1011
"fastlyDochubMap": "FASTLY_DOCHUB_MAP",
1112
"githubSecret": "GITHUB_SECRET",
13+
"githubDeletionSecret": "GITHUB_DELETION_SECRET",
1214
"githubBotPW": "GITHUB_BOT_PASSWORD",
1315
"slackSecret": "SLACK_SECRET",
1416
"slackAuthToken": "SLACK_TOKEN",

api/config/default.json

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,11 @@
66
"dbUsername": "username",
77
"dbPassword": "password",
88
"dbName": "db_name",
9+
"snootyDbName": "snooty_db_name",
910
"dbUrl": "url",
1011
"fastlyDochubMap": "FASTLY_DOCHUB_MAP",
1112
"githubSecret": "GITHUB_SECRET",
13+
"githubDeletionSecret": "GITHUB_DELETION_SECRET",
1214
"githubBotPW": "GITHUB_BOT_PASSWORD",
1315
"slackSecret": "SLACK_SECRET",
1416
"slackAuthToken": "SLACK_TOKEN",

api/controllers/v1/github.ts

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,14 @@
11
import * as c from 'config';
22
import * as crypto from 'crypto';
33
import * as mongodb from 'mongodb';
4+
import { APIGatewayEvent } from 'aws-lambda';
5+
import { PullRequestEvent } from '@octokit/webhooks-types';
46
import { JobRepository } from '../../../src/repositories/jobRepository';
57
import { ConsoleLogger } from '../../../src/services/logger';
68
import { BranchRepository } from '../../../src/repositories/branchRepository';
9+
import { RepoBranchesRepository } from '../../../src/repositories/repoBranchesRepository';
10+
import { MetadataRepository } from '../../../src/repositories/metadataRepository';
11+
import { UpdatedDocsRepository } from '../../../src/repositories/updatedDocsRepository';
712

813
// This function will validate your payload from GitHub
914
// See docs at https://developer.github.com/webhooks/securing/#validating-payloads-from-github
@@ -109,3 +114,101 @@ export const TriggerBuild = async (event: any = {}, context: any = {}): Promise<
109114
body: 'Job Queued',
110115
};
111116
};
117+
118+
/**
119+
* Deletes build artifacts for a given project + branch combination.
120+
* @param event
121+
*/
122+
export const MarkBuildArtifactsForDeletion = async (event: APIGatewayEvent) => {
123+
const consoleLogger = new ConsoleLogger();
124+
const defaultHeaders = { 'Content-Type': 'text/plain' };
125+
126+
const ghEventType = event.headers['X-GitHub-Event'];
127+
if (ghEventType !== 'pull_request') {
128+
const errMsg = 'GitHub event type is not of type "pull_request"';
129+
return {
130+
statusCode: 400,
131+
headers: defaultHeaders,
132+
body: errMsg,
133+
};
134+
}
135+
136+
if (!validateJsonWebhook(event, c.get<string>('githubDeletionSecret'))) {
137+
const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match";
138+
return {
139+
statusCode: 401,
140+
headers: defaultHeaders,
141+
body: errMsg,
142+
};
143+
}
144+
145+
if (!event.body) {
146+
const err = 'MarkBuildArtifactsForDeletion does not have a body in event payload';
147+
consoleLogger.error('MarkBuildArtifactsForDeletion', err);
148+
return {
149+
statusCode: 400,
150+
headers: defaultHeaders,
151+
body: err,
152+
};
153+
}
154+
155+
let payload: PullRequestEvent | undefined;
156+
try {
157+
payload = JSON.parse(event.body) as PullRequestEvent;
158+
} catch (e) {
159+
const errMsg = 'Payload is not valid JSON';
160+
return {
161+
statusCode: 400,
162+
headers: defaultHeaders,
163+
body: errMsg,
164+
};
165+
}
166+
167+
// Setting a webhook for PR events can have different actions: closed, opened, etc.
168+
// The "closed" action should occur whenever a PR is closed, regardless of merge or not.
169+
const { action } = payload;
170+
if (action !== 'closed') {
171+
const errMsg = `Unexpected GitHub action: ${action}`;
172+
return {
173+
statusCode: 400,
174+
headers: defaultHeaders,
175+
body: errMsg,
176+
};
177+
}
178+
179+
const { repository, pull_request: pullRequest } = payload;
180+
const branch = pullRequest.head.ref;
181+
182+
const client = new mongodb.MongoClient(c.get('dbUrl'));
183+
184+
try {
185+
await client.connect();
186+
const poolDb = client.db(c.get('dbName'));
187+
const repoBranchesRepository = new RepoBranchesRepository(poolDb, c, consoleLogger);
188+
const project = (await repoBranchesRepository.getProjectByRepoName(repository.name)) as string;
189+
190+
// Start marking build artifacts for deletion
191+
const snootyDb = client.db(c.get('snootyDbName'));
192+
const updatedDocsRepository = new UpdatedDocsRepository(snootyDb, c, consoleLogger);
193+
const metadataRepository = new MetadataRepository(snootyDb, c, consoleLogger);
194+
await Promise.all([
195+
updatedDocsRepository.marksAstsForDeletion(project, branch),
196+
metadataRepository.marksMetadataForDeletion(project, branch),
197+
]);
198+
} catch (e) {
199+
consoleLogger.error('MarkBuildArtifactsForDeletion', e);
200+
return {
201+
statusCode: 500,
202+
headers: defaultHeaders,
203+
body: e,
204+
};
205+
} finally {
206+
await client.close();
207+
}
208+
209+
return {
210+
statusCode: 200,
211+
headers: defaultHeaders,
212+
body: 'Build data successfully marked for deletion',
213+
};
214+
};

cdk-infra/lib/constructs/api/webhook-env-construct.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ export class WebhookEnvConstruct extends Construct {
2020
const env = getEnv();
2121

2222
const dbName = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/dbname`);
23+
const snootyDbName = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/collections/snooty`);
2324
const dbUsername = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/username`);
2425
const dbHost = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/host`);
2526
const jobCollection = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/collections/job/queue`);
@@ -35,6 +36,7 @@ export class WebhookEnvConstruct extends Construct {
3536
MONGO_ATLAS_HOST: dbHost,
3637
MONGO_ATLAS_URL: `mongodb+srv://${dbUsername}:${dbPassword}@${dbHost}/admin?retryWrites=true`,
3738
DB_NAME: dbName,
39+
SNOOTY_DB_NAME: snootyDbName,
3840
JOB_QUEUE_COL_NAME: jobCollection,
3941
NODE_CONFIG_DIR: './config',
4042
JOBS_QUEUE_URL: jobsQueue.queueUrl,

cdk-infra/lib/constructs/worker/worker-env-construct.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ export class WorkerEnvConstruct extends Construct {
2222
const ssmPrefix = getSsmPathPrefix();
2323

2424
const dbName = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/dbname`);
25+
const snootyDbName = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/collections/snooty`);
2526
const dbUsername = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/username`);
2627
const dbHost = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/host`);
2728

@@ -60,6 +61,7 @@ export class WorkerEnvConstruct extends Construct {
6061
MONGO_ATLAS_HOST: dbHost,
6162
MONGO_ATLAS_URL: `mongodb+srv://${dbUsername}:${dbPassword}@${dbHost}/admin?retryWrites=true`,
6263
DB_NAME: dbName,
64+
SNOOTY_DB_NAME: snootyDbName,
6365
JOBS_QUEUE_URL: jobsQueue.queueUrl,
6466
JOB_UPDATES_QUEUE_URL: jobUpdatesQueue.queueUrl,
6567
GITHUB_BOT_USERNAME: githubBotUsername,

cdk-infra/static/api/config/custom-environment-variables.json

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,11 @@
66
"dbUsername": "MONGO_ATLAS_USERNAME",
77
"dbPassword": "MONGO_ATLAS_PASSWORD",
88
"dbName": "DB_NAME",
9+
"snootyDbName": "SNOOTY_DB_NAME",
910
"dbUrl": "MONGO_ATLAS_URL",
1011
"fastlyDochubMap": "FASTLY_DOCHUB_MAP",
1112
"githubSecret": "GITHUB_SECRET",
13+
"githubDeletionSecret": "GITHUB_DELETION_SECRET",
1214
"slackSecret": "SLACK_SECRET",
1315
"slackAuthToken": "SLACK_TOKEN",
1416
"slackViewOpenUrl": "https://slack.com/api/views.open",

cdk-infra/static/api/config/default.json

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,11 @@
66
"dbUsername": "username",
77
"dbPassword": "password",
88
"dbName": "db_name",
9+
"snootyDbName": "snooty_db_name",
910
"dbUrl": "url",
1011
"fastlyDochubMap": "FASTLY_DOCHUB_MAP",
1112
"githubSecret": "GITHUB_SECRET",
13+
"githubDeletionSecret": "GITHUB_DELETION_SECRET",
1214
"slackSecret": "SLACK_SECRET",
1315
"slackAuthToken": "SLACK_TOKEN",
1416
"slackViewOpenUrl": "https://slack.com/api/views.open",

config/custom-environment-variables.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
"dbPassword": "MONGO_ATLAS_PASSWORD",
66
"dbHost": "MONGO_ATLAS_HOST",
77
"dbName": "DB_NAME",
8+
"snootyDbName": "SNOOTY_DB_NAME",
89
"dbUrl": "MONGO_ATLAS_URL",
910
"jobQueueCollection": "JOB_QUEUE_COL_NAME",
1011
"awsKey": "AWS_ACCESS_KEY_ID",

serverless.yml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,11 +57,13 @@ custom:
5757
dbUsername: ${ssm:/env/${self:provider.stage}/docs/worker_pool/atlas/username}
5858
dbPassword: ${ssm:/env/${self:provider.stage}/docs/worker_pool/atlas/password}
5959
dbName: ${ssm:/env/${self:provider.stage}/docs/worker_pool/atlas/dbname}
60+
snootyDbName: ${ssm:/env/${self:provider.stage}/docs/worker_pool/atlas/collections/snooty}
6061
dbhost: ${ssm:/env/${self:provider.stage}/docs/worker_pool/atlas/host}
6162
jobCollection: ${ssm:/env/${self:provider.stage}/docs/worker_pool/atlas/collections/job/queue}
6263
entitlementCollection: ${ssm:/env/${self:provider.stage}/docs/worker_pool/atlas/collections/user/entitlements}
6364
repoBranchesCollection: ${ssm:/env/${self:provider.stage}/docs/worker_pool/atlas/collections/repo}
6465
githubSecret: ${ssm:/env/${self:provider.stage}/docs/worker_pool/github/webhook/secret}
66+
githubDeletionSecret: ${ssm:/env/${self:provider.stage}/docs/worker_pool/github/webhook/deletionSecret}
6567
githubBotPW: ${ssm:/env/${self:provider.stage}/docs/worker_pool/github/bot/password}
6668
slackSecret: ${ssm:/env/${self:provider.stage}/docs/worker_pool/slack/webhook/secret}
6769
slackAuthToken: ${ssm:/env/${self:provider.stage}/docs/worker_pool/slack/auth/token}
@@ -106,9 +108,11 @@ webhook-env-core: &webhook-env-core
106108
MONGO_ATLAS_HOST: ${self:custom.dbhost}
107109
MONGO_ATLAS_URL: mongodb+srv://${self:custom.dbUsername}:${self:custom.dbPassword}@${self:custom.dbhost}/admin?retryWrites=true
108110
DB_NAME: ${self:custom.dbName}
111+
SNOOTY_DB_NAME: ${self:custom.snootyDbName}
109112
USER_ENTITLEMENT_COL_NAME: ${self:custom.entitlementCollection}
110113
JOB_QUEUE_COL_NAME: ${self:custom.jobCollection}
111114
GITHUB_SECRET: ${self:custom.githubSecret}
115+
GITHUB_DELETION_SECRET: ${self:custom.githubDeletionSecret}
112116
GITHUB_BOT_PASSWORD: ${self:custom.githubBotPW}
113117
REPO_BRANCHES_COL_NAME: ${self:custom.repoBranchesCollection}
114118
SLACK_SECRET: ${self:custom.slackSecret}
@@ -272,6 +276,16 @@ functions:
272276
cors: true
273277
environment:
274278
<<: *webhook-env-core
279+
280+
v1GithubDeleteArtifacts:
281+
handler: api/controllers/v1/github.MarkBuildArtifactsForDeletion
282+
events:
283+
- http:
284+
path: /webhook/github/trigger/delete
285+
method: POST
286+
cors: true
287+
environment:
288+
<<: *webhook-env-core
275289

276290
Outputs:
277291
JobsQueueURL:

src/repositories/baseRepository.ts

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,9 +71,13 @@ export abstract class BaseRepository {
7171
);
7272
}
7373

74-
protected async findOne(query: any, errorMsg: string): Promise<any> {
74+
protected async findOne(query: any, errorMsg: string, options: mongodb.FindOptions = {}): Promise<any> {
7575
try {
76-
return await this.promiseTimeoutS(this._config.get('MONGO_TIMEOUT_S'), this._collection.findOne(query), errorMsg);
76+
return this.promiseTimeoutS(
77+
this._config.get('MONGO_TIMEOUT_S'),
78+
this._collection.findOne(query, options),
79+
errorMsg
80+
);
7781
} catch (error) {
7882
this._logger.error(`${this._repoName}:findOne`, `Failed to find (${JSON.stringify(query)}) error: ${error}`);
7983
throw error;
@@ -128,6 +132,7 @@ export abstract class BaseRepository {
128132
}
129133
return true;
130134
}
135+
131136
protected async findOneAndUpdate(query: any, update: any, options: any, errorMsg: string): Promise<any> {
132137
try {
133138
return await this.promiseTimeoutS(

0 commit comments

Comments
 (0)