DOP-3951: Call Gatsby Cloud webhook after persistence module (#889)

Author: rayangler

api/handlers/jobs.ts

@@ -107,6 +107,7 @@ export async function notifyBuildSummary(jobId: string, options: BuildSummaryOpt
   }
   const repoName = fullDocument.payload.repoName;
   const username = fullDocument.user;
+
   const githubCommenter = new GithubCommenter(consoleLogger, githubToken);
   const slackConnector = new SlackConnector(consoleLogger, c);
 
@@ -239,6 +240,7 @@ export async function snootyBuildComplete(event: APIGatewayEvent): Promise<APIGa
     await client.connect();
     const db = client.db(c.get<string>('dbName'));
     const jobRepository = new JobRepository(db, c, consoleLogger);
+    await jobRepository.updateExecutionTime(jobId, { gatsbyCloudEndTime: new Date() });
     const updateResponse = await jobRepository.updateWithStatus(jobId, null, payload.status || JobStatus.failed, false);
     const previewUrl = getPreviewUrl(updateResponse.payload, c.get<string>('env'));
     await notifyBuildSummary(jobId, { mongoClient: client, previewUrl });
@@ -270,5 +272,5 @@ function getPreviewUrl(payload: Payload | undefined, env: string): string | unde
   const { repoOwner, branchName, project } = payload;
   const githubUsernameNoHyphens = repoOwner.split('-').join('').toLowerCase();
   const possibleStagingSuffix = env === 'stg' ? 'stg' : '';
-  return `https://preview-mongodb${githubUsernameNoHyphens}${possibleStagingSuffix}.gatsbyjs.io/${project}/${branchName}/index`;
+  return `https://preview-mongodb${githubUsernameNoHyphens}${possibleStagingSuffix}.gatsbyjs.io/${project}/${branchName}/`;
 }

src/job/jobHandler.ts

@@ -99,7 +99,20 @@ export abstract class JobHandler {
           null,
           null
         );
-        await this.jobRepository.updateWithStatus(this.currJob._id, files, JobStatus.completed);
+
+        // Prevent the Autobuilder's usual build completion from being sent after content staging job
+        // if the user already has a Gatsby Cloud site. Job should be marked as
+        // completed after the Gatsby Cloud build via the SnootyBuildComplete lambda.
+        const { _id: jobId, user } = this.currJob;
+        const gatsbyCloudSiteId = await this._repoEntitlementsRepo.getGatsbySiteIdByGithubUsername(user);
+        if (gatsbyCloudSiteId && this.currJob.payload.jobType === 'githubPush') {
+          this.logger.info(
+            jobId,
+            `User ${user} has a Gatsby Cloud site. The Autobuilder will not mark the build as completed right now.`
+          );
+        } else {
+          await this.jobRepository.updateWithStatus(jobId, files, JobStatus.completed);
+        }
       } else {
         if (publishResult.error) {
           await this.jobRepository.updateWithErrorStatus(this.currJob._id, publishResult.error);
@@ -266,14 +279,15 @@ export abstract class JobHandler {
       [`${stage}EndTime`]: end,
     };
 
-    this._jobRepository.findOneAndUpdateExecutionTime(this.currJob._id, update);
+    this._jobRepository.updateExecutionTime(this.currJob._id, update);
     return resp;
   }
 
   private async exeBuildModified(): Promise<void> {
     const stages = {
       ['get-build-dependencies']: 'buildDepsExe',
       ['next-gen-parse']: 'parseExe',
+      ['persistence-module']: 'persistenceExe',
       ['next-gen-html']: 'htmlExe',
       ['oas-page-build']: 'oasPageBuildExe',
     };
@@ -303,6 +317,12 @@ export abstract class JobHandler {
         const makeCommandsResp = await this._commandExecutor.execute([command]);
         await this.logBuildDetails(makeCommandsResp);
       }
+
+      // Call Gatsby Cloud preview webhook after persistence module finishes for staging builds
+      const isFeaturePreviewWebhookEnabled = process.env.GATSBY_CLOUD_PREVIEW_WEBHOOK_ENABLED?.toLowerCase() === 'true';
+      if (key === 'persistence-module' && this.name === 'Staging' && isFeaturePreviewWebhookEnabled) {
+        await this.callGatsbyCloudWebhook();
+      }
     }
     await this._logger.save(this.currJob._id, `${'(BUILD)'.padEnd(15)}Finished Build`);
   }
@@ -635,27 +655,44 @@ export abstract class JobHandler {
     return this._logger;
   }
 
-  protected async previewWebhook(): Promise<AxiosResponse<string>> {
-    const previewWebhookURL = 'https://webhook.gatsbyjs.com/hooks/data_source';
-    const githubUsername = this.currJob.user;
-    const gatsbySiteId = await this._repoEntitlementsRepo.getGatsbySiteIdByGithubUsername(githubUsername);
-    if (!gatsbySiteId) {
-      const message = `User ${githubUsername} does not have a Gatsby Cloud Site ID.`;
-      this._logger.warn('Gatsby Cloud Preview Webhook', message);
-      throw new Error(message);
-    }
+  // Invokes Gatsby Cloud Preview Webhook
+  protected async callGatsbyCloudWebhook(): Promise<void> {
+    const featurePreviewWebhookEnabled = process.env.GATSBY_CLOUD_PREVIEW_WEBHOOK_ENABLED;
+    // Logging for Debugging purposes only will remove once we see the build working in Gatsby.
+    await this.logger.save(
+      this.currJob._id,
+      `${'(GATSBY_CLOUD_PREVIEW_WEBHOOK_ENABLED)'.padEnd(15)}${featurePreviewWebhookEnabled}`
+    );
 
-    const url = `${previewWebhookURL}/${gatsbySiteId}`;
-    return await axios.post(
-      url,
-      {
-        jobId: this.currJob._id,
-        status: this.currJob.status,
-      },
-      {
-        headers: { 'x-gatsby-cloud-data-source': 'gatsby-source-snooty-preview' },
+    try {
+      const previewWebhookURL = 'https://webhook.gatsbyjs.com/hooks/data_source';
+      const githubUsername = this.currJob.user;
+      const gatsbySiteId = await this._repoEntitlementsRepo.getGatsbySiteIdByGithubUsername(githubUsername);
+      if (!gatsbySiteId) {
+        const message = `User ${githubUsername} does not have a Gatsby Cloud Site ID.`;
+        this._logger.warn('Gatsby Cloud Preview Webhook', message);
+        throw new Error(message);
       }
-    );
+
+      const url = `${previewWebhookURL}/${gatsbySiteId}`;
+      const response = await axios.post(
+        url,
+        {
+          jobId: this.currJob._id,
+        },
+        {
+          headers: { 'x-gatsby-cloud-data-source': 'gatsby-source-snooty-preview' },
+        }
+      );
+      await this._jobRepository.updateExecutionTime(this.currJob._id, { gatsbyCloudStartTime: new Date() });
+      await this.logger.save(this.currJob._id, `${'(POST Webhook Status)'.padEnd(15)}${response.status}`);
+    } catch (err) {
+      await this.logger.save(
+        this.currJob._id,
+        `${'(POST Webhook)'.padEnd(15)}Failed to POST to Gatsby Cloud webhook: ${err}`
+      );
+      throw err;
+    }
   }
 }
 

src/job/productionJobHandler.ts

@@ -134,6 +134,7 @@ export class ProductionJobHandler extends JobHandler {
     if (this.currJob?.buildCommands) {
       this.currJob.buildCommands[this.currJob.buildCommands.length - 1] = 'make get-build-dependencies';
       this.currJob.buildCommands.push('make next-gen-parse');
+      this.currJob.buildCommands.push('make persistence-module');
       this.currJob.buildCommands.push('make next-gen-html');
       this.currJob.buildCommands.push(`make oas-page-build MUT_PREFIX=${this.currJob.payload.mutPrefix}`);
     }

src/job/stagingJobHandler.ts

@@ -58,7 +58,8 @@ export class StagingJobHandler extends JobHandler {
   prepStageSpecificNextGenCommands(): void {
     if (this.currJob.buildCommands) {
       this.currJob.buildCommands[this.currJob.buildCommands.length - 1] = 'make next-gen-parse';
-      this.currJob.buildCommands.push(`make next-gen-html GH_USER=${this.currJob.payload.repoOwner}`);
+      this.currJob.buildCommands.push(`make persistence-module GH_USER=${this.currJob.payload.repoOwner}`);
+      this.currJob.buildCommands.push('make next-gen-html');
       const project = this.currJob.payload.project === 'cloud-docs' ? this.currJob.payload.project : '';
       const branchName = /^[a-zA-Z0-9_\-\./]+$/.test(this.currJob.payload.branchName)
         ? this.currJob.payload.branchName
@@ -75,25 +76,6 @@ export class StagingJobHandler extends JobHandler {
       if (resp?.output?.includes('Summary')) {
         resp.output = resp.output.slice(resp.output.indexOf('Summary'));
       }
-      // Invoke Gatsby Preview Webhook
-      const featurePreviewWebhookEnabled = process.env.GATSBY_CLOUD_PREVIEW_WEBHOOK_ENABLED;
-      // Logging for Debugging purposes only will remove once we see the build working in Gatsby.
-      await this.logger.save(
-        this.currJob._id,
-        `${'(GATSBY_CLOUD_PREVIEW_WEBHOOK_ENABLED)'.padEnd(15)}${featurePreviewWebhookEnabled}`
-      );
-      if (featurePreviewWebhookEnabled?.toLowerCase() === 'true') {
-        try {
-          const response = await this.previewWebhook();
-          await this.logger.save(this.currJob._id, `${'(POST Webhook Status)'.padEnd(15)}${response.status}`);
-        } catch (err) {
-          await this.logger.save(
-            this.currJob._id,
-            `${'(POST Webhook)'.padEnd(15)}Failed to POST to Gatsby Cloud webhook: ${err}`
-          );
-        }
-      }
-
       await this.logger.save(this.currJob._id, `${'(stage)'.padEnd(15)}Finished pushing to staging`);
       await this.logger.save(this.currJob._id, `${'(stage)'.padEnd(15)}Staging push details:\n\n${summary}`);
       return resp;

src/repositories/jobRepository.ts

@@ -101,17 +101,14 @@ export class JobRepository extends BaseRepository {
     await this._queueConnector.sendMessage(new JobQueueMessage(jobId, status, 0, taskId), url, delay);
   }
 
-  async findOneAndUpdateExecutionTime(id: string, setValues: { [key: string]: number }): Promise<void> {
+  async updateExecutionTime(id: string, setValues: { [key: string]: number | Date }): Promise<void> {
     const query = {
       _id: new objectId(id),
     };
     const update = { $set: setValues };
-    const options = { sort: { priority: -1, createdTime: 1 }, returnNewDocument: true };
-    this.findOneAndUpdate(query, update, options, `Mongo Timeout Error: Timed out while retrieving job`).catch(
-      (err) => {
-        this._logger.error('findOneAndUpdateExecutionTime', `Error at findOneAndUpdate: ${err}`);
-      }
-    );
+    this.updateOne(query, update, `Mongo Timeout Error: Timed out while retrieving job`).catch((err) => {
+      this._logger.error('updateExecutionTime', `Error: ${err}`);
+    });
   }
 
   async findOneAndUpdateJob(query): Promise<Job | null> {

src/repositories/repoEntitlementsRepository.ts

@@ -47,7 +47,7 @@ export class RepoEntitlementsRepository extends BaseRepository {
     }
   }
 
-  async getGatsbySiteIdByGithubUsername(githubUsername: string) {
+  async getGatsbySiteIdByGithubUsername(githubUsername: string): Promise<string | undefined> {
     const query = { github_username: githubUsername };
     const projection = { _id: 0, gatsby_site_id: 1 };
     const res = await this.findOne(
@@ -57,7 +57,7 @@ export class RepoEntitlementsRepository extends BaseRepository {
     );
     if (!res) {
       this._logger.error('Fetching Gatsby Cloud Site ID', `Could not find user: ${githubUsername}`);
-      return null;
+      return undefined;
     }
     return res.gatsby_site_id;
   }

tests/data/data.ts

@@ -145,6 +145,7 @@ export class TestDataProvider {
     return Array<string>().concat(genericCommands.slice(0, genericCommands.length - 1), [
       'make get-build-dependencies',
       'make next-gen-parse',
+      'make persistence-module',
       'make next-gen-html',
       `make oas-page-build MUT_PREFIX=${job.payload.mutPrefix}`,
     ]);
@@ -162,7 +163,8 @@ export class TestDataProvider {
     const genericCommands = TestDataProvider.getCommonBuildCommands(job);
     const commands = Array<string>().concat(genericCommands.slice(0, genericCommands.length - 1), [
       'make next-gen-parse',
-      `make next-gen-html GH_USER=${job.payload.repoOwner}`,
+      `make persistence-module GH_USER=${job.payload.repoOwner}`,
+      `make next-gen-html`,
     ]);
     const project = job.payload.project === 'cloud-docs' ? job.payload.project : '';
     const branchName = /^[a-zA-Z0-9_\-\./]+$/.test(job.payload.branchName) ? job.payload.branchName : '';

tests/data/jobDef.ts

@@ -51,6 +51,47 @@ export const getBuildJobPlain = (): Job => ({
   } as Payload,
 });
 
+export const getStagingJobDef = (): Job => ({
+  _id: '5c5e0817ce099eaf874a9801',
+  title: 'GitHub Push: skerschb',
+  buildCommands: [],
+  comMessage: null,
+  createdTime: new Date('2022-04-20T17:10:09.432Z'),
+  deployCommands: [],
+  email: '[email protected]',
+  endTime: new Date('2022-04-20T17:10:09.432Z'),
+  error: {},
+  invalidationStatusURL: 'test-status-url',
+  logs: [],
+  manifestPrefix: '',
+  mutPrefix: '',
+  pathPrefix: '',
+  priority: 1,
+  purgedUrls: null,
+  result: [],
+  shouldGenerateSearchManifest: true,
+  startTime: new Date('2022-04-20T17:10:09.432Z'),
+  status: JobStatus.completed,
+  user: 'skerschb',
+  payload: {
+    jobType: 'githubPush',
+    action: 'push',
+    branchName: 'DOCSP-666',
+    isFork: false,
+    manifestPrefix: undefined,
+    mutPrefix: undefined,
+    newHead: '38b805e9c7c4f1c364476682e93f9d24a87f470a',
+    pathPrefix: undefined,
+    prefix: 'compass',
+    project: 'testauth',
+    repoName: 'testauth',
+    repoOwner: 'skerschb',
+    source: 'github',
+    url: 'https://github.com/skerschb/testauth.git',
+    urlSlug: 'UsingAlias',
+  } as Payload,
+});
+
 // Represents a manifestJob at time of insert to queue
 export const getManifestJobDef = (): Omit<Job, '_id'> => ({
   buildCommands: [],

tests/unit/job/api/jobs.test.ts

@@ -15,6 +15,7 @@ jest.mock('config', () => ({
 
 jest.mock('../../../../src/repositories/jobRepository', () => ({
   JobRepository: jest.fn().mockImplementation(() => ({
+    updateExecutionTime: jest.fn(),
     updateWithStatus: jest.fn(() => mockJob),
     getJobById: jest.fn(),
   })),

tests/unit/job/stagingJobHandler.test.ts

@@ -1,6 +1,6 @@
 import { TestDataProvider } from '../../data/data';
 import { JobHandlerTestHelper } from '../../utils/jobHandlerTestHelper';
-import { getBuildJobDef } from '../../data/jobDef';
+import { getStagingJobDef } from '../../data/jobDef';
 
 describe('StagingJobHandler Tests', () => {
   let jobHandlerTestHelper: JobHandlerTestHelper;
@@ -71,6 +71,7 @@ describe('StagingJobHandler Tests', () => {
     expect(jobHandlerTestHelper.job.buildCommands).toEqual(
       TestDataProvider.getExpectedStagingBuildNextGenCommands(jobHandlerTestHelper.job)
     );
+    expect(jobHandlerTestHelper.jobRepo.updateWithStatus).toBeCalledTimes(1);
     expect(jobHandlerTestHelper.jobRepo.insertNotificationMessages).toBeCalledWith(
       jobHandlerTestHelper.job._id,
       'Summary: All good'
@@ -90,12 +91,18 @@ describe('StagingJobHandler Tests', () => {
     jobHandlerTestHelper.jobRepo.insertJob = jest.fn();
     const queueManifestJobSpy = jest.spyOn(jobHandlerTestHelper.jobHandler, 'queueManifestJob');
 
-    expect(jobHandlerTestHelper.job).toEqual(getBuildJobDef());
+    expect(jobHandlerTestHelper.job).toEqual(getStagingJobDef());
 
     jobHandlerTestHelper.setupForSuccess();
     await jobHandlerTestHelper.jobHandler.execute();
 
     expect(queueManifestJobSpy).toBeCalledTimes(0);
     expect(jobHandlerTestHelper.jobRepo.insertJob).toBeCalledTimes(0);
   });
+
+  test('Staging deploy with Gatsby Cloud site does not result in job completion', async () => {
+    jobHandlerTestHelper.setStageForDeploySuccess(true, false, undefined, { hasGatsbySiteId: true });
+    await jobHandlerTestHelper.jobHandler.execute();
+    expect(jobHandlerTestHelper.jobRepo.updateWithStatus).toBeCalledTimes(0);
+  });
 });