DOP-3598 (#826)

* ⚡ DOP-3598 passing test with benchmark mods

* 🔇 DOP-3598 removes console log

* ⚡ DOP-3598 properly use await to wait until each step is done

* 🎉 DOP-3598 successfully passes all test and collects timestamp

* 🎨 DOP-3598 cleans up the Dockerfile and deploy-stage-ecs

* ✅ DOP-3598 writes success test

* ✅ DOP-3598 writes success test

* 👌 DOP-3598 makes changes based on feedback

* 💡 DOP-3598 adds comment for clairty

* 👌 DOP-3598 adds feedmore from the team

* 🏷️ DOP-3598 typescript error fix

* 🐛 DOP-3598 updates broken test

* 👷 DOP-3598 puts back the docker and deploy configs to original state for cleanup

* 👌 DOP-3598 additonal improvements based on teams feedback

* ⏪ DOP-3598 revert configs for cleanup

* 👌 DOP-3598 provides better non-blocking erroring

* 🐛 DOP-3598 handles error and doesn't await for update but logs error if found

* 🔧 DOP-3598 configuration for testing changes with working job queue

* 🔧 DOP-3598 reverts config files to keep things clean

* 👌 DOP-3598 sets findOneAndUpdateExecutionTime to be a promise while not propagated the error upward

* 🔧 DOP-3598 testing against meta base branch

* 🔧 DOP-3598 reverts config to original state

---------

Co-authored-by: Caesar Bell <[email protected]>

Author: caesarbell

api/controllers/v1/github.ts

@@ -34,6 +34,14 @@ async function prepGithubPushPayload(githubEvent: any, branchRepository: BranchR
     priority: 1,
     error: {},
     result: null,
+    buildDepsExeStartTime: 0,
+    buildDepsExeEndTime: 0,
+    parseExeStartTime: 0,
+    parseExeEndTime: 0,
+    htmlExeStartTime: 0,
+    htmlExeEndTime: 0,
+    oasPageBuildExeStartTime: 0,
+    oasPageBuildExeEndTime: 0,
     payload: {
       jobType: 'githubPush',
       source: 'github',

api/controllers/v1/slack.ts

@@ -264,6 +264,14 @@ function createJob(payload: any, jobTitle: string, jobUserName: string, jobUserE
     startTime: null,
     endTime: null,
     priority: 1,
+    buildDepsExeStartTime: 0,
+    buildDepsExeEndTime: 0,
+    parseExeStartTime: 0,
+    parseExeEndTime: 0,
+    htmlExeStartTime: 0,
+    htmlExeEndTime: 0,
+    oasPageBuildExeStartTime: 0,
+    oasPageBuildExeEndTime: 0,
     error: {},
     result: null,
     payload: payload,

src/entities/job.ts

@@ -48,6 +48,7 @@ export type Payload = {
 
 export type Job = {
   _id: string;
+  useWithBenchmark?: boolean;
   payload: Payload;
   createdTime: Date;
   endTime: Date | null | undefined;

src/job/jobHandler.ts

@@ -234,21 +234,88 @@ export abstract class JobHandler {
     }
   }
 
+  private async logBuildDetails(resp: CommandExecutorResponse): Promise<void> {
+    await this._logger.save(
+      this.currJob._id,
+      `${'(BUILD)'.padEnd(15)}worker.sh run details:\n\n${resp.output}\n---\n${resp.error}`
+    );
+    if (resp.status != 'success') {
+      const error = new AutoBuilderError(resp.error, 'BuildError');
+      await this.logError(error);
+      throw error;
+    }
+  }
+
+  // call this method when we want benchmarks and uses cwd option to call command outside of a one liner.
+  private async callWithBenchmark(command: string, stage: string): Promise<CommandExecutorResponse> {
+    const start = performance.now();
+    const resp = await this._commandExecutor.execute([command], `repos/${this.currJob.payload.repoName}`);
+    await this._logger.save(
+      this.currJob._id,
+      `${'(COMMAND)'.padEnd(15)} ${command} run details in ${this.currJob.payload.repoName}`
+    );
+    const end = performance.now();
+    const update = {
+      [`${stage}StartTime`]: start,
+      [`${stage}EndTime`]: end,
+    };
+
+    this._jobRepository.findOneAndUpdateExecutionTime(this.currJob._id, update);
+    return resp;
+  }
+
+  private async exeBuildModified(): Promise<void> {
+    const stages = {
+      ['get-build-dependencies']: 'buildDepsExe',
+      ['next-gen-parse']: 'parseExe',
+      ['next-gen-html']: 'htmlExe',
+      ['oas-page-build']: 'oasPageBuildExe',
+    };
+
+    // get the prerequisite commands which should be all commands up to `rm -f makefile`
+    const endOfPrerequisiteCommands = this.currJob.buildCommands.indexOf('rm -f makefile');
+    const index = endOfPrerequisiteCommands + 1;
+    const prerequisiteCommands = this.currJob.buildCommands.slice(0, index);
+    await this._logger.save(
+      this.currJob._id,
+      `${'(PREREQUISITE COMMANDS)'.padEnd(15)} ${prerequisiteCommands.join(' && ')}`
+    );
+
+    const makeCommands = this.currJob.buildCommands.slice(index);
+    await this._logger.save(this.currJob._id, `${'(MAKE COMMANDS)'.padEnd(15)} ${makeCommands.join(' && ')}`);
+
+    // call prerequisite commands
+    await this._commandExecutor.execute(prerequisiteCommands);
+
+    for (const command of makeCommands) {
+      // works for any make command with the following signature make <make-rule>
+      const key = command.split(' ')[1].trim();
+      if (stages[key]) {
+        const makeCommandsWithBenchmarksResponse = await this.callWithBenchmark(command, stages[key]);
+        await this.logBuildDetails(makeCommandsWithBenchmarksResponse);
+      } else {
+        const makeCommandsResp = await this._commandExecutor.execute([command]);
+        await this.logBuildDetails(makeCommandsResp);
+      }
+    }
+    await this._logger.save(this.currJob._id, `${'(BUILD)'.padEnd(15)}Finished Build`);
+  }
+
+  private async exeBuild(): Promise<void> {
+    const resp = await this._commandExecutor.execute(this.currJob.buildCommands);
+    this.logBuildDetails(resp);
+    await this._logger.save(this.currJob._id, `${'(BUILD)'.padEnd(15)}Finished Build`);
+  }
+
   @throwIfJobInterupted()
   private async executeBuild(): Promise<boolean> {
     if (this.currJob.buildCommands && this.currJob.buildCommands.length > 0) {
       await this._logger.save(this.currJob._id, `${'(BUILD)'.padEnd(15)}Running Build`);
       await this._logger.save(this.currJob._id, `${'(BUILD)'.padEnd(15)}running worker.sh`);
-      const resp = await this._commandExecutor.execute(this.currJob.buildCommands);
-      await this._logger.save(this.currJob._id, `${'(BUILD)'.padEnd(15)}Finished Build`);
-      await this._logger.save(
-        this.currJob._id,
-        `${'(BUILD)'.padEnd(15)}worker.sh run details:\n\n${resp.output}\n---\n${resp.error}`
-      );
-      if (resp.status != 'success') {
-        const error = new AutoBuilderError(resp.error, 'BuildError');
-        await this.logError(error);
-        throw error;
+      if (this.currJob.useWithBenchmark) {
+        await this.exeBuildModified();
+      } else {
+        await this.exeBuild();
       }
     } else {
       const error = new AutoBuilderError('No commands to execute', 'BuildError');

src/job/jobManager.ts

@@ -116,6 +116,8 @@ export class JobManager {
     try {
       this._jobHandler = null;
       if (job?.payload) {
+        // Can easily rollback with commenting out this flag.
+        job.useWithBenchmark = true;
         await this.createHandlerAndExecute(job);
       } else {
         this._logger.info('JobManager', `No Jobs Found: ${new Date()}`);

src/job/productionJobHandler.ts

@@ -65,6 +65,7 @@ export class ProductionJobHandler extends JobHandler {
   prepStageSpecificNextGenCommands(): void {
     if (this.currJob?.buildCommands) {
       this.currJob.buildCommands[this.currJob.buildCommands.length - 1] = 'make get-build-dependencies';
+      this.currJob.buildCommands.push('make next-gen-parse');
       this.currJob.buildCommands.push('make next-gen-html');
       this.currJob.buildCommands.push(`make oas-page-build MUT_PREFIX=${this.currJob.payload.mutPrefix}`);
     }

src/job/stagingJobHandler.ts

@@ -54,7 +54,8 @@ export class StagingJobHandler extends JobHandler {
 
   prepStageSpecificNextGenCommands(): void {
     if (this.currJob.buildCommands) {
-      this.currJob.buildCommands[this.currJob.buildCommands.length - 1] = 'make next-gen-html';
+      this.currJob.buildCommands[this.currJob.buildCommands.length - 1] = 'make next-gen-parse';
+      this.currJob.buildCommands.push('make next-gen-html');
       const project = this.currJob.payload.project === 'cloud-docs' ? this.currJob.payload.project : '';
       const branchName = /^[a-zA-Z0-9_\-\./]+$/.test(this.currJob.payload.branchName)
         ? this.currJob.payload.branchName

src/repositories/jobRepository.ts

@@ -93,6 +93,19 @@ export class JobRepository extends BaseRepository {
     await this._queueConnector.sendMessage(new JobQueueMessage(jobId, status, 0, taskId), url, delay);
   }
 
+  async findOneAndUpdateExecutionTime(id: string, setValues: { [key: string]: number }): Promise<void> {
+    const query = {
+      _id: new objectId(id),
+    };
+    const update = { $set: setValues };
+    const options = { sort: { priority: -1, createdTime: 1 }, returnNewDocument: true };
+    this.findOneAndUpdate(query, update, options, `Mongo Timeout Error: Timed out while retrieving job`).catch(
+      (err) => {
+        this._logger.error('findOneAndUpdateExecutionTime', `Error at findOneAndUpdate: ${err}`);
+      }
+    );
+  }
+
   async findOneAndUpdateJob(query): Promise<Job | null> {
     const update = { $set: { startTime: new Date(), status: 'inProgress' } };
     const options = { sort: { priority: -1, createdTime: 1 }, returnNewDocument: true };

src/services/commandExecutor.ts

@@ -21,7 +21,7 @@ export class CommandExecutorResponse {
 }
 
 export interface ICommandExecutor {
-  execute(commands: Array<string>): Promise<CommandExecutorResponse>;
+  execute(commands: Array<string>, cwd?: string): Promise<CommandExecutorResponse>;
 }
 
 export interface IJobCommandExecutor extends ICommandExecutor {
@@ -43,11 +43,17 @@ export interface IGithubCommandExecutor {
 }
 
 export class ShellCommandExecutor implements ICommandExecutor {
-  async execute(commands: string[]): Promise<CommandExecutorResponse> {
+  async execute(commands: string[], cwd?: string): Promise<CommandExecutorResponse> {
     const exec = promisify(cp.exec);
     const resp = new CommandExecutorResponse();
     try {
-      const { stdout, stderr } = await exec(commands.join(' && '), { maxBuffer: c.get('MAX_STDOUT_BUFFER_SIZE') });
+      const options: { maxBuffer: number; cwd?: string } = {
+        maxBuffer: c.get('MAX_STDOUT_BUFFER_SIZE'),
+      };
+      if (cwd) {
+        options.cwd = cwd;
+      }
+      const { stdout, stderr } = await exec(commands.join(' && '), options);
 
       resp.output = stdout.trim();
       resp.error = stderr;

tests/data/data.ts

@@ -144,6 +144,7 @@ export class TestDataProvider {
     const genericCommands = TestDataProvider.getCommonBuildCommands(job);
     return Array<string>().concat(genericCommands.slice(0, genericCommands.length - 1), [
       'make get-build-dependencies',
+      'make next-gen-parse',
       'make next-gen-html',
       `make oas-page-build MUT_PREFIX=${job.payload.mutPrefix}`,
     ]);
@@ -160,6 +161,7 @@ export class TestDataProvider {
   static getExpectedStagingBuildNextGenCommands(job: Job): Array<string> {
     const genericCommands = TestDataProvider.getCommonBuildCommands(job);
     const commands = Array<string>().concat(genericCommands.slice(0, genericCommands.length - 1), [
+      'make next-gen-parse',
       'make next-gen-html',
     ]);
     const project = job.payload.project === 'cloud-docs' ? job.payload.project : '';