DOP-3304, DOP-3335, DOP-3359: update manifest job handler to use snooty manifest (#705)

* DOP-3304: update manifest job handler to use snooty manifest

* call parse only during manifest job

* exclude the 404 and meta sites from search

* DOP-3359: fix global search flags?

* fix test

* update mut to v0.10.2 so slugs are correct

* fix path

* idiomaticness ++

Author: schmalliso

Dockerfile

@@ -34,7 +34,9 @@ RUN /venv/bin/pip install -r https://raw.githubusercontent.com/mongodb/docs-tool
 RUN apt-get update && apt-get install -y python3 python3-dev python3-pip
 RUN apt-get -y install vim
 RUN apt-get -y install git pkg-config libxml2-dev
-RUN python3 -m pip install -r https://raw.githubusercontent.com/mongodb/mut/master/requirements.txt
+RUN python3 -m pip install https://github.com/mongodb/mut/releases/download/v0.10.2/mut-0.10.2-py3-none-any.whl
+
+
 ENV PATH="${PATH}:/home/docsworker-xlarge/.local/bin:/usr/local/lib/python2.7/dist-packages/virtualenv/bin"
 
 # get node 14
@@ -65,7 +67,7 @@ RUN git clone -b v${SNOOTY_PARSER_VERSION} --depth 1 https://github.com/mongodb/
 
 # install snooty frontend and docs-tools
 RUN git clone -b v${SNOOTY_FRONTEND_VERSION} --depth 1 https://github.com/mongodb/snooty.git       \
-    && cd snooty                                                                                   \                                                                        \
+    && cd snooty                                                                                   \
     && npm install                                                                                 \
     && git clone --depth 1 https://github.com/mongodb/docs-tools.git                               \
     && mkdir -p ./static/images                                                                    \

api/controllers/v1/slack.ts

@@ -140,9 +140,11 @@ export const getDeployableJobs = async (values, entitlement, branchRepository: B
       urlSlug,
       false,
       false,
-      '-g'
+      false
     );
 
+    newPayload.stable = !!isStableBranch;
+
     if (!aliases || aliases.length === 0) {
       if (non_versioned) {
         newPayload.urlSlug = '';
@@ -151,8 +153,6 @@ export const getDeployableJobs = async (values, entitlement, branchRepository: B
       continue;
     }
 
-    // if this is stable branch, we want autobuilder to know this is unaliased branch and therefore can reindex for search
-    newPayload.stable = isStableBranch ? '-g' : '';
     newPayload.aliased = true;
 
     // we use the primary alias for indexing search, not the original branch name (ie 'master'), for aliased repos
@@ -174,7 +174,7 @@ export const getDeployableJobs = async (values, entitlement, branchRepository: B
 
     aliases.forEach(async (alias: string) => {
       if (alias !== urlSlug) {
-        newPayload.stable = '';
+        newPayload.stable = false;
         newPayload.urlSlug = alias;
         newPayload.primaryAlias = false;
         deployHelper(deployable, newPayload, jobTitle, jobUserName, jobUserEmail);
@@ -231,7 +231,7 @@ function createPayload(
   urlSlug,
   aliased = false,
   primaryAlias = false,
-  stable = ''
+  stable = false
 ) {
   return {
     jobType,

src/entities/job.ts

@@ -36,7 +36,7 @@ export type Payload = {
   aliased: boolean | null | undefined;
   primaryAlias: string | null | undefined;
   repoBranches: any;
-  stable: string | null | undefined;
+  stable: boolean | null | undefined;
   isNextGen: boolean | null | undefined;
   regression: boolean | null | undefined;
   urlSlug: string | null | undefined;

src/job/jobHandler.ts

@@ -442,7 +442,7 @@ export abstract class JobHandler {
   // based on information about this build&deploy job
   // TODO: Give 'shouldGenerateSearchManifest' boolean to users' control
   shouldGenerateSearchManifest(): boolean {
-    const doNotSearchProperties = ['docs-landing'];
+    const doNotSearchProperties = ['docs-landing', 'docs-404', 'docs-meta'];
     if (doNotSearchProperties.includes(this.currJob.payload.repoName)) {
       return false;
     }

src/job/manifestJobHandler.ts

@@ -94,15 +94,15 @@ export class ManifestJobHandler extends JobHandler {
       `cd repos/${this.currJob.payload.repoName}`,
       'echo IGNORE: testing manifest generation deploy commands',
       'ls -al',
-      `mut-index upload public -b ${b} -o ${f}/${maP}.json -u ${jUaP(url, muP)} ${globalSearch}`,
+      `mut-index upload bundle.zip -b ${b} -o ${f}/${maP}.json -u ${jUaP(url, muP)} ${globalSearch}`,
     ];
   }
 
   // TODO: Can this function be merged with prepBuildCommands?
   prepStageSpecificNextGenCommands(): void {
     if (this.currJob?.buildCommands) {
       this.currJob.buildCommands[this.currJob.buildCommands.length - 1] = 'make get-build-dependencies';
-      this.currJob.buildCommands.push('make next-gen-html');
+      this.currJob.buildCommands.push('make next-gen-parse');
     }
   }
 

src/job/productionJobHandler.ts

@@ -54,7 +54,7 @@ export class ProductionJobHandler extends JobHandler {
       ] = `make next-gen-deploy MUT_PREFIX=${this.currJob.payload.mutPrefix}`;
       // TODO: Remove when satisfied with new manifestJobHandler infrastructure
       if (manifestPrefix) {
-        const searchFlag = this.currJob.payload.stable;
+        const searchFlag = this.currJob.payload.stable ? '-g' : '';
         this.currJob.deployCommands[
           this.currJob.deployCommands.length - 1
         ] += ` MANIFEST_PREFIX=${manifestPrefix} GLOBAL_SEARCH_FLAG=${searchFlag}`;

tests/data/data.ts

@@ -148,6 +148,14 @@ export class TestDataProvider {
     ]);
   }
 
+  static getExpectedManifestBuildNextGenCommands(job: Job): Array<string> {
+    const genericCommands = TestDataProvider.getCommonBuildCommands(job);
+    return Array<string>().concat(genericCommands.slice(0, genericCommands.length - 1), [
+      'make get-build-dependencies',
+      'make next-gen-parse',
+    ]);
+  }
+
   static getExpectedStagingBuildNextGenCommands(job: Job): Array<string> {
     const genericCommands = TestDataProvider.getCommonBuildCommands(job);
     const commands = Array<string>().concat(genericCommands.slice(0, genericCommands.length - 1), [
@@ -283,9 +291,8 @@ export class TestDataProvider {
     ]);
     if (job.payload.manifestPrefix) {
       // TODO: is job.payload.stableBranch supposed to be includeInGlobalSearch?
-      ret[
-        ret.length - 1
-      ] += ` MANIFEST_PREFIX=${job.payload.manifestPrefix} GLOBAL_SEARCH_FLAG=${job.payload.stableBranch}`;
+      const stable = job.payload.stable ? '-g' : '';
+      ret[ret.length - 1] += ` MANIFEST_PREFIX=${job.payload.manifestPrefix} GLOBAL_SEARCH_FLAG=${stable}`;
     }
     return ret;
   }

tests/data/fullDoc.ts

@@ -41,7 +41,7 @@ export default {
     url: 'https://github.com/mongodb/docs-java',
     newHead: null,
     primaryAlias: true,
-    stable: '',
+    stable: false,
   },
   logs: [
     '* Starting Job with ID: 6228fbb8f8498d4be9c237f9 and type: productionDeploy',

tests/data/fullDocWrongUrl.ts

@@ -41,7 +41,7 @@ export default {
     url: 'https://github.com/mongodb/docs-java',
     newHead: null,
     primaryAlias: true,
-    stable: '',
+    stable: false,
   },
   logs: [
     '* Starting Job with ID: 6228fbb8f8498d4be9c237f9 and type: productionDeploy',