DOP-3341: Integrate OAS Page Builder module as part of build process (#723)

Author: rayangler

Dockerfile

@@ -8,10 +8,19 @@ RUN npm install
 COPY . ./
 RUN npm run build
 
+# Build modules
+# OAS Page Builder
+RUN cd ./modules/oas-page-builder \
+    && npm install \
+    && npm run build
+
 # where repo work will happen
 FROM ubuntu:20.04
 ARG SNOOTY_PARSER_VERSION=0.13.15
 ARG SNOOTY_FRONTEND_VERSION=0.13.35
+# The Redoc CLI branch will most likely stay static. Updates to the branch should 
+# be limited to CLI bug fixes and Redoc dependency version bumps
+ARG REDOC_CLI_VERSION=0.1.0
 ARG FLIT_VERSION=3.0.0
 ARG NPM_BASE_64_AUTH
 ARG NPM_EMAIL
@@ -54,7 +63,8 @@ RUN useradd -ms /bin/bash docsworker-xlarge
 RUN npm -g config set user root
 USER docsworker-xlarge
 
-WORKDIR /home/docsworker-xlarge
+ARG WORK_DIRECTORY=/home/docsworker-xlarge
+WORKDIR ${WORK_DIRECTORY}
 
 # get shared.mk
 RUN curl https://raw.githubusercontent.com/mongodb/docs-worker-pool/meta/makefiles/shared.mk -o shared.mk
@@ -79,10 +89,32 @@ RUN git clone --depth 1 https://github.com/mongodb/devhub.git snooty-devhub
     && cd snooty-devhub                                                                            \
     && npm install --production
 
+# install redoc fork
+RUN git clone -b redoc-cli@${REDOC_CLI_VERSION} --depth 1 https://github.com/mongodb-forks/redoc.git redoc \
+    # Install dependencies for Redoc CLI
+    && cd redoc/cli \
+    && npm ci --omit=dev \
+    # Install dependencies for Redoc component and building/compiling
+    && cd ../ \
+    && npm ci --omit=dev --ignore-scripts \
+    && npm run compile:cli
+
 COPY --from=ts-compiler /home/docsworker-xlarge/package*.json ./
 COPY --from=ts-compiler /home/docsworker-xlarge/config config/
 COPY --from=ts-compiler /home/docsworker-xlarge/build ./
 RUN npm install
+
+# OAS Page Builder module copy
+# Create directory and add permissions to allow node module installation
+RUN mkdir -p modules/oas-page-builder && chmod 755 modules/oas-page-builder
+COPY --from=ts-compiler /home/docsworker-xlarge/modules/oas-page-builder/package*.json ./modules/oas-page-builder/
+COPY --from=ts-compiler /home/docsworker-xlarge/modules/oas-page-builder/dist ./modules/oas-page-builder/
+RUN cd ./modules/oas-page-builder/ && npm install
+
+# Needed for OAS Page Builder module in shared.mk
+ENV REDOC_PATH=${WORK_DIRECTORY}/redoc/cli/index.js
+ENV OAS_MODULE_PATH=${WORK_DIRECTORY}/modules/oas-page-builder/index.js
+
 RUN mkdir repos && chmod 755 repos
 EXPOSE 3000
 CMD ["node", "app.js"]

modules/oas-page-builder/README.md

@@ -60,3 +60,7 @@ npm run start -- --bundle /home/docsworker-xlarge/repos/cloud-docs/bundle.zip --
 | output | Path to the directory that the OpenAPI content pages should be built to. Typically, this would be the same output `public/` directory of a Snooty frontend build.                      |
 | redoc  | Path to the local installation of Redoc CLI to use. This should point to the team's [fork of Redoc](https://github.com/mongodb-forks/redoc), with the target being a compiled JS file. |
 | repo   | Path to the parsed docs repo's directory. This is to ensure that OpenAPI content pages using local OAS files can be properly sourced and passed down as an argument to Redoc CLI.      |
+
+## Usage in Production
+
+In production, this module is installed as part of the [Docker](https://github.com/mongodb/docs-worker-pool/blob/master/Dockerfile) image build and is called as part of a target in [shared.mk](https://github.com/mongodb/docs-worker-pool/blob/meta/makefiles/shared.mk) by the autobuilder. The module is intended to be used after the frontend has completed building pages, but before `mut` uploads them.

modules/oas-page-builder/index.ts

@@ -5,6 +5,7 @@ import { Command } from 'commander';
 import { getOASMetadata } from './src/services/buildMetadata';
 import { buildOpenAPIPages } from './src/services/pageBuilder';
 import { ModuleOptions } from './src/types';
+import { normalizeUrl } from './src/utils/normalizeUrl';
 
 const program = new Command();
 program
@@ -31,6 +32,9 @@ const app = async (options: ModuleOptions) => {
   const numOASPages = oasMetadataEntries.length;
   console.log(`OpenAPI content pages found: ${numOASPages}.`);
 
+  // Normalize url since Autobuilder's MUT_PREFIX could be malformed
+  options.siteUrl = normalizeUrl(options.siteUrl);
+
   await buildOpenAPIPages(oasMetadataEntries, { ...options, siteTitle });
 };
 

modules/oas-page-builder/sample.env

@@ -1,6 +1,11 @@
+# The following env variables are typically expected to be passed down from the Autobuilder.
+# For local development, populate them as needed in a separate .env file.
+
 # Mongo instance username
-DB_USER=user
+MONGO_ATLAS_USERNAME=user
 # Mongo instance password, corresponding to user
-DB_PASSWORD=password
+MONGO_ATLAS_PASSWORD=password
 # Mongo instance host url
-DB_HOST=localhost:27017
+MONGO_ATLAS_HOST=localhost:27017
+# Snooty environment to select appropriate snooty_* database
+SNOOTY_ENV=dotcomprd

modules/oas-page-builder/src/services/database.ts

@@ -1,28 +1,46 @@
 import { Db, MongoClient } from 'mongodb';
 import { OASFile, OASFileGitHash } from './models/OASFile';
 
-const DB_NAME = process.env.DB_NAME || 'snooty_dotcomprd';
 const COLLECTION_NAME = 'oas_files';
 
 const getAtlasURL = () => {
   const isHostLocal = process.env.DB_HOST?.includes('localhost');
   if (isHostLocal) {
-    return `mongodb://${process.env.DB_HOST}/?retryWrites=true&w=majority`;
+    return `mongodb://${process.env.MONGO_ATLAS_HOST}/?retryWrites=true&w=majority`;
   }
-  return `mongodb+srv://${process.env.DB_USER}:${process.env.DB_PASSWORD}@${process.env.DB_HOST}/?retryWrites=true&w=majority`;
+  return `mongodb+srv://${process.env.MONGO_ATLAS_USERNAME}:${process.env.MONGO_ATLAS_PASSWORD}@${process.env.MONGO_ATLAS_HOST}/?retryWrites=true&w=majority`;
 };
 
 const atlasURL = getAtlasURL();
 const client = new MongoClient(atlasURL);
 // cached db object, so we can handle initial connection process once if unitialized
 let dbInstance: Db;
 
+const getDbName = () => {
+  const env = process.env.SNOOTY_ENV ?? '';
+
+  switch (env) {
+    // Autobuilder's prd env
+    case 'production':
+    case 'dotcomprd':
+      return 'snooty_dotcomprd';
+    // Autobuilder's pre-prd env
+    case 'staging':
+    case 'dotcomstg':
+      return 'snooty_dotcomstg';
+    default:
+      // snooty_dotcomprd.oas_files should be guaranteed to have the latest data
+      return 'snooty_dotcomprd';
+  }
+};
+
 // Handles memoization of db object, and initial connection logic if needs to be initialized
 const db = async () => {
   if (!dbInstance) {
     try {
       await client.connect();
-      dbInstance = client.db(DB_NAME);
+      const dbName = getDbName();
+      dbInstance = client.db(dbName);
     } catch (error) {
       console.error(`Error at db client connection: ${error}`);
       throw error;

modules/oas-page-builder/src/utils/normalizeUrl.ts

@@ -0,0 +1,8 @@
+import { normalizePath } from './normalizePath';
+
+// Normalizes the pathname of a URL by removing repeated forward slashes
+export const normalizeUrl = (url: string) => {
+  const urlObject = new URL(url);
+  urlObject.pathname = normalizePath(urlObject.pathname);
+  return urlObject.href;
+};

modules/oas-page-builder/tests/unit/utils/normalizeUrl.test.ts

@@ -0,0 +1,14 @@
+import { normalizeUrl } from '../../../src/utils/normalizeUrl';
+
+describe('normalizeUrl', () => {
+  it('removes extra consecutive forward slashes', () => {
+    const httpsUrl = 'https://mongodbcom-cdn.website.staging.corp.mongodb.com//docs-qa/atlas/app-services';
+    let result = normalizeUrl(httpsUrl);
+    expect(result).toBe('https://mongodbcom-cdn.website.staging.corp.mongodb.com/docs-qa/atlas/app-services');
+
+    // Should work regardless of protocol
+    const httpUrl = 'http://mongodbcom-cdn.website.staging.corp.mongodb.com//docs-qa/atlas/app-services';
+    result = normalizeUrl(httpUrl);
+    expect(result).toBe('http://mongodbcom-cdn.website.staging.corp.mongodb.com/docs-qa/atlas/app-services');
+  });
+});

src/job/productionJobHandler.ts

@@ -66,6 +66,7 @@ export class ProductionJobHandler extends JobHandler {
     if (this.currJob?.buildCommands) {
       this.currJob.buildCommands[this.currJob.buildCommands.length - 1] = 'make get-build-dependencies';
       this.currJob.buildCommands.push('make next-gen-html');
+      this.currJob.buildCommands.push(`make oas-page-build MUT_PREFIX=${this.currJob.payload.mutPrefix}`);
     }
   }
 

src/job/stagingJobHandler.ts

@@ -57,6 +57,8 @@ export class StagingJobHandler extends JobHandler {
       this.currJob.buildCommands[this.currJob.buildCommands.length - 1] = 'make next-gen-html';
       if (this.currJob.payload.repoName === 'devhub-content-integration') {
         this.currJob.buildCommands[this.currJob.buildCommands.length - 1] += ` STRAPI_PUBLICATION_STATE=preview`;
+      } else {
+        this.currJob.buildCommands.push(`make oas-page-build MUT_PREFIX=${this.currJob.payload.mutPrefix}`);
       }
     }
   }

tests/data/data.ts

@@ -145,6 +145,7 @@ export class TestDataProvider {
     return Array<string>().concat(genericCommands.slice(0, genericCommands.length - 1), [
       'make get-build-dependencies',
       'make next-gen-html',
+      `make oas-page-build MUT_PREFIX=${job.payload.mutPrefix}`,
     ]);
   }
 
@@ -163,6 +164,8 @@ export class TestDataProvider {
     ]);
     if (job.payload.repoName == 'devhub-content-integration') {
       commands[commands.length - 1] += ` STRAPI_PUBLICATION_STATE=preview`;
+    } else {
+      commands.push(`make oas-page-build MUT_PREFIX=${job.payload.mutPrefix}`);
     }
     return commands;
   }