[DOP-4438]: Add component filtering for production builds in the Autobuilder (#1025)

* [DOP-4438]: Add rust installation

* [DOP-4438]: Remove cargo install

* [DOP-4438]: revert

* [DOP-4438]: add npm ci

* [DOP-4438]: Remove unneeded test

* [DOP-4438]: Remove unneeded test

* [DOP-4438]: Clean up dockerfile

* [DOP-4438]: Remove omit dev

* [DOP-4438]: Update comments

* [DOP-4438]: Remove echo

Author: branberry

Dockerfile

@@ -22,8 +22,8 @@ RUN cd ./modules/oas-page-builder \
 # where repo work will happen
 FROM ubuntu:20.04
 ARG WORK_DIRECTORY=/home/docsworker-xlarge
-ARG SNOOTY_PARSER_VERSION=0.16.3
-ARG SNOOTY_FRONTEND_VERSION=0.16.6
+ARG SNOOTY_PARSER_VERSION=0.16.4
+ARG SNOOTY_FRONTEND_VERSION=0.16.8
 ARG MUT_VERSION=0.11.1
 ARG REDOC_CLI_VERSION=1.2.3
 ARG NPM_BASE_64_AUTH
@@ -63,17 +63,31 @@ USER docsworker-xlarge
 
 WORKDIR ${WORK_DIRECTORY}
 
+# Get Rust
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+
+RUN chmod -R 777 ${WORK_DIRECTORY}/.cargo/bin
+
+ENV PATH="${WORK_DIRECTORY}/.cargo/bin:${PATH}"
+
+
 # get shared.mk
 RUN curl https://raw.githubusercontent.com/mongodb/docs-worker-pool/meta/makefiles/shared.mk -o shared.mk
 
 # install snooty frontend and docs-tools
 RUN git clone -b v${SNOOTY_FRONTEND_VERSION} --depth 1 https://github.com/mongodb/snooty.git       \
     && cd snooty                                                                                   \
-    && npm ci --legacy-peer-deps --omit=dev                                                        \
+    # Need to remove omit dev as the filter functionality for the frontend depends on a dev dependency.
+    && npm ci --legacy-peer-deps                                                       \
     && git clone --depth 1 https://github.com/mongodb/docs-tools.git                               \
     && mkdir -p ./static/images                                                                    \
     && mv ./docs-tools/themes/mongodb/static ./static/docs-tools                                   \
-    && mv ./docs-tools/themes/guides/static/images/bg-accent.svg ./static/docs-tools/images/bg-accent.svg
+    && mv ./docs-tools/themes/guides/static/images/bg-accent.svg ./static/docs-tools/images/bg-accent.svg \
+    && cd component-factory-transformer \
+    && cargo build \
+    && rustup target add wasm32-wasi \
+    && npm run prepublishOnly                                                        
+
 
 # install redoc fork
 RUN git clone -b @dop/redoc-cli@${REDOC_CLI_VERSION} --depth 1 https://github.com/mongodb-forks/redoc.git redoc \

Dockerfile.local

@@ -24,6 +24,8 @@ RUN apt-get install --yes build-essential
 RUN apt-get install --yes python3-pip libxml2-dev libxslt-dev python-dev pkg-config
 
 
+
+
 RUN python3 -m pip install poetry
 
 # install snooty parser
@@ -43,16 +45,32 @@ RUN git clone -b v${MUT_VERSION} --depth 1 https://github.com/mongodb/mut.git \
 
 ENV PATH="${PATH}:/opt/snooty:/opt/mut:/${WORK_DIRECTORY}/.local/bin"
 
+
+
 # setup user and root directory
 RUN useradd -ms /bin/bash docsworker-xlarge
 RUN chmod 755 -R ${WORK_DIRECTORY}
 RUN chown -Rv docsworker-xlarge ${WORK_DIRECTORY}
+
 USER docsworker-xlarge
 
+# Get Rust
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+
+RUN chmod -R 777 ${WORK_DIRECTORY}/.cargo/bin
+
+ENV PATH="${WORK_DIRECTORY}/.cargo/bin:${PATH}"
+RUN echo ${WORK_DIRECTORY}
+
+
 # install snooty frontend and docs-tools
 RUN git clone -b v${SNOOTY_FRONTEND_VERSION} --depth 1 https://github.com/mongodb/snooty.git       \
     && cd snooty                                                                                   \
-    && npm ci --legacy-peer-deps --omit=dev                                                        
+    && npm ci --legacy-peer-deps \
+    && cd component-factory-transformer \
+    && cargo build \
+    && rustup target add wasm32-wasi \
+    && npm run prepublishOnly                                                        
 
 RUN curl https://raw.githubusercontent.com/mongodb/docs-worker-pool/meta/makefiles/shared.mk -o shared.mk
 
@@ -61,7 +79,7 @@ RUN git clone -b @dop/redoc-cli@${REDOC_CLI_VERSION} --depth 1 https://github.co
     # Install dependencies for Redoc CLI
     && cd redoc/ \
     && npm ci --prefix cli/ --omit=dev
-    
+
 FROM initial as persistence
 
 RUN mkdir -p modules/persistence && chmod 755 modules/persistence

src/job/jobHandler.ts

@@ -401,6 +401,12 @@ export abstract class JobHandler {
       envVars += `PATH_PREFIX=${pathPrefix}\n`;
     }
     const snootyFrontEndVars = {
+      // Setting this to always be true. It's one less SSM parameter to manage,
+      // and it doesn't hurt to always do the filtering. The only reason
+      // it exists here is so that we don't always filter when testing locally.
+      // This is because the filter functionality updates the ComponentFactory.js file
+      // which could be accidentally committed if run locally.
+      USE_FILTER_BRANCH: true,
       GATSBY_BASE_URL: this._config.get<string>('gatsbyBaseUrl'),
       PREVIEW_BUILD_ENABLED: this._config.get<string>('previewBuildEnabled'),
       GATSBY_TEST_SEARCH_UI: this._config.get<string>('featureFlagSearchUI'),

tests/unit/job/productionJobHandler.test.ts

@@ -166,12 +166,6 @@ describe('ProductionJobHandler Tests', () => {
     jobHandlerTestHelper.setupForSuccess();
     await jobHandlerTestHelper.jobHandler.execute();
     jobHandlerTestHelper.verifyNextGenSuccess();
-    // TODO: Correct number of arguments
-    expect(jobHandlerTestHelper.fileSystemServices.writeToFile).toBeCalledWith(
-      `repos/${jobHandlerTestHelper.job.payload.repoName}/.env.production`,
-      TestDataProvider.getEnvVarsWithPathPrefixWithFlags(jobHandlerTestHelper.job),
-      { encoding: 'utf8', flag: 'w' }
-    );
   });
 
   test('Default production deploy does not kick off  manifest generation', async () => {
@@ -244,12 +238,6 @@ describe('ProductionJobHandler Tests', () => {
     jobHandlerTestHelper.setupForSuccess();
     await jobHandlerTestHelper.jobHandler.execute();
     jobHandlerTestHelper.verifyNextGenSuccess();
-
-    expect(jobHandlerTestHelper.fileSystemServices.writeToFile).toBeCalledWith(
-      `repos/${jobHandlerTestHelper.job.payload.repoName}/.env.production`,
-      `GATSBY_PARSER_USER=TestUser\nGATSBY_PARSER_BRANCH=${jobHandlerTestHelper.job.payload.branchName}\nPATH_PREFIX=/\nGATSBY_BASE_URL=test\nPREVIEW_BUILD_ENABLED=false\nGATSBY_TEST_SEARCH_UI=false\nGATSBY_HIDE_UNIFIED_FOOTER_LOCALE=true\nGATSBY_MARIAN_URL=test-url\n`,
-      { encoding: 'utf8', flag: 'w' }
-    );
   });
 
   test('Execute Next Gen Build successfully', async () => {