DOP-3508: Save successful builds (#779)

* add version data to atlas fetch, create save util

* add check for consistent data with saved version data

* added gitignore, implemented flags for successful builds with githash, invoked save util to db

* created createFetchGitHash with cache

* further modularized, added ensureSavedVersionDataMatches util

* added mocks and assertions to pageBuilder.test.ts

* reset db env, created const for githash

* move totalSuccess instantiation into closer scope

* PR feedback

* deleted comment

Author: mmeigs

modules/oas-page-builder/.gitignore

@@ -0,0 +1 @@
+.env

modules/oas-page-builder/src/services/database.ts

@@ -1,5 +1,5 @@
 import { Db, MongoClient } from 'mongodb';
-import { OASFile, OASFileGitHash } from './models/OASFile';
+import { OASFile, OASFilePartial, VersionData } from './models/OASFile';
 
 const COLLECTION_NAME = 'oas_files';
 
@@ -49,17 +49,42 @@ const db = async () => {
   return dbInstance;
 };
 
-// Finds the last saved git hash in our db for an OpenAPI spec file. This git hash
+// Finds the last saved git hash and version data in our db for an OpenAPI spec file. The git hash
 // should have an existing spec file but the hash may be subject to change every 24 hours.
-export const findLastSavedGitHash = async (apiKeyword: string) => {
+export const findLastSavedVersionData = async (apiKeyword: string) => {
   const dbSession = await db();
   try {
-    const projection = { gitHash: 1 };
+    const projection = { gitHash: 1, versions: 1 };
     const filter = { api: apiKeyword };
     const oasFilesCollection = dbSession.collection<OASFile>(COLLECTION_NAME);
-    return oasFilesCollection.findOne<OASFileGitHash>(filter, { projection });
+    return oasFilesCollection.findOne<OASFilePartial>(filter, { projection });
   } catch (error) {
     console.error(`Error fetching lastest git hash for API: ${apiKeyword}.`);
     throw error;
   }
 };
+
+export const saveSuccessfulBuildVersionData = async (apiKeyword: string, gitHash: string, versionData: VersionData) => {
+  const dbSession = await db();
+  try {
+    const query = {
+      api: apiKeyword,
+    };
+    const update = {
+      $set: {
+        gitHash: gitHash,
+        versions: versionData,
+        lastUpdated: new Date().toISOString(),
+      },
+    };
+    const options = {
+      upsert: true,
+    };
+
+    const oasFilesCollection = dbSession.collection<OASFile>(COLLECTION_NAME);
+    await oasFilesCollection.updateOne(query, update, options);
+  } catch (error) {
+    console.error(`Error updating lastest git hash and versions for API: ${apiKeyword}.`);
+    throw error;
+  }
+};

modules/oas-page-builder/src/services/models/OASFile.ts

@@ -1,8 +1,15 @@
 // Model for documents in the "oas_files" collection.
+
+export interface VersionData {
+  [k: string]: string[];
+}
+
 export interface OASFile {
   api: string;
   fileContent: string;
   gitHash: string;
+  lastUpdated: string;
+  versions: VersionData;
 }
 
-export type OASFileGitHash = Pick<OASFile, 'gitHash'>;
+export type OASFilePartial = Pick<OASFile, 'gitHash' | 'versions'>;

modules/oas-page-builder/src/services/pageBuilder.ts

@@ -1,10 +1,13 @@
 import fetch from 'node-fetch';
 import { normalizePath } from '../utils/normalizePath';
+import { fetchVersionData } from '../utils/fetchVersionData';
 import { RedocExecutor } from './redocExecutor';
-import { findLastSavedGitHash } from './database';
+import { findLastSavedVersionData, saveSuccessfulBuildVersionData } from './database';
 import { OASPageMetadata, PageBuilderOptions, RedocBuildOptions } from './types';
+import { VersionData } from './models/OASFile';
 
 const OAS_FILE_SERVER = 'https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/';
+const GIT_HASH_URL = 'https://cloud.mongodb.com/version';
 
 const fetchTextData = async (url: string, errMsg: string) => {
   const res = await fetch(url);
@@ -15,12 +18,44 @@ const fetchTextData = async (url: string, errMsg: string) => {
   return res.text();
 };
 
+const createFetchGitHash = () => {
+  let gitHashCache: string;
+  return {
+    fetchGitHash: async () => {
+      if (gitHashCache) return gitHashCache;
+      try {
+        const gitHash = await fetchTextData(GIT_HASH_URL, 'Could not find current version or git hash');
+        gitHashCache = gitHash;
+        return gitHash;
+      } catch (e) {
+        console.error(e);
+        throw new Error(`Unsuccessful git hash fetch`);
+      }
+    },
+    resetGitHashCache: () => {
+      gitHashCache = '';
+    },
+  };
+};
+
+const { fetchGitHash, resetGitHashCache } = createFetchGitHash();
+
 interface AtlasSpecUrlParams {
   apiKeyword: string;
   apiVersion?: string;
   resourceVersion?: string;
 }
 
+const ensureSavedVersionDataMatches = (versions: VersionData, apiVersion?: string, resourceVersion?: string) => {
+  // Check that requested versions are included in saved version data
+  if (apiVersion) {
+    if (!versions.major.includes(apiVersion) || (resourceVersion && !versions[apiVersion].includes(resourceVersion))) {
+      throw new Error(`Last successful build data does not include necessary version data:\n
+      Version requested: ${apiVersion}${resourceVersion ? ` - ${resourceVersion}` : ``}`);
+    }
+  }
+};
+
 const getAtlasSpecUrl = async ({ apiKeyword, apiVersion, resourceVersion }: AtlasSpecUrlParams) => {
   // Currently, the only expected API fetched programmatically is the Cloud Admin API,
   // but it's possible to have more in the future with varying processes.
@@ -34,10 +69,10 @@ const getAtlasSpecUrl = async ({ apiKeyword, apiVersion, resourceVersion }: Atla
   }`;
 
   let oasFileURL;
+  let successfulGitHash = true;
 
   try {
-    const versionURL = 'https://cloud.mongodb.com/version';
-    const gitHash = await fetchTextData(versionURL, 'Could not find current version or git hash');
+    const gitHash = await fetchGitHash();
     oasFileURL = `${OAS_FILE_SERVER}${gitHash}${versionExtension}.json`;
 
     // Sometimes the latest git hash might not have a fully available spec file yet.
@@ -46,17 +81,22 @@ const getAtlasSpecUrl = async ({ apiKeyword, apiVersion, resourceVersion }: Atla
     await fetchTextData(oasFileURL, `Error fetching data from ${oasFileURL}`);
   } catch (e) {
     console.error(e);
+    successfulGitHash = false;
 
-    const res = await findLastSavedGitHash(apiKeyword);
+    const res = await findLastSavedVersionData(apiKeyword);
     if (res) {
+      ensureSavedVersionDataMatches(res.versions, apiVersion, resourceVersion);
       oasFileURL = `${OAS_FILE_SERVER}${res.gitHash}${versionExtension}.json`;
       console.log(`Using ${oasFileURL}`);
     } else {
       throw new Error(`Could not find a saved hash for API: ${apiKeyword}`);
     }
   }
 
-  return oasFileURL;
+  return {
+    oasFileURL,
+    successfulGitHash,
+  };
 };
 
 interface GetOASpecParams {
@@ -82,14 +122,21 @@ async function getOASpec({
 }: GetOASpecParams) {
   try {
     let spec = '';
+    let isSuccessfulBuild = true;
     const buildOptions: RedocBuildOptions = {};
     if (sourceType === 'url') {
       spec = source;
     } else if (sourceType === 'local') {
       const localFilePath = normalizePath(`${repoPath}/source/${source}`);
       spec = localFilePath;
     } else if (sourceType === 'atlas') {
-      spec = await getAtlasSpecUrl({ apiKeyword: source, apiVersion, resourceVersion });
+      const { oasFileURL, successfulGitHash } = await getAtlasSpecUrl({
+        apiKeyword: source,
+        apiVersion,
+        resourceVersion,
+      });
+      spec = oasFileURL;
+      isSuccessfulBuild = successfulGitHash;
       // Ignore "incompatible types" warnings for Atlas Admin API/cloud-docs
 
       buildOptions['ignoreIncompatibleTypes'] = true;
@@ -102,8 +149,10 @@ async function getOASpec({
     const path = `${output}/${pageSlug}${filePathExtension}/index.html`;
     const finalFilename = normalizePath(path);
     await redocExecutor.execute(spec, finalFilename, buildOptions);
+    return isSuccessfulBuild;
   } catch (e) {
     console.error(e);
+    return false;
   }
 }
 
@@ -114,6 +163,7 @@ export const buildOpenAPIPages = async (
   const redocExecutor = new RedocExecutor(redocPath, siteUrl, siteTitle);
 
   for (const [pageSlug, data] of entries) {
+    let totalSuccess = true;
     const { source_type: sourceType, source, api_version: apiVersion, resource_versions: resourceVersions } = data;
 
     if (!apiVersion && resourceVersions && resourceVersions.length > 0) {
@@ -127,10 +177,41 @@ export const buildOpenAPIPages = async (
       // if a resource versions array is provided, then we can loop through the resourceVersions array and call the getOASpec
       // for each minor version
       for (const resourceVersion of resourceVersions) {
-        await getOASpec({ source, sourceType, output, pageSlug, redocExecutor, repoPath, apiVersion, resourceVersion });
+        const isSuccessfulBuild = await getOASpec({
+          source,
+          sourceType,
+          output,
+          pageSlug,
+          redocExecutor,
+          repoPath,
+          apiVersion,
+          resourceVersion,
+        });
+        if (!isSuccessfulBuild) totalSuccess = false;
       }
     }
     // apiVersion can be undefined, this case is handled within the getOASpec function
-    await getOASpec({ source, sourceType, output, pageSlug, redocExecutor, repoPath, apiVersion });
+    const isSuccessfulBuild = await getOASpec({
+      source,
+      sourceType,
+      output,
+      pageSlug,
+      redocExecutor,
+      repoPath,
+      apiVersion,
+    });
+    if (!isSuccessfulBuild) totalSuccess = false;
+
+    // If all builds successful, persist git hash and version data in db
+    if (totalSuccess && sourceType == 'atlas') {
+      try {
+        const gitHash = await fetchGitHash();
+        const versions = await fetchVersionData(gitHash);
+        await saveSuccessfulBuildVersionData(source, gitHash, versions);
+      } catch (e) {
+        console.error(e);
+      }
+    }
+    resetGitHashCache();
   }
 };

modules/oas-page-builder/src/utils/fetchVersionData.ts

@@ -0,0 +1,6 @@
+export const fetchVersionData = async (gitHash: string) => {
+  const versionUrl = `https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/${gitHash}-api-versions.json`;
+  const res = await fetch(versionUrl);
+  const { versions } = await res.json();
+  return versions;
+};

modules/oas-page-builder/tests/unit/services/pageBuilder.test.ts

@@ -1,7 +1,8 @@
 import fetch from 'node-fetch';
-import { findLastSavedGitHash } from '../../../src/services/database';
+import { findLastSavedVersionData, saveSuccessfulBuildVersionData } from '../../../src/services/database';
 import { buildOpenAPIPages } from '../../../src/services/pageBuilder';
 import { OASPageMetadata, PageBuilderOptions } from '../../../src/services/types';
+import { fetchVersionData } from '../../../src/utils/fetchVersionData';
 
 const MOCKED_GIT_HASH = '1234';
 const LAST_SAVED_GIT_HASH = '4321';
@@ -17,7 +18,13 @@ jest.mock('../../../src/services/redocExecutor', () => ({
 // Mock database since implementation relies on database instance. Returned values
 // are mocked for each test.
 jest.mock('../../../src/services/database', () => ({
-  findLastSavedGitHash: jest.fn(),
+  findLastSavedVersionData: jest.fn(),
+  saveSuccessfulBuildVersionData: jest.fn(),
+}));
+
+// Mock version data fetch to override mocked node-fetch
+jest.mock('../../../src/utils/fetchVersionData', () => ({
+  fetchVersionData: jest.fn(),
 }));
 
 // Helper function for concatenated output path
@@ -50,6 +57,7 @@ describe('pageBuilder', () => {
   beforeEach(() => {
     // Reset mock to reset call count
     mockExecute.mockReset();
+    jest.clearAllMocks();
   });
 
   it('builds OpenAPI pages', async () => {
@@ -130,6 +138,9 @@ describe('pageBuilder', () => {
   });
 
   it('builds OpenAPI pages with api version and resource version', async () => {
+    const expectedVersionData = { major: ['1.0', '2.0'], '2.0': ['01-01-2020'] };
+    // @ts-ignore
+    fetchVersionData.mockReturnValue(expectedVersionData);
     mockFetchImplementation(true);
 
     const testEntries: [string, OASPageMetadata][] = [
@@ -191,12 +202,15 @@ describe('pageBuilder', () => {
       getExpectedOutputPath(testOptions.output, testEntries[2][0], '2.0'),
       expectedAtlasBuildOptions
     );
+
+    expect(saveSuccessfulBuildVersionData).toBeCalledTimes(1);
+    expect(saveSuccessfulBuildVersionData).toBeCalledWith('cloud', MOCKED_GIT_HASH, expectedVersionData);
   });
 
   it('builds Atlas Cloud API with backup git hash', async () => {
     mockFetchImplementation(false);
     // @ts-ignore
-    findLastSavedGitHash.mockReturnValue({ gitHash: LAST_SAVED_GIT_HASH });
+    findLastSavedVersionData.mockReturnValue({ gitHash: LAST_SAVED_GIT_HASH });
 
     const testEntries: [string, OASPageMetadata][] = [['path/to/page/1', { source_type: 'atlas', source: 'cloud' }]];
 
@@ -206,16 +220,18 @@ describe('pageBuilder', () => {
       getExpectedOutputPath(testOptions.output, testEntries[0][0]),
       expectedAtlasBuildOptions
     );
+    expect(saveSuccessfulBuildVersionData).toBeCalledTimes(0);
   });
 
   it('does not build atlas OAS when backup git hash is missing', async () => {
     mockFetchImplementation(false);
     // @ts-ignore
-    findLastSavedGitHash.mockReturnValue(null);
+    findLastSavedVersionData.mockReturnValue(null);
 
     const testEntries: [string, OASPageMetadata][] = [['path/to/page/1', { source_type: 'atlas', source: 'cloud' }]];
 
     await buildOpenAPIPages(testEntries, testOptions);
     expect(mockExecute).toBeCalledTimes(0);
+    expect(saveSuccessfulBuildVersionData).toBeCalledTimes(0);
   });
 });