DOP-3511 (#771)

* remove associated products if they are not in repo branches. separate repo branches and cache response from DB

* update failing test

* update metadata when getting from zip

* destructure for one prop

Author: seungpark

modules/persistence/index.ts

@@ -7,7 +7,12 @@ import minimist from 'minimist';
 import * as mongodb from 'mongodb';
 import { teardown as closeDBConnection } from './src/services/connector';
 import { insertPages } from './src/services/pages';
-import { insertMetadata, insertMergedMetadataEntries, deleteStaleMetadata } from './src/services/metadata';
+import {
+  insertMetadata,
+  insertMergedMetadataEntries,
+  deleteStaleMetadata,
+  metadataFromZip,
+} from './src/services/metadata';
 import { upsertAssets } from './src/services/assets';
 
 interface ModuleArgs {
@@ -31,11 +36,11 @@ const app = async (path: string) => {
     // atomic buildId for all artifacts read by this module - fundamental assumption
     // that only one build will be used per run of this module.
     const buildId = new mongodb.ObjectId();
-
-    await Promise.all([insertPages(buildId, zip), insertMetadata(buildId, zip), upsertAssets(zip)]);
-    await insertMergedMetadataEntries(buildId, zip);
+    const metadata = await metadataFromZip(zip);
+    await Promise.all([insertPages(buildId, zip), insertMetadata(buildId, metadata), upsertAssets(zip)]);
+    await insertMergedMetadataEntries(buildId, metadata);
     // DOP-3447 clean up stale metadata
-    await deleteStaleMetadata(zip);
+    await deleteStaleMetadata(metadata);
     closeDBConnection();
     process.exit(0);
   } catch (error) {

modules/persistence/src/services/metadata/ToC/index.ts

@@ -13,8 +13,9 @@ export interface ToC {
   [key: string]: any;
 }
 
-type project = string;
-type branchName = string;
+// TODO: move this typing to outer scope ie. metadata/index.ts
+export type project = string;
+export type branchName = string;
 type branch = {
   [key: branchName]: ToCCopies;
 };

modules/persistence/src/services/metadata/associated_products/index.ts

@@ -1,12 +1,13 @@
 import { AggregationCursor } from 'mongodb';
 import { Metadata } from '..';
-import { pool, db } from '../../connector';
-import { ToCInsertions, TocOrderInsertions, traverseAndMerge, copyToCTree } from '../ToC';
+import { db } from '../../connector';
+import { getAllAssociatedRepoBranchesEntries, getRepoBranchesEntry } from '../repos_branches';
+import { ToCInsertions, TocOrderInsertions, traverseAndMerge, copyToCTree, project } from '../ToC';
 import { prefixFromEnvironment } from '../ToC/utils/prefixFromEnvironment';
 
 export interface AssociatedProduct {
-  name: string;
-  versions: string[];
+  name: project;
+  versions?: string[];
 }
 
 export type SharedMetadata = Metadata;
@@ -41,33 +42,6 @@ export interface ReposBranchesDocument {
   [key: string]: any;
 }
 
-// Queries pool*.repos_branches for any entries for the given project and branch from a metadata entry.
-const getRepoBranchesEntry = async (project, branch = ''): Promise<ReposBranchesDocument> => {
-  const db = await pool();
-  const query = {
-    project,
-  };
-  if (branch) {
-    query['branches'] = {
-      $elemMatch: { gitBranchName: branch },
-    };
-  }
-  return db.collection('repos_branches').findOne(query) as unknown as ReposBranchesDocument;
-};
-
-// Queries pool*.repos_branches for all entries for associated_products in a shared metadata entry
-const getAllAssociatedRepoBranchesEntries = async (metadata: Metadata) => {
-  const { associated_products } = metadata;
-  if (!associated_products || !associated_products.length) return [];
-  const associatedProductNames = associated_products.map((a) => a.name);
-  const db = await pool();
-  const entries = await db
-    .collection('repos_branches')
-    .find({ project: { $in: associatedProductNames } })
-    .toArray();
-  return entries as unknown as ReposBranchesDocument[];
-};
-
 const mapRepoBranches = (repoBranches: ReposBranchesDocument[]) =>
   Object.fromEntries(
     repoBranches.map((entry) => {
@@ -194,8 +168,6 @@ const getAssociatedProducts = async (umbrellaMetadata) => {
 export const mergeAssociatedToCs = async (metadata: Metadata) => {
   try {
     const { project, branch } = metadata;
-    // TODO: DOP-3518
-    // should get an umbrella metadata entry that is in repos branches
     const umbrellaMetadata = hasAssociations(metadata) ? metadata : await umbrellaMetadataEntry(project);
 
     // Short circuit execution here if there's no umbrella product metadata found

modules/persistence/src/services/metadata/index.ts

@@ -3,7 +3,8 @@ import { deserialize } from 'bson';
 import { ObjectId } from 'mongodb';
 import { db, deleteDocuments, insert } from '../connector';
 import { mergeAssociatedToCs, AssociatedProduct } from './associated_products';
-import { ToC } from './ToC';
+import { getRepoBranchesEntry } from './repos_branches';
+import { project, ToC } from './ToC';
 
 const COLLECTION_NAME = 'metadata';
 
@@ -18,26 +19,55 @@ export interface Metadata {
 // Service responsible for memoization of metadata entries.
 // Any extraneous logic performed on metadata entries as part of upload should be added here
 // or within subfolders of this module
-const metadataFromZip = (zip: AdmZip) => {
+export const metadataFromZip = async (zip: AdmZip) => {
   const zipEntries = zip.getEntries();
-  return zipEntries
+  const metadata = zipEntries
     .filter((entry) => entry.entryName === 'site.bson')
     .map((entry) => deserialize(entry.getData()))[0] as Metadata;
+  await verifyMetadata(metadata);
+  return metadata;
 };
 
-export const insertMetadata = async (buildId: ObjectId, zip: AdmZip) => {
+// Verifies the entries for associated_products in metadata
+const verifyMetadata = async (metadata: Metadata) => {
+  try {
+    if (!metadata['associated_products']?.length) {
+      return metadata;
+    }
+    const invalidNames: project[] = [];
+    const promises = metadata['associated_products'].map(async (ap) => {
+      const branchEntry = await getRepoBranchesEntry(ap.name);
+      if (!branchEntry) {
+        invalidNames.push(ap.name);
+      }
+    });
+    await Promise.all(promises);
+    if (invalidNames.length) {
+      console.warn(`No branches found for associated project(s) [${invalidNames}]. Removing such associated_products`);
+      metadata.associated_products = metadata.associated_products.filter((ap) => !invalidNames.includes(ap.name));
+    }
+    if (!metadata['associated_products'].length) {
+      delete metadata['associated_products'];
+    }
+    return metadata;
+  } catch (e) {
+    console.error(`Error while verifying metadata ${e}`);
+    throw e;
+  }
+};
+
+export const insertMetadata = async (buildId: ObjectId, metadata: Metadata) => {
   try {
-    const metadata = metadataFromZip(zip);
     return insert([metadata], COLLECTION_NAME, buildId);
   } catch (error) {
     console.error(`Error at insertion time for ${COLLECTION_NAME}: ${error}`);
     throw error;
   }
 };
 
-export const insertMergedMetadataEntries = async (buildId: ObjectId, zip: AdmZip) => {
+export const insertMergedMetadataEntries = async (buildId: ObjectId, metadata: Metadata) => {
   try {
-    const mergedMetadataEntries = await mergeAssociatedToCs(metadataFromZip(zip));
+    const mergedMetadataEntries = await mergeAssociatedToCs(metadata);
     return mergedMetadataEntries
       ? await Promise.all(mergedMetadataEntries.map((m) => insert([m], COLLECTION_NAME, buildId)))
       : [];
@@ -47,9 +77,9 @@ export const insertMergedMetadataEntries = async (buildId: ObjectId, zip: AdmZip
   }
 };
 
-export const deleteStaleMetadata = async (zip: AdmZip) => {
+export const deleteStaleMetadata = async (metadata: Metadata) => {
   try {
-    const { project, branch } = metadataFromZip(zip);
+    const { project, branch } = metadata;
     const LIMIT = 4;
     // get most recent metadata for this project-branch
     const snooty = await db();

modules/persistence/src/services/metadata/repos_branches/index.ts

@@ -0,0 +1,101 @@
+import { pool, db } from '../../connector';
+import { Metadata } from '..';
+import { project } from '../ToC';
+import { WithId } from 'mongodb';
+
+type EnvKeyedObject = {
+  prd: any;
+  preprd: any;
+  dotcomstg: any;
+  dotcomprd: any;
+};
+
+export interface BranchEntry {
+  name: string;
+  gitBranchName: string;
+  [key: string]: any;
+}
+
+export interface ReposBranchesDocument extends WithId<Document> {
+  repoName: string;
+  project: string;
+  branches: BranchEntry[];
+  url: EnvKeyedObject;
+  prefix: EnvKeyedObject;
+  [key: string]: any;
+}
+
+const internals: { [key: project]: ReposBranchesDocument } = {};
+
+// Queries pool*.repos_branches for all entries for associated_products in a shared metadata entry
+export const getAllAssociatedRepoBranchesEntries = async (metadata: Metadata) => {
+  const { associated_products = [] } = metadata;
+  if (!associated_products.length) return [];
+
+  const res: ReposBranchesDocument[] = [],
+    fetch: project[] = [];
+  associated_products.forEach((ap) => {
+    if (internals[ap.name]) {
+      res.push(internals[ap.name]);
+    } else {
+      fetch.push(ap.name);
+    }
+  });
+
+  if (!fetch.length) {
+    return res;
+  }
+
+  try {
+    const db = await pool();
+    await db
+      .collection('repos_branches')
+      .find({ project: { $in: fetch } })
+      .forEach((doc: ReposBranchesDocument) => {
+        // TODO: store in cache
+        internals[doc['project']] = doc;
+        res.push(doc);
+      });
+    return res;
+  } catch (e) {
+    console.error(`Error while getting associated repo branches: ${e}`);
+    throw e;
+  }
+};
+
+// Queries pool*.repos_branches for any entries for the given project and branch from a metadata entry.
+export const getRepoBranchesEntry = async (project: project, branch = ''): Promise<ReposBranchesDocument> => {
+  const cachedDoc = internals[project];
+  // return cached repo doc if exists
+  if (cachedDoc !== undefined) {
+    if (!branch) {
+      return cachedDoc;
+    }
+
+    return cachedDoc.branches.map((b) => b.gitBranchName).includes(branch)
+      ? cachedDoc
+      : (null as unknown as ReposBranchesDocument);
+  }
+
+  // get from DB if not cached
+  try {
+    const db = await pool();
+    const query = {
+      project,
+    };
+    if (branch) {
+      query['branches'] = {
+        $elemMatch: { gitBranchName: branch },
+      };
+    }
+    const res = (await db.collection('repos_branches').findOne(query)) as unknown as ReposBranchesDocument;
+    // if not already set, set cache value for repo_branches
+    if (!internals[project]) {
+      internals[project] = res;
+    }
+    return res;
+  } catch (e) {
+    console.error(`Error while getting repo branches entry: ${e}`);
+    throw e;
+  }
+};

modules/persistence/tests/metadata/metadata.test.ts

@@ -64,8 +64,8 @@ describe('metadata module', () => {
   });
 
   describe('metadataFromZip', () => {
-    const metaFromZip = _metadataFromZip(zip);
-    it('should get metadata from site.bson', () => {
+    it('should get metadata from site.bson', async () => {
+      const metaFromZip = await _metadataFromZip(zip);
       expect(metaFromZip).toEqual(meta);
     });
   });
@@ -74,7 +74,8 @@ describe('metadata module', () => {
     const buildId = new ObjectId();
     it('should insert metadata docs into metadata collection', async () => {
       try {
-        await insertMetadata(buildId, zip);
+        const metaFromZip = await _metadataFromZip(zip);
+        await insertMetadata(buildId, metaFromZip);
       } catch (e) {
         console.log(e);
       }
@@ -100,7 +101,8 @@ describe('metadata module', () => {
 
     it('removes copies of metadata for same project-branch, keeping the most recent ones', async () => {
       await mockDb.collection('metadata').insertMany(testData);
-      await deleteStaleMetadata(zip);
+      const metaFromZip = await _metadataFromZip(zip);
+      await deleteStaleMetadata(metaFromZip);
       const res = await mockDb
         .collection('metadata')
         .find({ project, branch })