Dop 3203 (#703)

casthewiz · web-flow · commit 9b3e7a07df60 · 2022-12-12T13:20:45.000-07:00
* DOP-3203: Add upsert functionality, asset upserts * fixup * DOP-3203: Stub functions, Stub tests, finalized merge query + logic * DOP-3203: Reorder logic * Improve the typing * fixup * stub toctreeorder * fixup * fixup * fixup * fixup * fixup * Use project for node options, not name * fixup * DOP-3203: Invoke the merging logic * DOP-3203: Coalesce feedback * DOP-3203: Rename associations to associated_products * DOP-3203: Null safety w/ default empty objects in insertions * DOP-3203: Unset _id in new shared metadata documents * DOP-3203: Address feedback further * fixup * fixup * fixup * fixup
diff --git a/modules/README.md b/modules/README.md
@@ -1,3 +1,5 @@
 # Overview
 
 This directory contains various build-time modules for use in makefiles or for use as published modules utilized in other CI/CD routines.
+Each entry in this directory should NOT be coupled to another module, or code within the autobuilder.
+It is the express intent of maintainers that each entry is standalone, and capable of being run or exported as needed with dependencies only within their constituent directories.
diff --git a/modules/persistence/README.md b/modules/persistence/README.md
@@ -1,4 +1,4 @@
-# Snooty Transport Module
+# Snooty Persistence Module
 
 As part of integrating Snooty Parser with the unified Snooty toolchain, this module holds exclusive responsibility over transporting and persisting AST and metadata output artifacts from the parser into datastores.
 
diff --git a/modules/persistence/index.ts b/modules/persistence/index.ts
@@ -5,8 +5,10 @@ dotenv.config();
 import AdmZip from 'adm-zip';
 import minimist from 'minimist';
 import * as mongodb from 'mongodb';
+import { teardown as closeDBConnection } from './src/services/connector';
 import { insertPages } from './src/services/pages';
-import { insertMetadata } from './src/services/metadata';
+import { insertMetadata, insertUmbrellaMetadata } from './src/services/metadata';
+import { upsertAssets } from './src/services/assets';
 
 interface ModuleArgs {
   path: string;
@@ -29,10 +31,17 @@ const app = async (path: string) => {
     // atomic buildId for all artifacts read by this module - fundamental assumption
     // that only one build will be used per run of this module.
     const buildId = new mongodb.ObjectId();
-    await Promise.all([insertPages(buildId, zip), insertMetadata(buildId, zip)]);
+    await Promise.all([
+      insertPages(buildId, zip),
+      insertMetadata(buildId, zip),
+      upsertAssets(zip),
+      insertUmbrellaMetadata(buildId, zip),
+    ]);
+    closeDBConnection();
     process.exit(0);
   } catch (error) {
     console.error(`Persistence Module encountered a terminal error: ${error}`);
+    closeDBConnection();
     throw error;
   }
 };
diff --git a/modules/persistence/src/services/assets/index.ts b/modules/persistence/src/services/assets/index.ts
@@ -0,0 +1,23 @@
+import AdmZip from 'adm-zip';
+import { upsert } from '../connector';
+
+const COLLECTION_NAME = 'documents';
+
+// Service responsible for upsertion of any image or blob assets.
+
+const assetsFromZip = (zip: AdmZip) => {
+  const assets = zip.getEntries();
+  return assets
+    .filter((entry) => entry.entryName?.startsWith('assets/'))
+    .map((entry) => ({ _id: entry.entryName.replace('assets/', ''), data: entry.getData() }));
+};
+
+export const upsertAssets = async (zip: AdmZip) => {
+  try {
+    const assets = await assetsFromZip(zip);
+    return Promise.all(assets.map((asset) => upsert(asset, COLLECTION_NAME, asset._id)));
+  } catch (error) {
+    console.error(`Error at upsertion time for ${COLLECTION_NAME}: ${error}`);
+    throw error;
+  }
+};
diff --git a/modules/persistence/src/services/connector/index.ts b/modules/persistence/src/services/connector/index.ts
@@ -1,14 +1,30 @@
+// Service that holds responsibility for initializing and exposing mdb interfaces.
+// Also exports helper functions for common operations (insert, upsert one by _id, etc.)
+// When adding helpers here, ask yourself if the helper will be used by more than one service
+// If no, the helper should be implemented in that service, not here
+
 import * as mongodb from 'mongodb';
 import { ObjectId, Db } from 'mongodb';
+import { db as poolDb } from './pool';
 
 // We should only ever have one client active at a time.
 const atlasURL = `mongodb+srv://${process.env.DB_USER}:${process.env.DB_PASSWORD}@${process.env.DB_HOST}/?retryWrites=true&w=majority`;
 const client = new mongodb.MongoClient(atlasURL);
+
+export const teardown = () => {
+  client.close();
+};
+
+// Initialize and export our pool connection
+// Try to limit access to pool as much as possible - we mostly want it for just repo_branches.
+export const pool = async () => {
+  return poolDb(client);
+};
+
 // cached db object, so we can handle initial connection process once if unitialized
 let dbInstance: Db;
-
 // Handles memoization of db object, and initial connection logic if needs to be initialized
-const db = async () => {
+export const db = async () => {
   if (!dbInstance) {
     try {
       await client.connect();
@@ -33,3 +49,16 @@ export const insert = async (docs: any[], collection: string, buildId: ObjectId)
     throw error;
   }
 };
+
+// Upsert wrapper, requires an _id field.
+export const upsert = async (payload: any, collection: string, _id: string | ObjectId) => {
+  const upsertSession = await db();
+  try {
+    const query = { _id };
+    const update = { $set: payload };
+    const options = { upsert: true };
+    return upsertSession.collection(collection).updateOne(query, update, options);
+  } catch (error) {
+    console.error(`Error at upsertion time for ${collection}: ${error}`);
+  }
+};
diff --git a/modules/persistence/src/services/connector/pool/index.ts b/modules/persistence/src/services/connector/pool/index.ts
@@ -0,0 +1,17 @@
+import { MongoClient, Db } from 'mongodb';
+
+// cached db object, so we can handle initial connection process once if unitialized
+let dbInstance: Db;
+// Handles memoization of db object, and initial connection logic if needs to be initialized
+export const db = async (client: MongoClient) => {
+  if (!dbInstance) {
+    try {
+      await client.connect();
+      dbInstance = client.db(process.env.POOL_DB_NAME);
+    } catch (error) {
+      console.error(`Error at db client connection: ${error}`);
+      throw error;
+    }
+  }
+  return dbInstance;
+};
diff --git a/modules/persistence/src/services/metadata/ToC/index.ts b/modules/persistence/src/services/metadata/ToC/index.ts
@@ -0,0 +1,77 @@
+import { SharedMetadata, AssociatedProduct } from '../associated_products';
+
+export interface ToC {
+  title: string;
+  slug: string;
+  children: ToC[];
+  options?: {
+    [key: string]: any;
+  };
+  [key: string]: any;
+}
+
+type project = string;
+type branchName = string;
+type branch = { [key: branchName]: ToC };
+
+export interface ToCInsertions {
+  [key: project]: branch;
+}
+
+export interface TocOrderInsertions {
+  [key: string]: {
+    [key: string]: any[];
+  };
+}
+
+const isInsertionCandidateNode = (node: ToC, associated_products: AssociatedProduct[] = []) => {
+  const nodeInProducts = node.options?.project && associated_products.find((p) => p.name === node.options?.project);
+  const nodeHasNoChildren = !node.children || node.children.length === 0;
+  return nodeHasNoChildren && nodeInProducts;
+};
+
+const mergeNode = (node: ToC, tocs: ToCInsertions) => {
+  // Options might be undefined, so safely cast to {} if nullish
+  node.options = node.options ?? {};
+
+  const project = tocs[node?.options?.project];
+  const branches = Object.keys(project);
+  node.options.versions = branches;
+  node.children = branches.map((branch) => {
+    const child = project[branch];
+    const options = {
+      ...child.options,
+      version: branch,
+    };
+    child.options = options;
+    return child;
+  });
+  return node;
+};
+
+const mergeTocTreeOrder = (metadata: SharedMetadata, node, insertions: TocOrderInsertions) => {
+  const insertion = insertions[metadata.project]?.[metadata.branch] || [];
+  const index = metadata.toctreeOrder.indexOf(node.options?.project);
+  return metadata.toctreeOrder.splice(index, 0, ...insertion);
+};
+
+// BFS through the toctree from the metadata entry provided as an arg
+// and insert matching tocInsertion entries + tocOrders
+export const traverseAndMerge = (
+  metadata: SharedMetadata,
+  tocInsertions: ToCInsertions,
+  tocOrderInsertions: TocOrderInsertions
+) => {
+  const { toctree, associated_products } = metadata;
+
+  let queue = [toctree];
+  while (queue?.length) {
+    let next = queue.shift();
+    if (next && isInsertionCandidateNode(next, associated_products)) {
+      next = mergeNode(next, tocInsertions);
+      metadata.toctreeorder = mergeTocTreeOrder(metadata, next, tocOrderInsertions);
+    }
+    if (next?.children) queue = [...queue, ...next.children];
+  }
+  return metadata;
+};
diff --git a/modules/persistence/src/services/metadata/associated_products/index.ts b/modules/persistence/src/services/metadata/associated_products/index.ts
@@ -0,0 +1,160 @@
+import { AggregationCursor } from 'mongodb';
+import { pool, db } from '../../connector';
+import { ToC, ToCInsertions, TocOrderInsertions, traverseAndMerge } from '../ToC';
+
+export interface AssociatedProduct {
+  name: string;
+  versions: string[];
+}
+
+export interface SharedMetadata {
+  project: string;
+  branch: string;
+  associated_products?: AssociatedProduct[];
+  toctree: ToC;
+  toctreeOrder: any[];
+  [key: string]: any;
+}
+
+// TODO: move the branch/repobranch interfaces into their own file, or into a seperate abstraction?
+interface BranchEntry {
+  name: string;
+  gitBranchName: string;
+  [key: string]: any;
+}
+
+export interface ReposBranchesDocument {
+  repoName: string;
+  project: string;
+  branches: BranchEntry[];
+  [key: string]: any;
+}
+
+// Queries pool*.repos_branches for any entries for the given project and branch from a metadata entry.
+const getRepoBranchesEntry = async (project, branch) => {
+  const db = await pool();
+  return db.collection('repos_branches').findOne({ branches: { $elemMatch: { gitBranchName: branch } }, project });
+};
+
+// Queries pool*.repos_branches for all entries for associated_products in a shared metadata entry
+const getAllAssociatedRepoBranchesEntries = async (metadata: SharedMetadata) => {
+  const { associated_products } = metadata;
+  if (!associated_products || !associated_products.length) return [];
+  const associatedProductNames = associated_products.map((a) => a.name);
+  const db = await pool();
+  const entries = await db
+    .collection('repos_branches')
+    .find({ project: { $in: associatedProductNames } })
+    .toArray();
+  return entries as unknown as ReposBranchesDocument[];
+};
+
+const mapRepoBranches = (repoBranches: ReposBranchesDocument[]) =>
+  Object.fromEntries(
+    repoBranches.map((entry) => {
+      const branches = Object.fromEntries(entry.branches.map((branch) => [branch.gitBranchName, branch]));
+      return [entry.project, branches];
+    })
+  );
+
+const hasAssociations = (metadata) => !!metadata.associated_products?.length;
+
+const umbrellaMetadataEntry = async (metadata): Promise<SharedMetadata> => {
+  try {
+    const snooty = await db();
+    const entry = await snooty
+      .collection('metadata')
+      .find({
+        'associated_products.name': metadata.project,
+      })
+      .sort({ build_id: -1 })
+      .limit(1)
+      .toArray();
+    return entry[0] as unknown as SharedMetadata;
+  } catch (error) {
+    console.log(`Error at time of querying for umbrella metadata entry: ${error}`);
+    throw error;
+  }
+};
+
+// Convert our cursor from the shared ToC aggregation query into a series of ToC objects
+const shapeToCsCursor = async (
+  tocCursor: AggregationCursor,
+  repoBranchesMap
+): Promise<{ tocInsertions: ToCInsertions; tocOrderInsertions: TocOrderInsertions }> => {
+  const tocInsertions = {};
+  const tocOrderInsertions = {};
+
+  await tocCursor.forEach((doc) => {
+    // Initialize to empty object if we haven't already, for a given project.
+    if (!tocInsertions[doc._id.project]) tocInsertions[doc._id.project] = {};
+    if (!tocOrderInsertions[doc._id.project]) tocOrderInsertions[doc._id.project] = {};
+
+    // TODO: If we want staging builds with embedded versions, it needs to be added here
+    if (repoBranchesMap?.[doc._id.project]?.[doc._id.branch]) {
+      tocInsertions[doc._id.project][doc._id.branch] = doc.most_recent.toctree;
+      tocOrderInsertions[doc._id.project][doc._id.branch] = doc.most_recent.toctreeOrder;
+    }
+  });
+
+  return { tocInsertions, tocOrderInsertions };
+};
+
+const getAssociatedProducts = async (umbrellaMetadata) => {
+  try {
+    const { associated_products } = umbrellaMetadata;
+    const associatedProductNames = associated_products.map((a) => a.name);
+    const snooty = await db();
+
+    // This query matches on projects in associated_products for our given metadata that have a build_id and that do not have merged tocs
+    // then groups per branch and per project from those matches
+    // and gets the most recent doc entry (by build_id), with the toctree and toctreeOrder fields.
+    const tocs = snooty.collection('metadata').aggregate([
+      {
+        $match: { project: { $in: associatedProductNames }, build_id: { $exists: true }, is_merged_toc: { $ne: true } },
+      },
+      {
+        $group: {
+          _id: { project: '$project', branch: '$branch' },
+          most_recent: {
+            $max: {
+              build_id: '$build_id',
+              toctree: '$toctree',
+              toctreeOrder: '$toctreeOrder',
+            },
+          },
+        },
+      },
+    ]);
+    return tocs;
+  } catch (error) {
+    console.log(`Error at time of aggregating existing associated product metadata entries: ${error}`);
+    throw error;
+  }
+};
+
+export const mergeAssociatedToCs = async (metadata) => {
+  const { project, branch } = metadata;
+  const umbrellaMetadata = hasAssociations(metadata) ? metadata : await umbrellaMetadataEntry(metadata);
+
+  // Short circuit execution here if there's no umbrella product metadata found
+  if (!umbrellaMetadata) return;
+  // Short circuit execution if the project branch is NOT in repo branches
+  // If we want to embed with staging builds, then this needs to be turned off
+  // or converted so that local metadata ToC is added to tocInsertions
+  const isStagingBranch = await !getRepoBranchesEntry(project, branch);
+  if (isStagingBranch) return;
+
+  const repoBranchesEntries = await getAllAssociatedRepoBranchesEntries(umbrellaMetadata);
+  const repoBranchesMap = mapRepoBranches(repoBranchesEntries);
+  const tocsCursor = await getAssociatedProducts(umbrellaMetadata);
+  const { tocInsertions, tocOrderInsertions } = await shapeToCsCursor(tocsCursor, repoBranchesMap);
+  const mergedMetadataEntry = traverseAndMerge(umbrellaMetadata, tocInsertions, tocOrderInsertions);
+
+  // Remove the _id and treat the entry as a brand new document.
+  delete mergedMetadataEntry._id;
+  // Add a flag to denote that the entry contains a merged ToC.
+  mergedMetadataEntry.is_merged_toc = true;
+
+  return mergedMetadataEntry;
+};
diff --git a/modules/persistence/src/services/metadata/index.ts b/modules/persistence/src/services/metadata/index.ts
diff --git a/modules/persistence/tests/data/metadata.json b/modules/persistence/tests/data/metadata.json

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-# Snooty Transport Module`
	`1`	`+# Snooty Persistence Module`
`2`	`2`
`3`	`3`	`As part of integrating Snooty Parser with the unified Snooty toolchain, this module holds exclusive responsibility over transporting and persisting AST and metadata output artifacts from the parser into datastores.`
`4`	`4`