CLOUDP-294461: [Product Metrics/Observability] Implement Metric Collection workflow (#357)

Author: yelizhenden-mdb

.gitignore

@@ -21,3 +21,6 @@
 
 *.out
 **/*ipa-collector-results-combined.log
+**/*metric-collection-results.json
+**/*spectral-output.txt
+**/*spectral-report.xml

package.json

@@ -1,6 +1,7 @@
 {
     "name": "mongodb-openapi",
     "description": "MongoDB repository with OpenAPI specification",
+    "type": "module",
     "scripts": {
         "format": "npx prettier . --write",
         "format-check": "npx prettier . --check",

tools/spectral/ipa/metrics/config.js

@@ -0,0 +1,18 @@
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const dirname = path.dirname(fileURLToPath(import.meta.url));
+const rootDir = path.resolve(dirname, '../../../../');
+
+const config = {
+  defaultOasFilePath: path.join(rootDir, 'openapi', 'v2.json'),
+  defaultRulesetFilePath: path.join(dirname, '..', 'ipa-spectral.yaml'),
+  defaultCollectorResultsFilePath: path.join(dirname, 'ipa-collector-results-combined.log'),
+  defaultOutputsDir: path.join(dirname, 'outputs'),
+};
+
+config.defaultMetricCollectionResultsFilePath = path.join(config.defaultOutputsDir, 'metric-collection-results.json');
+config.defaultSpectralReportFile = path.join(config.defaultOutputsDir, 'spectral-report.xml');
+config.defaultSpectralOutputFile = path.join(config.defaultOutputsDir, 'spectral-output.txt');
+
+export default config;

tools/spectral/ipa/metrics/metricCollection.js

@@ -0,0 +1,80 @@
+import spectral from '@stoplight/spectral-core';
+import * as fs from 'node:fs';
+import { spawnSync } from 'child_process';
+import {
+  loadOpenAPIFile,
+  extractTeamOwnership,
+  loadRuleset,
+  loadCollectorResults,
+  getSeverityPerRule,
+  merge,
+} from './utils.js';
+import config from './config.js';
+const { Spectral } = spectral;
+
+async function runMetricCollectionJob(oasFilePath = config.defaultOasFilePath) {
+  try {
+    console.log(`Loading OpenAPI file: ${oasFilePath}`);
+    const oasContent = loadOpenAPIFile(oasFilePath);
+
+    console.log('Extracting team ownership data...');
+    const ownershipData = extractTeamOwnership(oasContent);
+
+    console.log('Getting rule severities...');
+    const spectral = new Spectral();
+    const ruleset = await loadRuleset(config.defaultRulesetFilePath, spectral);
+    const ruleSeverityMap = getSeverityPerRule(ruleset);
+
+    console.log('Loading collector results...');
+    const collectorResults = loadCollectorResults(config.defaultCollectorResultsFilePath);
+
+    console.log('Merging results...');
+    const mergedResults = merge(ownershipData, collectorResults, ruleSeverityMap);
+
+    console.log('Metric collection job complete.');
+    return mergedResults;
+  } catch (error) {
+    console.error('Error during metric collection:', error.message);
+    throw error;
+  }
+}
+
+const args = process.argv.slice(2);
+const customOasFile = args[0];
+
+if (!fs.existsSync(config.defaultOutputsDir)) {
+  fs.mkdirSync('outputs');
+  console.log(`Output directory created successfully`);
+}
+
+const result = spawnSync(
+  'spectral',
+  [
+    'lint',
+    '--ruleset',
+    config.defaultRulesetFilePath,
+    '--format',
+    'stylish',
+    '--verbose',
+    '--format',
+    'junit',
+    '--output.junit',
+    config.defaultSpectralReportFile,
+    config.defaultOasFilePath,
+  ],
+  {
+    encoding: 'utf-8',
+  }
+);
+
+if (result.error) {
+  console.error('Error running Spectral lint:', result.error);
+  process.exit(1);
+}
+
+console.log('Spectral lint completed successfully.');
+fs.writeFileSync(config.defaultSpectralOutputFile, result.stdout);
+
+runMetricCollectionJob(customOasFile)
+  .then((results) => fs.writeFileSync(config.defaultMetricCollectionResultsFilePath, JSON.stringify(results)))
+  .catch((error) => console.error(error.message));

tools/spectral/ipa/metrics/utils.js

@@ -0,0 +1,117 @@
+import fs from 'node:fs';
+import { bundleAndLoadRuleset } from '@stoplight/spectral-ruleset-bundler/with-loader';
+import { EntryType } from './collector.js';
+
+export function loadOpenAPIFile(filePath) {
+  try {
+    const content = fs.readFileSync(filePath, 'utf8');
+    return JSON.parse(content);
+  } catch (error) {
+    throw new Error(`Failed to load OpenAPI file: ${error.message}`);
+  }
+}
+
+export function getSeverityPerRule(ruleset) {
+  const rules = ruleset.rules || {};
+  const map = {};
+  for (const [name, ruleObject] of Object.entries(rules)) {
+    map[name] = ruleObject.definition.severity;
+  }
+  return map;
+}
+
+export async function loadRuleset(rulesetPath, spectral) {
+  try {
+    const ruleset = await bundleAndLoadRuleset(rulesetPath, { fs, fetch });
+    await spectral.setRuleset(ruleset);
+    return ruleset;
+  } catch (error) {
+    throw new Error(`Failed to load ruleset: ${error.message}`);
+  }
+}
+
+export function extractTeamOwnership(oasContent) {
+  const ownerTeams = {};
+  const paths = oasContent.paths || {};
+
+  for (const [path, pathItem] of Object.entries(paths)) {
+    for (const [, operation] of Object.entries(pathItem)) {
+      const ownerTeam = operation['x-xgen-owner-team'];
+
+      if (ownerTeam) {
+        if (!ownerTeams[path]) {
+          ownerTeams[path] = ownerTeam;
+        } else if (ownerTeams[path] !== ownerTeam) {
+          console.warn(`Conflict on path ${path}: ${ownerTeams[path]} vs ${ownerTeam}`);
+        }
+      }
+    }
+  }
+
+  return ownerTeams;
+}
+
+export function loadCollectorResults(collectorResultsFilePath) {
+  try {
+    const content = fs.readFileSync(collectorResultsFilePath, 'utf8');
+    const contentParsed = JSON.parse(content);
+    return {
+      [EntryType.VIOLATION]: contentParsed[EntryType.VIOLATION],
+      [EntryType.ADOPTION]: contentParsed[EntryType.ADOPTION],
+      [EntryType.EXCEPTION]: contentParsed[EntryType.EXCEPTION],
+    };
+  } catch (error) {
+    throw new Error(`Failed to load Collector Results file: ${error.message}`);
+  }
+}
+
+function getIPAFromIPARule(ipaRule) {
+  const pattern = /IPA-\d{3}/;
+  const match = ipaRule.match(pattern);
+  if (match) {
+    return match[0];
+  }
+}
+
+export function merge(ownershipData, collectorResults, ruleSeverityMap) {
+  const results = [];
+
+  function addEntry(entryType, adoptionStatus) {
+    for (const entry of collectorResults[entryType]) {
+      const existing = results.find(
+        (result) => result.component_id === entry.componentId && result.ipa_rule === entry.ruleName
+      );
+
+      if (existing) {
+        console.warn('Duplicate entries found', existing);
+        continue;
+      }
+
+      let ownerTeam = null;
+      if (entry.componentId.startsWith('paths')) {
+        const pathParts = entry.componentId.split('.');
+        if (pathParts.length === 2) {
+          const path = pathParts[1];
+          ownerTeam = ownershipData[path];
+        }
+      }
+
+      results.push({
+        component_id: entry.componentId,
+        ipa_rule: entry.ruleName,
+        ipa: getIPAFromIPARule(entry.ruleName),
+        severity_level: ruleSeverityMap[entry.ruleName],
+        adoption_status: adoptionStatus,
+        exception_reason: entryType === EntryType.EXCEPTION ? entry.exceptionReason : null,
+        owner_team: ownerTeam,
+        timestamp: new Date().toISOString(),
+      });
+    }
+  }
+
+  addEntry(EntryType.VIOLATION, 'violated');
+  addEntry(EntryType.ADOPTION, 'adopted');
+  addEntry(EntryType.EXCEPTION, 'exempted');
+
+  return results;
+}