ci(ipa): automatic warning-level violation detection

Author: yelizhenden-mdb

.github/scripts/handle_warning_violations.sh

@@ -0,0 +1,113 @@
+#!/bin/bash
+set -eou pipefail
+
+WARNING_COUNT=$1
+TEAM_ID=$2
+JIRA_API_TOKEN=$3
+SLACK_BEARER_TOKEN=$4
+SLACK_CHANNEL_ID=$5
+DRY_RUN=${6:-false}  # Optional 6th parameter for dry run
+
+if [ "$WARNING_COUNT" -eq 0 ]; then
+  echo "No warning violations found, skipping ticket creation"
+  exit 0
+fi
+
+# Read violation details if available
+VIOLATION_DETAILS=""
+if [ -f "tools/spectral/ipa/metrics/outputs/warning-violations.json" ]; then
+  VIOLATION_DETAILS=$(jq -r '
+    group_by(.code) | 
+    map("• " + .[0].code + " (" + (length | tostring) + " violations)") | 
+    join("\n")
+  ' tools/spectral/ipa/metrics/outputs/warning-violations.json)
+fi
+
+if [ "$DRY_RUN" = "true" ]; then
+  echo "=== DRY RUN MODE ==="
+  echo "Would create Jira ticket with:"
+  echo "Summary: Warning-level IPA violations found - $WARNING_COUNT violations"
+  echo "Description:"
+  echo "Warning-level violations were found during IPA validation.
+
+Violation Summary:
+$VIOLATION_DETAILS
+
+Total violations: $WARNING_COUNT"
+  echo ""
+  echo "Would send Slack message:"
+  SLACK_SUMMARY=$(echo "$VIOLATION_DETAILS" | head -3)
+  if [ $(echo "$VIOLATION_DETAILS" | wc -l) -gt 3 ]; then
+    SLACK_SUMMARY="$SLACK_SUMMARY\n... and more"
+  fi
+  echo "Warning-level IPA violations found ($WARNING_COUNT violations). 
+
+Top violations:
+$SLACK_SUMMARY
+
+Jira ticket: [DRY RUN - no ticket created]"
+  exit 0
+fi
+
+# Check if warning ticket already exists
+EXISTING_TICKET=$(curl -s -H "Authorization: Bearer $JIRA_API_TOKEN" \
+  "https://jira.mongodb.org/rest/api/2/search?jql=project=CLOUDP AND summary~'Warning-level IPA violations' AND status!=Done" \
+  | jq -r '.issues[0].key // empty')
+
+if [ -n "$EXISTING_TICKET" ]; then
+  echo "Warning ticket already exists: $EXISTING_TICKET"
+  exit 0
+fi
+
+# Create detailed description
+DESCRIPTION="Warning-level violations were found during IPA validation. Please review and add exceptions if valid, or address false positives.
+
+Violation Summary:
+$VIOLATION_DETAILS
+
+Total violations: $WARNING_COUNT"
+
+# Create new Jira ticket
+TICKET_RESPONSE=$(curl -s -X POST -H "Authorization: Bearer $JIRA_API_TOKEN" \
+  -H "Content-Type: application/json" \
+  -d "{
+    \"fields\": {
+      \"project\": {\"key\": \"CLOUDP\"},
+      \"summary\": \"Warning-level IPA violations found - $WARNING_COUNT violations\",
+      \"description\": \"$DESCRIPTION\",
+      \"issuetype\": {\"name\": \"Task\"},
+      \"assignee\": {\"id\": \"$TEAM_ID\"}
+    }
+  }" \
+  "https://jira.mongodb.org/rest/api/2/issue/")
+
+TICKET_KEY=$(echo "$TICKET_RESPONSE" | jq -r '.key')
+
+if [ "$TICKET_KEY" != "null" ]; then
+  echo "Created Jira ticket: $TICKET_KEY"
+  
+  # Create summary for Slack
+  SLACK_SUMMARY=""
+  if [ -n "$VIOLATION_DETAILS" ]; then
+    SLACK_SUMMARY=$(echo "$VIOLATION_DETAILS" | head -3)
+    if [ $(echo "$VIOLATION_DETAILS" | wc -l) -gt 3 ]; then
+      SLACK_SUMMARY="$SLACK_SUMMARY\n... and more"
+    fi
+  fi
+  
+  # Send Slack notification with violation summary
+  SLACK_MESSAGE="Warning-level IPA violations found ($WARNING_COUNT violations). 
+
+Top violations:
+$SLACK_SUMMARY
+
+Jira ticket: https://jira.mongodb.org/browse/$TICKET_KEY"
+  
+  curl -X POST -H "Authorization: Bearer $SLACK_BEARER_TOKEN" \
+    -H "Content-type: application/json" \
+    --data "{\"channel\":\"$SLACK_CHANNEL_ID\",\"text\":\"$SLACK_MESSAGE\"}" \
+    https://slack.com/api/chat.postMessage
+else
+  echo "Failed to create Jira ticket"
+  exit 1
+fi

.github/workflows/release-IPA-metrics.yml

@@ -38,8 +38,16 @@ jobs:
         working-directory: ${{ github.workspace }}
 
       - name: Run Metric Collection Job
+        id: metric-collection
         working-directory: tools/spectral/ipa/metrics/scripts
-        run: node runMetricCollection.js "${{ github.workspace }}/v2.json"
+        run: |
+          node runMetricCollection.js "${{ github.workspace }}/v2.json"
+          if [ -f "../outputs/warning-count.txt" ]; then
+            warning_count=$(cat ../outputs/warning-count.txt)
+            echo "warning_count=${warning_count}" >> $GITHUB_OUTPUT
+          else
+            echo "warning_count=0" >> $GITHUB_OUTPUT
+          fi
 
       - name: aws configure
         uses: aws-actions/configure-aws-credentials@v4
@@ -54,6 +62,18 @@ jobs:
         working-directory: tools/spectral/ipa/metrics/scripts
         run: node dataDump.js
 
+      - name: Handle Warning Violations
+        if: ${{ steps.metric-collection.outputs.warning_count > 0 }}
+        env:
+          WARNING_COUNT: ${{ steps.metric-collection.outputs.warning_count }}
+          TEAM_ID: ${{ vars.JIRA_TEAM_ID_APIX_1 }}
+          JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}
+          SLACK_BEARER_TOKEN: ${{ secrets.SLACK_BEARER_TOKEN }}
+          SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID_APIX_1_DEV }}
+        run: |
+          chmod +x .github/scripts/handle_warning_violations.sh
+          .github/scripts/handle_warning_violations.sh "$WARNING_COUNT" "$TEAM_ID" "$JIRA_API_TOKEN" "$SLACK_BEARER_TOKEN" "$SLACK_CHANNEL_ID"
+
   failure-handler:
     name: Failure Handler
     needs: [ release-IPA-metrics ]

tools/spectral/ipa/__tests__/metrics/data/collector-results.log

@@ -1,5 +1,9 @@
 {
-  "violations": [],
+  "violations": [
+  {
+    "componentId": "paths./api/atlas/v2/federationSettings/{federationSettingsId}/connectedOrgConfigs/{orgId}.get",
+    "ruleName": "xgen-IPA-104-valid-operation-id"
+  }],
   "adoptions": [
     {
       "componentId": "paths./api/atlas/v2",

tools/spectral/ipa/__tests__/metrics/data/expected-metric-results.json

@@ -24,14 +24,15 @@ describe('tools/spectral/ipa/metrics/metricCollection.js runMetricCollectionJob'
 
   it('Outputs the expected metrics collection results', async () => {
     const expectedResults = JSON.parse(fs.readFileSync(expectedResultFilePath, 'utf8'));
+    console.log(expectedResults[expectedResults.length-1]);
     const spectral = new Spectral();
 
     const results = await runMetricCollectionJob(testConfig, spectral);
 
     expect(results).not.toBe(undefined);
-    expect(results.length).toEqual(expectedResults.length);
+    expect(results.metrics.length).toEqual(expectedResults.length);
 
-    results.forEach((entry, index) => {
+    results.metrics.forEach((entry, index) => {
       const expectedEntry = getEntry(expectedResults, entry['component_id'], entry['ipa_rule']);
       expect(entry['component_id']).toEqual(expectedEntry['component_id']);
       expect(entry['adoption_status']).toEqual(expectedEntry['adoption_status']);

tools/spectral/ipa/__tests__/metrics/metricCollection.test.js

@@ -22,7 +22,7 @@ export async function runMetricCollectionJob(
     console.log('Extracting team ownership data...');
     const ownershipData = extractTeamOwnership(oasContent);
 
-    console.log('Getting rule severities...');
+    console.log(`Getting rule severities... ${rulesetFilePath}`);
     const ruleset = await loadRuleset(rulesetFilePath, spectral);
     const ruleSeverityMap = getSeverityPerRule(ruleset);
 
@@ -32,8 +32,28 @@ export async function runMetricCollectionJob(
     console.log('Merging results...');
     const mergedResults = merge(ownershipData, collectorResults, ruleSeverityMap);
 
+    const warningViolations = mergedResults.filter(result =>
+      result.severity_level === 1 && result.adoption_status === 'violated'
+    );
+
+    const processedWarnings = warningViolations.map(violation => ({
+      code: violation.ipa_rule,
+      message: `IPA rule ${violation.ipa_rule} violated`,
+      path: violation.component_id,
+      source: null
+    }));
+
+    console.log(`Found ${warningViolations.length} warning-level violations`);
+
+
     console.log('Metric collection job complete.');
-    return mergedResults;
+    return {
+      metrics: mergedResults,
+      warnings: {
+        count: warningViolations.length,
+        violations: processedWarnings
+      }
+    };
   } catch (error) {
     console.error('Error during metric collection:', error.message);
     throw error;

tools/spectral/ipa/metrics/metricCollection.js

@@ -1,7 +1,13 @@
 import fs from 'node:fs';
+import path from 'path';
 import { spawnSync } from 'child_process';
 import spectral from '@stoplight/spectral-core';
-import { Compression, Table, writeParquet, WriterPropertiesBuilder } from 'parquet-wasm';
+import {
+  Compression,
+  Table,
+  writeParquet,
+  WriterPropertiesBuilder,
+} from 'parquet-wasm';
 import { tableFromJSON, tableToIPC } from 'apache-arrow';
 import config from '../config.js';
 import { runMetricCollectionJob } from '../metricCollection.js';
@@ -53,12 +59,21 @@ runMetricCollectionJob(
 )
   .then((results) => {
     console.log('Writing results');
-    const table = tableFromJSON(results);
+    const table = tableFromJSON(results.metrics);
     const wasmTable = Table.fromIPCStream(tableToIPC(table, 'stream'));
     const parquetUint8Array = writeParquet(
       wasmTable,
       new WriterPropertiesBuilder().setCompression(Compression.GZIP).build()
     );
     fs.writeFileSync(config.defaultMetricCollectionResultsFilePath, parquetUint8Array);
+    fs.writeFileSync(
+      path.join(config.defaultOutputsDir, 'warning-count.txt'),
+      results.warnings.count.toString()
+    );
+
+    fs.writeFileSync(
+      path.join(config.defaultOutputsDir, 'warning-violations.json'),
+      JSON.stringify(results.warnings.violations, null, 2)
+    );
   })
   .catch((error) => console.error(error.message));