ci(ipa): automatic warning-level violation detection (#905)

Author: yelizhenden-mdb

.github/scripts/handle_warning_violations.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+set -eou pipefail
+
+if [ "${WARNING_COUNT}" -eq 0 ]; then
+  echo "No warning violations found, skipping ticket creation"
+  exit 0
+fi
+
+# Read violation details if available
+VIOLATION_DETAILS=""
+if [ -f "tools/spectral/ipa/metrics/outputs/warning-violations.json" ]; then
+  VIOLATION_DETAILS=$(jq -r '
+    group_by(.code) | 
+    map("• " + .[0].code + " (" + (length | tostring) + " violations)") | 
+    join("\n")
+  ' tools/spectral/ipa/metrics/outputs/warning-violations.json)
+fi
+
+# Check if warning ticket already exists
+EXISTING_TICKET=$(curl -s -H "Authorization: Bearer ${JIRA_API_TOKEN}" \
+  "https://jira.mongodb.org/rest/api/2/search?jql=project=CLOUDP AND summary~'Warning-level IPA violations' AND status!=Done" \
+  | jq -r '.issues[0].key // empty')
+
+if [ -n "${EXISTING_TICKET}" ]; then
+  echo "Warning ticket already exists: ${EXISTING_TICKET}"
+  exit 0
+fi
+
+# Create detailed description
+DESCRIPTION="Warning-level violations were found during IPA validation. Please review and add exceptions if valid, or address false positives.
+These warning-level checks are part of the rule rollout process. See the IPA Validation Technical Documentation for details:
+https://wiki.corp.mongodb.com/spaces/MMS/pages/315003555/IPA+Validation+Technical+Documentation+Runbook#IPAValidationTechnicalDocumentation%26Runbook-RolloutofNewRule
+
+Violation Summary:
+${VIOLATION_DETAILS}
+
+Total violations: ${WARNING_COUNT}"
+
+# Create new Jira ticket
+TICKET_RESPONSE=$(curl -s -X POST -H "Authorization: Bearer ${JIRA_API_TOKEN}" \
+  -H "Content-Type: application/json" \
+  -d "{
+    \"fields\": {
+      \"project\": {\"key\": \"CLOUDP\"},
+      \"summary\": \"Warning-level IPA violations found\",
+      \"description\": \"${DESCRIPTION}\",
+      \"issuetype\": {\"name\": \"Task\"},
+      \"assignee\": {\"id\": \"${TEAM_ID}\"}
+    }
+  }" \
+  "https://jira.mongodb.org/rest/api/2/issue/")
+
+TICKET_KEY=$(echo "${TICKET_RESPONSE}" | jq -r '.key')
+
+if [ "${TICKET_KEY}" != "null" ]; then
+  echo "Created Jira ticket: ${TICKET_KEY}"
+  
+  # Create summary for Slack
+  SLACK_SUMMARY=""
+  if [ -n "${VIOLATION_DETAILS}" ]; then
+    SLACK_SUMMARY=$(echo "${VIOLATION_DETAILS}" | head -3)
+    if [ "$(echo "${VIOLATION_DETAILS}" | wc -l)" -gt 3 ]; then
+      SLACK_SUMMARY="${SLACK_SUMMARY}\n... and more"
+    fi
+  fi
+  
+  # Send Slack notification with violation summary
+  SLACK_MESSAGE="Warning-level IPA violations found (${WARNING_COUNT} violations) (${SLACK_ONCALL_USER}).
+
+Jira ticket: https://jira.mongodb.org/browse/${TICKET_KEY}"
+  
+  curl -X POST -H "Authorization: Bearer ${SLACK_BEARER_TOKEN}" \
+    -H "Content-type: application/json" \
+    --data "{\"channel\":\"${SLACK_CHANNEL_ID}\",\"text\":\"${SLACK_MESSAGE}\"}" \
+    https://slack.com/api/chat.postMessage
+else
+  echo "Failed to create Jira ticket"
+  exit 1
+fi

.github/workflows/release-IPA-metrics.yml

@@ -23,6 +23,7 @@ jobs:
             tools/spectral/ipa
             package.json
             package-lock.json
+            .github/scripts
 
       - name: Setup Node
         uses: actions/setup-node@v4
@@ -38,8 +39,10 @@ jobs:
         working-directory: ${{ github.workspace }}
 
       - name: Run Metric Collection Job
+        id: metric-collection
         working-directory: tools/spectral/ipa/metrics/scripts
-        run: node runMetricCollection.js "${{ github.workspace }}/v2.json"
+        run: |
+          node runMetricCollection.js "${{ github.workspace }}/v2.json"
 
       - name: aws configure
         uses: aws-actions/configure-aws-credentials@v4
@@ -54,6 +57,18 @@ jobs:
         working-directory: tools/spectral/ipa/metrics/scripts
         run: node dataDump.js
 
+# Enable this step in scope of CLOUDP-339852
+#      - name: Handle Warning Violations
+#        if: ${{ steps.metric-collection.outputs.warning_count > 0 }}
+#        env:
+#          WARNING_COUNT: ${{ steps.metric-collection.outputs.warning_count }}
+#          TEAM_ID: ${{ vars.JIRA_TEAM_ID_APIX_PLATFORM }}
+#          JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}
+#          SLACK_BEARER_TOKEN: ${{ secrets.SLACK_BEARER_TOKEN }}
+#          SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID_APIX_PLATFORM_DEV }}
+#          SLACK_ONCALL_USER: ${{ secrets.SLACK_APIX_PLATFORM_ONCALL_USER }}
+#        run: .github/scripts/handle_warning_violations.sh
+
   failure-handler:
     name: Failure Handler
     needs: [ release-IPA-metrics ]

tools/spectral/ipa/__tests__/metrics/data/collector-results.log

@@ -1,5 +1,9 @@
 {
-  "violations": [],
+  "violations": [
+  {
+    "componentId": "paths./api/atlas/v2/federationSettings/{federationSettingsId}/connectedOrgConfigs/{orgId}.get",
+    "ruleName": "xgen-IPA-104-valid-operation-id"
+  }],
   "adoptions": [
     {
       "componentId": "paths./api/atlas/v2",

tools/spectral/ipa/__tests__/metrics/data/expected-metric-results.json

@@ -29,9 +29,8 @@ describe('tools/spectral/ipa/metrics/metricCollection.js runMetricCollectionJob'
     const results = await runMetricCollectionJob(testConfig, spectral);
 
     expect(results).not.toBe(undefined);
-    expect(results.length).toEqual(expectedResults.length);
-
-    results.forEach((entry, index) => {
+    expect(results.metrics.length).toEqual(expectedResults.length);
+    results.metrics.forEach((entry, index) => {
       const expectedEntry = getEntry(expectedResults, entry['component_id'], entry['ipa_rule']);
       expect(entry['component_id']).toEqual(expectedEntry['component_id']);
       expect(entry['adoption_status']).toEqual(expectedEntry['adoption_status']);
@@ -41,6 +40,14 @@ describe('tools/spectral/ipa/metrics/metricCollection.js runMetricCollectionJob'
       expect(entry['owner_team']).toEqual(expectedEntry['owner_team']);
       expect(entry['severity_level']).toEqual(expectedEntry['severity_level']);
     });
+
+    expect(results.warnings.count).toEqual(1);
+    const violations = [
+      {
+        code: 'xgen-IPA-104-valid-operation-id',
+      },
+    ];
+    expect(results.warnings.violations).toEqual(violations);
   });
 });
 

tools/spectral/ipa/__tests__/metrics/metricCollection.test.js

@@ -32,8 +32,24 @@ export async function runMetricCollectionJob(
     console.log('Merging results...');
     const mergedResults = merge(ownershipData, collectorResults, ruleSeverityMap);
 
+    const warningViolations = mergedResults.filter(
+      (result) => result.severity_level === 'warn' && result.adoption_status === 'violated'
+    );
+
+    const processedWarnings = warningViolations.map((violation) => ({
+      code: violation.ipa_rule,
+    }));
+
+    console.log(`Found ${warningViolations.length} warning-level violations`);
+
     console.log('Metric collection job complete.');
-    return mergedResults;
+    return {
+      metrics: mergedResults,
+      warnings: {
+        count: warningViolations.length,
+        violations: processedWarnings,
+      },
+    };
   } catch (error) {
     console.error('Error during metric collection:', error.message);
     throw error;

tools/spectral/ipa/metrics/metricCollection.js

@@ -1,4 +1,5 @@
 import fs from 'node:fs';
+import path from 'path';
 import { spawnSync } from 'child_process';
 import spectral from '@stoplight/spectral-core';
 import { Compression, Table, writeParquet, WriterPropertiesBuilder } from 'parquet-wasm';
@@ -53,12 +54,18 @@ runMetricCollectionJob(
 )
   .then((results) => {
     console.log('Writing results');
-    const table = tableFromJSON(results);
+    const table = tableFromJSON(results.metrics);
     const wasmTable = Table.fromIPCStream(tableToIPC(table, 'stream'));
     const parquetUint8Array = writeParquet(
       wasmTable,
       new WriterPropertiesBuilder().setCompression(Compression.GZIP).build()
     );
     fs.writeFileSync(config.defaultMetricCollectionResultsFilePath, parquetUint8Array);
+    fs.writeFileSync(path.join(config.defaultOutputsDir, 'warning-count.txt'), results.warnings.count.toString());
+
+    fs.writeFileSync(
+      path.join(config.defaultOutputsDir, 'warning-violations.json'),
+      JSON.stringify(results.warnings.violations, null, 2)
+    );
   })
   .catch((error) => console.error(error.message));