Merge pull request #35419 from dims/install-pyyaml-for-experiment/audit-scripts

Install pyyaml for experiment/audit scripts

Author: k8s-ci-robot

config/jobs/kubernetes/sig-arch/conformance-audit.yaml

@@ -33,6 +33,7 @@ presubmits:
                   curl -sSL https://kind.sigs.k8s.io/dl/latest/linux-amd64.tgz | tar xvfz - -C "${PATH%%:*}/"
                   && curl -sO https://raw.githubusercontent.com/ii/kind/ci-audit-logging/hack/ci/e2e-k8s.sh
                   && bash e2e-k8s.sh
+                  && apt-get update && apt-get install -y --no-install-recommends python3-pip && pip3 install --no-cache-dir --break-system-packages pyyaml
                   && python3 ./../test-infra/experiment/audit/audit_log_parser.py --audit-logs ${ARTIFACTS}/audit/audit*.log --output "${ARTIFACTS}/audit/audit-endpoints.txt" --audit-operations-json "${ARTIFACTS}/audit/audit-operations.json" --swagger-url "file://$PWD/api/openapi-spec/swagger.json" --ineligible-endpoints-url "file://$PWD/test/conformance/testdata/ineligible_endpoints.yaml" --pending-eligible-endpoints-url "file://$PWD/test/conformance/testdata/pending_eligible_endpoints.yaml"
                   && python3 ./../test-infra/experiment/audit/kubernetes_api_analysis.py --pull-audit-endpoints "${ARTIFACTS}/audit/audit-endpoints.txt" --swagger-url "file://$PWD/api/openapi-spec/swagger.json"
             env:
@@ -91,6 +92,7 @@ periodics:
             curl -sSL https://kind.sigs.k8s.io/dl/latest/linux-amd64.tgz | tar xvfz - -C "${PATH%%:*}/"
             && curl -sO https://raw.githubusercontent.com/ii/kind/ci-audit-logging/hack/ci/e2e-k8s.sh
             && bash e2e-k8s.sh
+            && apt-get update && apt-get install -y --no-install-recommends python3-pip && pip3 install --no-cache-dir --break-system-packages pyyaml
             && python3 ./../test-infra/experiment/audit/audit_log_parser.py --audit-logs ${ARTIFACTS}/audit/audit*.log --output "${ARTIFACTS}/audit/audit-endpoints.txt" --audit-operations-json "${ARTIFACTS}/audit/audit-operations.json" --swagger-url "file://$PWD/api/openapi-spec/swagger.json"
       env:
       - name: BUILD_TYPE

experiment/audit/audit_log_parser.py

@@ -35,13 +35,7 @@
 import urllib.request
 from collections import Counter
 from pathlib import Path
-
-try:
-    import yaml
-
-    HAS_YAML = True
-except ImportError:
-    HAS_YAML = False
+import yaml
 
 
 def load_endpoint_list_from_yaml(url, endpoint_type="endpoints"):
@@ -60,21 +54,17 @@ def load_endpoint_list_from_yaml(url, endpoint_type="endpoints"):
         with urllib.request.urlopen(url, timeout=30) as response:
             content = response.read().decode()
 
-        # Parse YAML content - use PyYAML if available, otherwise manual parsing
-        if HAS_YAML:
-            try:
-                yaml_data = yaml.safe_load(content)
-                return _process_yaml_data(yaml_data, endpoint_type)
-            except yaml.YAMLError as e:
-                print(f"Error: Failed to parse YAML content from {url}")
-                print(f"YAML parsing error: {e}")
-                print(f"Content preview (first 500 chars):")
-                print(content[:500])
-                print(f"Cannot proceed with malformed YAML for {endpoint_type}")
-                sys.exit(1)
-        else:
-            print(f"Warning: PyYAML not available, using manual YAML parsing for {endpoint_type}")
-            return _manual_yaml_parse(content, endpoint_type)
+        # Parse YAML content using PyYAML
+        try:
+            yaml_data = yaml.safe_load(content)
+            return _process_yaml_data(yaml_data, endpoint_type)
+        except yaml.YAMLError as e:
+            print(f"Error: Failed to parse YAML content from {url}")
+            print(f"YAML parsing error: {e}")
+            print(f"Content preview (first 500 chars):")
+            print(content[:500])
+            print(f"Cannot proceed with malformed YAML for {endpoint_type}")
+            sys.exit(1)
 
     except urllib.error.URLError as e:
         print(f"Error: Failed to download {endpoint_type} from {url}")
@@ -121,50 +111,6 @@ def _process_yaml_data(yaml_data, endpoint_type):
     return endpoints
 
 
-def _manual_yaml_parse(content, endpoint_type):
-    """
-    Manual YAML parsing for simple structures when PyYAML is not available.
-
-    Handles two formats:
-    1. List of objects: - endpoint: name
-    2. Simple list: - name
-
-    Args:
-        content (str): YAML content as string
-        endpoint_type (str): Type of endpoints being loaded
-
-    Returns:
-        set: Set of endpoint operation IDs
-    """
-    endpoints = set()
-
-    for line in content.split('\n'):
-        line = line.strip()
-
-        # Skip empty lines, comments, and document separators
-        if not line or line.startswith('#') or line == '---':
-            continue
-
-        if line.startswith('- endpoint:'):
-            # Format: - endpoint: endpointName (ineligible_endpoints.yaml)
-            endpoint = line.replace('- endpoint:', '').strip()
-            if endpoint:
-                endpoints.add(endpoint)
-        elif line.startswith('- ') and ':' not in line:
-            # Format: - endpointName (pending_eligible_endpoints.yaml)
-            # Only if no colon (to avoid matching other object keys)
-            endpoint = line.replace('- ', '').strip()
-            if endpoint:
-                endpoints.add(endpoint)
-
-    if not endpoints:
-        print(f"Warning: No endpoints found using manual YAML parsing for {endpoint_type}")
-        print("This might indicate a parsing issue or unexpected YAML format")
-
-    print(f"Loaded {len(endpoints)} {endpoint_type} (manual parsing)")
-    return endpoints
-
-
 def load_ineligible_endpoints(ineligible_endpoints_url=None):
     """
     Load the list of ineligible endpoints from URL or local file.