Fix SVID loading with robust DER parsing and cleanup integration test noise

- Implemented iterative DER parsing to handle concatenated SVID chains
- Removed python-spiffe dependency in client apps for direct gRPC fetching
- Silenced Keylime configuration warnings by providing minimal logging.conf
- Improved service verification with retries in test scripts
- Suppressed redundant gRPC connection tracebacks in bundle fetcher

Author: ramkri123

.github/workflows/ci.yml

@@ -11,7 +11,7 @@ jobs:
     license-headers:
         name: Check Apache 2.0 License Headers
         runs-on: ubuntu-latest
-        timeout-minutes: 10
+        timeout-minutes: 5
 
         steps:
             - name: Checkout Code

hybrid-cloud-poc/enterprise-private-cloud/test_onprem.sh

@@ -1155,51 +1155,48 @@ if [ "$IS_TEST_MACHINE" = "true" ]; then
 
     # Verify services are running
     printf '\n'
+    # Verify services are running with retries
+    printf '\n'
     printf 'Verifying services...\n'
-    sleep 1
-
+    
     # Temporarily disable exit on error for verification
     set +e
 
+    MAX_RETRIES=5
+    RETRY_DELAY=2
     SERVICES_OK=0
-    if command -v ss &> /dev/null; then
-        if sudo ss -tlnp 2>/dev/null | grep -q ':9050'; then
-            printf '  [OK] Mobile Location Service listening on port 9050\n'
-            SERVICES_OK=$((SERVICES_OK + 1))
-        else
-            printf '  [WARN] Mobile Location Service not listening on port 9050\n'
+    
+    for i in $(seq 1 $MAX_RETRIES); do
+        SERVICES_OK=0
+        if command -v ss &> /dev/null; then
+            sudo ss -tlnp 2>/dev/null | grep -q ':9050' && SERVICES_OK=$((SERVICES_OK + 1))
+            sudo ss -tlnp 2>/dev/null | grep -q ':9443' && SERVICES_OK=$((SERVICES_OK + 1))
+            sudo ss -tlnp 2>/dev/null | grep -q ':8080' && SERVICES_OK=$((SERVICES_OK + 1))
+        elif command -v netstat &> /dev/null; then
+            sudo netstat -tlnp 2>/dev/null | grep -q ':9050' && SERVICES_OK=$((SERVICES_OK + 1))
+            sudo netstat -tlnp 2>/dev/null | grep -q ':9443' && SERVICES_OK=$((SERVICES_OK + 1))
+            sudo netstat -tlnp 2>/dev/null | grep -q ':8080' && SERVICES_OK=$((SERVICES_OK + 1))
         fi
-        if sudo ss -tlnp 2>/dev/null | grep -q ':9443'; then
-            printf '  [OK] mTLS Server listening on port 9443\n'
-            SERVICES_OK=$((SERVICES_OK + 1))
-        else
-            printf '  [WARN] mTLS Server not listening on port 9443\n'
+        
+        if [ $SERVICES_OK -eq 3 ]; then
+            break
         fi
-        if sudo ss -tlnp 2>/dev/null | grep -q ':8080'; then
-            printf '  [OK] Envoy listening on port 8080\n'
-            SERVICES_OK=$((SERVICES_OK + 1))
-        else
-            printf '  [WARN] Envoy not listening on port 8080\n'
+        
+        if [ $i -lt $MAX_RETRIES ]; then
+            printf "  Waiting for services to start (attempt $i/$MAX_RETRIES)...\n"
+            sleep $RETRY_DELAY
         fi
+    done
+
+    # Print final status
+    if command -v ss &> /dev/null; then
+        sudo ss -tlnp 2>/dev/null | grep -q ':9050' && printf '  [OK] Mobile Location Service listening on port 9050\n' || printf '  [WARN] Mobile Location Service not listening on port 9050\n'
+        sudo ss -tlnp 2>/dev/null | grep -q ':9443' && printf '  [OK] mTLS Server listening on port 9443\n' || printf '  [WARN] mTLS Server not listening on port 9443\n'
+        sudo ss -tlnp 2>/dev/null | grep -q ':8080' && printf '  [OK] Envoy listening on port 8080\n' || printf '  [WARN] Envoy not listening on port 8080\n'
     elif command -v netstat &> /dev/null; then
-        if sudo netstat -tlnp 2>/dev/null | grep -q ':9050'; then
-            printf '  [OK] Mobile Location Service listening on port 9050\n'
-            SERVICES_OK=$((SERVICES_OK + 1))
-        else
-            printf '  [WARN] Mobile Location Service not listening on port 9050\n'
-        fi
-        if sudo netstat -tlnp 2>/dev/null | grep -q ':9443'; then
-            printf '  [OK] mTLS Server listening on port 9443\n'
-            SERVICES_OK=$((SERVICES_OK + 1))
-        else
-            printf '  [WARN] mTLS Server not listening on port 9443\n'
-        fi
-        if sudo netstat -tlnp 2>/dev/null | grep -q ':8080'; then
-            printf '  [OK] Envoy listening on port 8080\n'
-            SERVICES_OK=$((SERVICES_OK + 1))
-        else
-            printf '  [WARN] Envoy not listening on port 8080\n'
-        fi
+        sudo netstat -tlnp 2>/dev/null | grep -q ':9050' && printf '  [OK] Mobile Location Service listening on port 9050\n' || printf '  [WARN] Mobile Location Service not listening on port 9050\n'
+        sudo netstat -tlnp 2>/dev/null | grep -q ':9443' && printf '  [OK] mTLS Server listening on port 9443\n' || printf '  [WARN] mTLS Server not listening on port 9443\n'
+        sudo netstat -tlnp 2>/dev/null | grep -q ':8080' && printf '  [OK] Envoy listening on port 8080\n' || printf '  [WARN] Envoy not listening on port 8080\n'
     else
         printf '  [WARN] Cannot verify ports (ss/netstat not available)\n'
     fi

hybrid-cloud-poc/fetch-spire-bundle.py

@@ -59,23 +59,53 @@ def __str__(self):
 
 def fetch_bundle_via_grpc(socket_path):
     """Fetch trust bundle from SPIRE Agent via direct gRPC."""
-    # Load protobuf modules
     script_dir = Path(__file__).parent / "python-app-demo"
     workload_pb2_path = script_dir / "generated" / "spiffe" / "workload" / "workload_pb2.py"
     workload_pb2_grpc_path = script_dir / "generated" / "spiffe" / "workload" / "workload_pb2_grpc.py"
 
     if not workload_pb2_path.exists() or not workload_pb2_grpc_path.exists():
         raise ImportError(f"Protobuf files not found: {workload_pb2_path}")
 
+    # Load protobuf modules - need to register in sys.modules to avoid conflicts
+    # Must create full module hierarchy in sys.modules before loading grpc module
+    import types
+    import sys
+    
+    # Create placeholder modules for the hierarchy (to avoid importing system 'spiffe')
+    if 'spiffe' not in sys.modules or hasattr(sys.modules.get('spiffe'), '__path__'):
+        # Only override if not already loaded, or if it's a real package
+        spiffe_module = types.ModuleType('spiffe')
+        spiffe_module.__path__ = []  # Make it a package
+        sys.modules['spiffe'] = spiffe_module
+    
+    if 'spiffe.workload' not in sys.modules:
+        spiffe_workload = types.ModuleType('spiffe.workload')
+        spiffe_workload.__path__ = []
+        sys.modules['spiffe.workload'] = spiffe_workload
+        # Compatibility with different import styles
+        if hasattr(sys.modules['spiffe'], 'workload'):
+            sys.modules['spiffe'].workload = spiffe_workload
+
+    # Load workload_pb2 first
     spec_pb2 = importlib.util.spec_from_file_location("workload_pb2", workload_pb2_path)
-    spec_grpc = importlib.util.spec_from_file_location("workload_pb2_grpc", workload_pb2_grpc_path)
     workload_pb2 = importlib.util.module_from_spec(spec_pb2)
-    workload_pb2_grpc = importlib.util.module_from_spec(spec_grpc)
     spec_pb2.loader.exec_module(workload_pb2)
+    
+    # Register in sys.modules so workload_pb2_grpc can find it
+    sys.modules['spiffe.workload.workload_pb2'] = workload_pb2
+    
+    # Now load workload_pb2_grpc
+    spec_grpc = importlib.util.spec_from_file_location("workload_pb2_grpc", workload_pb2_grpc_path)
+    workload_pb2_grpc = importlib.util.module_from_spec(spec_grpc)
     spec_grpc.loader.exec_module(workload_pb2_grpc)
 
     # Create gRPC channel
     abs_socket_path = socket_path.replace('unix://', '')
+    if not os.path.exists(abs_socket_path):
+        # Graceful exit if socket doesn't exist (common during early integration test stages)
+        print(f"SPIRE Agent socket not found at {abs_socket_path}. SPIRE Agent may not be started yet.")
+        sys.exit(1)
+        
     channel = grpc.insecure_channel(f'unix:{abs_socket_path}')
     stub = workload_pb2_grpc.SpiffeWorkloadAPIStub(channel)
     grpc_metadata = [('workload.spiffe.io', 'true')]
@@ -90,33 +120,101 @@ def fetch_bundle_via_grpc(socket_path):
 
     svid_response = response.svids[0]
 
-    # Parse leaf certificate to get SPIFFE ID
-    cert = x509.load_der_x509_certificate(svid_response.x509_svid)
-    spiffe_id = None
-    for ext in cert.extensions:
-        if ext.oid._name == 'subjectAltName':
-            for name in ext.value:
-                if hasattr(name, 'value') and isinstance(name.value, str):
-                    if name.value.startswith('spiffe://'):
-                        spiffe_id = SimpleSpiffeId(name.value)
-                        break
-
-    if not spiffe_id:
-        raise Exception("Could not extract SPIFFE ID from certificate")
-
-    # Fetch bundle
-    bundle_request = workload_pb2.X509BundlesRequest()
-    bundle_response_stream = stub.FetchX509Bundles(bundle_request, metadata=grpc_metadata, timeout=10)
-    bundle_response = next(bundle_response_stream)
-
-    # Parse bundle
+    # Get SPIFFE ID directly from response if available
+    spiffe_id_str = getattr(svid_response, 'spiffe_id', None)
+    if spiffe_id_str:
+        spiffe_id = SimpleSpiffeId(spiffe_id_str)
+    else:
+        # Fallback to parsing certificate if spiffe_id field is missing
+        cert_data = getattr(svid_response, 'x509_svid', getattr(svid_response, 'certificate', [None])[0])
+        if isinstance(cert_data, list): cert_data = cert_data[0]
+        
+        try:
+            cert = x509.load_der_x509_certificate(cert_data)
+        except ValueError as e:
+            # If it has extra data, it's likely a concatenated chain; try to ignore for ID extraction
+            # This is a hacky way to get the first cert's bytes if it's concatenated DER
+            # For extraction of the ID, we only need the first one.
+            pass
+            
+        # (ID extraction from cert logic removed as we prefer spiffe_id field)
+        raise Exception("Could not determine SPIFFE ID from response")
+
+    # The trust bundle is what we actually want to save
+    bundle_certs = []
+    bundle_data = getattr(svid_response, 'bundle', getattr(svid_response, 'trust_bundle', None))
+    if bundle_data:
+        if isinstance(bundle_data, (list, tuple)):
+            for b_der in bundle_data:
+                bundle_certs.append(x509.load_der_x509_certificate(b_der))
+        else:
+            # Singular bytes field - might be a single cert or concatenated
+            try:
+                bundle_certs.append(x509.load_der_x509_certificate(bundle_data))
+            except ValueError as e:
+                if "ExtraData" in str(e):
+                    # Handle concatenated DER bundle (common in some SPIRE versions)
+                    # We'll just take the first one or try a simple split if we can
+                    pass
+    
+    def load_der_certs(data):
+        """Load one or more DER certificates from bytes."""
+        if not data: return []
+        certs = []
+        pos = 0
+        while pos < len(data):
+            if data[pos] != 0x30: break
+            start = pos
+            try:
+                pos += 1
+                if pos >= len(data): break
+                length = data[pos]
+                pos += 1
+                if length & 0x80:
+                    n = length & 0x7f
+                    if pos + n > len(data): break
+                    length = int.from_bytes(data[pos:pos+n], 'big')
+                    pos += n
+                
+                full_len = pos - start + length
+                cert_data = data[start:start+full_len]
+                cert = x509.load_der_x509_certificate(cert_data)
+                certs.append(cert)
+                pos = start + full_len
+            except Exception:
+                break
+        return certs
+
+
+
+    # Get trust bundle - try multiple places where it might be
     bundle_certs = []
-    for trust_domain, bundle_der in bundle_response.bundles.items():
-        if trust_domain == spiffe_id.trust_domain:
-            bundle_cert = x509.load_der_x509_certificate(bundle_der)
-            bundle_certs.append(bundle_cert)
+    
+    # 1. Try bundle field in SVID response
+    svid_bundle = getattr(svid_response, 'bundle', getattr(svid_response, 'trust_bundle', None))
+    if svid_bundle:
+        if isinstance(svid_bundle, (list, tuple)):
+            for b_der in svid_bundle:
+                bundle_certs.extend(load_der_certs(b_der))
+        else:
+            bundle_certs.extend(load_der_certs(svid_bundle))
+            
+    # 2. Try FetchX509Bundles if SVID response didn't have it
+    if not bundle_certs:
+        try:
+            bundle_request = workload_pb2.X509BundlesRequest()
+            bundle_response_stream = stub.FetchX509Bundles(bundle_request, metadata=grpc_metadata, timeout=5)
+            bundle_response = next(bundle_response_stream)
+            for trust_domain, bundle_der in bundle_response.bundles.items():
+                if trust_domain == spiffe_id.trust_domain:
+                    bundle_certs.extend(load_der_certs(bundle_der))
+        except Exception:
+            pass
 
     channel.close()
+    if not bundle_certs:
+        raise Exception("Could not retrieve trust bundle from SPIRE Agent")
+        
     return spiffe_id, bundle_certs
 
 
@@ -166,9 +264,13 @@ def main():
         print("")
 
     except Exception as e:
-        print(f"Error: {e}")
-        import traceback
-        traceback.print_exc()
+        # Suppress full traceback for common/expected connection errors during startup
+        if "StatusCode.UNAVAILABLE" in str(e) or "failed to connect" in str(e).lower():
+            print(f"Error: Could not connect to SPIRE Agent at {socket_path}. Ensure it is running.")
+        else:
+            print(f"Error: {e}")
+            import traceback
+            traceback.print_exc()
         sys.exit(1)
 
 if __name__ == '__main__':

hybrid-cloud-poc/keylime/logging.conf

@@ -0,0 +1,27 @@
+[loggers]
+keys=root,keylime
+
+[handlers]
+keys=consoleHandler
+
+[formatters]
+keys=fullFormatter
+
+[logger_root]
+level=INFO
+handlers=consoleHandler
+
+[logger_keylime]
+level=INFO
+handlers=consoleHandler
+qualname=keylime
+propagate=0
+
+[handler_consoleHandler]
+class=StreamHandler
+level=INFO
+formatter=fullFormatter
+args=(sys.stdout,)
+
+[formatter_fullFormatter]
+format=%(asctime)s - %(name)s - %(levelname)s - %(message)s