fix: resolve KEDA APIService conflict for external metrics

Three fixes for the nightly E2E test failures:

1. DEFAULT_MODEL_ID: Update from Qwen/Qwen3-32B to Qwen/Qwen3-0.6B to
   match the llm-d repo's current default model. The stale value caused
   yq replacement to silently fail, making WVA query wrong model metrics.

2. WVA_METRICS_SECURE: Add env var to control bearer token auth on the
   WVA /metrics endpoint. OpenShift's user-workload-monitoring cannot
   authenticate with the controller-manager SA token.

3. KEDA APIService conflict: On clusters with KEDA, the
   v1beta1.external.metrics.k8s.io APIService points to KEDA's metrics
   server, which only serves ScaledObject metrics. After deploying
   Prometheus Adapter, detect and patch the APIService to point to
   Prometheus Adapter instead.

Supersedes #720.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
Signed-off-by: Andrew Anderson <andy@clubanderson.com>

Author: clubanderson

deploy/install.sh

@@ -68,7 +68,7 @@ INSTALL_GATEWAY_CTRLPLANE_ORIGINAL="${INSTALL_GATEWAY_CTRLPLANE:-}"
 INSTALL_GATEWAY_CTRLPLANE="${INSTALL_GATEWAY_CTRLPLANE:-false}"
 
 # Model and SLO Configuration
-DEFAULT_MODEL_ID=${DEFAULT_MODEL_ID:-"Qwen/Qwen3-32B"}
+DEFAULT_MODEL_ID=${DEFAULT_MODEL_ID:-"Qwen/Qwen3-0.6B"}
 MODEL_ID=${MODEL_ID:-"unsloth/Meta-Llama-3.1-8B"}
 ACCELERATOR_TYPE=${ACCELERATOR_TYPE:-"H100"}
 SLO_TPOT=${SLO_TPOT:-10}  # Target time-per-output-token SLO (in ms)
@@ -96,6 +96,8 @@ HPA_STABILIZATION_SECONDS=${HPA_STABILIZATION_SECONDS:-240}
 HPA_MIN_REPLICAS=${HPA_MIN_REPLICAS:-1}
 SKIP_CHECKS=${SKIP_CHECKS:-false}
 E2E_TESTS_ENABLED=${E2E_TESTS_ENABLED:-false}
+# WVA metrics endpoint security (set false to disable bearer token auth on /metrics)
+WVA_METRICS_SECURE=${WVA_METRICS_SECURE:-true}
 # vLLM max-num-seqs (max concurrent sequences per replica, lower = easier to saturate for testing)
 VLLM_MAX_NUM_SEQS=${VLLM_MAX_NUM_SEQS:-""}
 # Decode replicas override (useful for e2e testing with limited GPUs)
@@ -454,8 +456,9 @@ deploy_wva_controller() {
         --set wva.logging.level=$WVA_LOG_LEVEL \
         --set wva.prometheus.tls.insecureSkipVerify=$SKIP_TLS_VERIFY \
         --set wva.namespaceScoped=$NAMESPACE_SCOPED \
+        --set wva.metrics.secure=$WVA_METRICS_SECURE \
         ${CONTROLLER_INSTANCE:+--set wva.controllerInstance=$CONTROLLER_INSTANCE}
-    
+
     # Wait for WVA to be ready
     log_info "Waiting for WVA controller to be ready..."
     kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=workload-variant-autoscaler -n $WVA_NS --timeout=60s || \
@@ -877,6 +880,30 @@ deploy_prometheus_adapter() {
         }
 
     log_success "Prometheus Adapter deployment initiated (may still be starting)"
+
+    # On clusters with KEDA, the v1beta1.external.metrics.k8s.io APIService may
+    # point to KEDA's metrics server instead of Prometheus Adapter. KEDA's server
+    # only serves metrics for ScaledObjects, not arbitrary external metrics like
+    # wva_desired_replicas. Detect and fix this conflict.
+    local current_svc
+    current_svc=$(kubectl get apiservice v1beta1.external.metrics.k8s.io -o jsonpath='{.spec.service.name}' 2>/dev/null || echo "")
+
+    if [ -n "$current_svc" ] && [ "$current_svc" != "prometheus-adapter" ]; then
+        log_warning "external.metrics.k8s.io APIService points to '$current_svc' (likely KEDA)"
+        log_info "Patching APIService to point to Prometheus Adapter in $MONITORING_NAMESPACE"
+        kubectl patch apiservice v1beta1.external.metrics.k8s.io --type=merge -p "{
+            \"spec\": {
+                \"insecureSkipTLSVerify\": true,
+                \"service\": {
+                    \"name\": \"prometheus-adapter\",
+                    \"namespace\": \"$MONITORING_NAMESPACE\"
+                }
+            }
+        }" && log_success "APIService patched to use Prometheus Adapter" \
+           || log_warning "Failed to patch external.metrics.k8s.io APIService — HPA may not work"
+    else
+        log_info "external.metrics.k8s.io APIService already points to prometheus-adapter"
+    fi
 }
 
 verify_deployment() {