|
14 | 14 | import string |
15 | 15 | import subprocess |
16 | 16 | import time |
| 17 | +import os |
17 | 18 |
|
18 | 19 | import mlflow |
19 | 20 | import mlflow.sklearn |
@@ -97,6 +98,36 @@ def _skip_if_mlflow_not_ready(): |
97 | 98 | # --- Debug helpers to surface CI failures --- |
98 | 99 |
|
99 | 100 |
|
| 101 | +def _setup_minikube_docker_env(): |
| 102 | + """Ensure we're using minikube's docker daemon for image builds.""" |
| 103 | + try: |
| 104 | + # Get minikube docker env |
| 105 | + result = subprocess.run( |
| 106 | + ["minikube", "docker-env", "--shell", "bash"], |
| 107 | + capture_output=True, |
| 108 | + text=True, |
| 109 | + timeout=30, |
| 110 | + ) |
| 111 | + if result.returncode == 0: |
| 112 | + # Parse the env vars from minikube docker-env output |
| 113 | + env_lines = [line.strip() for line in result.stdout.split("\n") if line.startswith("export ")] |
| 114 | + for line in env_lines: |
| 115 | + if "=" in line: |
| 116 | + # Extract VAR=value from "export VAR=value" |
| 117 | + var_assignment = line.replace("export ", "", 1) |
| 118 | + if "=" in var_assignment: |
| 119 | + key, value = var_assignment.split("=", 1) |
| 120 | + # Remove quotes from value if present |
| 121 | + value = value.strip("\"'") |
| 122 | + os.environ[key] = value |
| 123 | + print(f"[DEBUG] Set {key}={value}") |
| 124 | + print("[DEBUG] Minikube docker environment configured") |
| 125 | + else: |
| 126 | + print(f"[DEBUG] Failed to get minikube docker-env: {result.stderr}") |
| 127 | + except Exception as exc: |
| 128 | + print(f"[DEBUG] Error setting up minikube docker env: {exc}") |
| 129 | + |
| 130 | + |
100 | 131 | def _run_debug_cmd(label, cmd): |
101 | 132 | print(f"[DEBUG] {label}: {' '.join(cmd)}") |
102 | 133 | try: |
@@ -137,7 +168,11 @@ def _dump_deployment_debug_info(deployment_name): |
137 | 168 | pod_name = _first_pod_name(PROJECT_NAME) |
138 | 169 | if pod_name: |
139 | 170 | _run_debug_cmd("kubectl describe pod", ["kubectl", "describe", "pod", pod_name, "-n", PROJECT_NAME]) |
140 | | - _run_debug_cmd("kubectl logs", ["kubectl", "logs", pod_name, "-n", PROJECT_NAME]) |
| 171 | + _run_debug_cmd("kubectl logs current", ["kubectl", "logs", pod_name, "-n", PROJECT_NAME]) |
| 172 | + _run_debug_cmd("kubectl logs previous", ["kubectl", "logs", pod_name, "-n", PROJECT_NAME, "--previous"]) |
| 173 | + |
| 174 | + # Also check available images in minikube |
| 175 | + _run_debug_cmd("minikube image ls", ["minikube", "image", "ls"]) |
141 | 176 |
|
142 | 177 |
|
143 | 178 | def _dump_registry_status(): |
@@ -174,7 +209,16 @@ def test_train_and_push_model_to_mlflow(): |
174 | 209 | def test_deploy_model(): |
175 | 210 | """Test model deployment.""" |
176 | 211 | _skip_if_mlflow_not_ready() |
| 212 | + # Ensure we use minikube's docker daemon for image builds |
| 213 | + _setup_minikube_docker_env() |
177 | 214 | result = run_cli("projects", "deploy", PROJECT_NAME, "--model-name", MODEL_NAME, "--model-version", MODEL_VERSION) |
| 215 | + |
| 216 | + # Check if image exists in minikube after deployment |
| 217 | + expected_image_name = ( |
| 218 | + f"{PROJECT_NAME.lower().replace('_', '-')}-{MODEL_NAME.lower().replace('_', '-')}-{MODEL_VERSION}-ctr:latest" |
| 219 | + ) |
| 220 | + _run_debug_cmd("minikube image ls", ["minikube", "image", "ls"]) |
| 221 | + |
178 | 222 | assert result.returncode == 0, f"Deploy failed: {result.stderr}" |
179 | 223 |
|
180 | 224 |
|
|
0 commit comments