|
15 | 15 | package util |
16 | 16 |
|
17 | 17 | import ( |
| 18 | + "bytes" |
18 | 19 | "context" |
19 | 20 | "fmt" |
| 21 | + "io" |
20 | 22 | "os" |
21 | 23 | "os/signal" |
22 | 24 | "regexp" |
23 | 25 | "strings" |
24 | 26 | "sync" |
25 | 27 | "time" |
26 | 28 |
|
| 29 | + "github.com/argoproj-labs/argocd-autopilot/pkg/kube" |
27 | 30 | "github.com/briandowns/spinner" |
28 | 31 | "github.com/codefresh-io/cli-v2/pkg/log" |
29 | 32 | "github.com/codefresh-io/cli-v2/pkg/reporter" |
30 | 33 | "github.com/codefresh-io/cli-v2/pkg/store" |
| 34 | + kubeutil "github.com/codefresh-io/cli-v2/pkg/util/kube" |
| 35 | + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
31 | 36 |
|
| 37 | + v1 "k8s.io/api/core/v1" |
| 38 | + kerrors "k8s.io/apimachinery/pkg/api/errors" |
| 39 | + "k8s.io/client-go/kubernetes" |
32 | 40 | "k8s.io/client-go/tools/clientcmd" |
33 | 41 | ) |
34 | 42 |
|
@@ -188,3 +196,148 @@ func reportCancel(status reporter.CliStepStatus) { |
188 | 196 | }) |
189 | 197 | } |
190 | 198 |
|
| 199 | +func RunNetworkTest(ctx context.Context, kubeFactory kube.Factory, urls ...string) error { |
| 200 | + const networkTestsTimeout = 120 * time.Second |
| 201 | + var testerPodName string |
| 202 | + |
| 203 | + envVars := map[string]string{ |
| 204 | + "URLS": strings.Join(urls, ","), |
| 205 | + "IN_CLUSTER": "1", |
| 206 | + } |
| 207 | + env := prepareEnvVars(envVars) |
| 208 | + |
| 209 | + client, err := kubeFactory.KubernetesClientSet() |
| 210 | + if err != nil { |
| 211 | + return fmt.Errorf("failed to create kubernetes client: %w", err) |
| 212 | + } |
| 213 | + |
| 214 | + err = kubeutil.LaunchJob(ctx, kubeutil.LaunchJobOptions{ |
| 215 | + Client: client, |
| 216 | + Namespace: store.Get().DefaultNamespace, |
| 217 | + JobName: &store.Get().NetworkTesterName, |
| 218 | + Image: &store.Get().NetworkTesterImage, |
| 219 | + Env: env, |
| 220 | + RestartPolicy: v1.RestartPolicyNever, |
| 221 | + BackOffLimit: 0, |
| 222 | + }) |
| 223 | + if err != nil { |
| 224 | + return err |
| 225 | + } |
| 226 | + |
| 227 | + defer func() { |
| 228 | + deferErr := client.BatchV1().Jobs(store.Get().DefaultNamespace).Delete(ctx, store.Get().NetworkTesterName, metav1.DeleteOptions{}) |
| 229 | + if deferErr != nil { |
| 230 | + log.G(ctx).Error("fail to delete job resource '%s': %s", store.Get().NetworkTesterName, deferErr.Error()) |
| 231 | + } |
| 232 | + }() |
| 233 | + |
| 234 | + log.G(ctx).Info("Running network test...") |
| 235 | + |
| 236 | + ticker := time.NewTicker(5 * time.Second) |
| 237 | + defer ticker.Stop() |
| 238 | + var podLastState *v1.Pod |
| 239 | + timeoutChan := time.After(networkTestsTimeout) |
| 240 | + |
| 241 | +Loop: |
| 242 | + for { |
| 243 | + select { |
| 244 | + case <-ticker.C: |
| 245 | + log.G(ctx).Debug("Waiting for network tester to finish") |
| 246 | + |
| 247 | + if testerPodName == "" { |
| 248 | + testerPodName, err = getTesterPodName(ctx, client) |
| 249 | + if err != nil { |
| 250 | + return err |
| 251 | + } |
| 252 | + } |
| 253 | + |
| 254 | + pod, err := client.CoreV1().Pods(store.Get().DefaultNamespace).Get(ctx, testerPodName, metav1.GetOptions{}) |
| 255 | + if err != nil { |
| 256 | + if statusError, errIsStatusError := err.(*kerrors.StatusError); errIsStatusError { |
| 257 | + if statusError.ErrStatus.Reason == metav1.StatusReasonNotFound { |
| 258 | + log.G(ctx).Debug("Network tester pod not found") |
| 259 | + } |
| 260 | + } |
| 261 | + } |
| 262 | + if len(pod.Status.ContainerStatuses) == 0 { |
| 263 | + log.G(ctx).Debug("Network tester pod: creating container") |
| 264 | + continue |
| 265 | + } |
| 266 | + if pod.Status.ContainerStatuses[0].State.Running != nil { |
| 267 | + log.G(ctx).Debug("Network tester pod: running") |
| 268 | + } |
| 269 | + if pod.Status.ContainerStatuses[0].State.Waiting != nil { |
| 270 | + log.G(ctx).Debug("Network tester pod: waiting") |
| 271 | + } |
| 272 | + if pod.Status.ContainerStatuses[0].State.Terminated != nil { |
| 273 | + log.G(ctx).Debug("Network tester pod: terminated") |
| 274 | + podLastState = pod |
| 275 | + break Loop |
| 276 | + } |
| 277 | + case <-timeoutChan: |
| 278 | + return fmt.Errorf("network test timeout reached!") |
| 279 | + } |
| 280 | + } |
| 281 | + |
| 282 | + defer func() { |
| 283 | + deferErr := client.CoreV1().Pods(store.Get().DefaultNamespace).Delete(ctx, testerPodName, metav1.DeleteOptions{}) |
| 284 | + if deferErr != nil { |
| 285 | + log.G(ctx).Error("fail to delete tester pod '%s': %s", testerPodName, deferErr.Error()) |
| 286 | + } |
| 287 | + }() |
| 288 | + |
| 289 | + return checkPodLastState(ctx, client, testerPodName,podLastState) |
| 290 | +} |
| 291 | + |
| 292 | +func prepareEnvVars(vars map[string]string) []v1.EnvVar { |
| 293 | + var env []v1.EnvVar |
| 294 | + |
| 295 | + for key, value := range vars { |
| 296 | + env = append(env, v1.EnvVar{ |
| 297 | + Name: key, |
| 298 | + Value: value, |
| 299 | + }) |
| 300 | + } |
| 301 | + |
| 302 | + return env |
| 303 | +} |
| 304 | + |
| 305 | +func getTesterPodName(ctx context.Context, client kubernetes.Interface) (string, error) { |
| 306 | + pods, err := client.CoreV1().Pods(store.Get().DefaultNamespace).List(ctx, metav1.ListOptions{}) |
| 307 | + if err != nil { |
| 308 | + return "", fmt.Errorf("failed to get pods from cluster: %w", err) |
| 309 | + } |
| 310 | + |
| 311 | + for _, pod := range pods.Items { |
| 312 | + if pod.ObjectMeta.GenerateName == store.Get().NetworkTesterGenerateName { |
| 313 | + return pod.ObjectMeta.Name, nil |
| 314 | + } |
| 315 | + } |
| 316 | + |
| 317 | + return "", nil |
| 318 | +} |
| 319 | + |
| 320 | +func checkPodLastState(ctx context.Context, client kubernetes.Interface, name string, podLastState *v1.Pod) error { |
| 321 | + req := client.CoreV1().Pods(store.Get().DefaultNamespace).GetLogs(name, &v1.PodLogOptions{}) |
| 322 | + podLogs, err := req.Stream(ctx) |
| 323 | + if err != nil { |
| 324 | + return fmt.Errorf("Failed to get network-tester pod logs: %w", err) |
| 325 | + } |
| 326 | + defer podLogs.Close() |
| 327 | + |
| 328 | + logsBuf := new(bytes.Buffer) |
| 329 | + _, err = io.Copy(logsBuf, podLogs) |
| 330 | + if err != nil { |
| 331 | + return fmt.Errorf("Failed to read network-tester pod logs: %w", err) |
| 332 | + } |
| 333 | + logs := strings.Trim(logsBuf.String(), "\n") |
| 334 | + log.G(ctx).Debug(logs) |
| 335 | + |
| 336 | + if podLastState.Status.ContainerStatuses[0].State.Terminated.ExitCode != 0 { |
| 337 | + terminationMessage := strings.Trim(podLastState.Status.ContainerStatuses[0].State.Terminated.Message, "\n") |
| 338 | + return fmt.Errorf("Network test failed with: %s", terminationMessage) |
| 339 | + } |
| 340 | + |
| 341 | + return nil |
| 342 | +} |
| 343 | + |
0 commit comments