Merge pull request #878 from nebius/SCHED-1167/flacky-fluxcd-install

Backport: Add retry wrapper for FluxCD kubectl provisioners

Author: theyoprst

soperator/modules/fluxcd/main.tf

@@ -1,13 +1,15 @@
 resource "terraform_data" "flux_namespace" {
   provisioner "local-exec" {
     interpreter = ["/bin/bash", "-c"]
-    command = join(
-      " ",
-      [
-        "kubectl", "create", "namespace", "flux-system",
-        "--context", var.k8s_cluster_context,
-      ]
-    )
+    # `kubectl create namespace` fails with AlreadyExists on re-runs, making it
+    # non-retryable. Piping through `--dry-run=client -o yaml | kubectl apply`
+    # produces an idempotent apply that succeeds whether the namespace exists or not.
+    # The pipe requires a shell, so the inner command is passed as a bash -c string.
+    command = join(" ", [
+      "${path.module}/../scripts/retry.sh", "--", "bash", "-c",
+      "'kubectl create namespace flux-system --context ${var.k8s_cluster_context} --dry-run=client -o yaml",
+      "| kubectl apply --context ${var.k8s_cluster_context} -f -'",
+    ])
   }
   triggers_replace = {
     first_run = "true"
@@ -21,6 +23,7 @@ resource "terraform_data" "flux2" {
     command = join(
       " ",
       [
+        "${path.module}/../scripts/retry.sh", "--",
         "kubectl", "--context", var.k8s_cluster_context,
         "apply", "-f", "https://github.com/fluxcd/flux2/releases/download/${var.flux_version}/install.yaml",
       ]

soperator/modules/scripts/retry.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+retries=5
+interval=2
+
+while [[ $# -gt 0 && "$1" != "--" ]]; do
+    case "$1" in
+        -n) retries="$2"; shift 2 ;;
+        -i) interval="$2"; shift 2 ;;
+        *)  echo "Usage: retry.sh [-n retries] [-i interval] -- <command>" >&2; exit 1 ;;
+    esac
+done
+[[ "${1:-}" == "--" ]] && shift
+
+if [[ $# -eq 0 ]]; then
+    echo "Usage: retry.sh [-n retries] [-i interval] -- <command>" >&2
+    exit 1
+fi
+
+for i in $(seq 1 "$retries"); do
+    if "$@"; then
+        exit 0
+    fi
+    if [[ $i -lt $retries ]]; then
+        echo "($i/$retries) Command failed, retrying in ${interval}s..."
+        sleep "$interval"
+    fi
+done
+
+echo "Command failed after $retries attempts" >&2
+exit 1