Skip to content

Commit 24b1d0f

Browse files
TaylorMutchTaylorMutch
committed
ci(helm-e2e): use kind in CI, keep k3d for local dev
The previous attempts to make the in-container kubectl reach the host's k3d API server kept hitting tooling gaps (missing iproute2, gawk-only strtonum). Step back and follow the conventional pattern instead: - Drop the `container:` block from the helm-e2e jobs and run on the bare runner. Install mise via `curl https://mise.run | sh`. - Use `helm/kind-action` to provision a kind cluster on the runner. Because the workflow steps run on the runner directly, the kind API server is reachable through the standard kubeconfig the action writes. - Add HELM_E2E_SKIP_CLUSTER and HELM_E2E_IMAGE_LOADER env vars to helm-e2e.sh so it can drive the existing flow against either a self- managed k3d cluster (default; what 'mise run e2e:helm:*' uses locally) or a caller-managed kind cluster (CI). Image loading switches between 'k3d image import' and 'kind load docker-image' accordingly. - Revert the in-container kubeconfig-rewrite hacks in helm-k3s-local.sh; they are no longer needed once CI runs on the bare runner.
1 parent ecfd83b commit 24b1d0f

3 files changed

Lines changed: 72 additions & 68 deletions

File tree

‎.github/workflows/branch-helm-e2e.yml‎

Lines changed: 28 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -59,30 +59,33 @@ jobs:
5959
permissions:
6060
contents: read
6161
packages: read
62-
container:
63-
image: ghcr.io/nvidia/openshell/ci:latest
64-
credentials:
65-
username: ${{ github.actor }}
66-
password: ${{ secrets.GITHUB_TOKEN }}
67-
options: --privileged
68-
volumes:
69-
- /var/run/docker.sock:/var/run/docker.sock
7062
env:
7163
MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
7264
HELM_E2E_CLUSTER_NAME: helm-e2e-${{ github.run_id }}-rust
7365
HELM_E2E_IMAGE_TAG: ${{ github.sha }}
66+
HELM_E2E_SKIP_CLUSTER: "1"
67+
HELM_E2E_IMAGE_LOADER: kind
7468
steps:
7569
- uses: actions/checkout@v6
7670

77-
- name: Mark workspace safe for git
78-
run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
79-
8071
- name: Log in to GHCR
8172
run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
8273

74+
- name: Install mise
75+
run: |
76+
curl https://mise.run | sh
77+
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
78+
echo "$HOME/.local/share/mise/shims" >> "$GITHUB_PATH"
79+
8380
- name: Install tools
8481
run: mise install --locked
8582

83+
- name: Create kind cluster
84+
uses: helm/kind-action@v1
85+
with:
86+
cluster_name: ${{ env.HELM_E2E_CLUSTER_NAME }}
87+
wait: 120s
88+
8689
- name: Run Helm E2E (Rust)
8790
run: mise run e2e:helm:rust
8891

@@ -95,32 +98,35 @@ jobs:
9598
permissions:
9699
contents: read
97100
packages: read
98-
container:
99-
image: ghcr.io/nvidia/openshell/ci:latest
100-
credentials:
101-
username: ${{ github.actor }}
102-
password: ${{ secrets.GITHUB_TOKEN }}
103-
options: --privileged
104-
volumes:
105-
- /var/run/docker.sock:/var/run/docker.sock
106101
env:
107102
MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
108103
HELM_E2E_CLUSTER_NAME: helm-e2e-${{ github.run_id }}-python
109104
HELM_E2E_IMAGE_TAG: ${{ github.sha }}
105+
HELM_E2E_SKIP_CLUSTER: "1"
106+
HELM_E2E_IMAGE_LOADER: kind
110107
steps:
111108
- uses: actions/checkout@v6
112109

113-
- name: Mark workspace safe for git
114-
run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
115-
116110
- name: Log in to GHCR
117111
run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
118112

113+
- name: Install mise
114+
run: |
115+
curl https://mise.run | sh
116+
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
117+
echo "$HOME/.local/share/mise/shims" >> "$GITHUB_PATH"
118+
119119
- name: Install tools
120120
run: mise install --locked
121121

122122
- name: Install Python dependencies
123123
run: uv sync --frozen && mise run --no-deps python:proto
124124

125+
- name: Create kind cluster
126+
uses: helm/kind-action@v1
127+
with:
128+
cluster_name: ${{ env.HELM_E2E_CLUSTER_NAME }}
129+
wait: 120s
130+
125131
- name: Run Helm E2E (Python)
126132
run: mise run e2e:helm:python

‎tasks/scripts/helm-e2e.sh‎

Lines changed: 44 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,15 @@
1919
# HELM_E2E_SUITE rust | python | all (default: all)
2020
# HELM_E2E_PKI pki-init | cert-manager (default: pki-init)
2121
# HELM_E2E_KEEP_CLUSTER 1 to skip cluster deletion on exit (default: 0)
22-
# HELM_E2E_CLUSTER_NAME override k3d cluster name (default: derived from branch)
22+
# HELM_E2E_CLUSTER_NAME override cluster name (default: derived from branch)
23+
# HELM_E2E_SKIP_CLUSTER 1 if the caller has already provisioned the cluster
24+
# (and KUBECONFIG points at it). The script will not
25+
# create or delete the cluster. Used by CI, where
26+
# helm/kind-action provisions a kind cluster before
27+
# this script runs.
28+
# HELM_E2E_IMAGE_LOADER k3d | kind | none — which loader to use to import
29+
# the gateway and supervisor images into the cluster
30+
# (default: k3d for local dev; CI sets kind)
2331
# HELM_E2E_IMAGE_TAG if set, pull gateway+supervisor images from
2432
# HELM_E2E_IMAGE_REGISTRY at this tag instead of
2533
# building them locally (used by CI to reuse the
@@ -35,6 +43,8 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
3543
SUITE="${HELM_E2E_SUITE:-all}"
3644
PKI_MODE="${HELM_E2E_PKI:-pki-init}"
3745
KEEP_CLUSTER="${HELM_E2E_KEEP_CLUSTER:-0}"
46+
SKIP_CLUSTER="${HELM_E2E_SKIP_CLUSTER:-0}"
47+
IMAGE_LOADER="${HELM_E2E_IMAGE_LOADER:-k3d}"
3848

3949
# Derive cluster name the same way helm-k3s-local.sh does (last path component of branch).
4050
_branch_cluster_name() {
@@ -92,12 +102,27 @@ require_cmd() {
92102
fi
93103
}
94104

95-
require_cmd k3d
96105
require_cmd helm
97106
require_cmd kubectl
98107
require_cmd docker
99108
require_cmd openssl
100109

110+
# k3d is only needed when this script manages the cluster lifecycle. CI hands
111+
# us a pre-existing kind cluster via HELM_E2E_SKIP_CLUSTER=1.
112+
if [ "${SKIP_CLUSTER}" != "1" ]; then
113+
require_cmd k3d
114+
fi
115+
case "${IMAGE_LOADER}" in
116+
k3d|kind|none) ;;
117+
*)
118+
echo "ERROR: unknown HELM_E2E_IMAGE_LOADER '${IMAGE_LOADER}' (must be k3d, kind, or none)" >&2
119+
exit 2
120+
;;
121+
esac
122+
if [ "${IMAGE_LOADER}" = "kind" ]; then
123+
require_cmd kind
124+
fi
125+
101126
if ! docker info >/dev/null 2>&1; then
102127
echo "ERROR: docker daemon is not reachable" >&2
103128
exit 2
@@ -106,7 +131,9 @@ fi
106131
echo "=== helm-e2e: suite=${SUITE} pki=${PKI_MODE} cluster=${CLUSTER_NAME} ==="
107132

108133
# ── Cluster ──────────────────────────────────────────────────────────────────
109-
if k3d cluster get "${CLUSTER_NAME}" >/dev/null 2>&1; then
134+
if [ "${SKIP_CLUSTER}" = "1" ]; then
135+
echo "Using pre-existing cluster '${CLUSTER_NAME}' (HELM_E2E_SKIP_CLUSTER=1)."
136+
elif k3d cluster get "${CLUSTER_NAME}" >/dev/null 2>&1; then
110137
echo "Reusing existing k3d cluster '${CLUSTER_NAME}'."
111138
# Refresh kubeconfig in case it's stale.
112139
k3d kubeconfig write "${CLUSTER_NAME}" --output "${KUBECONFIG}" >/dev/null
@@ -162,9 +189,20 @@ else
162189
"${ROOT}" 2>&1
163190
fi
164191

165-
# Load images into the k3d cluster nodes.
166-
echo "Loading images into k3d cluster..."
167-
k3d image import "${GATEWAY_IMAGE}" "${SUPERVISOR_IMAGE}" -c "${CLUSTER_NAME}" 2>&1
192+
# Load images into the cluster nodes.
193+
case "${IMAGE_LOADER}" in
194+
k3d)
195+
echo "Loading images into k3d cluster '${CLUSTER_NAME}'..."
196+
k3d image import "${GATEWAY_IMAGE}" "${SUPERVISOR_IMAGE}" -c "${CLUSTER_NAME}" 2>&1
197+
;;
198+
kind)
199+
echo "Loading images into kind cluster '${CLUSTER_NAME}'..."
200+
kind load docker-image "${GATEWAY_IMAGE}" "${SUPERVISOR_IMAGE}" --name "${CLUSTER_NAME}" 2>&1
201+
;;
202+
none)
203+
echo "Skipping image load (HELM_E2E_IMAGE_LOADER=none); the cluster must already have ${GATEWAY_IMAGE} and ${SUPERVISOR_IMAGE}."
204+
;;
205+
esac
168206

169207
# ── Deploy via Helm ───────────────────────────────────────────────────────────
170208
HELM_VALUES_FLAGS=(

‎tasks/scripts/helm-k3s-local.sh‎

Lines changed: 0 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -114,46 +114,6 @@ merge_kubeconfig() {
114114
rm -f "${tmp}"
115115

116116
kubectl --kubeconfig="${KUBECONFIG_TARGET}" config use-context "$(k3d_context_name)"
117-
118-
# When this script runs inside a container (e.g., a GitHub Actions
119-
# `container:` job mounting /var/run/docker.sock), k3d publishes the API
120-
# server on the host's `0.0.0.0:<port>` but `0.0.0.0` from inside the
121-
# container is not the host. Rewrite the server URL to the default-route
122-
# gateway, which routes to the docker host. The API server cert is signed
123-
# for `0.0.0.0` / `127.0.0.1` and won't have the gateway IP as a SAN, so
124-
# mark the cluster insecure-skip-tls-verify (CI-only path; local dev keeps
125-
# the default secure setup).
126-
if [[ -f /.dockerenv ]]; then
127-
local context old_server new_server host_addr
128-
context="$(k3d_context_name)"
129-
old_server=$(kubectl --kubeconfig="${KUBECONFIG_TARGET}" config view --raw \
130-
-o "jsonpath={.clusters[?(@.name=='${context}')].cluster.server}")
131-
if [[ "${old_server}" == https://0.0.0.0:* ]]; then
132-
# Read the default-route gateway from /proc/net/route directly to avoid
133-
# depending on the `ip` command, which is not in the CI image. The
134-
# gateway field is a little-endian 32-bit hex value, so we read pairs
135-
# of hex digits in reverse and format as dotted decimal.
136-
host_addr=$(awk '$2=="00000000" {
137-
gw = $3
138-
printf "%d.%d.%d.%d",
139-
strtonum("0x" substr(gw,7,2)),
140-
strtonum("0x" substr(gw,5,2)),
141-
strtonum("0x" substr(gw,3,2)),
142-
strtonum("0x" substr(gw,1,2))
143-
exit
144-
}' /proc/net/route 2>/dev/null) || host_addr=""
145-
if [[ -n "${host_addr}" ]]; then
146-
new_server="${old_server//0.0.0.0/${host_addr}}"
147-
echo "Inside container; rewriting kubeconfig server ${old_server} -> ${new_server} (insecure-skip-tls-verify)."
148-
kubectl --kubeconfig="${KUBECONFIG_TARGET}" config unset \
149-
"clusters.${context}.certificate-authority-data" >/dev/null 2>&1 || true
150-
kubectl --kubeconfig="${KUBECONFIG_TARGET}" config set-cluster "${context}" \
151-
--server="${new_server}" --insecure-skip-tls-verify=true >/dev/null
152-
else
153-
echo "warning: running inside a container but could not detect a default-route gateway; kubectl may fail to reach the API server." >&2
154-
fi
155-
fi
156-
fi
157117
}
158118

159119
apply_base_manifests() {

0 commit comments

Comments
 (0)