Skip to content

Commit 39ebc15

Browse files
pohlypohly
committed
DRA canary: fix determining previous release, add n - 2
We want to use the latest release candidate or the latest stable patch release, whatever is newer. With master = 1.34, we can do n - 2 testing with 1.32 kubelet.
1 parent e8fcda1 commit 39ebc15

File tree

3 files changed

+142
-23
lines changed

3 files changed

+142
-23
lines changed

‎config/jobs/kubernetes/sig-node/dra-canary.yaml‎

Lines changed: 118 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -47,10 +47,9 @@ presubmits:
4747
GINKGO_E2E_PID=
4848
trap 'if [ "${GINKGO_E2E_PID}" ]; then kill -TERM "${GINKGO_E2E_PID}"; fi' TERM
4949
trap 'if [ "${GINKGO_E2E_PID}" ]; then kill -INT "${GINKGO_E2E_PID}"; fi' INT
50-
# The final config gets dumped to stderr of the job.
51-
# It's the result of getting the original kind.yaml, manipulating it with sed,
50+
# The final kind.yaml the result of getting the original kind.yaml, manipulating it with sed,
5251
# and adding something at the end.
53-
kind create cluster --retain --config <( (
52+
(
5453
${kind_yaml_cmd[@]} |
5554
# Configure potentially different images for control plane and workers.
5655
sed -e "/^- role: control-plane/ a \ image: $control_plane_image" -e "/^- role: worker/ a \ image: $worker_image"
@@ -69,7 +68,9 @@ presubmits:
6968
local:
7069
dataDir: /tmp/etcd
7170
EOF
72-
) | tee /dev/stderr )
71+
) >/tmp/kind.yaml
72+
cat /tmp/kind.yaml
73+
kind create cluster --retain --config /tmp/kind.yaml
7374
atexit () {
7475
kind export logs "${ARTIFACTS}/kind"
7576
kind delete cluster
@@ -135,10 +136,9 @@ presubmits:
135136
GINKGO_E2E_PID=
136137
trap 'if [ "${GINKGO_E2E_PID}" ]; then kill -TERM "${GINKGO_E2E_PID}"; fi' TERM
137138
trap 'if [ "${GINKGO_E2E_PID}" ]; then kill -INT "${GINKGO_E2E_PID}"; fi' INT
138-
# The final config gets dumped to stderr of the job.
139-
# It's the result of getting the original kind.yaml, manipulating it with sed,
139+
# The final kind.yaml the result of getting the original kind.yaml, manipulating it with sed,
140140
# and adding something at the end.
141-
kind create cluster --retain --config <( (
141+
(
142142
${kind_yaml_cmd[@]} |
143143
# Configure potentially different images for control plane and workers.
144144
sed -e "/^- role: control-plane/ a \ image: $control_plane_image" -e "/^- role: worker/ a \ image: $worker_image"
@@ -157,7 +157,9 @@ presubmits:
157157
local:
158158
dataDir: /tmp/etcd
159159
EOF
160-
) | tee /dev/stderr )
160+
) >/tmp/kind.yaml
161+
cat /tmp/kind.yaml
162+
kind create cluster --retain --config /tmp/kind.yaml
161163
atexit () {
162164
kind export logs "${ARTIFACTS}/kind"
163165
kind delete cluster
@@ -189,7 +191,7 @@ presubmits:
189191
preset-kind-volume-mounts: "true"
190192
annotations:
191193
testgrid-dashboards: sig-node-dynamic-resource-allocation, sig-node-presubmits
192-
description: Runs E2E tests for Dynamic Resource Allocation beta features against a Kubernetes master cluster created with sigs.k8s.io/kind with kubelet from the previous release.
194+
description: Runs E2E tests for Dynamic Resource Allocation beta features against a Kubernetes master cluster created with sigs.k8s.io/kind with kubelet from the "current - 1" release.
193195
testgrid-alert-email: [email protected], [email protected]
194196
decorate: true
195197
decoration_config:
@@ -219,17 +221,20 @@ presubmits:
219221
kind build node-image --image="$control_plane_image" "${kind_node_source}"
220222
major=$(echo "$revision" | sed -e 's/^v\([0-9]*\).*/\1/')
221223
minor=$(echo "$revision" | sed -e 's/^v[0-9]*\([0-9]*\).*/\1/')
222-
# TODO: find latest patch release
224+
previous_minor=$((minor - 1))
225+
# latest-*.txt is only the latest release candidate and can be lower than stable-*.txt.
226+
# Pick whatever is more recent to ensure that we cover release candidates for older patch releases.
227+
$ TODO: only in the periodic job. In the presubmit, test against a known-good previous release.
228+
previous=$((curl --silent -L https://dl.k8s.io/release/latest-$major.$previous_minor.txt && echo && curl --silent -L https://dl.k8s.io/release/stable-$major.$previous_minor.txt && echo) | sort -n | tail -1)
223229
worker_image=dra/node:skewed1
224-
kind build node-image --image="$worker_image" "https://dl.k8s.io/v$major.$((minor - 1)).0/kubernetes-server-linux-amd64.tar.gz"
230+
kind build node-image --image="$worker_image" "https://dl.k8s.io/$previous/kubernetes-server-linux-amd64.tar.gz"
225231
# We might need support for disabling tests which need a recent kubelet. We'll see...
226232
GINKGO_E2E_PID=
227233
trap 'if [ "${GINKGO_E2E_PID}" ]; then kill -TERM "${GINKGO_E2E_PID}"; fi' TERM
228234
trap 'if [ "${GINKGO_E2E_PID}" ]; then kill -INT "${GINKGO_E2E_PID}"; fi' INT
229-
# The final config gets dumped to stderr of the job.
230-
# It's the result of getting the original kind.yaml, manipulating it with sed,
235+
# The final kind.yaml the result of getting the original kind.yaml, manipulating it with sed,
231236
# and adding something at the end.
232-
kind create cluster --retain --config <( (
237+
(
233238
${kind_yaml_cmd[@]} |
234239
# Configure potentially different images for control plane and workers.
235240
sed -e "/^- role: control-plane/ a \ image: $control_plane_image" -e "/^- role: worker/ a \ image: $worker_image"
@@ -248,7 +253,105 @@ presubmits:
248253
local:
249254
dataDir: /tmp/etcd
250255
EOF
251-
) | tee /dev/stderr )
256+
) >/tmp/kind.yaml
257+
cat /tmp/kind.yaml
258+
kind create cluster --retain --config /tmp/kind.yaml
259+
atexit () {
260+
kind export logs "${ARTIFACTS}/kind"
261+
kind delete cluster
262+
}
263+
trap atexit EXIT
264+
KUBECONFIG=${HOME}/.kube/config ${ginkgo} run --nodes=8 --timeout=24h --silence-skips --force-newlines --no-color --label-filter="DRA && Feature: isSubsetOf { OffByDefault, DynamicResourceAllocation } && !Alpha && !Flaky && !Slow" ${e2e_test} -- -provider=local -report-dir="${ARTIFACTS}" -report-complete-ginkgo -report-complete-junit &
265+
GINKGO_E2E_PID=$!
266+
wait "${GINKGO_E2E_PID}"
267+
# docker-in-docker needs privileged mode
268+
securityContext:
269+
privileged: true
270+
resources:
271+
limits:
272+
cpu: 2
273+
memory: 6Gi
274+
requests:
275+
cpu: 2
276+
memory: 6Gi
277+
278+
- name: pull-kubernetes-kind-dra-n-2-canary
279+
cluster: eks-prow-build-cluster
280+
skip_branches:
281+
- release-\d+\.\d+ # per-release image
282+
always_run: false
283+
optional: true
284+
labels:
285+
preset-service-account: "true"
286+
preset-dind-enabled: "true"
287+
preset-kind-volume-mounts: "true"
288+
annotations:
289+
testgrid-dashboards: sig-node-dynamic-resource-allocation, sig-node-presubmits
290+
description: Runs E2E tests for Dynamic Resource Allocation beta features against a Kubernetes master cluster created with sigs.k8s.io/kind with kubelet from the "current - 2" release.
291+
testgrid-alert-email: [email protected], [email protected]
292+
decorate: true
293+
decoration_config:
294+
timeout: 90m
295+
path_alias: k8s.io/kubernetes
296+
spec:
297+
containers:
298+
- image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20250527-1b2b10e804-master
299+
command:
300+
- runner.sh
301+
args:
302+
- /bin/bash
303+
- -xce
304+
- |
305+
set -o pipefail
306+
# A presubmit job uses the checked out and merged source code.
307+
revision=$(git describe --tags)
308+
kind_yaml_cmd=(cat test/e2e/dra/kind.yaml)
309+
kind_node_source=.
310+
features=( )
311+
make WHAT="github.com/onsi/ginkgo/v2/ginkgo k8s.io/kubernetes/test/e2e/e2e.test"
312+
ginkgo=_output/bin/ginkgo
313+
e2e_test=_output/bin/e2e.test
314+
# The latest kind is assumed to work also for older release branches, should this job get forked.
315+
curl --fail --silent --show-error --location https://kind.sigs.k8s.io/dl/latest/linux-amd64.tgz | tar xvfz - -C "${PATH%%:*}/" kind
316+
control_plane_image=dra/node:latest
317+
kind build node-image --image="$control_plane_image" "${kind_node_source}"
318+
major=$(echo "$revision" | sed -e 's/^v\([0-9]*\).*/\1/')
319+
minor=$(echo "$revision" | sed -e 's/^v[0-9]*\([0-9]*\).*/\1/')
320+
previous_minor=$((minor - 2))
321+
# latest-*.txt is only the latest release candidate and can be lower than stable-*.txt.
322+
# Pick whatever is more recent to ensure that we cover release candidates for older patch releases.
323+
$ TODO: only in the periodic job. In the presubmit, test against a known-good previous release.
324+
previous=$((curl --silent -L https://dl.k8s.io/release/latest-$major.$previous_minor.txt && echo && curl --silent -L https://dl.k8s.io/release/stable-$major.$previous_minor.txt && echo) | sort -n | tail -1)
325+
worker_image=dra/node:skewed2
326+
kind build node-image --image="$worker_image" "https://dl.k8s.io/$previous/kubernetes-server-linux-amd64.tar.gz"
327+
# We might need support for disabling tests which need a recent kubelet. We'll see...
328+
GINKGO_E2E_PID=
329+
trap 'if [ "${GINKGO_E2E_PID}" ]; then kill -TERM "${GINKGO_E2E_PID}"; fi' TERM
330+
trap 'if [ "${GINKGO_E2E_PID}" ]; then kill -INT "${GINKGO_E2E_PID}"; fi' INT
331+
# The final kind.yaml the result of getting the original kind.yaml, manipulating it with sed,
332+
# and adding something at the end.
333+
(
334+
${kind_yaml_cmd[@]} |
335+
# Configure potentially different images for control plane and workers.
336+
sed -e "/^- role: control-plane/ a \ image: $control_plane_image" -e "/^- role: worker/ a \ image: $worker_image"
337+
338+
# Additional features are not in kind.yaml, but they can be added at the end.
339+
for feature in ${features[@]}; do echo " ${feature}: true"; done
340+
341+
# Append ClusterConfiguration which causes etcd to use /tmp
342+
# (https://github.com/kubernetes-sigs/kind/issues/845#issuecomment-1261248420).
343+
# There's no kubeadmConfigPatches in any kind.yaml, so we can append at the end.
344+
cat <<EOF
345+
kubeadmConfigPatches:
346+
- |
347+
kind: ClusterConfiguration
348+
etcd:
349+
local:
350+
dataDir: /tmp/etcd
351+
EOF
352+
) >/tmp/kind.yaml
353+
cat /tmp/kind.yaml
354+
kind create cluster --retain --config /tmp/kind.yaml
252355
atexit () {
253356
kind export logs "${ARTIFACTS}/kind"
254357
kind delete cluster

‎config/jobs/kubernetes/sig-node/dra.generate.conf‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,17 +41,28 @@ all_features = true
4141
use_dind = true
4242
run_if_changed = /(dra|dynamicresources|resourceclaim|deviceclass|resourceslice|resourceclaimtemplate|dynamic-resource-allocation|pkg/apis/resource|api/resource)/.*.go
4343

44-
# This job runs the current e2e.test against a cluster where the kubelet is from the previous release (n - 1).
44+
# This job runs the current e2e.test against a cluster where the kubelet is from the "current - 1" release.
4545
#
4646
# It enables and tests the same features as kind-dra.
4747
[kind-dra-n-1]
48-
description = Runs E2E tests for Dynamic Resource Allocation beta features against a Kubernetes master cluster created with sigs.k8s.io/kind with kubelet from the previous release.
48+
description = Runs E2E tests for Dynamic Resource Allocation beta features against a Kubernetes master cluster created with sigs.k8s.io/kind with kubelet from the "current - 1" release.
4949
use_dind = true
5050
cluster = eks-prow-build-cluster
5151
run_if_changed = /(dra|dynamicresources|resourceclaim|deviceclass|resourceslice|resourceclaimtemplate|dynamic-resource-allocation|pkg/apis/resource|api/resource)/.*.go
5252
kubelet_skew = 1
5353
generate = canary # not ready for periodic yet
5454

55+
# This job runs the current e2e.test against a cluster where the kubelet is from the "current - 2" release.
56+
#
57+
# It enables and tests the same features as kind-dra.
58+
[kind-dra-n-2]
59+
description = Runs E2E tests for Dynamic Resource Allocation beta features against a Kubernetes master cluster created with sigs.k8s.io/kind with kubelet from the "current - 2" release.
60+
use_dind = true
61+
cluster = eks-prow-build-cluster
62+
run_if_changed = /(dra|dynamicresources|resourceclaim|deviceclass|resourceslice|resourceclaimtemplate|dynamic-resource-allocation|pkg/apis/resource|api/resource)/.*.go
63+
kubelet_skew = 2
64+
generate = canary # not ready for periodic yet
65+
5566
# This job runs e2e_node.test with a focus on tests for the Dynamic Resource Allocation feature (currently beta)
5667
[node-e2e-crio-cgrpv1-dra]
5768
job_type = node

‎config/jobs/kubernetes/sig-node/dra.jinja‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -156,20 +156,23 @@ presubmits:
156156
{%- if kubelet_skew|int > 0 %}
157157
major=$(echo "$revision" | sed -e 's/^v\([0-9]*\).*/\1/')
158158
minor=$(echo "$revision" | sed -e 's/^v[0-9]*\([0-9]*\).*/\1/')
159-
# TODO: find latest patch release
159+
previous_minor=$((minor - {{kubelet_skew}}))
160+
# latest-*.txt is only the latest release candidate and can be lower than stable-*.txt.
161+
# Pick whatever is more recent to ensure that we cover release candidates for older patch releases.
162+
$ TODO: only in the periodic job. In the presubmit, test against a known-good previous release.
163+
previous=$((curl --silent -L https://dl.k8s.io/release/latest-$major.$previous_minor.txt && echo && curl --silent -L https://dl.k8s.io/release/stable-$major.$previous_minor.txt && echo) | sort -n | tail -1)
160164
worker_image=dra/node:skewed{{kubelet_skew}}
161-
kind build node-image --image="$worker_image" "https://dl.k8s.io/v$major.$((minor - {{kubelet_skew|int}})).0/kubernetes-server-linux-amd64.tar.gz"
165+
kind build node-image --image="$worker_image" "https://dl.k8s.io/$previous/kubernetes-server-linux-amd64.tar.gz"
162166
# We might need support for disabling tests which need a recent kubelet. We'll see...
163167
{%- else %}
164168
worker_image="$control_plane_image"
165169
{%- endif %}
166170
GINKGO_E2E_PID=
167171
trap 'if [ "${GINKGO_E2E_PID}" ]; then kill -TERM "${GINKGO_E2E_PID}"; fi' TERM
168172
trap 'if [ "${GINKGO_E2E_PID}" ]; then kill -INT "${GINKGO_E2E_PID}"; fi' INT
169-
# The final config gets dumped to stderr of the job.
170-
# It's the result of getting the original kind.yaml, manipulating it with sed,
173+
# The final kind.yaml the result of getting the original kind.yaml, manipulating it with sed,
171174
# and adding something at the end.
172-
kind create cluster --retain --config <( (
175+
(
173176
${kind_yaml_cmd[@]} |
174177
# Configure potentially different images for control plane and workers.
175178
sed -e "/^- role: control-plane/ a \ image: $control_plane_image" -e "/^- role: worker/ a \ image: $worker_image"
@@ -188,7 +191,9 @@ presubmits:
188191
local:
189192
dataDir: /tmp/etcd
190193
EOF
191-
) | tee /dev/stderr )
194+
) >/tmp/kind.yaml
195+
cat /tmp/kind.yaml
196+
kind create cluster --retain --config /tmp/kind.yaml
192197
atexit () {
193198
kind export logs "${ARTIFACTS}/kind"
194199
kind delete cluster

0 commit comments

Comments
 (0)