Skip to content

Commit bb0e969

Browse files
authored
Merge pull request #34836 from ynfx8/copy
add emulated version upgrade test scripts and config prow job
2 parents 27416db + 8e6ded6 commit bb0e969

File tree

3 files changed

+287
-2
lines changed

3 files changed

+287
-2
lines changed

config/jobs/kubernetes/sig-testing/compatibility-versions-e2e.yaml

Lines changed: 57 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ periodics:
2828
spec:
2929
containers:
3030
- image: gcr.io/k8s-staging-test-infra/krte:v20250513-98d205aae3-master
31-
imagePullPolicy: Always # pull latest image for canary testing
3231
command:
3332
- wrapper.sh
3433
- bash
@@ -86,7 +85,6 @@ periodics:
8685
spec:
8786
containers:
8887
- image: gcr.io/k8s-staging-test-infra/krte:v20250513-98d205aae3-master
89-
imagePullPolicy: Always # pull latest image for canary testing
9088
command:
9189
- wrapper.sh
9290
- bash
@@ -165,3 +163,60 @@ periodics:
165163
# this is mostly for building kubernetes
166164
memory: 14Gi
167165
cpu: 7
166+
- interval: 1h
167+
cluster: k8s-infra-prow-build
168+
name: ci-kubernetes-e2e-kind-two-steps-upgrade-n-minus-1
169+
annotations:
170+
testgrid-dashboards: sig-testing-kind
171+
testgrid-tab-name: two-steps-upgrade-test-n-minus-1
172+
description: Uses kind to run e2e tests from the n-1 kubernetes release against a latest kubernetes master components w/ --emulated-version=n-1 set.
173+
# TODO(#34269) update owners in experiment/compatibility-versions
174+
testgrid-alert-email: [email protected]
175+
testgrid-num-columns-recent: '6'
176+
labels:
177+
preset-dind-enabled: "true"
178+
preset-kind-volume-mounts: "true"
179+
decorate: true
180+
decoration_config:
181+
timeout: 60m
182+
extra_refs:
183+
- org: kubernetes
184+
repo: kubernetes
185+
base_ref: master
186+
path_alias: k8s.io/kubernetes
187+
workdir: true
188+
- org: kubernetes
189+
repo: test-infra
190+
base_ref: master
191+
path_alias: k8s.io/test-infra
192+
spec:
193+
containers:
194+
- image: gcr.io/k8s-staging-test-infra/krte:v20250513-98d205aae3-master
195+
command:
196+
- wrapper.sh
197+
- bash
198+
- -c
199+
- curl -sSL https://kind.sigs.k8s.io/dl/latest/linux-amd64.tgz | tar xvfz - -C "${PATH%%:*}/" && ./../test-infra/experiment/compatibility-versions/e2e-two-steps-upgrade.sh
200+
env:
201+
- name: VERSION_DELTA
202+
value: "1"
203+
- name: SKIP
204+
value: Alpha|Disruptive|Slow|Flaky|IPv6|LoadBalancer|PodSecurityPolicy|nfs
205+
- name: PARALLEL
206+
value: "true"
207+
- name: FEATURE_GATES
208+
value: '{"AllBeta":true}'
209+
- name: RUNTIME_CONFIG
210+
value: '{"api/beta":"true", "api/ga":"true"}'
211+
# we need privileged mode in order to do docker in docker
212+
securityContext:
213+
privileged: true
214+
resources:
215+
limits:
216+
memory: 9Gi
217+
cpu: 7
218+
requests:
219+
# these are both a bit below peak usage during build
220+
# this is mostly for building kubernetes
221+
memory: 9Gi
222+
cpu: 7
Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
#!/usr/bin/env bash
2+
# Copyright 2024 The Kubernetes Authors.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
# hack script for running a kind e2e
17+
# must be run with a kubernetes checkout in $PWD (IE from the checkout)
18+
# Usage: SKIP="ginkgo skip regex" FOCUS="ginkgo focus regex" kind-e2e.sh
19+
20+
set -o errexit -o nounset -o xtrace
21+
22+
# Settings:
23+
# SKIP: ginkgo skip regex
24+
# FOCUS: ginkgo focus regex
25+
# LABEL_FILTER: ginkgo label query for selecting tests (see "Spec Labels" in https://onsi.github.io/ginkgo/#filtering-specs)
26+
#
27+
# The default is to focus on conformance tests. Serial tests get skipped when
28+
# parallel testing is enabled. Using LABEL_FILTER instead of combining SKIP and
29+
# FOCUS is recommended (more expressive, easier to read than regexp).
30+
#
31+
# GA_ONLY: true - limit to GA APIs/features as much as possible
32+
# false - (default) APIs and features left at defaults
33+
# FEATURE_GATES:
34+
# JSON or YAML encoding of a string/bool map: {"FeatureGateA": true, "FeatureGateB": false}
35+
# Enables or disables feature gates in the entire cluster.
36+
# Cannot be used when GA_ONLY=true.
37+
# RUNTIME_CONFIG:
38+
# JSON or YAML encoding of a string/string (!) map: {"apia.example.com/v1alpha1": "true", "apib.example.com/v1beta1": "false"}
39+
# Enables API groups in the apiserver via --runtime-config.
40+
# Cannot be used when GA_ONLY=true.
41+
42+
COMMON_SCRIPT="${COMMON_SCRIPT:-${PWD}/../test-infra/experiment/compatibility-versions/common.sh}"
43+
source "${COMMON_SCRIPT}"
44+
45+
# setup signal handlers
46+
# shellcheck disable=SC2317 # this is not unreachable code
47+
signal_handler() {
48+
if [ -n "${GINKGO_PID:-}" ]; then
49+
kill -TERM "$GINKGO_PID" || true
50+
fi
51+
cleanup
52+
}
53+
trap signal_handler INT TERM
54+
55+
main() {
56+
# create temp dir and setup cleanup
57+
TMP_DIR=$(mktemp -d -p /tmp kind-e2e-XXXXXX)
58+
echo "Created temporary directory: ${TMP_DIR}"
59+
60+
# ensure artifacts (results) directory exists when not in CI
61+
export ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}"
62+
mkdir -p "${ARTIFACTS}"
63+
64+
export VERSION_DELTA=${VERSION_DELTA:-1}
65+
66+
WORKSPACE_STATUS=$(./hack/print-workspace-status.sh)
67+
GIT_VERSION=$(echo "$WORKSPACE_STATUS" | awk '/^gitVersion / {print $2}')
68+
# Check if gitVersion contains alpha.0 and increment VERSION_DELTA if needed
69+
# If the current version is alpha.0, it means the previous *stable* or developed
70+
# branch is actually n-2 relative to the current minor number for compatibility purposes.
71+
if [[ "${GIT_VERSION}" == *alpha.0* ]]; then
72+
echo "Detected alpha.0 in gitVersion (${GIT_VERSION}), treating as still the previous minor version."
73+
VERSION_DELTA=$((VERSION_DELTA + 1))
74+
echo "Adjusted VERSION_DELTA: ${VERSION_DELTA}"
75+
fi
76+
77+
MAJOR_VERSION=$(echo "$WORKSPACE_STATUS" | awk '/^STABLE_BUILD_MAJOR_VERSION / {print $2}')
78+
MINOR_VERSION=$(echo "$WORKSPACE_STATUS" | awk '/^STABLE_BUILD_MINOR_VERSION / {split($2, minor, "+"); print minor[1]}')
79+
export CURRENT_VERSION="${MAJOR_VERSION}.${MINOR_VERSION}"
80+
export PREV_VERSION="${MAJOR_VERSION}.$((MINOR_VERSION - VERSION_DELTA))"
81+
export EMULATED_VERSION="${PREV_VERSION}"
82+
83+
# export the KUBECONFIG to a unique path for testing
84+
KUBECONFIG="${HOME}/.kube/kind-test-config"
85+
export KUBECONFIG
86+
echo "exported KUBECONFIG=${KUBECONFIG}"
87+
88+
# debug kind version
89+
kind version
90+
91+
# in CI attempt to release some memory after building
92+
if [ -n "${KUBETEST_IN_DOCKER:-}" ]; then
93+
sync || true
94+
echo 1 > /proc/sys/vm/drop_caches || true
95+
fi
96+
97+
res=0
98+
create_cluster || res=$?
99+
100+
101+
# Perform the upgrade. Assume kind-upgrade.sh is in the same directory as this script.
102+
UPGRADE_SCRIPT="${UPGRADE_SCRIPT:-${PWD}/../test-infra/experiment/compatibility-versions/kind-upgrade.sh}"
103+
echo "Upgrading cluster with ${UPGRADE_SCRIPT}"
104+
105+
upgrade_cluster_components || res=$?
106+
# If upgrade fails, we shouldn't run tests but should still clean up.
107+
if [[ "$res" -ne 0 ]]; then
108+
cleanup
109+
exit $res
110+
fi
111+
112+
EMULATED_VERSION_UPGRADE_SCRIPT="${EMULATED_VERSION_UPGRADE_SCRIPT:-${PWD}/../test-infra/experiment/compatibility-versions/emulated-version-upgrade.sh}"
113+
echo "Upgrading cluster with ${EMULATED_VERSION_UPGRADE_SCRIPT}"
114+
"${EMULATED_VERSION_UPGRADE_SCRIPT}" | tee "${ARTIFACTS}/emulated-upgrade-output.txt"
115+
116+
# Clone the previous versions Kubernetes release branch
117+
# TODO(aaron-prindle) extend the branches to test from n-1 -> n-1..3 as more k8s releases are done that support compatibility versions
118+
export PREV_RELEASE_BRANCH="release-${EMULATED_VERSION}"
119+
# Define the path within the temp directory for the cloned repo
120+
PREV_RELEASE_REPO_PATH="${TMP_DIR}/prev-release-k8s"
121+
echo "Cloning branch ${PREV_RELEASE_BRANCH} into ${PREV_RELEASE_REPO_PATH}"
122+
git clone --filter=blob:none --single-branch --branch "${PREV_RELEASE_BRANCH}" https://github.com/kubernetes/kubernetes.git "${PREV_RELEASE_REPO_PATH}"
123+
124+
# enter the cloned prev repo branch (in temp) and run tests
125+
pushd "${PREV_RELEASE_REPO_PATH}"
126+
build_prev_version_bins || res=$?
127+
run_prev_version_tests || res=$?
128+
popd
129+
130+
131+
cleanup || res=$?
132+
exit $res
133+
}
134+
135+
main
Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
#!/usr/bin/env bash
2+
# Copyright 2025 The Kubernetes Authors.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
# script adapted from - https://gist.github.com/aojea/2c94034f8e86d08842e5916231eb3fe1
17+
18+
set -e
19+
set -o pipefail
20+
21+
# Set default values
22+
CLUSTER_NAME=${CLUSTER_NAME:-kind}
23+
BUILD_MODE=${BUILD_MODE:-docker}
24+
UPDATE_KUBE_PROXY=${UPDATE_KUBE_PROXY:-true}
25+
UPDATE_KUBELET=${UPDATE_KUBE_PROXY:-true}
26+
# TODO: we can have more granularity here
27+
UPDATE_CONTROL_PLANE=${UPDATE_CONTROL_PLANE:-true}
28+
CONTROL_PLANE_COMPONENTS="kube-apiserver kube-controller-manager kube-scheduler"
29+
30+
# Initialize variables
31+
# Assume go installed
32+
KUBE_ROOT="."
33+
source "${KUBE_ROOT}/hack/lib/version.sh"
34+
kube::version::get_version_vars
35+
DOCKER_TAG=${KUBE_GIT_VERSION/+/_}
36+
DOCKER_REGISTRY=${KUBE_DOCKER_REGISTRY:-registry.k8s.io}
37+
export GOFLAGS="-tags=providerless"
38+
export KUBE_BUILD_CONFORMANCE=n
39+
40+
# KIND nodes
41+
NODES=$(kind get nodes --name ${CLUSTER_NAME})
42+
CONTROL_PLANE_NODES=$(kind get nodes --name ${CLUSTER_NAME} | grep control)
43+
WORKER_NODES=$(kind get nodes --name ${CLUSTER_NAME} | grep worker)
44+
45+
update_kubelet() {
46+
for n in $NODES; do
47+
# Backup previous kubelet
48+
docker exec $n cp /usr/bin/kubelet /usr/bin/kubelet.bak
49+
# Install new kubelet binary
50+
docker cp ${KUBELET_BINARY} $n:/usr/bin/kubelet
51+
docker exec $n systemctl restart kubelet
52+
echo "Updated kubelet on node $n"
53+
done
54+
}
55+
56+
upgrade_emulated_version(){
57+
for n in $CONTROL_PLANE_NODES; do
58+
for i in $CONTROL_PLANE_COMPONENTS; do
59+
docker exec $n sed -e '/\s*--emulated-version=.*/d' -i /etc/kubernetes/manifests/$i.yaml
60+
echo "Removed emulated-version flag of component $i static pod on node $n if present"
61+
docker exec $n sed -e '/\s*--emulation-forward-compatible=true/d' -i /etc/kubernetes/manifests/$i.yaml
62+
echo "Removed emulation-forward-compatible flag of component $i static pod on node $n if present"
63+
sleep 10
64+
done
65+
done
66+
}
67+
68+
check_emulated_version_removed(){
69+
for n in $CONTROL_PLANE_NODES; do
70+
for i in $CONTROL_PLANE_COMPONENTS; do
71+
if docker exec $n ps aux | grep $i | grep emulated-version; then
72+
echo "Error: Component $i still have emulated-version flag"
73+
exit 1
74+
fi
75+
echo "Confirm component $i does not have --emulated-version flag"
76+
done
77+
done
78+
}
79+
80+
# Main
81+
IMAGES_PATH="${KUBE_ROOT}/_output/release-images/amd64"
82+
KUBELET_BINARY=$(find ${KUBE_ROOT}/_output/ -type f -name kubelet)
83+
84+
if [[ "$UPDATE_CONTROL_PLANE" == "true" ]]; then
85+
upgrade_emulated_version
86+
fi
87+
88+
if [[ "$UPDATE_KUBELET" == "true" ]]; then
89+
update_kubelet
90+
fi
91+
92+
if [[ "$UPDATE_CONTROL_PLANE" == "true" ]]; then
93+
check_emulated_version_removed
94+
exit 0
95+
fi

0 commit comments

Comments
 (0)