1+ #! /usr/bin/env bash
2+ # Copyright 2025 The Kubernetes Authors.
3+ #
4+ # Licensed under the Apache License, Version 2.0 (the "License");
5+ # you may not use this file except in compliance with the License.
6+ # You may obtain a copy of the License at
7+ #
8+ # http://www.apache.org/licenses/LICENSE-2.0
9+ #
10+ # Unless required by applicable law or agreed to in writing, software
11+ # distributed under the License is distributed on an "AS IS" BASIS,
12+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+ # See the License for the specific language governing permissions and
14+ # limitations under the License.
15+
16+ # hack script for running kind clusters, fetching kube-apiserver metrics, and validating feature gates
17+ # must be run with a kubernetes checkout in $PWD (IE from the checkout)
18+ # Usage: compatibility-versions-feature-gate-test.sh
19+
20+ set -o errexit -o nounset -o pipefail
21+ set -o xtrace
22+
23+ # Settings:
24+ # GA_ONLY: true - limit to GA APIs/features as much as possible
25+ # false - (default) APIs and features left at defaults
26+
27+ # FEATURE_GATES:
28+ # JSON or YAML encoding of a string/bool map: {"FeatureGateA": true, "FeatureGateB": false}
29+ # Enables or disables feature gates in the entire cluster.
30+ # Cannot be used when GA_ONLY=true.
31+
32+ # RUNTIME_CONFIG:
33+ # JSON or YAML encoding of a string/string (!) map: {"apia.example.com/v1alpha1": "true", "apib.example.com/v1beta1": "false"}
34+ # Enables API groups in the apiserver via --runtime-config.
35+ # Cannot be used when GA_ONLY=true.
36+
37+ # cleanup logic for cleanup on exit
38+ CLEANED_UP=false
39+ cleanup () {
40+ if [ " $CLEANED_UP " = " true" ; then
41+ return
42+ fi
43+ # KIND_CREATE_ATTEMPTED is true once we: kind create
44+ if [ " ${KIND_CREATE_ATTEMPTED:- } " = true ]; then
45+ kind " export" " ${ARTIFACTS} " || true
46+ kind delete cluster || true
47+ fi
48+ rm -f _output/bin/kubectl || true
49+ # remove our tempdir, this needs to be last, or it will prevent kind delete
50+ if [ -n " ${TMP_DIR:- } " ; then
51+ rm -rf " ${TMP_DIR:? } "
52+ fi
53+ CLEANED_UP=true
54+ }
55+
56+ # setup signal handlers
57+ # shellcheck disable=SC2317 # this is not unreachable code
58+ signal_handler () {
59+ cleanup
60+ }
61+ trap signal_handler INT TERM
62+
63+ # build kubernetes / node image, kubectl binary
64+ build () {
65+ # build the node image w/ kubernetes
66+ kind build node-image -v 1
67+ # make sure we have kubectl
68+ make all WHAT=" cmd/kubectl"
69+
70+ # Ensure the built kubectl is used instead of system
71+ export PATH=" ${PWD} /_output/bin:$PATH "
72+ }
73+
74+ check_structured_log_support () {
75+ case " ${KUBE_VERSION} " in
76+ v1.1[0-8].* )
77+ echo " $1 is only supported on versions >= v1.19, got ${KUBE_VERSION} "
78+ exit 1
79+ ;;
80+ esac
81+ }
82+
83+ # up a cluster with kind
84+ create_cluster () {
85+ # Grab the version of the cluster we're about to start
86+ KUBE_VERSION=" $( docker run --rm --entrypoint=cat " kindest/node:latest" ) "
87+
88+ # Default Log level for all components in test clusters
89+ KIND_CLUSTER_LOG_LEVEL=${KIND_CLUSTER_LOG_LEVEL:- 4}
90+
91+ EMULATED_VERSION=${EMULATED_VERSION:- }
92+
93+ # potentially enable --logging-format
94+ CLUSTER_LOG_FORMAT=${CLUSTER_LOG_FORMAT:- }
95+ scheduler_extra_args=" \" v\" : \" ${KIND_CLUSTER_LOG_LEVEL} \" "
96+ controllerManager_extra_args=" \" v\" : \" ${KIND_CLUSTER_LOG_LEVEL} \" "
97+ apiServer_extra_args=" \" v\" : \" ${KIND_CLUSTER_LOG_LEVEL} \" "
98+ kubelet_extra_args=" \" v\" : \" ${KIND_CLUSTER_LOG_LEVEL} \" "
99+
100+ if [ -n " $CLUSTER_LOG_FORMAT " ; then
101+ check_structured_log_support " CLUSTER_LOG_FORMAT"
102+ scheduler_extra_args=" ${scheduler_extra_args}
103+ \" logging-format\" : \" ${CLUSTER_LOG_FORMAT} \" "
104+ controllerManager_extra_args=" ${controllerManager_extra_args}
105+ \" logging-format\" : \" ${CLUSTER_LOG_FORMAT} \" "
106+ apiServer_extra_args=" ${apiServer_extra_args}
107+ \" logging-format\" : \" ${CLUSTER_LOG_FORMAT} \" "
108+ fi
109+
110+ KUBELET_LOG_FORMAT=${KUBELET_LOG_FORMAT:- $CLUSTER_LOG_FORMAT }
111+ if [ -n " $KUBELET_LOG_FORMAT " ; then
112+ check_structured_log_support " KUBECTL_LOG_FORMAT"
113+ kubelet_extra_args=" ${kubelet_extra_args}
114+ \" logging-format\" : \" ${KUBELET_LOG_FORMAT} \" "
115+ fi
116+
117+ # JSON or YAML map injected into featureGates config
118+ feature_gates=" ${FEATURE_GATES:- {\} } "
119+ # --runtime-config argument value passed to the API server, again as a map
120+ runtime_config=" ${RUNTIME_CONFIG:- {\} } "
121+
122+ case " ${GA_ONLY:- false} " in
123+ false)
124+ :
125+ ;;
126+ true)
127+ if [ " ${feature_gates} " != " {}" ; then
128+ echo " GA_ONLY=true and FEATURE_GATES=${feature_gates} are mutually exclusive."
129+ exit 1
130+ fi
131+ if [ " ${runtime_config} " != " {}" ; then
132+ echo " GA_ONLY=true and RUNTIME_CONFIG=${runtime_config} are mutually exclusive."
133+ exit 1
134+ fi
135+
136+ echo " Limiting to GA APIs and features for ${KUBE_VERSION} "
137+ feature_gates=' {"AllAlpha":false,"AllBeta":false}'
138+ runtime_config=' {"api/alpha":"false", "api/beta":"false"}'
139+ ;;
140+ * )
141+ echo " \$ GA_ONLY set to '${GA_ONLY} '; supported values are true and false (default)"
142+ exit 1
143+ ;;
144+ esac
145+
146+ # create the config file
147+ cat << EOF > "${ARTIFACTS} /kind-config.yaml"
148+ # config for 1 control plane node and 2 workers (necessary for conformance)
149+ kind: Cluster
150+ apiVersion: kind.x-k8s.io/v1alpha4
151+ networking:
152+ ipFamily: ${IP_FAMILY:- ipv4}
153+ kubeProxyMode: ${KUBE_PROXY_MODE:- iptables}
154+ # don't pass through host search paths
155+ # TODO: possibly a reasonable default in the future for kind ...
156+ dnsSearch: []
157+ nodes:
158+ - role: control-plane
159+ featureGates: ${feature_gates}
160+ runtimeConfig: ${runtime_config}
161+ kubeadmConfigPatches:
162+ - |
163+ kind: ClusterConfiguration
164+ metadata:
165+ name: config
166+ apiServer:
167+ extraArgs:
168+ ${apiServer_extra_args}
169+ "emulated-version": "${EMULATED_VERSION} "
170+ controllerManager:
171+ extraArgs:
172+ ${controllerManager_extra_args}
173+ "emulated-version": "${EMULATED_VERSION} "
174+ scheduler:
175+ extraArgs:
176+ ${scheduler_extra_args}
177+ "emulated-version": "${EMULATED_VERSION} "
178+ ---
179+ kind: InitConfiguration
180+ nodeRegistration:
181+ kubeletExtraArgs:
182+ ${kubelet_extra_args}
183+ ---
184+ kind: JoinConfiguration
185+ nodeRegistration:
186+ kubeletExtraArgs:
187+ ${kubelet_extra_args}
188+ EOF
189+
190+ KIND_CREATE_ATTEMPTED=true
191+ kind create cluster \
192+ --image=kindest/node:latest \
193+ --retain \
194+ --wait=1m \
195+ -v=3 \
196+ " --config=${ARTIFACTS} /kind-config.yaml"
197+
198+ # debug cluster version
199+ kubectl version
200+
201+ # Patch kube-proxy to set the verbosity level
202+ kubectl patch -n kube-system daemonset/kube-proxy \
203+ --type=' json' ' [{"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--v=' " ${KIND_CLUSTER_LOG_LEVEL} " ' " }]'
204+ }
205+
206+ fetch_metrics () {
207+ local output_file=" $1 "
208+ echo " Fetching metrics to ${output_file} ..."
209+ kubectl get --raw /metrics > " ${output_file} "
210+ }
211+
212+
213+ main () {
214+ TMP_DIR=$( mktemp -d)
215+ export ARTIFACTS=" ${ARTIFACTS:- ${PWD} / _artifacts} "
216+ mkdir -p " ${ARTIFACTS} "
217+
218+ export EMULATED_VERSION=$( get_latest_release_version)
219+ export PREV_VERSIONED_FEATURE_LIST=${PREV_VERSIONED_FEATURE_LIST:- " release-${EMULATED_VERSION} /test/featuregates_linter/test_data/versioned_feature_list.yaml"
220+ export UNVERSIONED_FEATURE_LIST=${UNVERSIONED_FEATURE_LIST:- " release-${EMULATED_VERSION} /test/featuregates_linter/test_data/unversioned_feature_list.yaml"
221+
222+ # Create and validate previous cluster
223+ git clone --filter=blob:none --single-branch --branch " release-${EMULATED_VERSION} " " release-${EMULATED_VERSION} "
224+
225+ # Build current version
226+ build
227+
228+ # Create and validate latest cluster
229+ KUBECONFIG=" ${HOME} /.kube/kind-test-config-latest"
230+ export KUBECONFIG
231+ create_cluster
232+ LATEST_METRICS=" ${ARTIFACTS} /latest_metrics.txt"
233+ fetch_metrics " ${LATEST_METRICS} "
234+ LATEST_RESULTS=" ${ARTIFACTS} /latest_results.txt"
235+
236+ VALIDATE_SCRIPT=" ${VALIDATE_SCRIPT:- ${PWD} / ../ test-infra/ experiment/ compatibility-versions/ validate-compatibility-versions-feature-gates.sh} "
237+ " ${VALIDATE_SCRIPT} " " ${EMULATED_VERSION} " " ${LATEST_METRICS} " " ${PREV_VERSIONED_FEATURE_LIST} " " ${UNVERSIONED_FEATURE_LIST} " " ${LATEST_RESULTS} "
238+
239+ # Report results
240+ echo " === Latest Cluster (${EMULATED_VERSION} ) Validation ==="
241+ cat " ${LATEST_RESULTS} "
242+
243+ if grep -q " FAIL" " ${LATEST_RESULTS} " ; then
244+ echo " Validation failures detected"
245+ exit 1
246+ fi
247+
248+ cleanup
249+ }
250+
251+ get_latest_release_version () {
252+ git ls-remote --heads https://github.com/kubernetes/kubernetes.git | \
253+ grep -o ' release-[0-9]\+\.[0-9]\+' | \
254+ sort -t. -k1,1n -k2,2n | \
255+ tail -n1 | \
256+ cut -d- -f2
257+ }
258+
259+ main
0 commit comments