DRA canary: filter by kubelet version, use stable release for presubmit

As expected some tests fail because they test aspects of the kubelet
which were added in more recent releases, like seamless upgrades.
That test can labeled with KubeletMinVersion:1.33 to avoid running it with
kubelet 1.32.

The CI artifacts worked okay in previous canary tests, but want we want
for presubmits are the stable releases to avoid breaking PR testing for
reasons that are not under control of the PR author.

Author: pohly

config/jobs/kubernetes/sig-node/dra-canary.yaml

@@ -248,9 +248,9 @@ presubmits:
           major=$(echo "$revision" | sed -e 's/^v\([0-9]*\).*/\1/')
           minor=$(echo "$revision" | sed -e 's/^v[0-9]*\.\([0-9]*\).*/\1/')
           previous_minor=$((minor - 1))
-          # TODO: use dl.k8s.io/release for presubmits.
-          previous=$(curl --silent -L "https://dl.k8s.io/ci/latest-$major.$previous_minor.txt" )
-          curl --silent -L "https://dl.k8s.io/ci/$previous/kubernetes-server-linux-amd64.tar.gz" | tar zxOf - kubernetes/server/bin/kubelet >/tmp/kubelet
+          # Test with the lastest stable release to avoid breaking presubmits because of unrelated issues in a release candidate.
+          previous=$(curl --silent -L "https://dl.k8s.io/release/stable-$major.$previous_minor.txt" )
+          curl --silent -L "https://dl.k8s.io/release/$previous/kubernetes-server-linux-amd64.tar.gz" | tar zxOf - kubernetes/server/bin/kubelet >/tmp/kubelet
           chmod a+rx /tmp/kubelet
           /tmp/kubelet --version
           worker_nodes=$(kind get nodes | grep worker)
@@ -259,9 +259,15 @@ presubmits:
               docker exec $n systemctl restart kubelet
           done
 
-          # We might need support for disabling tests which need a recent kubelet. We'll see...
+          # We need support for disabling tests which need a recent kubelet.
+          # If a test is labeled with `KubeletMinVersion:1.34`, then it cannot run
+          # when the deployed kubelet is 1.32. This is enforced by
+          # generating `KubeletMinVersion: isSubsetOf { 1.33, 1.34 }`, i.e.
+          # not including the actual kubelet version and anything older
+          # in an allow list.
+          kubelet_label_filter="&& KubeletMinVersion: isSubsetOf { $( for v in $(seq $((previous_minor + 1)) $((minor - 1))); do echo "1.$v, "; done)1.$minor }"
 
-          KUBECONFIG=${HOME}/.kube/config ${ginkgo} run --nodes=8 --timeout=24h --silence-skips --force-newlines --no-color --label-filter="DRA && Feature: isSubsetOf { OffByDefault, DynamicResourceAllocation } && !Alpha && !Flaky && !Slow" ${e2e_test} -- -provider=local -report-dir="${ARTIFACTS}" -report-complete-ginkgo -report-complete-junit &
+          KUBECONFIG=${HOME}/.kube/config ${ginkgo} run --nodes=8 --timeout=24h --silence-skips --force-newlines --no-color --label-filter="DRA && Feature: isSubsetOf { OffByDefault, DynamicResourceAllocation }$kubelet_label_filter && !Alpha && !Flaky && !Slow" ${e2e_test} -- -provider=local -report-dir="${ARTIFACTS}" -report-complete-ginkgo -report-complete-junit &
           GINKGO_E2E_PID=$!
           wait "${GINKGO_E2E_PID}"
         # docker-in-docker needs privileged mode
@@ -350,9 +356,9 @@ presubmits:
           major=$(echo "$revision" | sed -e 's/^v\([0-9]*\).*/\1/')
           minor=$(echo "$revision" | sed -e 's/^v[0-9]*\.\([0-9]*\).*/\1/')
           previous_minor=$((minor - 2))
-          # TODO: use dl.k8s.io/release for presubmits.
-          previous=$(curl --silent -L "https://dl.k8s.io/ci/latest-$major.$previous_minor.txt" )
-          curl --silent -L "https://dl.k8s.io/ci/$previous/kubernetes-server-linux-amd64.tar.gz" | tar zxOf - kubernetes/server/bin/kubelet >/tmp/kubelet
+          # Test with the lastest stable release to avoid breaking presubmits because of unrelated issues in a release candidate.
+          previous=$(curl --silent -L "https://dl.k8s.io/release/stable-$major.$previous_minor.txt" )
+          curl --silent -L "https://dl.k8s.io/release/$previous/kubernetes-server-linux-amd64.tar.gz" | tar zxOf - kubernetes/server/bin/kubelet >/tmp/kubelet
           chmod a+rx /tmp/kubelet
           /tmp/kubelet --version
           worker_nodes=$(kind get nodes | grep worker)
@@ -361,9 +367,15 @@ presubmits:
               docker exec $n systemctl restart kubelet
           done
 
-          # We might need support for disabling tests which need a recent kubelet. We'll see...
+          # We need support for disabling tests which need a recent kubelet.
+          # If a test is labeled with `KubeletMinVersion:1.34`, then it cannot run
+          # when the deployed kubelet is 1.32. This is enforced by
+          # generating `KubeletMinVersion: isSubsetOf { 1.33, 1.34 }`, i.e.
+          # not including the actual kubelet version and anything older
+          # in an allow list.
+          kubelet_label_filter="&& KubeletMinVersion: isSubsetOf { $( for v in $(seq $((previous_minor + 1)) $((minor - 1))); do echo "1.$v, "; done)1.$minor }"
 
-          KUBECONFIG=${HOME}/.kube/config ${ginkgo} run --nodes=8 --timeout=24h --silence-skips --force-newlines --no-color --label-filter="DRA && Feature: isSubsetOf { OffByDefault, DynamicResourceAllocation } && !Alpha && !Flaky && !Slow" ${e2e_test} -- -provider=local -report-dir="${ARTIFACTS}" -report-complete-ginkgo -report-complete-junit &
+          KUBECONFIG=${HOME}/.kube/config ${ginkgo} run --nodes=8 --timeout=24h --silence-skips --force-newlines --no-color --label-filter="DRA && Feature: isSubsetOf { OffByDefault, DynamicResourceAllocation }$kubelet_label_filter && !Alpha && !Flaky && !Slow" ${e2e_test} -- -provider=local -report-dir="${ARTIFACTS}" -report-complete-ginkgo -report-complete-junit &
           GINKGO_E2E_PID=$!
           wait "${GINKGO_E2E_PID}"
         # docker-in-docker needs privileged mode

config/jobs/kubernetes/sig-node/dra.jinja

@@ -189,9 +189,16 @@ presubmits:
           major=$(echo "$revision" | sed -e 's/^v\([0-9]*\).*/\1/')
           minor=$(echo "$revision" | sed -e 's/^v[0-9]*\.\([0-9]*\).*/\1/')
           previous_minor=$((minor - {{kubelet_skew}}))
-          # TODO: use dl.k8s.io/release for presubmits.
+          {%- if ci %}
+          # Test with the most recent CI build, doesn't even need to be released yet.
+          # We want to know if those are broken.
           previous=$(curl --silent -L "https://dl.k8s.io/ci/latest-$major.$previous_minor.txt" )
           curl --silent -L "https://dl.k8s.io/ci/$previous/kubernetes-server-linux-amd64.tar.gz" | tar zxOf - kubernetes/server/bin/kubelet >/tmp/kubelet
+          {%- else %}
+          # Test with the lastest stable release to avoid breaking presubmits because of unrelated issues in a release candidate.
+          previous=$(curl --silent -L "https://dl.k8s.io/release/stable-$major.$previous_minor.txt" )
+          curl --silent -L "https://dl.k8s.io/release/$previous/kubernetes-server-linux-amd64.tar.gz" | tar zxOf - kubernetes/server/bin/kubelet >/tmp/kubelet
+          {%- endif %}
           chmod a+rx /tmp/kubelet
           /tmp/kubelet --version
           worker_nodes=$(kind get nodes | grep worker)
@@ -200,10 +207,16 @@ presubmits:
               docker exec $n systemctl restart kubelet
           done
 
-          # We might need support for disabling tests which need a recent kubelet. We'll see...
+          # We need support for disabling tests which need a recent kubelet.
+          # If a test is labeled with `KubeletMinVersion:1.34`, then it cannot run
+          # when the deployed kubelet is 1.32. This is enforced by
+          # generating `KubeletMinVersion: isSubsetOf { 1.33, 1.34 }`, i.e.
+          # not including the actual kubelet version and anything older
+          # in an allow list.
+          kubelet_label_filter="&& KubeletMinVersion: isSubsetOf { $( for v in $(seq $((previous_minor + 1)) $((minor - 1))); do echo "1.$v, "; done)1.$minor }"
           {%- endif %}
 
-          KUBECONFIG=${HOME}/.kube/config ${ginkgo} run --nodes=8 --timeout=24h --silence-skips --force-newlines --no-color --label-filter="DRA && Feature: isSubsetOf { OffByDefault, DynamicResourceAllocation } {%- if not all_features %} && !Alpha {%- endif %} && !Flaky {%- if not ci %} && !Slow {%- endif %}" ${e2e_test} -- -provider=local -report-dir="${ARTIFACTS}" -report-complete-ginkgo -report-complete-junit &
+          KUBECONFIG=${HOME}/.kube/config ${ginkgo} run --nodes=8 --timeout=24h --silence-skips --force-newlines --no-color --label-filter="DRA && Feature: isSubsetOf { OffByDefault, DynamicResourceAllocation } {%- if kubelet_skew|int > 0 %}$kubelet_label_filter{%- endif %} {%- if not all_features %} && !Alpha {%- endif %} && !Flaky {%- if not ci %} && !Slow {%- endif %}" ${e2e_test} -- -provider=local -report-dir="${ARTIFACTS}" -report-complete-ginkgo -report-complete-junit &
           GINKGO_E2E_PID=$!
           wait "${GINKGO_E2E_PID}"
           {%- else -%}