Merge pull request #34909 from bitoku/update-crio

k8s-ci-robot · web-flow · commit 1a36b1d6a738 · 2025-06-03T19:22:38.000-07:00
Test a new cri-o version with canary job
diff --git a/config/jobs/kubernetes/sig-node/sig-node-presubmit.yaml b/config/jobs/kubernetes/sig-node/sig-node-presubmit.yaml
@@ -1624,6 +1624,62 @@ presubmits:
         - --skip-regex=\[Flaky\]|\[Slow\]|\[Serial\]|\[Feature:InPlacePodVerticalScaling\]|\[Feature:UserNamespacesSupport]|\[Feature:PodLifecycleSleepActionAllowZero\]|\[Feature:UserNamespacesPodSecurityStandards\]|\[Feature:KubeletCredentialProviders\]|\[Feature:LockContention\]|\[Feature:SCTPConnectivity\]|\[Feature:PodLevelResources\]
         - '--test-args=--container-runtime-endpoint=unix:///var/run/crio/crio.sock --container-runtime-process-name=/usr/local/bin/crio --container-runtime-pid-file= --kubelet-flags="--cgroup-driver=systemd --cgroups-per-qos=true --cgroup-root=/ --runtime-cgroups=/system.slice/crio.service --kubelet-cgroups=/system.slice/kubelet.service" --extra-log="{\"name\": \"crio.log\", \"journalctl\": [\"-u\", \"crio\"]}"'
         - --image-config-file=/home/prow/go/src/k8s.io/test-infra/jobs/e2e_node/crio/latest/image-config-cgroupv2.yaml
+  - name: pull-kubernetes-node-crio-cgrpv2-e2e-canary
+    cluster: k8s-infra-prow-build
+    # explicitly needs /test pull-kubernetes-node-crio-cgrpv2-e2e-canary to run
+    always_run: false
+    # if at all it is run and fails, don't block the PR
+    optional: true
+    branches:
+      # TODO(releng): Remove once repo default branch has been renamed
+      - master
+      - main
+    decorate: true
+    path_alias: k8s.io/kubernetes
+    extra_refs:
+      - org: kubernetes
+        repo: test-infra
+        base_ref: master
+        path_alias: k8s.io/test-infra
+    decoration_config:
+      timeout: 180m
+    labels:
+      preset-service-account: "true"
+      preset-k8s-ssh: "true"
+      preset-pull-kubernetes-e2e: "true"
+      preset-pull-kubernetes-e2e-gce: "true"
+    annotations:
+      testgrid-dashboards: sig-node-cri-o, sig-node-presubmits
+      testgrid-tab-name: pr-crio-cgrpv2-gce-e2e-canary
+    spec:
+      containers:
+        - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20250527-1b2b10e804-master
+          resources:
+            limits:
+              cpu: 4
+              memory: 6Gi
+            requests:
+              cpu: 4
+              memory: 6Gi
+          env:
+            - name: KUBE_SSH_USER
+              value: core
+            - name: IGNITION_INJECT_GCE_SSH_PUBLIC_KEY_FILE
+              value: "1"
+          command:
+            - runner.sh
+          args:
+            - kubetest2
+            - noop
+            - --test=node
+            - --
+            - --repo-root=.
+            - --gcp-zone=us-west1-b
+            - --parallelism=8
+            - --focus-regex=\[NodeConformance\]|\[Feature:.+\]|\[Feature\]
+            - --skip-regex=\[Flaky\]|\[Slow\]|\[Serial\]|\[Feature:InPlacePodVerticalScaling\]|\[Feature:UserNamespacesSupport]|\[Feature:PodLifecycleSleepActionAllowZero\]|\[Feature:UserNamespacesPodSecurityStandards\]|\[Feature:KubeletCredentialProviders\]|\[Feature:LockContention\]|\[Feature:SCTPConnectivity\]|\[Feature:PodLevelResources\]
+            - '--test-args=--container-runtime-endpoint=unix:///var/run/crio/crio.sock --container-runtime-process-name=/usr/local/bin/crio --container-runtime-pid-file= --kubelet-flags="--cgroup-driver=systemd --cgroups-per-qos=true --cgroup-root=/ --runtime-cgroups=/system.slice/crio.service --kubelet-cgroups=/system.slice/kubelet.service" --extra-log="{\"name\": \"crio.log\", \"journalctl\": [\"-u\", \"crio\"]}"'
+            - --image-config-file=/home/prow/go/src/k8s.io/test-infra/jobs/e2e_node/crio/latest/image-config-cgroupv2-canary.yaml
   - name: pull-crio-cgroupv1-node-e2e-features
     cluster: k8s-infra-prow-build
     # explicitly needs /test pull-crio-cgroupv1-node-e2e-features to run
diff --git a/jobs/e2e_node/crio/README.md b/jobs/e2e_node/crio/README.md
@@ -67,3 +67,10 @@ the instance.
 To change the version of CRI-O being used for a single ignition file, just copy
 [env.yaml](./templates/base/env.yaml) and adapt
 [`./templates/generate`](./templates/generate) accordingly.
+
+Make sure the specified cri-o version is uploaded to
+`https://storage.googleapis.com/cri-o/artifacts/cri-o.amd64.{{ CRIO_COMMIT }}.tar.gz`,
+otherwise the tests should fail.
+
+You can test the cri-o version change by changing [env-canary.yaml](./templates/base/env-canary.yaml)
+and run `pull-kubernetes-node-crio-cgrpv2-e2e-canary`.
diff --git a/jobs/e2e_node/crio/crio_cgroupv2_canary.ign b/jobs/e2e_node/crio/crio_cgroupv2_canary.ign
@@ -0,0 +1,90 @@
+{
+  "ignition": {
+    "version": "3.3.0"
+  },
+  "kernelArguments": {
+    "shouldNotExist": [
+      "mitigations=auto,nosmt"
+    ]
+  },
+  "storage": {
+    "files": [
+      {
+        "path": "/etc/zincati/config.d/90-disable-auto-updates.toml",
+        "contents": {
+          "compression": "",
+          "source": "data:,%5Bupdates%5D%0Aenabled%20%3D%20false%0A"
+        },
+        "mode": 420
+      },
+      {
+        "path": "/root/kubelet-e2e.te",
+        "contents": {
+          "compression": "gzip",
+          "source": "data:;base64,H4sIAAAAAAAC/6RRy07DMBC8+ytG4gwCjkT9lsp1BrTKxjbOWhWK+u+ItiFpE07scWcf83B9aqsSXT1QaY98JV6enhvnCj+rFGJ0uJZ9ZUKy+YNy2FtzC4SPkmpetevAsrd+DUgUWx9J0bzEn5UZCeqHAa0UjJAUTBEKvREpM+JYxIjT/fi7KDHC58zYTguaQofe58tmoW9RGH1P1KgSu/nYybmH3bKufHHT3Dmvmo4TNmt9++f7++cLX7YZLAcWNC6mbdq10vcb7B8aZ3yK+nz+nEjjvgMAAP//f1Vw5EkCAAA="
+        },
+        "mode": 420
+      },
+      {
+        "path": "/etc/crio/crio.conf.d/20-crio.conf",
+        "contents": {
+          "compression": "gzip",
+          "source": "data:;base64,H4sIAAAAAAAC/7SP0UrFMAyG7/MUY/duT3CeREbp6bIukiYjTUXfXuomgoII4lVp+P98Xx6TkU5UYsYFKmWJ3gzDoUzpdbgN44ye5h6az9n0VFVGgLNoTZwKLsCaA+Mzcu+seG95hNX0CCSbxZDchtuwRa4IK26xsYer2wvWJH3d+fHWKVmTBa5vOKLv72Kt2sx0xxe8BHtuhKJCrvZTTqV8v+GT121+wzut/85zrP6wR1kZ7T+4bwEAAP//V7kNseQBAAA="
+        },
+        "mode": 420
+      },
+      {
+        "path": "/etc/sysctl.d/99-e2e-sysctl.conf",
+        "contents": {
+          "compression": "gzip",
+          "source": "data:;base64,H4sIAAAAAAAC/2SQvU7DQBCEez/FSG7BduKf4Eh0NBR06dH6bo1P8d1F3r3EvD2KCJFQqpW+LWbmy/EezMIkjCMvgefi5OynpxUU7B/SaWGy8nzFGuFpdT75AofJCZwgMFu2GOOS5Tg5Cz47oy4GKIvKEwY2lIQh36LsLS5uniGsoBvCxVnGG4+UZj2QHOWD1izHQMIWMcA7n0LyiCN0YmGcaU4sRZZjXXjcY1I9yb4sv5xOaShM9OUt7X5N9N5pWVM1Vp1pNrTth11XjdyZ2tKu3fZmbPuW25eaqOqyHIeJYX9L3QJBC0NjxBwv18HXNo+Ti+y/zNdm0zd11WSPQu+vnwAAAP//xE8bG4oBAAA="
+        },
+        "mode": 420
+      },
+      {
+        "path": "/etc/ssh-key-secret/ssh-public",
+        "contents": {
+          "source": "data:text/plain;base64,R0NFX1NTSF9QVUJMSUNfS0VZX0ZJTEVfQ09OVEVOVA=="
+        },
+        "mode": 420
+      },
+      {
+        "path": "/etc/systemd/system.conf.d/10-env.conf",
+        "contents": {
+          "compression": "",
+          "source": "data:,%5BManager%5D%0ADefaultEnvironment%3D%22CRIO_SCRIPT_COMMIT%3Dc54e56dea6a3175198e3bd9b306f681a67c48a09%22%0ADefaultEnvironment%3D%22CRIO_COMMIT%3Dc5c41f21fa802d1c19a4114e88ecd91a270e3a15%22%0A"
+        },
+        "mode": 420
+      }
+    ]
+  },
+  "systemd": {
+    "units": [
+      {
+        "contents": "[Unit]\nDescription=Configure required sysctls.\n\n[Service]\nType=oneshot\nExecStart=/usr/lib/systemd/systemd-sysctl\n\n[Install]\nWantedBy=multi-user.target\n",
+        "enabled": true,
+        "name": "configure-sysctl.service"
+      },
+      {
+        "contents": "[Unit]\nDescription=Download and install required tools.\nBefore=crio-install.service\nAfter=NetworkManager-wait-online.service\n\n[Service]\nType=oneshot\nExecStart=rpm-ostree install \\\n  -y \\\n  --apply-live \\\n  --allow-inactive \\\n  dbus-tools \\\n  checkpolicy\n\n[Install]\nWantedBy=multi-user.target\n",
+        "enabled": true,
+        "name": "tools-install.service"
+      },
+      {
+        "contents": "[Unit]\nDescription=Setup SELinux policy\nAfter=tools-install.service\n\n[Service]\nType=oneshot\nExecStartPre=setenforce 1\nExecStartPre=checkmodule -M -m -o /root/kubelet-e2e.mod /root/kubelet-e2e.te\nExecStartPre=semodule_package -o /root/kubelet-e2e.pp -m /root/kubelet-e2e.mod\nExecStartPre=semodule -i /root/kubelet-e2e.pp\nExecStartPre=mkdir -p /var/lib/kubelet\nExecStart=chcon -R -u system_u -r object_r -t var_lib_t /var/lib/kubelet\n\n[Install]\nWantedBy=multi-user.target\n",
+        "enabled": true,
+        "name": "selinux-install.service"
+      },
+      {
+        "contents": "[Unit]\nDescription=Download and install crio binaries and configurations.\nAfter=selinux-install.service\n\n[Service]\nType=oneshot\nExecStartPre=mount /tmp /tmp -o remount,exec,suid\nExecStartPre=mount -o remount,rw /dev/sda4 /usr\nExecStartPre=bash -c '\\\n  curl --fail --retry 5 --retry-delay 3 --silent --show-error \\\n    https://raw.githubusercontent.com/cri-o/packaging/$CRIO_SCRIPT_COMMIT/get |\\\n      bash -s -- -t $CRIO_COMMIT'\nExecStartPre=rm -f /etc/cni/net.d/87-podman-bridge.conflist\nExecStartPre=rm -f /etc/crio/crio.conf.d/10-crio.conf\nExecStartPre=mv /etc/cni/net.d/10-crio-bridge.conflist.disabled /etc/cni/net.d/10-crio-bridge.conflist\nExecStart=systemctl enable --now crio.service\nRestart=on-failure\n\n[Install]\nWantedBy=multi-user.target\n",
+        "enabled": true,
+        "name": "crio-install.service"
+      },
+      {
+        "contents": "[Unit]\nDescription=Copy authorized keys\nBefore=crio-install.service\nAfter=NetworkManager-wait-online.service\n\n[Service]\nType=oneshot\nExecStart=/bin/sh -c '\\\n  /usr/bin/mkdir -m 0700 -p /home/core/.ssh \u0026\u0026 \\\n  /usr/bin/cat /etc/ssh-key-secret/ssh-public \\\n    \u003e\u003e /home/core/.ssh/authorized_keys \u0026\u0026 \\\n  /usr/bin/chown -R core:core /home/core/.ssh \u0026\u0026 \\\n  /usr/bin/chmod 0600 /home/core/.ssh/authorized_keys'\n\n[Install]\nWantedBy=multi-user.target\n",
+        "enabled": true,
+        "name": "authorized-key.service"
+      }
+    ]
+  }
+}
diff --git a/jobs/e2e_node/crio/latest/image-config-cgroupv2-canary.yaml b/jobs/e2e_node/crio/latest/image-config-cgroupv2-canary.yaml
@@ -0,0 +1,5 @@
+images:
+  fedora:
+    image_family: fedora-coreos-stable
+    project: fedora-coreos-cloud
+    metadata: "user-data</home/prow/go/src/k8s.io/test-infra/jobs/e2e_node/crio/crio_cgroupv2_canary.ign"
diff --git a/jobs/e2e_node/crio/templates/base/env-canary.yaml b/jobs/e2e_node/crio/templates/base/env-canary.yaml
@@ -0,0 +1,10 @@
+---
+storage:
+  files:
+    - path: /etc/systemd/system.conf.d/10-env.conf
+      mode: 0644
+      contents:
+        inline: |
+          [Manager]
+          DefaultEnvironment="CRIO_SCRIPT_COMMIT=c54e56dea6a3175198e3bd9b306f681a67c48a09"
+          DefaultEnvironment="CRIO_COMMIT=c5c41f21fa802d1c19a4114e88ecd91a270e3a15"
diff --git a/jobs/e2e_node/crio/templates/crio_cgroupv2_canary.yaml b/jobs/e2e_node/crio/templates/crio_cgroupv2_canary.yaml
@@ -0,0 +1,129 @@
+---
+variant: fcos
+version: 1.4.0
+kernel_arguments:
+  should_not_exist:
+    - mitigations=auto,nosmt
+storage:
+  files:
+    - path: /etc/zincati/config.d/90-disable-auto-updates.toml
+      contents:
+        local: 90-disable-auto-updates.toml
+      mode: 0644
+    - path: /root/kubelet-e2e.te
+      contents:
+        local: kubelet-e2e.te
+      mode: 0644
+    - path: /etc/crio/crio.conf.d/20-crio.conf
+      contents:
+        local: 20-crio.conf
+      mode: 0644
+    - path: /etc/sysctl.d/99-e2e-sysctl.conf
+      contents:
+        local: 99-e2e-sysctl.conf
+      mode: 0644
+    - path: /etc/ssh-key-secret/ssh-public
+      contents:
+        # base64 encoded "GCE_SSH_PUBLIC_KEY_FILE_CONTENT"
+        source: data:text/plain;base64,R0NFX1NTSF9QVUJMSUNfS0VZX0ZJTEVfQ09OVEVOVA==
+      mode: 0644
+    - path: /etc/systemd/system.conf.d/10-env.conf
+      mode: 0644
+      contents:
+        inline: |
+          [Manager]
+          DefaultEnvironment="CRIO_SCRIPT_COMMIT=c54e56dea6a3175198e3bd9b306f681a67c48a09"
+          DefaultEnvironment="CRIO_COMMIT=c5c41f21fa802d1c19a4114e88ecd91a270e3a15"
+systemd:
+  units:
+    - name: configure-sysctl.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Configure required sysctls.
+
+        [Service]
+        Type=oneshot
+        ExecStart=/usr/lib/systemd/systemd-sysctl
+
+        [Install]
+        WantedBy=multi-user.target
+    - name: tools-install.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Download and install required tools.
+        Before=crio-install.service
+        After=NetworkManager-wait-online.service
+
+        [Service]
+        Type=oneshot
+        ExecStart=rpm-ostree install \
+          -y \
+          --apply-live \
+          --allow-inactive \
+          dbus-tools \
+          checkpolicy
+
+        [Install]
+        WantedBy=multi-user.target
+    - name: selinux-install.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Setup SELinux policy
+        After=tools-install.service
+
+        [Service]
+        Type=oneshot
+        ExecStartPre=setenforce 1
+        ExecStartPre=checkmodule -M -m -o /root/kubelet-e2e.mod /root/kubelet-e2e.te
+        ExecStartPre=semodule_package -o /root/kubelet-e2e.pp -m /root/kubelet-e2e.mod
+        ExecStartPre=semodule -i /root/kubelet-e2e.pp
+        ExecStartPre=mkdir -p /var/lib/kubelet
+        ExecStart=chcon -R -u system_u -r object_r -t var_lib_t /var/lib/kubelet
+
+        [Install]
+        WantedBy=multi-user.target
+    - name: crio-install.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Download and install crio binaries and configurations.
+        After=selinux-install.service
+
+        [Service]
+        Type=oneshot
+        ExecStartPre=mount /tmp /tmp -o remount,exec,suid
+        ExecStartPre=mount -o remount,rw /dev/sda4 /usr
+        ExecStartPre=bash -c '\
+          curl --fail --retry 5 --retry-delay 3 --silent --show-error \
+            https://raw.githubusercontent.com/cri-o/packaging/$CRIO_SCRIPT_COMMIT/get |\
+              bash -s -- -t $CRIO_COMMIT'
+        ExecStartPre=rm -f /etc/cni/net.d/87-podman-bridge.conflist
+        ExecStartPre=rm -f /etc/crio/crio.conf.d/10-crio.conf
+        ExecStartPre=mv /etc/cni/net.d/10-crio-bridge.conflist.disabled /etc/cni/net.d/10-crio-bridge.conflist
+        ExecStart=systemctl enable --now crio.service
+        Restart=on-failure
+
+        [Install]
+        WantedBy=multi-user.target
+    - name: authorized-key.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Copy authorized keys
+        Before=crio-install.service
+        After=NetworkManager-wait-online.service
+
+        [Service]
+        Type=oneshot
+        ExecStart=/bin/sh -c '\
+          /usr/bin/mkdir -m 0700 -p /home/core/.ssh && \
+          /usr/bin/cat /etc/ssh-key-secret/ssh-public \
+            >> /home/core/.ssh/authorized_keys && \
+          /usr/bin/chown -R core:core /home/core/.ssh && \
+          /usr/bin/chmod 0600 /home/core/.ssh/authorized_keys'
+
+        [Install]
+        WantedBy=multi-user.target
diff --git a/jobs/e2e_node/crio/templates/generate b/jobs/e2e_node/crio/templates/generate
@@ -29,6 +29,7 @@ declare -A CONFIGURATIONS=(
     ["crio_cgroupv1_eventedpleg"]="root env cgroupv1 eventedpleg"
     ["crio_cgroupv1_hugepages"]="root env cgroupv1 hugepages"
     ["crio_cgroupv2"]="root env"
+    ["crio_cgroupv2_canary"]="root env-canary"
     ["crio_cgroupv2_drop_infra_ctr"]="root env drop-infra-ctr"
     ["crio_cgroupv2_swap1g"]="root env swap-1G"
     ["crio_cgroupv2_imagefs"]="root env imagefs"
