From 9447f725afff608ba132a95a6f493c3f64464d92 Mon Sep 17 00:00:00 2001 From: Lennart Jern Date: Wed, 1 Oct 2025 10:09:15 +0000 Subject: [PATCH] E2E: Use secret instead of host path for CCM config Signed-off-by: Lennart Jern --- .../data/ccm/cloud-controller-manager.yaml | 106 +++++++++--------- .../components/cluster-resource-sets/ccm.yaml | 18 +++ .../patch-ccm-cloud-config.yaml | 8 -- 3 files changed, 73 insertions(+), 59 deletions(-) diff --git a/test/e2e/data/ccm/cloud-controller-manager.yaml b/test/e2e/data/ccm/cloud-controller-manager.yaml index 90ef0035c8..7fb50d4835 100644 --- a/test/e2e/data/ccm/cloud-controller-manager.yaml +++ b/test/e2e/data/ccm/cloud-controller-manager.yaml @@ -1,5 +1,5 @@ # From: https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/master/manifests/controller-manager/openstack-cloud-controller-manager-ds.yaml ---- +# NOTE! We modify the node-selector to have empty value (""). This matches what kubeadm does. apiVersion: v1 kind: ServiceAccount metadata: @@ -26,64 +26,67 @@ spec: spec: nodeSelector: node-role.kubernetes.io/control-plane: "" - # we need user root to read the cloud.conf from the host securityContext: - runAsUser: 0 + runAsUser: 1001 tolerations: - - key: node.cloudprovider.kubernetes.io/uninitialized - value: "true" - effect: NoSchedule - - key: node-role.kubernetes.io/master - effect: NoSchedule - - key: node-role.kubernetes.io/control-plane - effect: NoSchedule + - key: "CriticalAddonsOnly" + operator: "Equal" + value: "true" + effect: "NoExecute" + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule serviceAccountName: cloud-controller-manager containers: - - name: openstack-cloud-controller-manager - image: >- - registry.k8s.io/provider-os/openstack-cloud-controller-manager:v1.32.0 - args: - - /bin/openstack-cloud-controller-manager - - --v=1 - - --cluster-name=$(CLUSTER_NAME) - - --cloud-config=$(CLOUD_CONFIG) - - --cloud-provider=openstack - - --use-service-account-credentials=false - - --bind-address=127.0.0.1 - volumeMounts: - - mountPath: /etc/kubernetes - name: k8s - readOnly: true - - mountPath: /etc/kubernetes/pki - name: k8s-certs - readOnly: true - - mountPath: /etc/ssl/certs - name: ca-certs - readOnly: true - resources: - requests: - cpu: 200m - env: - - name: CLOUD_CONFIG - value: /etc/kubernetes/cloud.conf - - name: CLUSTER_NAME - value: kubernetes - hostNetwork: true - volumes: - - hostPath: - path: /etc/kubernetes - type: DirectoryOrCreate - name: k8s - - hostPath: - path: /etc/kubernetes/pki - type: DirectoryOrCreate + - name: openstack-cloud-controller-manager + image: registry.k8s.io/provider-os/openstack-cloud-controller-manager:v1.33.0 + args: + - /bin/openstack-cloud-controller-manager + - --v=1 + - --cluster-name=$(CLUSTER_NAME) + - --cloud-config=$(CLOUD_CONFIG) + - --cloud-provider=openstack + - --use-service-account-credentials=false + - --bind-address=127.0.0.1 + volumeMounts: + - mountPath: /etc/kubernetes/pki name: k8s-certs - - hostPath: - path: /etc/ssl/certs - type: DirectoryOrCreate + readOnly: true + - mountPath: /etc/ssl/certs name: ca-certs + readOnly: true + - mountPath: /etc/config + name: cloud-config-volume + readOnly: true + resources: + requests: + cpu: 200m + env: + - name: CLOUD_CONFIG + value: /etc/config/cloud.conf + - name: CLUSTER_NAME + value: kubernetes + dnsPolicy: ClusterFirst + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes/pki + type: DirectoryOrCreate + name: k8s-certs + - hostPath: + path: /etc/ssl/certs + type: DirectoryOrCreate + name: ca-certs + - name: cloud-config-volume + secret: + secretName: cloud-config --- # https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/refs/heads/master/manifests/controller-manager/cloud-controller-manager-role-bindings.yaml +# NOTE! We need to "extract" the List or the CRS will fail to apply. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -111,6 +114,7 @@ subjects: namespace: kube-system --- # https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/refs/heads/master/manifests/controller-manager/cloud-controller-manager-roles.yaml +# NOTE! We need to "extract" the List or the CRS will fail to apply. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/test/e2e/data/kustomize/components/cluster-resource-sets/ccm.yaml b/test/e2e/data/kustomize/components/cluster-resource-sets/ccm.yaml index f567cb5ada..3ae3644d1d 100644 --- a/test/e2e/data/kustomize/components/cluster-resource-sets/ccm.yaml +++ b/test/e2e/data/kustomize/components/cluster-resource-sets/ccm.yaml @@ -4,6 +4,22 @@ metadata: name: ccm-${CLUSTER_NAME}-crs-1 data: ${CCM_RESOURCES} --- +apiVersion: v1 +kind: Secret +metadata: + name: ccm-${CLUSTER_NAME}-crs-1 +type: addons.cluster.x-k8s.io/resource-set +stringData: + cloud-config-secret.yaml: | + apiVersion: v1 + kind: Secret + metadata: + # This name is referenced in the CCM deployment manifest + name: cloud-config + namespace: kube-system + data: + cloud.conf: ${OPENSTACK_CLOUD_PROVIDER_CONF_B64} +--- apiVersion: addons.cluster.x-k8s.io/v1beta2 kind: ClusterResourceSet metadata: @@ -15,4 +31,6 @@ spec: resources: - kind: ConfigMap name: ccm-${CLUSTER_NAME}-crs-1 + - kind: Secret + name: ccm-${CLUSTER_NAME}-crs-1 strategy: ApplyOnce diff --git a/test/e2e/data/kustomize/components/cluster-resource-sets/patch-ccm-cloud-config.yaml b/test/e2e/data/kustomize/components/cluster-resource-sets/patch-ccm-cloud-config.yaml index b88d9eda92..f12a04eb7d 100644 --- a/test/e2e/data/kustomize/components/cluster-resource-sets/patch-ccm-cloud-config.yaml +++ b/test/e2e/data/kustomize/components/cluster-resource-sets/patch-ccm-cloud-config.yaml @@ -1,14 +1,6 @@ - op: add path: /spec/kubeadmConfigSpec/files value: [] -- op: add - path: /spec/kubeadmConfigSpec/files/- - value: - content: ${OPENSTACK_CLOUD_PROVIDER_CONF_B64} - encoding: base64 - owner: root - path: /etc/kubernetes/cloud.conf - permissions: "0600" - op: add path: /spec/kubeadmConfigSpec/files/- value: