E2E: Use secret instead of host path for CCM config

Signed-off-by: Lennart Jern <[email protected]>

Author: lentzi90

test/e2e/data/ccm/cloud-controller-manager.yaml

@@ -1,5 +1,5 @@
 # From: https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/master/manifests/controller-manager/openstack-cloud-controller-manager-ds.yaml
----
+# NOTE! We modify the node-selector to have empty value (""). This matches what kubeadm does.
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@@ -26,64 +26,67 @@ spec:
     spec:
       nodeSelector:
         node-role.kubernetes.io/control-plane: ""
-      # we need user root to read the cloud.conf from the host
       securityContext:
-        runAsUser: 0
+        runAsUser: 1001
       tolerations:
-        - key: node.cloudprovider.kubernetes.io/uninitialized
-          value: "true"
-          effect: NoSchedule
-        - key: node-role.kubernetes.io/master
-          effect: NoSchedule
-        - key: node-role.kubernetes.io/control-plane
-          effect: NoSchedule
+      - key: "CriticalAddonsOnly"
+        operator: "Equal"
+        value: "true"
+        effect: "NoExecute"
+      - key: node.cloudprovider.kubernetes.io/uninitialized
+        value: "true"
+        effect: NoSchedule
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+      - key: node-role.kubernetes.io/control-plane
+        effect: NoSchedule
       serviceAccountName: cloud-controller-manager
       containers:
-        - name: openstack-cloud-controller-manager
-          image: >-
-            registry.k8s.io/provider-os/openstack-cloud-controller-manager:v1.32.0
-          args:
-            - /bin/openstack-cloud-controller-manager
-            - --v=1
-            - --cluster-name=$(CLUSTER_NAME)
-            - --cloud-config=$(CLOUD_CONFIG)
-            - --cloud-provider=openstack
-            - --use-service-account-credentials=false
-            - --bind-address=127.0.0.1
-          volumeMounts:
-            - mountPath: /etc/kubernetes
-              name: k8s
-              readOnly: true
-            - mountPath: /etc/kubernetes/pki
-              name: k8s-certs
-              readOnly: true
-            - mountPath: /etc/ssl/certs
-              name: ca-certs
-              readOnly: true
-          resources:
-            requests:
-              cpu: 200m
-          env:
-            - name: CLOUD_CONFIG
-              value: /etc/kubernetes/cloud.conf
-            - name: CLUSTER_NAME
-              value: kubernetes
-      hostNetwork: true
-      volumes:
-        - hostPath:
-            path: /etc/kubernetes
-            type: DirectoryOrCreate
-          name: k8s
-        - hostPath:
-            path: /etc/kubernetes/pki
-            type: DirectoryOrCreate
+      - name: openstack-cloud-controller-manager
+        image: registry.k8s.io/provider-os/openstack-cloud-controller-manager:v1.33.0
+        args:
+        - /bin/openstack-cloud-controller-manager
+        - --v=1
+        - --cluster-name=$(CLUSTER_NAME)
+        - --cloud-config=$(CLOUD_CONFIG)
+        - --cloud-provider=openstack
+        - --use-service-account-credentials=false
+        - --bind-address=127.0.0.1
+        volumeMounts:
+        - mountPath: /etc/kubernetes/pki
           name: k8s-certs
-        - hostPath:
-            path: /etc/ssl/certs
-            type: DirectoryOrCreate
+          readOnly: true
+        - mountPath: /etc/ssl/certs
           name: ca-certs
+          readOnly: true
+        - mountPath: /etc/config
+          name: cloud-config-volume
+          readOnly: true
+        resources:
+          requests:
+            cpu: 200m
+        env:
+        - name: CLOUD_CONFIG
+          value: /etc/config/cloud.conf
+        - name: CLUSTER_NAME
+          value: kubernetes
+      dnsPolicy: ClusterFirst
+      hostNetwork: true
+      volumes:
+      - hostPath:
+          path: /etc/kubernetes/pki
+          type: DirectoryOrCreate
+        name: k8s-certs
+      - hostPath:
+          path: /etc/ssl/certs
+          type: DirectoryOrCreate
+        name: ca-certs
+      - name: cloud-config-volume
+        secret:
+          secretName: cloud-config
 ---
 # https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/refs/heads/master/manifests/controller-manager/cloud-controller-manager-role-bindings.yaml
+# NOTE! We need to "extract" the List or the CRS will fail to apply.
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
@@ -111,6 +114,7 @@ subjects:
   namespace: kube-system
 ---
 # https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/refs/heads/master/manifests/controller-manager/cloud-controller-manager-roles.yaml
+# NOTE! We need to "extract" the List or the CRS will fail to apply.
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:

test/e2e/data/kustomize/components/cluster-resource-sets/ccm.yaml

@@ -4,6 +4,22 @@ metadata:
   name: ccm-${CLUSTER_NAME}-crs-1
 data: ${CCM_RESOURCES}
 ---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ccm-${CLUSTER_NAME}-crs-1
+type: addons.cluster.x-k8s.io/resource-set
+stringData:
+  cloud-config-secret.yaml: |
+    apiVersion: v1
+    kind: Secret
+    metadata:
+      # This name is referenced in the CCM deployment manifest
+      name: cloud-config
+      namespace: kube-system
+    data:
+      cloud.conf: ${OPENSTACK_CLOUD_PROVIDER_CONF_B64}
+---
 apiVersion: addons.cluster.x-k8s.io/v1beta2
 kind: ClusterResourceSet
 metadata:
@@ -15,4 +31,6 @@ spec:
   resources:
   - kind: ConfigMap
     name: ccm-${CLUSTER_NAME}-crs-1
+  - kind: Secret
+    name: ccm-${CLUSTER_NAME}-crs-1
   strategy: ApplyOnce

test/e2e/data/kustomize/components/cluster-resource-sets/patch-ccm-cloud-config.yaml

@@ -1,14 +1,6 @@
 - op: add
   path: /spec/kubeadmConfigSpec/files
   value: []
-- op: add
-  path: /spec/kubeadmConfigSpec/files/-
-  value:
-    content: ${OPENSTACK_CLOUD_PROVIDER_CONF_B64}
-    encoding: base64
-    owner: root
-    path: /etc/kubernetes/cloud.conf
-    permissions: "0600"
 - op: add
   path: /spec/kubeadmConfigSpec/files/-
   value: