fix external secrets for new cluster (#7883)

* fix external secrets for new cluster

* use workload identity fully with kyverno

* fix typos

Author: upodroid

infra/gcp/terraform/k8s-infra-prow/buckets.tf

@@ -115,6 +115,11 @@ module "prow_bucket" {
       role   = "roles/storage.objectAdmin"
       member = "serviceAccount:prow-build-trusted@k8s-infra-prow-build-trusted.iam.gserviceaccount.com"
     },
+    {
+      // IBM build clusters, pods in the test-pods namespace only
+      role   = "roles/storage.objectAdmin"
+      member = "principalSet://iam.googleapis.com/projects/16065310909/locations/global/workloadIdentityPools/ibm-clusters/attribute.namespace/test-pods"
+    },
     {
       role   = "roles/storage.objectViewer"
       member = "allUsers"

kubernetes/apps/external-secrets.yaml

@@ -13,26 +13,26 @@ spec:
               operator: Exists
   template:
     metadata:
-      name: 'external-secrets-{{ .name }}'
+      name: "external-secrets-{{ .name }}"
     spec:
       destination:
         namespace: external-secrets
         server: "{{ .server }}"
       project: default
       sources:
         - chart: external-secrets
-          repoURL: 'https://charts.external-secrets.io'
-          targetRevision: 0.10.3
+          repoURL: "https://charts.external-secrets.io"
+          targetRevision: 0.14.4
           helm:
             releaseName: external-secrets
             parameters:
               - name: installCRDs
-                value: 'true'
+                value: "true"
               - name: serviceAccount.name
                 value: external-secrets
             valueFiles:
-            - $values/kubernetes/{{ .name }}/helm/external-secrets.yaml
-        - repoURL: 'https://github.com/kubernetes/k8s.io.git'
+              - $values/kubernetes/{{ .name }}/helm/external-secrets.yaml
+        - repoURL: "https://github.com/kubernetes/k8s.io.git"
           targetRevision: main
           ref: values
       syncPolicy:

kubernetes/apps/kustomization.yaml

@@ -5,6 +5,7 @@ resources:
   - atlantis.yaml
   - external-secrets.yaml
   - cert-manager.yaml
+  - kyverno.yaml
   - prow.yaml
   - istio.yaml
   - oauth2-proxy.yaml

kubernetes/apps/kyverno.yaml

@@ -0,0 +1,36 @@
+apiVersion: argoproj.io/v1alpha1
+kind: ApplicationSet
+metadata:
+  name: kyverno
+spec:
+  goTemplate: true
+  generators:
+    - clusters:
+        selector:
+          matchLabels:
+            cloud: ibm
+  template:
+    metadata:
+      name: "kyverno-{{ .name }}"
+    spec:
+      destination:
+        namespace: kyverno
+        server: "{{ .server }}"
+      project: default
+      sources:
+        - chart: kyverno
+          repoURL: "https://kyverno.github.io/kyverno"
+          targetRevision: 3.3.7
+          helm:
+            releaseName: kyverno
+            valueFiles:
+              - $values/kubernetes/{{ .name }}/helm/kyverno.yaml
+        - repoURL: "https://github.com/kubernetes/k8s.io.git"
+          targetRevision: main
+          ref: values
+      syncPolicy:
+        automated:
+          prune: true
+          selfHeal: true
+        syncOptions:
+          - CreateNamespace=true

kubernetes/ibm-ppc64le/helm/external-secrets.yaml

@@ -7,3 +7,47 @@ extraObjects:
       provider:
         gcpsm:
           projectID: k8s-infra-prow-build
+  - apiVersion: v1
+    kind: ConfigMap
+    metadata:
+      name: google-adc
+    data:
+      adc.json: |
+        {
+          "universe_domain": "googleapis.com",
+          "type": "external_account",
+          "audience": "//iam.googleapis.com/projects/16065310909/locations/global/workloadIdentityPools/ibm-clusters/providers/ppc64le",
+          "subject_token_type": "urn:ietf:params:oauth:token-type:jwt",
+          "token_url": "https://sts.googleapis.com/v1/token",
+          "credential_source": {
+            "file": "/var/run/secrets/google-iam-token/serviceaccount/token",
+            "format": {
+              "type": "text"
+            }
+          }
+        }
+
+extraVolumes:
+  - name: google-iam-token
+    projected:
+      defaultMode: 420
+      sources:
+        - serviceAccountToken:
+            audience: sts.googleapis.com
+            expirationSeconds: 86400
+            path: token
+  - name: google-adc
+    configMap:
+      name: google-adc
+
+extraEnv:
+  - name: GOOGLE_APPLICATION_CREDENTIALS
+    value: /etc/google/adc.json
+
+extraVolumeMounts:
+  - mountPath: /var/run/secrets/google-iam-token/serviceaccount
+    name: google-iam-token
+    readOnly: true
+  - mountPath: /etc/google
+    name: google-adc
+    readOnly: true

kubernetes/ibm-ppc64le/prow/kustomization.yaml

@@ -7,6 +7,8 @@ resources:
   - boskos-reaper.yaml
   - boskos-resources-configmap.yaml
   - boskos.yaml
-  - build-serviceaccounts.yaml
+  - kyverno.yaml
   - limit-range.yaml
+  - secrets.yaml
+  - oidc.yaml
   - test-pods-poddisruptionbudget.yaml

kubernetes/ibm-ppc64le/prow/kyverno.yaml

@@ -0,0 +1,52 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: insert-gcp-credentials
+spec:
+  rules:
+    - name: add-creds
+      match:
+        any:
+          - resources:
+              kinds:
+                - Pod
+      preconditions:
+        any:
+          - key: '{{request.object.metadata.labels."created-by-prow"}}'
+            operator: Equals
+            value: "true"
+      mutate:
+        patchStrategicMerge:
+          spec:
+            initContainers:
+              # pod order matters
+              - name: clonerefs
+              - (name): "initupload"
+                # prow passes the json path directly, uncomment this once the feature is disabled in prow
+                # env:
+                #   - name: GOOGLE_APPLICATION_CREDENTIALS
+                #     value: /secrets/gcs/service-account.json
+                volumeMounts:
+                  - mountPath: /var/run/secrets/google-iam-token/serviceaccount
+                    name: google-iam-token
+                    readOnly: true
+            containers:
+              - name: test
+              - (name): sidecar
+                # prow passes the json path directly, uncomment this once the feature is disabled in prow
+                # env:
+                #   - name: GOOGLE_APPLICATION_CREDENTIALS
+                #     value: /secrets/gcs/service-account.json
+                volumeMounts:
+                  - mountPath: /var/run/secrets/google-iam-token/serviceaccount
+                    name: google-iam-token
+                    readOnly: true
+            volumes:
+              - name: google-iam-token
+                projected:
+                  defaultMode: 420
+                  sources:
+                    - serviceAccountToken:
+                        audience: sts.googleapis.com
+                        expirationSeconds: 86400
+                        path: token

kubernetes/ibm-ppc64le/prow/secrets.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: service-account
+  namespace: test-pods
+stringData:
+  service-account.json: |
+    {
+      "universe_domain": "googleapis.com",
+      "type": "external_account",
+      "audience": "//iam.googleapis.com/projects/16065310909/locations/global/workloadIdentityPools/ibm-clusters/providers/ppc64le",
+      "subject_token_type": "urn:ietf:params:oauth:token-type:jwt",
+      "token_url": "https://sts.googleapis.com/v1/token",
+      "credential_source": {
+        "file": "/var/run/secrets/google-iam-token/serviceaccount/token",
+        "format": {
+          "type": "text"
+        }
+      }
+    }