feat: add sidecar container (#389)

mikhail-klimko · web-flow · commit c0d62b9f6b05 · 2023-08-31T17:01:31.000+03:00
diff --git a/charts/cf-runtime/.ci/values-ci.yaml b/charts/cf-runtime/.ci/values-ci.yaml
@@ -17,7 +17,9 @@ monitor:
   enabled: true
   rbac:
     namespaced: true
-runner: {}
+runner:
+  sidecar:
+    enabled: true
 runtime:
   dind:
     image:
diff --git a/charts/cf-runtime/Chart.yaml b/charts/cf-runtime/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v2
 description: A Helm chart for Codefresh Runner
 name: cf-runtime
-version: 6.0.1
+version: 6.1.0
 keywords:
   - codefresh
   - runner
@@ -14,8 +14,8 @@ maintainers:
     url: https://codefresh-io.github.io/
 annotations:
   artifacthub.io/changes: |
-    - kind: fixed
-      description: Fix imageRegistry prefix for runtime images
+    - kind: added
+      description: Add optional sidecar container in runner to reconcile runtime-environment spec from Codefresh API
 dependencies:
   - name: cf-common
     repository: https://chartmuseum.codefresh.io/cf-common
diff --git a/charts/cf-runtime/README.md b/charts/cf-runtime/README.md
@@ -1,6 +1,6 @@
 ## Codefresh Runner
 
-![Version: 6.0.1](https://img.shields.io/badge/Version-6.0.1-informational?style=flat-square)
+![Version: 6.1.0](https://img.shields.io/badge/Version-6.1.0-informational?style=flat-square)
 
 Helm chart for deploying [Codefresh Runner](https://codefresh.io/docs/docs/installation/codefresh-runner/) to Kubernetes.
 
@@ -235,6 +235,16 @@ global:
   runtimeName: "my-cluster-name/my-namespace" # optional
 ```
 
+> **Note!** Though it's still possible to update runtime-environment via [get](https://codefresh-io.github.io/cli/runtime-environments/get-runtime-environments/) and [patch](https://codefresh-io.github.io/cli/runtime-environments/apply-runtime-environments/) commands, it's recommended to enable sidecar container to pull runtime spec from Codefresh API to detect any drift in configuration.
+
+```yaml
+runner:
+  # -- Sidecar container
+  # Reconciles runtime spec from Codefresh API for drift detection
+  sidecar:
+    enabled: true
+```
+
 ## Architecture
 
 [Codefresh Runner architecture](https://codefresh.io/docs/docs/installation/codefresh-runner/#codefresh-runner-architecture)
@@ -886,6 +896,7 @@ Go to [https://<YOUR_ONPREM_DOMAIN_HERE>/admin/runtime-environments/system](http
 | runner.enabled | bool | `true` | Enable the runner |
 | runner.env | object | `{}` | Add additional env vars |
 | runner.image | object | `{"registry":"quay.io","repository":"codefresh/venona","tag":"1.9.17"}` | Set image |
+| runner.init | object | `{"image":{"registry":"quay.io","repository":"codefresh/cli","tag":"0.85.0-rootless"},"resources":{"limits":{"cpu":"1","memory":"512Mi"},"requests":{"cpu":"0.2","memory":"256Mi"}}}` | Init container |
 | runner.nodeSelector | object | `{}` | Set node selector |
 | runner.podAnnotations | object | `{}` | Set pod annotations |
 | runner.podSecurityContext | object | See below | Set security context for the pod |
@@ -899,6 +910,7 @@ Go to [https://<YOUR_ONPREM_DOMAIN_HERE>/admin/runtime-environments/system](http
 | runner.serviceAccount.annotations | object | `{}` | Additional service account annotations |
 | runner.serviceAccount.create | bool | `true` | Create service account |
 | runner.serviceAccount.name | string | `""` | Override service account name |
+| runner.sidecar | object | `{"enabled":false,"env":{"RECONCILE_INTERVAL":300},"image":{"registry":"quay.io","repository":"codefresh/codefresh-shell","tag":"0.0.2"},"resources":{}}` | Sidecar container Reconciles runtime spec from Codefresh API for drift detection |
 | runner.tolerations | list | `[]` | Set tolerations |
 | runner.updateStrategy | object | `{"type":"RollingUpdate"}` | Upgrade strategy |
 | runtime | object | See below | Set runtime parameters |
diff --git a/charts/cf-runtime/README.md.gotmpl b/charts/cf-runtime/README.md.gotmpl
@@ -235,6 +235,16 @@ global:
   runtimeName: "my-cluster-name/my-namespace" # optional
 ```
 
+> **Note!** Though it's still possible to update runtime-environment via [get](https://codefresh-io.github.io/cli/runtime-environments/get-runtime-environments/) and [patch](https://codefresh-io.github.io/cli/runtime-environments/apply-runtime-environments/) commands, it's recommended to enable sidecar container to pull runtime spec from Codefresh API to detect any drift in configuration.
+
+```yaml
+runner:
+  # -- Sidecar container
+  # Reconciles runtime spec from Codefresh API for drift detection
+  sidecar:
+    enabled: true
+```
+
 ## Architecture
 
 [Codefresh Runner architecture](https://codefresh.io/docs/docs/installation/codefresh-runner/#codefresh-runner-architecture)
diff --git a/charts/cf-runtime/files/reconcile-runtime.sh b/charts/cf-runtime/files/reconcile-runtime.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+echo "-----"
+echo "API_HOST:             ${API_HOST}"
+echo "KUBE_CONTEXT:         ${KUBE_CONTEXT}"
+echo "KUBE_NAMESPACE:       ${KUBE_NAMESPACE}"
+echo "OWNER_NAME:           ${OWNER_NAME}"
+echo "RUNTIME_NAME:         ${RUNTIME_NAME}"
+echo "CONFIGMAP_NAME:       ${CONFIGMAP_NAME}"
+echo "RECONCILE_INTERVAL:   ${RECONCILE_INTERVAL}"
+echo "-----"
+
+msg() { echo -e "\e[32mINFO ---> $1\e[0m"; }
+err() { echo -e "\e[31mERR ---> $1\e[0m" ; return 1; }
+
+
+if [ -z "${USER_CODEFRESH_TOKEN}" ]; then
+    err "missing codefresh user token. must supply \".global.codefreshToken\" if agent-codefresh-token does not exist"
+    exit 1
+fi
+
+codefresh auth create-context --api-key ${USER_CODEFRESH_TOKEN} --url ${API_HOST}
+
+while true; do
+  msg "Reconciling ${RUNTIME_NAME} runtime"
+
+  sleep $RECONCILE_INTERVAL
+
+  codefresh get re \
+      --name ${RUNTIME_NAME} \
+      -o yaml \
+      | yq 'del(.version, .metadata.changedBy, .metadata.creationTime)' > /tmp/runtime.yaml
+
+  sed -i "s/'/\"/g" /tmp/runtime.yaml
+
+  kubectl get cm ${CONFIGMAP_NAME} -n ${KUBE_NAMESPACE} -o yaml \
+  | yq 'del(.metadata.resourceVersion, .metadata.uid)' \
+  | yq eval '.data["runtime.yaml"] = load_str("/tmp/runtime.yaml")' \
+  | kubectl apply -f -
+done
diff --git a/charts/cf-runtime/templates/_components/runner/_deployment.yaml b/charts/cf-runtime/templates/_components/runner/_deployment.yaml
@@ -69,6 +69,23 @@ spec:
         volumeMounts:
           {{- toYaml . | nindent 8 }}
         {{- end }}
+      {{- if .Values.sidecar.enabled }}
+      - name: reconcile-runtime
+        image: {{ include (printf "%s.image.name" $cfCommonTplSemver ) (dict "image" .Values.sidecar.image "context" .) }}
+        imagePullPolicy: {{ .Values.sidecar.image.pullPolicy | default "IfNotPresent" }}
+        command:
+        - /bin/bash
+        args:
+        - -ec
+        - |
+          {{ .Files.Get "files/reconcile-runtime.sh" | nindent 10 }}
+        env:
+        {{- include "runner-sidecar.environment-variables" . | nindent 8 }}
+        {{- with .Values.sidecar.resources }}
+        resources:
+          {{- toYaml . | nindent 10  }}
+        {{- end }}
+      {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
diff --git a/charts/cf-runtime/templates/_components/runner/_rbac.yaml b/charts/cf-runtime/templates/_components/runner/_rbac.yaml
@@ -25,7 +25,7 @@ rules:
     verbs: [ "get", "create", "delete" ]
   - apiGroups: [ "" ]
     resources: [ "configmaps", "secrets" ]
-    verbs: [ "get", "create" ]
+    verbs: [ "get", "create", "update", patch ]
   - apiGroups: [ "apps" ]
     resources: [ "deployments" ]
     verbs: [ "get" ]
diff --git a/charts/cf-runtime/templates/_components/runner/environment-variables/_sidecar-container.yaml b/charts/cf-runtime/templates/_components/runner/environment-variables/_sidecar-container.yaml
@@ -0,0 +1,22 @@
+{{- define "runner-sidecar.environment-variables.defaults" }}
+HOME: /tmp
+{{- end }}
+
+{{- define "runner-sidecar.environment-variables.calculated" }}
+API_HOST: {{ include "runtime.runtime-environment-spec.codefresh-host" . }}
+USER_CODEFRESH_TOKEN: {{ include "runtime.installation-token-env-var-value" . | nindent 2 }}
+KUBE_CONTEXT: {{ include "runtime.runtime-environment-spec.context-name" . }}
+KUBE_NAMESPACE: {{ .Release.Namespace }}
+OWNER_NAME: {{ include "runner.fullname" . }}
+RUNTIME_NAME: {{ include "runtime.runtime-environment-spec.runtime-name" . }}
+CONFIGMAP_NAME: {{ printf "%s-%s" (include "runtime.fullname" .) "spec" }}
+{{- end }}
+
+{{- define "runner-sidecar.environment-variables" }}
+  {{- $cfCommonTplSemver := printf "cf-common-%s" (index .Subcharts "cf-common").Chart.Version }}
+  {{- $defaults := (include "runner-sidecar.environment-variables.defaults" . | fromYaml) }}
+  {{- $calculated := (include "runner-sidecar.environment-variables.calculated" . | fromYaml) }}
+  {{- $overrides := .Values.sidecar.env }}
+  {{- $mergedValues := mergeOverwrite (merge $defaults $calculated) $overrides }}
+  {{- include (printf "%s.env-vars" $cfCommonTplSemver) (dict "Values" $mergedValues "context" .) }}
+{{- end }}
diff --git a/charts/cf-runtime/templates/hooks/post-install/cm-update-runtime.yaml b/charts/cf-runtime/templates/hooks/post-install/cm-update-runtime.yaml
@@ -13,6 +13,6 @@ metadata:
       {{- toYaml . | nindent 4 }}
     {{- end }}
 data:
-  runtime.yaml: |-
-    {{ include "runtime.runtime-environment-spec.template" . | nindent 4 }}
+  runtime.yaml: |
+    {{ include "runtime.runtime-environment-spec.template" . | nindent 4 | trim }}
 {{- end }}
diff --git a/charts/cf-runtime/templates/runtime/runtime-env-spec-tmpl.yaml b/charts/cf-runtime/templates/runtime/runtime-env-spec-tmpl.yaml
@@ -8,17 +8,10 @@
 metadata:
   name: {{ include "runtime.runtime-environment-spec.runtime-name" . }}
   agent: {{ .Values.runtime.agent }}
-extends: {{- toYaml .Values.runtime.runtimeExtends | nindent 2 }}
-{{- with .Values.runtime.description }}
-description: {{ . }}
-{{- end }}
-{{- if not .Values.runtime.agent }}
-accounts: {{- toYaml .Values.runtime.accounts | nindent 2 }}
-{{- end }}
 runtimeScheduler:
   type: KubernetesPod
   {{- if $engineContext.image }}
-  image: {{ include (printf "%s.image.name" $cfCommonTplSemver ) (dict "image" $engineContext.image "context" .) }}
+  image: {{ include (printf "%s.image.name" $cfCommonTplSemver ) (dict "image" $engineContext.image "context" .) | quote }}
   {{- end }}
   {{- with $engineContext.command }}
   command: {{- toYaml . | nindent 4 }}
@@ -27,19 +20,19 @@ runtimeScheduler:
   {{- with $engineContext.env }}
    {{- toYaml . | nindent 4 }}
   {{- end }}
-    COMPOSE_IMAGE: '{{ include "runtime.runtimeImageName" (dict "registry" $imageRegistry "imageFullName" $engineContext.runtimeImages.COMPOSE_IMAGE) }}'
-    CONTAINER_LOGGER_IMAGE: '{{ include "runtime.runtimeImageName" (dict "registry" $imageRegistry "imageFullName" $engineContext.runtimeImages.CONTAINER_LOGGER_IMAGE) }}'
-    DOCKER_BUILDER_IMAGE: '{{ include "runtime.runtimeImageName" (dict "registry" $imageRegistry "imageFullName" $engineContext.runtimeImages.DOCKER_BUILDER_IMAGE) }}'
-    DOCKER_PULLER_IMAGE: '{{ include "runtime.runtimeImageName" (dict "registry" $imageRegistry "imageFullName" $engineContext.runtimeImages.DOCKER_PULLER_IMAGE) }}'
-    DOCKER_PUSHER_IMAGE: '{{ include "runtime.runtimeImageName" (dict "registry" $imageRegistry "imageFullName" $engineContext.runtimeImages.DOCKER_PUSHER_IMAGE) }}'
-    DOCKER_TAG_PUSHER_IMAGE: '{{ include "runtime.runtimeImageName" (dict "registry" $imageRegistry "imageFullName" $engineContext.runtimeImages.DOCKER_TAG_PUSHER_IMAGE) }}'
-    FS_OPS_IMAGE: '{{ include "runtime.runtimeImageName" (dict "registry" $imageRegistry "imageFullName" $engineContext.runtimeImages.FS_OPS_IMAGE) }}'
-    GIT_CLONE_IMAGE: '{{ include "runtime.runtimeImageName" (dict "registry" $imageRegistry "imageFullName" $engineContext.runtimeImages.GIT_CLONE_IMAGE) }}'
-    KUBE_DEPLOY: '{{ include "runtime.runtimeImageName" (dict "registry" $imageRegistry "imageFullName" $engineContext.runtimeImages.KUBE_DEPLOY) }}'
-    PIPELINE_DEBUGGER_IMAGE: '{{ include "runtime.runtimeImageName" (dict "registry" $imageRegistry "imageFullName" $engineContext.runtimeImages.PIPELINE_DEBUGGER_IMAGE) }}'
-    TEMPLATE_ENGINE: '{{ include "runtime.runtimeImageName" (dict "registry" $imageRegistry "imageFullName" $engineContext.runtimeImages.TEMPLATE_ENGINE) }}'
+    COMPOSE_IMAGE: {{ include "runtime.runtimeImageName" (dict "registry" $imageRegistry "imageFullName" $engineContext.runtimeImages.COMPOSE_IMAGE) | quote }}
+    CONTAINER_LOGGER_IMAGE: {{ include "runtime.runtimeImageName" (dict "registry" $imageRegistry "imageFullName" $engineContext.runtimeImages.CONTAINER_LOGGER_IMAGE) | quote }}
+    DOCKER_BUILDER_IMAGE: {{ include "runtime.runtimeImageName" (dict "registry" $imageRegistry "imageFullName" $engineContext.runtimeImages.DOCKER_BUILDER_IMAGE) | quote }}
+    DOCKER_PULLER_IMAGE: {{ include "runtime.runtimeImageName" (dict "registry" $imageRegistry "imageFullName" $engineContext.runtimeImages.DOCKER_PULLER_IMAGE) | quote }}
+    DOCKER_PUSHER_IMAGE: {{ include "runtime.runtimeImageName" (dict "registry" $imageRegistry "imageFullName" $engineContext.runtimeImages.DOCKER_PUSHER_IMAGE) | quote }}
+    DOCKER_TAG_PUSHER_IMAGE: {{ include "runtime.runtimeImageName" (dict "registry" $imageRegistry "imageFullName" $engineContext.runtimeImages.DOCKER_TAG_PUSHER_IMAGE) | quote }}
+    FS_OPS_IMAGE: {{ include "runtime.runtimeImageName" (dict "registry" $imageRegistry "imageFullName" $engineContext.runtimeImages.FS_OPS_IMAGE) | quote }}
+    GIT_CLONE_IMAGE: {{ include "runtime.runtimeImageName" (dict "registry" $imageRegistry "imageFullName" $engineContext.runtimeImages.GIT_CLONE_IMAGE) | quote }}
+    KUBE_DEPLOY: {{ include "runtime.runtimeImageName" (dict "registry" $imageRegistry "imageFullName" $engineContext.runtimeImages.KUBE_DEPLOY) | quote }}
+    PIPELINE_DEBUGGER_IMAGE: {{ include "runtime.runtimeImageName" (dict "registry" $imageRegistry "imageFullName" $engineContext.runtimeImages.PIPELINE_DEBUGGER_IMAGE) | quote }}
+    TEMPLATE_ENGINE: {{ include "runtime.runtimeImageName" (dict "registry" $imageRegistry "imageFullName" $engineContext.runtimeImages.TEMPLATE_ENGINE) | quote }}
   {{- with $engineContext.userEnvVars }}
-  userEnvVars:  {{- toYaml . | nindent 2 }}
+  userEnvVars:  {{- toYaml . | nindent 4 }}
   {{- end }}
   {{- with $engineContext.workflowLimits }}
   workflowLimits: {{ toYaml . | nindent 4 }}
@@ -82,7 +75,7 @@ runtimeScheduler:
 dockerDaemonScheduler:
   type: DindKubernetesPod
   {{- if $dindContext.image }}
-  dindImage: {{ include (printf "%s.image.name" $cfCommonTplSemver ) (dict "image" $dindContext.image "context" .) }}
+  dindImage: {{ include (printf "%s.image.name" $cfCommonTplSemver ) (dict "image" $dindContext.image "context" .) | quote }}
   {{- end }}
   {{- with $dindContext.userAccess }}
   userAccess: {{ . }}
@@ -125,7 +118,10 @@ dockerDaemonScheduler:
   pvcs:
   {{- range $index, $pvc := $dindContext.pvcs }}
     - name: {{ $pvc.name }}
-  {{- include (printf "%v.tplrender" $cfCommonTplSemver) (dict "Values" (omit $pvc "name" ) "context" $) | nindent 6 }}
+      reuseVolumeSelector: {{ $pvc.reuseVolumeSelector | quote }}
+      reuseVolumeSortOrder: {{ $pvc.reuseVolumeSortOrder }}
+      storageClassName: {{ include (printf "%v.tplrender" $cfCommonTplSemver) (dict "Values" $pvc.storageClassName "context" $) }}
+      volumeSize: {{ $pvc.volumeSize }}
   {{- end }}
   {{- end }}
   defaultDindResources:
@@ -159,4 +155,16 @@ dockerDaemonScheduler:
       secret:
         secretName: codefresh-certs-server
   {{- end }}
+extends: {{- toYaml .Values.runtime.runtimeExtends | nindent 2 }}
+  {{- if .Values.runtime.description }}
+description: {{ .Values.runtime.description }}
+  {{- else }}
+description: null
+  {{- end }}
+{{- if .Values.global.accountId }}
+accountId: {{ .Values.global.accountId }}
+{{- end }}
+{{- if not .Values.runtime.agent }}
+accounts: {{- toYaml .Values.runtime.accounts | nindent 2 }}
+{{- end }}
 {{- end }}
diff --git a/charts/cf-runtime/tests/private-registry/private_registry_test.yaml b/charts/cf-runtime/tests/private-registry/private_registry_test.yaml
@@ -23,33 +23,31 @@ tests:
   - it: Test private registry in runtime spec
     template: templates/hooks/post-install/cm-update-runtime.yaml
     asserts:
-      - equal:
+      - matchRegex:
           path: data["runtime.yaml"]
-          value: |-
+          pattern: |
             metadata:
               name: my-context/codefresh
               agent: true
-            extends:
-              - system/default/hybrid/k8s_low_limits
             runtimeScheduler:
               type: KubernetesPod
-              image: somedomain.io/codefresh/engine:tagoverride
+              image: "somedomain.io/codefresh/engine:tagoverride"
               command:
                 - npm
                 - run
                 - start
               envVars:
-                COMPOSE_IMAGE: 'somedomain.io/codefresh/compose:tagoverride'
-                CONTAINER_LOGGER_IMAGE: 'somedomain.io/codefresh/cf-container-logger:tagoverride'
-                DOCKER_BUILDER_IMAGE: 'somedomain.io/codefresh/cf-docker-builder:tagoverride'
-                DOCKER_PULLER_IMAGE: 'somedomain.io/codefresh/cf-docker-puller:tagoverride'
-                DOCKER_PUSHER_IMAGE: 'somedomain.io/codefresh/cf-docker-pusher:tagoverride'
-                DOCKER_TAG_PUSHER_IMAGE: 'somedomain.io/codefresh/cf-docker-tag-pusher:tagoverride'
-                FS_OPS_IMAGE: 'somedomain.io/codefresh/fs-ops:tagoverride'
-                GIT_CLONE_IMAGE: 'somedomain.io/codefresh/cf-git-cloner:tagoverride'
-                KUBE_DEPLOY: 'somedomain.io/codefresh/cf-deploy-kubernetes:tagoverride'
-                PIPELINE_DEBUGGER_IMAGE: 'somedomain.io/codefresh/cf-debugger:tagoverride'
-                TEMPLATE_ENGINE: 'somedomain.io/codefresh/pikolo:tagoverride'
+                COMPOSE_IMAGE: "somedomain.io/codefresh/compose:tagoverride"
+                CONTAINER_LOGGER_IMAGE: "somedomain.io/codefresh/cf-container-logger:tagoverride"
+                DOCKER_BUILDER_IMAGE: "somedomain.io/codefresh/cf-docker-builder:tagoverride"
+                DOCKER_PULLER_IMAGE: "somedomain.io/codefresh/cf-docker-puller:tagoverride"
+                DOCKER_PUSHER_IMAGE: "somedomain.io/codefresh/cf-docker-pusher:tagoverride"
+                DOCKER_TAG_PUSHER_IMAGE: "somedomain.io/codefresh/cf-docker-tag-pusher:tagoverride"
+                FS_OPS_IMAGE: "somedomain.io/codefresh/fs-ops:tagoverride"
+                GIT_CLONE_IMAGE: "somedomain.io/codefresh/cf-git-cloner:tagoverride"
+                KUBE_DEPLOY: "somedomain.io/codefresh/cf-deploy-kubernetes:tagoverride"
+                PIPELINE_DEBUGGER_IMAGE: "somedomain.io/codefresh/cf-debugger:tagoverride"
+                TEMPLATE_ENGINE: "somedomain.io/codefresh/pikolo:tagoverride"
               cluster:
                 namespace: codefresh
                 serviceAccount: codefresh-engine
@@ -65,7 +63,7 @@ tests:
                   memory: 128Mi
             dockerDaemonScheduler:
               type: DindKubernetesPod
-              dindImage: somedomain.io/codefresh/dind:tagoverride
+              dindImage: "somedomain.io/codefresh/dind:tagoverride"
               userAccess: true
               cluster:
                 namespace: codefresh
@@ -75,15 +73,19 @@ tests:
                   selector: my-context
               pvcs:
                 - name: dind
-                  reuseVolumeSelector: codefresh-app,io.codefresh.accountName
+                  reuseVolumeSelector: "codefresh-app,io.codefresh.accountName"
                   reuseVolumeSortOrder: pipeline_id
-                  storageClassName: 'dind-local-volumes-runner-codefresh'
+                  storageClassName: dind-local-volumes-runner-codefresh
                   volumeSize: 16Gi
               defaultDindResources:
                 limits:
                   cpu: 400m
                   memory: 800Mi
                 requests: null
+            extends:
+              - system/default/hybrid/k8s_low_limits
+            description: null
+            accountId: 7890
 
   - it: Test private registy in runner
     template: templates/runner/deployment.yaml
diff --git a/charts/cf-runtime/tests/runtime/runtime_onprem_test.yaml b/charts/cf-runtime/tests/runtime/runtime_onprem_test.yaml
diff --git a/charts/cf-runtime/tests/runtime/runtime_test.yaml b/charts/cf-runtime/tests/runtime/runtime_test.yaml
diff --git a/charts/cf-runtime/tests/runtime/runtime_values.yaml b/charts/cf-runtime/tests/runtime/runtime_values.yaml
diff --git a/charts/cf-runtime/values.yaml b/charts/cf-runtime/values.yaml
