feat: adding configurations for grpc

Signed-off-by: John Zakrzewski <Jozakrzewski@microsoft.com>

Author: John Zakrzewski

chart/Chart.yaml

@@ -48,11 +48,6 @@ dependencies:
     version: 4.2.0
     repository: https://openebs.github.io/dynamic-localpv-provisioner
     condition: localpv-provisioner.enabled
-  - name: cert-manager
-    version: v1.12.10
-    repository: https://charts.jetstack.io
-    alias: cert-manager
-    condition: cert-manager.enabled
 annotations:
   helm.sh/images: |
     - name: bats

chart/README.md

@@ -56,7 +56,6 @@ This removes all the Kubernetes components associated with the chart and deletes
 |------------|------|---------|
 |  | crds | 0.0.0 |
 | https://charts.bitnami.com/bitnami | etcd | 8.6.0 |
-| https://charts.jetstack.io | cert-manager(cert-manager) | v1.17.0 |
 | https://grafana.github.io/helm-charts | loki-stack | 2.9.11 |
 | https://jaegertracing.github.io/helm-charts | jaeger-operator | 2.50.1 |
 | https://nats-io.github.io/k8s/helm/charts/ | nats | 0.19.14 |
@@ -120,7 +119,7 @@ This removes all the Kubernetes components associated with the chart and deletes
 | base.​logging.​format | Valid values for format are pretty, json and compact | `"pretty"` |
 | base.​logging.​silenceLevel | Silence specific module components | `nil` |
 | base.​metrics.​enabled | Enable the metrics exporter | `true` |
-| cert-manager.​enabled | Enable cert-manager only if tls is enabled | `true` |
+| cert-manager.​enabled | Enable cert-manager only if tls is enabled | `false` |
 | crds.​csi.​volumeSnapshots.​enabled | Install Volume Snapshot CRDs | `true` |
 | crds.​enabled | Disables the installation of all CRDs if set to false | `true` |
 | csi.​controller.​logLevel | Log level for the csi controller | `"info"` |

chart/templates/hooks/wait-for-cert-manager.yaml

@@ -106,7 +106,45 @@ spec:
                 issuerRef:
                   name: ca-issuer
                   kind: Issuer
-                secretName: api-rest-tls
+                secretName: rest-api-server-cert
+              EOF
+              echo "Creating core-agent-server certificate..."
+              kubectl apply -f - <<EOF
+              apiVersion: cert-manager.io/v1
+              kind: Certificate
+              metadata:
+                name: agent-core-server
+                namespace: {{ .Release.Namespace }}
+              spec:
+                duration: 175200h
+                isCA: false
+                dnsNames:
+                  - {{ .Release.Name }}-agent-core.{{ .Release.Namespace }}.svc
+                  - {{ .Release.Name }}-agent-core.{{ .Release.Namespace }}.svc.cluster.local
+                  - {{ .Release.Name }}-agent-core
+                issuerRef:
+                  name: ca-issuer
+                  kind: Issuer
+                secretName: agent-core-server-cert
+              EOF
+              echo "Creating io-engine-server certificate..."
+              kubectl apply -f - <<EOF
+              apiVersion: cert-manager.io/v1
+              kind: Certificate
+              metadata:
+                name: io-engine-server
+                namespace: {{ .Release.Namespace }}
+              spec:
+                duration: 175200h
+                isCA: false
+                dnsNames:
+                  - {{ .Release.Name }}-io-engine.{{ .Release.Namespace }}.svc
+                  - {{ .Release.Name }}-io-engine.{{ .Release.Namespace }}.svc.cluster.local
+                  - {{ .Release.Name }}-io-engine
+                issuerRef:
+                  name: ca-issuer
+                  kind: Issuer
+                secretName: io-engine-server-cert
               EOF
       restartPolicy: OnFailure
 {{- end }}

chart/templates/mayastor/agents/core/agent-core-deployment.yaml

@@ -69,6 +69,11 @@ spec:
             - "--max-rebuilds={{ .Values.agents.core.rebuild.maxConcurrent }}"{{ end }}
             {{- if eq ((.Values.agents.core.rebuild).partial).enabled false }}
             - "--disable-partial-rebuild"{{ end }}
+            {{- if .Values.tls.enabled }}
+            - --tls-server-cert-path=/etc/cert/tls.crt
+            - --tls-server-key-path=/etc/cert/tls.key
+            - --tls-client-ca-path=/etc/client_cert/ca.crt # CA cert for client verification with io-engine
+            {{- end }}
           ports:
             - containerPort: 50051
           env:
@@ -86,6 +91,16 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+          {{- if .Values.tls.enabled }}
+          volumeMounts:
+            - name: agent-core-server-cert
+              mountPath: /etc/cert
+              readOnly: true
+            - name: ca-cert
+              mountPath:  /etc/client_cert/ca.crt
+              subPath: ca.crt
+              readOnly: true
+          {{- end }}
         - name: agent-ha-cluster
           resources:
             limits:
@@ -94,12 +109,17 @@ spec:
             requests:
               cpu: {{ .Values.agents.ha.cluster.resources.requests.cpu | quote }}
               memory: {{ .Values.agents.ha.cluster.resources.requests.memory | quote }}
-          image: "{{ .Values.image.registry }}/{{ .Values.image.repo }}/{{ include "image_prefix" . }}-agent-ha-cluster:{{ default .Values.image.tag .Values.image.repoTags.controlPlane }}"
+          image: "mayastor.azurecr.io/johnaius/mayastor-agent-ha-cluster:grpc"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           args:
             - "-g=[::]:50052"
             - "--store=http://{{ include "etcdUrl" . }}"
-            - "--core-grpc=https://{{ .Release.Name }}-agent-core:50051"{{ if .Values.base.jaeger.enabled }}
+            {{- if .Values.tls.enabled }}
+            - "--core-grpc=https://{{ .Release.Name }}-agent-core:50051"
+            {{- else }}
+            - "--core-grpc=http://{{ .Release.Name }}-agent-core:50051"
+            {{- end }}
+            {{ if .Values.base.jaeger.enabled }}
             - "--jaeger={{ include "jaeger_url" . }}"{{ end }}{{ if .Values.eventing.enabled }}
             - "--events-url=nats://{{ .Release.Name }}-nats:4222"{{ end }}
             - "--ansi-colors={{ .Values.base.logging.color }}"
@@ -121,3 +141,13 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+      {{- if .Values.tls.enabled }}
+      volumes:
+        - name: agent-core-server-cert
+          secret:
+            secretName: agent-core-server-cert
+        - name: ca-cert
+          secret:
+            defaultMode: 420
+            secretName: ca-root-cert
+      {{- end }}

chart/templates/mayastor/apis/api-rest-deployment.yaml

@@ -49,7 +49,11 @@ spec:
             - "--http=[::]:8081"
             - "--request-timeout={{ .Values.base.default_req_timeout }}"{{ if .Values.base.jaeger.enabled }}
             - "--jaeger={{ include "jaeger_url" . }}"{{ end }}
+            {{- if .Values.tls.enabled }}
             - "--core-grpc=https://{{ .Release.Name }}-agent-core:50051"
+            {{- else }}
+            - "--core-grpc=http://{{ .Release.Name }}-agent-core:50051"
+            {{- end }}
             - "--ansi-colors={{ .Values.base.logging.color }}"
             - "--fmt-style={{ include "logFormat" . }}"
             {{- if .Values.apis.rest.healthProbes.readiness.enabled }}
@@ -60,16 +64,11 @@ spec:
           {{- else }}
             - --cert-file=/etc/tls/tls.crt
             - --key-file=/etc/tls/tls.key
-            # - --tls-client-ca-path=/etc/client_cert/ca.crt # CA cert for client verification with core-agent
+            - --tls-client-ca-path=/etc/tls/ca.crt # CA cert for client verification with core-agent
           volumeMounts:
             - name: certs
               mountPath: /etc/tls
               readOnly: true
-            # - name: ca-cert
-            #   mountPath: /etc/client_cert/ca.crt
-            #   subPath: ca.crt
-            #   readOnly: true
-
           {{- end }}
           ports:
             - containerPort: 8080
@@ -105,9 +104,5 @@ spec:
       volumes:
         - name: certs
           secret:
-            secretName: api-rest-tls
-        # - name: ca-cert
-        #   secret:
-        #     defaultMode: 420
-        #     secretName: agent-core-server-cert
+            secretName: rest-api-server-cert
       {{- end }}

chart/templates/mayastor/csi/csi-controller-deployment.yaml

@@ -155,5 +155,5 @@ spec:
         - name: ca-cert
           secret:
             defaultMode: 420
-            secretName: api-rest-tls
+            secretName: ca-root-cert
         {{- end }}

chart/templates/mayastor/csi/csi-node-daemonset.yaml

@@ -107,10 +107,10 @@ spec:
         - "--fmt-style={{ include "logFormat" . }}"
         - "--ansi-colors={{ .Values.base.logging.color }}"
         {{- if .Values.tls.enabled }}
-        - "--endpoint=https://{{ .Release.Name }}-api-rest:8080"
+        - "--rest-endpoint=https://{{ .Release.Name }}-api-rest:8080"
         - "--tls-client-ca-path=/etc/client_cert/ca.crt" # CA cert for client verification with rest
         {{- else }}
-        - "--endpoint=http://{{ .Release.Name }}-api-rest:8081"
+        - "--rest-endpoint=http://{{ .Release.Name }}-api-rest:8081"
         {{- end }}
         command:
         - csi-node
@@ -191,5 +191,5 @@ spec:
       - name: ca-cert
         secret:
           defaultMode: 420
-          secretName: api-rest-tls
+          secretName: ca-root-cert
       {{- end }}

chart/templates/mayastor/io/io-engine-daemonset.yaml

@@ -100,7 +100,11 @@ spec:
         #       If you use 2 CPUs, the CPU: field should also read 2.
         - "--grpc-ip=$(MY_POD_IP)"
         - "-N$(MY_NODE_NAME)"
+        {{- if .Values.tls.enabled }}
         - "-Rhttps://{{ .Release.Name }}-agent-core:50051"
+        {{- else }}
+        - "-Rhttp://{{ .Release.Name }}-agent-core:50051"
+        {{- end }}
         - "-y/var/local/{{ .Release.Name }}/io-engine/config.yaml"
         - "-l{{ include "cpuFlag" . }}"
         - "-p={{ include "etcdUrl" . }}"{{ if .Values.io_engine.target.nvmf.ptpl }}
@@ -113,6 +117,11 @@ spec:
         - "--tgt-crdt={{ .Values.io_engine.target.nvmf.hostCmdRetryDelay.crdt1 }}"{{ if .Values.eventing.enabled }}
         - "--events-url=nats://{{ .Release.Name }}-nats:4222"{{ end }}
         - "--ps-retries={{ default 300 .Values.io_engine.pstorRetries }}"
+        {{- if .Values.tls.enabled }}
+        - "--tls-server-key-path=/etc/server_cert/tls.key"
+        - "--tls-server-cert-path=/etc/server_cert/tls.crt"
+        - "--tls-client-ca-path=/etc/cert/ca.crt"
+        {{- end }}
         command:
         - io-engine
         securityContext:
@@ -128,6 +137,14 @@ spec:
           mountPath: /var/local/{{ .Release.Name }}/io-engine/
         - name: hugepage
           mountPath: /dev/hugepages
+        {{- if .Values.tls.enabled }}
+        - name: io-engine-server-cert
+          mountPath: "/etc/server_cert/"
+          readOnly: true
+        - name: ca-cert
+          mountPath: "/etc/cert"
+          readOnly: true
+        {{- end }}
         resources:
           limits:
             cpu: {{ .Values.io_engine.resources.limits.cpu | default (include "coreCount" .) | quote }}
@@ -161,3 +178,11 @@ spec:
         hostPath:
           path: /var/local/{{ .Release.Name }}/io-engine/
           type: DirectoryOrCreate
+      {{- if .Values.tls.enabled }}
+      - name: io-engine-server-cert
+        secret:
+          secretName: io-engine-server-cert
+      - name: ca-cert
+        secret:
+          secretName: ca-root-cert
+      {{- end }}

chart/templates/mayastor/obs/obs-callhome-deployment.yaml

@@ -100,6 +100,6 @@ spec:
         - name: ca-cert
           secret:
             defaultMode: 420
-            secretName: api-rest-tls
+            secretName: ca-root-cert
         {{- end }}
 {{- end }}

chart/templates/mayastor/operators/operator-diskpool-deployment.yaml

@@ -75,17 +75,9 @@ spec:
               mountPath: /etc/client_cert/ca.crt
               subPath: ca.crt
               readOnly: true
-            # - name: agent-ca-cert
-            #   mountPath: /etc/client_cert/agent-ca.crt
-            #   subPath: agent-ca.crt
-            #   readOnly: true
       volumes:
         - name: ca-cert
           secret:
             defaultMode: 420
-            secretName: api-rest-tls
-        # - name: agent-ca-cert
-        #   secret:
-        #     defaultMode: 420
-        #     secretName: agent-core-server-cert
+            secretName: ca-root-cert
       {{- end}}