Use PEM files directly as ZooKeeper keystore and truststore (#613)

Author: lhotari

README.md

@@ -160,7 +160,7 @@ It includes support for:
         - [x] Broker
         - [x] Toolset
         - [x] Bookie
-        - [x] ZooKeeper
+        - [x] ZooKeeper (requires the `AdditionalCertificateOutputFormats=true` feature gate to be enabled in the cert-manager deployment when using cert-manager versions below 1.15.0)
     - [x] Authentication
         - [x] JWT
         - [x] OpenID
@@ -402,6 +402,15 @@ helm upgrade -n <namespace> -f values.yaml <pulsar-release-name> apachepulsar/pu
 
 For more detailed information, see our [Upgrading](http://pulsar.apache.org/docs/helm-upgrade/) guide.
 
+## Upgrading to Helm chart version 4.2.0 (not released yet)
+
+### TLS configuration for ZooKeeper has changed
+
+The TLS configuration for ZooKeeper has been changed to fix certificate and private key expiration issues.
+This change impacts configurations that have `tls.enabled` and `tls.zookeeper.enabled` set in `values.yaml`.
+The revised solution requires the `AdditionalCertificateOutputFormats=true` feature gate to be enabled in the `cert-manager` deployment when using cert-manager versions below 1.15.0.
+If you installed `cert-manager` using `./scripts/cert-manager/install-cert-manager.sh`, you can re-run the updated script to set the feature gate. The script currently installs or upgrades cert-manager LTS version 1.12.17, where the feature gate must be explicitly enabled.
+
 ## Upgrading from Helm Chart versions before 4.0.0 to 4.0.0 version and above
 
 ### Pulsar Proxy service's default type has been changed from `LoadBalancer` to `ClusterIP`

charts/pulsar/templates/_autorecovery.tpl

@@ -36,7 +36,7 @@ Define autorecovery zookeeper client tls settings
 */}}
 {{- define "pulsar.autorecovery.zookeeper.tls.settings" -}}
 {{- if and .Values.tls.enabled .Values.tls.zookeeper.enabled }}
-/pulsar/keytool/keytool.sh autorecovery {{ template "pulsar.autorecovery.hostname" . }} true;
+{{- include "pulsar.component.zookeeper.tls.settings" (dict "component" "autorecovery" "isClient" true) -}}
 {{- end }}
 {{- end }}
 
@@ -51,11 +51,6 @@ Define autorecovery tls certs mounts
 - name: ca
   mountPath: "/pulsar/certs/ca"
   readOnly: true
-{{- if .Values.tls.zookeeper.enabled }}
-- name: keytool
-  mountPath: "/pulsar/keytool/keytool.sh"
-  subPath: keytool.sh
-{{- end }}
 {{- end }}
 {{- end }}
 
@@ -72,18 +67,14 @@ Define autorecovery tls certs volumes
       path: tls.crt
     - key: tls.key
       path: tls.key
+    - key: tls-combined.pem
+      path: tls-combined.pem
 - name: ca
   secret:
     secretName: "{{ template "pulsar.certs.issuers.ca.secretName" . }}"
     items:
     - key: ca.crt
       path: ca.crt
-{{- if .Values.tls.zookeeper.enabled }}
-- name: keytool
-  configMap:
-    name: "{{ template "pulsar.fullname" . }}-keytool-configmap"
-    defaultMode: 0755
-{{- end }}
 {{- end }}
 {{- end }}
 
@@ -93,7 +84,7 @@ Define autorecovery init container : verify cluster id
 {{- define "pulsar.autorecovery.init.verify_cluster_id" -}}
 bin/apply-config-from-env.py conf/bookkeeper.conf;
 export BOOKIE_MEM="-Xmx128M";
-{{- include "pulsar.autorecovery.zookeeper.tls.settings" . -}}
+{{- include "pulsar.autorecovery.zookeeper.tls.settings" . }}
 until timeout 15 bin/bookkeeper shell whatisinstanceid; do
   sleep 3;
 done;

charts/pulsar/templates/_bookkeeper.tpl

@@ -37,7 +37,7 @@ Define bookie zookeeper client tls settings
 */}}
 {{- define "pulsar.bookkeeper.zookeeper.tls.settings" -}}
 {{- if and .Values.tls.enabled .Values.tls.zookeeper.enabled }}
-/pulsar/keytool/keytool.sh bookie {{ template "pulsar.bookkeeper.hostname" . }} true;
+{{- include "pulsar.component.zookeeper.tls.settings" (dict "component" "bookie" "isClient" true) -}}
 {{- end }}
 {{- end }}
 
@@ -52,11 +52,6 @@ Define bookie tls certs mounts
 - name: ca
   mountPath: "/pulsar/certs/ca"
   readOnly: true
-{{- if .Values.tls.zookeeper.enabled }}
-- name: keytool
-  mountPath: "/pulsar/keytool/keytool.sh"
-  subPath: keytool.sh
-{{- end }}
 {{- end }}
 {{- end }}
 
@@ -73,18 +68,16 @@ Define bookie tls certs volumes
       path: tls.crt
     - key: tls.key
       path: tls.key
+{{- if .Values.tls.zookeeper.enabled }}
+    - key: tls-combined.pem
+      path: tls-combined.pem
+{{- end }}
 - name: ca
   secret:
     secretName: "{{ template "pulsar.certs.issuers.ca.secretName" . }}"
     items:
     - key: ca.crt
       path: ca.crt
-{{- if .Values.tls.zookeeper.enabled }}
-- name: keytool
-  configMap:
-    name: "{{ template "pulsar.fullname" . }}-keytool-configmap"
-    defaultMode: 0755
-{{- end }}
 {{- end }}
 {{- end }}
 
@@ -147,7 +140,7 @@ Define bookie init container : verify cluster id
 {{- if not (and .Values.volumes.persistence .Values.bookkeeper.volumes.persistence) }}
 bin/apply-config-from-env.py conf/bookkeeper.conf;
 export BOOKIE_MEM="-Xmx128M";
-{{- include "pulsar.bookkeeper.zookeeper.tls.settings" . -}}
+{{- include "pulsar.bookkeeper.zookeeper.tls.settings" . }}
 until timeout 15 bin/bookkeeper shell whatisinstanceid; do
   sleep 3;
 done;
@@ -157,7 +150,7 @@ bin/bookkeeper shell bookieformat -nonInteractive -force -deleteCookie || true
 set -e;
 bin/apply-config-from-env.py conf/bookkeeper.conf;
 export BOOKIE_MEM="-Xmx128M";
-{{- include "pulsar.bookkeeper.zookeeper.tls.settings" . -}}
+{{- include "pulsar.bookkeeper.zookeeper.tls.settings" . }}
 until timeout 15 bin/bookkeeper shell whatisinstanceid; do
   sleep 3;
 done;

charts/pulsar/templates/_broker.tpl

@@ -43,7 +43,7 @@ Define broker zookeeper client tls settings
 */}}
 {{- define "pulsar.broker.zookeeper.tls.settings" -}}
 {{- if and .Values.tls.enabled .Values.tls.zookeeper.enabled }}
-/pulsar/keytool/keytool.sh broker {{ template "pulsar.broker.hostname" . }} true;
+{{- include "pulsar.component.zookeeper.tls.settings" (dict "component" "broker" "isClient" true) -}}
 {{- end }}
 {{- end }}
 
@@ -58,11 +58,6 @@ Define broker tls certs mounts
 - name: ca
   mountPath: "/pulsar/certs/ca"
   readOnly: true
-{{- if .Values.tls.zookeeper.enabled }}
-- name: keytool
-  mountPath: "/pulsar/keytool/keytool.sh"
-  subPath: keytool.sh
-{{- end }}
 {{- end }}
 {{- end }}
 
@@ -79,17 +74,15 @@ Define broker tls certs volumes
       path: tls.crt
     - key: tls.key
       path: tls.key
+{{- if .Values.tls.zookeeper.enabled }}
+    - key: tls-combined.pem
+      path: tls-combined.pem
+{{- end }}
 - name: ca
   secret:
     secretName: "{{ template "pulsar.certs.issuers.ca.secretName" . }}"
     items:
     - key: ca.crt
       path: ca.crt
-{{- if .Values.tls.zookeeper.enabled }}
-- name: keytool
-  configMap:
-    name: "{{ template "pulsar.fullname" . }}-keytool-configmap"
-    defaultMode: 0755
-{{- end }}
 {{- end }}
 {{- end }}

charts/pulsar/templates/_certs.tpl

@@ -57,4 +57,58 @@ Define the pulsar certs ca issuer secret name
 {{- fail "certs.issuers.ca.secretName is required when TLS is enabled and certs.internal_issuer.enabled is false" -}}
 {{- end -}}
 {{- end -}}
+{{- end -}}
+
+{{/*
+Common certificate template
+Usage: {{- include "pulsar.cert.template" (dict "root" . "componentConfig" .Values.proxy "tlsConfig" .Values.tls.proxy) -}}
+*/}}
+{{- define "pulsar.cert.template" -}}
+{{- if eq .root.Values.certs.internal_issuer.apiVersion "cert-manager.io/v1beta1" -}}
+{{- fail "cert-manager.io/v1beta1 is no longer supported. Please set certs.internal_issuer.apiVersion to cert-manager.io/v1" -}}
+{{- end -}}
+apiVersion: "{{ .root.Values.certs.internal_issuer.apiVersion }}"
+kind: Certificate
+metadata:
+  name: "{{ template "pulsar.fullname" .root }}-{{ .tlsConfig.cert_name }}"
+  namespace: {{ template "pulsar.namespace" .root }}
+spec:
+  # Secret names are always required.
+  secretName: "{{ .root.Release.Name }}-{{ .tlsConfig.cert_name }}"
+{{- if .root.Values.tls.zookeeper.enabled }}
+  additionalOutputFormats:
+    - type: CombinedPEM
+{{- end }}
+  duration: "{{ .root.Values.tls.common.duration }}"
+  renewBefore: "{{ .root.Values.tls.common.renewBefore }}"
+  subject:
+    organizations:
+{{ toYaml .root.Values.tls.common.organization | indent 4 }}
+  # The use of the common name field has been deprecated since 2000 and is
+  # discouraged from being used.
+  commonName: "{{ template "pulsar.fullname" .root }}-{{ .componentConfig.component }}"
+  isCA: false
+  privateKey:
+    size: {{ .root.Values.tls.common.keySize }}
+    algorithm: {{ .root.Values.tls.common.keyAlgorithm }}
+    encoding: {{ .root.Values.tls.common.keyEncoding }}
+  usages:
+    - server auth
+    - client auth
+  # At least one of a DNS Name, USI SAN, or IP address is required.
+  dnsNames:
+{{- if .tlsConfig.dnsNames }}
+{{ toYaml .tlsConfig.dnsNames | indent 4 }}
+{{- end }}
+    - {{ printf "*.%s-%s.%s.svc.%s" (include "pulsar.fullname" .root) .componentConfig.component (include "pulsar.namespace" .root) .root.Values.clusterDomain | quote }}
+    - {{ printf "%s-%s" (include "pulsar.fullname" .root) .componentConfig.component | quote }}
+  # Issuer references are always required.
+  issuerRef:
+    name: "{{ template "pulsar.certs.issuers.ca.name" .root }}"
+    # We can reference ClusterIssuers by changing the kind here.
+    # The default value is Issuer (i.e. a locally namespaced Issuer)
+    kind: Issuer
+    # This is optional since cert-manager will default to this value however
+    # if you are using an external issuer, change this to that issuer group.
+    group: cert-manager.io
 {{- end -}}

charts/pulsar/templates/_toolset.tpl

@@ -36,7 +36,7 @@ Define toolset zookeeper client tls settings
 */}}
 {{- define "pulsar.toolset.zookeeper.tls.settings" -}}
 {{- if and .Values.tls.enabled .Values.tls.zookeeper.enabled -}}
-/pulsar/keytool/keytool.sh toolset {{ template "pulsar.toolset.hostname" . }} true;
+{{- include "pulsar.component.zookeeper.tls.settings" (dict "component" "toolset" "isClient" true) -}}
 {{- end -}}
 {{- end }}
 
@@ -51,11 +51,6 @@ Define toolset tls certs mounts
 - name: ca
   mountPath: "/pulsar/certs/ca"
   readOnly: true
-{{- if .Values.tls.zookeeper.enabled }}
-- name: keytool
-  mountPath: "/pulsar/keytool/keytool.sh"
-  subPath: keytool.sh
-{{- end }}
 {{- end }}
 {{- end }}
 
@@ -72,17 +67,13 @@ Define toolset tls certs volumes
       path: tls.crt
     - key: tls.key
       path: tls.key
+    - key: tls-combined.pem
+      path: tls-combined.pem
 - name: ca
   secret:
     secretName: "{{ template "pulsar.certs.issuers.ca.secretName" . }}"
     items:
     - key: ca.crt
       path: ca.crt
-{{- if .Values.tls.zookeeper.enabled }}
-- name: keytool
-  configMap:
-    name: "{{ template "pulsar.fullname" . }}-keytool-configmap"
-    defaultMode: 0755
-{{- end }}
 {{- end }}
 {{- end }}

charts/pulsar/templates/_zookeeper.tpl

@@ -53,7 +53,23 @@ Define zookeeper tls settings
 */}}
 {{- define "pulsar.zookeeper.tls.settings" -}}
 {{- if and .Values.tls.enabled .Values.tls.zookeeper.enabled }}
-/pulsar/keytool/keytool.sh zookeeper {{ template "pulsar.zookeeper.hostname" . }} false;
+{{- include "pulsar.component.zookeeper.tls.settings" (dict "component" "zookeeper" "isClient" false) -}}
+{{- end }}
+{{- end }}
+
+{{- define "pulsar.component.zookeeper.tls.settings" }}
+{{- $component := .component -}}
+{{- $isClient := .isClient -}}
+{{- $keyFile := printf "/pulsar/certs/%s/tls-combined.pem" $component -}}
+{{- $caFile := "/pulsar/certs/ca/ca.crt" -}}
+{{- if $isClient }}
+echo $'\n' >> conf/pulsar_env.sh
+echo "PULSAR_EXTRA_OPTS=\"\${PULSAR_EXTRA_OPTS} -Dzookeeper.clientCnxnSocket=org.apache.zookeeper.ClientCnxnSocketNetty -Dzookeeper.client.secure=true -Dzookeeper.client.certReload=true -Dzookeeper.ssl.keyStore.location={{- $keyFile }} -Dzookeeper.ssl.keyStore.type=PEM -Dzookeeper.ssl.trustStore.location={{- $caFile }} -Dzookeeper.ssl.trustStore.type=PEM\"" >> conf/pulsar_env.sh
+echo $'\n' >> conf/bkenv.sh
+echo "BOOKIE_EXTRA_OPTS=\"\${BOOKIE_EXTRA_OPTS} -Dzookeeper.clientCnxnSocket=org.apache.zookeeper.ClientCnxnSocketNetty -Dzookeeper.client.secure=true -Dzookeeper.client.certReload=true -Dzookeeper.ssl.keyStore.location={{- $keyFile }} -Dzookeeper.ssl.keyStore.type=PEM -Dzookeeper.ssl.trustStore.location={{- $caFile }} -Dzookeeper.ssl.trustStore.type=PEM\"" >> conf/bkenv.sh
+{{- else }}
+echo $'\n' >> conf/pulsar_env.sh
+echo "PULSAR_EXTRA_OPTS=\"\${PULSAR_EXTRA_OPTS} -Dzookeeper.ssl.keyStore.location={{- $keyFile }} -Dzookeeper.ssl.keyStore.type=PEM -Dzookeeper.ssl.trustStore.location={{- $caFile }} -Dzookeeper.ssl.trustStore.type=PEM\"" >> conf/pulsar_env.sh
 {{- end }}
 {{- end }}