Swagger at / + some tls tuneability (#162)

* Complex pipeline where evertything is TLS encrypted and certs are rotated

Signed-off-by: Jirka Kremser <jiri.kremser@gmail.com>

* Update examples/pipelines-tls/setup.sh

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update examples/vllm/setup.sh

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update examples/otel-operator/setup.sh

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Signed-off-by: Jirka Kremser <jiri.kremser@gmail.com>

* Swagger + own certs for keda <-> scaler comm

Signed-off-by: Jirka Kremser <jiri.kremser@gmail.com>

---------

Signed-off-by: Jirka Kremser <jiri.kremser@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: jkremser

examples/pipelines-tls/prometheus-values.yaml

@@ -20,7 +20,7 @@ configmapReload:
 server:
   extraFlags:
     - web.enable-remote-write-receiver
-  readinessProbeInitialDelay: 240
+  readinessProbeInitialDelay: 90
   retention: "7d"
   retentionSize: "1GB"
   resources:

examples/pipelines-tls/scaler-pipelines-tls-values.yaml

@@ -8,6 +8,10 @@ settings:
 #    caFile: "/etc/scaler-tls/ca.crt"
     certFile: "/etc/scaler-tls/tls.crt"
     keyFile: "/etc/scaler-tls/tls.key"
+    keda:
+      enabled: true
+      certFile: "/etc/scaler-tls/tls.crt"
+      keyFile: "/etc/scaler-tls/tls.key"
     reloadInterval: "15s"
     secrets:
       - name: keda-otel-root-ca-bundle

examples/pipelines-tls/setup.sh

@@ -83,7 +83,6 @@ kubectl rollout status -n observability --timeout=600s deploy/prometheus-server
 
 # SO
 kubectl wait -nkeda --for condition=ready --timeout=300s cert/keda-operator-tls-certificates
-#certSecret="-nkeda secret/kedaorg-certs"
 certSecret="-nkeda secret/keda-otel-scaler-cert-secret"
 export _caCertPem=$(kubectl get $(echo $certSecret) -o'go-template={{index .data "ca.crt"}}' | base64 -d | awk '{ print "          " $0 }')
 export _tlsClientKey=$(kubectl get $(echo $certSecret) -o'go-template={{index .data "tls.key"}}' | base64 -d | awk '{ print "          " $0 }')
@@ -109,18 +108,17 @@ k get otelcol -A
 k get cert -A -owide
 
 # create traffic
-(hey -z 60s http://localhost:8080 &> /dev/null)&
+(hey -z 30s http://localhost:8080 &> /dev/null)&
 
 # check how it scales out
 k get hpa -A && k get so -A
 
 # verify SSL works
-_pod=$(kubectl get po -nobservability -lapp.kubernetes.io/name=router-collector --no-headers -ocustom-columns=":metadata.name")
-kubectl debug -it -n observability ${_pod} --image=dockersec/tcpdump --target otc-container -- tcpdump -i any 'port 4317 and (tcp[tcpflags] & (tcp-syn|tcp-ack) == tcp-syn)'
-and you should be able to observe beginnings of SSL handshakes (kill nginx pod to force one)
+kubectl debug -it -n observability $(kubectl get po -nobservability -lapp.kubernetes.io/name=router-collector --no-headers -ocustom-columns=":metadata.name") --image=dockersec/tcpdump --target otc-container -- tcpdump -i any 'port 4317 and (tcp[tcpflags] & (tcp-syn|tcp-ack) == tcp-syn)'
+# and you should be able to observe beginnings of SSL handshakes (kill nginx pod to force one)
 
-# force cert rotation in app ns
-cmctl renew --namespace=app --all
+# force cert rotations
+cmctl renew -A --all
 
 🚀
 USAGE

helmchart/otel-add-on/templates/deployment.yaml

@@ -65,7 +65,7 @@ spec:
             - name: IS_ACTIVE_POLLING_INTERVAL_MS
               value: {{ .Values.settings.isActivePollingIntervalMilliseconds | quote }}
 
-            # TLS
+            # TLS OTLP
             - name: OTLP_TLS_CA_FILE
               value: {{ .Values.settings.tls.caFile | quote }}
             - name: OTLP_TLS_CERT_FILE
@@ -74,6 +74,13 @@ spec:
               value: {{ .Values.settings.tls.keyFile | quote }}
             - name: OTLP_CERTIFICATE_RELOAD_INTERVAL
               value: {{ .Values.settings.tls.reloadInterval | quote }}
+            # TLS KEDA
+            - name: KEDA_TLS_ENABLED
+              value: {{ .Values.settings.tls.keda.enabled | quote }}
+            - name: KEDA_TLS_CERT_FILE
+              value: {{ .Values.settings.tls.keda.certFile | default .Values.settings.tls.certFile | quote }}
+            - name: KEDA_TLS_KEY_FILE
+              value: {{ .Values.settings.tls.keda.keyFile | default .Values.settings.tls.keyFile | quote }}
       {{- if .Values.settings.logs.noColor }}
             - name: NO_COLOR
               value: {{ .Values.settings.logs.noColor | quote }}

helmchart/otel-add-on/values.yaml

@@ -65,6 +65,12 @@ settings:
     # -- (optional) specifies the duration after which the certificates will be reloaded.
     # This is useful when using the CertManager for rotating the certs mounted as Secrets.
     reloadInterval: "5m"
+    keda:
+      enabled: false
+      # -- (optional) path to TLS certificate that will be used for KEDA gRPC server. If empty, defaults to `settings.tls.certFile`
+      certFile: ""
+      # -- (optional) path to TLS key that will be used for KEDA gRPC server. If empty, defaults to `settings.tls.keyFile`
+      keyFile: ""
     # -- (optional) list of secrets that will be mounted to deployment's pod. One entry in this list, will create one volume and one volumeMount for pod.
     # This is a convenient way for mounting the certs for TLS, but using `.volumes & .volumeMounts` for anything advanced will also work.
     secrets: []

main.go

@@ -80,7 +80,7 @@ func main() {
 			setupLog.Info("🔒 TLS for gRPC server enabled (OTLP receiver)", "tlsSettings", tlsSettings)
 		}
 
-		if e = startGrpcServer(ctx, ctrl.Log, ms, mp, cfg); !util.IsIgnoredErr(e) {
+		if e = startKEDAGrpcServer(ctx, ctrl.Log, ms, mp, cfg); !util.IsIgnoredErr(e) {
 			setupLog.Error(e, "gRPC server failed (KEDA external scaler)")
 			return e
 		}
@@ -182,7 +182,7 @@ func startReceiver(ctx context.Context, otlpReceiverPort int, tlsSettings *confi
 	return nil
 }
 
-func startGrpcServer(
+func startKEDAGrpcServer(
 	ctx context.Context,
 	lggr logr.Logger,
 	ms types.MemStore,
@@ -197,14 +197,17 @@ func startGrpcServer(
 	}
 
 	var serverOpts []grpc.ServerOption
-	tlsSettings := makeTlsSettings(cfg)
-	if tlsSettings.CertFile != "" && tlsSettings.KeyFile != "" {
-		creds, e := credentials.NewServerTLSFromFile(tlsSettings.CertFile, tlsSettings.KeyFile)
-		if e != nil {
-			setupLog.Error(e, "failed to get certificates")
+	if cfg.TLSKedaComm {
+		if cfg.TLSKedaCertFile != "" && cfg.TLSKedaKeyFile != "" {
+			creds, e := credentials.NewServerTLSFromFile(cfg.TLSKedaCertFile, cfg.TLSKedaKeyFile)
+			if e != nil {
+				setupLog.Error(e, "failed to get certificates")
+				os.Exit(1)
+			}
+			setupLog.Info("🔒 TLS for gRPC server enabled (KEDA scaler <-> KEDA comm)", "cert", cfg.TLSKedaCertFile, "key", cfg.TLSKedaKeyFile)
+			setupLog.Info("🔒 caveat: ^ these are not being actively watched and automatically reloaded")
+			serverOpts = append(serverOpts, grpc.Creds(creds))
 		}
-		setupLog.Info("🔒 gRPC server for KEDA scaler has TLS enabled")
-		serverOpts = append(serverOpts, grpc.Creds(creds))
 	}
 
 	grpcServer := grpc.NewServer(serverOpts...)

rest/api.go

@@ -58,13 +58,16 @@ func Init(restApiPort int, info prometheus.Labels, ms types.MemStore, isDebug bo
 		a.lggr.Error(err, "Disabling trusted proxies failed")
 	}
 	docs.SwaggerInfo.BasePath = "/"
+	router.GET("/", func(ctx *gin.Context) {
+		ctx.Redirect(http.StatusPermanentRedirect, "/swagger/index.html")
+	})
 	router.GET("/swagger/*any", ginSwagger.WrapHandler(swaggerfiles.Handler))
 	router.GET("/memstore/names", a.getMetricNames)
 	router.GET("/memstore/data", a.getMetricData)
 	router.POST("/memstore/query", a.query)
 	router.POST("/memstore/reset", a.reset)
 	router.GET("/info", a.getInfo)
-	a.lggr.Info(fmt.Sprintf("Swagger docs available at: http://localhost:%d/swagger/index.html", restApiPort))
+	a.lggr.Info(fmt.Sprintf("Swagger docs available at: http://localhost:%d", restApiPort))
 	return router.Run(fmt.Sprintf(":%d", restApiPort))
 }
 

util/config.go

@@ -25,6 +25,9 @@ type Config struct {
 	TLSCertFile        string        `envconfig:"OTLP_TLS_CERT_FILE" default:""`
 	TLSKeyFile         string        `envconfig:"OTLP_TLS_KEY_FILE" default:""`
 	CertReloadInterval time.Duration `envconfig:"OTLP_CERTIFICATE_RELOAD_INTERVAL" default:"5m"`
+	TLSKedaComm        bool          `envconfig:"KEDA_TLS_ENABLED" default:"false"`
+	TLSKedaCertFile    string        `envconfig:"KEDA_TLS_CERT_FILE" default:""`
+	TLSKedaKeyFile     string        `envconfig:"KEDA_TLS_KEY_FILE" default:""`
 
 	// Other
 	NoColor  bool `envconfig:"NO_COLOR" default:"false"`