Add support for TLS & tests (#146)

* Add support for TLS & tests

Signed-off-by: Jirka Kremser <jiri.kremser@gmail.com>

* TLS support: Helm charts & custom examples in dev Makefile

Signed-off-by: Jirka Kremser <jiri.kremser@gmail.com>

---------

Signed-off-by: Jirka Kremser <jiri.kremser@gmail.com>

Author: jkremser

.gitignore

@@ -15,3 +15,7 @@ helmchart/otel-add-on/*.tgz
 
 # Other
 /vendor/
+
+# Tests
+/certs
+values_tmp

Makefile

@@ -3,9 +3,9 @@
 ###############################
 SHELL       = /bin/bash
 GH_REPO_ORG = kedify
-VERSION 		?= main
+VERSION 	?= main
 GIT_COMMIT  ?= $(shell git rev-list -1 HEAD)
-LATEST_TAG ?= $(shell git fetch --force --tags &> /dev/null ; git describe --tags $(git rev-list --tags --max-count=1))
+LATEST_TAG  ?= $(shell git fetch --force --tags &> /dev/null ; git describe --tags $(git rev-list --tags --max-count=1))
 GO_LDFLAGS="-X github.com/${GH_REPO_ORG}/otel-add-on/build.version=${VERSION} -X github.com/${GH_REPO_ORG}/otel-add-on/build.gitCommit=${GIT_COMMIT}"
 BUILD_PLATFORMS ?= linux/amd64,linux/arm64
 
@@ -18,43 +18,61 @@ endif
 CGO        ?=0
 TARGET_OS  ?=linux
 
+define SERVER_DOMAINS
+basicConstraints=CA:FALSE
+keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
+extendedKeyUsage = serverAuth, clientAuth
+subjectAltName = @alt_names
+[alt_names]
+DNS.1 = localhost
+DNS.2 = *.keda.svc.cluster.local
+DNS.3 = *.keda.svc
+DNS.4 = *.keda
+IP.1 = 127.0.0.1
+endef
+export SERVER_DOMAINS
+
 GO_BUILD_VARS= GO111MODULE=on CGO_ENABLED=$(CGO) GOOS=$(TARGET_OS) GOARCH=$(ARCH)
 
 ###############################
 #		TARGETS
 ###############################
 all: help
 
+##@ Build
+
 .PHONY: build
-build:  ## Builds the binary.
+build: ## Builds the binary.
 	@$(call say,Build the binary)
 	${GO_BUILD_VARS} go build -ldflags $(GO_LDFLAGS) -o bin/otel-add-on .
 
-.PHONY: run
-run:  ## Runs the scaler locally.
-	go run ./main.go
-
 .PHONY: build-image
-build-image: build  ## Builds the container image for current arch.
+build-image: build ## Builds the container image for current arch.
 	@$(call say,Build container image $(CONTAINER_IMAGE))
 	docker build . -t ${CONTAINER_IMAGE} --build-arg VERSION=${VERSION} --build-arg GIT_COMMIT=${GIT_COMMIT}
 
 .PHONY: build-image-multiarch
-build-image-multiarch:  ## Builds the container image for arm64 and amd64.
+build-image-multiarch: ## Builds the container image for arm64 and amd64.
 	@$(call say,Build container image $(CONTAINER_IMAGE))
 	docker buildx build --output=type=registry --platform=${BUILD_PLATFORMS} . -t ${CONTAINER_IMAGE} --build-arg VERSION=${VERSION} --build-arg GIT_COMMIT=${GIT_COMMIT}
 
 .PHONY: build-image-goreleaser
 build-image-goreleaser: ## Builds the multi-arch container image using goreleaser.
 	goreleaser release --skip=validate,publish,sbom --clean --snapshot
 
+##@ General
+
+.PHONY: run
+run: ## Runs the scaler locally.
+	go run ./main.go
+
 .PHONY: test
-test:  ## Runs golang unit tests.
+test: test-certs ## Runs golang unit tests.
 	@$(call say,Running golang unit tests)
 	go test -race -v ./...
 
 .PHONY: e2e-test
-e2e-test:  ## Runs end to end tests. This will spawn a k3d cluster.
+e2e-test: ## Runs end to end tests. This will spawn a k3d cluster.
 	@$(call say,Running end to end tests)
 	cd e2e-tests && go test -count=1 -race -v ./...
 
@@ -75,12 +93,32 @@ codegen: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and
 	./hack/update-codegen.sh
 
 .PHONY: deploy-helm
-deploy-helm:  ## Deploys helm chart with otel-collector and otel scaler.
+deploy-helm: ## Deploys helm chart with otel-collector and otel scaler.
 	@$(call say,Deploy helm chart to current k8s context)
 	cd helmchart/otel-add-on && \
 	helm dependency build && \
 	helm upgrade -i kedify-otel .
 
+.PHONY: rootca-test-certs
+rootca-test-certs:
+	@$(call say,CA cert)
+	rm -rf certs
+	mkdir -p certs
+	openssl req -x509 -nodes -new -sha256 -days 1024 -newkey rsa:2048 -keyout certs/rootCA.key -out certs/rootCA.crt -subj "/C=US/CN=Keda-OTel-Scaler-Root-CA"
+	rm -rf certs/rootCA.srl
+
+.PHONY: rootca-test-certs
+test-certs: rootca-test-certs ## Generates certs for local unit and e2e tests
+	@$(call say,Server cert)
+	echo "$$SERVER_DOMAINS" > certs/domains.ext
+	openssl req -new -nodes -newkey rsa:2048 -keyout certs/server.key -out certs/server.csr -subj "/C=US/ST=KedaState/L=KedaCity/O=Test-Certificates/CN=keda-otel-scaler.keda.svc"
+	openssl x509 -req -sha256 -days 1024 -in certs/server.csr -CA certs/rootCA.crt -CAkey certs/rootCA.key -CAcreateserial -extfile certs/domains.ext -out certs/server.crt
+
+	@$(call say,Client cert)
+	openssl req -new -nodes -newkey rsa:2048 -keyout certs/client.key -out certs/client.csr -subj "/C=US/ST=KedaState/L=KedaCity/O=Test-Certificates/CN=client"
+	openssl x509 -req -sha256 -days 1024 -in certs/client.csr -CA certs/rootCA.crt -CAkey certs/rootCA.key -CAcreateserial -out certs/client.crt
+	rm -rf certs/*.{csr,srl,ext}
+
 .PHONY: logs
 logs:
 	@$(call say,logs)
@@ -95,8 +133,8 @@ gomodifytags: ## Download gomodifytags locally if necessary.
 	GOBIN=$(shell pwd)/bin go install github.com/fatih/gomodifytags@v1.17.0
 
 .PHONY: help
-help: ## Show this help.
-	@egrep -h '\s##\s' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-25s\033[0m %s\n", $$1, $$2}'
+help: ## Display this help.
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-24s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
 
 ###############################

README.md

@@ -98,6 +98,26 @@ helm upgrade -i keda-otel-scaler -nkeda oci://ghcr.io/kedify/charts/otel-add-on
 k apply -f examples/so.yaml
 ```
 
+### Advanced setups
+
+Check some prepared examples in the [`./examples`](./examples) directory and also checked the `dev.Makefile` if you want to
+set up mTLS between a collector and this scaler. 
+
+```bash
+λ make -f dev.Makefile
+Usage:
+  make <target>
+Demos
+  demo-podinfo              setup ./examples/metric-pull
+  demo-podinfo-tls          setup ./examples/metric-pull with TLS
+  demo-otel-upstream        setup ./examples/metric-push
+  demo-operator             setup ./examples/otel-operator
+  demo-operator-tls         setup ./examples/otel-operator with TLS
+  
+λ make -f dev.Makefile demo-podinfo-tls
+...
+```
+
 ## Troubleshooting
 
 To figure out the actual value of a metric query, there is a simple REST api that can be used:

dev.Makefile

@@ -3,34 +3,110 @@ include Makefile
 LOCAL_ENDPOINT ?= host.k3d.internal
 SO_NAME ?= otel-example
 
+## helpers
+check_defined = \
+	$(strip $(foreach 1,$1, \
+		$(call __check_defined,$1,$(strip $(value 2)))))
+__check_defined = \
+	$(if $(value $1),, \
+		$(error Undefined $1$(if $2, ($2))))
+check_k3d = \
+	@if [ -z $$(kubectl config current-context | grep "^k3d-") ]; then \
+		echo "Create k3d cluster first!" ;\
+		exit 1 ;\
+	fi
+
+##@ Dev
 .PHONY: dev-k3d
-dev-k3d: build-image  ## Builds the container image for current arch, imports it to running k3d and restarts the scaler.
+dev-k3d: ## Builds the container image for current arch, imports it to running k3d and restarts the scaler.
 	@$(call say,Doing the dev cycle)
-	k3d image import ghcr.io/kedify/otel-add-on:latest
+	@$(call check_k3d)
+	k3d image import -c $(shell kubectl config current-context | sed -e "s/^k3d-//") ghcr.io/kedify/otel-add-on:latest
 	helm upgrade --reuse-values \
-        kedify-otel helmchart/otel-add-on \
-		--set image.tag=latest  \
+		keda-otel-scaler helmchart/otel-add-on \
+		-nkeda \
+		--set image.tag=latest \
 		--set image.pullPolicy=IfNotPresent \
-		--set settings.logs.logLvl=debug \
-	kubectl rollout restart deploy/otel-add-on-scaler
+		--set settings.logs.logLvl=debug
+	kubectl -nkeda rollout restart deploy/keda-otel-scaler
 
 .PHONY: dev-local
 dev-local: ## Prepare the SO and otel collector for local debug
 	@$(call say,Prepare the conditions for local debug)
+	@$(call check_k3d)
 	helm upgrade --reuse-values \
- 		kedify-otel helmchart/otel-add-on \
- 		--set replicaCount=1 \
- 		--set opentelemetry-collector.config.exporters.otlp.endpoint=$(LOCAL_ENDPOINT):4317
+		keda-otel-scaler helmchart/otel-add-on \
+		-nkeda \
+		--set replicaCount=1 \
+		--set opentelemetry-collector.config.exporters.otlp.endpoint=$(LOCAL_ENDPOINT):4317
 	kubectl patch so $(SO_NAME) --type=json -p '[{"op":"replace","path":"/spec/triggers/0/metadata/scalerAddress","value":"$(LOCAL_ENDPOINT):4318"}]'
-	@$(call say,Continue by running the scaler locally from your favorite IDE outsice of K8s)
+	@$(call say,Continue by running the scaler locally from your favorite IDE outside of K8s)
 	@echo "Make sure $(LOCAL_ENDPOINT):4317 and $(LOCAL_ENDPOINT):4318 are listening.."
 
 .PHONY: undo-dev-local
 undo-dev-local: ## Revers the SO and otel collector for local debug
 	@$(call say,Revert the conditions for local debug)
+	@$(call check_k3d)
 	helm upgrade --reuse-values \
- 		kedify-otel helmchart/otel-add-on \
- 		--set replicaCount=1 \
- 		--set opentelemetry-collector.config.exporters.otlp.endpoint=keda-otel-scaler:4317
- 	kubectl patch so $(SO_NAME) --type=json -p '[{"op":"replace","path":"/spec/triggers/0/metadata/scalerAddress","value":"keda-otel-scaler:4318"}]'
- 	kubectl scale deploy/otel-add-on-scaler --replicas=1
+		keda-otel-scaler helmchart/otel-add-on \
+		-nkeda \
+		--set replicaCount=1 \
+		--set opentelemetry-collector.config.exporters.otlp.endpoint=keda-otel-scaler.keda.svc:4317
+	kubectl patch so $(SO_NAME) --type=json -p '[{"op":"replace","path":"/spec/triggers/0/metadata/scalerAddress","value":"keda-otel-scaler.keda.svc:4318"}]'
+	kubectl scale -nkeda deploy/keda-otel-scaler --replicas=1
+
+.PHONY: k8s-certs
+k8s-certs: test-certs ## Creates k8s secrets from the generated certificates
+	@$(call say,Preparing certs)
+	@$(call check_k3d)
+	kubectl -nkeda delete secret --ignore-not-found server-tls client-tls root-ca
+	kubectl -nkeda create secret tls server-tls --cert=certs/server.crt --key=certs/server.key
+	kubectl -nkeda create secret tls client-tls --cert=certs/client.crt --key=certs/client.key
+	kubectl -nkeda create secret generic root-ca --from-file=rootCA.crt=certs/rootCA.crt
+
+##@ Demos
+.PHONY: demo-podinfo
+demo-podinfo: ## setup ./examples/metric-pull
+	./examples/metric-pull/setup.sh
+	$(MAKE) -f dev.Makefile dev-k3d
+
+.PHONY: demo-podinfo-tls
+demo-podinfo-tls: ## setup ./examples/metric-pull with TLS
+	SETUP_ONLY=true ./examples/metric-pull/setup.sh
+	$(MAKE) -f dev.Makefile k8s-certs
+	helm upgrade -i -nkeda keda-otel-scaler ./helmchart/otel-add-on -f ./examples/metric-pull/scaler-with-collector-pull-tls-values.yaml
+	$(MAKE) -f dev.Makefile dev-k3d
+	kubectl apply -f ./examples/metric-pull/podinfo-so.yaml
+	@$(call say,Done)
+	@echo "Continue with: (hey -n 7000 -z 180s http://localhost:8181/delay/2 &> /dev/null)&"
+
+.PHONY: demo-otel-upstream
+demo-otel-upstream: ## setup ./examples/metric-push
+	./examples/metric-push/setup.sh
+	$(MAKE) -f dev.Makefile dev-k3d
+
+.PHONY: demo-operator
+demo-operator: ## setup ./examples/otel-operator
+	@:$(call check_defined, PR_BRANCH GH_PAT)
+	$(call check_k3d)
+	./examples/otel-operator/setup.sh
+	$(MAKE) -f dev.Makefile dev-k3d
+
+.PHONY: demo-operator-tls
+demo-operator-tls: ## setup ./examples/otel-operator with TLS
+	SETUP_ONLY=true ./examples/otel-operator/setup.sh
+	$(MAKE) -f dev.Makefile k8s-certs
+	# helm cant merge correctly value files when the later overrides an item in an array (w/ unique name)
+	rm -rf values_tmp && ./hack/mergeValues.sh \
+		examples/otel-operator/scaler-with-operator-with-collector-values.yaml \
+		examples/otel-operator/tls-overlay-values.yaml > values_tmp
+	@$(call say,Merged values:)
+	yq values_tmp
+	helm upgrade -i \
+			keda-otel-scaler helmchart/otel-add-on \
+			-nkeda \
+			-f ./values_tmp
+	rm -rf values_tmp
+	$(MAKE) -f dev.Makefile dev-k3d
+	@$(call say,Creating SO)
+	kubectl apply -f <(cat ./examples/otel-operator/so.yaml | envsubst)