chore: replace k3s, reduce log level, update deps, multi-arch builds (#662)

* chore: reduce log level on metrics

Metrics reporter is spamming the logs even in production flavor
because it logs as info when reporting metrics for each scheduler

* chore(github): update deprecated ubuntu img

* fix(runtime): retry pdb update

Sometimes, PDB update can fail due to conflicts of changing
the resource (in-memory copy vs runtime). To fix, k8s provides
and API to use retry on conflicts

* feat(forwarder): add dial options

Optional parameter that the caller can add specifying dial
parameters such as WithBlock(), that blocks grpc.Dial() until
connection is established rather than doing in background.
This is useful in tests

* chore: update to go 1.23

* fix(runtime): avoid race condition in GR watcher

* chore: enable multi-arch images

Build images based on platform, by default we always publish to
linux/amd64 and linux/arm64 - whereas locally we rely on the hosting
OS arch

* refactor: replace k3s with k3d

K3d is a newer tool to bring up a Kubernetes mock cluster. Previous
K3S tool is deprecated and did not support CI runners anymore, thus
refactoring the application setup and tests to use k3d. Also adjusted
docs and Makefile commands to bring up the k3d/up when starting Maestro
or running integration tests

* chore: redis 8 and postgres 17

Update local setup to use latest redis and postgres version available

* chore(ci): fix test yaml with k3d

* fix(runtime): indempotent Stop with nil check

* chore(ci): fix build on CI for multi-arch

* fix(test): flaky test assert only room in terminating

* fix(forwarder): type assertion on cache

* fix: simplify Makefile and test setup

Author: hspedro

.github/workflows/lint.yaml

@@ -8,7 +8,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v1
         with:
-          go-version: '1.22'
+          go-version: '1.23'
       - name: Checkout
         uses: actions/checkout@v2
       - name: Restore cache

.github/workflows/mkdocs.yaml

@@ -19,14 +19,13 @@ jobs:
         with:
           python-version: 3.x
       - run: cp proto/apidocs.swagger.json docs/reference/apidocs.json
-      - run: pip install mkdocs-material 
+      - run: pip install mkdocs-material
       - run: pip install mkdocs-git-revision-date-plugin
       - run: pip install mkdocs-render-swagger-plugin
       - run: pip install mkdocs-mermaid2-plugin
       - run: git config user.name 'github-actions[bot]' && git config user.email 'github-actions[bot]@users.noreply.github.com'
       - name: Publish docs
         run: mkdocs gh-deploy --force
-        
   update-docs:
     runs-on: ubuntu-latest
     steps:

.github/workflows/publish.yaml

@@ -10,20 +10,23 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Get the version (release tag)
         run: echo "IMAGE_VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Build and push
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           context: ./
           file: ./Dockerfile
+          platforms: linux/amd64,linux/arm64
           push: true
           tags: tfgco/maestro:${{ env.IMAGE_VERSION }}

.github/workflows/test.yaml

@@ -3,14 +3,14 @@ on: push
 jobs:
   unit:
     name: Unit tests
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
     - name: Set up Go
-      uses: actions/setup-go@v1
+      uses: actions/setup-go@v5
       with:
-        go-version: '1.22'
+        go-version: '1.23'
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
     - name: Restore cache
       uses: actions/cache@v4
       with:
@@ -23,46 +23,54 @@ jobs:
       run: bash <(curl -s https://codecov.io/bash)
   integration:
     name: Integration tests
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
     - name: Set up Go
-      uses: actions/setup-go@v1
+      uses: actions/setup-go@v5
       with:
-        go-version: '1.22'
+        go-version: '1.23'
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
     - name: Restore cache
       uses: actions/cache@v4
       with:
         path: ~/go/pkg/mod
         key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+    - name: Set up KinD cluster
+      uses: helm/kind-action@v1.10.0
+      with:
+        node_image: kindest/node:v1.27.3
     - name: Run tests
       run: |
-        make run/integration-tests
+        echo "INFO: Running integration tests (using KUBECONFIG from kind-action)..."
+        go test -tags=integration -count=1 -timeout 20m -coverprofile=coverage.out -covermode=atomic ./...
     - name: Upload coverage to Codecov
       run: bash <(curl -s https://codecov.io/bash)
 
   runtime-integration:
     name: Runtime Integration tests
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
-        k3s-version: [v1.19.11, v1.20.7, v1.21.1]
-    env:
-      K3S_VERSION:  ${{ matrix.k3s-version }}
+        k8s-version: [v1.25.11, v1.26.6, v1.27.3]
     steps:
     - name: Set up Go
-      uses: actions/setup-go@v1
+      uses: actions/setup-go@v5
       with:
-        go-version: '1.22'
+        go-version: '1.23'
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
     - name: Restore cache
       uses: actions/cache@v4
       with:
         path: ~/go/pkg/mod
         key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+    - name: Set up KinD cluster
+      uses: helm/kind-action@v1.10.0
+      with:
+        node_image: kindest/node:${{ matrix.k8s-version }}
     - name: Run tests
       run: |
-        make run/runtime-integration-tests
+        echo "INFO: Running runtime integration tests (using KUBECONFIG from kind-action for k8s ${{ matrix.k8s-version }} )..."
+        go test -tags=integration -count=1 -timeout 20m ./internal/adapters/runtime/kubernetes/...

.gitignore

@@ -51,7 +51,7 @@ env
 
 # K3S files
 kubeconfig.yaml
-e2e/framework/maestro/kubeconfig.yaml
+e2e/framework/maestro/.k3d-kubeconfig.yaml
 
 # MkDocs
 site/

.golangci.yaml

@@ -1,16 +1,17 @@
+version: "2"
 run:
   timeout: 5m
   build-tags:
     - unit
     - integration
     - wireinject
-  skip-dirs:
-    - ".*/mock/.*"
-  skip-files:
+
+linters:
+  exclusions:
+    paths:
+    - "mock"
     - "_gen.go$"
     - "pb.go$"
     - "pb.gw.go$"
-
-linters:
   enable:
     - goimports

Dockerfile

@@ -19,7 +19,11 @@
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #
 
-FROM golang:1.22-alpine AS build-env
+FROM golang:1.23-alpine AS build-env
+
+# TARGETARCH and TARGETOS are automatically set by buildx
+ARG TARGETARCH
+ARG TARGETOS
 
 WORKDIR /build
 
@@ -30,13 +34,12 @@ COPY . /build
 
 RUN mkdir -p /app
 
-RUN apk add --update make
-
-RUN cd /build && \
-    make build-linux-x86_64 && \
-    mv ./bin/maestro-linux-x86_64 /app/maestro && \
-    mv internal/service/migrations /app/migrations && \
-    mv ./config /app/config
+# Instead of 'make build', compile directly for the target platform
+# Output the binary directly to /app/maestro
+RUN echo "Building for $TARGETOS/$TARGETARCH" && \
+    CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -a -installsuffix cgo -o /app/maestro ./ && \
+    mv /build/internal/service/migrations /app/migrations && \
+    mv /build/config /app/config
 
 
 FROM alpine
@@ -49,4 +52,4 @@ COPY --from=build-env /app/config /app/config
 
 EXPOSE 8080 8081
 
-CMD /app/maestro
+CMD ["/app/maestro"]

Makefile

@@ -34,7 +34,7 @@ lint: lint/go lint/protobuf ## Execute linters.
 
 .PHONY: lint/go
 lint/go: ## Execute golangci-lint.
-	@go run github.com/golangci/golangci-lint/cmd/golangci-lint@v1.51.0 run
+	@go run github.com/golangci/golangci-lint/cmd/golangci-lint@v1.64.5 run
 
 .PHONY: lint/protobuf
 lint/protobuf: ## Execute buf linter.
@@ -48,35 +48,63 @@ run/unit-tests: ## Execute unit tests.
 	@go test -count=1 -tags=unit -coverprofile=coverage.out -covermode=atomic $(OPTS) ./...
 
 .PHONY: run/integration-tests
-run/integration-tests: ## Execute integration tests.
-	@go test -tags=integration -count=1 -timeout 20m -coverprofile=coverage.out -covermode=atomic $(OPTS) ./...
+run/integration-tests: k3d/up ## Execute integration tests.
+	@KUBECONFIG_PATH="$(shell pwd)/e2e/framework/maestro/.k3d-kubeconfig.yaml"; \
+	echo "INFO: Checking for Kubeconfig at $$KUBECONFIG_PATH for integration tests..."; \
+	if [ -f "$$KUBECONFIG_PATH" ] && [ -s "$$KUBECONFIG_PATH" ]; then \
+		echo "INFO: Using kubeconfig from $$KUBECONFIG_PATH for integration tests..."; \
+		KUBECONFIG="$$KUBECONFIG_PATH" go test -tags=integration -count=1 -timeout 20m -coverprofile=coverage.out -covermode=atomic $(OPTS) ./... ; \
+	else \
+		echo "ERROR: Kubeconfig $$KUBECONFIG_PATH not found or is empty. Please ensure 'make k3d/up' completed successfully before running integration tests."; \
+		exit 1; \
+	fi
 
 .PHONY: run/runtime-integration-tests
-run/runtime-integration-tests: ## Execute runtime integration tests.
-	@go test -tags=integration -count=1 -timeout 20m ./internal/adapters/runtime/kubernetes/...
+run/runtime-integration-tests: k3d/up ## Execute runtime integration tests.
+	@KUBECONFIG_PATH="$(shell pwd)/e2e/framework/maestro/.k3d-kubeconfig.yaml"; \
+	echo "INFO: Checking for Kubeconfig at $$KUBECONFIG_PATH for runtime integration tests..."; \
+	if [ -f "$$KUBECONFIG_PATH" ] && [ -s "$$KUBECONFIG_PATH" ]; then \
+		echo "INFO: Using kubeconfig from $$KUBECONFIG_PATH for runtime integration tests..."; \
+		KUBECONFIG="$$KUBECONFIG_PATH" go test -tags=integration -count=1 -timeout 20m ./internal/adapters/runtime/kubernetes/... ; \
+	else \
+		echo "ERROR: Kubeconfig $$KUBECONFIG_PATH not found or is empty. Please ensure 'make k3d/up' completed successfully before running runtime integration tests."; \
+		exit 1; \
+	fi
 
 .PHONY: license-check
 license-check: ## Execute license check.
 	@go run github.com/google/addlicense -skip yaml -skip yml -skip proto -check .
 
 .PHONY: run/e2e-tests
-run/e2e-tests: build-linux-x86_64 setup/e2e deps/stop ## Execute end-to-end tests.
+run/e2e-tests: k3d/up build setup/e2e deps/stop ## Execute end-to-end tests.
 	@cd e2e; go test -count=1 -v $(OPTS) ./suites/...
 
 #-------------------------------------------------------------------------------
 #  Build and run
 #-------------------------------------------------------------------------------
 
 .PHONY: build
-build: build-linux-x86_64 ## Build the project and generates a binary.
-	@rm -f ./bin/maestro || true
-	@go build -o ./bin/maestro ./
+build: ## Build the project and generates a binary.
+ifeq ($(shell go env GOARCH),amd64)
+	@$(MAKE) build-linux-x86_64
+	@ln -sf ./bin/maestro-linux-x86_64 ./bin/maestro
+else ifeq ($(shell go env GOARCH),arm64)
+	@$(MAKE) build-linux-arm64
+	@ln -sf ./bin/maestro-linux-arm64 ./bin/maestro
+else
+	@echo "Unsupported architecture: $(shell go env GOARCH)"
+	@exit 1
+endif
 
 .PHONY: build-linux-x86_64
 build-linux-x86_64: ## Build the project and generates a binary for x86_64 architecture.
 	@rm -f ./bin/maestro-linux-x86_64 || true
 	@env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -installsuffix cgo -o ./bin/maestro-linux-x86_64 ./
 
+.PHONY: build-linux-arm64
+build-linux-arm64: ## Build the project and generates a binary for arm64 architecture.
+	@rm -f ./bin/maestro-linux-arm64 || true
+	@env CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -a -installsuffix cgo -o ./bin/maestro-linux-arm64 ./
 
 .PHONY: run/worker
 run/worker: ## Runs maestro worker.
@@ -150,15 +178,56 @@ deps/down: ## Delete containers dependencies.
 #  Easily start Maestro
 #-------------------------------------------------------------------------------
 .PHONY: maestro/start
-maestro/start: build-linux-x86_64 ## Start Maestro with all of its dependencies.
+maestro/start: k3d/up build ## Start Maestro with all of its dependencies.
 	@echo "Starting maestro..."
 	@cd ./e2e/framework/maestro; docker compose up --build -d
 	@MAESTRO_MIGRATION_PATH="file://internal/service/migrations" go run main.go migrate;
 	@cd ./e2e/framework/maestro; docker compose up --build -d worker runtime-watcher #Worker and watcher do not work before migration, so we start them after it.
 	@echo "Maestro is up and running!"
 
 .PHONY: maestro/down
-maestro/down: ## Delete Maestro and all of its dependencies.
+maestro/down: k3d/down ## Delete Maestro and all of its dependencies.
 	@echo "Deleting maestro..."
 	@cd ./e2e/framework/maestro; docker compose down
+	@$(MAKE) k3d/down
 	@echo "Maestro was deleted with success!"
+
+#-------------------------------------------------------------------------------
+#  Local Kubernetes cluster (k3d)
+#-------------------------------------------------------------------------------
+
+K3D_CLUSTER_NAME ?= maestro-dev
+
+.PHONY: k3d/up
+k3d/up: ## Create/Start the k3d cluster via docker-compose.
+	@echo "INFO: Ensuring k3d service is up and cluster '$(K3D_CLUSTER_NAME)' is created via docker-compose..."
+	@docker-compose up -d k3d
+	@echo "INFO: Waiting for k3d cluster and kubeconfig file at ./e2e/framework/maestro/.k3d-kubeconfig.yaml..."
+	@timeout=120; \
+	interval=5; \
+	elapsed=0; \
+	until [ -f ./e2e/framework/maestro/.k3d-kubeconfig.yaml ] && [ -s ./e2e/framework/maestro/.k3d-kubeconfig.yaml ]; do \
+		if [ $$elapsed -ge $$timeout ]; then \
+			echo "ERROR: Timeout waiting for kubeconfig file."; \
+			exit 1; \
+		fi; \
+		echo "INFO: Waiting for ./e2e/framework/maestro/.k3d-kubeconfig.yaml to be created and populated by k3d service (elapsed $$elapsed sec)..."; \
+		sleep $$interval; \
+		elapsed=$$((elapsed + interval)); \
+	done
+	@echo "INFO: k3d cluster '$(K3D_CLUSTER_NAME)' should be ready and kubeconfig exported."
+	@echo "To use with kubectl, run: export KUBECONFIG=$(shell pwd)/e2e/framework/maestro/.k3d-kubeconfig.yaml"
+
+.PHONY: k3d/down
+k3d/down: ## Delete the local k3d cluster and stop the k3d docker-compose service.
+	@echo "INFO: Attempting to delete k3d cluster '$(K3D_CLUSTER_NAME)' via k3d service (if running)..."
+	@if docker-compose ps k3d | grep -q "k3d"; then \
+		docker-compose exec k3d k3d cluster delete $(K3D_CLUSTER_NAME) || echo "INFO: k3d cluster delete command failed or cluster was not found."; \
+	else \
+		echo "INFO: k3d service not running, skipping cluster deletion command."; \
+	fi
+	@echo "INFO: Stopping and removing k3d service..."
+	@docker-compose rm -sf k3d
+	@echo "INFO: Removing k3d kubeconfig file..."
+	@rm -f ./e2e/framework/maestro/.k3d-kubeconfig.yaml
+	@echo "INFO: k3d cluster and service are down."