chore: fix local setup with k3d (#670)

Add docker deployments into a common service and make it
accessible to k3d load balancer exposing kubernetes cluster

Author: hspedro

Makefile

@@ -4,6 +4,9 @@ SOURCES := $(shell \
 
 BUF := github.com/bufbuild/buf/cmd/buf@v1.32.1
 
+# Detect Docker Compose command - prefer 'docker compose' over 'docker-compose'
+DOCKER_COMPOSE := $(shell if command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then echo "docker compose"; elif command -v docker-compose >/dev/null 2>&1; then echo "docker-compose"; else echo "docker-compose"; fi)
+
 .PHONY: help
 help: Makefile ## Show list of commands.
 	@echo "Choose a command to run in "$(APP_NAME)":"
@@ -152,25 +155,25 @@ migrate: ## Execute migration.
 .PHONY: deps/up
 deps/up: ## Create containers dependencies.
 	@echo "Creating dependencies "
-	@docker-compose up -d --build
+	@$(DOCKER_COMPOSE) up -d --build
 	@echo "Dependencies created successfully."
 
 .PHONY: deps/start
 deps/start: ## Start containers dependencies.
 	@echo "Starting dependencies "
-	@docker-compose start
+	@$(DOCKER_COMPOSE) start
 	@echo "Dependencies started successfully."
 
 .PHONY: deps/stop
 deps/stop: ## Stop containers dependencies.
 	@echo "Stopping dependencies "
-	@docker-compose stop
+	@$(DOCKER_COMPOSE) stop
 	@echo "Dependencies stopped successfully."
 
 .PHONY: deps/down
 deps/down: ## Delete containers dependencies.
 	@echo "Deleting dependencies "
-	@docker-compose down
+	@$(DOCKER_COMPOSE) down
 	@echo "Dependencies deleted successfully."
 
 
@@ -179,16 +182,18 @@ deps/down: ## Delete containers dependencies.
 #-------------------------------------------------------------------------------
 .PHONY: maestro/start
 maestro/start: k3d/up build ## Start Maestro with all of its dependencies.
-	@echo "Starting maestro..."
-	@cd ./e2e/framework/maestro; docker compose up --build -d
+	@echo "Starting maestro dependencies (postgres, redis)..."
+	@cd ./e2e/framework/maestro; $(DOCKER_COMPOSE) up -d postgres redis
+	@echo "Running database migrations..."
 	@MAESTRO_MIGRATION_PATH="file://internal/service/migrations" go run main.go migrate;
-	@cd ./e2e/framework/maestro; docker compose up --build -d worker runtime-watcher #Worker and watcher do not work before migration, so we start them after it.
+	@echo "Starting maestro application services..."
+	@cd ./e2e/framework/maestro; $(DOCKER_COMPOSE) up --build -d
 	@echo "Maestro is up and running!"
 
 .PHONY: maestro/down
 maestro/down: k3d/down ## Delete Maestro and all of its dependencies.
 	@echo "Deleting maestro..."
-	@cd ./e2e/framework/maestro; docker compose down
+	@cd ./e2e/framework/maestro; $(DOCKER_COMPOSE) down
 	@$(MAKE) k3d/down
 	@echo "Maestro was deleted with success!"
 
@@ -201,7 +206,7 @@ K3D_CLUSTER_NAME ?= maestro-dev
 .PHONY: k3d/up
 k3d/up: ## Create/Start the k3d cluster via docker-compose.
 	@echo "INFO: Ensuring k3d service is up and cluster '$(K3D_CLUSTER_NAME)' is created via docker-compose..."
-	@docker-compose up -d k3d
+	@$(DOCKER_COMPOSE) up -d k3d
 	@echo "INFO: Waiting for k3d cluster and kubeconfig file at ./e2e/framework/maestro/.k3d-kubeconfig.yaml..."
 	@timeout=120; \
 	interval=5; \
@@ -221,13 +226,13 @@ k3d/up: ## Create/Start the k3d cluster via docker-compose.
 .PHONY: k3d/down
 k3d/down: ## Delete the local k3d cluster and stop the k3d docker-compose service.
 	@echo "INFO: Attempting to delete k3d cluster '$(K3D_CLUSTER_NAME)' via k3d service (if running)..."
-	@if docker-compose ps k3d | grep -q "k3d"; then \
-		docker-compose exec k3d k3d cluster delete $(K3D_CLUSTER_NAME) || echo "INFO: k3d cluster delete command failed or cluster was not found."; \
+	@if $(DOCKER_COMPOSE) ps k3d | grep -q "k3d"; then \
+		$(DOCKER_COMPOSE) exec k3d k3d cluster delete $(K3D_CLUSTER_NAME) || echo "INFO: k3d cluster delete command failed or cluster was not found."; \
 	else \
 		echo "INFO: k3d service not running, skipping cluster deletion command."; \
 	fi
 	@echo "INFO: Stopping and removing k3d service..."
-	@docker-compose rm -sf k3d
+	@$(DOCKER_COMPOSE) rm -sf k3d
 	@echo "INFO: Removing k3d kubeconfig file..."
 	@rm -f ./e2e/framework/maestro/.k3d-kubeconfig.yaml
 	@echo "INFO: k3d cluster and service are down."

cmd/k3d/start.sh

@@ -0,0 +1,77 @@
+# MIT License
+#
+# Copyright (c) 2021 TFG Co
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+#!/bin/sh
+
+set -e
+
+K3D_CLUSTER_NAME=${K3D_CLUSTER_NAME:-maestro-dev}
+KUBECONFIG_OUT_DIR="/kubeconfig-out"
+KUBECONFIG_FILE_PATH="$KUBECONFIG_OUT_DIR/.k3d-kubeconfig.yaml"
+KUBECONFIG_TMP_FILE_PATH="$KUBECONFIG_FILE_PATH.tmp"
+
+# Check if cluster exists
+if ! k3d cluster get $K3D_CLUSTER_NAME > /dev/null 2>&1; then
+  echo "INFO: k3d cluster '$K3D_CLUSTER_NAME' not found. Creating it now..."
+  k3d cluster create $K3D_CLUSTER_NAME \
+    --agents 1 \
+    --port '38080:80@loadbalancer' \
+    --api-port '127.0.0.1:6443' \
+    --network 'k3d-maestro-dev' \
+    --wait
+  echo "INFO: k3d cluster '$K3D_CLUSTER_NAME' created successfully."
+else
+  # Check if running, start if not
+  if ! k3d cluster get $K3D_CLUSTER_NAME --no-headers 2>/dev/null | grep -q 'running'; then
+    echo "INFO: k3d cluster '$K3D_CLUSTER_NAME' exists but is not running. Starting it..."
+    k3d cluster start $K3D_CLUSTER_NAME --wait
+  else
+    echo "INFO: k3d cluster '$K3D_CLUSTER_NAME' already exists and is running."
+  fi
+fi
+
+echo "INFO: Exporting kubeconfig for cluster '$K3D_CLUSTER_NAME'..."
+mkdir -p $KUBECONFIG_OUT_DIR
+
+# Get the kubeconfig and create two versions:
+# 1. Host-accessible version (for external tools)
+# 2. Container-internal version (for kubectl inside the container)
+k3d kubeconfig get $K3D_CLUSTER_NAME > $KUBECONFIG_TMP_FILE_PATH
+
+# Create host-accessible version (external tools can use this)
+sed 's/server: https:\/\/127.0.0.1:6443/server: https:\/\/k3d-maestro-dev-serverlb:6443/g' $KUBECONFIG_TMP_FILE_PATH > $KUBECONFIG_FILE_PATH
+
+# Create container-internal version (for kubectl inside the container)
+# This should point to the load balancer from within the k3d network
+cp $KUBECONFIG_TMP_FILE_PATH /root/.kube/config
+sed -i 's/server: https:\/\/127.0.0.1:6443/server: https:\/\/k3d-maestro-dev-serverlb:6443/g' /root/.kube/config
+
+rm $KUBECONFIG_TMP_FILE_PATH
+
+echo "INFO: Kubeconfig exported. k3d service ready and will keep running."
+echo "INFO: Container-internal kubeconfig configured at /root/.kube/config for kubectl access"
+echo " ---- KUBECONFIG START ---- "
+cat $KUBECONFIG_FILE_PATH
+echo " ---- KUBECONFIG END ---- "
+
+# Keep container running
+tail -f /dev/null

docker-compose.yaml

@@ -11,7 +11,7 @@ services:
     - POSTGRES_USER=maestro
     - POSTGRES_PASSWORD=maestro
     healthcheck:
-      test: [ "CMD-SHELL", "pg_isready -U postgres" ]
+      test: [ "CMD-SHELL", "pg_isready -U maestro" ]
       interval: 10s
       timeout: 10s
       retries: 5
@@ -28,42 +28,21 @@ services:
 
   k3d:
     image: rancher/k3d:5.3.0
-    command: >
-      sh -c "
-        K3D_CLUSTER_NAME=$${K3D_CLUSTER_NAME:-maestro-dev};
-        if ! k3d cluster get $$K3D_CLUSTER_NAME > /dev/null 2>&1; then
-          echo 'INFO: k3d cluster '$$K3D_CLUSTER_NAME' not found. Creating it now...';
-          k3d cluster create $$K3D_CLUSTER_NAME \\
-            --agents 1 \\
-            --port '38080:80@loadbalancer' \\
-            --api-port '127.0.0.1:6443' \\
-            --wait;
-          echo 'INFO: k3d cluster '$$K3D_CLUSTER_NAME' created successfully.';
-        else
-          # Check if running, start if not
-          # Redirecting stderr for grep to avoid issues if cluster doesn't exist during check
-          if ! k3d cluster get $$K3D_CLUSTER_NAME --no-headers 2>/dev/null | grep -q 'running'; then
-            echo 'INFO: k3d cluster '$$K3D_CLUSTER_NAME' exists but is not running. Starting it...';
-            k3d cluster start $$K3D_CLUSTER_NAME --wait;
-          else
-            echo 'INFO: k3d cluster '$$K3D_CLUSTER_NAME' already exists and is running.';
-          fi
-        fi &&
-        echo 'INFO: Exporting kubeconfig for cluster '$$K3D_CLUSTER_NAME'...';
-        mkdir -p /kubeconfig-out &&
-        k3d kubeconfig get $$K3D_CLUSTER_NAME > /kubeconfig-out/.k3d-kubeconfig.yaml &&
-        echo 'INFO: Kubeconfig exported. k3d service ready and will keep running.' &&
-        echo ' ---- KUBECONFIG START ---- ' &&
-        cat /kubeconfig-out/.k3d-kubeconfig.yaml &&
-        echo ' ---- KUBECONFIG END ---- ' &&
-        tail -f /dev/null
-      "
+    command: [ "/bin/sh", "/app/start.sh" ]
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
-      - ./e2e/framework/maestro:/kubeconfig-out
+      - ${PWD}/e2e/framework/maestro:/kubeconfig-out
+      - ./cmd/k3d/start.sh:/app/start.sh
     environment:
       - K3D_CLUSTER_NAME=maestro-dev
+    networks:
+      - default
+      - k3d
 
 volumes:
   k3s-server: {}
   postgres: {}
+
+networks:
+  k3d:
+    name: k3d-maestro-dev

docs/reference/Kubernetes.md

@@ -93,11 +93,11 @@ Maestro needs the following permissions for managing resources in a kubernetes c
 Maestro provides two ways for configuring kubernetes cluster access.
 
 ### Using inCluster mode
-Set `adapters.runtime.kubernetes.inCluster` config value to true or use its env var equivalent, the kubernetes client will be configured 
+Set `adapters.runtime.kubernetes.inCluster` config value to true or use its env var equivalent, the kubernetes client will be configured
 automatically using the same _service account_ of the maestro component running pod.
 
 The kubernetes client has a default rate limiter which implements a token bucket approach.
-The inCluster mode allows rate limit changes via `adapters.runtime.kubernetes.qps` 
+The inCluster mode allows rate limit changes via `adapters.runtime.kubernetes.qps`
 and `adapters.runtime.kubernetes.burst` configs.
 
 This mode is recommended to be used when running maestro components in the same cluster

e2e/framework/maestro/Dockerfile

@@ -19,14 +19,32 @@
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #
 
-FROM alpine:3
+FROM golang:1.23-alpine as builder
 
-MAINTAINER TFG Co <backend@tfgco.com>
+WORKDIR /app
+
+# Copy go.mod and go.sum files to download dependencies
+COPY go.mod ./
+COPY go.sum ./
+RUN go get ./...
+RUN go mod download
+
+# Copy the entire source code
+COPY . .
+
+# Build the binary statically, ensuring it includes our latest changes
+# The -a flag forces rebuilding of packages that are already up-to-date.
+RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o ./maestro ./
+
+# Stage 2: Create the final, minimal image
+FROM alpine:3.15
 
 WORKDIR /app
 
-COPY ./bin/maestro-linux-x86_64 /app/maestro
-COPY ./config /app/config
+# Copy only the compiled binary from the builder stage
+COPY --from=builder /app/maestro .
+# Copy the static configuration files
+COPY ./config ./config
 
 ENTRYPOINT ["/app/maestro"]
 CMD ["--help"]

e2e/framework/maestro/docker-compose.yml

@@ -4,12 +4,14 @@ services:
     build:
       context: ../../..
       dockerfile: ./e2e/framework/maestro/Dockerfile
+      no_cache: true
     environment:
       - MAESTRO_ADAPTERS_ROOMSTORAGE_REDIS_URL=redis://redis:6379/0
       - MAESTRO_ADAPTERS_SCHEDULERSTORAGE_POSTGRES_URL=postgres://maestro:maestro@postgres:5432/maestro?sslmode=disable
       - MAESTRO_ADAPTERS_OPERATIONFLOW_REDIS_URL=redis://redis:6379/0
       - MAESTRO_ADAPTERS_OPERATIONSTORAGE_REDIS_URL=redis://redis:6379/0
       - MAESTRO_ADAPTERS_OPERATIONLEASESTORAGE_REDIS_URL=redis://redis:6379/0
+      - MAESTRO_ADAPTERS_RUNTIME_KUBERNETES_MASTERURL=
       - MAESTRO_INTERNALAPI_PORT=8081
       - MAESTRO_API_PORT=8080
     ports:
@@ -21,14 +23,19 @@ services:
         condition: service_healthy
       redis:
         condition: service_healthy
+    networks:
+      - default
+      - k3d
 
   rooms-api:
     build:
       context: ../../..
       dockerfile: ./e2e/framework/maestro/Dockerfile
+      no_cache: true
     environment:
       - MAESTRO_ADAPTERS_ROOMSTORAGE_REDIS_URL=redis://redis:6379/0
       - MAESTRO_ADAPTERS_INSTANCESTORAGE_REDIS_URL=redis://redis:6379/0
+      - MAESTRO_ADAPTERS_RUNTIME_KUBERNETES_MASTERURL=k3d-maestro-dev-serverlb:6443
       - MAESTRO_ADAPTERS_RUNTIME_KUBERNETES_KUBECONFIG=/kubeconfig/config
       - MAESTRO_ADAPTERS_SCHEDULERCACHE_REDIS_URL=redis://redis:6379/0
       - MAESTRO_ADAPTERS_SCHEDULERSTORAGE_POSTGRES_URL=postgres://maestro:maestro@postgres:5432/maestro?sslmode=disable
@@ -45,15 +52,20 @@ services:
         condition: service_healthy
       redis:
         condition: service_healthy
+    networks:
+      - default
+      - k3d
 
   runtime-watcher:
     build:
       context: ../../..
       dockerfile: ./e2e/framework/maestro/Dockerfile
+      no_cache: true
     environment:
       - MAESTRO_ADAPTERS_SCHEDULERSTORAGE_POSTGRES_URL=postgres://maestro:maestro@postgres:5432/maestro?sslmode=disable
       - MAESTRO_ADAPTERS_ROOMSTORAGE_REDIS_URL=redis://redis:6379/0
       - MAESTRO_ADAPTERS_INSTANCESTORAGE_REDIS_URL=redis://redis:6379/0
+      - MAESTRO_ADAPTERS_RUNTIME_KUBERNETES_MASTERURL=
       - MAESTRO_ADAPTERS_RUNTIME_KUBERNETES_KUBECONFIG=/kubeconfig/config
       - MAESTRO_ADAPTERS_SCHEDULERCACHE_REDIS_URL=redis://redis:6379/0
       - MAESTRO_INTERNALAPI_PORT=8061
@@ -62,18 +74,23 @@ services:
     volumes:
       - ./.k3d-kubeconfig.yaml:/kubeconfig/config:ro
     command: [start, runtime-watcher, -l, development]
+    networks:
+      - default
+      - k3d
 
   worker:
     build:
       context: ../../..
       dockerfile: ./e2e/framework/maestro/Dockerfile
+      no_cache: true
     environment:
       - MAESTRO_ADAPTERS_SCHEDULERSTORAGE_POSTGRES_URL=postgres://maestro:maestro@postgres:5432/maestro?sslmode=disable
       - MAESTRO_ADAPTERS_OPERATIONFLOW_REDIS_URL=redis://redis:6379/0
       - MAESTRO_ADAPTERS_OPERATIONSTORAGE_REDIS_URL=redis://redis:6379/0
       - MAESTRO_ADAPTERS_ROOMSTORAGE_REDIS_URL=redis://redis:6379/0
       - MAESTRO_ADAPTERS_OPERATIONLEASESTORAGE_REDIS_URL=redis://redis:6379/0
       - MAESTRO_ADAPTERS_INSTANCESTORAGE_REDIS_URL=redis://redis:6379/0
+      - MAESTRO_ADAPTERS_RUNTIME_KUBERNETES_MASTERURL=
       - MAESTRO_ADAPTERS_RUNTIME_KUBERNETES_KUBECONFIG=/kubeconfig/config
       - MAESTRO_ADAPTERS_RUNTIME_KUBERNETES_TOPOLOGYSPREADCONSTRAINT_ENABLED=false
       - MAESTRO_ADAPTERS_SCHEDULERCACHE_REDIS_URL=redis://redis:6379/0
@@ -90,6 +107,9 @@ services:
         condition: service_healthy
       redis:
         condition: service_healthy
+    networks:
+      - default
+      - k3d
 
   postgres:
     image: postgres:17-alpine
@@ -100,10 +120,12 @@ services:
     - POSTGRES_USER=maestro
     - POSTGRES_PASSWORD=maestro
     healthcheck:
-      test: [ "CMD-SHELL", "pg_isready -U postgres" ]
+      test: [ "CMD-SHELL", "pg_isready -U maestro" ]
       interval: 10s
       timeout: 10s
       retries: 5
+    networks:
+      - default
 
   redis:
     image: redis:8.0-alpine
@@ -114,6 +136,14 @@ services:
       interval: 10s
       timeout: 10s
       retries: 5
+    networks:
+      - default
 
 volumes:
   eventsproto: {}
+
+networks:
+  default:
+  k3d:
+    name: k3d-maestro-dev
+    external: true