feature: debug builds for the host operator

In order to be able to debug the operator live in the local cluster, we
need the binary to be executed with Delve instead, and we need it to be
built without any code optimizations or minimization so that the
breakpoints work.

For that purpose, a new Dockerfile is added which can build the image
that way, and the corresponding Make targets make it easy to kick off
the whole process with a single command.

The Makefile targets also enable the "toolchain-e2e" repository to
easily build the "debug" image.

Also, when Developer Sandbox is deployed locally, usually the
registration service gets managed by the "ToolChain config controller",
which launches it with a specific command. In order to be able to launch
the registration service with Delve, a few minor modifications are made
so that that launch command can be overridden.

SANDBOX-1561

Author: MikelAlejoBR

build/Dockerfile.debug

@@ -0,0 +1,55 @@
+# Compile Delve in an intermediate step.
+FROM registry.access.redhat.com/ubi8/ubi:latest AS delve-builder
+
+# The Golang version must be provided as a build arguments, which will ensure
+# that Delve gets built with the same version the service's binary gets built
+# with, to avoid any discrepancies.
+ARG GOLANG_VERSION
+
+# Install Tar to be able to extract Golang.
+RUN yum install --assumeyes \
+    tar \
+    && yum clean all
+
+# Download and extract Golang.
+RUN curl --location --output "${GOLANG_VERSION}.linux-amd64.tar.gz" "https://go.dev/dl/${GOLANG_VERSION}.linux-amd64.tar.gz" \
+    && curl --location --output "${GOLANG_VERSION}.linux-amd64.tar.gz.sha256" "https://go.dev/dl/${GOLANG_VERSION}.linux-amd64.tar.gz.sha256" \
+    && echo "$(cat ${GOLANG_VERSION}.linux-amd64.tar.gz.sha256) ${GOLANG_VERSION}.linux-amd64.tar.gz" | sha256sum --check \
+    && tar --directory /usr/local --extract --file "${GOLANG_VERSION}.linux-amd64.tar.gz" \
+    && rm --force "${GOLANG_VERSION}.linux-amd64.tar.gz"
+
+# Put Golang in the path so that we can Delve with it.
+ENV PATH=$PATH:/usr/local/go/bin
+
+# Build Delve and leave it in a temporary directory.
+RUN GOBIN=/tmp/bin go install github.com/go-delve/delve/cmd/dlv@latest
+
+# Build the operator as normal.
+FROM registry.access.redhat.com/ubi8/ubi-minimal:latest
+
+LABEL maintainer = "KubeSaw <devsandbox@redhat.com>"
+LABEL author = "KubeSaw <devsandbox@redhat.com>"
+
+# Ensure that the "DEBUG_MODE" is enabled so that the binary executes with
+# Delve.
+ENV DEBUG_MODE=true \
+    OPERATOR=/usr/local/bin/host-operator \
+    USER_UID=1001 \
+    USER_NAME=host-operator \
+    LANG=en_US.utf8
+
+# Install operator binary in the image.
+COPY build/_output/bin/host-operator ${OPERATOR}
+
+COPY build/bin /usr/local/bin
+RUN  /usr/local/bin/user_setup
+
+# Copy Delve to the image so that we can execute the operator with it.
+COPY --from=delve-builder /tmp/bin/dlv /usr/local/bin/dlv
+
+ENTRYPOINT ["/usr/local/bin/entrypoint"]
+
+# Expose the debugger's port.
+EXPOSE 50000
+
+USER ${USER_UID}

build/bin/entrypoint

@@ -9,4 +9,9 @@ if ! whoami &>/dev/null; then
   fi
 fi
 
-exec ${OPERATOR} $@
+if [ -n "${DEBUG_MODE}" ]
+then
+  exec /usr/local/bin/dlv --listen=:50000 --headless --continue --api-version=2 --accept-multiclient exec "${OPERATOR}" "$@"
+else
+  exec ${OPERATOR} "$@"
+fi

controllers/toolchainconfig/env.go

@@ -1,3 +1,6 @@
 package toolchainconfig
 
-const RegistrationServiceImageEnvKey = "REGISTRATION_SERVICE_IMAGE"
+const (
+	RegistrationServiceImageEnvKey   = "REGISTRATION_SERVICE_IMAGE"
+	RegistrationServiceCommandEnvKey = "REGISTRATION_SERVICE_COMMAND"
+)

controllers/toolchainconfig/toolchainconfig_controller.go

@@ -151,6 +151,16 @@ func getVars(namespace string, cfg ToolchainConfig) templateVars {
 	vars["IMAGE"] = image
 	vars.addIfNotEmpty("NAMESPACE", namespace)
 	vars.addIfNotEmpty("REPLICAS", fmt.Sprint(cfg.RegistrationService().Replicas()))
+
+	// Allow overriding the registration service's command via an environment
+	// variable.
+	command := os.Getenv(RegistrationServiceCommandEnvKey)
+	if command != "" {
+		vars["COMMAND"] = command
+	} else {
+		vars["COMMAND"] = `["registration-service"]`
+	}
+
 	return vars
 }
 

deploy/templates/registration-service/registration-service.yaml

@@ -96,13 +96,12 @@ objects:
               image: ${IMAGE}
               ports:
                 - containerPort: 8080 # registration service
-                - containerPort: 8081 # proxy 
+                - containerPort: 8081 # proxy
                 - containerPort: 8082 # proxy metrics
                   name: metrics
                 - containerPort: 8083 # registration service metrics
                   name: regsvc-metrics
-              command:
-                - registration-service
+              command: ${{COMMAND}}
               imagePullPolicy: IfNotPresent
               livenessProbe:
                 failureThreshold: 3
@@ -141,7 +140,7 @@ objects:
                 requests:
                   cpu: "50m"
                   memory: "100M"
-  
+
   # route for the registration service
   - kind: Route
     apiVersion: v1
@@ -202,7 +201,7 @@ objects:
         run: registration-service
       type: ClusterIP
       sessionAffinity: null
-  
+
   # route for the proxy
   - kind: Route
     apiVersion: v1
@@ -224,7 +223,7 @@ objects:
       tls:
         termination: edge
       wildcardPolicy: None
- 
+
   # service associated with the proxy route
   - kind: Service
     apiVersion: v1
@@ -244,7 +243,7 @@ objects:
         run: registration-service
       type: ClusterIP
       sessionAffinity: null
-  
+
   # internal service for the proxy, used by Prometheus to scrape the metrics
   - kind: Service
     apiVersion: v1
@@ -271,3 +270,5 @@ parameters:
     value: quay.io/openshiftio/codeready-toolchain/registration-service:latest
   - name: REPLICAS
     value: '3'
+  - name: COMMAND
+    value: '["registration-service"]'

make/go.mk

@@ -20,6 +20,19 @@ $(OUT_DIR)/operator:
 		-o $(OUT_DIR)/bin/host-operator \
 		./cmd/main.go
 
+.PHONY: build-debug
+## Build the operator without the optimizations and the inlining, to enable
+## remote debugging via Delve.
+build-debug: generate
+	@echo "building host-operator in ${GO_PACKAGE_PATH}"
+	$(Q)go version
+	$(Q)CGO_ENABLED=0 GOARCH=${goarch} GOOS=linux \
+		go build ${V_FLAG} \
+		-gcflags "all=-N -l" \
+		-ldflags "-X ${GO_PACKAGE_PATH}/version.Commit=${GIT_COMMIT_ID} -X ${GO_PACKAGE_PATH}/version.BuildTime=${BUILD_TIME}" \
+		-o $(OUT_DIR)/bin/host-operator \
+		./cmd/main.go
+
 .PHONY: vendor
 vendor:
 	$(Q)go mod vendor

make/podman.mk

@@ -10,11 +10,21 @@ IMAGE_PLATFORM ?= linux/amd64
 podman-image: build
 	$(Q)podman build --platform ${IMAGE_PLATFORM} -f build/Dockerfile -t ${IMAGE} .
 
+## Build the operator's image with Delve on it so that it is ready to attach a
+## debugger to it.
+podman-image-debug: build-debug
+	$(Q) podman build --platform ${IMAGE_PLATFORM} --build-arg GOLANG_VERSION="$$(go version | awk '{print $$3}')" --file build/Dockerfile.debug --tag ${IMAGE} .
+
 .PHONY: podman-push
 ## Push the binary image to quay.io registry
 podman-push: check-namespace podman-image
 	$(Q)podman push ${IMAGE}
 
+.PHONY: podman-push-debug
+## Push the image with the debugger in it to the repository.
+podman-push-debug: check-namespace podman-image-debug
+	$(Q)podman push ${IMAGE}
+
 .PHONY: check-namespace
 check-namespace:
 ifeq ($(QUAY_NAMESPACE),${GO_PACKAGE_ORG_NAME})