Implement health probe and configuration (#199)

* Implement health probes without configuration

* Development fix

* Update services/health/probes_service.go

Co-authored-by: Copilot <[email protected]>

* Fix incorrect listenAndServe behavior

* Add coverage exclusions

* One more fix

* Implement configuration subsystem

* Finalize the implementation

* Finalize the implementation

* Final fixes

---------

Co-authored-by: Copilot <[email protected]>

Author: s-vitaliy

.container/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.25-bookworm AS build-stage
+FROM --platform=$BUILDPLATFORM golang:1.25-bookworm AS build-stage
 
 ARG TARGETOS 
 ARG TARGETARCH
@@ -26,7 +26,7 @@ WORKDIR /app
 
 COPY --from=build-stage /app /app
 
-COPY .container/appconfig.yaml /app/appconfig.yaml
+COPY appconfig.yaml /app/appconfig.yaml
 
 USER nonroot:nonroot
 

.container/appconfig.yaml

@@ -46,7 +46,7 @@ spec:
         imagePullPolicy: "{{ .Values.image.pullPolicy }}"
         livenessProbe:
           httpGet:
-            port: 8080
+            port: {{ .Values.settings.probes.port }}
             path: /health
           initialDelaySeconds: 10
           timeoutSeconds: 5
@@ -55,8 +55,8 @@ spec:
           failureThreshold: 10
         readinessProbe:
           httpGet:
-            port: 8080
-            path: /health
+            port: {{ .Values.settings.probes.port }}
+            path: /health/ready
           initialDelaySeconds: 10
           timeoutSeconds: 5
           periodSeconds: 30
@@ -67,6 +67,14 @@ spec:
             value: {{ .Values.environment }}
           - name: APPLICATION_VERSION
             value: "{{ (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }}"
+          - name: ARCANE_OPERATOR__PROBES__ADDR
+            value: {{ .Values.settings.probes.listenAddrOverride | default (printf "0.0.0.0:%v" .Values.settings.probes.port) }}
+          - name: ARCANE_OPERATOR__PROBES__WRITE_TIMEOUT
+            value: {{ .Values.settings.probes.writeTimeout | default  "15s"}}
+          - name: ARCANE_OPERATOR__PROBES__READ_TIMEOUT
+            value: {{ .Values.settings.probes.writeTimeout | default  "15s"}}
+          - name: ARCANE_OPERATOR__PROBES__SHUTDOWN_TIMEOUT
+            value: {{ .Values.settings.probes.writeTimeout | default  "5s"}}
           {{- with .Values.extraEnv }}
             {{- toYaml . | nindent 12 }}
           {{- end }}

.helm/templates/deployment.yaml

@@ -184,32 +184,25 @@ securityContext:
     type: RuntimeDefault
 
 settings:
-  # Settings for the streaming Job Maintenance controller
-  jobMaintenanceController:
-    
-    # The namespace where the streaming jobs are created and maintained
-    # Defaults to the namespace where the operator is deployed
-    jobNamespace: ""
-    
-    # The maximum number of streaming job events in the Akka event buffer
-    # If the number of events exceeds this value, the service throws an exception
-    # and will be restarted by the Deployment controller
-    maxEventBufferCapacity: 10000
-    
-  # Settings for the StreamClass controller
-  streamClassController:
-    
-    # The namespace where the streaming jobs are created and maintained
-    # Defaults to the namespace where the operator is deployed
-    streamClassesNamespace: ""
-    
-    # The maximum number of streaming job events in the Akka event buffer
-    # If the number of events exceeds this value, the service throws an exception
-    # and will be restarted by the Deployment controller
-    maxEventBufferCapacity: 10000
 
-    # Custom logger configuration
-  customLoggingProperties: { }
-  
-  # Custom logger minimum level overrides
-  customLoggingMinimumLevel: { }
+  # Configuration for the health probes
+  probes:
+
+    ## Address override for the probes to listen on (overrides the port setting or interface binding)
+    ## If not set, defaults to "0.0.0.0:<port>"
+    listenAddrOverride: ""
+
+    ## Port for the health probes to listen on
+    port: 8888
+
+    ## Timeout for write operations
+    ## If not set, defaults to 15s
+    writeTimeout: ""
+
+    ## Timeout for read operations
+    ## If not set, defaults to 15s
+    readTimeout: ""
+
+    ## Timeout for server shutdown
+    ## If not set, defaults to 5s
+    shutdownTimeout: ""

.helm/values.yaml

@@ -0,0 +1,5 @@
+probes:
+  addr: ":8888"
+  write-timeout: 15s
+  read-timeout: 15s
+  shutdown-timeout: 5s

appconfig.yaml

@@ -0,0 +1,10 @@
+package config
+
+import "github.com/SneaksAndData/arcane-operator/services/health"
+
+// AppConfig holds the application configuration settings.
+type AppConfig struct {
+
+	// ProbesConfiguration holds the configuration for health probes.
+	ProbesConfiguration health.ProbesConfig `mapstructure:"probes,omitempty"`
+}

config/configuration.go

@@ -0,0 +1,62 @@
+package config
+
+import (
+	"context"
+	"fmt"
+	"github.com/spf13/viper"
+	"k8s.io/klog/v2"
+	"os"
+	"strings"
+)
+
+const (
+	EnvPrefix = "ARCANE_OPERATOR_" // var names will be ARCANE_OPERATOR__MY_ENV_VAR or ARCANE_OPERATOR__SECTION1__SECTION2__MY_ENV_VAR
+)
+
+func configExists(configPath string) (bool, error) { // coverage-ignore (this should be covered in integration tests)
+	if _, err := os.Stat(configPath); os.IsNotExist(err) {
+		return false, nil
+	} else if err != nil { // coverage-ignore
+		return false, err
+	} else {
+		return true, nil
+	}
+}
+
+func LoadConfig[T any](ctx context.Context) (*T, error) { // coverage-ignore (this should be covered in integration tests)
+	logger := klog.FromContext(ctx)
+	customViper := viper.NewWithOptions(viper.KeyDelimiter("__"))
+	customViper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
+
+	localConfig := fmt.Sprintf("appconfig.%s.yaml", strings.ToLower(os.Getenv("APPLICATION_ENVIRONMENT")))
+
+	exists, err := configExists(localConfig)
+	if err != nil {
+		return nil, fmt.Errorf("error checking config for existance: %w", err)
+	}
+
+	if exists {
+		customViper.SetConfigFile(fmt.Sprintf("appconfig.%s.yaml", strings.ToLower(os.Getenv("APPLICATION_ENVIRONMENT"))))
+	} else {
+		logger.Info("no environment specific config found, loading default appconfig.yaml")
+		customViper.SetConfigFile("appconfig.yaml")
+	}
+
+	customViper.SetEnvPrefix(EnvPrefix)
+	customViper.AllowEmptyEnv(true)
+	customViper.AutomaticEnv()
+
+	err = customViper.ReadInConfig()
+	if err != nil { // coverage-ignore
+		return nil, fmt.Errorf("error reading config file: %w", err)
+	}
+
+	var appConfig T
+	err = customViper.Unmarshal(&appConfig)
+
+	if err != nil { // coverage-ignore
+		return nil, fmt.Errorf("error unmarshalling config: %w", err)
+	}
+
+	return &appConfig, nil
+}

config/management.go

@@ -10,6 +10,7 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/samber/slog-datadog/v2 v2.10.2
 	github.com/samber/slog-multi v1.6.0
+	github.com/spf13/viper v1.21.0
 	github.com/stretchr/testify v1.11.1
 	go.uber.org/mock v0.6.0
 	golang.org/x/sync v0.18.0
@@ -34,6 +35,7 @@ require (
 	github.com/go-openapi/jsonpointer v0.21.0 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/google/btree v1.1.3 // indirect
@@ -45,14 +47,20 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.22.0 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.62.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/sagikazarmark/locafero v0.11.0 // indirect
 	github.com/samber/lo v1.52.0 // indirect
 	github.com/samber/slog-common v0.19.0 // indirect
+	github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
+	github.com/spf13/afero v1.15.0 // indirect
+	github.com/spf13/cast v1.10.0 // indirect
 	github.com/spf13/pflag v1.0.10 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	go.yaml.in/yaml/v2 v2.4.3 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect

go.mod

@@ -24,6 +24,8 @@ github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8
 github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
 github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU=
 github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM=
+github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
+github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
 github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=
@@ -42,6 +44,8 @@ github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+Gr
 github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
 github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
 github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
+github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
+github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@@ -91,6 +95,8 @@ github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns
 github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo=
 github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A=
 github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k=
+github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
+github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -105,6 +111,8 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg
 github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
 github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
 github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
+github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
+github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
 github.com/samber/lo v1.52.0 h1:Rvi+3BFHES3A8meP33VPAxiBZX/Aws5RxrschYGjomw=
 github.com/samber/lo v1.52.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0=
 github.com/samber/slog-common v0.19.0 h1:fNcZb8B2uOLooeYwFpAlKjkQTUafdjfqKcwcC89G9YI=
@@ -114,8 +122,16 @@ github.com/samber/slog-datadog/v2 v2.10.2/go.mod h1:4OaxK5ESM2j6EhPXyFBEVXjjvWmd
 github.com/samber/slog-multi v1.6.0 h1:i1uBY+aaln6ljwdf7Nrt4Sys8Kk6htuYuXDHWJsHtZg=
 github.com/samber/slog-multi v1.6.0/go.mod h1:qTqzmKdPpT0h4PFsTN5rYRgLwom1v+fNGuIrl1Xnnts=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
+github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U=
+github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
+github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
+github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
+github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
 github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
 github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
+github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -128,6 +144,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
+github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

go.sum

@@ -27,4 +27,4 @@ mock-stream-plugin:
         --namespace default \
         --set jobTemplateSettings.podFailurePolicySettings.retryOnExitCodes="{120,121}" \
         --set jobTemplateSettings.backoffLimit=1 \
-        --version v1.0.2
+        --version v1.0.3