Added haproxy sidecar in its most basic form

.dockerignore

@@ -2,6 +2,6 @@
 ./Jenkinsfile
 ./Makefile
 ./README.md
-./bin
+# ./bin
 ./varnish-operator
 ./vendor

.gitignore

@@ -11,3 +11,5 @@ _output
 *.swp
 *.swo
 _book
+skaffold.yaml
+*.local

Dockerfile.controller

@@ -34,7 +34,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends varnish
 FROM debian:bullseye-slim
 LABEL maintainer="Alex Lytvynenko <[email protected]>, Tomash Sidei <[email protected]>"
 
-RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends libc6 libedit2 libncursesw6 libtinfo6 libvarnishapi2 \
+RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends libc6 libedit2 libncursesw6 libtinfo6 libvarnishapi2 procps \
     && rm -rf /var/lib/apt/lists/* \
                     /etc/varnish/* \
     && adduser --quiet --system --no-create-home --home /nonexistent --group varnish \

Dockerfile.haproxy

@@ -0,0 +1,7 @@
+FROM haproxytech/haproxy-debian:2.7
+
+RUN apt-get update && apt-get upgrade -y \
+    && adduser --quiet --system --no-create-home --home /nonexistent --group varnish \
+    && rm -rf /var/lib/apt/lists/*
+
+USER varnish

Makefile

@@ -12,6 +12,15 @@ VARNISH_METRICS_IMG ?= ${VARNISH_METRICS_PUBLISH_IMG}-dev
 NAMESPACE ?= "default"
 CRD_OPTIONS ?= "crd:crdVersions=v1"
 
+KUSTOMIZE = $(shell pwd)/bin/kustomize
+ifeq (, $(wildcard $KUSTOMIZE))
+  KUSTOMIZE = $(shell which kustomize)
+endif
+
+# ifeq (, $(wildcard $KUSTOMIZE))
+#   $(error kustomize must exist)
+# endif
+
 # all: test varnish-operator
 all: test varnish-operator varnish-controller
 
@@ -149,16 +158,20 @@ e2e-tests:
 	KUBECONFIG=$(ROOT_DIR)e2e-tests-kubeconfig go test ./tests
 	sh $(ROOT_DIR)hack/delete_dev_cluster.sh
 
-KUSTOMIZE = $(shell pwd)/bin/kustomize
 kustomize:
 	$(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/[email protected])
 
 # Generate bundle manifests and metadata, then validate generated files.
 .PHONY: bundle
 bundle: manifests kustomize
+ifeq ($(shell yq --version | cut -d" " -f3 | cut -d"." -f1), 3)
 	yq w -i config/manager/deployment.yaml 'spec.template.spec.containers(name==varnish-operator).env(name==CONTAINER_IMAGE).value' $(PUBLISH_IMG)
 	yq w -i config/manifests/bases/varnish-operator.clusterserviceversion.yaml 'metadata.annotations.containerImage' $(PUBLISH_IMG)
 	yq w -i config/manifests/bases/varnish-operator.clusterserviceversion.yaml 'metadata.annotations.createdAt' $(date +"%Y-%m-%d")
+else
+	yq e '(.spec.template.spec.containers[] | select(.name == "varnish-operator") | .env[] | select(.name == "CONTAINER_IMAGE") | .value) = "'$(PUBLISH_IMG)'"' -i config/manager/deployment.yaml
+	yq e '.metadata.annotations.containerImage = "'$(PUBLISH_IMG)'" | .metadata.annotations.createdAt = "'$(date +"%Y-%m-%d")'"' -i config/manifests/bases/varnish-operator.clusterserviceversion.yaml
+endif
 	operator-sdk generate kustomize manifests -q
 	cd config/manager && $(KUSTOMIZE) edit set image controller=$(PUBLISH_IMG)
 	$(KUSTOMIZE) build config/manifests | operator-sdk generate bundle -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS)

api/v1alpha1/defaults.go

@@ -90,9 +90,45 @@ func defaultVarnishClusterSpec(in *VarnishClusterSpec) {
 	if in.Backend.ZoneBalancing == nil {
 		in.Backend.ZoneBalancing = &VarnishClusterBackendZoneBalancing{}
 	}
+
+	DefaultHaproxySidecar(in.HaproxySidecar)
+
 	defaultVarnishZoneBalancingType(in.Backend.ZoneBalancing)
 }
 
+func DefaultHaproxySidecar(haproxySidecar *HaproxySidecar) {
+	if haproxySidecar == nil {
+		haproxySidecar = &HaproxySidecar{
+			Enabled: false,
+		}
+	} else if haproxySidecar.Enabled {
+		if haproxySidecar.MaxConnections == nil {
+			haproxySidecar.MaxConnections = proto.Int32(64)
+		}
+		if haproxySidecar.ConnectTimeout == nil {
+			haproxySidecar.ConnectTimeout = proto.Int32(5000)
+		}
+		if haproxySidecar.ClientTimeout == nil {
+			haproxySidecar.ClientTimeout = proto.Int32(50000)
+		}
+		if haproxySidecar.ServerTimeout == nil {
+			haproxySidecar.ServerTimeout = proto.Int32(50000)
+		}
+		if haproxySidecar.StatRefreshRate == nil {
+			haproxySidecar.StatRefreshRate = proto.Int32(10)
+		}
+		if haproxySidecar.BackendAdditionalFlags == "" {
+			haproxySidecar.BackendAdditionalFlags = "none"
+		}
+		if haproxySidecar.BackendServerMaxAgeHeader == nil {
+			haproxySidecar.BackendServerMaxAgeHeader = proto.Int32(31536000)
+		}
+		if haproxySidecar.BackendServerPort == nil {
+			haproxySidecar.BackendServerPort = proto.Int32(443)
+		}
+	}
+}
+
 func defaultVarnish(in *VarnishClusterVarnish) {
 	if in.ImagePullPolicy == "" {
 		in.ImagePullPolicy = v1.PullAlways

api/v1alpha1/varnishcluster_types.go

@@ -57,6 +57,14 @@ const (
 	VarnishClusterBackendZoneBalancingTypeDisabled   = "disabled"
 	VarnishClusterBackendZoneBalancingTypeAuto       = "auto"
 	VarnishClusterBackendZoneBalancingTypeThresholds = "thresholds"
+
+	HaproxyContainerName   = "haproxy-sidecar"
+	HaproxyConfigFileName  = "haproxy.cfg"
+	HaproxyConfigDir       = "/usr/local/etc/haproxy"
+	HaproxyConfigVolume    = "haproxy-config"
+	HaproxyMetricsPort     = 8404
+	HaproxyMetricsPortName = "haproxy-metrics"
+	HaproxyScriptsVolume   = "haproxy-scripts"
 )
 
 // +kubebuilder:object:root=true
@@ -96,7 +104,29 @@ type VarnishClusterSpec struct {
 	// +kubebuilder:validation:Enum=debug;info;warn;error;dpanic;panic;fatal
 	LogLevel string `json:"logLevel,omitempty"`
 	// +kubebuilder:validation:Enum=json;console
-	LogFormat string `json:"logFormat,omitempty"`
+	LogFormat      string          `json:"logFormat,omitempty"`
+	HaproxySidecar *HaproxySidecar `json:"haproxySidecar,omitempty"`
+}
+
+type HaproxySidecar struct {
+	Enabled bool   `json:"enabled,omitempty"`
+	Image   string `json:"image,omitempty"`
+	// +kubebuilder:validation:Enum=Always;Never;IfNotPresent
+	ImagePullPolicy           v1.PullPolicy           `json:"imagePullPolicy,omitempty"`
+	ImagePullSecret           string                  `json:"imagePullSecret,omitempty"`
+	Resources                 v1.ResourceRequirements `json:"resources,omitempty"`
+	MaxConnections            *int32                  `json:"maxConnections,omitempty"`
+	ConnectTimeout            *int32                  `json:"connectTimeout,omitempty"`  // in millis, 5000 default
+	ClientTimeout             *int32                  `json:"clientTimeout,omitempty"`   // in millis, 50000 default
+	ServerTimeout             *int32                  `json:"serverTimeout,omitempty"`   // in millis, 50000 default
+	StatRefreshRate           *int32                  `json:"statRefreshRate,omitempty"` // in seconds, 10 default
+	EnableFrontendMetrics     bool                    `json:"enableFrontendMetrics,omitempty"`
+	BackendAdditionalFlags    string                  `json:"backendAdditionalFlags,omitempty"`
+	BackendServerHostHeader   string                  `json:"backendServerHostHeader"`
+	BackendServerMaxAgeHeader *int32                  `json:"backendServerMaxAgeHeader,omitempty"`
+	BackendServerPort         *int32                  `json:"backendServerPort,omitempty"`
+	BackendServers            []string                `json:"backendServers"`
+	HttpChk                   []string                `json:"httpchk,omitempty"`
 }
 
 type VarnishClusterUpdateStrategyType string
@@ -241,19 +271,28 @@ type VarnishClusterMonitoringGrafanaDashboard struct {
 
 // VarnishClusterStatus defines the observed state of VarnishCluster
 type VarnishClusterStatus struct {
-	VCL                 VCLStatus `json:"vcl"`
-	VarnishArgs         string    `json:"varnishArgs,omitempty"`
-	Replicas            int32     `json:"replicas,omitempty"`
-	VarnishPodsSelector string    `json:"varnishPodsSelector,omitempty"`
+	VCL                 VCLStatus            `json:"vcl"`
+	HAProxy             HaproxySidecarStatus `json:"haproxy"`
+	VarnishArgs         string               `json:"varnishArgs,omitempty"`
+	Replicas            int32                `json:"replicas,omitempty"`
+	VarnishPodsSelector string               `json:"varnishPodsSelector,omitempty"`
 }
 
-// VCLStatus describes the VCL versions status
-type VCLStatus struct {
+type ConfigMapStatus struct {
 	Version          *string `json:"version,omitempty"`
 	ConfigMapVersion string  `json:"configMapVersion"`
 	Availability     string  `json:"availability"`
 }
 
+// VCLStatus describes the VCL versions status
+type VCLStatus struct {
+	ConfigMapStatus `json:",inline"`
+}
+
+type HaproxySidecarStatus struct {
+	ConfigMapStatus `json:",inline"`
+}
+
 // +kubebuilder:object:root=true
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 

cmd/varnish-controller/main.go

@@ -4,9 +4,17 @@
 package main
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"log"
+	"os"
+	"time"
+
+	"github.com/ibm/varnish-operator/pkg/names"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 
 	"github.com/ibm/varnish-operator/api/v1alpha1"
 	"github.com/ibm/varnish-operator/pkg/logger"
@@ -69,6 +77,11 @@ func main() {
 		log.Fatalf("could not load rest client config. Error: %s", err)
 	}
 
+	if _, ok := os.LookupEnv("INIT_CONTAINER"); ok {
+		initHaproxyConfig(clientConfig, logr, varnishControllerConfig)
+		return
+	}
+
 	vMetrics := varnishMetrics.NewVarnishControllerMetrics()
 	controllerMetrics.Registry.MustRegister(vMetrics.VCLCompilationError)
 
@@ -107,3 +120,27 @@ func main() {
 		logr.With(err).Fatalf("Failed to start manager")
 	}
 }
+
+func initHaproxyConfig(clientConfig *rest.Config, logr *logger.Logger, varnishControllerConfig *config.Config) {
+	cs, err := kubernetes.NewForConfig(clientConfig)
+	if err != nil {
+		logr.With(zap.Error(err)).Fatalf("unable to create k8s client")
+	}
+	haproxyConfigMapName := names.HaproxyConfigMap(varnishControllerConfig.VarnishClusterName)
+	configFound := false
+	for !configFound {
+		logr.Infof("attempting to retrieve configmap: %s", haproxyConfigMapName)
+		configMaps := cs.CoreV1().ConfigMaps(varnishControllerConfig.Namespace)
+		if haproxyConfigMap, err := configMaps.Get(context.Background(), haproxyConfigMapName, metav1.GetOptions{}); err == nil {
+			cfgData := haproxyConfigMap.Data[v1alpha1.HaproxyConfigFileName]
+			haproxyConfigFileName := v1alpha1.HaproxyConfigDir + "/" + v1alpha1.HaproxyConfigFileName
+			if err := os.WriteFile(haproxyConfigFileName, []byte(cfgData), 0644); err != nil {
+				logr.With(zap.Error(err)).Fatalf("unable to write haproxy config file: %s", haproxyConfigFileName)
+			}
+			logr.Infof("haproxy config has been written\n%s", cfgData)
+			configFound = true
+		} else {
+			time.Sleep(2 * time.Second)
+		}
+	}
+}