0.25.4

.github/workflows/build_branch.yaml

@@ -23,7 +23,8 @@ jobs:
           DOCKER_PASS: ${{ secrets.DOCKER_PASS }}
         run: |
           export CHO_RELEASE=$(cat release)
+          export GO_VERSION=$(grep '^go ' go.mod | awk '{print $2}')
 
           echo "${DOCKER_PASS}" | docker login -u $DOCKER_USER --password-stdin docker.io
-          docker buildx build --progress plain --platform=linux/amd64,linux/arm64 -f dockerfile/operator/Dockerfile -t docker.io/${DOCKER_ORG}/clickhouse-operator:${CHO_RELEASE} --pull --push  .
-          docker buildx build --progress plain --platform=linux/amd64,linux/arm64 -f dockerfile/metrics-exporter/Dockerfile -t docker.io/${DOCKER_ORG}/metrics-exporter:${CHO_RELEASE} --pull --push .
+          docker buildx build --progress plain --platform=linux/amd64,linux/arm64 -f dockerfile/operator/Dockerfile         --build-arg GO_VERSION=${GO_VERSION} -t docker.io/${DOCKER_ORG}/clickhouse-operator:${CHO_RELEASE} --pull --push .
+          docker buildx build --progress plain --platform=linux/amd64,linux/arm64 -f dockerfile/metrics-exporter/Dockerfile --build-arg GO_VERSION=${GO_VERSION} -t docker.io/${DOCKER_ORG}/metrics-exporter:${CHO_RELEASE}    --pull --push .

.github/workflows/build_master.yaml

@@ -23,8 +23,8 @@ jobs:
           DOCKER_PASS: ${{ secrets.DOCKER_PASS }}
         run: |
           export CHO_RELEASE=latest
+          export GO_VERSION=$(grep '^go ' go.mod | awk '{print $2}')
 
           echo "${DOCKER_PASS}" | docker login -u $DOCKER_USER --password-stdin docker.io
-          docker buildx build --progress plain --platform=linux/amd64,linux/arm64 -f dockerfile/operator/Dockerfile -t docker.io/${DOCKER_ORG}/clickhouse-operator:${CHO_RELEASE} --pull --push  .
-          docker buildx build --progress plain --platform=linux/amd64,linux/arm64 -f dockerfile/metrics-exporter/Dockerfile -t docker.io/${DOCKER_ORG}/metrics-exporter:${CHO_RELEASE} --pull --push .
-
+          docker buildx build --progress plain --platform=linux/amd64,linux/arm64 -f dockerfile/operator/Dockerfile         --build-arg GO_VERSION=${GO_VERSION} -t docker.io/${DOCKER_ORG}/clickhouse-operator:${CHO_RELEASE} --pull --push .
+          docker buildx build --progress plain --platform=linux/amd64,linux/arm64 -f dockerfile/metrics-exporter/Dockerfile --build-arg GO_VERSION=${GO_VERSION} -t docker.io/${DOCKER_ORG}/metrics-exporter:${CHO_RELEASE}    --pull --push .

.github/workflows/release_chart.yaml

@@ -28,13 +28,6 @@ jobs:
       - name: Package Chart
         run: cr package deploy/helm/clickhouse-operator
 
-      - name: Install Helm
-        run: |
-          curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
-
-      - name: Login to GitHub Container Registry
-        run: echo "${{ secrets.GITHUB_TOKEN }}" | helm registry login ghcr.io -u ${{ github.actor }} --password-stdin
-
       - name: Get Release Assets
         id: get_assets
         run: |
@@ -62,13 +55,38 @@ jobs:
             -H "Content-Type: application/gzip" \
             -T "${CHART_PATH}" \
             "https://uploads.github.com/repos/${GITHUB_REPOSITORY}/releases/${{ github.event.release.id }}/assets?name=$(basename ${CHART_PATH})"
-
+      - name: Validate Helm Repository Configuration
+        run: |
+          if [ -z "${{ secrets.HELM_GITHUB_TOKEN }}" ]; then
+            echo "ERROR: HELM_GITHUB_TOKEN secret is not set or is empty"
+            echo "Please add HELM_GITHUB_TOKEN to repository secrets with write access to the helm repository"
+            exit 1
+          fi
+
+          if [ -z "${{ vars.HELM_GITHUB_REPOSITORY }}" ]; then
+            echo "ERROR: HELM_GITHUB_REPOSITORY variable is not set or is empty"
+            echo "Please add HELM_GITHUB_REPOSITORY to repository variables (Settings -> Secrets and variables -> Actions -> Variables)"
+            exit 1
+          fi
+
+          echo "Configuration validated:"
+          echo "  HELM_GITHUB_REPOSITORY: ${{ vars.HELM_GITHUB_REPOSITORY }}"
+          echo "  HELM_GITHUB_TOKEN: [SET]"
+
+      - name: Upload Release Artifacts to Helm Repo
+        run: |
+          cr upload \
+            --git-repo=${{ vars.HELM_GITHUB_REPOSITORY }} \
+            --owner=${GITHUB_REPOSITORY_OWNER} \
+            --release-name-template=${{ github.event.release.name }} \
+            --token=${{ secrets.HELM_GITHUB_TOKEN }} \
+            --package-path=.cr-release-packages \
+            --skip-existing
       - name: Configure Git
         run: |
           git config user.name "$GITHUB_ACTOR"
           git config user.email "[email protected]"
-
-      - name: Release Chart
+      - name: Release Chart to Operator Repo
         run: |
           git remote add httpsorigin "https://github.com/${GITHUB_REPOSITORY}.git"
           git fetch httpsorigin
@@ -80,8 +98,86 @@ jobs:
             --index-path=index.yaml \
             --remote=httpsorigin \
             --push
-
-      - name: Push Helm Chart to OCI Registry
+      - name: Release Chart to Helm Repo
         run: |
-          CHART_PATH=$(ls .cr-release-packages/altinity-clickhouse-operator-*.tgz)
-          helm push "${CHART_PATH}" oci://ghcr.io/altinity/clickhouse-operator-helm-chart
+          # Validate configuration before attempting to push
+          if [ -z "${{ vars.HELM_GITHUB_REPOSITORY }}" ]; then
+            echo "ERROR: HELM_GITHUB_REPOSITORY variable is not set or is empty"
+            echo "This step requires HELM_GITHUB_REPOSITORY to be set in repository variables"
+            echo "Go to: Settings -> Secrets and variables -> Actions -> Variables"
+            exit 1
+          fi
+
+          if [ -z "${{ secrets.HELM_GITHUB_TOKEN }}" ]; then
+            echo "ERROR: HELM_GITHUB_TOKEN secret is not set or is empty"
+            echo "This step requires HELM_GITHUB_TOKEN with write access to: ${GITHUB_REPOSITORY_OWNER}/${{ vars.HELM_GITHUB_REPOSITORY }}"
+            echo "Go to: Settings -> Secrets and variables -> Actions -> Secrets"
+            exit 1
+          fi
+
+          echo "Attempting to push to helm repository: ${GITHUB_REPOSITORY_OWNER}/${{ vars.HELM_GITHUB_REPOSITORY }}"
+
+          # Test token authentication
+          echo "Testing token authentication..."
+          TOKEN_USER=$(curl -sS -H "Authorization: token ${{ secrets.HELM_GITHUB_TOKEN }}" https://api.github.com/user | jq -r '.login')
+          echo "Token authenticated as user: ${TOKEN_USER}"
+
+          # Save current directory
+          WORK_DIR=$(pwd)
+
+          # Create a temporary directory for helm repo operations
+          TEMP_DIR=$(mktemp -d)
+          cd "$TEMP_DIR"
+
+          # Clone the helm repository WITHOUT token in URL to avoid masking issues
+          echo "Cloning helm repository to temporary directory..."
+          git clone https://github.com/${GITHUB_REPOSITORY_OWNER}/${{ vars.HELM_GITHUB_REPOSITORY }}.git helm-repo || {
+            echo "ERROR: Failed to clone helm repository"
+            echo "Please verify:"
+            echo "  1. Repository exists: ${GITHUB_REPOSITORY_OWNER}/${{ vars.HELM_GITHUB_REPOSITORY }}"
+            exit 1
+          }
+
+          cd helm-repo
+
+          # Configure git credentials for push
+          git config user.email "[email protected]"
+          git config user.name "$GITHUB_ACTOR"
+
+          # Set up authentication using git credential helper
+          git config credential.helper "store --file=.git/credentials"
+          echo "https://x-access-token:${{ secrets.HELM_GITHUB_TOKEN }}@github.com" > .git/credentials
+
+          # Now use cr index from within the helm repo to avoid history conflicts
+          echo "Generating index.yaml within helm repository context..."
+
+          # Copy the package to a local directory within helm repo
+          mkdir -p .cr-release-packages
+          cp "$WORK_DIR"/.cr-release-packages/*.tgz .cr-release-packages/ || {
+            echo "ERROR: No chart packages found in .cr-release-packages"
+            exit 1
+          }
+
+          # Generate index with cr (this will handle the gh-pages branch automatically)
+          cr index \
+            --git-repo=${{ vars.HELM_GITHUB_REPOSITORY }} \
+            --owner=${GITHUB_REPOSITORY_OWNER} \
+            --release-name-template=${{ github.event.release.name }} \
+            --token=${{ secrets.HELM_GITHUB_TOKEN }} \
+            --package-path=.cr-release-packages \
+            --index-path=index.yaml \
+            --push || {
+            echo "ERROR: Failed to generate or push index to helm repository"
+            echo "Debug: Current directory is $(pwd)"
+            echo "Debug: Git remotes:"
+            git remote -v
+            echo "Debug: Git status:"
+            git status
+            exit 1
+          }
+
+          echo "Successfully updated helm repository index"
+
+          # Cleanup
+          cd /
+          rm -rf "$TEMP_DIR"

.github/workflows/run_tests.yaml

@@ -55,13 +55,15 @@ jobs:
         run: |
           minikube status
           export CHO_RELEASE=$(cat release)
+          export GO_VERSION=$(grep '^go ' go.mod | awk '{print $2}')
           echo "current release=$CHO_RELEASE"
+          echo "current go version=$GO_VERSION"
 
-          docker build -f dockerfile/operator/Dockerfile -t altinity/clickhouse-operator:${CHO_RELEASE} --pull .
-          docker build -f dockerfile/metrics-exporter/Dockerfile -t altinity/metrics-exporter:${CHO_RELEASE} --pull .
+          docker build -f dockerfile/operator/Dockerfile         --build-arg GO_VERSION=${GO_VERSION} -t altinity/clickhouse-operator:${CHO_RELEASE} --pull .
+          docker build -f dockerfile/metrics-exporter/Dockerfile --build-arg GO_VERSION=${GO_VERSION} -t altinity/metrics-exporter:${CHO_RELEASE}    --pull .
 
           docker image save altinity/clickhouse-operator:${CHO_RELEASE} -o operator.tar
-          docker image save altinity/metrics-exporter:${CHO_RELEASE} -o metrics-exporter.tar
+          docker image save altinity/metrics-exporter:${CHO_RELEASE}    -o metrics-exporter.tar
 
           minikube image load operator.tar
           minikube image load metrics-exporter.tar

cmd/operator/app/thread_chi.go

@@ -63,6 +63,13 @@ func initClickHouse(ctx context.Context) {
 	log.V(1).F().Info("Config parsed:")
 	log.Info("\n" + chop.Config().String(true))
 
+	// Log namespace deny list configuration
+	if chop.Config().Watch.Namespaces.Exclude.Len() > 0 {
+		log.Info("Namespace deny list configured: %v - these namespaces will NOT be reconciled", chop.Config().Watch.Namespaces.Exclude.Value())
+	} else {
+		log.V(1).Info("No namespace deny list configured - all watched namespaces will be reconciled")
+	}
+
 	// Create Informers
 	kubeInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(
 		kubeClient,

cmd/operator/app/thread_keeper.go

@@ -93,6 +93,12 @@ func keeperPredicate() predicate.Funcs {
 				return false
 			}
 
+			// Check if namespace should be watched (includes deny list check)
+			if !chop.Config().IsNamespaceWatched(obj.Namespace) {
+				logger.V(2).Info("chkInformer: skip event, namespace is not watched or is in deny list", "namespace", obj.Namespace)
+				return false
+			}
+
 			if obj.Spec.Suspend.Value() {
 				return false
 			}
@@ -107,6 +113,12 @@ func keeperPredicate() predicate.Funcs {
 				return false
 			}
 
+			// Check if namespace should be watched (includes deny list check)
+			if !chop.Config().IsNamespaceWatched(obj.Namespace) {
+				logger.V(2).Info("chkInformer: skip event, namespace is not watched or is in deny list", "namespace", obj.Namespace)
+				return false
+			}
+
 			if obj.Spec.Suspend.Value() {
 				return false
 			}

config/config-dev.yaml

@@ -1,17 +1,17 @@
+# IMPORTANT
+# This file is auto-generated
+# Do not edit this file - all changes would be lost
+# Edit appropriate template in the following folder:
+# deploy/builder/templates-config
+# IMPORTANT
 #
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
+# Template parameters available:
+#   WATCH_NAMESPACES=
+#   CH_USERNAME_PLAIN=
+#   CH_PASSWORD_PLAIN=
+#   CH_CREDENTIALS_SECRET_NAMESPACE=
+#   CH_CREDENTIALS_SECRET_NAME=clickhouse-operator
+#   VERBOSITY=1
 
 ################################################
 ##
@@ -23,8 +23,9 @@ watch:
   # Concurrently running operators should watch on different namespaces.
   # IMPORTANT
   # Regexp is applicable.
-  #namespaces: ["dev", "test"]
-  namespaces: [dev, test]
+  namespaces:
+    include: [dev, test]
+    exclude: []
 
 clickhouse:
   configuration:
@@ -276,8 +277,8 @@ template:
   chi:
     # CHI template updates handling policy
     # Possible policy values:
-    #   - ReadOnStart. Accept CHIT updates on the operators start only.
-    #   - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI
+    #   - ReadOnStart. Accept CHIT updates on the operator's start only.
+    #   - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply new CHITs on next regular reconcile of the CHI
     policy: ApplyOnNextReconcile
 
     # Path to the folder where ClickHouseInstallation templates .yaml manifests are located.
@@ -288,7 +289,7 @@ template:
     # CHK template updates handling policy
     # Possible policy values:
     #   - ReadOnStart. Accept CHIT updates on the operators start only.
-    #   - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI
+    #   - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply new CHITs on next regular reconcile of the CHI
     policy: ApplyOnNextReconcile
 
     # Path to the folder where ClickHouseInstallation templates .yaml manifests are located.
@@ -315,9 +316,9 @@ reconcile:
     #   3. The first shard is always reconciled alone. Concurrency starts from the second shard and onward.
     # Thus limiting number of shards being reconciled (and thus having hosts down) in each CHI by both number and percentage
 
-    # Max number of concurrent shard reconciles within one CHI in progress
+    # Max number of concurrent shard reconciles within one cluster in progress
     reconcileShardsThreadsNumber: 1
-    # Max percentage of concurrent shard reconciles within one CHI in progress
+    # Max percentage of concurrent shard reconciles within one cluster in progress
     reconcileShardsMaxConcurrencyPercent: 50
 
   # Reconcile StatefulSet scenario
@@ -356,15 +357,31 @@ reconcile:
     #   - to be excluded from a ClickHouse cluster
     #   - to complete all running queries
     #   - to be included into a ClickHouse cluster
-    # respectfully before moving forward
+    # respectfully before moving forward with host reconcile
     wait:
       exclude: true
       queries: true
       include: false
+      # The operator during reconcile procedure should wait for replicas to catch-up
+      # replication delay a.k.a replication lag for the following replicas
       replicas:
+        # All replicas (new and known earlier) are explicitly requested to wait for replication to catch-up
         all: no
+        # New replicas only are requested to wait for replication to catch-up
         new: yes
+        # Replication catch-up is considered to be completed as soon as replication delay
+        # a.k.a replication lag - calculated as "MAX(absolute_delay) FROM system.replicas"
+        # is within this specified delay (in seconds)
         delay: 10
+      probes:
+        # Whether the operator during host launch procedure should wait for startup probe to succeed.
+        # In case probe is unspecified wait is assumed to be completed successfully.
+        # Default option value is to do not wait.
+        startup: no
+        # Whether the operator during host launch procedure should wait for readiness probe to succeed.
+        # In case probe is unspecified wait is assumed to be completed successfully.
+        # Default option value is to wait.
+        readiness: yes
 
 ################################################
 ##

config/config.yaml

@@ -23,7 +23,6 @@ watch:
   # Concurrently running operators should watch on different namespaces.
   # IMPORTANT
   # Regexp is applicable.
-  #namespaces: ["dev", "test"]
   namespaces: []
 
 clickhouse:
@@ -356,15 +355,31 @@ reconcile:
     #   - to be excluded from a ClickHouse cluster
     #   - to complete all running queries
     #   - to be included into a ClickHouse cluster
-    # respectfully before moving forward
+    # respectfully before moving forward with host reconcile
     wait:
       exclude: true
       queries: true
       include: false
+      # The operator during reconcile procedure should wait for replicas to catch-up
+      # replication delay a.k.a replication lag for the following replicas
       replicas:
+        # All replicas (new and known earlier) are explicitly requested to wait for replication to catch-up
         all: no
+        # New replicas only are requested to wait for replication to catch-up
         new: yes
+        # Replication catch-up is considered to be completed as soon as replication delay
+        # a.k.a replication lag - calculated as "MAX(absolute_delay) FROM system.replicas"
+        # is within this specified delay (in seconds)
         delay: 10
+      probes:
+        # Whether the operator during host launch procedure should wait for startup probe to succeed.
+        # In case probe is unspecified wait is assumed to be completed successfully.
+        # Default option value is to do not wait.
+        startup: no
+        # Whether the operator during host launch procedure should wait for readiness probe to succeed.
+        # In case probe is unspecified wait is assumed to be completed successfully.
+        # Default option value is to wait.
+        readiness: yes
 
 ################################################
 ##