charts/redpanda: use new sidecar --selector flag

This commit builds upon a previous commit that added the `--selector` flag to
the sidecar. It updates the chart to appropriate set this new flag and adds
acceptance tests to assert the functionality works as expected.

This commit is separate as the v2.x.x branches need to release the updated
sidecar before it can be used.

Author: chrisseto

acceptance/features/helm-chart.feature

@@ -0,0 +1,53 @@
+@operator:none
+Feature: Redpanda Helm Chart
+
+  Scenario: Tolerating Node Failure
+    # TODO make this an outline and run through multiple configs:
+    # - mTLS
+    # - TLS
+    # - name overrides
+    # - ???
+    Given I helm install "redpanda" "../charts/redpanda/chart" with values:
+    ```yaml
+     nameOverride: foobar
+     fullnameOverride: bazquux
+
+     statefulset:
+       sideCars:
+         # TODO remove before merging
+         image:
+           tag: dev
+           repository: localhost/redpanda-operator
+         pvcUnbinder:
+           enabled: true
+           unbindAfter: 15s
+         brokerDecommissioner:
+           enabled: true
+           decommissionAfter: 15s
+    ```
+    # May not be required?
+    # And Pods matching "app.kubernetes.io/instance=redpanda,app.kubernetes.io/name=foobar" are Ready
+    When I stop the Node running Pod "bazquux-2"
+    And Pod "bazquux-2" is eventually Pending
+    # This helps speed up the test and makes the following wait for Running to
+    # be less likely to assert on a phantom Pod.
+    # TODO: Something between these two steps is wrong. Pod transitions from
+    # Pending -> Terminating. Seems like a race between manual deletion and eviction?
+    # Maybe have the previous step block until the Node is actually registered as dead?
+    # And I force delete Pod "bazquux-2"
+    Then Pod "bazquux-2" will eventually be Running
+    And kubectl exec -it "bazquux-0" "rpk redpanda admin brokers list | sed -E 's/\s+/ /gm' | cut -d ' ' -f 1,6" will eventually output:
+    ```
+    ID MEMBERSHIP
+    0 active
+    1 active
+    3 active
+    ```
+    And kubectl exec -it "bazquux-0" "rpk redpanda admin brokers list --include-decommissioned | sed -E 's/\s+/ /gm' | cut -d ' ' -f 1,6" will eventually output:
+    ```
+    ID MEMBERSHIP
+    0 active
+    1 active
+    3 active
+    2 -
+    ```

acceptance/features/migration.feature

@@ -2,7 +2,7 @@ Feature: Helm chart to Redpanda Operator migration
 
   @skip:gke @skip:aks @skip:eks
   Scenario: Migrate from a Helm chart release to a Redpanda custom resource
-        Given a Helm release named "redpanda-migration-example" of the "redpanda/redpanda" helm chart with the values:
+        Given I helm install "redpanda-migration-example" "../charts/redpanda/chart" with values:
         """
     # tag::helm-values[]
         fullnameOverride: name-override

acceptance/features/operator-upgrades.feature

@@ -3,10 +3,8 @@ Feature: Upgrading the operator
   @skip:gke @skip:aks @skip:eks
   Scenario: Operator upgrade from 2.4.5
     Given I install local CRDs from "../operator/config/crd/bases"
-    And I install redpanda helm chart version "v2.4.5" with the values:
+    And I helm install "redpanda-operator" "redpanda/operator" --version v2.4.5 with values:
     """
-    console:
-      enabled: false
     """
     And I apply Kubernetes manifest:
     """
@@ -29,7 +27,7 @@ Feature: Upgrading the operator
     # use just a Ready status check here since that's all the
     # old operator supports
     And cluster "operator-upgrade" is available
-    Then I can upgrade to the latest operator with the values:
+    Then I can helm upgrade "redpanda-operator" "../operator/chart" with values:
     """
     image:
       tag: dev
@@ -42,7 +40,7 @@ Feature: Upgrading the operator
 
   @skip:gke @skip:aks @skip:eks
   Scenario: Operator upgrade from 25.1.3
-    And I install redpanda helm chart version "v25.1.3" with the values:
+    Given I helm install "redpanda-operator" "redpanda/operator" --version v25.1.3 with values:
     """
     crds:
       enabled: true
@@ -68,7 +66,7 @@ Feature: Upgrading the operator
     # use just a Ready status check here since that's all the
     # old operator supports
     And cluster "operator-upgrade" is available
-    Then I can upgrade to the latest operator with the values:
+    Then I can helm upgrade "redpanda-operator" "../operator/chart" with values:
     """
     image:
       tag: dev

acceptance/main_test.go

@@ -93,7 +93,7 @@ var setupSuite = sync.OnceValues(func() (*framework.Suite, error) {
 				return
 			}
 			t.Log("Installing default Redpanda operator chart")
-			t.InstallLocalHelmChart(ctx, "../operator/chart", helm.InstallOptions{
+			t.InstallHelmChart(ctx, "../operator/chart", helm.InstallOptions{
 				Name:      "redpanda-operator",
 				Namespace: namespace,
 				Values: operatorchart.PartialValues{
@@ -205,7 +205,7 @@ func OperatorTag(ctx context.Context, t framework.TestingT, args ...string) cont
 	}
 
 	t.Logf("Installing Redpanda operator chart: %q", name)
-	t.InstallLocalHelmChart(ctx, "../operator/chart", helm.InstallOptions{
+	t.InstallHelmChart(ctx, "../operator/chart", helm.InstallOptions{
 		Name:       "redpanda-operator",
 		Namespace:  t.Namespace(),
 		ValuesFile: filepath.Join("operator", fmt.Sprintf("%s.yaml", name)),

acceptance/steps/cluster.go

@@ -166,6 +166,26 @@ func shutdownRandomClusterNode(ctx context.Context, t framework.TestingT, cluste
 	t.ShutdownNode(ctx, pod.Spec.NodeName)
 }
 
+func shutdownNodeOfPod(ctx context.Context, t framework.TestingT, podName string) {
+	t.ResourceKey(podName)
+
+	var pod corev1.Pod
+	require.NoError(t, t.Get(ctx, t.ResourceKey(podName), &pod))
+
+
+	var node corev1.Node
+	require.NoError(t, t.Get(ctx, t.ResourceKey(pod.Spec.NodeName), &node))
+
+	node.Spec.Taints = append(node.Spec.Taints, corev1.Taint{
+		Key:    "node.kubernetes.io/out-of-service",
+		Effect: corev1.TaintEffectNoExecute,
+	})
+
+	require.NoError(t, t.Update(ctx, &node))
+
+	t.ShutdownNode(ctx, pod.Spec.NodeName)
+}
+
 func deleteNotReadyKubernetesNodes(ctx context.Context, t framework.TestingT) {
 	var nodes corev1.NodeList
 	require.NoError(t, t.List(ctx, &nodes))

acceptance/steps/helm.go

@@ -12,40 +12,45 @@ package steps
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	"github.com/cucumber/godog"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/rest"
 	"sigs.k8s.io/yaml"
 
 	framework "github.com/redpanda-data/redpanda-operator/harpoon"
 	"github.com/redpanda-data/redpanda-operator/pkg/helm"
 )
 
-// The unused parameter is meant to specify a Helm chart place (remote or local in the file system).
-func iInstallHelmRelease(ctx context.Context, t framework.TestingT, helmReleaseName, _ string, values *godog.DocString) {
+func iHelmInstall(ctx context.Context, t framework.TestingT, name, chart, version string, values *godog.DocString) {
+	// We don't really reference anything other than the redpanda repo, so just
+	// handle repos as a naive check here.
+	if strings.HasPrefix(chart, "redpanda/") {
+		t.AddHelmRepo(ctx, "redpanda", "https://charts.redpanda.com")
+	}
+
 	var valuesMap map[string]any
 	require.NoError(t, yaml.Unmarshal([]byte(values.Content), &valuesMap))
 
-	helmClient, err := helm.New(helm.Options{
-		KubeConfig: rest.CopyConfig(t.RestConfig()),
+	t.InstallHelmChart(ctx, chart, helm.InstallOptions{
+		Name:      name,
+		Version:   version,
+		Values:    valuesMap,
+		Namespace: t.Namespace(),
 	})
-	require.NoError(t, err)
-
-	require.NoError(t, helmClient.RepoAdd(ctx, "console", "https://charts.redpanda.com"))
+}
 
-	path := "../charts/redpanda/chart"
-	require.NoError(t, helmClient.DependencyBuild(ctx, path))
+func iHelmUpgrade(ctx context.Context, t framework.TestingT, name, chart, version string, values *godog.DocString) {
+	var valuesMap map[string]any
+	require.NoError(t, yaml.Unmarshal([]byte(values.Content), &valuesMap))
 
-	t.Logf("installing chart %q", path)
-	_, err = helmClient.Install(ctx, path, helm.InstallOptions{
-		Name:      helmReleaseName,
-		Namespace: t.Namespace(),
+	t.UpgradeHelmChart(ctx, name, chart, helm.UpgradeOptions{
+		Version:   version,
 		Values:    valuesMap,
+		Namespace: t.Namespace(),
 	})
-	require.NoError(t, err)
 }
 
 func iDeleteHelmReleaseSecret(ctx context.Context, t framework.TestingT, helmReleaseName string) {

acceptance/steps/k8s.go

@@ -36,6 +36,15 @@ import (
 // for type assertions
 var _ client.Object = (client.Object)(nil)
 
+func podWillEventuallyBeInPhase(ctx context.Context, t framework.TestingT, podName string, phase string) {
+	require.EventuallyWithT(t, func(c *assert.CollectT) {
+		var pod corev1.Pod
+		require.NoError(c, t.Get(ctx, t.ResourceKey(podName), &pod))
+
+		require.Equal(c, corev1.PodPhase(phase), pod.Status.Phase)
+	}, 5*time.Minute, 5*time.Second)
+}
+
 func kubernetesObjectHasClusterOwner(ctx context.Context, t framework.TestingT, groupVersionKind, resourceName, clusterName string) {
 	var cluster redpandav1alpha2.Redpanda
 
@@ -163,7 +172,23 @@ func execJSONPath(ctx context.Context, t framework.TestingT, jsonPath, groupVers
 	return nil
 }
 
-func iExecInPodMatching(
+func execInPodEventuallyMatches(
+	ctx context.Context,
+	t framework.TestingT,
+	podName string,
+	cmd string,
+	expected *godog.DocString,
+) {
+	ctl, err := kube.FromRESTConfig(t.RestConfig())
+	require.NoError(t, err)
+
+	pod, err := kube.Get[corev1.Pod](ctx, ctl, kube.ObjectKey{Namespace: t.Namespace(), Name: podName})
+	require.NoErrorf(t, err, "Pod with name %q not found", podName)
+
+	execInPod(t, ctx, ctl, pod, cmd, expected)
+}
+
+func execInPodMatchingEventuallyMatches(
 	ctx context.Context,
 	t framework.TestingT,
 	cmd,
@@ -181,11 +206,24 @@ func iExecInPodMatching(
 
 	require.True(t, len(pods.Items) > 0, "selector %q found no Pods", selector.String())
 
-	var stdout bytes.Buffer
-	require.NoError(t, ctl.Exec(ctx, &pods.Items[0], kube.ExecOptions{
-		Command: []string{"sh", "-c", cmd},
-		Stdout:  &stdout,
-	}))
+	execInPod(t, ctx, ctl, &pods.Items[0], cmd, expected)
+}
 
-	assert.Equal(t, strings.TrimSpace(expected.Content), strings.TrimSpace(stdout.String()))
+func execInPod(
+	t framework.TestingT,
+	ctx context.Context,
+	ctl *kube.Ctl,
+	pod *corev1.Pod,
+	cmd string,
+	expected *godog.DocString,
+) {
+	require.EventuallyWithT(t, func(collect *assert.CollectT) {
+		var stdout bytes.Buffer
+		require.NoError(collect, ctl.Exec(ctx, pod, kube.ExecOptions{
+			Command: []string{"sh", "-c", cmd},
+			Stdout:  &stdout,
+		}))
+
+		assert.Equal(collect, strings.TrimSpace(expected.Content), strings.TrimSpace(stdout.String()))
+	}, 5*time.Minute, 5*time.Second)
 }

acceptance/steps/operator.go

@@ -11,21 +11,14 @@ package steps
 
 import (
 	"context"
-	"fmt"
-	"os"
 	"regexp"
-	"strconv"
-	"strings"
 
-	"github.com/cucumber/godog"
 	"github.com/stretchr/testify/require"
 	appsv1 "k8s.io/api/apps/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime/schema"
 
 	framework "github.com/redpanda-data/redpanda-operator/harpoon"
-	"github.com/redpanda-data/redpanda-operator/pkg/helm"
 )
 
 func operatorIsRunning(ctx context.Context, t framework.TestingT) {
@@ -93,70 +86,3 @@ func createClusterRoleBinding(ctx context.Context, serviceAccountName, clusterRo
 		}))
 	})
 }
-
-func iCanUpgradeToTheLatestOperatorWithTheValues(ctx context.Context, t framework.TestingT, values *godog.DocString) {
-	file, err := os.CreateTemp("", "values-*.yaml")
-	require.NoError(t, err)
-
-	_, err = file.Write([]byte(values.Content))
-	require.NoError(t, err)
-	require.NoError(t, file.Close())
-
-	t.Cleanup(func(ctx context.Context) {
-		require.NoError(t, os.RemoveAll(file.Name()))
-	})
-
-	t.UpgradeLocalHelmChart(ctx, "../operator/chart", "redpanda-operator", helm.UpgradeOptions{
-		Namespace:  t.Namespace(),
-		ValuesFile: file.Name(),
-		NoWait:     true,
-	})
-}
-
-func iInstallRedpandaHelmChartVersionWithTheValues(ctx context.Context, t framework.TestingT, version string, values *godog.DocString) {
-	file, err := os.CreateTemp("", "values-*.yaml")
-	require.NoError(t, err)
-
-	_, err = file.Write([]byte(values.Content))
-	require.NoError(t, err)
-	require.NoError(t, file.Close())
-
-	t.Cleanup(func(ctx context.Context) {
-		require.NoError(t, os.RemoveAll(file.Name()))
-	})
-
-	major, err := strconv.Atoi(strings.Split(strings.TrimPrefix(version, "v"), ".")[0])
-	require.NoError(t, err)
-
-	if major < 25 {
-		// these are needed for old versions of the operator
-		t.ApplyManifest(ctx, fmt.Sprintf("https://raw.githubusercontent.com/redpanda-data/redpanda-operator/refs/tags/%s/operator/config/crd/bases/toolkit.fluxcd.io/helm-controller.yaml", version))
-		t.ApplyManifest(ctx, fmt.Sprintf("https://raw.githubusercontent.com/redpanda-data/redpanda-operator/refs/tags/%s/operator/config/crd/bases/toolkit.fluxcd.io/source-controller.yaml", version))
-	}
-
-	t.Cleanup(func(ctx context.Context) {
-		// make sure we remove all finalizers for these or the CRD cleanup will get wedged
-		removeAllFinalizers(ctx, t, schema.GroupVersionKind{
-			Group:   "helm.toolkit.fluxcd.io",
-			Kind:    "HelmRelease",
-			Version: "v2beta2",
-		})
-		removeAllFinalizers(ctx, t, schema.GroupVersionKind{
-			Group:   "source.toolkit.fluxcd.io",
-			Kind:    "HelmRepository",
-			Version: "v1beta1",
-		})
-		removeAllFinalizers(ctx, t, schema.GroupVersionKind{
-			Group:   "source.toolkit.fluxcd.io",
-			Kind:    "HelmChart",
-			Version: "v1beta2",
-		})
-	})
-
-	t.InstallHelmChart(ctx, "https://charts.redpanda.com", "redpanda", "operator", helm.InstallOptions{
-		Name:       "redpanda-operator",
-		Namespace:  t.Namespace(),
-		Version:    version,
-		ValuesFile: file.Name(),
-	})
-}