operator: Include ClusterRole permission for redpanda controller

In the redpanda package the kubebuilder comment does not have all possible
variants of ClusterRole permissions neccessery to handle creation of all
Redpanda helm chart resources. When rpk bundle ClusterRole was included to
the Redpanda helm chart deployment, then the integration test suite failed with
flux reporting the following:
```
creation of clusterroles.rbac.authorization.k8s.io "rp-9gd31r-rpk-bundle" is forbidden:
user "system:serviceaccount:testenv-g5jfk:testenv-pzy3ce"
(groups=["system:serviceaccounts" "system:serviceaccounts:testenv-g5jfk" "system:authenticated"])
is attempting to grant RBAC permissions not currently held:

{APIGroups:[""], Resources:["endpoints"], Verbs:["get" "list"]}
{APIGroups:[""], Resources:["events"], Verbs:["get" "list"]}
{APIGroups:[""], Resources:["limitranges"], Verbs:["get" "list"]}
{APIGroups:[""], Resources:["persistentvolumeclaims"], Verbs:["get" "list"]}
{APIGroups:[""], Resources:["pods"], Verbs:["get" "list"]}
{APIGroups:[""], Resources:["pods/log"], Verbs:["get" "list"]}
{APIGroups:[""], Resources:["replicationcontrollers"], Verbs:["get" "list"]}
{APIGroups:[""], Resources:["resourcequotas"], Verbs:["get" "list"]}
{APIGroups:[""], Resources:["serviceaccounts"], Verbs:["get" "list"]}
{APIGroups:[""], Resources:["services"], Verbs:["get" "list"]}
```

The setup of integration test suite included only permissions defined in redpanda package.
Kustomize and Operator helm chart includes those missing permissions.

Author: RafalKorepta

operator/config/rbac/bases/operator/role.yaml

@@ -16,6 +16,17 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  - limitranges
+  - pods/log
+  - replicationcontrollers
+  - resourcequotas
+  verbs:
+  - get
+  - list
 - apiGroups:
   - ""
   resources:

operator/config/rbac/v2-manager-role/role.yaml

@@ -14,6 +14,22 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  - events
+  - limitranges
+  - persistentvolumeclaims
+  - pods
+  - pods/log
+  - replicationcontrollers
+  - resourcequotas
+  - serviceaccounts
+  - services
+  verbs:
+  - get
+  - list
 - apiGroups:
   - ""
   resources:

operator/internal/controller/redpanda/redpanda_controller.go

@@ -140,6 +140,10 @@ type RedpandaReconciler struct {
 // The leases is used by controller-runtime in sidecar. Operator main reconciliation needs to have leases permissions in order to create role that have the same permissions.
 // +kubebuilder:rbac:groups=coordination.k8s.io,namespace=default,resources=leases,verbs=get;list;watch;create;update;patch;delete
 
+// rpk bundle additional rbac permissions
+// When Redpanda chart values has set `rbac.enabled = true`, then operator needs to have elevated permissions to create ClusterRole
+// +kubebuilder:rbac:groups=core,resources=endpoints;events;limitranges;persistentvolumeclaims;pods;pods/log;replicationcontrollers;resourcequotas;serviceaccounts;services,verbs=get;list
+
 // SetupWithManager sets up the controller with the Manager.
 func (r *RedpandaReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
 	if err := registerHelmReferencedIndex(ctx, mgr, "statefulset", &appsv1.StatefulSet{}); err != nil {

operator/internal/controller/redpanda/redpanda_controller_test.go

@@ -731,7 +731,7 @@ func (s *RedpandaControllerSuite) TestStableUIDAndGeneration() {
 		flipped := s.snapshotCluster(filter)
 		s.compareSnapshot(fresh, flipped, isStable)
 
-		s.T().Logf("toggling useFlux: %t -> %t", useFlux, !useFlux)
+		s.T().Logf("toggling useFlux: %t -> %t", !useFlux, useFlux)
 		rp.Spec.ChartRef.UseFlux = ptr.To(useFlux)
 		s.applyAndWait(rp)
 
@@ -743,9 +743,15 @@ func (s *RedpandaControllerSuite) TestStableUIDAndGeneration() {
 		// HelmRelease and HelmChart are checked explicitly here, but any test that would left behind mentioned resource
 		// will prevent from namespace deletion. In other words if test suite can not delete namespace, then with high
 		// probability resource with finalizer prevents from namespace deletion.
-		var hr v2beta2.HelmRelease
-		err := s.client.Get(s.ctx, types.NamespacedName{Name: rp.GetHelmReleaseName(), Namespace: rp.Namespace}, &hr)
-		s.True(apierrors.IsNotFound(err))
+
+		// In the flux base deployment the HelmRelease will be deleted after Redpanda is fully removed from
+		// Kubernetes API server. The ownerReference tight together Redpanda with HelmRelease, but there is
+		// delay between HelmRelease and Redpadna custom resource deletion.
+		s.EventuallyWithT(func(t *assert.CollectT) {
+			var hr v2beta2.HelmRelease
+			err := s.client.Get(s.ctx, types.NamespacedName{Name: rp.GetHelmReleaseName(), Namespace: rp.Namespace}, &hr)
+			assert.True(t, apierrors.IsNotFound(err))
+		}, time.Minute, time.Second, "HelmRelease not GC'd")
 
 		var hc sourcecontrollerv1beta2.HelmChart
 		err = s.client.Get(s.ctx, types.NamespacedName{Name: rp.Namespace + "-" + rp.Name, Namespace: rp.Namespace}, &hc)
@@ -958,7 +964,7 @@ func (s *RedpandaControllerSuite) minimalRP(useFlux bool) *redpandav1alpha2.Redp
 					},
 				},
 				RBAC: &redpandav1alpha2.RBAC{
-					Enabled: ptr.To(false),
+					Enabled: ptr.To(true),
 				},
 			},
 		},

operator/internal/controller/redpanda/role.yaml

@@ -14,6 +14,22 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  - events
+  - limitranges
+  - persistentvolumeclaims
+  - pods
+  - pods/log
+  - replicationcontrollers
+  - resourcequotas
+  - serviceaccounts
+  - services
+  verbs:
+  - get
+  - list
 - apiGroups:
   - ""
   resources: