feat: enhance toolhive k8s e2e tests with multi-tenant tests (#1029)

Signed-off-by: ChrisJBurns <[email protected]>

Author: ChrisJBurns

.github/workflows/operator-ci.yml

@@ -156,7 +156,6 @@ jobs:
         # but we want to make sure renovate bumps the versions when new ones are released. Doing that with
         # just the number is a bit more difficult and i like simple things.
         version: [
-          "kindest/node:v1.30.13",
           "kindest/node:v1.31.9",
           "kindest/node:v1.32.5",
           "kindest/node:v1.33.1"
@@ -198,12 +197,16 @@ jobs:
       with:
         cluster_name: toolhive
         version: v0.29.0
+        cloud_provider: true
         node_image: ${{ matrix.version }}
 
     - name: Run Chainsaw tests
       run: |
         kind get kubeconfig --name toolhive > kconfig.yaml
         export KUBECONFIG=kconfig.yaml
-        chainsaw test --test-dir test/e2e/chainsaw/operator/setup --config .chainsaw.yaml
-        chainsaw test --test-dir test/e2e/chainsaw/operator/test-scenarios --config .chainsaw.yaml
-        chainsaw test --test-dir test/e2e/chainsaw/operator/cleanup --config .chainsaw.yaml
+        chainsaw test --test-dir test/e2e/chainsaw/operator/multi-tenancy/setup --config .chainsaw.yaml
+        chainsaw test --test-dir test/e2e/chainsaw/operator/multi-tenancy/test-scenarios --config .chainsaw.yaml
+        chainsaw test --test-dir test/e2e/chainsaw/operator/multi-tenancy/cleanup --config .chainsaw.yaml
+        chainsaw test --test-dir test/e2e/chainsaw/operator/single-tenancy/setup --config .chainsaw.yaml
+        chainsaw test --test-dir test/e2e/chainsaw/operator/single-tenancy/test-scenarios --config .chainsaw.yaml
+        chainsaw test --test-dir test/e2e/chainsaw/operator/single-tenancy/cleanup --config .chainsaw.yaml

cmd/thv-operator/Taskfile.yml

@@ -111,7 +111,8 @@ tasks:
         --set operator.toolhiveRunnerImage={{.TOOLHIVE_IMAGE}} \
         --namespace toolhive-system \
         --create-namespace \
-        --kubeconfig kconfig.yaml
+        --kubeconfig kconfig.yaml \
+        {{ .CLI_ARGS }}
 
   operator-undeploy:
     desc: Undeploy operator from the K8s cluster
@@ -143,9 +144,12 @@ tasks:
   operator-e2e-test:
     desc: Run E2E tests for the operator
     cmds:
-      - chainsaw test --test-dir test/e2e/chainsaw/operator/setup
-      - chainsaw test --test-dir test/e2e/chainsaw/operator/test-scenarios
-      - chainsaw test --test-dir test/e2e/chainsaw/operator/cleanup
+      - chainsaw test --test-dir test/e2e/chainsaw/operator/multi-tenancy/setup
+      - chainsaw test --test-dir test/e2e/chainsaw/operator/multi-tenancy/test-scenarios
+      - chainsaw test --test-dir test/e2e/chainsaw/operator/multi-tenancy/cleanup
+      - chainsaw test --test-dir test/e2e/chainsaw/operator/single-tenancy/setup
+      - chainsaw test --test-dir test/e2e/chainsaw/operator/single-tenancy/test-scenarios
+      - chainsaw test --test-dir test/e2e/chainsaw/operator/single-tenancy/cleanup
 
   crdref-install:
     desc: Install elastic/crd-ref-docs

test/e2e/chainsaw/operator/cleanup/assert-crd.yaml renamed to test/e2e/chainsaw/operator/multi-tenancy/cleanup/assert-crd.yaml

@@ -0,0 +1,145 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: toolhive-operator-manager-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  - services
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - "rbac.authorization.k8s.io"
+  resources:
+  - roles
+  - rolebindings
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - secrets
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - apps
+  resources:
+  - deployments
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - toolhive.stacklok.dev
+  resources:
+  - mcpservers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - toolhive.stacklok.dev
+  resources:
+  - mcpservers/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - toolhive.stacklok.dev
+  resources:
+  - mcpservers/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - "apps"
+  resources:
+  - statefulsets
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - patch
+  - delete
+  - apply
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - patch
+  - delete
+  - apply
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - pods/attach
+  verbs:
+  - create
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - pods/log
+  verbs:
+  - get

test/e2e/chainsaw/operator/cleanup/assert-operator-ready.yaml renamed to test/e2e/chainsaw/operator/multi-tenancy/cleanup/assert-operator-ready.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: toolhive-operator-manager-rolebinding
+  namespace: test-namespace
+subjects:
+- kind: ServiceAccount
+  name: toolhive-operator
+  namespace: toolhive-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: toolhive-operator-manager-role

test/e2e/chainsaw/operator/cleanup/chainsaw-test.yaml renamed to test/e2e/chainsaw/operator/multi-tenancy/cleanup/chainsaw-test.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: toolhive-operator-manager-rolebinding
+  namespace: toolhive-system
+subjects:
+- kind: ServiceAccount
+  name: toolhive-operator
+  namespace: toolhive-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: toolhive-operator-manager-role