WIP - Kubeconfig RBAC

Makefile

@@ -127,27 +127,33 @@ build-installer: manifests generate kustomize ## Generate a consolidated YAML wi
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
 	$(KUSTOMIZE) build config/default > dist/install.yaml
 
+KIND_CLUSTER_NAME ?= kind
+
+.PHONY: kind-load
+kind-load: ## Loads the docker image into a local kind cluster.
+	kind load docker-image ${IMG} --name "$(KIND_CLUSTER_NAME)"
+
 ##@ Deployment
 
 ifndef ignore-not-found
   ignore-not-found = false
 endif
 
 .PHONY: install
-install: kubectl kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
+install: kubectl kustomize ## Install CRDs into the K8s cluster specified by $KUBECONFIG.
 	$(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f -
 
 .PHONY: uninstall
-uninstall: kubectl kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
+uninstall: kubectl kustomize ## Uninstall CRDs from the K8s cluster specified by $KUBECONFIG. Call with ignore-not-found=true to ignore resource not found errors during deletion.
 	$(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -
 
 .PHONY: deploy
-deploy: kubectl kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
+deploy: kubectl kustomize ## Deploy controller to the K8s cluster specified by $KUBECONFIG.
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
 	$(KUSTOMIZE) build config/default | $(KUBECTL) apply -f -
 
 .PHONY: undeploy
-undeploy: kubectl kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
+undeploy: kubectl kustomize ## Undeploy controller from the K8s cluster specified by $KUBECONFIG. Call with ignore-not-found=true to ignore resource not found errors during deletion.
 	$(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -
 
 ##@ Dependencies

cmd/main.go

@@ -39,6 +39,7 @@ import (
 	"github.com/kcp-dev/kcp-operator/internal/controller/cacheserver"
 	"github.com/kcp-dev/kcp-operator/internal/controller/frontproxy"
 	"github.com/kcp-dev/kcp-operator/internal/controller/kubeconfig"
+	kubeconfigrbac "github.com/kcp-dev/kcp-operator/internal/controller/kubeconfig-rbac"
 	"github.com/kcp-dev/kcp-operator/internal/controller/rootshard"
 	"github.com/kcp-dev/kcp-operator/internal/controller/shard"
 	"github.com/kcp-dev/kcp-operator/internal/reconciling"
@@ -188,6 +189,13 @@ func main() {
 		setupLog.Error(err, "unable to create controller", "controller", "Kubeconfig")
 		os.Exit(1)
 	}
+	if err = (&kubeconfigrbac.KubeconfigRBACReconciler{
+		Client: mgr.GetClient(),
+		Scheme: mgr.GetScheme(),
+	}).SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "KubeconfigRBAC")
+		os.Exit(1)
+	}
 	// +kubebuilder:scaffold:builder
 
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {

config/crd/bases/operator.kcp.io_frontproxies.yaml

@@ -210,7 +210,7 @@ spec:
                             DNS names determined automatically by the kcp-operator.
                             If DNSNames is used together with IssuerRef, DNSNames will be uses as-is and not merged.
                             If IssuerRef is not set, DNSNames will be merged with the defaults. This is to avoid
-                            trying to guess what DNSNames configued issuer might support.
+                            trying to guess what DNSNames configured issuer might support.
                           items:
                             type: string
                           type: array

config/crd/bases/operator.kcp.io_kubeconfigs.yaml

@@ -49,6 +49,27 @@ spec:
           spec:
             description: KubeconfigSpec defines the desired state of Kubeconfig.
             properties:
+              authorization:
+                description: Authorization allows to provision permissions for this
+                  kubeconfig.
+                properties:
+                  clusterRoleBindings:
+                    properties:
+                      cluster:
+                        description: Cluster can be either a cluster name or a workspace
+                          path.
+                        type: string
+                      clusterRoles:
+                        items:
+                          type: string
+                        type: array
+                    required:
+                    - cluster
+                    - clusterRoles
+                    type: object
+                required:
+                - clusterRoleBindings
+                type: object
               certificateTemplate:
                 description: |-
                   CertificateTemplate allows to customize the properties on the generated
@@ -77,7 +98,7 @@ spec:
                           DNS names determined automatically by the kcp-operator.
                           If DNSNames is used together with IssuerRef, DNSNames will be uses as-is and not merged.
                           If IssuerRef is not set, DNSNames will be merged with the defaults. This is to avoid
-                          trying to guess what DNSNames configued issuer might support.
+                          trying to guess what DNSNames configured issuer might support.
                         items:
                           type: string
                         type: array
@@ -350,6 +371,13 @@ spec:
           status:
             description: KubeconfigStatus defines the observed state of Kubeconfig
             properties:
+              authorization:
+                properties:
+                  provisionedCluster:
+                    type: string
+                required:
+                - provisionedCluster
+                type: object
               conditions:
                 items:
                   description: Condition contains details for one aspect of the current

config/crd/bases/operator.kcp.io_rootshards.yaml

@@ -296,7 +296,7 @@ spec:
                             DNS names determined automatically by the kcp-operator.
                             If DNSNames is used together with IssuerRef, DNSNames will be uses as-is and not merged.
                             If IssuerRef is not set, DNSNames will be merged with the defaults. This is to avoid
-                            trying to guess what DNSNames configued issuer might support.
+                            trying to guess what DNSNames configured issuer might support.
                           items:
                             type: string
                           type: array
@@ -1766,7 +1766,7 @@ spec:
                                 DNS names determined automatically by the kcp-operator.
                                 If DNSNames is used together with IssuerRef, DNSNames will be uses as-is and not merged.
                                 If IssuerRef is not set, DNSNames will be merged with the defaults. This is to avoid
-                                trying to guess what DNSNames configued issuer might support.
+                                trying to guess what DNSNames configured issuer might support.
                               items:
                                 type: string
                               type: array

config/crd/bases/operator.kcp.io_shards.yaml

@@ -280,7 +280,7 @@ spec:
                             DNS names determined automatically by the kcp-operator.
                             If DNSNames is used together with IssuerRef, DNSNames will be uses as-is and not merged.
                             If IssuerRef is not set, DNSNames will be merged with the defaults. This is to avoid
-                            trying to guess what DNSNames configued issuer might support.
+                            trying to guess what DNSNames configured issuer might support.
                           items:
                             type: string
                           type: array

config/manager/kustomization.yaml

@@ -5,3 +5,4 @@ resources:
 images:
 - name: controller
   newName: ghcr.io/kcp-dev/kcp-operator
+  newTag: e2e

config/manager/manager.yaml

@@ -50,19 +50,15 @@ spec:
       #               - linux
       securityContext:
         runAsNonRoot: true
-        # TODO(user): For common cases that do not require escalating privileges
-        # it is recommended to ensure that all your Pods/Containers are restrictive.
-        # More info: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted
-        # Please uncomment the following code if your project does NOT have to work on old Kubernetes
-        # versions < 1.19 or on vendors versions which do NOT support this field by default (i.e. Openshift < 4.11 ).
-        # seccompProfile:
-        #   type: RuntimeDefault
+        seccompProfile:
+          type: RuntimeDefault
       containers:
       - command:
         - /manager
         args:
           - --leader-elect
           - --health-probe-bind-address=:8081
+          - --zap-time-encoding=iso8601
         image: controller:latest
         name: manager
         securityContext:

docs/content/architecture/.pages

@@ -2,4 +2,5 @@ nav:
   - index.md
   - basics.md
   - front-proxy.md
+  - kubeconfig.md
   - Certificate Management: pki.md

docs/content/architecture/index.md

@@ -7,7 +7,7 @@ This section describes how the kcp-operator is designed and meant to be used.
 - [Basics](basics.md) – A general overview over the resources provided by the kcp-operator.
 - [front-proxy](front-proxy.md) – Explains how the kcp front-proxy can be used to ingest traffic.
 - [Certificate Management](pki.md) – This page describes the various CAs and certificates used in a kcp installation.
+- [Kubeconfig](kubeconfig.md) – Shows how `Kubeconfig` objects can be used to provide credentials to kcp.
 <!--
 - [Sharding](sharding.md) – How `RootShards` and `Shards` work together to create a scalable kcp setup.
-- [Kubeconfigs](kubeconfigs.md) – Shows how `Kubeconfig` objects can be used to provide credentials to kcp.
 -->

docs/content/architecture/kubeconfig.md

@@ -0,0 +1,84 @@
+---
+description: >
+    Shows how `Kubeconfig` objects can be used to provide credentials to kcp.
+---
+
+# Kubeconfigs
+
+Besides provisioning kcp itself, the kcp-operator can also provide [kubeconfigs](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) to access kcp. Each kubeconfig will internally be backed by a dedicated client certificate.
+
+## Basics
+
+A minimal `Kubeconfig` object typically looks like this:
+
+```yaml
+apiVersion: operator.kcp.io/v1alpha1
+kind: Kubeconfig
+metadata:
+  name: susan
+  namespace: my-kcp
+spec:
+  # Required: the username inside Kubernetes;
+  # this will be the client certificate's common name.
+  username: susan
+
+  # required: groups to attach to the user;
+  # this will be the organizations in the client cert.
+  groups:
+    - system:kcp:admin
+
+  # Required: in what Secret the generated kubeconfig should be stored.
+  secretRef:
+    name: susan-kubeconfig
+
+  # Required: a Kubeconfig must target either a FrontProxy, Shard or RootShard.
+  target:
+    frontProxyRef:
+      name: my-front-proxy
+
+  # Required: how long the certificate should be valid for;
+  # the operator will automatically renew the certificate, after which the
+  # Secret will be renewed and have to be re-downloaded.
+  validity: 8766h
+```
+
+`Kubeconfig` objects must exist in the same namespace as the kcp installation they are meant for.
+
+Once the `Kubeconfig` has been created, you can observe its status to wait for it to be ready. After that, retrieve the Secret mentioned in the `secretRef` to find the finished kubeconfig, ready to use.
+
+!!! warning
+    Deleting a `Kubeconfig` will also delete the underlying Secret from the hosting cluster, however this will not invalidate the existing certificate that is embedded in the kubeconfig. This means anyone with a copy of the kubeconfig can keep using it until the certificate expires.
+
+    To disarm an old kubeconfig, make sure to revoke any permissions granted through RBAC for the user and/or their groups.
+
+!!! note
+    The `Kubeconfig`'s name is embedded into the certificate in form of a group (organization) named `kubeconfig:<name>`. This is to allow a unique mapping from RBAC rules to `Kubeconfig` objects for the authorization (see further down). Take note that this means the `Kubeconfig`' name is leaked to whoever gets the kubeconfig.
+
+## Authorization
+
+Without any further configuration than shown in the basics section above, the created identity (username + groups) will not get any permissions in kcp. So while the kubeconfig is valid and allows proper authentication, pretty much no actions will be permitted yet.
+
+The administrator has to either rely on externally-managed RBAC rules to provide permissions, or use the kcp-operator to provision such RBAC in a workspace.
+
+To make the kcp-operator manage RBAC, use `spec.authorization` inside a `Kubeconfig`:
+
+```yaml
+apiVersion: operator.kcp.io/v1alpha1
+kind: Kubeconfig
+metadata:
+  name: susan
+  namespace: my-kcp
+spec:
+  #...snip...
+
+  authorization:
+    clusterRoleBindings:
+      # This can be a workspace path (root:something) or a cluster name (ID).
+      cluster: root:initech:teamgibbons
+      clusterRoles:
+        - cluster-admin
+```
+
+This configuration would bind the group `kubeconfig:susan` to the ClusterRole `cluster-admin` inside the given workspace. Note that this is specifically not bound to the user (common name), so that two `Kubeconfig` objects that both have the same `spec.name` to not have colliding RBAC.
+
+When deleting a `Kubeconfig` with authorization settings, the kcp-operator will first unprovision (delete) the `ClusterRoleBindings` before the `Kubeconfig` can be deleted.

docs/content/contributing/local-setup.md

@@ -58,19 +58,20 @@ run the operator as a binary.
 Build the image:
 
 ```sh
-make docker-build IMG=ghcr.io/kcp-dev/kcp-operator:1
+export IMG=ghcr.io/kcp-dev/kcp-operator:local
+make docker-build
 ```
 
 Load the image into the kind cluster:
 
 ```sh
-kind load docker-image ghcr.io/kcp-dev/kcp-operator:1
+kind load docker-image "$IMG"
 ```
 
 Deploy the operator manifests into the cluster:
 
 ```sh
-make deploy IMG=ghcr.io/kcp-dev/kcp-operator:1
+make deploy
 ```
 
 ### Option 2: Run Operator Directly
@@ -87,12 +88,12 @@ Then start the operator via `go run`:
 go run ./cmd/main.go
 ```
 
-## Create kcp Instance 
+## Create kcp Instance
 
 Now you can create a root shard:
 
 ```sh
-kubectl apply -f config/samples/operator.kcp.io_v1alpha1_rootshard.yaml       
+kubectl apply -f config/samples/operator.kcp.io_v1alpha1_rootshard.yaml
 ```
 
 Create the additional shard:

go.mod

@@ -36,6 +36,7 @@ require (
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/egymgmbh/go-prefix-writer v0.0.0-20180609083313-7326ea162eca // indirect
 	github.com/emicklei/go-restful/v3 v3.12.1 // indirect
 	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
 	github.com/fatih/color v1.17.0 // indirect

go.sum

@@ -21,6 +21,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/egymgmbh/go-prefix-writer v0.0.0-20180609083313-7326ea162eca h1:7oodhZp9MZW0DBkrZXyUsJWKQFy35SVxjZ8K4vHXnk8=
+github.com/egymgmbh/go-prefix-writer v0.0.0-20180609083313-7326ea162eca/go.mod h1:UhMFM+dnOcm1f0Pve8uqRaxAhEYki+/CuA2BTDp2T04=
 github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU=
 github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls=

hack/run-e2e-tests.sh

@@ -16,7 +16,7 @@
 
 set -euo pipefail
 
-KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-e2e}"
+export KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-e2e}"
 DATA_DIR=".e2e-$KIND_CLUSTER_NAME"
 OPERATOR_PID=0
 PROTOKOL_PID=0
@@ -35,12 +35,6 @@ kind create cluster --name "$KIND_CLUSTER_NAME"
 chmod 600 "$KUBECONFIG"
 
 teardown_kind() {
-  if [[ $OPERATOR_PID -gt 0 ]]; then
-    echo "Stopping kcp-operator..."
-    kill -TERM $OPERATOR_PID
-    wait $OPERATOR_PID
-  fi
-
   if [[ $PROTOKOL_PID -gt 0 ]]; then
     echo "Stopping protokol..."
     kill -TERM $PROTOKOL_PID
@@ -59,7 +53,7 @@ fi
 echo "Kubeconfig is in $KUBECONFIG."
 
 # apply kernel limits job first and wait for completion
-echo "Applying kernel limits job…"
+echo "Applying kernel limits job..."
 kubectl apply --filename hack/ci/kernel.yaml
 kubectl wait --for=condition=Complete job/kernel-limits --timeout=300s
 echo "Kernel limits job completed."
@@ -85,20 +79,13 @@ _tools/helm upgrade \
 
 kubectl apply --filename hack/ci/testdata/clusterissuer.yaml
 
-# start the operator locally
-echo "Starting kcp-operator..."
-_build/manager \
-  -kubeconfig "$KUBECONFIG" \
-  -zap-log-level debug \
-  -zap-encoder console \
-  -zap-time-encoding iso8601 \
-  -health-probe-bind-address="" \
-  >"$DATA_DIR/kcp-operator.log" 2>&1 &
-OPERATOR_PID=$!
-echo "Running as process $OPERATOR_PID."
+# build operator image and deploy it into kind
+echo "Building and deploying kcp-operator..."
+export IMG="ghcr.io/kcp-dev/kcp-operator:e2e"
+make --no-print-directory docker-build kind-load deploy
 
 if command -v protokol &> /dev/null; then
-  protokol --namespace 'e2e-*' --output "$DATA_DIR/kind-logs" 2>/dev/null &
+  protokol --namespace 'e2e-*' --namespace kcp-operator-system --output "$DATA_DIR/kind-logs" 2>/dev/null &
   PROTOKOL_PID=$!
 else
   echo "Install https://codeberg.org/xrstf/protokol to automatically"