✨ Add clusteraddon controller (#28)

Add clusteraddon controller which is responsible to reconcile
ClusterAddons. It templates the ClusterAddon Helm Chart from the cluster
stack assets and applies its objects. It also updates the objects if the
Helm Chart has been updates and only then - not necessarily on each
update of a ClusterStack version.

Signed-off-by: janiskemper <[email protected]>

Author: janiskemper

README.md

@@ -64,4 +64,30 @@ To tear down the kind cluster, use:
 $ make delete-bootstrap-cluster
 ```
 
-If you have any trouble finding the right command, then you can use `make help` to get a list of all available make targets. 
+If you have any trouble finding the right command, then you can use `make help` to get a list of all available make targets. 
+
+## Troubleshooting
+
+Check the latest events:
+
+```shell
+kubectl get events -A  --sort-by=.lastTimestamp
+```
+
+Check the conditions:
+
+```shell
+go run github.com/guettli/check-conditions@latest all 
+```
+
+Check with `clusterctl`:
+
+```shell
+clusterctl describe cluster -n cluster my-cluster
+```
+
+Check the logs. List all logs from all deployments. Show the logs of the last ten minutes:
+
+```shell
+kubectl get deployment -A --no-headers | while read -r ns d _; do echo; echo "====== $ns $d"; kubectl logs --since=10m -n $ns deployment/$d; done
+```

cmd/main.go

@@ -31,6 +31,7 @@ import (
 	githubclient "github.com/SovereignCloudStack/cluster-stack-operator/pkg/github/client"
 	"github.com/SovereignCloudStack/cluster-stack-operator/pkg/kube"
 	"github.com/SovereignCloudStack/cluster-stack-operator/pkg/utillog"
+	"github.com/SovereignCloudStack/cluster-stack-operator/pkg/workloadcluster"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
@@ -54,19 +55,23 @@ func init() {
 	//+kubebuilder:scaffold:scheme
 }
 
-func main() {
-	var metricsAddr string
-	var probeAddr string
-	var enableLeaderElection bool
-	var leaderElectionNamespace string
-	var watchFilterValue string
-	var watchNamespace string
-	var clusterStackConcurrency int
-	var clusterStackReleaseConcurrency int
-	var clusterAddonConcurrency int
-	var logLevel string
-	var releaseDir string
+var (
+	metricsAddr                    string
+	probeAddr                      string
+	enableLeaderElection           bool
+	leaderElectionNamespace        string
+	watchFilterValue               string
+	watchNamespace                 string
+	clusterStackConcurrency        int
+	clusterStackReleaseConcurrency int
+	clusterAddonConcurrency        int
+	logLevel                       string
+	releaseDir                     string
+	qps                            float64
+	burst                          int
+)
 
+func main() {
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":9440", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", true, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
@@ -78,6 +83,8 @@ func main() {
 	flag.IntVar(&clusterAddonConcurrency, "clusteraddon-concurrency", 1, "Number of ClusterAddons to process simultaneously")
 	flag.StringVar(&logLevel, "log-level", "info", "Specifies log level. Options are 'debug', 'info' and 'error'")
 	flag.StringVar(&releaseDir, "release-dir", "/tmp/downloads/", "Specify release directory for cluster-stack releases")
+	flag.Float64Var(&qps, "qps", 50, "Enable custom query per second for kubernetes API server")
+	flag.IntVar(&burst, "burst", 100, "Enable custom burst defines how many queries the API server will accept before enforcing the limit established by qps")
 
 	flag.Parse()
 
@@ -111,6 +118,15 @@ func main() {
 
 	gitFactory := githubclient.NewFactory()
 
+	restConfig := mgr.GetConfig()
+	restConfig.QPS = float32(qps)
+	restConfig.Burst = burst
+
+	restConfigSettings := controller.RestConfigSettings{
+		QPS:   float32(qps),
+		Burst: burst,
+	}
+
 	var wg sync.WaitGroup
 	wg.Add(1)
 
@@ -123,9 +139,10 @@ func main() {
 		setupLog.Error(err, "unable to create controller", "controller", "ClusterStack")
 		os.Exit(1)
 	}
+
 	if err = (&controller.ClusterStackReleaseReconciler{
 		Client:              mgr.GetClient(),
-		RESTConfig:          mgr.GetConfig(),
+		RESTConfig:          restConfig,
 		ReleaseDirectory:    releaseDir,
 		WatchFilterValue:    watchFilterValue,
 		KubeClientFactory:   kube.NewFactory(),
@@ -135,6 +152,18 @@ func main() {
 		os.Exit(1)
 	}
 
+	if err = (&controller.ClusterAddonReconciler{
+		Client:                 mgr.GetClient(),
+		ReleaseDirectory:       releaseDir,
+		RestConfigSettings:     restConfigSettings,
+		WatchFilterValue:       watchFilterValue,
+		KubeClientFactory:      kube.NewFactory(),
+		WorkloadClusterFactory: workloadcluster.NewFactory(),
+	}).SetupWithManager(ctx, mgr, controllerruntimecontroller.Options{MaxConcurrentReconciles: clusterAddonConcurrency}); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "ClusterAddon")
+		os.Exit(1)
+	}
+
 	if err = (&controller.ClusterAddonCreateReconciler{
 		Client: mgr.GetClient(),
 	}).SetupWithManager(ctx, mgr, controllerruntimecontroller.Options{}); err != nil {

config/crd/kustomization.yaml

@@ -7,7 +7,7 @@ commonlabels:
 resources:
 - bases/clusterstack.x-k8s.io_clusterstacks.yaml
 - bases/clusterstack.x-k8s.io_clusterstackreleases.yaml
-#- bases/clusterstack.x-k8s.io_clusteraddons.yaml
+- bases/clusterstack.x-k8s.io_clusteraddons.yaml
 #+kubebuilder:scaffold:crdkustomizeresource
 
 patchesStrategicMerge:

config/cso/cluster.yaml

@@ -2,7 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1
 kind: Cluster
 metadata:
   name: "${CLUSTER_NAME}"
-  namespace: ${NAMESPACE}
+  namespace: cluster
 spec:
   clusterNetwork:
     services:
@@ -23,4 +23,4 @@ spec:
       machineDeployments:
         - class: workeramd64
           name: md-0
-          replicas: 1
+          replicas: 1

config/rbac/role.yaml

@@ -39,12 +39,38 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - cluster.x-k8s.io
+  resources:
+  - clusters
+  verbs:
+  - get
 - apiGroups:
   - clusterstack.x-k8s.io
   resources:
   - clusteraddons
   verbs:
   - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - clusterstack.x-k8s.io
+  resources:
+  - clusteraddons/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - clusterstack.x-k8s.io
+  resources:
+  - clusteraddons/status
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - clusterstack.x-k8s.io
   resources:

go.mod

@@ -4,6 +4,7 @@ go 1.19
 
 require (
 	github.com/go-logr/logr v1.2.4
+	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572
 	github.com/google/go-github/v52 v52.0.0
 	github.com/onsi/ginkgo/v2 v2.11.0
 	github.com/onsi/gomega v1.27.8
@@ -21,7 +22,6 @@ require (
 )
 
 require (
-	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect

hack/kind-dev.sh

@@ -44,6 +44,9 @@ kindV1Alpha4Cluster:
   nodes:
   - role: control-plane
     image: kindest/node:${CLUSTER_VERSION}
+    extraMounts:
+    - hostPath: /var/run/docker.sock
+      containerPath: /var/run/docker.sock
   networking:
     podSubnet: "10.244.0.0/16"
     serviceSubnet: "10.96.0.0/12"