✨ Add hook server into same container as operator

Add hook server into same container as operator

Adding the hook server into the same container as CSO
Signed-off-by: janiskemper <[email protected]>

Author: janiskemper

Tiltfile

@@ -18,7 +18,6 @@ settings = {
         "kind-cso",
     ],
     "local_mode": False,
-    "runtime_sdk": True,
     "deploy_cert_manager": True,
     "preload_images_for_kind": True,
     "kind_cluster_name": "cso",
@@ -110,12 +109,6 @@ def fixup_yaml_empty_arrays(yaml_str):
     yaml_str = yaml_str.replace("conditions: null", "conditions: []")
     return yaml_str.replace("storedVersions: null", "storedVersions: []")
 
-tilt_dockerfile_header_runtime = """
-FROM docker.io/library/alpine:3.18.0
-WORKDIR /
-COPY extension/.tiltbuild/manager .
-"""
-
 ## This should have the same versions as the Dockerfile
 if settings.get("local_mode"):
     tilt_dockerfile_header_cso = """
@@ -218,53 +211,11 @@ def deploy_cso():
         labels = ["CSO"],
     )
 
-def deploy_runtime_extension():
-    yaml = str(kustomizesub("./extension/config/default"))
-
-    local_resource(
-        name = "runtime-components",
-        cmd = ["sh", "-ec", sed_cmd, yaml, "|", envsubst_cmd],
-        labels = ["runtime"],
-    )
-
-    local_resource(
-        "runtime-manager",
-        cmd = 'cd extension; mkdir -p .tiltbuild;CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags \'-extldflags "-static"\' -o .tiltbuild/manager main.go',
-        deps = ["extension/handlers", "extension/main.go"],
-        labels = ["runtime"],
-    )
-
-    entrypoint = ["/manager"]
-    extra_args = settings.get("extra_args")
-    if extra_args:
-        entrypoint.extend(extra_args)
-
-    docker_build_with_restart(
-    ref = "ghcr.io/sovereigncloudstack/runtime-sdk-staging",
-    context = ".",
-    dockerfile_contents = tilt_dockerfile_header_runtime,
-    entrypoint = entrypoint,
-    live_update = [
-        sync("extension/.tiltbuild/manager", "/manager"),
-        # sync(".release", "/tmp/cluster-stacks"),
-    ],
-    ignore = ["templates"],
-    )
+def deploy_capd():
+    yaml = './capd.yaml'
+    cmd = "kubectl apply -f capd.yaml"
+    local(cmd, quiet = True)
 
-    k8s_yaml(blob(yaml))
-    k8s_resource(workload = "test-runtime-sdk", labels = ["runtime"])
-    k8s_resource(
-        objects = [
-            "runtimesdk:namespace",
-            # "test-runtime-sdk:deployment",
-            "test-runtime-sdk-sa:serviceaccount",
-            "test-runtime-sdk-role:clusterrole",
-            "test-runtime-sdk-role-rolebinding:clusterrolebinding",
-            "runtime-sdk-selfsigned-issuer:issuer",
-        ],
-        new_name = "runtime-misc",
-        labels = ["runtime"],
-    )
 
 def clusterstack():
     k8s_resource(objects = ["clusterstack:clusterstack"], new_name = "clusterstack", labels = ["CLUSTERSTACK"])
@@ -333,9 +284,6 @@ deploy_cso()
 
 clusterstack()
 
-if settings.get("runtime_sdk"):
-    deploy_runtime_extension()
-
 waitforsystem()
 
 prepare_environment()

cmd/main.go

@@ -26,6 +26,7 @@ import (
 
 	//+kubebuilder:scaffold:imports
 	csov1alpha1 "github.com/SovereignCloudStack/cluster-stack-operator/api/v1alpha1"
+	"github.com/SovereignCloudStack/cluster-stack-operator/extension/handlers"
 	"github.com/SovereignCloudStack/cluster-stack-operator/internal/controller"
 	"github.com/SovereignCloudStack/cluster-stack-operator/pkg/assetsclient"
 	"github.com/SovereignCloudStack/cluster-stack-operator/pkg/assetsclient/fake"
@@ -39,6 +40,10 @@ import (
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/cluster-api/controllers/remote"
+	runtimecatalog "sigs.k8s.io/cluster-api/exp/runtime/catalog"
+	runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
+	"sigs.k8s.io/cluster-api/exp/runtime/server"
 	dockerv1beta1 "sigs.k8s.io/cluster-api/test/infrastructure/docker/api/v1beta1"
 	"sigs.k8s.io/cluster-api/util/record"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -51,9 +56,15 @@ import (
 var (
 	scheme   = runtime.NewScheme()
 	setupLog = ctrl.Log.WithName("setup")
+
+	// catalog contains all information about RuntimeHooks.
+	catalog = runtimecatalog.New()
 )
 
 func init() {
+	// Adds to the catalog all the RuntimeHooks defined in cluster API.
+	_ = runtimehooksv1.AddToCatalog(catalog)
+
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 	utilruntime.Must(csov1alpha1.AddToScheme(scheme))
 	utilruntime.Must(dockerv1beta1.AddToScheme(scheme))
@@ -76,6 +87,8 @@ var (
 	localMode                      bool
 	qps                            float64
 	burst                          int
+	hookPort                       int
+	hookCertDir                    string
 )
 
 func main() {
@@ -93,7 +106,8 @@ func main() {
 	flag.BoolVar(&localMode, "local", false, "Enable local mode where no release assets will be downloaded from a remote Git repository. Useful for implementing cluster stacks.")
 	flag.Float64Var(&qps, "qps", 50, "Enable custom query per second for kubernetes API server")
 	flag.IntVar(&burst, "burst", 100, "Enable custom burst defines how many queries the API server will accept before enforcing the limit established by qps")
-
+	flag.IntVar(&hookPort, "hook-port", 9442, "hook server port")
+	flag.StringVar(&hookCertDir, "hook-cert-dir", "/tmp/k8s-hook-server/serving-certs/", "hook cert dir, only used when hook-port is specified.")
 	flag.Parse()
 
 	ctrl.SetLogger(utillog.GetDefaultLogger(logLevel))
@@ -151,7 +165,6 @@ func main() {
 	}
 
 	var wg sync.WaitGroup
-	wg.Add(1)
 
 	if err = (&controller.ClusterStackReconciler{
 		Client:              mgr.GetClient(),
@@ -207,14 +220,88 @@ func main() {
 		os.Exit(1)
 	}
 
-	setupLog.Info("starting manager", "version", csoversion.Get().String())
-	if err := mgr.Start(ctx); err != nil {
-		setupLog.Error(err, "problem running manager")
+	// Create a http server for serving runtime extensions
+	hookServer, err := server.New(server.Options{
+		Catalog: catalog,
+		Port:    hookPort,
+		CertDir: hookCertDir,
+	})
+	if err != nil {
+		setupLog.Error(err, "error creating webhook server")
+		os.Exit(1)
+	}
+
+	// Lifecycle Hooks
+	// Gets a client to access the Kubernetes cluster where this RuntimeExtension will be deployed to;
+	// this is a requirement specific of the lifecycle hooks implementation for Cluster APIs E2E tests.
+	restConfig.UserAgent = remote.DefaultClusterAPIUserAgent("cluster-stack-operator-extension-manager")
+
+	// Create the ExtensionHandlers for the lifecycle hooks
+	lifecycleExtensionHandlers := handlers.NewExtensionHandlers(mgr.GetClient(), scheme)
+
+	setupLog.Info("Add extension handlers")
+	if err := hookServer.AddExtensionHandler(server.ExtensionHandler{
+		Hook:        runtimehooksv1.BeforeClusterUpgrade,
+		Name:        "before-cluster-upgrade",
+		HandlerFunc: lifecycleExtensionHandlers.DoBeforeClusterUpgrade,
+	}); err != nil {
+		setupLog.Error(err, "error adding handler")
+		os.Exit(1)
+	}
+
+	if err := hookServer.AddExtensionHandler(server.ExtensionHandler{
+		Hook:        runtimehooksv1.AfterClusterUpgrade,
+		Name:        "after-cluster-upgrade",
+		HandlerFunc: lifecycleExtensionHandlers.DoAfterClusterUpgrade,
+	}); err != nil {
+		setupLog.Error(err, "error adding handler")
 		os.Exit(1)
 	}
 
-	wg.Done()
-	// Wait for all target cluster managers to gracefully shut down.
+	if err := hookServer.AddExtensionHandler(server.ExtensionHandler{
+		Hook:        runtimehooksv1.AfterControlPlaneInitialized,
+		Name:        "after-control-plane-initialized",
+		HandlerFunc: lifecycleExtensionHandlers.DoAfterControlPlaneInitialized,
+	}); err != nil {
+		setupLog.Error(err, "error adding handler")
+		os.Exit(1)
+	}
+
+	errChan := make(chan error, 1)
+
+	wg.Add(1)
+
+	go func() {
+		setupLog.Info("starting manager", "version", csoversion.Get().String())
+		if err := mgr.Start(ctx); err != nil {
+			setupLog.Error(err, "problem running manager")
+			errChan <- err
+		}
+		wg.Done()
+	}()
+
+	wg.Add(1)
+
+	go func() {
+		setupLog.Info("starting hook server")
+		if err := hookServer.Start(ctx); err != nil {
+			setupLog.Error(err, "problem running hook server")
+			errChan <- err
+		}
+		wg.Done()
+	}()
+
+	go func() {
+		select {
+		case err := <-errChan:
+			setupLog.Error(err, "Received error")
+			os.Exit(1)
+		case <-ctx.Done():
+			setupLog.Info("shutting down")
+		}
+	}()
+
+	// wait for all processes to shut down
 	wg.Wait()
 }
 

config/certmanager/certificate.yaml

@@ -26,3 +26,20 @@ spec:
   subject:
     organizations:
     - k8s-sig-cluster-lifecycle
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: hook-server-serving-cert  # this name should match the one appeared in kustomizeconfig.yaml
+  namespace: system
+spec:
+  dnsNames:
+  - $(HOOK_SERVER_SERVICE_NAME).$(HOOK_SERVER_SERVICE_NAMESPACE).svc
+  - $(HOOK_SERVER_SERVICE_NAME).$(HOOK_SERVER_SERVICE_NAMESPACE).svc.cluster.local
+  issuerRef:
+    kind: Issuer
+    name: selfsigned-issuer
+  secretName: cso-hook-server-server-cert  # this secret will not be prefixed, since it's not managed by kustomize
+  subject:
+    organizations:
+    - k8s-sig-cluster-lifecycle

config/default/kustomization.yaml

@@ -10,11 +10,13 @@ resources:
   - ../rbac
   - ../manager
   - ../webhook
+  - ../hookserver
   - ../certmanager
 
 patchesStrategicMerge:
   - manager_config_patch.yaml
   - manager_webhook_patch.yaml
+  - manager_hookserver_patch.yaml
   - webhookcainjection_patch.yaml
   - manager_pull_policy.yaml
 vars:
@@ -26,21 +28,47 @@ vars:
       name: serving-cert # this name should match the one in certificate.yaml
     fieldref:
       fieldpath: metadata.namespace
+  - name: HOOK_SERVER_CERTIFICATE_NAMESPACE  # namespace of the certificate CR
+    objref:
+      kind: Certificate
+      group: cert-manager.io
+      version: v1
+      name: hook-server-serving-cert  # this name should match the one in certificate.yaml
+    fieldref:
+      fieldpath: metadata.namespace
   - name: CERTIFICATE_NAME
     objref:
       kind: Certificate
       group: cert-manager.io
       version: v1
       name: serving-cert # this name should match the one in certificate.yaml
+  - name: HOOK_SERVER_CERTIFICATE_NAME
+    objref:
+      kind: Certificate
+      group: cert-manager.io
+      version: v1
+      name: hook-server-serving-cert  # this name should match the one in certificate.yaml
   - name: SERVICE_NAMESPACE # namespace of the service
     objref:
       kind: Service
       version: v1
       name: webhook-service
     fieldref:
       fieldpath: metadata.namespace
+  - name: HOOK_SERVER_SERVICE_NAMESPACE  # namespace of the service
+    objref:
+      kind: Service
+      version: v1
+      name: hook-server-svc
+    fieldref:
+      fieldpath: metadata.namespace
   - name: SERVICE_NAME
     objref:
       kind: Service
       version: v1
       name: webhook-service
+  - name: HOOK_SERVER_SERVICE_NAME
+    objref:
+      kind: Service
+      version: v1
+      name: hook-server-svc

config/default/manager_hookserver_patch.yaml

@@ -0,0 +1,23 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: manager
+        ports:
+        - containerPort: 9442
+          name: hook-server-svc
+          protocol: TCP
+        volumeMounts:
+        - mountPath: /tmp/k8s-hook-server/serving-certs
+          name: hook-server-cert
+          readOnly: true
+      volumes:
+      - name: hook-server-cert
+        secret:
+          defaultMode: 420
+          secretName: cso-hook-server-server-cert

config/hookserver/extensionconfig.yaml

@@ -0,0 +1,13 @@
+apiVersion: runtime.cluster.x-k8s.io/v1alpha1
+kind: ExtensionConfig
+metadata:
+  annotations:
+    runtime.cluster.x-k8s.io/inject-ca-from-secret: $(HOOK_SERVER_CERTIFICATE_NAMESPACE)/$(HOOK_SERVER_CERTIFICATE_NAME)
+  name: hook-server-extensionconfig
+spec:
+  clientConfig:
+    service:
+      name: hook-server-svc
+      namespace: system # Note: this assumes the test extension get deployed in the default namespace defined in its own runtime-extensions-components.yaml
+      port: 443
+  namespaceSelector: {}

config/hookserver/kustomization.yaml

@@ -0,0 +1,6 @@
+resources:
+- service.yaml
+- extensionconfig.yaml
+
+configurations:
+- kustomizeconfig.yaml

config/hookserver/kustomizeconfig.yaml

@@ -0,0 +1,18 @@
+# the following config is for teaching kustomize where to look at when substituting vars.
+# It requires kustomize v2.1.0 or newer to work properly.
+nameReference:
+- kind: Service
+  version: v1
+  fieldSpecs:
+  - kind: ExtensionConfig
+    group: runtime.cluster.x-k8s.io
+    path: spec/clientConfig/service/name
+
+namespace:
+- kind: ExtensionConfig
+  group: runtime.cluster.x-k8s.io
+  path: spec/clientConfig/service/namespace
+  create: true
+
+varReference:
+- path: metadata/annotations