feat: multi cluster support (#240)

feat: multi-cluster listener & gateway support

- Implemented multi-cluster support for the listener and gateway.
- Introduced condition management and strategy pattern in reconcilers.
- Integrated golang-commons lifecycle.
- Removed singlecluster mode and reconciler factory.
- Added RoundTripperFactory and moved config builder to common package.

Signed-off-by: Artem Shcherbatiuk <[email protected]>
On-behalf-of: @SAP [email protected]

Author: vertex451

.gitignore

@@ -24,6 +24,7 @@ coverage.html
 *.swp
 *.swo
 *~
+*.log
 
 # MacOS
 **/.DS_Store
@@ -40,4 +41,5 @@ coverage.html
 go.work
 
 # binary files
-main
+main
+kubernetes-graphql-gateway

.mockery.yaml

@@ -4,21 +4,21 @@ with-expecter: true
 packages:
   k8s.io/client-go/discovery:
     config:
-      dir: listener/kcp/mocks
+      dir: listener/reconciler/kcp/mocks
       outpkg: mocks
     interfaces:
       DiscoveryInterface:
 
   k8s.io/apimachinery/pkg/api/meta:
     config:
-      dir: listener/kcp/mocks
+      dir: listener/reconciler/kcp/mocks
       outpkg: mocks
     interfaces:
       RESTMapper:
 
   sigs.k8s.io/controller-runtime/pkg/client:
     config:
-      dir: gateway/resolver/mocks
+      dir: common/mocks
       outpkg: mocks
     interfaces:
       WithWatch:
@@ -31,45 +31,46 @@ packages:
     interfaces:
       RoundTripper:
 
-  github.com/openmfp/kubernetes-graphql-gateway/listener/workspacefile:
+  github.com/openmfp/kubernetes-graphql-gateway/listener/pkg/workspacefile:
     config:
-      dir: listener/workspacefile/mocks
+      dir: listener/pkg/workspacefile/mocks
       outpkg: mocks
     interfaces:
       IOHandler:
 
-  github.com/openmfp/kubernetes-graphql-gateway/listener/discoveryclient:
+  github.com/openmfp/kubernetes-graphql-gateway/listener/reconciler/kcp/discoveryclient:
     config:
-      dir: listener/discoveryclient/mocks
+      dir: listener/reconciler/kcp/discoveryclient/mocks
       outpkg: mocks
     interfaces:
       Factory:
 
-  github.com/openmfp/kubernetes-graphql-gateway/listener/apischema:
+  github.com/openmfp/kubernetes-graphql-gateway/listener/pkg/apischema:
     config:
-      dir: listener/apischema/mocks
+      dir: listener/pkg/apischema/mocks
       outpkg: mocks
     interfaces:
       Resolver:
 
-  github.com/openmfp/kubernetes-graphql-gateway/listener/clusterpath:
+  github.com/openmfp/kubernetes-graphql-gateway/listener/reconciler/kcp/clusterpath:
     config:
-      dir: listener/clusterpath/mocks
+      dir: listener/reconciler/kcp/clusterpath/mocks
       outpkg: mocks
     interfaces:
       Resolver:
 
-  github.com/openmfp/kubernetes-graphql-gateway/listener/controller:
+  k8s.io/client-go/openapi:
     config:
-      dir: listener/controller/mocks
+      dir: listener/pkg/apischema/mocks
       outpkg: mocks
     interfaces:
-      CRDResolver:
+      GroupVersion:
+      Client:
 
-  k8s.io/client-go/openapi:
+  github.com/openmfp/kubernetes-graphql-gateway/gateway/manager:
     config:
-      dir: listener/apischema/mocks
+      dir: gateway/manager/mocks
       outpkg: mocks
     interfaces:
-      GroupVersion:
-      Client:
+      ClusterManager:
+      SchemaWatcher:

.testcoverage.yml

@@ -4,4 +4,8 @@ exclude:
     - cmd
     - tests
     - common/config/config.go
-    - mocks
+    - mocks
+    - common/apis/*
+    # remove it later:
+    - listener/reconciler/clusteraccess/subroutines.go
+    - listener/reconciler/singlecluster

README.md

@@ -13,6 +13,38 @@ This repository contains two main components:
 - [Listener](./docs/listener.md): watches a cluster and stores its openAPI spec in a directory.
 - [Gateway](./docs/gateway.md): exposes the openAPI spec as a GraphQL endpoints.
 
+## MultiCluster Support
+
+The system supports three modes of operation:
+
+1. **Single Cluster** (`ENABLE_KCP=false`, `MULTICLUSTER=false`): Gateway connects to the same cluster as the listener
+2. **KCP Mode** (`ENABLE_KCP=true`): Designed for KCP-based multi-cluster scenarios  
+3. **MultiCluster Mode** (`ENABLE_KCP=false`, `MULTICLUSTER=true`): Gateway connects to multiple external clusters via ClusterAccess resources
+
+### MultiCluster with ClusterAccess
+
+In MultiCluster mode, the system uses ClusterAccess resources to store kubeconfig data and connection information. The listener processes these resources and embeds connection metadata into schema files, which the gateway then uses to establish cluster-specific connections.
+
+For complete setup instructions, see:
+- [ClusterAccess documentation](./docs/clusteraccess.md) - Manual setup
+- [MultiCluster Kubeconfig Flow](./docs/multicluster-kubeconfig-flow.md) - Detailed flow explanation
+
+### Quick Setup Scripts
+
+```bash
+# Create ClusterAccess with secure token authentication  
+./scripts/create-clusteraccess.sh --target-kubeconfig ~/.kube/prod-config
+
+# Test end-to-end integration
+./scripts/test-clusteraccess-integration.sh
+```
+
+### Gateway Requirements
+
+- **Single Cluster Mode**: Requires KUBECONFIG to connect to the local cluster
+- **KCP Mode**: Requires KUBECONFIG to connect to KCP management cluster  
+- **MultiCluster Mode**: Does NOT require KUBECONFIG - gets all connection info from schema files
+
 ## Authorization
 
 All information about authorization can be found in the [authorization](./docs/authorization.md) section.

Taskfile.yml

@@ -6,6 +6,7 @@ vars:
   ENVTEST_K8S_VERSION: "1.30.0" # to get latest version run $(pwd)/bin/setup-envtest list
   ENVTEST_VERSION: "release-0.20" # https://github.com/kubernetes-sigs/controller-runtime/releases
   MOCKERY_VERSION: v2.52.3 # https://github.com/vektra/mockery/releases
+  CONTROLLER_GEN_VERSION: v0.18.0 # https://github.com/kubernetes-sigs/controller-tools/releases
 tasks:
   ## Setup
   setup:mockery:
@@ -16,6 +17,10 @@ tasks:
     internal: true
     cmds:
       - test -s {{.LOCAL_BIN}}/setup-envtest || GOBIN=$(pwd)/{{.LOCAL_BIN}} go install sigs.k8s.io/controller-runtime/tools/setup-envtest@{{.ENVTEST_VERSION}}
+  setup:controller-gen:
+    internal: true
+    cmds:
+      - test -s {{.LOCAL_BIN}}/controller-gen || GOBIN=$(pwd)/{{.LOCAL_BIN}} go install sigs.k8s.io/controller-tools/cmd/controller-gen@{{.CONTROLLER_GEN_VERSION}}
   update:crd:
     desc: "Download the latest CRD from OpenMFP"
     cmds:
@@ -31,6 +36,25 @@ tasks:
     cmds:
       - test -s {{.LOCAL_BIN}}/go-test-coverage || GOBIN=$(pwd)/{{.LOCAL_BIN}} go install github.com/vladopajic/go-test-coverage/v2@latest
 
+  ## Code Generation
+  generate:crd:
+    desc: "Generate CRD manifests from Go types"
+    deps: [setup:controller-gen]
+    cmds:
+      - "{{.LOCAL_BIN}}/controller-gen crd:crdVersions=v1 paths=./common/apis/v1alpha1 output:crd:artifacts:config=config/crd"
+      - echo "CRD manifests generated successfully in config/crd/"
+  generate:deepcopy:
+    desc: "Generate deepcopy methods for API types"
+    deps: [setup:controller-gen]
+    cmds:
+      - "{{.LOCAL_BIN}}/controller-gen object paths=./common/apis/v1alpha1"
+      - echo "Deepcopy methods generated successfully"
+  generate:
+    desc: "Generate all CRD-related files (manifests + deepcopy methods)"
+    deps: [generate:crd, generate:deepcopy]
+    cmds:
+      - echo "All CRD generation completed successfully!"
+
   ## Development
   mockery:
     deps: [ setup:mockery ]
@@ -62,7 +86,7 @@ tasks:
         vars:
           ADDITIONAL_COMMAND_ARGS: -coverprofile=./cover.out -covermode=atomic -coverpkg=./...
   cover:
-    deps: [ setup:envtest, setup:go-test-coverage ]
+    deps: [ setup:envtest, update:crd, setup:go-test-coverage ]
     cmds:
       - task: envtest
         vars:
@@ -75,16 +99,36 @@ tasks:
       - go tool cover -html=cover.out -o coverage.html
       - open coverage.html || xdg-open coverage.html || start coverage.html
   validate:
+    desc: "Run all validation checks including code generation, linting, and testing"
     cmds:
+      - task: generate
       - task: mockery
       - task: lint
       - task: test
 
 
   gateway:
-    cmds:
+    desc: "Start the GraphQL gateway server (kills existing process on port 8080 if needed)"
+    cmds:
+      - |
+        # Check if port 8080 is in use and kill the process if found
+        PID=$(lsof -ti:8080 2>/dev/null || echo "")
+        if [ ! -z "$PID" ]; then
+          echo "Found existing process $PID on port 8080, killing it..."
+          kill $PID 2>/dev/null || true
+          sleep 2
+        fi
       - go run main.go gateway
 
   listener:
-    cmds:
+    desc: "Start the listener server (kills existing process on port 8090 if needed)"
+    cmds:
+      - |
+        # Check if port 8090 is in use and kill the process if found
+        PID=$(lsof -ti:8090 2>/dev/null || echo "")
+        if [ ! -z "$PID" ]; then
+          echo "Found existing process $PID on port 8090, killing it..."
+          kill $PID 2>/dev/null || true
+          sleep 2
+        fi
       - go run main.go listener

cmd/gateway.go

@@ -13,8 +13,6 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/cobra"
 	ctrl "sigs.k8s.io/controller-runtime"
-	restCfg "sigs.k8s.io/controller-runtime/pkg/client/config"
-	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
 	"github.com/openmfp/golang-commons/logger"
 
@@ -48,12 +46,12 @@ var gatewayCmd = &cobra.Command{
 			defer openmfpcontext.Recover(log)
 		}
 
-		ctrl.SetLogger(zap.New(zap.UseDevMode(true)))
+		ctrl.SetLogger(log.Logr())
 
-		// Get Kubernetes restCfg
-		restCfg, err := restCfg.GetConfig()
+		gatewayInstance, err := manager.NewGateway(log, appCfg)
 		if err != nil {
-			log.Fatal().Err(err).Msg("Error getting Kubernetes restCfg, exiting")
+			log.Error().Err(err).Msg("Error creating gateway")
+			return fmt.Errorf("failed to create gateway: %w", err)
 		}
 
 		// Initialize tracing provider
@@ -76,15 +74,14 @@ var gatewayCmd = &cobra.Command{
 			}
 		}()
 
-		// Initialize Manager
-		managerInstance, err := manager.NewManager(log, restCfg, appCfg)
-		if err != nil {
-			log.Error().Err(err).Msg("Error creating manager")
-			return fmt.Errorf("failed to create manager: %w", err)
-		}
+		defer func() {
+			if err := providerShutdown(ctx); err != nil {
+				log.Fatal().Err(err).Msg("failed to shutdown TracerProvider")
+			}
+		}()
 
 		// Set up HTTP handler
-		http.Handle("/", managerInstance)
+		http.Handle("/", gatewayInstance)
 
 		// Replace the /metrics endpoint handler
 		http.Handle("/metrics", promhttp.Handler())
@@ -113,6 +110,10 @@ var gatewayCmd = &cobra.Command{
 			log.Fatal().Err(err).Msg("HTTP server shutdown failed")
 		}
 
+		if err := gatewayInstance.Close(); err != nil {
+			log.Error().Err(err).Msg("Error closing gateway services")
+		}
+
 		// Call the shutdown cleanup
 		shutdown()