diff --git a/api/constants/conditions.go b/api/constants/conditions.go index 5fcf846..449aef7 100644 --- a/api/constants/conditions.go +++ b/api/constants/conditions.go @@ -3,4 +3,10 @@ package constants const ( // ConditionMCPSuccessful is an aggregated condition showing whether all component resources could be reconciled successfully. ConditionMCPSuccessful = "MCPSuccessful" + + ConditionClusterRequestGranted = "ClusterRequestGranted" + ConditionClusterReady = "ClusterReady" + ConditionAccessRequestGranted = "AccessRequestGranted" + ConditionAccessRequestDeleted = "AccessRequestDeleted" + ConditionClusterRequestDeleted = "ClusterRequestDeleted" ) diff --git a/api/constants/reasons.go b/api/constants/reasons.go index 1cdaddc..39790ef 100644 --- a/api/constants/reasons.go +++ b/api/constants/reasons.go @@ -97,3 +97,11 @@ const ( // ReasonNotAllComponentsReconciledSuccessfully indicates that not all components have been reconciled successfully. ReasonNotAllComponentsReconciledSuccessfully = "NotAllComponentsReconciledSuccessfully" ) + +const ( + ReasonClusterRequestNotGranted = "ClusterRequestNotGranted" + ReasonClusterNotReady = "ClusterNotReady" + ReasonAccessRequestNotGranted = "AccessRequestNotGranted" + ReasonAccessRequestNotDeleted = "AccessRequestNotDeleted" + ReasonClusterRequestNotDeleted = "ClusterRequestNotDeleted" +) diff --git a/api/core/v1alpha1/component_types.go b/api/core/v1alpha1/component_types.go index 5d6c6d0..7de8287 100644 --- a/api/core/v1alpha1/component_types.go +++ b/api/core/v1alpha1/component_types.go @@ -139,3 +139,15 @@ func (ct ComponentType) ReconciliationCondition() string { func (ct ComponentType) HealthyCondition() string { return fmt.Sprintf("%sHealthy", string(ct)) } + +// ArchitectureLabelPrefix returns the component-specific architecture label prefix. +// Note that this label is only used on the MCP resource itself, on the component resources, the static ArchitectureLabelPrefix is used. +func (ct ComponentType) ArchitectureLabelPrefix() string { + return fmt.Sprintf("%s.%s", strings.ToLower(string(ct)), ArchitectureLabelPrefix) +} + +// ArchitectureVersionLabel returns the component-specific architecture version label. +// Note that this label is only used on the MCP resource itself, on the component resources, the static ArchitectureVersionLabel is used. +func (ct ComponentType) ArchitectureVersionLabel() string { + return fmt.Sprintf("%s%s", ct.ArchitectureLabelPrefix(), "version") +} diff --git a/api/core/v1alpha1/constants.go b/api/core/v1alpha1/constants.go index 99652a4..04898c0 100644 --- a/api/core/v1alpha1/constants.go +++ b/api/core/v1alpha1/constants.go @@ -39,6 +39,11 @@ const ( // ManagedByLabel is added to resources created by the operator. ManagedByLabel = BaseDomain + "/managed-by" + // ManagedPurposeLabel holds the purpose of a managed resource. + ManagedPurposeLabel = "managed." + BaseDomain + "/purpose" + // ManagedPurposeArchitectureImmutability is the value of the managed purpose label for resources that are used to enforce architecture immutability. + ManagedPurposeArchitectureImmutability = "architecture-immutability" + CreatedByAnnotation = BaseDomain + "/created-by" DisplayNameAnnotation = BaseDomain + "/display-name" @@ -76,4 +81,12 @@ const ( APIServerDomain = "apiserver." + BaseDomain ManagedByAPIServerLabel = APIServerDomain + "/managed" + + // Architecture Switch Labels + ArchitectureLabelPrefix = "architecture." + BaseDomain + "/" + ArchitectureVersionLabel = ArchitectureLabelPrefix + "version" + ArchitectureV1 = "v1" + ArchitectureV2 = "v2" + V1MCPReferenceLabelName = "v1." + BaseDomain + "/mcp-name" + V1MCPReferenceLabelNamespace = "v1." + BaseDomain + "/mcp-namespace" ) diff --git a/charts/mcp-operator/templates/configmap-mcp-operator-config.yaml b/charts/mcp-operator/templates/configmap-mcp-operator-config.yaml new file mode 100644 index 0000000..4563794 --- /dev/null +++ b/charts/mcp-operator/templates/configmap-mcp-operator-config.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mcp-operator-config + namespace: {{ .Release.Namespace }} + labels: + {{- include "mcp-operator.labels" . | nindent 4 }} +data: + config.yaml: | + architecture: + {{- if and .Values.architecture .Values.architecture.immutability }} + immutability: + {{- .Values.architecture.immutability | toYaml | nindent 8 }} + {{- end }} + {{- if and .Values.apiServer .Values.apiServer.architecture }} + apiServer: + version: {{ .Values.apiServer.architecture.version | default "v1" }} + allowOverride: {{ .Values.apiServer.architecture.allowOverride | default false }} + {{- end }} diff --git a/charts/mcp-operator/templates/deployment.yaml b/charts/mcp-operator/templates/deployment.yaml index e14962c..0e70ce0 100644 --- a/charts/mcp-operator/templates/deployment.yaml +++ b/charts/mcp-operator/templates/deployment.yaml @@ -30,6 +30,7 @@ spec: checksum/co-clusters: {{ include (print $.Template.BasePath "/secrets-cloudorchestrator-clusters.yaml") . | sha256sum }} checksum/auth-config: {{ include (print $.Template.BasePath "/secret-auth-config.yaml") . | sha256sum }} checksum/authz-config: {{ include (print $.Template.BasePath "/secret-authz-config.yaml") . | sha256sum }} + checksum/mcp-operator-config: {{ include (print $.Template.BasePath "/configmap-mcp-operator-config.yaml") . | sha256sum }} {{- with .Values.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} @@ -50,6 +51,7 @@ spec: command: - /mcp-operator - --controllers={{ include "mcp-operator.activeControllersString" .Values }} + - --config=/etc/config/mcp-operator/config.yaml {{- if .Values.deployment.leaderElection.enabled }} - --leader-elect - --lease-namespace={{ .Values.deployment.leaderElection.leaseNamespace }} @@ -140,6 +142,9 @@ spec: mountPath: /tmp/k8s-webhook-server/serving-certs/ readOnly: true {{- end }} + - name: mcp-operator-config + mountPath: /etc/config/mcp-operator + readOnly: true - name: common mountPath: /etc/config/common readOnly: true @@ -247,6 +252,11 @@ spec: {{- end }} {{- end }} {{- end }} + - name: mcp-operator-config + projected: + sources: + - configMap: + name: mcp-operator-config - name: common projected: sources: diff --git a/charts/mcp-operator/values.yaml b/charts/mcp-operator/values.yaml index 97ef6e4..427124c 100644 --- a/charts/mcp-operator/values.yaml +++ b/charts/mcp-operator/values.yaml @@ -32,7 +32,10 @@ clusters: # caData: ... # caConfigMapName: ... - +# architecture: # architecture configuration +# immutability: +# policyName: mcp-architecture-immutability # name of the ValidatingAdmissionPolicy to enforce architecture immutability +# disabled: false # whether architecture immutability should be enforced (strongly recommended to leave this enabled) crds: manage: true @@ -53,6 +56,9 @@ managedcontrolplane: apiserver: disabled: false + # architecture: + # version: v1 + # allowOverride: false worker: maxWorkers: 10 intervalTime: 10s diff --git a/cmd/mcp-operator/app/app.go b/cmd/mcp-operator/app/app.go index 7184c37..0f45214 100644 --- a/cmd/mcp-operator/app/app.go +++ b/cmd/mcp-operator/app/app.go @@ -9,6 +9,7 @@ import ( "github.com/openmcp-project/mcp-operator/internal/releasechannel" "github.com/openmcp-project/mcp-operator/internal/utils/apiserver" + mcpocfg "github.com/openmcp-project/mcp-operator/internal/config" apiservercontroller "github.com/openmcp-project/mcp-operator/internal/controller/core/apiserver" authenticationcontroller "github.com/openmcp-project/mcp-operator/internal/controller/core/authentication" authorizationcontroller "github.com/openmcp-project/mcp-operator/internal/controller/core/authorization" @@ -17,12 +18,16 @@ import ( landscapercontroller "github.com/openmcp-project/mcp-operator/internal/controller/core/landscaper" mcpcontroller "github.com/openmcp-project/mcp-operator/internal/controller/core/managedcontrolplane" + admissionv1 "k8s.io/api/admissionregistration/v1" "sigs.k8s.io/controller-runtime/pkg/cluster" + v2install "github.com/openmcp-project/openmcp-operator/api/install" + laasinstall "github.com/gardener/landscaper-service/pkg/apis/core/install" cocorev1beta1 "github.com/openmcp-project/control-plane-operator/api/v1beta1" "github.com/openmcp-project/controller-utils/pkg/init/webhooks" "github.com/openmcp-project/controller-utils/pkg/logging" + "github.com/openmcp-project/controller-utils/pkg/resources" "github.com/spf13/cobra" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -45,6 +50,8 @@ import ( openmcpinstall "github.com/openmcp-project/mcp-operator/api/install" ) +const OperatorName = "ManagedControlPlaneOperator" + func NewMCPOperatorCommand(ctx context.Context) *cobra.Command { options := NewOptions() @@ -143,6 +150,125 @@ func (o *Options) runInit(ctx context.Context) error { } } + // manage architecture immutability + labelSelector := client.MatchingLabels{ + openmcpv1alpha1.ManagedByLabel: OperatorName, + openmcpv1alpha1.ManagedPurposeLabel: openmcpv1alpha1.ManagedPurposeArchitectureImmutability, + } + evapbs := &admissionv1.ValidatingAdmissionPolicyBindingList{} + if err := crateClient.List(ctx, evapbs, labelSelector); err != nil { + return fmt.Errorf("error listing ValidatingAdmissionPolicyBindings: %w", err) + } + for _, evapb := range evapbs.Items { + if mcpocfg.Config.Architecture.Immutability.Disabled || evapb.Name != mcpocfg.Config.Architecture.Immutability.PolicyName { + setupLog.Info("Deleting existing ValidatingAdmissionPolicyBinding with architecture immutability purpose", "name", evapb.Name) + if err := crateClient.Delete(ctx, &evapb); client.IgnoreNotFound(err) != nil { + return fmt.Errorf("error deleting ValidatingAdmissionPolicyBinding '%s': %w", evapb.Name, err) + } + } + } + evaps := &admissionv1.ValidatingAdmissionPolicyList{} + if err := crateClient.List(ctx, evaps, labelSelector); err != nil { + return fmt.Errorf("error listing ValidatingAdmissionPolicies: %w", err) + } + for _, evap := range evaps.Items { + if mcpocfg.Config.Architecture.Immutability.Disabled || evap.Name != mcpocfg.Config.Architecture.Immutability.PolicyName { + setupLog.Info("Deleting existing ValidatingAdmissionPolicy with architecture immutability purpose", "name", evap.Name) + if err := crateClient.Delete(ctx, &evap); client.IgnoreNotFound(err) != nil { + return fmt.Errorf("error deleting ValidatingAdmissionPolicy '%s': %w", evap.Name, err) + } + } + } + if !mcpocfg.Config.Architecture.Immutability.Disabled { + setupLog.Info("Architecture immutability validation enabled, creating/updating ValidatingAdmissionPolicies ...") + vapm := resources.NewValidatingAdmissionPolicyMutator(mcpocfg.Config.Architecture.Immutability.PolicyName, admissionv1.ValidatingAdmissionPolicySpec{ + FailurePolicy: ptr.To(admissionv1.Fail), + MatchConstraints: &admissionv1.MatchResources{ + ResourceRules: []admissionv1.NamedRuleWithOperations{ + { + RuleWithOperations: admissionv1.RuleWithOperations{ + Operations: []admissionv1.OperationType{ + admissionv1.Create, + admissionv1.Update, + }, + Rule: admissionv1.Rule{ // match all resources, actual restriction happens in the binding + APIGroups: []string{"*"}, + APIVersions: []string{"*"}, + Resources: []string{"*"}, + }, + }, + }, + }, + }, + Variables: []admissionv1.Variable{ + { + Name: "archLabel", + Expression: fmt.Sprintf(`(has(object.metadata.labels) && "%s" in object.metadata.labels) ? object.metadata.labels["%s"] : ""`, openmcpv1alpha1.ArchitectureVersionLabel, openmcpv1alpha1.ArchitectureVersionLabel), + }, + { + Name: "oldArchLabel", + Expression: fmt.Sprintf(`(oldObject != null && has(oldObject.metadata.labels) && "%s" in oldObject.metadata.labels) ? oldObject.metadata.labels["%s"] : ""`, openmcpv1alpha1.ArchitectureVersionLabel, openmcpv1alpha1.ArchitectureVersionLabel), + }, + }, + Validations: []admissionv1.Validation{ + { + Expression: fmt.Sprintf(`variables.archLabel == "%s" || variables.archLabel == "%s"`, openmcpv1alpha1.ArchitectureV1, openmcpv1alpha1.ArchitectureV2), + Message: fmt.Sprintf(`The label "%s" must be set and its value must be either "%s" or "%s".`, openmcpv1alpha1.ArchitectureVersionLabel, openmcpv1alpha1.ArchitectureV1, openmcpv1alpha1.ArchitectureV2), + }, + { + Expression: fmt.Sprintf(`request.operation == "CREATE" || (variables.oldArchLabel == "" && variables.archLabel == "%s") || (variables.oldArchLabel == variables.archLabel)`, openmcpv1alpha1.ArchitectureV1), + Message: fmt.Sprintf(`The label "%s" is immutable, it may not be changed or removed once set. Adding it to existing resources is only allowed with "%s" as value.`, openmcpv1alpha1.ArchitectureVersionLabel, openmcpv1alpha1.ArchitectureV1), + }, + }, + }) + vapm.MetadataMutator().WithLabels(map[string]string{ + openmcpv1alpha1.ManagedByLabel: OperatorName, + openmcpv1alpha1.ManagedPurposeLabel: openmcpv1alpha1.ManagedPurposeArchitectureImmutability, + }) + if err := resources.CreateOrUpdateResource(ctx, crateClient, vapm); err != nil { + return fmt.Errorf("error creating/updating ValidatingAdmissionPolicy for architecture immutability: %w", err) + } + + vapbm := resources.NewValidatingAdmissionPolicyBindingMutator(mcpocfg.Config.Architecture.Immutability.PolicyName, admissionv1.ValidatingAdmissionPolicyBindingSpec{ + PolicyName: mcpocfg.Config.Architecture.Immutability.PolicyName, + ValidationActions: []admissionv1.ValidationAction{ + admissionv1.Deny, + }, + MatchResources: &admissionv1.MatchResources{ + ResourceRules: []admissionv1.NamedRuleWithOperations{ + { + RuleWithOperations: admissionv1.RuleWithOperations{ + Operations: []admissionv1.OperationType{ + admissionv1.Create, + admissionv1.Update, + }, + Rule: admissionv1.Rule{ + APIGroups: []string{openmcpv1alpha1.GroupVersion.Group}, + APIVersions: []string{openmcpv1alpha1.GroupVersion.Version}, + Resources: []string{ + "apiservers", + "landscapers", + "cloudorchestrators", + "authentications", + "authorizations", + }, + }, + }, + }, + }, + }, + }) + vapbm.MetadataMutator().WithLabels(map[string]string{ + openmcpv1alpha1.ManagedByLabel: OperatorName, + openmcpv1alpha1.ManagedPurposeLabel: openmcpv1alpha1.ManagedPurposeArchitectureImmutability, + }) + if err := resources.CreateOrUpdateResource(ctx, crateClient, vapbm); err != nil { + return fmt.Errorf("error creating/updating ValidatingAdmissionPolicyBinding for architecture immutability: %w", err) + } + + setupLog.Info("ValidatingAdmissionPolicy and ValidatingAdmissionPolicyBinding for architecture immutability created/updated") + } + return nil } @@ -219,7 +345,15 @@ func (o *Options) run(ctx context.Context) error { if o.ActiveControllers.Has(ControllerIDAPIServer) { // APIServer controller - apiServerProvider, err := apiservercontroller.NewAPIServerProvider(ctx, mgr.GetClient(), o.APIServerConfig) + // build platform cluster client for v2 path + v2scheme := v2install.InstallOperatorAPIs(runtime.NewScheme()) + platformClient, err := client.New(o.LaaSClusterConfig, client.Options{ + Scheme: v2scheme, + }) + if err != nil { + return fmt.Errorf("error creating platform cluster client: %w", err) + } + apiServerProvider, err := apiservercontroller.NewAPIServerProvider(ctx, mgr.GetClient(), platformClient, o.APIServerConfig) if err != nil { return fmt.Errorf("error creating %s: %w", apiservercontroller.ControllerName, err) } diff --git a/cmd/mcp-operator/app/options.go b/cmd/mcp-operator/app/options.go index e430dff..148906f 100644 --- a/cmd/mcp-operator/app/options.go +++ b/cmd/mcp-operator/app/options.go @@ -10,6 +10,7 @@ import ( "github.com/openmcp-project/mcp-operator/internal/components" + mcpocfg "github.com/openmcp-project/mcp-operator/internal/config" "github.com/openmcp-project/mcp-operator/internal/controller/core/apiserver/config" configauthn "github.com/openmcp-project/mcp-operator/internal/controller/core/authentication/config" configauthz "github.com/openmcp-project/mcp-operator/internal/controller/core/authorization/config" @@ -56,6 +57,7 @@ type rawOptions struct { // raw options that need to be evaluated APIServerConfigPath string `json:"apiServerConfigPath"` + MCPOConfigPath string `json:"configPath"` LaaSClusterPath string `json:"laasClusterConfigPath"` CrateClusterPath string `json:"crateClusterConfigPath"` CloudOrchestratorClusterPath string `json:"cloudOrchestratorClusterConfigPath"` @@ -131,6 +133,9 @@ func (o *Options) String(includeHeader bool, includeRawOptions bool) (string, er // API server config opts["apiServerConfig"] = o.APIServerConfig + // architecture config + opts["config"] = mcpocfg.Config + // clusters opts["crateClusterHost"] = nil if o.CrateClusterConfig != nil { @@ -219,6 +224,7 @@ func (o *Options) AddFlags(fs *flag.FlagSet) { fs.StringVar(&o.AuthzConfigPath, "authz-config", "", "Path to the authorization config file.") // common + fs.StringVar(&o.MCPOConfigPath, "config", "", "Path to the MCP operator config file.") fs.BoolVar(&o.DryRun, "dry-run", false, "If true, the CLI args are evaluated as usual, but the program exits before the controllers are started.") fs.StringVar(&o.CrateClusterPath, "crate-cluster", "", "Path to the crate cluster kubeconfig file or directory containing either a kubeconfig or host, token, and ca file. Leave empty to use in-cluster config.") fs.StringVar(&o.ControllerList, "controllers", strings.Join([]string{ControllerIDManagedControlPlane, ControllerIDAPIServer, ControllerIDLandscaper, ControllerIDCloudOrchestrator}, ","), "Comma-separated list of controllers that should be active.") @@ -337,6 +343,19 @@ func (o *Options) Complete() error { } } + // load config + if o.MCPOConfigPath != "" { + cfg, err := mcpocfg.LoadConfig(o.MCPOConfigPath) + if err != nil { + return err + } + err = cfg.Validate().ToAggregate() + if err != nil { + return fmt.Errorf("invalid config: %w", err) + } + mcpocfg.Config = *cfg + } + // print options optsString, err := o.String(true, false) if err != nil { diff --git a/docs/README.md b/docs/README.md index f3d8e5a..d0223ac 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,14 @@ # Documentation Index +## v2 Architecture + +- [v2 Architecture Bridge](architecture-v2/bridge.md) + +## Configuration + +- [Configuration](config/config.md) + ## Controllers - [APIServer Controller](controllers/apiserver.md) diff --git a/docs/architecture-v2/.docnames b/docs/architecture-v2/.docnames new file mode 100644 index 0000000..5a7e71c --- /dev/null +++ b/docs/architecture-v2/.docnames @@ -0,0 +1,3 @@ +{ + "header": "v2 Architecture" +} \ No newline at end of file diff --git a/docs/architecture-v2/bridge.md b/docs/architecture-v2/bridge.md new file mode 100644 index 0000000..bb94fd3 --- /dev/null +++ b/docs/architecture-v2/bridge.md @@ -0,0 +1,47 @@ +# v2 Architecture Bridge + +In order to migrate an existing MCP landscape to the new v2 architecture step by step, additional logic was added to the MCP operator, which allows it to switch between the old and the new logic for each component. The 'new logic' is usually just some bridge logic that transforms the v1 api type into the v2 api type and transforms the v2 api type's status back into the v1 format. + +The bridge is currently implemented for the following components: +- `APIServer` + +## Architecture Configuration + +To configure for which components the bridge is enabled, set the architecture config in the [general configuration](../config/config.md) file: +```yaml +immutability: + policyName: mcp-architecture-immutability + disabled: false +apiServer: + version: v1 + allowOverride: false +# more components are to follow +``` + +The component configuration should look similar, if not identical, for each component: +- `version` describes the architecture version that is used for this component by default. + - Valid values are `v1` (meaning old logic) and `v2` (using the v2 bridge). + - Defaults to `v1` if not specified for a component. +- `allowOverride` specifies whether the version specified in `version` should be able to be overridden by a corresponding label on the `ManagedControlPlane` resource. + - If this is `true` for a specific component and an MCP resource has a label `.architecture.openmcp.cloud/version`, the label's value will be used instead of the version configured in the architecture configuration. + - For example, the label for the `APIServer` component is named `apiserver.architecture.openmcp.cloud/version`. + - If `allowOverride` is `false`, setting such a label on the MCP resource causes an error during reconciliation. + - If the label's value is not a valid version, an error will occur during reconciliation. + - Defaults to `false` if not specified for a component. + +## Architecture Version Labels and Immutability + +The architecture that is used for a specific component of a specific MCP must not be changed after it has been initially decided. The reason for this is simple: If the version was changed from `v1` to `v2` after the component has already been deployed, the `v2` bridge logic would not detect the resources that were already deployed by the `v1` logic and re-deploy it 'the v2 way', leading to duplicated resources and potential conflicts. The same is true vice-versa. + +To ensure that the architecture version does not change, we use a combination of labels and [ValidatingAdmissionPolicies](https://kubernetes.io/docs/reference/access-authn-authz/validating-admission-policy/): +- The 'ground truth' of which version is being used is stored in a label on each component resource. + - This is used by the components' controllers to decide which logic they use for reconciliation. + - The label's key is `architecture.openmcp.cloud/version`. + - As a kind of migration, component resources that don't have the label are treated as having it set to `v1`. +- The value of the label is never allowed to change. + - If the label is missing, it is allowed to be added with `v1` as value. +- Newly created or updated component resources must have the label set. + +If `immutability.disabled` in the architecture configuration is not set to `true` (it is `false` by default), the MCP operator will deploy a `ValidatingAdmissionPolicy` and `ValidatingAdmissionPolicyBinding` on startup to ensure the architecture version immutability. `immutability.policyName` specifies the name for both resources and defaults to `mcp-architecture-immutability` if not specified. + +Both resources are removed during startup if `immutability.disabled` is set to `true`. diff --git a/docs/config/.docnames b/docs/config/.docnames new file mode 100644 index 0000000..2c8f8e5 --- /dev/null +++ b/docs/config/.docnames @@ -0,0 +1,3 @@ +{ + "header": "Configuration" +} \ No newline at end of file diff --git a/docs/config/config.md b/docs/config/config.md new file mode 100644 index 0000000..8b3ae86 --- /dev/null +++ b/docs/config/config.md @@ -0,0 +1,14 @@ +# Configuration + +The MCP Operator takes a - currently optional - configuration file via the `--config` argument. + +```yaml +architecture: # the architecture configuration + apiServer: + version: v2 + allowOverride: true +``` + +The following fields can be specified: +- `architecture` _(optional)_ + - The architecture configuration has its own [documentation](../architecture-v2/bridge.md), see there for further details. diff --git a/go.mod b/go.mod index f730510..6c5c93d 100644 --- a/go.mod +++ b/go.mod @@ -13,9 +13,12 @@ require ( github.com/gardener/landscaper-service v0.132.0 github.com/onsi/ginkgo/v2 v2.23.4 github.com/onsi/gomega v1.38.0 + github.com/openmcp-project/cluster-provider-gardener/api v0.2.0 github.com/openmcp-project/control-plane-operator v0.1.10 github.com/openmcp-project/controller-utils v0.13.1 github.com/openmcp-project/mcp-operator/api v0.32.0 + github.com/openmcp-project/openmcp-operator/api v0.7.0 + github.com/openmcp-project/openmcp-operator/lib v0.8.3 github.com/spf13/cobra v1.9.1 github.com/spf13/pflag v1.0.7 github.com/stretchr/testify v1.10.0 diff --git a/go.sum b/go.sum index 959ccbe..e59440e 100644 --- a/go.sum +++ b/go.sum @@ -109,10 +109,16 @@ github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8= github.com/onsi/gomega v1.38.0 h1:c/WX+w8SLAinvuKKQFh77WEucCnPk4j2OTUr7lt7BeY= github.com/onsi/gomega v1.38.0/go.mod h1:OcXcwId0b9QsE7Y49u+BTrL4IdKOBOKnD6VQNTJEB6o= +github.com/openmcp-project/cluster-provider-gardener/api v0.2.0 h1:afEPFJe5tSnGBxUPkxEBM3gZeL7AgtpwNhsSM8Qh5Xs= +github.com/openmcp-project/cluster-provider-gardener/api v0.2.0/go.mod h1:k6YmDIb2Gl13teO9BnfN9a7MVG/4/oJ+FDwK07i8dA8= github.com/openmcp-project/control-plane-operator v0.1.10 h1:5ticEP3llBmIHQkBzkZwgTdcHJ7Z9rSaw5BsY3QQ2pM= github.com/openmcp-project/control-plane-operator v0.1.10/go.mod h1:GNu9LBTPWoE3dKsBo2kS+SeKSLU2qLtu3VjpaznsB2o= github.com/openmcp-project/controller-utils v0.13.1 h1:+06c0bs1BIO+hBsTcuiEK5y8vpDFoZPml59WNm8fagM= github.com/openmcp-project/controller-utils v0.13.1/go.mod h1:Z1ytVshYcgJq3VQVGqkuZsjO/BCr4UYAaVpHl6JSIMI= +github.com/openmcp-project/openmcp-operator/api v0.7.0 h1:DvaMS3xtAvahGOQm9sI26aotupa8XkwZP52HfOhZ9K0= +github.com/openmcp-project/openmcp-operator/api v0.7.0/go.mod h1:TuAq8Fbrzuykxw/h589M8+QfHotwero5MPWVzdFAqkw= +github.com/openmcp-project/openmcp-operator/lib v0.8.3 h1:2bb1zbP6Si7/fUfXT9M5B0xQnd7O4zGJXaUoe9pDmcA= +github.com/openmcp-project/openmcp-operator/lib v0.8.3/go.mod h1:oydIXRZoNDxtI4DI/JBUB08UPzvfdaKLqHvC4S4HXHQ= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/internal/config/architecture/config.go b/internal/config/architecture/config.go new file mode 100644 index 0000000..09eda47 --- /dev/null +++ b/internal/config/architecture/config.go @@ -0,0 +1,179 @@ +//nolint:revive +package architecture + +import ( + "fmt" + "maps" + "reflect" + "strings" + + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/validation/field" + + openmcpv1alpha1 "github.com/openmcp-project/mcp-operator/api/core/v1alpha1" + "github.com/openmcp-project/mcp-operator/internal/components" +) + +var AllowedVersions = sets.New(openmcpv1alpha1.ArchitectureV1, openmcpv1alpha1.ArchitectureV2) + +//////////////// +// ArchConfig // +//////////////// + +type ArchConfig struct { + // Immutability contains the configuration for the immutability check. + Immutability ImmutabilityConfig `json:"immutability"` + // APIServer contains the configuration for the APIServer component v1-v2 bridge. + APIServer BridgeConfig `json:"apiServer"` + // Landscaper contains the configuration for the Landscaper component v1-v2 bridge. + Landscaper BridgeConfig `json:"landscaper"` +} + +func (cfg *ArchConfig) Validate() field.ErrorList { + if cfg == nil { + return nil + } + + allErrs := field.ErrorList{} + + allErrs = append(allErrs, cfg.Immutability.Validate()...) + + cbcs := cfg.componentBridgeConfigs() + for ct, bridgeConfig := range cbcs { + allErrs = append(allErrs, bridgeConfig.Validate(field.NewPath(string(ct)))...) + } + + return allErrs +} + +func (cfg *ArchConfig) Default() { + if cfg == nil { + return + } + + cfg.Immutability.Default() + + cbcs := cfg.componentBridgeConfigs() + for _, bridgeConfig := range cbcs { + bridgeConfig.Default() + } +} + +// GetBridgeConfigForComponent returns the bridge configuration for the given component type. +// If the component type is not recognized, it returns a default BridgeConfig. +func (cfg *ArchConfig) GetBridgeConfigForComponent(compType openmcpv1alpha1.ComponentType) BridgeConfig { + if cfg != nil { + cbcs := cfg.componentBridgeConfigs() + if bridgeConfig, ok := cbcs[compType]; ok { + return *bridgeConfig + } + } + res := BridgeConfig{} + res.Default() + return res +} + +// componentBridgeConfigs returns a mapping from component types to their respective bridge configurations. +// Note that pointers to the BridgeConfigs are returned, but you should only modify them if you know what you're doing. +func (cfg *ArchConfig) componentBridgeConfigs() map[openmcpv1alpha1.ComponentType]*BridgeConfig { + res := make(map[openmcpv1alpha1.ComponentType]*BridgeConfig) + + if cfg == nil { + return res + } + + v := reflect.ValueOf(cfg).Elem() + for i := range v.NumField() { + field := v.Type().Field(i) + compIter := maps.Keys(components.Registry.GetKnownComponents()) + for ct := range compIter { + if field.Name == string(ct) { + if bridgeConfig, ok := v.Field(i).Addr().Interface().(*BridgeConfig); ok { + res[ct] = bridgeConfig + } + break + } + } + } + + return res +} + +////////////////// +// BridgeConfig // +////////////////// + +type BridgeConfig struct { + // Version specifies the default version of the architecture to use. + Version string `json:"version"` + // AllowOverride specifies if the used version can be overridden by setting the appropriate label on the resource. + AllowOverride bool `json:"allowOverride"` +} + +// IsAllowedVersion returns whether the given version is allowed in the architecture configuration. +func (cfg BridgeConfig) IsAllowedVersion(version string) bool { + return AllowedVersions.Has(version) +} + +func (cfg *BridgeConfig) Default() { + if cfg == nil { + return + } + + if !cfg.IsAllowedVersion(cfg.Version) { + cfg.Version = openmcpv1alpha1.ArchitectureV1 // default to v1 if not set or invalid + } +} + +func (cfg *BridgeConfig) Validate(fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + if cfg == nil { + return allErrs + } + + if !AllowedVersions.Has(cfg.Version) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("version"), cfg.Version, fmt.Sprintf("version must be one of [%s]", strings.Join(sets.List(AllowedVersions), ", ")))) + } + + return allErrs +} + +//////////////////////// +// ImmutabilityConfig // +//////////////////////// + +type ImmutabilityConfig struct { + // PolicyName is the name of the ValidatingAdmissionPolicy and the corresponding ValidatingAdmissionPolicyBinding + // that is created to prevent switching between architecture versions. + // Defaults to 'mcp-architecture-immutability' if not specified. + PolicyName string `json:"policyName"` + // Disabled disables the immutability check if set to 'true'. + // This will cause the MCP operator to delete any previously (for the purpose of preventing architecture version changes) + // created ValidatingAdmissionPolicy and ValidatingAdmissionPolicyBinding. + Disabled bool `json:"disabled"` +} + +func (cfg *ImmutabilityConfig) Default() { + if cfg == nil { + return + } + + if cfg.PolicyName == "" { + cfg.PolicyName = "mcp-architecture-immutability" + } +} + +func (cfg *ImmutabilityConfig) Validate() field.ErrorList { + if cfg == nil { + return nil + } + + allErrs := field.ErrorList{} + + if cfg.PolicyName == "" { + allErrs = append(allErrs, field.Required(field.NewPath("policyName"), "policy name must not be empty")) + } + + return allErrs +} diff --git a/internal/config/architecture/version.go b/internal/config/architecture/version.go new file mode 100644 index 0000000..b147ad4 --- /dev/null +++ b/internal/config/architecture/version.go @@ -0,0 +1,26 @@ +package architecture + +import ( + openmcpv1alpha1 "github.com/openmcp-project/mcp-operator/api/core/v1alpha1" + "github.com/openmcp-project/mcp-operator/internal/components" +) + +// DecideVersion determines the architecture version for a given component. +// This basically just checks the component's labels for the architecture version label. +// If the label is missing, the configured default version for the component type is returned. +// If the component is nil, 'v1' is returned. +func (cfg *ArchConfig) DecideVersion(comp components.Component) string { + if comp == nil { + return openmcpv1alpha1.ArchitectureV1 + } + + bridgeConfig := cfg.GetBridgeConfigForComponent(comp.Type()) + version := bridgeConfig.Version + + labelVersion, ok := comp.GetLabels()[openmcpv1alpha1.ArchitectureVersionLabel] + if ok && bridgeConfig.IsAllowedVersion(labelVersion) { + version = labelVersion + } + + return version +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..5321a45 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,60 @@ +package config + +import ( + "fmt" + "os" + + "k8s.io/apimachinery/pkg/util/validation/field" + "sigs.k8s.io/yaml" + + "github.com/openmcp-project/mcp-operator/internal/config/architecture" +) + +var Config MCPOperatorConfig + +func init() { + Config.Default() +} + +type MCPOperatorConfig struct { + // Architecture contains the configuration regarding v1 and v2 architecture. + Architecture architecture.ArchConfig `json:"architecture"` +} + +func LoadConfig(path string) (*MCPOperatorConfig, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("error reading config file: %w", err) + } + return LoadConfigFromBytes(data) +} + +func LoadConfigFromBytes(data []byte) (*MCPOperatorConfig, error) { + cfg := &MCPOperatorConfig{} + err := yaml.Unmarshal(data, cfg) + if err != nil { + return nil, fmt.Errorf("error parsing config file: %w", err) + } + cfg.Default() + return cfg, nil +} + +func (cfg *MCPOperatorConfig) Default() { + if cfg == nil { + return + } + + cfg.Architecture.Default() +} + +func (cfg *MCPOperatorConfig) Validate() field.ErrorList { + if cfg == nil { + return nil + } + + allErrs := field.ErrorList{} + + allErrs = append(allErrs, cfg.Architecture.Validate()...) + + return allErrs +} diff --git a/internal/controller/core/apiserver/controller.go b/internal/controller/core/apiserver/controller.go index c1ed062..725dc7e 100644 --- a/internal/controller/core/apiserver/controller.go +++ b/internal/controller/core/apiserver/controller.go @@ -10,6 +10,7 @@ import ( "github.com/openmcp-project/mcp-operator/internal/utils" componentutils "github.com/openmcp-project/mcp-operator/internal/utils/components" + mcpocfg "github.com/openmcp-project/mcp-operator/internal/config" apiserverconfig "github.com/openmcp-project/mcp-operator/internal/controller/core/apiserver/config" apiserverhandler "github.com/openmcp-project/mcp-operator/internal/controller/core/apiserver/handler" "github.com/openmcp-project/mcp-operator/internal/controller/core/apiserver/handler/gardener" @@ -42,7 +43,7 @@ func (r *APIServerProvider) GetAPIServerHandlerForType(ctx context.Context, t op return nil, fmt.Errorf("unknown API server type '%s'", string(t)) } -func NewAPIServerProvider(ctx context.Context, client client.Client, cfg *apiserverconfig.APIServerProviderConfiguration) (*APIServerProvider, error) { +func NewAPIServerProvider(ctx context.Context, crateClient, platformClient client.Client, cfg *apiserverconfig.APIServerProviderConfiguration) (*APIServerProvider, error) { log, ctx := utils.InitializeControllerLogger(ctx, ControllerName) ccfg, err := cfg.Complete(ctx) if err != nil { @@ -55,7 +56,8 @@ func NewAPIServerProvider(ctx context.Context, client client.Client, cfg *apiser return &APIServerProvider{ CompletedAPIServerProviderConfiguration: *ccfg, - Client: client, + CrateClient: crateClient, + PlatformClient: platformClient, }, nil } @@ -63,8 +65,10 @@ func NewAPIServerProvider(ctx context.Context, client client.Client, cfg *apiser type APIServerProvider struct { apiserverconfig.CompletedAPIServerProviderConfiguration - // Client is the registration cluster client. - Client client.Client + // CrateClient is the registration cluster client. + CrateClient client.Client + // PlatformClient is the platform cluster client. + PlatformClient client.Client // FakeHandler is a fake APIServerHandler for testing purposes. // It should only be non-nil in tests. @@ -84,7 +88,7 @@ func (r *APIServerProvider) Reconcile(ctx context.Context, req ctrl.Request) (ct if rr.Component == nil { return rr.Result, rr.ReconcileError } - return componentutils.UpdateStatus(ctx, r.Client, rr) + return componentutils.UpdateStatus(ctx, r.CrateClient, rr) } func (r *APIServerProvider) reconcile(ctx context.Context, req ctrl.Request) componentutils.ReconcileResult[*openmcpv1alpha1.APIServer] { @@ -92,7 +96,7 @@ func (r *APIServerProvider) reconcile(ctx context.Context, req ctrl.Request) com // get internal APIServer resource as := &openmcpv1alpha1.APIServer{} - if err := r.Client.Get(ctx, req.NamespacedName, as); err != nil { + if err := r.CrateClient.Get(ctx, req.NamespacedName, as); err != nil { if apierrors.IsNotFound(err) { log.Debug("Resource not found") return componentutils.ReconcileResult[*openmcpv1alpha1.APIServer]{} @@ -110,7 +114,7 @@ func (r *APIServerProvider) reconcile(ctx context.Context, req ctrl.Request) com return componentutils.ReconcileResult[*openmcpv1alpha1.APIServer]{} case openmcpv1alpha1.OperationAnnotationValueReconcile: log.Debug("Removing reconcile operation annotation from resource") - if err := componentutils.PatchAnnotation(ctx, r.Client, as, openmcpv1alpha1.OperationAnnotation, "", componentutils.ANNOTATION_DELETE); err != nil { + if err := componentutils.PatchAnnotation(ctx, r.CrateClient, as, openmcpv1alpha1.OperationAnnotation, "", componentutils.ANNOTATION_DELETE); err != nil { return componentutils.ReconcileResult[*openmcpv1alpha1.APIServer]{ReconcileError: openmcperrors.WithReason(fmt.Errorf("error removing operation annotation: %w", err), cconst.ReasonCrateClusterInteractionProblem)} } } @@ -133,28 +137,41 @@ func (r *APIServerProvider) reconcile(ctx context.Context, req ctrl.Request) com old := as.DeepCopy() if controllerutil.AddFinalizer(as, openmcpv1alpha1.APIServerComponent.Finalizer()) { - if err := r.Client.Patch(ctx, as, client.MergeFrom(old)); err != nil { + if err := r.CrateClient.Patch(ctx, as, client.MergeFrom(old)); err != nil { return componentutils.ReconcileResult[*openmcpv1alpha1.APIServer]{Component: as, ReconcileError: openmcperrors.WithReason(fmt.Errorf("error patching finalizer on APIServer: %w", err), cconst.ReasonCrateClusterInteractionProblem)} } } } - apiServerHandler, err := r.GetAPIServerHandlerForType(ctx, as.Spec.Type, r.CompletedAPIServerProviderConfiguration) - if err != nil { - return componentutils.ReconcileResult[*openmcpv1alpha1.APIServer]{Component: as, ReconcileError: openmcperrors.WithReason(fmt.Errorf("error getting APIServer handler: %w", err), cconst.ReasonConfigurationProblem)} - } - ctx = logging.NewContext(ctx, log.WithValues("apiServerType", string(as.Spec.Type))) - old := as.DeepCopy() var res ctrl.Result var usf apiserverhandler.UpdateStatusFunc var cons []openmcpv1alpha1.ComponentCondition var errr openmcperrors.ReasonableError - if !deleteAPIServer { - res, usf, cons, errr = apiServerHandler.HandleCreateOrUpdate(ctx, as, r.Client) + + if mcpocfg.Config.Architecture.DecideVersion(as) == openmcpv1alpha1.ArchitectureV2 { + // v2 logic + log.Info("Using v2 logic for APIServer") + if !deleteAPIServer { + res, usf, cons, errr = v2HandleCreateOrUpdate(ctx, as, r.PlatformClient) + } else { + res, usf, cons, errr = v2HandleDelete(ctx, as, r.PlatformClient) + } } else { - res, usf, cons, errr = apiServerHandler.HandleDelete(ctx, as, r.Client) + // v1 logic + apiServerHandler, err := r.GetAPIServerHandlerForType(ctx, as.Spec.Type, r.CompletedAPIServerProviderConfiguration) + if err != nil { + return componentutils.ReconcileResult[*openmcpv1alpha1.APIServer]{Component: as, ReconcileError: openmcperrors.WithReason(fmt.Errorf("error getting APIServer handler: %w", err), cconst.ReasonConfigurationProblem)} + } + ctx = logging.NewContext(ctx, log.WithValues("apiServerType", string(as.Spec.Type))) + + if !deleteAPIServer { + res, usf, cons, errr = apiServerHandler.HandleCreateOrUpdate(ctx, as, r.CrateClient) + } else { + res, usf, cons, errr = apiServerHandler.HandleDelete(ctx, as, r.CrateClient) + } } + errs := openmcperrors.NewReasonableErrorList(errr) if usf != nil { @@ -170,7 +187,7 @@ func (r *APIServerProvider) reconcile(ctx context.Context, req ctrl.Request) com old := as.DeepCopy() changed := controllerutil.RemoveFinalizer(as, openmcpv1alpha1.APIServerComponent.Finalizer()) if changed { - if err := r.Client.Patch(ctx, as, client.MergeFrom(old)); err != nil { + if err := r.CrateClient.Patch(ctx, as, client.MergeFrom(old)); err != nil { errs.Append(fmt.Errorf("error removing finalizer from APIServer: %w", err)) } } diff --git a/internal/controller/core/apiserver/controller_test.go b/internal/controller/core/apiserver/controller_test.go index 5201b51..5d0bd09 100644 --- a/internal/controller/core/apiserver/controller_test.go +++ b/internal/controller/core/apiserver/controller_test.go @@ -2,23 +2,34 @@ package apiserver_test import ( "context" + "encoding/json" "fmt" + "strconv" + "time" + mcpocfg "github.com/openmcp-project/mcp-operator/internal/config" "github.com/openmcp-project/mcp-operator/internal/controller/core/apiserver" apiserverhandler "github.com/openmcp-project/mcp-operator/internal/controller/core/apiserver/handler" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" . "github.com/openmcp-project/mcp-operator/test/matchers" "github.com/openmcp-project/controller-utils/pkg/testing" + clustersv1alpha1 "github.com/openmcp-project/openmcp-operator/api/clusters/v1alpha1" + openmcpclusterutils "github.com/openmcp-project/openmcp-operator/lib/utils" + + gardenv1beta1 "github.com/openmcp-project/mcp-operator/api/external/gardener/pkg/apis/core/v1beta1" cconst "github.com/openmcp-project/mcp-operator/api/constants" openmcpv1alpha1 "github.com/openmcp-project/mcp-operator/api/core/v1alpha1" @@ -27,7 +38,7 @@ import ( ) func getReconciler(c ...client.Client) reconcile.Reconciler { - r, err := apiserver.NewAPIServerProvider(constructorContext, c[0], defaultConfig) + r, err := apiserver.NewAPIServerProvider(constructorContext, c[0], c[1], defaultConfig) Expect(err).NotTo(HaveOccurred()) r.FakeHandler = fakeHandler return r @@ -39,7 +50,12 @@ const ( ) func testEnvSetup(testDirPathSegments ...string) *testing.ComplexEnvironment { - return testutils.DefaultTestSetupBuilder(testDirPathSegments...).WithFakeClient(testutils.APIServerCluster, testutils.Scheme).WithReconcilerConstructor(apiServerReconciler, getReconciler, testutils.CrateCluster).Build() + return testutils.DefaultTestSetupBuilder(testDirPathSegments...). + WithFakeClient(testutils.APIServerCluster, testutils.Scheme). + WithFakeClient(testutils.LaaSCoreCluster, testutils.Scheme). + WithDynamicObjectsWithStatus(testutils.LaaSCoreCluster, &clustersv1alpha1.AccessRequest{}, &clustersv1alpha1.ClusterRequest{}, &clustersv1alpha1.Cluster{}). + WithReconcilerConstructor(apiServerReconciler, getReconciler, testutils.CrateCluster, testutils.LaaSCoreCluster). + Build() } func mockReadyConditions(ready bool) []openmcpv1alpha1.ComponentCondition { @@ -53,6 +69,7 @@ func mockReadyConditions(ready bool) []openmcpv1alpha1.ComponentCondition { } var _ = Describe("CO-1153 APIServer Controller", func() { + It("should do nothing if the reconciled resource is not found", func() { env := testEnvSetup() @@ -314,4 +331,236 @@ var _ = Describe("CO-1153 APIServer Controller", func() { Expect(as.Status.ExternalAPIServerStatus.ServiceAccountIssuer).To(Equal("https://k8s-sa.ondemand.com")) }) + Context("v2", func() { + + BeforeEach(func() { + mcpocfg.Config.Architecture.APIServer.Version = openmcpv1alpha1.ArchitectureV2 + }) + + It("should create a ClusterRequest and AccessRequest instead of a Shoot", func() { + env := testEnvSetup("testdata", "test-07") + + as := &openmcpv1alpha1.APIServer{} + err := env.Client(testutils.CrateCluster).Get(env.Ctx, types.NamespacedName{Name: "test", Namespace: "test"}, as) + Expect(err).NotTo(HaveOccurred()) + + req := testing.RequestFromObject(as) + rr := env.ShouldReconcile(apiServerReconciler, req) + Expect(rr.RequeueAfter).To(BeNumerically(">", 0)) + + cr := &clustersv1alpha1.ClusterRequest{} + cr.Name = as.Name + cr.Namespace = openmcpclusterutils.StableRequestNamespace(as.Namespace) + Expect(env.Client(testutils.LaaSCoreCluster).Get(env.Ctx, client.ObjectKeyFromObject(cr), cr)).To(Succeed()) + + Expect(env.Client(testutils.CrateCluster).Get(env.Ctx, client.ObjectKeyFromObject(as), as)).To(Succeed()) + Expect(as.Status.Conditions).To(ContainElements( + MatchComponentCondition(openmcpv1alpha1.ComponentCondition{ + Type: cconst.ConditionClusterRequestGranted, + Status: openmcpv1alpha1.ComponentConditionStatusFalse, + }), + MatchComponentCondition(openmcpv1alpha1.ComponentCondition{ + Type: cconst.ConditionClusterReady, + Status: openmcpv1alpha1.ComponentConditionStatusUnknown, + }), + MatchComponentCondition(openmcpv1alpha1.ComponentCondition{ + Type: cconst.ConditionAccessRequestGranted, + Status: openmcpv1alpha1.ComponentConditionStatusUnknown, + }), + )) + + // mock Cluster and ClusterRequest status + cluster := &clustersv1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-namespace", + }, + Spec: clustersv1alpha1.ClusterSpec{ + Purposes: []string{"mcp"}, + Tenancy: clustersv1alpha1.TENANCY_EXCLUSIVE, + }, + } + Expect(env.Client(testutils.LaaSCoreCluster).Create(env.Ctx, cluster)).To(Succeed()) + + cr.Status.Phase = clustersv1alpha1.REQUEST_GRANTED + cr.Status.Cluster = &clustersv1alpha1.NamespacedObjectReference{ + ObjectReference: clustersv1alpha1.ObjectReference{ + Name: cluster.Name, + }, + Namespace: cluster.Namespace, + } + Expect(env.Client(testutils.LaaSCoreCluster).Status().Update(env.Ctx, cr)).To(Succeed()) + + // reconcile again, should now get further + rr = env.ShouldReconcile(apiServerReconciler, req) + Expect(rr.RequeueAfter).To(BeNumerically(">", 0)) + + Expect(env.Client(testutils.CrateCluster).Get(env.Ctx, client.ObjectKeyFromObject(as), as)).To(Succeed()) + Expect(as.Status.Conditions).To(ContainElements( + MatchComponentCondition(openmcpv1alpha1.ComponentCondition{ + Type: cconst.ConditionClusterRequestGranted, + Status: openmcpv1alpha1.ComponentConditionStatusTrue, + }), + MatchComponentCondition(openmcpv1alpha1.ComponentCondition{ + Type: cconst.ConditionClusterReady, + Status: openmcpv1alpha1.ComponentConditionStatusFalse, + }), + MatchComponentCondition(openmcpv1alpha1.ComponentCondition{ + Type: cconst.ConditionAccessRequestGranted, + Status: openmcpv1alpha1.ComponentConditionStatusUnknown, + }), + )) + + // mock Cluster status + dummyShoot := &gardenv1beta1.Shoot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-shoot", + Namespace: "test-namespace", + }, + Spec: gardenv1beta1.ShootSpec{ + CloudProfileName: ptr.To("gcp"), + Purpose: ptr.To(gardenv1beta1.ShootPurposeProduction), + Region: "europe", + }, + } + dummyShootJson, err := json.Marshal(dummyShoot) + Expect(err).NotTo(HaveOccurred()) + cluster.Status = clustersv1alpha1.ClusterStatus{ + Phase: clustersv1alpha1.CLUSTER_PHASE_READY, + ProviderStatus: &runtime.RawExtension{ + Raw: dummyShootJson, + }, + } + Expect(env.Client(testutils.LaaSCoreCluster).Status().Update(env.Ctx, cluster)).To(Succeed()) + + // reconcile again, should now get further + rr = env.ShouldReconcile(apiServerReconciler, req) + Expect(rr.RequeueAfter).To(BeNumerically(">", 0)) + + ar := &clustersv1alpha1.AccessRequest{} + ar.Name = as.Name + ar.Namespace = cr.Namespace + Expect(env.Client(testutils.LaaSCoreCluster).Get(env.Ctx, client.ObjectKeyFromObject(ar), ar)).To(Succeed()) + + Expect(env.Client(testutils.CrateCluster).Get(env.Ctx, client.ObjectKeyFromObject(as), as)).To(Succeed()) + Expect(as.Status.Conditions).To(ContainElements( + MatchComponentCondition(openmcpv1alpha1.ComponentCondition{ + Type: cconst.ConditionClusterRequestGranted, + Status: openmcpv1alpha1.ComponentConditionStatusTrue, + }), + MatchComponentCondition(openmcpv1alpha1.ComponentCondition{ + Type: cconst.ConditionClusterReady, + Status: openmcpv1alpha1.ComponentConditionStatusTrue, + }), + MatchComponentCondition(openmcpv1alpha1.ComponentCondition{ + Type: cconst.ConditionAccessRequestGranted, + Status: openmcpv1alpha1.ComponentConditionStatusFalse, + }), + )) + + // mock AccessRequest status and secret + creationTime := time.Now() + expirationTime := time.Now().Add(24 * time.Hour) + access := &corev1.Secret{} + access.Name = "test-access-secret" + access.Namespace = ar.Namespace + access.Data = map[string][]byte{ + "kubeconfig": []byte("fake"), + "creationTimestamp": []byte(strconv.FormatInt(creationTime.Unix(), 10)), + "expirationTimestamp": []byte(strconv.FormatInt(expirationTime.Unix(), 10)), + } + Expect(env.Client(testutils.LaaSCoreCluster).Create(env.Ctx, access)).To(Succeed()) + + ar.Status.Phase = clustersv1alpha1.REQUEST_GRANTED + ar.Status.SecretRef = &clustersv1alpha1.NamespacedObjectReference{ + ObjectReference: clustersv1alpha1.ObjectReference{ + Name: access.Name, + }, + Namespace: access.Namespace, + } + Expect(env.Client(testutils.LaaSCoreCluster).Status().Update(env.Ctx, ar)).To(Succeed()) + + // reconcile again, should now get further + rr = env.ShouldReconcile(apiServerReconciler, req) + + Expect(env.Client(testutils.CrateCluster).Get(env.Ctx, client.ObjectKeyFromObject(as), as)).To(Succeed()) + Expect(as.Status.Conditions).To(ContainElements( + MatchComponentCondition(openmcpv1alpha1.ComponentCondition{ + Type: cconst.ConditionClusterRequestGranted, + Status: openmcpv1alpha1.ComponentConditionStatusTrue, + }), + MatchComponentCondition(openmcpv1alpha1.ComponentCondition{ + Type: cconst.ConditionClusterReady, + Status: openmcpv1alpha1.ComponentConditionStatusTrue, + }), + MatchComponentCondition(openmcpv1alpha1.ComponentCondition{ + Type: cconst.ConditionAccessRequestGranted, + Status: openmcpv1alpha1.ComponentConditionStatusTrue, + }), + )) + + Expect(as.Status.AdminAccess).ToNot(BeNil()) + Expect(as.Status.AdminAccess.CreationTimestamp.Time).To(BeTemporally("~", creationTime, 1*time.Second)) + Expect(as.Status.AdminAccess.ExpirationTimestamp.Time).To(BeTemporally("~", expirationTime, 1*time.Second)) + Expect(as.Status.AdminAccess.Kubeconfig).To(BeEquivalentTo("fake")) + + reconcileAt := creationTime.Add(time.Duration(float64(expirationTime.Sub(creationTime)) * 0.85)) + Expect(rr.RequeueAfter).To(BeNumerically("~", time.Until(reconcileAt), 1*time.Second)) + + // add dummy finalizers to the ClusterRequest and AccessRequest to verify the deletion flow + cr.Finalizers = append(cr.Finalizers, "dummy") + Expect(env.Client(testutils.LaaSCoreCluster).Update(env.Ctx, cr)).To(Succeed()) + ar.Finalizers = append(ar.Finalizers, "dummy") + Expect(env.Client(testutils.LaaSCoreCluster).Update(env.Ctx, ar)).To(Succeed()) + + // delete the APIServer, should not be deleted because of the finalizers + Expect(env.Client(testutils.CrateCluster).Delete(env.Ctx, as)).To(Succeed()) + rr = env.ShouldReconcile(apiServerReconciler, req) + Expect(rr.RequeueAfter).To(BeNumerically(">", 0)) + + Expect(env.Client(testutils.CrateCluster).Get(env.Ctx, client.ObjectKeyFromObject(as), as)).To(Succeed()) + Expect(as.Status.Conditions).To(ContainElements( + MatchComponentCondition(openmcpv1alpha1.ComponentCondition{ + Type: cconst.ConditionAccessRequestDeleted, + Status: openmcpv1alpha1.ComponentConditionStatusFalse, + }), + MatchComponentCondition(openmcpv1alpha1.ComponentCondition{ + Type: cconst.ConditionClusterRequestDeleted, + Status: openmcpv1alpha1.ComponentConditionStatusUnknown, + }), + )) + + // remove the AccessRequest finalizers and reconcile again + Expect(env.Client(testutils.LaaSCoreCluster).Get(env.Ctx, client.ObjectKeyFromObject(ar), ar)).To(Succeed()) + Expect(ar.DeletionTimestamp).ToNot(BeZero()) + ar.Finalizers = nil + Expect(env.Client(testutils.LaaSCoreCluster).Update(env.Ctx, ar)).To(Succeed()) + rr = env.ShouldReconcile(apiServerReconciler, req) + Expect(rr.RequeueAfter).To(BeNumerically(">", 0)) + + Expect(env.Client(testutils.CrateCluster).Get(env.Ctx, client.ObjectKeyFromObject(as), as)).To(Succeed()) + Expect(as.Status.Conditions).To(ContainElements( + MatchComponentCondition(openmcpv1alpha1.ComponentCondition{ + Type: cconst.ConditionAccessRequestDeleted, + Status: openmcpv1alpha1.ComponentConditionStatusTrue, + }), + MatchComponentCondition(openmcpv1alpha1.ComponentCondition{ + Type: cconst.ConditionClusterRequestDeleted, + Status: openmcpv1alpha1.ComponentConditionStatusFalse, + }), + )) + + // remove the ClusterRequest finalizers and reconcile again + Expect(env.Client(testutils.LaaSCoreCluster).Get(env.Ctx, client.ObjectKeyFromObject(cr), cr)).To(Succeed()) + Expect(cr.DeletionTimestamp).ToNot(BeZero()) + cr.Finalizers = nil + Expect(env.Client(testutils.LaaSCoreCluster).Update(env.Ctx, cr)).To(Succeed()) + rr = env.ShouldReconcile(apiServerReconciler, req) + Expect(rr.RequeueAfter).ToNot(BeNumerically(">", 0)) + + Expect(env.Client(testutils.CrateCluster).Get(env.Ctx, client.ObjectKeyFromObject(as), as)).To(MatchError(apierrors.IsNotFound, "IsNotFound")) + }) + + }) + }) diff --git a/internal/controller/core/apiserver/testdata/test-07/apiserver.yaml b/internal/controller/core/apiserver/testdata/test-07/apiserver.yaml new file mode 100644 index 0000000..ff6c5cb --- /dev/null +++ b/internal/controller/core/apiserver/testdata/test-07/apiserver.yaml @@ -0,0 +1,15 @@ +apiVersion: core.openmcp.cloud/v1alpha1 +kind: APIServer +metadata: + generation: 1 + labels: + openmcp.cloud/mcp-generation: "1" + openmcp.cloud/mcp-name: "test" + openmcp.cloud/mcp-namespace: "test" + name: test + namespace: test +spec: + desiredRegion: + direction: central + name: europe + type: Gardener diff --git a/internal/controller/core/apiserver/v2.go b/internal/controller/core/apiserver/v2.go new file mode 100644 index 0000000..828f706 --- /dev/null +++ b/internal/controller/core/apiserver/v2.go @@ -0,0 +1,501 @@ +package apiserver + +import ( + "context" + "fmt" + "strconv" + "time" + + "github.com/openmcp-project/controller-utils/pkg/logging" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/openmcp-project/controller-utils/pkg/clusteraccess" + "github.com/openmcp-project/controller-utils/pkg/resources" + + gcpv1alpha1 "github.com/openmcp-project/cluster-provider-gardener/api/core/v1alpha1" + clustersv1alpha1 "github.com/openmcp-project/openmcp-operator/api/clusters/v1alpha1" + clustersconst "github.com/openmcp-project/openmcp-operator/api/clusters/v1alpha1/constants" + openmcpclusterutils "github.com/openmcp-project/openmcp-operator/lib/utils" + + gardenv1beta1 "github.com/openmcp-project/mcp-operator/api/external/gardener/pkg/apis/core/v1beta1" + + cconst "github.com/openmcp-project/mcp-operator/api/constants" + openmcpv1alpha1 "github.com/openmcp-project/mcp-operator/api/core/v1alpha1" + openmcperrors "github.com/openmcp-project/mcp-operator/api/errors" + handler "github.com/openmcp-project/mcp-operator/internal/controller/core/apiserver/handler" + componentutils "github.com/openmcp-project/mcp-operator/internal/utils/components" +) + +func v2HandleCreateOrUpdate(ctx context.Context, as *openmcpv1alpha1.APIServer, platformClient client.Client) (ctrl.Result, handler.UpdateStatusFunc, []openmcpv1alpha1.ComponentCondition, openmcperrors.ReasonableError) { + log := logging.FromContextOrPanic(ctx).WithName(openmcpv1alpha1.ArchitectureV2) + ctx = logging.NewContext(ctx, log) + + clusterRequestGrantedCon := openmcpv1alpha1.ComponentCondition{ + Type: cconst.ConditionClusterRequestGranted, + Status: openmcpv1alpha1.ComponentConditionStatusUnknown, + } + clusterReadyCon := openmcpv1alpha1.ComponentCondition{ + Type: cconst.ConditionClusterReady, + Status: openmcpv1alpha1.ComponentConditionStatusUnknown, + } + accessRequestGrantedCon := openmcpv1alpha1.ComponentCondition{ + Type: cconst.ConditionAccessRequestGranted, + Status: openmcpv1alpha1.ComponentConditionStatusUnknown, + } + + // instead of calling a handler, create a ClusterRequest and an AccessRequest + // ensure namespace, because this is created on the platform cluster + nsName := openmcpclusterutils.StableRequestNamespace(as.Namespace) + nsm := resources.NewNamespaceMutator(nsName) + nsm.MetadataMutator().WithLabels(map[string]string{ + openmcpv1alpha1.V1MCPReferenceLabelNamespace: as.Namespace, + }) + if err := resources.CreateOrUpdateResource(ctx, platformClient, nsm); err != nil { + rerr := openmcperrors.WithReason(fmt.Errorf("failed to create or update namespace %s: %w", nsName, err), clustersconst.ReasonPlatformClusterInteractionProblem) + return ctrl.Result{}, nil, clusterConditions(false, rerr.Reason(), rerr.Error(), clusterRequestGrantedCon, clusterReadyCon, accessRequestGrantedCon), rerr + } + + // create or update ClusterRequest + var purpose string + switch as.Spec.Type { + case openmcpv1alpha1.Gardener: + purpose = "mcp" + case openmcpv1alpha1.GardenerDedicated: + purpose = "mcp-worker" + default: + rerr := openmcperrors.WithReason(fmt.Errorf("unknown APIServer type '%s'", as.Spec.Type), clustersconst.ReasonConfigurationProblem) + return ctrl.Result{}, nil, clusterConditions(false, rerr.Reason(), rerr.Error(), clusterRequestGrantedCon, clusterReadyCon, accessRequestGrantedCon), rerr + } + cr := &clustersv1alpha1.ClusterRequest{} + cr.Name = as.Name + cr.Namespace = nsName + crm := NewClusterRequestMutator(cr.Name, cr.Namespace, purpose) + crm.MetadataMutator().WithLabels(map[string]string{ + openmcpv1alpha1.V1MCPReferenceLabelName: as.Name, + openmcpv1alpha1.V1MCPReferenceLabelNamespace: as.Namespace, + }) + if err := resources.CreateOrUpdateResource(ctx, platformClient, crm); err != nil { + rerr := openmcperrors.WithReason(fmt.Errorf("failed to create or update ClusterRequest %s/%s: %w", cr.Namespace, cr.Name, err), clustersconst.ReasonPlatformClusterInteractionProblem) + return ctrl.Result{}, nil, clusterConditions(false, rerr.Reason(), rerr.Error(), clusterRequestGrantedCon, clusterReadyCon, accessRequestGrantedCon), rerr + } + + // if the ClusterRequest is granted, fetch the corresponding cluster + if err := platformClient.Get(ctx, client.ObjectKeyFromObject(cr), cr); err != nil { + rerr := openmcperrors.WithReason(fmt.Errorf("failed to get ClusterRequest %s/%s: %w", cr.Namespace, cr.Name, err), clustersconst.ReasonPlatformClusterInteractionProblem) + return ctrl.Result{}, nil, clusterConditions(false, rerr.Reason(), rerr.Error(), clusterRequestGrantedCon, clusterReadyCon, accessRequestGrantedCon), rerr + } + var setShootInStatus handler.UpdateStatusFunc + if cr.Status.Phase == clustersv1alpha1.REQUEST_GRANTED && cr.Status.Cluster != nil { + clusterRequestGrantedCon.Status = openmcpv1alpha1.ComponentConditionStatusTrue + + // fetch Cluster resource + cluster := &clustersv1alpha1.Cluster{} + cluster.Name = cr.Status.Cluster.Name + cluster.Namespace = cr.Status.Cluster.Namespace + if err := platformClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster); err != nil { + rerr := openmcperrors.WithReason(fmt.Errorf("failed to get Cluster %s/%s: %w", cluster.Namespace, cluster.Name, err), clustersconst.ReasonPlatformClusterInteractionProblem) + return ctrl.Result{}, nil, clusterConditions(false, rerr.Reason(), rerr.Error(), clusterRequestGrantedCon, clusterReadyCon, accessRequestGrantedCon), rerr + } + + clusterReadyCon.Status = openmcpv1alpha1.ComponentConditionStatusFromBool(cluster.Status.Phase == clustersv1alpha1.CLUSTER_PHASE_READY) + if clusterReadyCon.Status != openmcpv1alpha1.ComponentConditionStatusTrue { + clusterReadyCon.Reason = cconst.ReasonClusterNotReady + clusterReadyCon.Message = cluster.Status.Message + if clusterReadyCon.Message == "" { + clusterReadyCon.Message = "Cluster is not ready yet, no further information available" + } + } + + // check if there is a shoot manifest in the Cluster status + // if so, copy it into the APIServer status + if cluster.Status.ProviderStatus != nil { + cs := &gcpv1alpha1.ClusterStatus{} + if err := cluster.Status.GetProviderStatus(cs); err != nil { + rerr := openmcperrors.WithReason(fmt.Errorf("error unmarshalling provider status: %w", err), clustersconst.ReasonInternalError) + return ctrl.Result{}, nil, clusterConditions(false, rerr.Reason(), rerr.Error(), clusterRequestGrantedCon, clusterReadyCon, accessRequestGrantedCon), rerr + } + log.Debug("Provider status found, checking for shoot manifest") + if cs.Shoot != nil { + log.Debug("Found shoot in provider status", "shootName", cs.Shoot.GetName(), "shootNamespace", cs.Shoot.GetNamespace()) + setShootInStatus = func(status *openmcpv1alpha1.APIServerStatus) error { + status.GardenerStatus = &openmcpv1alpha1.GardenerStatus{} + uShoot := &unstructured.Unstructured{} + data, err := runtime.DefaultUnstructuredConverter.ToUnstructured(cs.Shoot) + if err != nil { + return fmt.Errorf("unable to convert shoot to unstructured object: %w", err) + } + uShoot.SetUnstructuredContent(data) + // ensure type information is set + uShoot.SetAPIVersion(gardenv1beta1.SchemeGroupVersion.String()) + uShoot.SetKind("Shoot") + // delete fields that should not be part of the shoot manifest in the status + uShoot.SetFinalizers(nil) + uShoot.SetResourceVersion("") + uShoot.SetCreationTimestamp(metav1.Time{}) + uShoot.SetGenerateName("") + uShoot.SetGeneration(0) + uShoot.SetManagedFields(nil) + uShoot.SetDeletionGracePeriodSeconds(nil) + uShoot.SetDeletionTimestamp(nil) + uShoot.SetOwnerReferences(nil) + + status.GardenerStatus.Shoot = &runtime.RawExtension{Object: uShoot} + return nil + } + + } + } + + if clusterReadyCon.Status != openmcpv1alpha1.ComponentConditionStatusTrue { + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil, clusterConditions(false, cconst.ReasonClusterNotReady, clusterReadyCon.Message, clusterRequestGrantedCon, clusterReadyCon, accessRequestGrantedCon), nil + } + + } else { + clusterRequestGrantedCon.Status = openmcpv1alpha1.ComponentConditionStatusFalse + clusterRequestGrantedCon.Reason = cconst.ReasonClusterRequestNotGranted + crReason := cr.Status.Reason + crMessage := cr.Status.Message + if crReason == "" { + crReason = "" + } + if crMessage == "" { + crMessage = "" + } + clusterRequestGrantedCon.Message = fmt.Sprintf("ClusterRequest is not granted or does not reference a cluster: [%s] %s", crReason, crMessage) + + rr := ctrl.Result{RequeueAfter: 30 * time.Second} + if cr.Status.Phase == clustersv1alpha1.REQUEST_DENIED { + // a denied request will never become granted (at least that's the idea), so no reason to wait for it + rr = ctrl.Result{} + } + return rr, nil, clusterConditions(false, clusterRequestGrantedCon.Reason, clusterRequestGrantedCon.Message, clusterRequestGrantedCon, clusterReadyCon, accessRequestGrantedCon), nil + } + + // build the UpdateStatusFunc + apiAccess := &openmcpv1alpha1.APIServerAccess{} + var usf handler.UpdateStatusFunc = func(status *openmcpv1alpha1.APIServerStatus) error { + if status.ExternalAPIServerStatus == nil { + status.ExternalAPIServerStatus = &openmcpv1alpha1.ExternalAPIServerStatus{} + } + if setShootInStatus != nil { + if err := setShootInStatus(status); err != nil { + return fmt.Errorf("error setting shoot in status: %w", err) + } + } + if apiAccess != nil { + status.AdminAccess = apiAccess + } + return nil + } + + rr := ctrl.Result{} + if clusterReadyCon.Status == openmcpv1alpha1.ComponentConditionStatusTrue { + // ensure AccessRequest + ar := &clustersv1alpha1.AccessRequest{} + ar.Name = as.Name + ar.Namespace = nsName + arm := NewAccessRequestMutator(ar.Name, ar.Namespace, cr.Name, cr.Namespace, false, []clustersv1alpha1.PermissionsRequest{ + { + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"*"}, + Resources: []string{"*"}, + Verbs: []string{"*"}, + }, + }, + }, + }) + arm.MetadataMutator().WithLabels(map[string]string{ + openmcpv1alpha1.V1MCPReferenceLabelName: as.Name, + openmcpv1alpha1.V1MCPReferenceLabelNamespace: as.Namespace, + }) + if err := resources.CreateOrUpdateResource(ctx, platformClient, arm); err != nil { + rerr := openmcperrors.WithReason(fmt.Errorf("failed to create or update AccessRequest %s/%s: %w", ar.Namespace, ar.Name, err), clustersconst.ReasonPlatformClusterInteractionProblem) + return ctrl.Result{}, usf, clusterConditions(false, rerr.Reason(), rerr.Error(), clusterRequestGrantedCon, clusterReadyCon, accessRequestGrantedCon), rerr + } + // if the AccessRequest is granted, fetch the corresponding access + if err := platformClient.Get(ctx, client.ObjectKeyFromObject(ar), ar); err != nil { + rerr := openmcperrors.WithReason(fmt.Errorf("failed to get AccessRequest %s/%s: %w", ar.Namespace, ar.Name, err), clustersconst.ReasonPlatformClusterInteractionProblem) + return ctrl.Result{}, usf, clusterConditions(false, rerr.Reason(), rerr.Error(), clusterRequestGrantedCon, clusterReadyCon, accessRequestGrantedCon), rerr + } + if ar.Status.Phase != clustersv1alpha1.REQUEST_GRANTED && ar.Status.SecretRef == nil { + accessRequestGrantedCon.Status = openmcpv1alpha1.ComponentConditionStatusFalse + accessRequestGrantedCon.Reason = cconst.ReasonAccessRequestNotGranted + arReason := cr.Status.Reason + arMessage := cr.Status.Message + if arReason == "" { + arReason = "" + } + if arMessage == "" { + arMessage = "" + } + accessRequestGrantedCon.Message = fmt.Sprintf("AccessRequest '%s/%s' is not granted or does not reference a secret: [%s] %s", ar.Namespace, ar.Name, arReason, arMessage) + + rr := ctrl.Result{RequeueAfter: 30 * time.Second} + if ar.Status.Phase == clustersv1alpha1.REQUEST_DENIED { + // a denied request will never become granted (at least that's the idea), so no reason to wait for it + rr = ctrl.Result{} + } + return rr, usf, clusterConditions(false, accessRequestGrantedCon.Reason, accessRequestGrantedCon.Message, clusterRequestGrantedCon, clusterReadyCon, accessRequestGrantedCon), nil + } + + accessRequestGrantedCon.Status = openmcpv1alpha1.ComponentConditionStatusTrue + + // fetch the secret containing the kubeconfig + secret := &corev1.Secret{} + secret.Name = ar.Status.SecretRef.Name + secret.Namespace = ar.Status.SecretRef.Namespace + if err := platformClient.Get(ctx, client.ObjectKeyFromObject(secret), secret); err != nil { + rerr := openmcperrors.WithReason(fmt.Errorf("failed to get Secret %s/%s: %w", secret.Namespace, secret.Name, err), clustersconst.ReasonPlatformClusterInteractionProblem) + return ctrl.Result{}, usf, clusterConditions(false, rerr.Reason(), rerr.Error(), clusterRequestGrantedCon, clusterReadyCon, accessRequestGrantedCon), rerr + } + kcfg, ok := secret.Data["kubeconfig"] + if !ok { + rerr := openmcperrors.WithReason(fmt.Errorf("kubeconfig not found in secret %s/%s", secret.Namespace, secret.Name), clustersconst.ReasonInternalError) + return ctrl.Result{}, usf, clusterConditions(false, rerr.Reason(), rerr.Error(), clusterRequestGrantedCon, clusterReadyCon, accessRequestGrantedCon), rerr + } + rawCreationTime, ok := secret.Data["creationTimestamp"] + if !ok { + rerr := openmcperrors.WithReason(fmt.Errorf("creationTimestamp not found in secret %s/%s", secret.Namespace, secret.Name), clustersconst.ReasonInternalError) + return ctrl.Result{}, usf, clusterConditions(false, rerr.Reason(), rerr.Error(), clusterRequestGrantedCon, clusterReadyCon, accessRequestGrantedCon), rerr + } + creationSeconds, err := strconv.ParseInt(string(rawCreationTime), 10, 64) + if err != nil { + rerr := openmcperrors.WithReason(fmt.Errorf("error parsing creationTimestamp from secret %s/%s to int64: %w", secret.Namespace, secret.Name, err), clustersconst.ReasonInternalError) + return ctrl.Result{}, usf, clusterConditions(false, rerr.Reason(), rerr.Error(), clusterRequestGrantedCon, clusterReadyCon, accessRequestGrantedCon), rerr + } + expirationTime, ok := secret.Data["expirationTimestamp"] + if !ok { + rerr := openmcperrors.WithReason(fmt.Errorf("expirationTimestamp not found in secret %s/%s", secret.Namespace, secret.Name), clustersconst.ReasonInternalError) + return ctrl.Result{}, usf, clusterConditions(false, rerr.Reason(), rerr.Error(), clusterRequestGrantedCon, clusterReadyCon, accessRequestGrantedCon), rerr + } + expirationSeconds, err := strconv.ParseInt(string(expirationTime), 10, 64) + if err != nil { + rerr := openmcperrors.WithReason(fmt.Errorf("error parsing expirationTimestamp from secret %s/%s to int64: %w", secret.Namespace, secret.Name, err), clustersconst.ReasonInternalError) + return ctrl.Result{}, usf, clusterConditions(false, rerr.Reason(), rerr.Error(), clusterRequestGrantedCon, clusterReadyCon, accessRequestGrantedCon), rerr + } + apiAccess.Kubeconfig = string(kcfg) + apiAccess.CreationTimestamp = &metav1.Time{Time: time.Unix(creationSeconds, 0)} + apiAccess.ExpirationTimestamp = &metav1.Time{Time: time.Unix(expirationSeconds, 0)} + rr = ctrl.Result{ + RequeueAfter: time.Until(clusteraccess.ComputeTokenRenewalTimeWithRatio(apiAccess.CreationTimestamp.Time, apiAccess.ExpirationTimestamp.Time, 0.85)), + } + } + + return rr, usf, clusterConditions(true, "", "", clusterRequestGrantedCon, clusterReadyCon, accessRequestGrantedCon), nil +} + +func v2HandleDelete(ctx context.Context, as *openmcpv1alpha1.APIServer, platformClient client.Client) (ctrl.Result, handler.UpdateStatusFunc, []openmcpv1alpha1.ComponentCondition, openmcperrors.ReasonableError) { + log := logging.FromContextOrPanic(ctx).WithName(openmcpv1alpha1.ArchitectureV2) + ctx = logging.NewContext(ctx, log) + + accessRequestDeletedCon := openmcpv1alpha1.ComponentCondition{ + Type: cconst.ConditionAccessRequestDeleted, + Status: openmcpv1alpha1.ComponentConditionStatusUnknown, + } + clusterRequestDeletedCon := openmcpv1alpha1.ComponentCondition{ + Type: cconst.ConditionClusterRequestDeleted, + Status: openmcpv1alpha1.ComponentConditionStatusUnknown, + } + + // instead of calling a handler, remove AccessRequest and ClusterRequest + nsName := openmcpclusterutils.StableRequestNamespace(as.Namespace) + + // remove AccessRequest + ar := &clustersv1alpha1.AccessRequest{} + ar.Name = as.Name + ar.Namespace = nsName + if err := platformClient.Delete(ctx, ar); client.IgnoreNotFound(err) != nil { + rerr := openmcperrors.WithReason(fmt.Errorf("failed to delete AccessRequest %s/%s: %w", ar.Namespace, ar.Name, err), clustersconst.ReasonPlatformClusterInteractionProblem) + accessRequestDeletedCon.Status = openmcpv1alpha1.ComponentConditionStatusFalse + accessRequestDeletedCon.Reason = rerr.Reason() + accessRequestDeletedCon.Message = err.Error() + return ctrl.Result{}, nil, clusterConditions(false, rerr.Reason(), rerr.Error(), accessRequestDeletedCon, clusterRequestDeletedCon), rerr + } + + if err := platformClient.Get(ctx, client.ObjectKeyFromObject(ar), ar); err != nil { + if !apierrors.IsNotFound(err) { + rerr := openmcperrors.WithReason(fmt.Errorf("failed to verify deletion of AccessRequest '%s/%s': %w", ar.Namespace, ar.Name, err), clustersconst.ReasonPlatformClusterInteractionProblem) + accessRequestDeletedCon.Status = openmcpv1alpha1.ComponentConditionStatusFalse + accessRequestDeletedCon.Reason = rerr.Reason() + accessRequestDeletedCon.Message = rerr.Error() + return ctrl.Result{}, nil, clusterConditions(false, rerr.Reason(), rerr.Error(), accessRequestDeletedCon, clusterRequestDeletedCon), rerr + } + accessRequestDeletedCon.Status = openmcpv1alpha1.ComponentConditionStatusTrue + } else { + accessRequestDeletedCon.Status = openmcpv1alpha1.ComponentConditionStatusFalse + accessRequestDeletedCon.Reason = cconst.ReasonAccessRequestNotDeleted + accessRequestDeletedCon.Message = fmt.Sprintf("AccessRequest '%s/%s' has not been deleted yet", ar.Namespace, ar.Name) + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil, clusterConditions(false, accessRequestDeletedCon.Reason, accessRequestDeletedCon.Message, accessRequestDeletedCon, clusterRequestDeletedCon), nil + } + + var usf handler.UpdateStatusFunc = func(status *openmcpv1alpha1.APIServerStatus) error { + status.AdminAccess = nil + return nil + } + + // remove ClusterRequest + cr := &clustersv1alpha1.ClusterRequest{} + cr.Name = as.Name + cr.Namespace = nsName + if err := platformClient.Delete(ctx, cr); client.IgnoreNotFound(err) != nil { + rerr := openmcperrors.WithReason(fmt.Errorf("failed to delete ClusterRequest %s/%s: %w", cr.Namespace, cr.Name, err), clustersconst.ReasonPlatformClusterInteractionProblem) + clusterRequestDeletedCon.Status = openmcpv1alpha1.ComponentConditionStatusFalse + clusterRequestDeletedCon.Reason = rerr.Reason() + clusterRequestDeletedCon.Message = err.Error() + return ctrl.Result{}, usf, clusterConditions(false, rerr.Reason(), rerr.Error(), accessRequestDeletedCon, clusterRequestDeletedCon), rerr + } + + if err := platformClient.Get(ctx, client.ObjectKeyFromObject(cr), cr); err != nil { + if !apierrors.IsNotFound(err) { + rerr := openmcperrors.WithReason(fmt.Errorf("failed to verify deletion of ClusterRequest '%s/%s': %w", cr.Namespace, cr.Name, err), clustersconst.ReasonPlatformClusterInteractionProblem) + clusterRequestDeletedCon.Status = openmcpv1alpha1.ComponentConditionStatusFalse + clusterRequestDeletedCon.Reason = rerr.Reason() + clusterRequestDeletedCon.Message = rerr.Error() + return ctrl.Result{}, usf, clusterConditions(false, rerr.Reason(), rerr.Error(), accessRequestDeletedCon, clusterRequestDeletedCon), rerr + } + clusterRequestDeletedCon.Status = openmcpv1alpha1.ComponentConditionStatusTrue + } else { + clusterRequestDeletedCon.Status = openmcpv1alpha1.ComponentConditionStatusFalse + clusterRequestDeletedCon.Reason = cconst.ReasonClusterRequestNotDeleted + clusterRequestDeletedCon.Message = fmt.Sprintf("ClusterRequest '%s/%s' has not been deleted yet", cr.Namespace, cr.Name) + return ctrl.Result{RequeueAfter: 30 * time.Second}, usf, clusterConditions(false, clusterRequestDeletedCon.Reason, clusterRequestDeletedCon.Message, accessRequestDeletedCon, clusterRequestDeletedCon), nil + } + + usf = func(status *openmcpv1alpha1.APIServerStatus) error { + status.AdminAccess = nil + status.GardenerStatus = nil + return nil + } + + return ctrl.Result{}, usf, clusterConditions(true, "", "", accessRequestDeletedCon, clusterRequestDeletedCon), nil +} + +type ClusterRequestMutator struct { + name string + namespace string + purpose string + meta resources.MetadataMutator +} + +var _ resources.Mutator[*clustersv1alpha1.ClusterRequest] = &ClusterRequestMutator{} + +func NewClusterRequestMutator(name, namespace, purpose string) *ClusterRequestMutator { + return &ClusterRequestMutator{ + name: name, + namespace: namespace, + purpose: purpose, + meta: resources.NewMetadataMutator(), + } +} + +// Empty implements resources.Mutator. +func (m *ClusterRequestMutator) Empty() *clustersv1alpha1.ClusterRequest { + return &clustersv1alpha1.ClusterRequest{ + TypeMeta: metav1.TypeMeta{ + APIVersion: clustersv1alpha1.GroupVersion.String(), + Kind: "ClusterRequest", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: m.name, + Namespace: m.namespace, + }, + } +} + +// MetadataMutator implements resources.Mutator. +func (m *ClusterRequestMutator) MetadataMutator() resources.MetadataMutator { + return m.meta +} + +// Mutate implements resources.Mutator. +func (m *ClusterRequestMutator) Mutate(r *clustersv1alpha1.ClusterRequest) error { + r.Spec.Purpose = m.purpose + return m.meta.Mutate(r) +} + +// String implements resources.Mutator. +func (m *ClusterRequestMutator) String() string { + return fmt.Sprintf("ClusterRequest %s/%s", m.namespace, m.name) +} + +type AccessRequestMutator struct { + name string + namespace string + refName string + refNamespace string + isClusterRef bool + permissions []clustersv1alpha1.PermissionsRequest + meta resources.MetadataMutator +} + +var _ resources.Mutator[*clustersv1alpha1.AccessRequest] = &AccessRequestMutator{} + +func NewAccessRequestMutator(name, namespace, refName, refNamespace string, isClusterRef bool, permissions []clustersv1alpha1.PermissionsRequest) *AccessRequestMutator { + return &AccessRequestMutator{ + name: name, + namespace: namespace, + refName: refName, + refNamespace: refNamespace, + isClusterRef: isClusterRef, + permissions: permissions, + meta: resources.NewMetadataMutator(), + } +} + +// Empty implements resources.Mutator. +func (m *AccessRequestMutator) Empty() *clustersv1alpha1.AccessRequest { + return &clustersv1alpha1.AccessRequest{ + TypeMeta: metav1.TypeMeta{ + APIVersion: clustersv1alpha1.GroupVersion.String(), + Kind: "ClusterRequest", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: m.name, + Namespace: m.namespace, + }, + } +} + +// MetadataMutator implements resources.Mutator. +func (m *AccessRequestMutator) MetadataMutator() resources.MetadataMutator { + return m.meta +} + +// Mutate implements resources.Mutator. +func (m *AccessRequestMutator) Mutate(r *clustersv1alpha1.AccessRequest) error { + if m.isClusterRef && r.Spec.ClusterRef == nil { + r.Spec.ClusterRef = &clustersv1alpha1.NamespacedObjectReference{} + r.Spec.ClusterRef.Name = m.refName + r.Spec.ClusterRef.Namespace = m.refNamespace + } else if !m.isClusterRef && r.Spec.RequestRef == nil { + r.Spec.RequestRef = &clustersv1alpha1.NamespacedObjectReference{} + r.Spec.RequestRef.Name = m.refName + r.Spec.RequestRef.Namespace = m.refNamespace + } + r.Spec.Permissions = make([]clustersv1alpha1.PermissionsRequest, len(m.permissions)) + for i, perm := range m.permissions { + r.Spec.Permissions[i] = *perm.DeepCopy() + } + return m.meta.Mutate(r) +} + +// String implements resources.Mutator. +func (m *AccessRequestMutator) String() string { + return fmt.Sprintf("AccessRequest %s/%s", m.namespace, m.name) +} + +func clusterConditions(ready bool, reason, message string, additionalConditions ...openmcpv1alpha1.ComponentCondition) []openmcpv1alpha1.ComponentCondition { + conditions := []openmcpv1alpha1.ComponentCondition{ + componentutils.NewCondition(openmcpv1alpha1.APIServerComponent.HealthyCondition(), openmcpv1alpha1.ComponentConditionStatusFromBool(ready), reason, message), + } + conditions = append(conditions, additionalConditions...) + return conditions +} diff --git a/internal/controller/core/managedcontrolplane/controller.go b/internal/controller/core/managedcontrolplane/controller.go index aa8b651..f32852d 100644 --- a/internal/controller/core/managedcontrolplane/controller.go +++ b/internal/controller/core/managedcontrolplane/controller.go @@ -154,7 +154,7 @@ func (r *ManagedControlPlaneController) Reconcile(ctx context.Context, req ctrl. cp.Status.Status = openmcpv1alpha1.MCPStatusDeleting } - errs := []error{} + errs := []error{err} if err := r.Client.Status().Update(ctx, cp); err != nil { errs = append(errs, fmt.Errorf("error updating ManagedControlPlane status: %w", err)) } diff --git a/internal/controller/core/managedcontrolplane/controller_test.go b/internal/controller/core/managedcontrolplane/controller_test.go index 5cda107..34a2144 100644 --- a/internal/controller/core/managedcontrolplane/controller_test.go +++ b/internal/controller/core/managedcontrolplane/controller_test.go @@ -8,6 +8,8 @@ import ( "time" "github.com/openmcp-project/mcp-operator/internal/components" + mcpocfg "github.com/openmcp-project/mcp-operator/internal/config" + archconfig "github.com/openmcp-project/mcp-operator/internal/config/architecture" "github.com/openmcp-project/mcp-operator/internal/controller/core/managedcontrolplane" @@ -40,10 +42,21 @@ const ( ) var _ = Describe("CO-1153 ManagedControlPlane Controller", func() { + + BeforeEach(func() { + mcpocfg.Config.Architecture = archconfig.ArchConfig{} + mcpocfg.Config.Architecture.Default() + mcpocfg.Config.Architecture.APIServer.AllowOverride = true + }) + It("should create all component resources that are configured in the MCP and delete them again when they are unconfigured", func() { var err error env := testutils.DefaultTestSetupBuilder("testdata", "test-01").WithReconcilerConstructor(mcpReconciler, getReconciler, testutils.CrateCluster).Build() + mcpocfg.Config.Architecture.APIServer.Version = openmcpv1alpha1.ArchitectureV1 + mcpocfg.Config.Architecture.Landscaper.AllowOverride = true + mcpocfg.Config.Architecture.Landscaper.Version = openmcpv1alpha1.ArchitectureV2 + // get ManagedControlPlane mcp := &openmcpv1alpha1.ManagedControlPlane{} err = env.Client(testutils.CrateCluster).Get(env.Ctx, types.NamespacedName{Name: "test", Namespace: "test"}, mcp) @@ -68,6 +81,15 @@ var _ = Describe("CO-1153 ManagedControlPlane Controller", func() { Expect(ch.Resource().GetLabels()).To(HaveKeyWithValue(openmcpv1alpha1.ManagedControlPlaneBackReferenceLabelNamespace, mcp.Namespace)) Expect(ch.Resource().GetLabels()).To(HaveKeyWithValue(openmcpv1alpha1.ManagedControlPlaneGenerationLabel, fmt.Sprint(mcp.Generation))) Expect(ch.Resource().GetLabels()).ToNot(HaveKey(openmcpv1alpha1.InternalConfigurationGenerationLabel)) + + switch ct { + case openmcpv1alpha1.APIServerComponent: + fallthrough + case openmcpv1alpha1.LandscaperComponent: + Expect(ch.Resource().GetLabels()).To(HaveKeyWithValue(openmcpv1alpha1.ArchitectureVersionLabel, openmcpv1alpha1.ArchitectureV2)) + default: + Expect(ch.Resource().GetLabels()).To(HaveKeyWithValue(openmcpv1alpha1.ArchitectureVersionLabel, openmcpv1alpha1.ArchitectureV1)) + } } } @@ -516,6 +538,31 @@ var _ = Describe("CO-1153 ManagedControlPlane Controller", func() { reconcileAndTest() }) + It("should throw an error if the MCP has an architecture version label for a component that does not allow overrides", func() { + env := testutils.DefaultTestSetupBuilder("testdata", "test-01").WithReconcilerConstructor(mcpReconciler, getReconciler, testutils.CrateCluster).Build() + mcpocfg.Config.Architecture.APIServer.AllowOverride = false + + // get ManagedControlPlane + mcp := &openmcpv1alpha1.ManagedControlPlane{} + Expect(env.Client(testutils.CrateCluster).Get(env.Ctx, types.NamespacedName{Name: "test", Namespace: "test"}, mcp)).To(Succeed()) + + req := openmcptesting.RequestFromObject(mcp) + env.ShouldNotReconcileWithError(mcpReconciler, req, And(MatchError(ContainSubstring("override")), MatchError(ContainSubstring("APIServer")), MatchError(ContainSubstring("not allowed")))) + }) + + It("should throw an error if the MCP has an architecture version label with an invalid version", func() { + env := testutils.DefaultTestSetupBuilder("testdata", "test-01").WithReconcilerConstructor(mcpReconciler, getReconciler, testutils.CrateCluster).Build() + + // get ManagedControlPlane + mcp := &openmcpv1alpha1.ManagedControlPlane{} + Expect(env.Client(testutils.CrateCluster).Get(env.Ctx, types.NamespacedName{Name: "test", Namespace: "test"}, mcp)).To(Succeed()) + mcp.Labels[openmcpv1alpha1.APIServerComponent.ArchitectureVersionLabel()] = "invalid" + Expect(env.Client(testutils.CrateCluster).Update(env.Ctx, mcp)).To(Succeed()) + + req := openmcptesting.RequestFromObject(mcp) + env.ShouldNotReconcileWithError(mcpReconciler, req, And(MatchError(ContainSubstring("version")), MatchError(ContainSubstring("APIServer")), MatchError(ContainSubstring("not allowed")))) + }) + }) func TestConfig(t *testing.T) { diff --git a/internal/controller/core/managedcontrolplane/conversion.go b/internal/controller/core/managedcontrolplane/conversion.go index 53ad464..759ec92 100644 --- a/internal/controller/core/managedcontrolplane/conversion.go +++ b/internal/controller/core/managedcontrolplane/conversion.go @@ -2,9 +2,11 @@ package managedcontrolplane import ( "fmt" + "maps" "slices" "github.com/openmcp-project/mcp-operator/internal/components" + mcpocfg "github.com/openmcp-project/mcp-operator/internal/config" componentutils "github.com/openmcp-project/mcp-operator/internal/utils/components" corev1 "k8s.io/api/core/v1" @@ -35,8 +37,8 @@ func (*ManagedControlPlaneController) ManagedControlPlaneToSplitInternalResource } res := map[openmcpv1alpha1.ComponentType]*components.ComponentHandler{} - allCompHandlerss := components.Registry.GetKnownComponents() - for ct, ch := range allCompHandlerss { + allCompHandlers := components.Registry.GetKnownComponents() + for ct, ch := range allCompHandlers { if ch != nil && ch.Resource() != nil && ch.Converter() != nil && ch.Converter().IsConfigured(mcp) { ch.Resource().SetName(mcp.Name) ch.Resource().SetNamespace(mcp.Namespace) @@ -59,7 +61,26 @@ func (*ManagedControlPlaneController) ManagedControlPlaneToSplitInternalResource }) } - ch.Resource().SetLabels(labels) + // take over architecture version label from the MCP resource, if override is allowed for the component + bridgeConfig := mcpocfg.Config.Architecture.GetBridgeConfigForComponent(ct) + cLabels := make(map[string]string, len(labels)+1) + maps.Copy(cLabels, labels) + v, found := mcp.Labels[ct.ArchitectureVersionLabel()] + if found { + // check if version override is allowed for this component + if !bridgeConfig.AllowOverride { + return nil, fmt.Errorf("architecture version override is not allowed for component '%s', remove the '%s' label", string(ct), ct.ArchitectureVersionLabel()) + } + if !bridgeConfig.IsAllowedVersion(v) { + return nil, fmt.Errorf("architecture version '%s' is not allowed for component '%s'", v, string(ct)) + } + cLabels[openmcpv1alpha1.ArchitectureVersionLabel] = v + } else { + cLabels[openmcpv1alpha1.ArchitectureVersionLabel] = bridgeConfig.Version + } + + ch.Resource().SetLabels(cLabels) + componentutils.SetCreatedFromGeneration(ch.Resource(), mcp, icfg) if err := controllerutil.SetControllerReference(mcp, ch.Resource(), scheme); err != nil { return nil, fmt.Errorf("unable to set owner reference: %w", err) diff --git a/internal/controller/core/managedcontrolplane/testdata/test-01/mcp.yaml b/internal/controller/core/managedcontrolplane/testdata/test-01/mcp.yaml index 8a2a57c..d3d0252 100644 --- a/internal/controller/core/managedcontrolplane/testdata/test-01/mcp.yaml +++ b/internal/controller/core/managedcontrolplane/testdata/test-01/mcp.yaml @@ -4,6 +4,8 @@ metadata: name: test namespace: test generation: 5 + labels: + apiserver.architecture.openmcp.cloud/version: v2 spec: desiredRegion: name: europe diff --git a/test/utils/test.go b/test/utils/test.go index 764f6f4..687cdc3 100644 --- a/test/utils/test.go +++ b/test/utils/test.go @@ -9,6 +9,7 @@ import ( laasinstall "github.com/gardener/landscaper-service/pkg/apis/core/install" cocorev1beta1 "github.com/openmcp-project/control-plane-operator/api/v1beta1" "github.com/openmcp-project/controller-utils/pkg/testing" + v2install "github.com/openmcp-project/openmcp-operator/api/install" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -30,6 +31,7 @@ func init() { utilruntime.Must(gardenv1beta1.AddToScheme(Scheme)) utilruntime.Must(gardenauthenticationv1alpha1.AddToScheme(Scheme)) utilruntime.Must(clientgoscheme.AddToScheme(Scheme)) + v2install.InstallOperatorAPIs(Scheme) } const (