diff --git a/cmd/api-syncagent/main.go b/cmd/api-syncagent/main.go index c3edd66..03b1993 100644 --- a/cmd/api-syncagent/main.go +++ b/cmd/api-syncagent/main.go @@ -53,6 +53,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cluster" ctrlruntimelog "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" ) func main() { @@ -168,9 +169,11 @@ func setupLocalManager(ctx context.Context, opts *Options) (manager.Manager, err BaseContext: func() context.Context { return ctx }, + Metrics: metricsserver.Options{BindAddress: opts.MetricsAddr}, LeaderElection: opts.EnableLeaderElection, LeaderElectionID: "syncagent." + opts.AgentName, LeaderElectionNamespace: opts.Namespace, + HealthProbeBindAddress: opts.HealthAddr, }) if err != nil { return nil, err diff --git a/cmd/api-syncagent/options.go b/cmd/api-syncagent/options.go index 2530d70..1d75f4e 100644 --- a/cmd/api-syncagent/options.go +++ b/cmd/api-syncagent/options.go @@ -65,12 +65,16 @@ type Options struct { KubeconfigCAFileOverride string LogOptions log.Options + + MetricsAddr string + HealthAddr string } func NewOptions() *Options { return &Options{ LogOptions: log.NewDefaultOptions(), PublishedResourceSelector: labels.Everything(), + MetricsAddr: "127.0.0.1:8085", } } @@ -83,8 +87,10 @@ func (o *Options) AddFlags(flags *pflag.FlagSet) { flags.StringVar(&o.APIExportRef, "apiexport-ref", o.APIExportRef, "name of the APIExport in kcp that this Sync Agent is powering") flags.StringVar(&o.PublishedResourceSelectorString, "published-resource-selector", o.PublishedResourceSelectorString, "restrict this Sync Agent to only process PublishedResources matching this label selector (optional)") flags.BoolVar(&o.EnableLeaderElection, "enable-leader-election", o.EnableLeaderElection, "whether to perform leader election") - flags.StringVar(&o.KubeconfigHostOverride, "kubeconfig-host-override", o.KubeconfigHostOverride, "Override the host configured in the local kubeconfig") - flags.StringVar(&o.KubeconfigCAFileOverride, "kubeconfig-ca-file-override", o.KubeconfigCAFileOverride, "Override the server CA file configured in the local kubeconfig") + flags.StringVar(&o.KubeconfigHostOverride, "kubeconfig-host-override", o.KubeconfigHostOverride, "override the host configured in the local kubeconfig") + flags.StringVar(&o.KubeconfigCAFileOverride, "kubeconfig-ca-file-override", o.KubeconfigCAFileOverride, "override the server CA file configured in the local kubeconfig") + flags.StringVar(&o.MetricsAddr, "metrics-address", o.MetricsAddr, "host and port to serve Prometheus metrics via /metrics (HTTP)") + flags.StringVar(&o.HealthAddr, "health-address", o.HealthAddr, "host and port to serve probes via /readyz and /healthz (HTTP)") } func (o *Options) Validate() error { diff --git a/docs/faq.md b/docs/faq.md index 63e2a61..03e7bc2 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -36,3 +36,19 @@ Only those required for its own operation. If you configure a namespaced resourc automatically add a claim for `namespaces` in kcp, plus it will add either `configmaps` or `secrets` if related resources are configured in a `PublishedResource`. But you cannot specify additional permissions claims. + +## I am seeing errors in the agent logs, what's going on? + +Errors like + +> reflector.go:561] k8s.io/client-go@v0.31.2/tools/cache/reflector.go:243: failed to list +> example.com/v1, Kind=Dummy: the server could not find the requested resource + +or + +> reflector.go:158] "Unhandled Error" err="k8s.io/client-go@v0.31.2/tools/cache/reflector.go:243: +> Failed to watch kcp.example.com/v1, Kind=Dummy: failed to list kcp.example.com/v1, Kind=Dummy: +> the server could not find the requested resource" logger="UnhandledError" + +are typical when bootstrapping new APIExports in kcp. They are only cause for concern if they +persist after configuring all PublishedResources. diff --git a/go.mod b/go.mod index aa312e0..1c55793 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/evanphx/json-patch/v5 v5.9.0 github.com/go-logr/logr v1.4.2 github.com/go-logr/zapr v1.3.0 + github.com/google/go-cmp v0.6.0 github.com/kcp-dev/apimachinery/v2 v2.0.1-0.20240817110845-a9eb9752bfeb github.com/kcp-dev/client-go v0.0.0-20240912145314-f5949d81732a github.com/kcp-dev/code-generator/v2 v2.3.1 @@ -53,7 +54,6 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.1-0.20210504230335-f78f29fc09ea // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect diff --git a/internal/controller/apiresourceschema/controller.go b/internal/controller/apiresourceschema/controller.go index 1d40910..59821b4 100644 --- a/internal/controller/apiresourceschema/controller.go +++ b/internal/controller/apiresourceschema/controller.go @@ -133,7 +133,10 @@ func (r *Reconciler) reconcile(ctx context.Context, log *zap.SugaredLogger, pubR } // project the CRD - projectedCRD := r.projectResourceNames(r.apiExportName, crd, pubResource.Spec.Projection) + projectedCRD, err := r.applyProjection(r.apiExportName, crd, pubResource) + if err != nil { + return nil, fmt.Errorf("failed to apply projection rules: %w", err) + } // to prevent changing the source GVK e.g. from "apps/v1 Daemonset" to "core/v1 Pod", // we include the source GVK in hashed form in the final APIResourceSchema name. @@ -147,7 +150,7 @@ func (r *Reconciler) reconcile(ctx context.Context, log *zap.SugaredLogger, pubR err = r.kcpClient.Get(wsCtx, types.NamespacedName{Name: arsName}, ars, &ctrlruntimeclient.GetOptions{}) if apierrors.IsNotFound(err) { - if err := r.createAPIResourceSchema(wsCtx, log, r.apiExportName, projectedCRD, arsName, pubResource.Spec.Resource.Version); err != nil { + if err := r.createAPIResourceSchema(wsCtx, log, r.apiExportName, projectedCRD, arsName); err != nil { return nil, fmt.Errorf("failed to create APIResourceSchema: %w", err) } } else if err != nil { @@ -170,17 +173,7 @@ func (r *Reconciler) reconcile(ctx context.Context, log *zap.SugaredLogger, pubR return nil, nil } -func (r *Reconciler) createAPIResourceSchema(ctx context.Context, log *zap.SugaredLogger, apigroup string, projectedCRD *apiextensionsv1.CustomResourceDefinition, arsName string, selectedVersion string) error { - // At this moment we ignore every non-selected version in the CRD, as we have not fully - // decided on how to support the API version lifecycle yet. Having multiple versions in - // the CRD will make kcp require a `conversion` to also be configured. Since we cannot - // enforce that and want to instead work with existing CRDs as best as we can, we chose - // this option (instead of error'ing out if a conversion is missing). - projectedCRD.Spec.Conversion = nil - projectedCRD.Spec.Versions = slices.DeleteFunc(projectedCRD.Spec.Versions, func(v apiextensionsv1.CustomResourceDefinitionVersion) bool { - return v.Name != selectedVersion - }) - +func (r *Reconciler) createAPIResourceSchema(ctx context.Context, log *zap.SugaredLogger, apigroup string, projectedCRD *apiextensionsv1.CustomResourceDefinition, arsName string) error { // prefix is irrelevant as the reconciling framework will use arsName anyway converted, err := kcpdevv1alpha1.CRDToAPIResourceSchema(projectedCRD, "irrelevant") if err != nil { @@ -206,12 +199,35 @@ func (r *Reconciler) createAPIResourceSchema(ctx context.Context, log *zap.Sugar return r.kcpClient.Create(ctx, ars) } -func (r *Reconciler) projectResourceNames(apiGroup string, crd *apiextensionsv1.CustomResourceDefinition, projection *syncagentv1alpha1.ResourceProjection) *apiextensionsv1.CustomResourceDefinition { +func (r *Reconciler) applyProjection(apiGroup string, crd *apiextensionsv1.CustomResourceDefinition, pr *syncagentv1alpha1.PublishedResource) (*apiextensionsv1.CustomResourceDefinition, error) { result := crd.DeepCopy() result.Spec.Group = apiGroup + // At this moment we ignore every non-selected version in the CRD, as we have not fully + // decided on how to support the API version lifecycle yet. Having multiple versions in + // the CRD will make kcp require a `conversion` to also be configured. Since we cannot + // enforce that and want to instead work with existing CRDs as best as we can, we chose + // this option (instead of error'ing out if a conversion is missing). + result.Spec.Conversion = nil + result.Spec.Versions = slices.DeleteFunc(result.Spec.Versions, func(v apiextensionsv1.CustomResourceDefinitionVersion) bool { + return v.Name != pr.Spec.Resource.Version + }) + + if len(result.Spec.Versions) != 1 { + // This should never happen because of checks earlier in the reconciler. + return nil, fmt.Errorf("invalid CRD: cannot find selected version %q", pr.Spec.Resource.Version) + } + + result.Spec.Versions[0].Served = true + result.Spec.Versions[0].Storage = true + + projection := pr.Spec.Projection if projection == nil { - return result + return result, nil + } + + if projection.Version != "" { + result.Spec.Versions[0].Name = projection.Version } if projection.Kind != "" { @@ -238,7 +254,7 @@ func (r *Reconciler) projectResourceNames(apiGroup string, crd *apiextensionsv1. result.Spec.Names.ShortNames = projection.ShortNames } - return result + return result, nil } // getAPIResourceSchemaName generates the name for the ARS in kcp. Note that diff --git a/test/crds/backup.yaml b/test/crds/backup.yaml new file mode 100644 index 0000000..8de6059 --- /dev/null +++ b/test/crds/backup.yaml @@ -0,0 +1,27 @@ +# sourced from https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: backups.eksempel.no +spec: + group: eksempel.no + scope: Namespaced + names: + plural: backups + singular: backup + kind: Backup + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + source: + type: string + destination: + type: string diff --git a/test/crds/crontab-improved.yaml b/test/crds/crontab-improved.yaml new file mode 100644 index 0000000..ed8e59b --- /dev/null +++ b/test/crds/crontab-improved.yaml @@ -0,0 +1,32 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: crontabs.example.com +spec: + group: example.com + scope: Namespaced + names: + plural: crontabs + singular: crontab + kind: CronTab + shortNames: + - ct + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + cronSpec: + type: string + image: + type: string + command: + type: string + replicas: + type: integer diff --git a/test/crds/crontab-multi-versions.yaml b/test/crds/crontab-multi-versions.yaml new file mode 100644 index 0000000..51a0dd1 --- /dev/null +++ b/test/crds/crontab-multi-versions.yaml @@ -0,0 +1,47 @@ +# sourced from https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: crontabs.example.com +spec: + group: example.com + scope: Namespaced + names: + plural: crontabs + singular: crontab + kind: CronTab + shortNames: + - ct + versions: + - name: v1 + served: true + storage: false + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + cronSpec: + type: string + image: + type: string + replicas: + type: integer + - name: v2 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + cronSpec: + type: string + image: + type: string + replicas: + type: integer diff --git a/test/e2e/apiexport/apiexport_test.go b/test/e2e/apiexport/apiexport_test.go new file mode 100644 index 0000000..cee6d5e --- /dev/null +++ b/test/e2e/apiexport/apiexport_test.go @@ -0,0 +1,345 @@ +//go:build e2e + +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiexport + +import ( + "context" + "testing" + "time" + + "github.com/go-logr/logr" + + syncagentv1alpha1 "github.com/kcp-dev/api-syncagent/sdk/apis/syncagent/v1alpha1" + "github.com/kcp-dev/api-syncagent/test/utils" + + kcpapisv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha1" + + "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + ctrlruntime "sigs.k8s.io/controller-runtime" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestPermissionsClaims(t *testing.T) { + const ( + apiExportName = "kcp.example.com" + ) + + ctx := context.Background() + ctrlruntime.SetLogger(logr.Discard()) + + // setup a test environment in kcp + orgKubconfig := utils.CreateOrganization(t, ctx, "apiexport-no-pclaims-by-default", apiExportName) + + // start a service cluster + envtestKubeconfig, envtestClient, _ := utils.RunEnvtest(t, []string{ + "test/crds/crontab.yaml", + "test/crds/backup.yaml", + }) + + // publish Crontabs and Backups + t.Logf("Publishing CRDs…") + prCrontabs := &syncagentv1alpha1.PublishedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "publish-crontabs", + }, + Spec: syncagentv1alpha1.PublishedResourceSpec{ + Resource: syncagentv1alpha1.SourceResourceDescriptor{ + APIGroup: "example.com", + Version: "v1", + Kind: "CronTab", + }, + }, + } + + if err := envtestClient.Create(ctx, prCrontabs); err != nil { + t.Fatalf("Failed to create PublishedResource: %v", err) + } + + prBackups := &syncagentv1alpha1.PublishedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "publish-backups", + }, + Spec: syncagentv1alpha1.PublishedResourceSpec{ + Resource: syncagentv1alpha1.SourceResourceDescriptor{ + APIGroup: "eksempel.no", + Version: "v1", + Kind: "Backup", + }, + }, + } + + if err := envtestClient.Create(ctx, prBackups); err != nil { + t.Fatalf("Failed to create PublishedResource: %v", err) + } + + // let the agent do its thing + utils.RunAgent(ctx, t, "bob", orgKubconfig, envtestKubeconfig, apiExportName) + + // wait for the APIExport to be updated + t.Logf("Waiting for APIExport to be updated…") + orgClient := utils.GetClient(t, orgKubconfig) + apiExportKey := types.NamespacedName{Name: apiExportName} + + apiExport := &kcpapisv1alpha1.APIExport{} + err := wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 1*time.Minute, false, func(ctx context.Context) (done bool, err error) { + err = orgClient.Get(ctx, apiExportKey, apiExport) + if err != nil { + return false, err + } + + return len(apiExport.Spec.LatestResourceSchemas) == 2, nil + }) + if err != nil { + t.Fatalf("Failed to wait for APIExport to be updated: %v", err) + } + + if claims := apiExport.Spec.PermissionClaims; len(claims) > 0 { + t.Fatalf("APIExport should have no permissions claims, but has %v", claims) + } + + // let's configure some related resources + + // refresh the objects + if err := envtestClient.Get(ctx, ctrlruntimeclient.ObjectKeyFromObject(prCrontabs), prCrontabs); err != nil { + t.Fatalf("Failed to get PublishedResource: %v", err) + } + + if err := envtestClient.Get(ctx, ctrlruntimeclient.ObjectKeyFromObject(prBackups), prBackups); err != nil { + t.Fatalf("Failed to get PublishedResource: %v", err) + } + + t.Logf("Configuring related resources…") + prBackups.Spec.Related = []syncagentv1alpha1.RelatedResourceSpec{ + { + Identifier: "super-secret", + Origin: "kcp", + Kind: "Secret", + Reference: syncagentv1alpha1.RelatedResourceReference{ + Name: syncagentv1alpha1.ResourceLocator{ + Path: "spec.test.name", + }, + Namespace: &syncagentv1alpha1.ResourceLocator{ + Path: "spec.test.namespace", + }, + }, + }, + { + Identifier: "other-super-secret", + Origin: "service", + Kind: "Secret", + Reference: syncagentv1alpha1.RelatedResourceReference{ + Name: syncagentv1alpha1.ResourceLocator{ + Path: "spec.otherTest.name", + }, + Namespace: &syncagentv1alpha1.ResourceLocator{ + Path: "spec.otherTest.namespace", + }, + }, + }, + } + + prCrontabs.Spec.Related = []syncagentv1alpha1.RelatedResourceSpec{ + { + Identifier: "config", + Origin: "kcp", + Kind: "ConfigMap", + Reference: syncagentv1alpha1.RelatedResourceReference{ + Name: syncagentv1alpha1.ResourceLocator{ + Path: "spec.secretTest.name", + }, + Namespace: &syncagentv1alpha1.ResourceLocator{ + Path: "spec.secretTest.namespace", + }, + }, + }, + } + + if err := envtestClient.Update(ctx, prCrontabs); err != nil { + t.Fatalf("Failed to update PublishedResource: %v", err) + } + + if err := envtestClient.Update(ctx, prBackups); err != nil { + t.Fatalf("Failed to update PublishedResource: %v", err) + } + + // wait for the permission claims to be updated; note that since we have related resources at all, + // the agent will also claim namespaces (since both ConfigMaps and Secrets are always namespaced). + + t.Logf("Wait for the claims to be updated…") + apiExport = &kcpapisv1alpha1.APIExport{} + err = wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 1*time.Minute, false, func(ctx context.Context) (done bool, err error) { + err = orgClient.Get(ctx, apiExportKey, apiExport) + if err != nil { + return false, err + } + + return len(apiExport.Spec.PermissionClaims) == 3, nil + }) + if err != nil { + t.Fatalf("Failed to wait for APIExport to be updated: %v", err) + } + + expectedClaims := []kcpapisv1alpha1.PermissionClaim{ + { + GroupResource: kcpapisv1alpha1.GroupResource{ + Group: "", + Resource: "configmaps", + }, + All: true, + }, + { + GroupResource: kcpapisv1alpha1.GroupResource{ + Group: "", + Resource: "namespaces", + }, + All: true, + }, + { + GroupResource: kcpapisv1alpha1.GroupResource{ + Group: "", + Resource: "secrets", + }, + All: true, + }, + } + + // Do not use cmp.Equal() because the Equal() func on PermissionClaims does not check all fields. + if !equality.Semantic.DeepEqual(expectedClaims, apiExport.Spec.PermissionClaims) { + t.Fatalf("Expected permission claims %+v, but got %+v.", expectedClaims, apiExport.Spec.PermissionClaims) + } +} + +func TestExistingPermissionsClaimsAreKept(t *testing.T) { + const ( + apiExportName = "kcp.example.com" + ) + + ctx := context.Background() + ctrlruntime.SetLogger(logr.Discard()) + + // setup a test environment in kcp + orgKubconfig := utils.CreateOrganization(t, ctx, "apiexport-pclaims-are-kept", apiExportName) + + // start a service cluster + envtestKubeconfig, envtestClient, _ := utils.RunEnvtest(t, []string{ + "test/crds/crontab.yaml", + }) + + // set a random claim that is supposed to survive + orgClient := utils.GetClient(t, orgKubconfig) + apiExportKey := types.NamespacedName{Name: apiExportName} + + apiExport := &kcpapisv1alpha1.APIExport{} + if err := orgClient.Get(ctx, apiExportKey, apiExport); err != nil { + t.Fatalf("Failed to get APIExport: %v", err) + } + + apiExport.Spec.PermissionClaims = []kcpapisv1alpha1.PermissionClaim{ + { + GroupResource: kcpapisv1alpha1.GroupResource{ + Group: "", + Resource: "configmaps", + }, + All: true, + }, + } + + if err := orgClient.Update(ctx, apiExport); err != nil { + t.Fatalf("Failed to update APIExport: %v", err) + } + + // publish Crontabs + t.Logf("Publishing CRD…") + prCrontabs := &syncagentv1alpha1.PublishedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "publish-crontabs", + }, + Spec: syncagentv1alpha1.PublishedResourceSpec{ + Resource: syncagentv1alpha1.SourceResourceDescriptor{ + APIGroup: "example.com", + Version: "v1", + Kind: "CronTab", + }, + Related: []syncagentv1alpha1.RelatedResourceSpec{ + { + Identifier: "super-secret", + Origin: "kcp", + Kind: "Secret", + Reference: syncagentv1alpha1.RelatedResourceReference{ + Name: syncagentv1alpha1.ResourceLocator{ + Path: "spec.test.name", + }, + Namespace: &syncagentv1alpha1.ResourceLocator{ + Path: "spec.test.namespace", + }, + }, + }, + }, + }, + } + + if err := envtestClient.Create(ctx, prCrontabs); err != nil { + t.Fatalf("Failed to create PublishedResource: %v", err) + } + + // let the agent do its thing + utils.RunAgent(ctx, t, "bob", orgKubconfig, envtestKubeconfig, apiExportName) + + // wait for the APIExport to be updated + expectedClaims := []kcpapisv1alpha1.PermissionClaim{ + { + GroupResource: kcpapisv1alpha1.GroupResource{ + Group: "", + Resource: "configmaps", + }, + All: true, + }, + { + GroupResource: kcpapisv1alpha1.GroupResource{ + Group: "", + Resource: "namespaces", + }, + All: true, + }, + { + GroupResource: kcpapisv1alpha1.GroupResource{ + Group: "", + Resource: "secrets", + }, + All: true, + }, + } + + t.Logf("Waiting for APIExport to be updated…") + err := wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 1*time.Minute, false, func(ctx context.Context) (done bool, err error) { + err = orgClient.Get(ctx, apiExportKey, apiExport) + if err != nil { + return false, err + } + + // Do not use cmp.Equal() because the Equal() func on PermissionClaims does not check all fields. + return equality.Semantic.DeepEqual(expectedClaims, apiExport.Spec.PermissionClaims), nil + }) + if err != nil { + t.Fatalf("Failed to wait for APIExport to be updated: %v", err) + } +} diff --git a/test/e2e/apiresourceschema/apiresourceschema_test.go b/test/e2e/apiresourceschema/apiresourceschema_test.go index f55f7a0..080d3d8 100644 --- a/test/e2e/apiresourceschema/apiresourceschema_test.go +++ b/test/e2e/apiresourceschema/apiresourceschema_test.go @@ -24,12 +24,14 @@ import ( "time" "github.com/go-logr/logr" + "github.com/google/go-cmp/cmp" syncagentv1alpha1 "github.com/kcp-dev/api-syncagent/sdk/apis/syncagent/v1alpha1" "github.com/kcp-dev/api-syncagent/test/utils" kcpapisv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" @@ -106,3 +108,272 @@ func TestARSAreCreated(t *testing.T) { t.Fatalf("APIResourceSchema does not exist: %v", err) } } + +func TestARSAreNotUpdated(t *testing.T) { + const ( + apiExportName = "example.com" + ) + + ctx := context.Background() + ctrlruntime.SetLogger(logr.Discard()) + + // setup a test environment in kcp + orgKubconfig := utils.CreateOrganization(t, ctx, "ars-are-not-updated", apiExportName) + + // start a service cluster + envtestKubeconfig, envtestClient, _ := utils.RunEnvtest(t, []string{ + "test/crds/crontab.yaml", + }) + + // publish Crontabs + t.Logf("Publishing CronTabs…") + pr := &syncagentv1alpha1.PublishedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "publish-crontabs", + }, + Spec: syncagentv1alpha1.PublishedResourceSpec{ + Resource: syncagentv1alpha1.SourceResourceDescriptor{ + APIGroup: "example.com", + Version: "v1", + Kind: "CronTab", + }, + }, + } + + if err := envtestClient.Create(ctx, pr); err != nil { + t.Fatalf("Failed to create PublishedResource: %v", err) + } + + // let the agent do its thing + utils.RunAgent(ctx, t, "bob", orgKubconfig, envtestKubeconfig, apiExportName) + + // wait for the APIExport to be updated + t.Logf("Waiting for APIExport to be updated…") + orgClient := utils.GetClient(t, orgKubconfig) + apiExportKey := types.NamespacedName{Name: apiExportName} + + var arsName string + err := wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 1*time.Minute, false, func(ctx context.Context) (done bool, err error) { + apiExport := &kcpapisv1alpha1.APIExport{} + err = orgClient.Get(ctx, apiExportKey, apiExport) + if err != nil { + return false, err + } + + if len(apiExport.Spec.LatestResourceSchemas) == 0 { + return false, nil + } + + arsName = apiExport.Spec.LatestResourceSchemas[0] + + return true, nil + }) + if err != nil { + t.Fatalf("Failed to wait for APIExport to be updated: %v", err) + } + + // update the CRD + t.Logf("Updating CRD (same version, but new schema)…") + utils.ApplyCRD(t, ctx, envtestClient, "test/crds/crontab-improved.yaml") + + // give the agent some time to do nothing + time.Sleep(3 * time.Second) + + // validate that the APIExport has *not* changed + apiExport := &kcpapisv1alpha1.APIExport{} + err = orgClient.Get(ctx, apiExportKey, apiExport) + if err != nil { + t.Fatalf("APIExport disappeared: %v", err) + } + + if l := len(apiExport.Spec.LatestResourceSchemas); l != 1 { + t.Fatalf("APIExport should still have 1 resource schema, but has %d.", l) + } + + if currentName := apiExport.Spec.LatestResourceSchemas[0]; currentName != arsName { + t.Fatalf("APIExport should still refer to the original ARS %q, but now contains %q.", arsName, currentName) + } +} + +func TestARSDropsAllVersionsExceptTheSelectedOne(t *testing.T) { + const ( + apiExportName = "example.com" + theVersion = "v1" + ) + + ctx := context.Background() + ctrlruntime.SetLogger(logr.Discard()) + + // setup a test environment in kcp + orgKubconfig := utils.CreateOrganization(t, ctx, "ars-drops-crd-versions", apiExportName) + + // start a service cluster + envtestKubeconfig, envtestClient, _ := utils.RunEnvtest(t, []string{ + "test/crds/crontab-multi-versions.yaml", + }) + + // publish Crontabs + t.Logf("Publishing CronTabs…") + pr := &syncagentv1alpha1.PublishedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "publish-crontabs", + }, + Spec: syncagentv1alpha1.PublishedResourceSpec{ + Resource: syncagentv1alpha1.SourceResourceDescriptor{ + APIGroup: "example.com", + Version: theVersion, + Kind: "CronTab", + }, + }, + } + + if err := envtestClient.Create(ctx, pr); err != nil { + t.Fatalf("Failed to create PublishedResource: %v", err) + } + + // let the agent do its thing + utils.RunAgent(ctx, t, "bob", orgKubconfig, envtestKubeconfig, apiExportName) + + // wait for the APIExport to be updated + t.Logf("Waiting for APIExport to be updated…") + orgClient := utils.GetClient(t, orgKubconfig) + apiExportKey := types.NamespacedName{Name: apiExportName} + + var arsName string + err := wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 1*time.Minute, false, func(ctx context.Context) (done bool, err error) { + apiExport := &kcpapisv1alpha1.APIExport{} + err = orgClient.Get(ctx, apiExportKey, apiExport) + if err != nil { + return false, err + } + + if len(apiExport.Spec.LatestResourceSchemas) == 0 { + return false, nil + } + + arsName = apiExport.Spec.LatestResourceSchemas[0] + + return true, nil + }) + if err != nil { + t.Fatalf("Failed to wait for APIExport to be updated: %v", err) + } + + // check the APIResourceSchema + ars := &kcpapisv1alpha1.APIResourceSchema{} + err = orgClient.Get(ctx, types.NamespacedName{Name: arsName}, ars) + if err != nil { + t.Fatalf("APIResourceSchema does not exist: %v", err) + } + + if len(ars.Spec.Versions) != 1 { + t.Fatalf("Expected only one version to remain in ARS, but found %d.", len(ars.Spec.Versions)) + } + + if name := ars.Spec.Versions[0].Name; name != theVersion { + t.Fatalf("Expected ARS to contain %q, but contains %q.", theVersion, name) + } +} + +func TestProjection(t *testing.T) { + const ( + apiExportName = "example.com" + originalVersion = "v1" + ) + + ctx := context.Background() + ctrlruntime.SetLogger(logr.Discard()) + + // setup a test environment in kcp + orgKubconfig := utils.CreateOrganization(t, ctx, "ars-projections", apiExportName) + + // start a service cluster + envtestKubeconfig, envtestClient, _ := utils.RunEnvtest(t, []string{ + "test/crds/crontab.yaml", + }) + + // publish Crontabs + t.Logf("Publishing CronTabs…") + pr := &syncagentv1alpha1.PublishedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "publish-crontabs", + }, + Spec: syncagentv1alpha1.PublishedResourceSpec{ + Resource: syncagentv1alpha1.SourceResourceDescriptor{ + APIGroup: "example.com", + Version: originalVersion, + Kind: "CronTab", + }, + Projection: &syncagentv1alpha1.ResourceProjection{ + Version: "v6", + Scope: syncagentv1alpha1.ClusterScoped, + Kind: "CronusTabulatus", + Plural: "cronustabulati", + ShortNames: []string{"cront"}, + }, + }, + } + + if err := envtestClient.Create(ctx, pr); err != nil { + t.Fatalf("Failed to create PublishedResource: %v", err) + } + + // let the agent do its thing + utils.RunAgent(ctx, t, "bob", orgKubconfig, envtestKubeconfig, apiExportName) + + // wait for the APIExport to be updated + t.Logf("Waiting for APIExport to be updated…") + orgClient := utils.GetClient(t, orgKubconfig) + apiExportKey := types.NamespacedName{Name: apiExportName} + + var arsName string + err := wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 1*time.Minute, false, func(ctx context.Context) (done bool, err error) { + apiExport := &kcpapisv1alpha1.APIExport{} + err = orgClient.Get(ctx, apiExportKey, apiExport) + if err != nil { + return false, err + } + + if len(apiExport.Spec.LatestResourceSchemas) == 0 { + return false, nil + } + + arsName = apiExport.Spec.LatestResourceSchemas[0] + + return true, nil + }) + if err != nil { + t.Fatalf("Failed to wait for APIExport to be updated: %v", err) + } + + // check the APIResourceSchema + ars := &kcpapisv1alpha1.APIResourceSchema{} + err = orgClient.Get(ctx, types.NamespacedName{Name: arsName}, ars) + if err != nil { + t.Fatalf("APIResourceSchema does not exist: %v", err) + } + + if len(ars.Spec.Versions) != 1 { + t.Fatalf("Expected only one version to remain in ARS, but found %d.", len(ars.Spec.Versions)) + } + + if name := ars.Spec.Versions[0].Name; name != pr.Spec.Projection.Version { + t.Errorf("Expected ARS to contain version %q, but contains %q.", pr.Spec.Projection.Version, name) + } + + if ars.Spec.Scope != apiextensionsv1.ResourceScope(pr.Spec.Projection.Scope) { + t.Errorf("Expected ARS to be of scope %q, but is %q.", pr.Spec.Projection.Scope, ars.Spec.Scope) + } + + if ars.Spec.Names.Kind != pr.Spec.Projection.Kind { + t.Errorf("Expected ARS to be kind %q, but is %q.", pr.Spec.Projection.Kind, ars.Spec.Names.Kind) + } + + if ars.Spec.Names.Plural != pr.Spec.Projection.Plural { + t.Errorf("Expected ARS to have plural name %q, but has %q.", pr.Spec.Projection.Plural, ars.Spec.Names.Plural) + } + + if !cmp.Equal(ars.Spec.Names.ShortNames, pr.Spec.Projection.ShortNames) { + t.Errorf("Expected ARS to have short names %v, but has %v.", pr.Spec.Projection.ShortNames, ars.Spec.Names.ShortNames) + } +} diff --git a/test/utils/fixtures.go b/test/utils/fixtures.go index 8929411..4cc6cf2 100644 --- a/test/utils/fixtures.go +++ b/test/utils/fixtures.go @@ -19,6 +19,8 @@ package utils import ( "context" "fmt" + "os" + "path/filepath" "strings" "testing" "time" @@ -32,8 +34,10 @@ import ( "github.com/kcp-dev/kcp/sdk/apis/third_party/conditions/util/conditions" rbacv1 "k8s.io/api/rbac/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/util/yaml" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/kontext" ) @@ -301,3 +305,42 @@ func BindToAPIExport(t *testing.T, ctx context.Context, client ctrlruntimeclient return apiBinding } + +func ApplyCRD(t *testing.T, ctx context.Context, client ctrlruntimeclient.Client, filename string) { + t.Helper() + + crd := loadCRD(t, filename) + + existingCRD := &apiextensionsv1.CustomResourceDefinition{} + if err := client.Get(ctx, ctrlruntimeclient.ObjectKeyFromObject(crd), existingCRD); err != nil { + if err := client.Create(ctx, crd); err != nil { + t.Fatalf("Failed to create CRD: %v", err) + } + } else { + existingCRD.Spec = crd.Spec + + if err := client.Update(ctx, existingCRD); err != nil { + t.Fatalf("Failed to update CRD: %v", err) + } + } +} + +func loadCRD(t *testing.T, filename string) *apiextensionsv1.CustomResourceDefinition { + t.Helper() + + rootDirectory := requiredEnv(t, "ROOT_DIRECTORY") + + f, err := os.Open(filepath.Join(rootDirectory, filename)) + if err != nil { + t.Fatalf("Failed to read CRD: %v", err) + } + defer f.Close() + + crd := &apiextensionsv1.CustomResourceDefinition{} + dec := yaml.NewYAMLOrJSONDecoder(f, 1024) + if err := dec.Decode(crd); err != nil { + t.Fatalf("Failed to decode CRD: %v", err) + } + + return crd +} diff --git a/test/utils/process.go b/test/utils/process.go index d8be5fa..df4a4da 100644 --- a/test/utils/process.go +++ b/test/utils/process.go @@ -88,6 +88,8 @@ func RunAgent( "--kcp-kubeconfig", kcpKubeconfig, "--namespace", "kube-system", "--log-format", "Console", + "--health-address", "0", + "--metrics-address", "0", } logFile := filepath.Join(ArtifactsDirectory(t), uniqueLogfile(t, "")) diff --git a/test/utils/utils.go b/test/utils/utils.go index 054c89c..4811c4e 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -30,6 +30,7 @@ import ( corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/scale/scheme" "k8s.io/client-go/tools/clientcmd" @@ -60,6 +61,7 @@ func newScheme(t *testing.T) *runtime.Scheme { must(t, kcptenancyv1alpha1.AddToScheme(sc)) must(t, kcpapisv1alpha1.AddToScheme(sc)) must(t, syncagentv1alpha1.AddToScheme(sc)) + must(t, apiextensionsv1.AddToScheme(sc)) return sc }