Skip to content

Commit c513691

Browse files
kcp-ci-botkcp-ci-bot
authored
Merge pull request #24 from kcp-dev/more-e2e-tests
🌱 Add integration tests for the apiexport controller
2 parents 242e2e6 + 75dbdcf commit c513691

File tree

13 files changed

+829
-19
lines changed

13 files changed

+829
-19
lines changed

‎cmd/api-syncagent/main.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ import (
5353
"sigs.k8s.io/controller-runtime/pkg/cluster"
5454
ctrlruntimelog "sigs.k8s.io/controller-runtime/pkg/log"
5555
"sigs.k8s.io/controller-runtime/pkg/manager"
56+
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
5657
)
5758

5859
func main() {
@@ -168,9 +169,11 @@ func setupLocalManager(ctx context.Context, opts *Options) (manager.Manager, err
168169
BaseContext: func() context.Context {
169170
return ctx
170171
},
172+
Metrics: metricsserver.Options{BindAddress: opts.MetricsAddr},
171173
LeaderElection: opts.EnableLeaderElection,
172174
LeaderElectionID: "syncagent." + opts.AgentName,
173175
LeaderElectionNamespace: opts.Namespace,
176+
HealthProbeBindAddress: opts.HealthAddr,
174177
})
175178
if err != nil {
176179
return nil, err

‎cmd/api-syncagent/options.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,12 +65,16 @@ type Options struct {
6565
KubeconfigCAFileOverride string
6666

6767
LogOptions log.Options
68+
69+
MetricsAddr string
70+
HealthAddr string
6871
}
6972

7073
func NewOptions() *Options {
7174
return &Options{
7275
LogOptions: log.NewDefaultOptions(),
7376
PublishedResourceSelector: labels.Everything(),
77+
MetricsAddr: "127.0.0.1:8085",
7478
}
7579
}
7680

@@ -83,8 +87,10 @@ func (o *Options) AddFlags(flags *pflag.FlagSet) {
8387
flags.StringVar(&o.APIExportRef, "apiexport-ref", o.APIExportRef, "name of the APIExport in kcp that this Sync Agent is powering")
8488
flags.StringVar(&o.PublishedResourceSelectorString, "published-resource-selector", o.PublishedResourceSelectorString, "restrict this Sync Agent to only process PublishedResources matching this label selector (optional)")
8589
flags.BoolVar(&o.EnableLeaderElection, "enable-leader-election", o.EnableLeaderElection, "whether to perform leader election")
86-
flags.StringVar(&o.KubeconfigHostOverride, "kubeconfig-host-override", o.KubeconfigHostOverride, "Override the host configured in the local kubeconfig")
87-
flags.StringVar(&o.KubeconfigCAFileOverride, "kubeconfig-ca-file-override", o.KubeconfigCAFileOverride, "Override the server CA file configured in the local kubeconfig")
90+
flags.StringVar(&o.KubeconfigHostOverride, "kubeconfig-host-override", o.KubeconfigHostOverride, "override the host configured in the local kubeconfig")
91+
flags.StringVar(&o.KubeconfigCAFileOverride, "kubeconfig-ca-file-override", o.KubeconfigCAFileOverride, "override the server CA file configured in the local kubeconfig")
92+
flags.StringVar(&o.MetricsAddr, "metrics-address", o.MetricsAddr, "host and port to serve Prometheus metrics via /metrics (HTTP)")
93+
flags.StringVar(&o.HealthAddr, "health-address", o.HealthAddr, "host and port to serve probes via /readyz and /healthz (HTTP)")
8894
}
8995

9096
func (o *Options) Validate() error {

‎docs/faq.md‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,3 +36,19 @@ Only those required for its own operation. If you configure a namespaced resourc
3636
automatically add a claim for `namespaces` in kcp, plus it will add either `configmaps` or `secrets`
3737
if related resources are configured in a `PublishedResource`. But you cannot specify additional
3838
permissions claims.
39+
40+
## I am seeing errors in the agent logs, what's going on?
41+
42+
Errors like
43+
44+
> reflector.go:561] k8s.io/[email protected]/tools/cache/reflector.go:243: failed to list
45+
> example.com/v1, Kind=Dummy: the server could not find the requested resource
46+
47+
or
48+
49+
> reflector.go:158] "Unhandled Error" err="k8s.io/[email protected]/tools/cache/reflector.go:243:
50+
> Failed to watch kcp.example.com/v1, Kind=Dummy: failed to list kcp.example.com/v1, Kind=Dummy:
51+
> the server could not find the requested resource" logger="UnhandledError"
52+
53+
are typical when bootstrapping new APIExports in kcp. They are only cause for concern if they
54+
persist after configuring all PublishedResources.

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ require (
77
github.com/evanphx/json-patch/v5 v5.9.0
88
github.com/go-logr/logr v1.4.2
99
github.com/go-logr/zapr v1.3.0
10+
github.com/google/go-cmp v0.6.0
1011
github.com/kcp-dev/apimachinery/v2 v2.0.1-0.20240817110845-a9eb9752bfeb
1112
github.com/kcp-dev/client-go v0.0.0-20240912145314-f5949d81732a
1213
github.com/kcp-dev/code-generator/v2 v2.3.1
@@ -53,7 +54,6 @@ require (
5354
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
5455
github.com/golang/protobuf v1.5.4 // indirect
5556
github.com/google/gnostic-models v0.6.8 // indirect
56-
github.com/google/go-cmp v0.6.0 // indirect
5757
github.com/google/gofuzz v1.2.1-0.20210504230335-f78f29fc09ea // indirect
5858
github.com/google/uuid v1.6.0 // indirect
5959
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect

‎internal/controller/apiresourceschema/controller.go‎

Lines changed: 32 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,10 @@ func (r *Reconciler) reconcile(ctx context.Context, log *zap.SugaredLogger, pubR
133133
}
134134

135135
// project the CRD
136-
projectedCRD := r.projectResourceNames(r.apiExportName, crd, pubResource.Spec.Projection)
136+
projectedCRD, err := r.applyProjection(r.apiExportName, crd, pubResource)
137+
if err != nil {
138+
return nil, fmt.Errorf("failed to apply projection rules: %w", err)
139+
}
137140

138141
// to prevent changing the source GVK e.g. from "apps/v1 Daemonset" to "core/v1 Pod",
139142
// we include the source GVK in hashed form in the final APIResourceSchema name.
@@ -147,7 +150,7 @@ func (r *Reconciler) reconcile(ctx context.Context, log *zap.SugaredLogger, pubR
147150
err = r.kcpClient.Get(wsCtx, types.NamespacedName{Name: arsName}, ars, &ctrlruntimeclient.GetOptions{})
148151

149152
if apierrors.IsNotFound(err) {
150-
if err := r.createAPIResourceSchema(wsCtx, log, r.apiExportName, projectedCRD, arsName, pubResource.Spec.Resource.Version); err != nil {
153+
if err := r.createAPIResourceSchema(wsCtx, log, r.apiExportName, projectedCRD, arsName); err != nil {
151154
return nil, fmt.Errorf("failed to create APIResourceSchema: %w", err)
152155
}
153156
} else if err != nil {
@@ -170,17 +173,7 @@ func (r *Reconciler) reconcile(ctx context.Context, log *zap.SugaredLogger, pubR
170173
return nil, nil
171174
}
172175

173-
func (r *Reconciler) createAPIResourceSchema(ctx context.Context, log *zap.SugaredLogger, apigroup string, projectedCRD *apiextensionsv1.CustomResourceDefinition, arsName string, selectedVersion string) error {
174-
// At this moment we ignore every non-selected version in the CRD, as we have not fully
175-
// decided on how to support the API version lifecycle yet. Having multiple versions in
176-
// the CRD will make kcp require a `conversion` to also be configured. Since we cannot
177-
// enforce that and want to instead work with existing CRDs as best as we can, we chose
178-
// this option (instead of error'ing out if a conversion is missing).
179-
projectedCRD.Spec.Conversion = nil
180-
projectedCRD.Spec.Versions = slices.DeleteFunc(projectedCRD.Spec.Versions, func(v apiextensionsv1.CustomResourceDefinitionVersion) bool {
181-
return v.Name != selectedVersion
182-
})
183-
176+
func (r *Reconciler) createAPIResourceSchema(ctx context.Context, log *zap.SugaredLogger, apigroup string, projectedCRD *apiextensionsv1.CustomResourceDefinition, arsName string) error {
184177
// prefix is irrelevant as the reconciling framework will use arsName anyway
185178
converted, err := kcpdevv1alpha1.CRDToAPIResourceSchema(projectedCRD, "irrelevant")
186179
if err != nil {
@@ -206,12 +199,35 @@ func (r *Reconciler) createAPIResourceSchema(ctx context.Context, log *zap.Sugar
206199
return r.kcpClient.Create(ctx, ars)
207200
}
208201

209-
func (r *Reconciler) projectResourceNames(apiGroup string, crd *apiextensionsv1.CustomResourceDefinition, projection *syncagentv1alpha1.ResourceProjection) *apiextensionsv1.CustomResourceDefinition {
202+
func (r *Reconciler) applyProjection(apiGroup string, crd *apiextensionsv1.CustomResourceDefinition, pr *syncagentv1alpha1.PublishedResource) (*apiextensionsv1.CustomResourceDefinition, error) {
210203
result := crd.DeepCopy()
211204
result.Spec.Group = apiGroup
212205

206+
// At this moment we ignore every non-selected version in the CRD, as we have not fully
207+
// decided on how to support the API version lifecycle yet. Having multiple versions in
208+
// the CRD will make kcp require a `conversion` to also be configured. Since we cannot
209+
// enforce that and want to instead work with existing CRDs as best as we can, we chose
210+
// this option (instead of error'ing out if a conversion is missing).
211+
result.Spec.Conversion = nil
212+
result.Spec.Versions = slices.DeleteFunc(result.Spec.Versions, func(v apiextensionsv1.CustomResourceDefinitionVersion) bool {
213+
return v.Name != pr.Spec.Resource.Version
214+
})
215+
216+
if len(result.Spec.Versions) != 1 {
217+
// This should never happen because of checks earlier in the reconciler.
218+
return nil, fmt.Errorf("invalid CRD: cannot find selected version %q", pr.Spec.Resource.Version)
219+
}
220+
221+
result.Spec.Versions[0].Served = true
222+
result.Spec.Versions[0].Storage = true
223+
224+
projection := pr.Spec.Projection
213225
if projection == nil {
214-
return result
226+
return result, nil
227+
}
228+
229+
if projection.Version != "" {
230+
result.Spec.Versions[0].Name = projection.Version
215231
}
216232

217233
if projection.Kind != "" {
@@ -238,7 +254,7 @@ func (r *Reconciler) projectResourceNames(apiGroup string, crd *apiextensionsv1.
238254
result.Spec.Names.ShortNames = projection.ShortNames
239255
}
240256

241-
return result
257+
return result, nil
242258
}
243259

244260
// getAPIResourceSchemaName generates the name for the ARS in kcp. Note that

‎test/crds/backup.yaml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# sourced from https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/
2+
apiVersion: apiextensions.k8s.io/v1
3+
kind: CustomResourceDefinition
4+
metadata:
5+
name: backups.eksempel.no
6+
spec:
7+
group: eksempel.no
8+
scope: Namespaced
9+
names:
10+
plural: backups
11+
singular: backup
12+
kind: Backup
13+
versions:
14+
- name: v1
15+
served: true
16+
storage: true
17+
schema:
18+
openAPIV3Schema:
19+
type: object
20+
properties:
21+
spec:
22+
type: object
23+
properties:
24+
source:
25+
type: string
26+
destination:
27+
type: string

‎test/crds/crontab-improved.yaml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
apiVersion: apiextensions.k8s.io/v1
2+
kind: CustomResourceDefinition
3+
metadata:
4+
name: crontabs.example.com
5+
spec:
6+
group: example.com
7+
scope: Namespaced
8+
names:
9+
plural: crontabs
10+
singular: crontab
11+
kind: CronTab
12+
shortNames:
13+
- ct
14+
versions:
15+
- name: v1
16+
served: true
17+
storage: true
18+
schema:
19+
openAPIV3Schema:
20+
type: object
21+
properties:
22+
spec:
23+
type: object
24+
properties:
25+
cronSpec:
26+
type: string
27+
image:
28+
type: string
29+
command:
30+
type: string
31+
replicas:
32+
type: integer

‎test/crds/crontab-multi-versions.yaml‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
# sourced from https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/
2+
apiVersion: apiextensions.k8s.io/v1
3+
kind: CustomResourceDefinition
4+
metadata:
5+
name: crontabs.example.com
6+
spec:
7+
group: example.com
8+
scope: Namespaced
9+
names:
10+
plural: crontabs
11+
singular: crontab
12+
kind: CronTab
13+
shortNames:
14+
- ct
15+
versions:
16+
- name: v1
17+
served: true
18+
storage: false
19+
schema:
20+
openAPIV3Schema:
21+
type: object
22+
properties:
23+
spec:
24+
type: object
25+
properties:
26+
cronSpec:
27+
type: string
28+
image:
29+
type: string
30+
replicas:
31+
type: integer
32+
- name: v2
33+
served: true
34+
storage: true
35+
schema:
36+
openAPIV3Schema:
37+
type: object
38+
properties:
39+
spec:
40+
type: object
41+
properties:
42+
cronSpec:
43+
type: string
44+
image:
45+
type: string
46+
replicas:
47+
type: integer

0 commit comments

Comments
 (0)