Skip to content

Commit 02f8282

Browse files
fourthislefourthisle
authored
Bugfix: Add separate APISchema Resolver implementation for standard k8s clusters (#69)
* refactor apischema.Resolver and add prereconcile step for standard k8s clusters * cleanup duplicate code * fix obsolete naming * rm helper files * improve the CRD GVK lookup logic
1 parent 2a29944 commit 02f8282

File tree

4 files changed

+227
-106
lines changed

4 files changed

+227
-106
lines changed

‎listener/apischema/crd_resolver.go

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
package apischema
2+
3+
import (
4+
"encoding/json"
5+
"errors"
6+
"fmt"
7+
"maps"
8+
"slices"
9+
"strings"
10+
11+
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
12+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
13+
"k8s.io/client-go/discovery"
14+
"k8s.io/client-go/openapi"
15+
"k8s.io/kube-openapi/pkg/validation/spec"
16+
)
17+
18+
var (
19+
ErrInvalidPath = errors.New("path doesn't contain the / separator")
20+
ErrNotPreferred = errors.New("path ApiGroup does not belong to the server preferred APIs")
21+
ErrGVKNotPreferred = errors.New("failed to find CRD GVK in API preferred resources")
22+
)
23+
24+
type CRDResolver struct {
25+
*discovery.DiscoveryClient
26+
}
27+
28+
func (cr *CRDResolver) Resolve() ([]byte, error) {
29+
return resolveSchema(cr.DiscoveryClient)
30+
}
31+
32+
func (cr *CRDResolver) ResolveApiSchema(crd *apiextensionsv1.CustomResourceDefinition) ([]byte, error) {
33+
gvk, err := getCRDGroupVersionKind(crd.Spec)
34+
if err != nil {
35+
return nil, fmt.Errorf("failed to get CRD GVK: %w", err)
36+
}
37+
38+
apiResLists, err := cr.ServerPreferredResources()
39+
if err != nil {
40+
return nil, fmt.Errorf("failed to get server preferred resources: %w", err)
41+
}
42+
43+
preferredApiGroups, err := errorIfCRDNotInPreferredApiGroups(gvk, apiResLists)
44+
if err != nil {
45+
return nil, fmt.Errorf("failed to filter server preferred resources: %w", err)
46+
}
47+
48+
return resolveForPaths(cr.OpenAPIV3(), preferredApiGroups)
49+
}
50+
51+
func errorIfCRDNotInPreferredApiGroups(gvk *metav1.GroupVersionKind, apiResLists []*metav1.APIResourceList) ([]string, error) {
52+
targetGV := gvk.Group + "/" + gvk.Version
53+
isGVFound := false
54+
preferredApiGroups := make([]string, 0, len(apiResLists))
55+
for _, apiResources := range apiResLists {
56+
gv := apiResources.GroupVersion
57+
isGVFound = gv == targetGV
58+
if isGVFound && !isCRDKindIncluded(gvk, apiResources) {
59+
return nil, ErrGVKNotPreferred
60+
}
61+
preferredApiGroups = append(preferredApiGroups, gv)
62+
}
63+
64+
if !isGVFound {
65+
return nil, ErrGVKNotPreferred
66+
}
67+
return preferredApiGroups, nil
68+
}
69+
70+
func isCRDKindIncluded(gvk *metav1.GroupVersionKind, apiResources *metav1.APIResourceList) bool {
71+
for _, res := range apiResources.APIResources {
72+
if res.Kind == gvk.Kind {
73+
return true
74+
}
75+
}
76+
return false
77+
}
78+
79+
func getCRDGroupVersionKind(spec apiextensionsv1.CustomResourceDefinitionSpec) (*metav1.GroupVersionKind, error) {
80+
for _, v := range spec.Versions {
81+
if v.Storage {
82+
return &metav1.GroupVersionKind{
83+
Group: spec.Group,
84+
Version: v.Name,
85+
Kind: spec.Names.Kind,
86+
}, nil
87+
}
88+
}
89+
return nil, errors.New("failed to find storage version")
90+
}
91+
92+
func getSchemaForPath(preferredApiGroups []string, path string, gv openapi.GroupVersion) (map[string]*spec.Schema, error) {
93+
if !strings.Contains(path, separator) {
94+
return nil, ErrInvalidPath
95+
}
96+
pathApiGroupArray := strings.Split(path, separator)
97+
pathApiGroup := strings.Join(pathApiGroupArray[1:], separator)
98+
// filer out apiGroups that aren't in the preferred list
99+
if !slices.Contains(preferredApiGroups, pathApiGroup) {
100+
return nil, ErrNotPreferred
101+
}
102+
103+
b, err := gv.Schema(discovery.AcceptV1)
104+
if err != nil {
105+
return nil, fmt.Errorf("failed to get schema for path %s :%w", path, err)
106+
}
107+
108+
resp := &schemaResponse{}
109+
if err := json.Unmarshal(b, resp); err != nil {
110+
return nil, fmt.Errorf("failed to unmarshal schema for path %s :%w", path, err)
111+
}
112+
return resp.Components.Schemas, nil
113+
}
114+
115+
func resolveForPaths(oc openapi.Client, preferredApiGroups []string) ([]byte, error) {
116+
apiv3Paths, err := oc.Paths()
117+
if err != nil {
118+
return nil, fmt.Errorf("failed to get OpenAPI paths: %w", err)
119+
}
120+
121+
schemas := make(map[string]*spec.Schema)
122+
for path, gv := range apiv3Paths {
123+
schema, err := getSchemaForPath(preferredApiGroups, path, gv)
124+
if err != nil {
125+
//TODO: debug log?
126+
continue
127+
}
128+
maps.Copy(schemas, schema)
129+
}
130+
v3JSON, err := json.Marshal(&schemaResponse{
131+
Components: schemasComponentsWrapper{
132+
Schemas: schemas,
133+
},
134+
})
135+
if err != nil {
136+
return nil, fmt.Errorf("failed to marshal openAPI v3 schema: %w", err)
137+
}
138+
v2JSON, err := ConvertJSON(v3JSON)
139+
if err != nil {
140+
return nil, fmt.Errorf("failed to convert openAPI v3 schema to v2: %w", err)
141+
}
142+
143+
return v2JSON, nil
144+
}
145+
146+
func resolveSchema(dc discovery.DiscoveryInterface) ([]byte, error) {
147+
preferredApiGroups := []string{}
148+
apiResList, err := dc.ServerPreferredResources()
149+
if err != nil {
150+
return nil, fmt.Errorf("failed to get server preferred resources: %w", err)
151+
}
152+
for _, apiRes := range apiResList {
153+
preferredApiGroups = append(preferredApiGroups, apiRes.GroupVersion)
154+
}
155+
156+
return resolveForPaths(dc.OpenAPIV3(), preferredApiGroups)
157+
}

‎listener/apischema/resolver.go

Lines changed: 1 addition & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,8 @@
11
package apischema
22

33
import (
4-
"encoding/json"
5-
"fmt"
6-
"maps"
7-
"slices"
8-
"strings"
9-
10-
//nolint:staticcheck // SA1019 Keep using module since it's still being maintained and the api of google.golang.org/protobuf/proto differs
11-
124
"k8s.io/client-go/discovery"
135
"k8s.io/kube-openapi/pkg/validation/spec"
14-
// "github.com/getkin/kin-openapi/openapi2conv"
15-
// "github.com/getkin/kin-openapi/openapi3"
166
)
177

188
const (
@@ -39,58 +29,5 @@ type ResolverImpl struct {
3929
}
4030

4131
func (r *ResolverImpl) Resolve(dc discovery.DiscoveryInterface) ([]byte, error) {
42-
preferredApiGroups := []string{}
43-
apiResList, err := dc.ServerPreferredResources()
44-
if err != nil {
45-
return nil, fmt.Errorf("failed to get server preferred resources: %w", err)
46-
}
47-
for _, apiRes := range apiResList {
48-
preferredApiGroups = append(preferredApiGroups, apiRes.GroupVersion)
49-
}
50-
51-
apiv3Paths, err := dc.OpenAPIV3().Paths()
52-
if err != nil {
53-
return nil, fmt.Errorf("failed to get OpenAPI paths: %w", err)
54-
}
55-
56-
schemas := make(map[string]*spec.Schema)
57-
for key, path := range apiv3Paths {
58-
if !strings.Contains(key, separator) {
59-
continue
60-
}
61-
pathApiGroupArray := strings.Split(key, separator)
62-
pathApiGroup := strings.Join(pathApiGroupArray[1:], separator)
63-
// filer out apiGroups that aren't in the preferred list
64-
if !slices.Contains(preferredApiGroups, pathApiGroup) {
65-
continue
66-
}
67-
68-
b, err := path.Schema(discovery.AcceptV1)
69-
if err != nil {
70-
//TODO: debug log?
71-
continue
72-
}
73-
74-
resp := &schemaResponse{}
75-
err = json.Unmarshal(b, resp)
76-
if err != nil {
77-
//TODO: debug log?
78-
continue
79-
}
80-
maps.Copy(schemas, resp.Components.Schemas)
81-
}
82-
v3JSON, err := json.Marshal(&schemaResponse{
83-
Components: schemasComponentsWrapper{
84-
Schemas: schemas,
85-
},
86-
})
87-
if err != nil {
88-
return nil, fmt.Errorf("failed to marshal openAPI v3 schema: %w", err)
89-
}
90-
v2JSON, err := ConvertJSON(v3JSON)
91-
if err != nil {
92-
return nil, fmt.Errorf("failed to convert openAPI v3 schema to v2: %w", err)
93-
}
94-
95-
return v2JSON, nil
32+
return resolveSchema(dc)
9633
}

‎listener/controller/crd_controller.go

Lines changed: 44 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,13 @@ package controller
33
import (
44
"bytes"
55
"context"
6-
"errors"
7-
8-
"io/fs"
6+
"fmt"
97

108
"github.com/openmfp/crd-gql-gateway/listener/apischema"
119
"github.com/openmfp/crd-gql-gateway/listener/workspacefile"
12-
"k8s.io/client-go/discovery"
1310

1411
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
12+
apierrors "k8s.io/apimachinery/pkg/api/errors"
1513
ctrl "sigs.k8s.io/controller-runtime"
1614
"sigs.k8s.io/controller-runtime/pkg/client"
1715
"sigs.k8s.io/controller-runtime/pkg/log"
@@ -21,23 +19,20 @@ import (
2119
type CRDReconciler struct {
2220
ClusterName string
2321
client.Client
24-
*discovery.DiscoveryClient
22+
*apischema.CRDResolver
2523
io workspacefile.IOHandler
26-
sc apischema.Resolver
2724
}
2825

2926
func NewCRDReconciler(name string,
3027
clt client.Client,
31-
dc *discovery.DiscoveryClient,
28+
cr *apischema.CRDResolver,
3229
io workspacefile.IOHandler,
33-
sc apischema.Resolver,
3430
) *CRDReconciler {
3531
return &CRDReconciler{
36-
ClusterName: name,
37-
Client: clt,
38-
DiscoveryClient: dc,
39-
io: io,
40-
sc: sc,
32+
ClusterName: name,
33+
Client: clt,
34+
CRDResolver: cr,
35+
io: io,
4136
}
4237
}
4338

@@ -49,49 +44,57 @@ func (r *CRDReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R
4944
logger.Info("starting reconciliation...")
5045

5146
crd := &apiextensionsv1.CustomResourceDefinition{}
52-
if err := r.Client.Get(ctx, req.NamespacedName, crd); client.IgnoreNotFound(err) != nil {
47+
err := r.Client.Get(ctx, req.NamespacedName, crd)
48+
if apierrors.IsNotFound(err) {
49+
logger.Info("resource not found, updating schema...")
50+
return ctrl.Result{}, r.updateAPISchema()
51+
}
52+
if client.IgnoreNotFound(err) != nil {
5353
logger.Error(err, "failed to get reconciled object")
5454
return ctrl.Result{}, err
5555
}
5656

57+
return ctrl.Result{}, r.updateAPISchemaWith(crd)
58+
}
59+
60+
// SetupWithManager sets up the controller with the Manager.
61+
func (r *CRDReconciler) SetupWithManager(mgr ctrl.Manager) error {
62+
return ctrl.NewControllerManagedBy(mgr).
63+
For(&apiextensionsv1.CustomResourceDefinition{}).
64+
Named("CRD").
65+
Complete(r)
66+
}
67+
68+
func (r *CRDReconciler) updateAPISchema() error {
5769
savedJSON, err := r.io.Read(r.ClusterName)
58-
if errors.Is(err, fs.ErrNotExist) {
59-
actualJSON, err1 := r.sc.Resolve(r.DiscoveryClient)
60-
if err1 != nil {
61-
logger.Error(err1, "failed to resolve server JSON schema")
62-
return ctrl.Result{}, err1
63-
}
70+
if err != nil {
71+
return fmt.Errorf("failed to read JSON from filesystem: %w", err)
72+
}
73+
actualJSON, err := r.Resolve()
74+
if err != nil {
75+
return fmt.Errorf("failed to resolve server JSON schema: %w", err)
76+
}
77+
if !bytes.Equal(actualJSON, savedJSON) {
6478
if err := r.io.Write(actualJSON, r.ClusterName); err != nil {
65-
logger.Error(err, "failed to write JSON to filesystem")
66-
return ctrl.Result{}, err
79+
return fmt.Errorf("failed to write JSON to filesystem: %w", err)
6780
}
68-
return ctrl.Result{}, nil
6981
}
82+
return nil
83+
}
7084

85+
func (r *CRDReconciler) updateAPISchemaWith(crd *apiextensionsv1.CustomResourceDefinition) error {
86+
savedJSON, err := r.io.Read(r.ClusterName)
7187
if err != nil {
72-
logger.Error(err, "failed to read JSON from filesystem")
73-
return ctrl.Result{}, err
88+
return fmt.Errorf("failed to read JSON from filesystem: %w", err)
7489
}
75-
76-
actualJSON, err := r.sc.Resolve(r.DiscoveryClient)
90+
actualJSON, err := r.ResolveApiSchema(crd)
7791
if err != nil {
78-
logger.Error(err, "failed to resolve server JSON schema")
79-
return ctrl.Result{}, err
92+
return fmt.Errorf("failed to resolve server JSON schema: %w", err)
8093
}
8194
if !bytes.Equal(actualJSON, savedJSON) {
8295
if err := r.io.Write(actualJSON, r.ClusterName); err != nil {
83-
logger.Error(err, "failed to write JSON to filesystem")
84-
return ctrl.Result{}, err
96+
return fmt.Errorf("failed to write JSON to filesystem: %w", err)
8597
}
8698
}
87-
88-
return ctrl.Result{}, nil
89-
}
90-
91-
// SetupWithManager sets up the controller with the Manager.
92-
func (r *CRDReconciler) SetupWithManager(mgr ctrl.Manager) error {
93-
return ctrl.NewControllerManagedBy(mgr).
94-
For(&apiextensionsv1.CustomResourceDefinition{}).
95-
Named("CRD").
96-
Complete(r)
99+
return nil
97100
}

0 commit comments

Comments
 (0)