Improve CRD management performance (#5304)

Fixes #4574.

Author: matthchr

v2/internal/crdmanagement/helpers.go

@@ -5,6 +5,7 @@ package crdmanagement
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/go-logr/logr"
 	"github.com/rotisserie/eris"
@@ -17,6 +18,8 @@ import (
 	"github.com/Azure/azure-service-operator/v2/pkg/genruntime/registration"
 )
 
+const CRDFilePrefix = "apiextensions.k8s.io_v1_customresourcedefinition_"
+
 func MakeCRDMap(
 	crds []apiextensions.CustomResourceDefinition,
 ) map[string]apiextensions.CustomResourceDefinition {
@@ -101,3 +104,47 @@ func makeMatchString(crd apiextensions.CustomResourceDefinition) string {
 	// matchString should be "group/kind"
 	return fmt.Sprintf("%s/%s", group, kind)
 }
+
+// groupFromFilename extracts the API group from a CRD filename.
+// CRD files follow the convention: "apiextensions.k8s.io_v1_customresourcedefinition_{kindPlural}.{group}.yaml".
+// For example, "apiextensions.k8s.io_v1_customresourcedefinition_virtualnetworks.network.azure.com.yaml"
+// returns "network.azure.com".
+func groupFromFilename(filename string) (string, error) {
+	crdName, err := crdNameFromFilename(filename)
+	if err != nil {
+		return "", err
+	}
+
+	// CRD name is "{kindPlural}.{group}", extract group (everything after the first dot)
+	idx := strings.Index(crdName, ".")
+	if idx < 0 || idx == len(crdName)-1 {
+		return "", eris.Errorf("CRD name %q derived from filename %q has no group component", crdName, filename)
+	}
+
+	return crdName[idx+1:], nil
+}
+
+// crdNameFromFilename derives the CRD metadata.name from a CRD filename.
+// CRD files follow the convention: "apiextensions.k8s.io_v1_customresourcedefinition_{crdname}.yaml",
+// where {crdname} is the CRD's metadata.name (e.g. "virtualnetworks.network.azure.com").
+// For example, "apiextensions.k8s.io_v1_customresourcedefinition_virtualnetworks.network.azure.com.yaml"
+// returns "virtualnetworks.network.azure.com".
+func crdNameFromFilename(filename string) (string, error) {
+	if !strings.HasPrefix(filename, CRDFilePrefix) {
+		return "", eris.Errorf("filename %q does not have expected prefix %q", filename, CRDFilePrefix)
+	}
+
+	if !strings.HasSuffix(filename, ".yaml") {
+		return "", eris.Errorf("filename %q does not have .yaml extension", filename)
+	}
+
+	// Strip prefix and .yaml suffix
+	crdName := strings.TrimPrefix(filename, CRDFilePrefix)
+	crdName = strings.TrimSuffix(crdName, ".yaml")
+
+	if crdName == "" {
+		return "", eris.Errorf("filename %q has no CRD name between prefix and extension", filename)
+	}
+
+	return crdName, nil
+}

v2/internal/crdmanagement/helpers_test.go

@@ -5,6 +5,7 @@ package crdmanagement_test
 
 import (
 	"context"
+	"os"
 	"testing"
 
 	. "github.com/onsi/gomega"
@@ -41,9 +42,10 @@ func Test_AllCRDsReady_NoneAreFiltered(t *testing.T) {
 	g := NewGomegaWithT(t)
 
 	testData := testSetup(t)
+	g.Expect(testcommon.CheckBundledCRDsDirectory(testData.crdPath)).To(Succeed())
 
 	// load crds
-	goalCRDs, err := testData.crdManager.LoadOperatorCRDs(testData.crdPath, testData.namespace)
+	goalCRDs, err := testData.crdManager.LoadOperatorCRDs(testData.crdPath, testData.namespace, testData.crdManager.BuildCRDFileFilter("*", nil))
 	g.Expect(err).ToNot(HaveOccurred())
 
 	readyResources := crdmanagement.MakeCRDMap(goalCRDs)
@@ -63,9 +65,10 @@ func Test_FiveCRDsReady_AllOthersAreFiltered(t *testing.T) {
 	g := NewGomegaWithT(t)
 
 	testData := testSetup(t)
+	g.Expect(testcommon.CheckBundledCRDsDirectory(testData.crdPath)).To(Succeed())
 
 	// load crds
-	goalCRDs, err := testData.crdManager.LoadOperatorCRDs(testData.crdPath, testData.namespace)
+	goalCRDs, err := testData.crdManager.LoadOperatorCRDs(testData.crdPath, testData.namespace, testData.crdManager.BuildCRDFileFilter("*", nil))
 	g.Expect(err).ToNot(HaveOccurred())
 
 	// Filter all but the first 5 CRDs
@@ -88,6 +91,54 @@ func Test_FiveCRDsReady_AllOthersAreFiltered(t *testing.T) {
 	g.Expect(objs).To(HaveLen(5))
 }
 
+// This test requires that the task target `bundle-crds` has been run
+func Test_ResourceGroupPattern_AllOthersAreFiltered(t *testing.T) {
+	t.Parallel()
+	g := NewGomegaWithT(t)
+
+	testData := testSetup(t)
+	g.Expect(testcommon.CheckBundledCRDsDirectory(testData.crdPath)).To(Succeed())
+
+	// load crds
+	shouldLoad := testData.crdManager.BuildCRDFileFilter("resources.azure.com/ResourceGroup", nil)
+	goalCRDs, err := testData.crdManager.LoadOperatorCRDs(testData.crdPath, testData.namespace, shouldLoad)
+	g.Expect(err).ToNot(HaveOccurred())
+
+	readyResources := crdmanagement.MakeCRDMap(goalCRDs)
+
+	knownTypes, err := testData.getKnownStorageTypes()
+	g.Expect(err).ToNot(HaveOccurred())
+
+	// Filter the types to register
+	objs, err := crdmanagement.FilterStorageTypesByReadyCRDs(testData.logger, testData.s.GetScheme(), readyResources, knownTypes)
+	g.Expect(err).ToNot(HaveOccurred())
+	g.Expect(objs).To(HaveLen(1))
+}
+
+// This test requires that the task target `bundle-crds` has been run
+func Test_AllCRDFilenames_MatchExpectedPattern(t *testing.T) {
+	t.Parallel()
+	g := NewGomegaWithT(t)
+
+	testData := testSetup(t)
+	g.Expect(testcommon.CheckBundledCRDsDirectory(testData.crdPath)).To(Succeed())
+
+	entries, err := os.ReadDir(testData.crdPath)
+	g.Expect(err).ToNot(HaveOccurred())
+	g.Expect(entries).ToNot(BeEmpty())
+
+	for _, entry := range entries {
+		if entry.IsDir() {
+			continue
+		}
+		filename := entry.Name()
+
+		// Every file should have the expected prefix and .yaml suffix
+		g.Expect(filename).To(HavePrefix(crdmanagement.CRDFilePrefix), "file %q missing expected prefix", filename)
+		g.Expect(filename).To(HaveSuffix(".yaml"), "file %q missing .yaml suffix", filename)
+	}
+}
+
 /*
  * Helpers
  */