add regression testing

Signed-off-by: Per Goncalves da Silva <[email protected]>

Author: Per Goncalves da Silva

internal/operator-controller/rukpak/convert/registryv1_test.go

@@ -1,9 +1,13 @@
 package convert_test
 
 import (
+	"cmp"
 	"fmt"
 	"io/fs"
 	"os"
+	"path/filepath"
+	"reflect"
+	"slices"
 	"strings"
 	"testing"
 	"testing/fstest"
@@ -17,6 +21,7 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/cli-runtime/pkg/resource"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/operator-framework/api/pkg/operators/v1alpha1"
@@ -32,24 +37,80 @@ const (
 
 	bundlePathAnnotations = "metadata/annotations.yaml"
 	bundlePathCSV         = "manifests/csv.yaml"
+
+	regressionTestRoot = "testdata/regression-tests"
 )
 
-func getCsvAndService() (v1alpha1.ClusterServiceVersion, corev1.Service) {
-	csv := v1alpha1.ClusterServiceVersion{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: "testCSV",
-		},
-		Spec: v1alpha1.ClusterServiceVersionSpec{
-			InstallModes: []v1alpha1.InstallMode{{Type: v1alpha1.InstallModeTypeAllNamespaces, Supported: true}},
-		},
-	}
-	svc := corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: "testService",
+// Test_ConvertRegressions checks that the registry+v1 to plain manifest
+// conversion code against bundles in testdata. The expected output is pre-generated using the rukpak/convert
+// package with the `regression-tests/generate-manifests.go` tool. New test cases must also be added to
+// generate manifests, which will generate the expected manifests. These _must_ be manually inspected/tested
+// for correctness before changes are committed to the test-cases. If the expected manifests for existing test-cases
+// change, this is likely a regression. Inspect it carefully.
+func Test_ConvertRegressions(t *testing.T) {
+	for _, tc := range []struct {
+		name             string
+		installNamespace string
+		watchNamespace   string
+		bundle           string
+		testCaseName     string
+	}{
+		{
+			name:             "AllNamespaces",
+			installNamespace: "argocd-system",
+			bundle:           "argocd-operator.v0.6.0",
+			testCaseName:     "all-namespaces",
+		}, {
+			name:             "SingleNamespaces",
+			installNamespace: "argocd-system",
+			watchNamespace:   "argocd-watch",
+			bundle:           "argocd-operator.v0.6.0",
+			testCaseName:     "single-namespaces",
+		}, {
+			name:             "OwnNamespaces",
+			installNamespace: "argocd-system",
+			watchNamespace:   "argocd-system",
+			bundle:           "argocd-operator.v0.6.0",
+			testCaseName:     "own-namespaces",
 		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			var (
+				bundleRoot   = filepath.Join(regressionTestRoot, tc.bundle)
+				testCaseRoot = filepath.Join(bundleRoot, "test-cases")
+			)
+
+			t.Log("Build BundleFS")
+			bundleFs := os.DirFS(filepath.Join(bundleRoot, "bundle"))
+
+			t.Log("Parse BundleFS into registry+v1")
+			regv1, err := convert.ParseFS(bundleFs)
+			require.NoError(t, err)
+
+			t.Log("Convert registry+v1 to plain manifests")
+			plain, err := convert.Convert(regv1, tc.installNamespace, []string{tc.watchNamespace})
+			require.NoError(t, err)
+
+			t.Log("Compare to pre-generated manifests")
+			var actualObjs []client.Object
+			for _, obj := range plain.Objects {
+				uObj := convertToUnstructured(t, obj)
+				actualObjs = append(actualObjs, &uObj)
+			}
+
+			expectedObjs, err := loadGeneratedManifests(filepath.Join(testCaseRoot, tc.testCaseName))
+			require.NoError(t, err)
+
+			slices.SortFunc(actualObjs, sortByKindNamespaceName)
+			slices.SortFunc(expectedObjs, sortByKindNamespaceName)
+
+			for i := range actualObjs {
+				require.True(t, reflect.DeepEqual(actualObjs[i], expectedObjs[i]), "failed comparing manifest %d", i)
+				//require.Equal(t, expectedObjs[i], actualObjs[i])
+			}
+			require.Equal(t, len(expectedObjs), len(actualObjs))
+		})
 	}
-	svc.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Service"})
-	return csv, svc
 }
 
 func TestRegistryV1SuiteNamespaceNotAvailable(t *testing.T) {
@@ -192,61 +253,6 @@ func TestRegistryV1SuiteNamespaceClusterScoped(t *testing.T) {
 	require.Empty(t, resObj.GetNamespace())
 }
 
-func getBaseCsvAndService() (v1alpha1.ClusterServiceVersion, corev1.Service) {
-	// base CSV definition that each test case will deep copy and modify
-	baseCSV := v1alpha1.ClusterServiceVersion{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: "testCSV",
-			Annotations: map[string]string{
-				olmProperties: fmt.Sprintf("[{\"type\": %s, \"value\": \"%s\"}]", property.TypeConstraint, "value"),
-			},
-		},
-		Spec: v1alpha1.ClusterServiceVersionSpec{
-			InstallStrategy: v1alpha1.NamedInstallStrategy{
-				StrategySpec: v1alpha1.StrategyDetailsDeployment{
-					DeploymentSpecs: []v1alpha1.StrategyDeploymentSpec{
-						{
-							Name: "testDeployment",
-							Spec: appsv1.DeploymentSpec{
-								Template: corev1.PodTemplateSpec{
-									Spec: corev1.PodSpec{
-										Containers: []corev1.Container{
-											{
-												Name:  "testContainer",
-												Image: "testImage",
-											},
-										},
-									},
-								},
-							},
-						},
-					},
-					Permissions: []v1alpha1.StrategyDeploymentPermissions{
-						{
-							ServiceAccountName: "testServiceAccount",
-							Rules: []rbacv1.PolicyRule{
-								{
-									APIGroups: []string{"test"},
-									Resources: []string{"pods"},
-									Verbs:     []string{"*"},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-	}
-
-	svc := corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: "testService",
-		},
-	}
-	svc.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Service"})
-	return baseCSV, svc
-}
-
 func TestRegistryV1SuiteGenerateAllNamespace(t *testing.T) {
 	t.Log("RegistryV1 Suite Convert")
 	t.Log("It should generate objects successfully based on target namespaces")
@@ -650,3 +656,107 @@ func removePaths(mapFs fstest.MapFS, paths ...string) fstest.MapFS {
 	}
 	return mapFs
 }
+
+func getCsvAndService() (v1alpha1.ClusterServiceVersion, corev1.Service) {
+	csv := v1alpha1.ClusterServiceVersion{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "testCSV",
+		},
+		Spec: v1alpha1.ClusterServiceVersionSpec{
+			InstallModes: []v1alpha1.InstallMode{{Type: v1alpha1.InstallModeTypeAllNamespaces, Supported: true}},
+		},
+	}
+	svc := corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "testService",
+		},
+	}
+	svc.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Service"})
+	return csv, svc
+}
+
+func getBaseCsvAndService() (v1alpha1.ClusterServiceVersion, corev1.Service) {
+	// base CSV definition that each test case will deep copy and modify
+	baseCSV := v1alpha1.ClusterServiceVersion{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "testCSV",
+			Annotations: map[string]string{
+				olmProperties: fmt.Sprintf("[{\"type\": %s, \"value\": \"%s\"}]", property.TypeConstraint, "value"),
+			},
+		},
+		Spec: v1alpha1.ClusterServiceVersionSpec{
+			InstallStrategy: v1alpha1.NamedInstallStrategy{
+				StrategySpec: v1alpha1.StrategyDetailsDeployment{
+					DeploymentSpecs: []v1alpha1.StrategyDeploymentSpec{
+						{
+							Name: "testDeployment",
+							Spec: appsv1.DeploymentSpec{
+								Template: corev1.PodTemplateSpec{
+									Spec: corev1.PodSpec{
+										Containers: []corev1.Container{
+											{
+												Name:  "testContainer",
+												Image: "testImage",
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+					Permissions: []v1alpha1.StrategyDeploymentPermissions{
+						{
+							ServiceAccountName: "testServiceAccount",
+							Rules: []rbacv1.PolicyRule{
+								{
+									APIGroups: []string{"test"},
+									Resources: []string{"pods"},
+									Verbs:     []string{"*"},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	svc := corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "testService",
+		},
+	}
+	svc.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Service"})
+	return baseCSV, svc
+}
+
+func loadGeneratedManifests(generatedPath string) ([]client.Object, error) {
+	var expectedObjs []client.Object
+	err := fs.WalkDir(os.DirFS(generatedPath), ".", func(path string, info fs.DirEntry, err error) error {
+		if err != nil || info.IsDir() || filepath.Ext(path) != ".yaml" {
+			return err
+		}
+		file, err := os.Open(filepath.Join(generatedPath, path))
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+		result := resource.NewLocalBuilder().Unstructured().Stream(file, path).Do()
+		if result.Err() != nil {
+			return err
+		}
+		return result.Visit(func(info *resource.Info, err error) error {
+			expectedObjs = append(expectedObjs, info.Object.(*unstructured.Unstructured))
+			return nil
+		})
+	})
+	return expectedObjs, err
+}
+
+func sortByKindNamespaceName(a client.Object, b client.Object) int {
+	return cmp.Or(
+		cmp.Compare(a.GetObjectKind().GroupVersionKind().Kind, b.GetObjectKind().GroupVersionKind().Kind),
+		cmp.Compare(a.GetNamespace(), b.GetNamespace()),
+		cmp.Compare(a.GetName(), b.GetName()),
+	)
+}

internal/operator-controller/rukpak/convert/testdata/regression-tests/README.md

@@ -0,0 +1,7 @@
+## registry+v1 bundle generation regression tests
+
+This directory includes test cases for the rukpak/convert package based on real bundle data.
+The manifests are generated and manually/visually verified for correctness.
+
+The `generate-manifests.go` tool is used to generate the tests cases by calling convert.Convert on bundles
+in the `testdata` directory. 

internal/operator-controller/rukpak/convert/testdata/regression-tests/argocd-operator.v0.6.0/bundle/manifests/argocd-operator-controller-manager-metrics-service_v1_service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  creationTimestamp: null
+  labels:
+    control-plane: controller-manager
+  name: argocd-operator-controller-manager-metrics-service
+spec:
+  ports:
+  - name: https
+    port: 8443
+    targetPort: https
+  selector:
+    control-plane: controller-manager
+status:
+  loadBalancer: {}

internal/operator-controller/rukpak/convert/testdata/regression-tests/argocd-operator.v0.6.0/bundle/manifests/argocd-operator-manager-config_v1_configmap.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+data:
+  controller_manager_config.yaml: |
+    apiVersion: controller-runtime.sigs.k8s.io/v1alpha1
+    kind: ControllerManagerConfig
+    health:
+      healthProbeBindAddress: :8081
+    metrics:
+      bindAddress: 127.0.0.1:8080
+    webhook:
+      port: 9443
+    leaderElection:
+      leaderElect: true
+      resourceName: b674928d.argoproj.io
+kind: ConfigMap
+metadata:
+  name: argocd-operator-manager-config

internal/operator-controller/rukpak/convert/testdata/regression-tests/argocd-operator.v0.6.0/bundle/manifests/argocd-operator-metrics-reader_rbac.authorization.k8s.io_v1_clusterrole.yaml

@@ -0,0 +1,10 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  name: argocd-operator-metrics-reader
+rules:
+- nonResourceURLs:
+  - /metrics
+  verbs:
+  - get