Merge pull request #685 from zcahana/bundle_gzip

Add an option to store Gzip-encoded bundle data in ConfigMaps

Author: openshift-merge-robot

cmd/opm/alpha/bundle/extract.go

@@ -31,6 +31,7 @@ func init() {
 	extractCmd.Flags().StringP("configmapname", "c", "", "name of configmap to write bundle data")
 	extractCmd.Flags().StringP("namespace", "n", "openshift-operator-lifecycle-manager", "namespace to write configmap data")
 	extractCmd.Flags().Uint64P("datalimit", "l", 1<<20, "maximum limit in bytes for total bundle data")
+	extractCmd.Flags().BoolP("gzip", "z", false, "enable gzip compression of configmap data")
 	extractCmd.MarkPersistentFlagRequired("configmapname")
 }
 
@@ -55,8 +56,12 @@ func runExtractCmd(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
+	gzip, err := cmd.Flags().GetBool("gzip")
+	if err != nil {
+		return err
+	}
 
-	loader := configmap.NewConfigMapLoaderForDirectory(configmapName, namespace, manifestsDir, kubeconfig)
+	loader := configmap.NewConfigMapLoader(configmapName, namespace, manifestsDir, gzip, kubeconfig)
 	if err := loader.Populate(datalimit); err != nil {
 		return fmt.Errorf("error loading manifests from directory: %s", err)
 	}

pkg/configmap/configmap.go

@@ -9,6 +9,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 
 	"github.com/operator-framework/operator-registry/pkg/api"
+	"github.com/operator-framework/operator-registry/pkg/lib/encoding"
 	"github.com/operator-framework/operator-registry/pkg/registry"
 )
 
@@ -41,7 +42,7 @@ func (l *BundleLoader) Load(cm *corev1.ConfigMap) (bundle *api.Bundle, err error
 		"configmap": fmt.Sprintf("%s/%s", cm.GetNamespace(), cm.GetName()),
 	})
 
-	bundle, skipped, bundleErr := loadBundle(logger, cm.Data)
+	bundle, skipped, bundleErr := loadBundle(logger, cm)
 	if bundleErr != nil {
 		err = fmt.Errorf("failed to extract bundle from configmap - %v", bundleErr)
 		return
@@ -50,10 +51,21 @@ func (l *BundleLoader) Load(cm *corev1.ConfigMap) (bundle *api.Bundle, err error
 	return
 }
 
-func loadBundle(entry *logrus.Entry, data map[string]string) (bundle *api.Bundle, skipped map[string]string, err error) {
+func loadBundle(entry *logrus.Entry, cm *corev1.ConfigMap) (bundle *api.Bundle, skipped map[string]string, err error) {
 	bundle = &api.Bundle{Object: []string{}}
 	skipped = map[string]string{}
 
+	data := cm.Data
+	if hasGzipEncodingAnnotation(cm) {
+		entry.Debug("Decoding gzip-encoded bundle data")
+
+		var err error
+		data, err = decodeGzipBinaryData(cm)
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+
 	// Add kube resources to the bundle.
 	for name, content := range data {
 		reader := strings.NewReader(content)
@@ -85,3 +97,18 @@ func loadBundle(entry *logrus.Entry, data map[string]string) (bundle *api.Bundle
 
 	return
 }
+
+func decodeGzipBinaryData(cm *corev1.ConfigMap) (map[string]string, error) {
+	data := map[string]string{}
+
+	for name, content := range cm.BinaryData {
+		decoded, err := encoding.GzipBase64Decode(content)
+		if err != nil {
+			return nil, fmt.Errorf("error decoding gzip-encoded bundle data: %v", err)
+		}
+
+		data[name] = string(decoded)
+	}
+
+	return data, nil
+}

pkg/configmap/configmap_test.go

@@ -1,16 +1,23 @@
 package configmap
 
 import (
+	"context"
 	"os"
-	"strings"
 	"testing"
 
 	"github.com/operator-framework/operator-registry/pkg/api"
+	unstructuredlib "github.com/operator-framework/operator-registry/pkg/lib/unstructured"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/yaml"
+	"k8s.io/client-go/kubernetes/fake"
+)
+
+const (
+	configMapName      = "test-configmap"
+	configMapNamespace = "test-namespace"
 )
 
 func TestLoad(t *testing.T) {
@@ -40,7 +47,8 @@ func TestLoad(t *testing.T) {
 				assert.NotNil(t, objects)
 				assert.Equal(t, 1, len(objects))
 
-				unst := getUnstructured(t, objects[0])
+				unst, err := unstructuredlib.FromString(objects[0])
+				assert.NoError(t, err)
 				assert.True(t, unst.GetKind() == "Foo")
 			},
 		},
@@ -51,7 +59,8 @@ func TestLoad(t *testing.T) {
 				csvGot := bundleGot.GetCsvJson()
 				assert.NotNil(t, csvGot)
 
-				unst := getUnstructured(t, csvGot)
+				unst, err := unstructuredlib.FromString(csvGot)
+				assert.NoError(t, err)
 				assert.True(t, unst.GetName() == "first" || unst.GetName() == "second")
 			},
 		},
@@ -69,7 +78,8 @@ func TestLoad(t *testing.T) {
 			assertFunc: func(t *testing.T, bundleGot *api.Bundle) {
 				csvGot := bundleGot.GetCsvJson()
 				assert.NotNil(t, csvGot)
-				unst := getUnstructured(t, csvGot)
+				unst, err := unstructuredlib.FromString(csvGot)
+				assert.NoError(t, err)
 				assert.True(t, unst.GetName() == "kiali-operator.v1.4.2")
 
 				objects := bundleGot.GetObject()
@@ -83,7 +93,8 @@ func TestLoad(t *testing.T) {
 			assertFunc: func(t *testing.T, bundleGot *api.Bundle) {
 				csvGot := bundleGot.GetCsvJson()
 				assert.NotNil(t, csvGot)
-				unst := getUnstructured(t, csvGot)
+				unst, err := unstructuredlib.FromString(csvGot)
+				assert.NoError(t, err)
 				assert.True(t, unst.GetName() == "kiali-operator.v1.4.2")
 
 				objects := bundleGot.GetObject()
@@ -109,6 +120,56 @@ func TestLoad(t *testing.T) {
 	}
 }
 
+func TestLoadWriteRead(t *testing.T) {
+	tests := []struct {
+		name   string
+		source string
+		gzip   bool
+	}{
+		{
+			name:   "BundleUncompressed",
+			source: "testdata/bundles/etcd.0.9.2/",
+			gzip:   false,
+		},
+		{
+			name:   "BundleCompressed",
+			source: "testdata/bundles/etcd.0.9.2/",
+			gzip:   true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cm := &corev1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      configMapName,
+					Namespace: configMapNamespace,
+				},
+			}
+			clientset := fake.NewSimpleClientset()
+			clientset.CoreV1().ConfigMaps(configMapNamespace).Create(context.TODO(), cm, metav1.CreateOptions{})
+
+			cmLoader := NewConfigMapLoaderWithClient(configMapName, configMapNamespace, tt.source, tt.gzip, clientset)
+			err := cmLoader.Populate(1 << 20)
+			assert.NoError(t, err)
+
+			cm, err = clientset.CoreV1().ConfigMaps(configMapNamespace).Get(context.TODO(), configMapName, metav1.GetOptions{})
+			assert.NoError(t, err)
+
+			bundleLoader := NewBundleLoader()
+			bundle, err := bundleLoader.Load(cm)
+
+			expectedObjects, err := unstructuredlib.FromDir(tt.source + "manifests/")
+			assert.NoError(t, err)
+
+			bundleObjects, err := unstructuredlib.FromBundle(bundle)
+			assert.NoError(t, err)
+
+			assert.ElementsMatch(t, expectedObjects, bundleObjects)
+		})
+	}
+}
+
 func loadfromFile(t *testing.T, path string) *corev1.ConfigMap {
 	reader, err := os.Open(path)
 	require.NoError(t, err, "unable to load from file %s", path)
@@ -120,11 +181,3 @@ func loadfromFile(t *testing.T, path string) *corev1.ConfigMap {
 
 	return bundle
 }
-
-func getUnstructured(t *testing.T, str string) *unstructured.Unstructured {
-	dec := yaml.NewYAMLOrJSONDecoder(strings.NewReader(str), 1)
-	unst := &unstructured.Unstructured{}
-	err := dec.Decode(unst)
-	assert.NoError(t, err)
-	return unst
-}