Merge pull request #384 from gallettilance/check-before-insert

Check before insert

Author: openshift-merge-robot

cmd/opm/main.go

@@ -5,11 +5,13 @@ import (
 
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 
 	"github.com/operator-framework/operator-registry/cmd/opm/alpha"
 	"github.com/operator-framework/operator-registry/cmd/opm/index"
 	"github.com/operator-framework/operator-registry/cmd/opm/registry"
 	"github.com/operator-framework/operator-registry/cmd/opm/version"
+	registrylib "github.com/operator-framework/operator-registry/pkg/registry"
 )
 
 func main() {
@@ -35,6 +37,18 @@ func main() {
 	}
 
 	if err := rootCmd.Execute(); err != nil {
+		agg, ok := err.(utilerrors.Aggregate)
+		if !ok {
+			os.Exit(1)
+		}
+		for _, e := range agg.Errors() {
+			if _, ok := e.(registrylib.BundleImageAlreadyAddedErr); ok {
+				os.Exit(2)
+			}
+			if _, ok := e.(registrylib.PackageVersionAlreadyAddedErr); ok {
+				os.Exit(3)
+			}
+		}
 		os.Exit(1)
 	}
 }

pkg/lib/indexer/indexer.go

@@ -85,6 +85,7 @@ func (i ImageIndexer) AddToIndex(request AddToIndexRequest) error {
 	// Add the bundles to the registry
 	err = i.RegistryAdder.AddToRegistry(addToRegistryReq)
 	if err != nil {
+		i.Logger.WithError(err).Debugf("unable to add bundle to registry")
 		return err
 	}
 
@@ -222,7 +223,6 @@ func (i ImageIndexer) PruneFromIndex(request PruneFromIndexRequest) error {
 	return nil
 }
 
-
 // extractDatabase sets a temp directory for unpacking an image
 func (i ImageIndexer) extractDatabase(buildDir, fromIndex string) (string, error) {
 	tmpDir, err := ioutil.TempDir("./", tmpDirPrefix)

pkg/lib/registry/registry.go

@@ -5,16 +5,16 @@ import (
 	"crypto/x509"
 	"database/sql"
 	"fmt"
-	"github.com/operator-framework/operator-registry/pkg/containertools"
-	"github.com/operator-framework/operator-registry/pkg/image/execregistry"
 	"io/ioutil"
 	"os"
 
 	"github.com/sirupsen/logrus"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 
+	"github.com/operator-framework/operator-registry/pkg/containertools"
 	"github.com/operator-framework/operator-registry/pkg/image"
 	"github.com/operator-framework/operator-registry/pkg/image/containerdregistry"
+	"github.com/operator-framework/operator-registry/pkg/image/execregistry"
 	"github.com/operator-framework/operator-registry/pkg/registry"
 	"github.com/operator-framework/operator-registry/pkg/sqlite"
 )
@@ -34,8 +34,6 @@ type AddToRegistryRequest struct {
 }
 
 func (r RegistryUpdater) AddToRegistry(request AddToRegistryRequest) error {
-	var errs []error
-
 	db, err := sql.Open("sqlite3", request.InputDatabase)
 	if err != nil {
 		return err
@@ -96,16 +94,17 @@ func (r RegistryUpdater) AddToRegistry(request AddToRegistryRequest) error {
 	}
 
 	if err := populate(context.TODO(), dbLoader, graphLoader, dbQuerier, reg, simpleRefs, request.Mode); err != nil {
-		err = fmt.Errorf("error loading bundle from image: %s", err)
+		r.Logger.Debugf("unable to populate database: %s", err)
+
 		if !request.Permissive {
 			r.Logger.WithError(err).Error("permissive mode disabled")
-			errs = append(errs, err)
+			return err
 		} else {
 			r.Logger.WithError(err).Warn("permissive mode enabled")
 		}
 	}
 
-	return utilerrors.NewAggregate(errs) // nil if no errors
+	return nil
 }
 
 func populate(ctx context.Context, loader registry.Load, graphLoader registry.GraphLoader, querier registry.Query, reg image.Registry, refs []image.Reference, mode registry.Mode) error {

pkg/registry/populator.go

@@ -63,7 +63,50 @@ func (i *DirectoryPopulator) Populate(mode Mode) error {
 	return nil
 }
 
+func (i *DirectoryPopulator) globalSanityCheck(imagesToAdd []*ImageInput) error {
+	var errs []error
+	images := make(map[string]struct{})
+	for _, image := range imagesToAdd {
+		images[image.bundle.BundleImage] = struct{}{}
+	}
+
+	for _, image := range imagesToAdd {
+		bundlePaths, err := i.querier.GetBundlePathsForPackage(context.TODO(), image.bundle.Package)
+		if err != nil {
+			// Assume that this means that the bundle is empty
+			// Or that this is the first time the package is loaded.
+			return nil
+		}
+		for _, bundlePath := range bundlePaths {
+			if _, ok := images[bundlePath]; ok {
+				errs = append(errs, BundleImageAlreadyAddedErr{ErrorString: fmt.Sprintf("Bundle %s already exists", image.bundle.BundleImage)})
+				continue
+			}
+		}
+		for _, channel := range image.bundle.Channels {
+			bundle, err := i.querier.GetBundle(context.TODO(), image.bundle.Package, channel, image.bundle.csv.GetName())
+			if err != nil {
+				// Assume that if we can not find a bundle for the package, channel and or CSV Name that this is safe to add
+				continue
+			}
+			if bundle != nil {
+				// raise error that this package + channel + csv combo is already in the db
+				errs = append(errs, PackageVersionAlreadyAddedErr{ErrorString: "Bundle already added that provides package and csv"})
+				break
+			}
+		}
+	}
+
+	return utilerrors.NewAggregate(errs)
+}
+
 func (i *DirectoryPopulator) loadManifests(imagesToAdd []*ImageInput, mode Mode) error {
+	// global sanity checks before insertion
+	err := i.globalSanityCheck(imagesToAdd)
+	if err != nil {
+		return err
+	}
+
 	switch mode {
 	case ReplacesMode:
 		// TODO: This is relatively inefficient. Ideally, we should be able to use a replaces

pkg/registry/populator_test.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"math/rand"
 	"os"
+	"reflect"
 	"testing"
 	"time"
 
@@ -16,6 +17,8 @@ import (
 	"github.com/operator-framework/operator-registry/pkg/image"
 	"github.com/operator-framework/operator-registry/pkg/registry"
 	"github.com/operator-framework/operator-registry/pkg/sqlite"
+
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 )
 
 func init() {
@@ -246,6 +249,7 @@ func TestImageLoading(t *testing.T) {
 		addImage     img
 		wantPackages []*registry.Package
 		wantErr      bool
+		err          error
 	}{
 		{
 			name: "OneChannel/AddBundleToTwoChannels",
@@ -386,6 +390,76 @@ func TestImageLoading(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "AddBundleAlreadyExists",
+			initImages: []img{
+				{
+					// this is in the "preview" channel
+					ref: image.SimpleReference("quay.io/prometheus/operator:0.14.0"),
+					dir: "../../bundles/prometheus.0.14.0",
+				},
+			},
+			addImage: img{
+				//Adding same bundle different bundle
+				ref: image.SimpleReference("quay.io/prometheus/operator-test:testing"),
+				dir: "../../bundles/prometheus.0.14.0",
+			},
+			wantPackages: []*registry.Package{
+				{
+					Name:           "prometheus",
+					DefaultChannel: "stable",
+					Channels: map[string]registry.Channel{
+						"preview": {
+							Head: registry.BundleKey{
+								BundlePath: "quay.io/prometheus/operator:0.14.0",
+								Version:    "0.14.0",
+								CsvName:    "prometheusoperator.0.14.0",
+							},
+							Nodes: map[registry.BundleKey]map[registry.BundleKey]struct{}{
+								{BundlePath: "quay.io/prometheus/operator:0.14.0", Version: "0.14.0", CsvName: "prometheusoperator.0.14.0"}: {},
+							},
+						},
+					},
+				},
+			},
+			wantErr: true,
+			err:     registry.PackageVersionAlreadyAddedErr{},
+		},
+		{
+			name: "AddExactBundleAlreadyExists",
+			initImages: []img{
+				{
+					// this is in the "preview" channel
+					ref: image.SimpleReference("quay.io/prometheus/operator:0.14.0"),
+					dir: "../../bundles/prometheus.0.14.0",
+				},
+			},
+			addImage: img{
+				// Add the same package
+				ref: image.SimpleReference("quay.io/prometheus/operator:0.14.0"),
+				dir: "../../bundles/prometheus.0.14.0",
+			},
+			wantPackages: []*registry.Package{
+				{
+					Name:           "prometheus",
+					DefaultChannel: "stable",
+					Channels: map[string]registry.Channel{
+						"preview": {
+							Head: registry.BundleKey{
+								BundlePath: "quay.io/prometheus/operator:0.14.0",
+								Version:    "0.14.0",
+								CsvName:    "prometheusoperator.0.14.0",
+							},
+							Nodes: map[registry.BundleKey]map[registry.BundleKey]struct{}{
+								{BundlePath: "quay.io/prometheus/operator:0.14.0", Version: "0.14.0", CsvName: "prometheusoperator.0.14.0"}: {},
+							},
+						},
+					},
+				},
+			},
+			wantErr: true,
+			err:     registry.BundleImageAlreadyAddedErr{},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -411,7 +485,12 @@ func TestImageLoading(t *testing.T) {
 				graphLoader,
 				query,
 				map[image.Reference]string{tt.addImage.ref: tt.addImage.dir})
-			require.NoError(t, add.Populate(registry.ReplacesMode))
+			err = add.Populate(registry.ReplacesMode)
+			if tt.wantErr {
+				require.True(t, checkAggErr(err, tt.err))
+				return
+			}
+			require.NoError(t, err)
 
 			for _, p := range tt.wantPackages {
 				graphLoader, err := sqlite.NewSQLGraphLoaderFromDB(db)
@@ -426,6 +505,18 @@ func TestImageLoading(t *testing.T) {
 	}
 }
 
+func checkAggErr(aggErr, wantErr error) bool {
+	if a, ok := aggErr.(utilerrors.Aggregate); ok {
+		for _, e := range a.Errors() {
+			if reflect.TypeOf(e).String() == reflect.TypeOf(wantErr).String() {
+				return true
+			}
+		}
+		return false
+	}
+	return reflect.TypeOf(aggErr).String() == reflect.TypeOf(wantErr).String()
+}
+
 func TestQuerierForDependencies(t *testing.T) {
 	logrus.SetLevel(logrus.DebugLevel)
 	db, cleanup := CreateTestDb(t)

pkg/registry/types.go

@@ -14,6 +14,24 @@ var (
 	ErrPackageNotInDatabase = errors.New("Package not in database")
 )
 
+// BundleImageAlreadyAddedErr is an error that describes a bundle is already added
+type BundleImageAlreadyAddedErr struct {
+	ErrorString string
+}
+
+func (e BundleImageAlreadyAddedErr) Error() string {
+	return e.ErrorString
+}
+
+// PackageVersionAlreadyAddedErr is an error that describes that a bundle that is already in the databse that provides this package and version
+type PackageVersionAlreadyAddedErr struct {
+	ErrorString string
+}
+
+func (e PackageVersionAlreadyAddedErr) Error() string {
+	return e.ErrorString
+}
+
 const (
 	GVKType     = "olm.gvk"
 	PackageType = "olm.package"