Merge pull request #679 from joelanford/declcfg-ignore

Add declarative config `.indexignore` support

Author: openshift-merge-robot

go.mod

@@ -29,6 +29,7 @@ require (
 	github.com/h2non/filetype v1.1.1
 	github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c
 	github.com/hashicorp/go-multierror v1.0.0
+	github.com/joelanford/ignore v0.0.0-20210607151042-0d25dc18b62d
 	github.com/mattn/go-sqlite3 v1.10.0
 	github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2
 	github.com/onsi/ginkgo v1.14.1
@@ -48,7 +49,7 @@ require (
 	github.com/yvasiyarov/newrelic_platform_go v0.0.0-20160601141957-9c099fbc30e9 // indirect
 	go.etcd.io/bbolt v1.3.5
 	golang.org/x/mod v0.3.0
-	golang.org/x/net v0.0.0-20201110031124-69a78807bb2b
+	golang.org/x/net v0.0.0-20210326060303-6b1517762897
 	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
 	google.golang.org/grpc v1.30.0
 	google.golang.org/grpc/cmd/protoc-gen-go-grpc v0.0.0-20200709232328-d8193ee9cc3e

go.sum

@@ -270,7 +270,7 @@ func newRegistry() (image.Registry, error) {
 			},
 			image.SimpleReference("test.registry/foo-operator/foo-index-declcfg:v0.2.0"): &image.MockImage{
 				Labels: map[string]string{
-					"operators.operatorframework.io.index.configs.v1": "/foo/index.yaml",
+					"operators.operatorframework.io.index.configs.v1": "/foo",
 				},
 				FS: subDeclcfgImage,
 			},

internal/action/render_test.go

@@ -2,12 +2,14 @@ package declcfg
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"os"
 	"path/filepath"
 	"strings"
 
+	"github.com/joelanford/ignore"
 	"github.com/operator-framework/api/pkg/operators"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/util/yaml"
@@ -22,7 +24,16 @@ func LoadDir(configDir string) (*DeclarativeConfig, error) {
 
 func loadFS(root string, w fsWalker) (*DeclarativeConfig, error) {
 	cfg := &DeclarativeConfig{}
+
+	matcher, err := ignore.NewMatcher(os.DirFS(root), ".indexignore")
+	if err != nil {
+		return nil, err
+	}
+
 	if err := w.WalkFiles(root, func(path string, r io.Reader) error {
+		if matcher.Match(path, false) {
+			return nil
+		}
 		fileCfg, err := readYAMLOrJSON(r)
 		if err != nil {
 			return fmt.Errorf("could not load config file %q: %v", path, err)
@@ -78,14 +89,16 @@ func readYAMLOrJSON(r io.Reader) (*DeclarativeConfig, error) {
 	for {
 		doc := json.RawMessage{}
 		if err := dec.Decode(&doc); err != nil {
-			break
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			return nil, err
 		}
 		doc = []byte(strings.NewReplacer(`\u003c`, "<", `\u003e`, ">", `\u0026`, "&").Replace(string(doc)))
 
 		var in Meta
 		if err := json.Unmarshal(doc, &in); err != nil {
-			// Ignore JSON blobs if they are not parsable as meta objects.
-			continue
+			return nil, err
 		}
 
 		switch in.Schema {
@@ -102,8 +115,7 @@ func readYAMLOrJSON(r io.Reader) (*DeclarativeConfig, error) {
 			}
 			cfg.Bundles = append(cfg.Bundles, b)
 		case "":
-			// Ignore meta blobs that don't have a schema.
-			continue
+			return nil, fmt.Errorf("object '%s' is missing root schema field", string(doc))
 		default:
 			cfg.Others = append(cfg.Others, in)
 		}
@@ -129,6 +141,7 @@ func (w dirWalker) WalkFiles(root string, f func(string, io.Reader) error) error
 		if err != nil {
 			return err
 		}
+		defer file.Close()
 		return f(path, file)
 	})
 }

internal/declcfg/load.go

@@ -22,20 +22,19 @@ func TestReadYAMLOrJSON(t *testing.T) {
 	}
 	specs := []spec{
 		{
-			name:              "Ignored/NotYAMLOrJSON",
-			file:              "testdata/invalid/not-yaml-or-json.txt",
-			assertion:         require.NoError,
-			expectNumPackages: 0,
-			expectNumBundles:  0,
-			expectNumOthers:   0,
+			name:      "Error/NotYAMLOrJSON",
+			file:      "testdata/invalid/not-yaml-or-json.txt",
+			assertion: require.Error,
 		},
 		{
-			name:              "Ignored/NotJSONObject",
-			file:              "testdata/invalid/not-yaml-or-json-object.json",
-			assertion:         require.NoError,
-			expectNumPackages: 0,
-			expectNumBundles:  0,
-			expectNumOthers:   0,
+			name:      "Error/NotJSONObject",
+			file:      "testdata/invalid/not-yaml-or-json-object.json",
+			assertion: require.Error,
+		},
+		{
+			name:      "Error/NoSchema",
+			file:      "testdata/invalid/no-schema.yaml",
+			assertion: require.Error,
 		},
 		{
 			name:      "Error/InvalidPackageJSON",

internal/declcfg/load_test.go

@@ -0,0 +1 @@
+hello: world

internal/declcfg/testdata/invalid/no-schema.yaml

@@ -1 +1 @@
-This is not yaml or json.
+[This is not yaml or json.}

internal/declcfg/testdata/invalid/not-yaml-or-json.txt

@@ -0,0 +1,5 @@
+*
+!*.json
+!*.yaml
+
+*.clusterserviceversion.yaml

internal/declcfg/testdata/valid/.indexignore

@@ -0,0 +1,5 @@
+# Valid Declarative Config
+
+This is a README file about this declarative config. It should be ignored
+when loading this directory as declarative config due to the patterns
+present in the `.indexignore` file.