pkg.parser: add the ability to parse manifests with comments

Signed-off-by: Muvaffak Onus <[email protected]>

Author: muvaf

pkg/parser/parser.go

@@ -22,7 +22,6 @@ import (
 	"context"
 	"io"
 	"strings"
-	"unicode"
 
 	"github.com/spf13/afero"
 	corev1 "k8s.io/api/core/v1"
@@ -91,7 +90,7 @@ func New(meta, obj ObjectCreaterTyper) *PackageParser {
 // decode objects recognized by the meta scheme, then attempts to decode objects
 // recognized by the object scheme. Objects not recognized by either scheme
 // return an error rather than being skipped.
-func (p *PackageParser) Parse(_ context.Context, reader io.ReadCloser) (*Package, error) { //nolint:gocyclo // Only at 11.
+func (p *PackageParser) Parse(_ context.Context, reader io.ReadCloser) (*Package, error) {
 	pkg := NewPackage()
 	if reader == nil {
 		return pkg, nil
@@ -101,27 +100,25 @@ func (p *PackageParser) Parse(_ context.Context, reader io.ReadCloser) (*Package
 	dm := json.NewSerializerWithOptions(json.DefaultMetaFactory, p.metaScheme, p.metaScheme, json.SerializerOptions{Yaml: true})
 	do := json.NewSerializerWithOptions(json.DefaultMetaFactory, p.objScheme, p.objScheme, json.SerializerOptions{Yaml: true})
 	for {
-		bytes, err := yr.Read()
+		content, err := yr.Read()
 		if err != nil && !errors.Is(err, io.EOF) {
 			return pkg, err
 		}
 		if errors.Is(err, io.EOF) {
 			break
 		}
-		if len(bytes) == 0 {
+		content = cleanYAML(content)
+		if len(content) == 0 {
 			continue
 		}
-		if isWhiteSpace(bytes) {
-			continue
-		}
-		m, _, err := dm.Decode(bytes, nil, nil)
+		m, _, err := dm.Decode(content, nil, nil)
 		if err != nil {
 			// NOTE(hasheddan): we only try to decode with object scheme if the
 			// error is due the object not being registered in the meta scheme.
 			if !runtime.IsNotRegisteredError(err) {
 				return pkg, annotateErr(err, reader)
 			}
-			o, _, err := do.Decode(bytes, nil, nil)
+			o, _, err := do.Decode(content, nil, nil)
 			if err != nil {
 				return pkg, annotateErr(err, reader)
 			}
@@ -133,17 +130,27 @@ func (p *PackageParser) Parse(_ context.Context, reader io.ReadCloser) (*Package
 	return pkg, nil
 }
 
-// isWhiteSpace determines whether the passed in bytes are all unicode white
-// space.
-func isWhiteSpace(bytes []byte) bool {
+// cleanYAML cleans up YAML by removing empty and commented out lines which
+// cause issues with decoding.
+func cleanYAML(y []byte) []byte {
+	lines := []string{}
 	empty := true
-	for _, b := range bytes {
-		if !unicode.IsSpace(rune(b)) {
+	for _, line := range strings.Split(string(y), "\n") {
+		trimmed := strings.TrimSpace(line)
+		if trimmed == "" || strings.HasPrefix(trimmed, "#") {
+			continue
+		}
+		// We don't want to return an empty document with only separators that
+		// have nothing in-between.
+		if empty && trimmed != "---" && trimmed != "..." {
 			empty = false
-			break
 		}
+		lines = append(lines, line)
+	}
+	if empty {
+		return nil
 	}
-	return empty
+	return []byte(strings.Join(lines, "\n"))
 }
 
 // annotateErr annotates an error if the reader is an AnnotatedReadCloser.

pkg/parser/parser_test.go

@@ -55,6 +55,17 @@ metadata:
 
 	deployBytes = []byte(`apiVersion: apps/v1
 kind: Deployment
+metadata:
+  name: test`)
+
+	commentedOutBytes = []byte(`# apiVersion: apps/v1
+# kind: Deployment
+# metadata:
+#   name: test`)
+	manifestWithComments = []byte(`
+apiVersion: apiextensions.k8s.io/v1beta1
+# Some Comment
+kind: CustomResourceDefinition
 metadata:
   name: test`)
 
@@ -79,6 +90,9 @@ func TestParser(t *testing.T) {
 	emptyFs := afero.NewMemMapFs()
 	_ = afero.WriteFile(emptyFs, "empty.yaml", []byte(""), 0o644)
 	_ = afero.WriteFile(emptyFs, "bad.yam", []byte("definitely not yaml"), 0o644)
+	commentedFs := afero.NewMemMapFs()
+	_ = afero.WriteFile(commentedFs, "commented.yaml", commentedOutBytes, 0o644)
+	_ = afero.WriteFile(commentedFs, ".crossplane/realmanifest.yaml", manifestWithComments, 0o644)
 	objScheme := runtime.NewScheme()
 	_ = apiextensions.AddToScheme(objScheme)
 	metaScheme := runtime.NewScheme()
@@ -128,6 +142,24 @@ func TestParser(t *testing.T) {
 				objects: []runtime.Object{crd, crd, crd, crd},
 			},
 		},
+		"FsBackendCommentedOut": {
+			reason:  "should parse filesystem successfully even if all the files are commented out",
+			parser:  New(metaScheme, objScheme),
+			backend: NewFsBackend(commentedFs, FsDir("."), FsFilters(SkipDirs(), SkipNotYAML(), SkipPath(".crossplane/*"))),
+			pkg: &Package{
+				meta:    nil,
+				objects: nil,
+			},
+		},
+		"FsBackendWithComments": {
+			reason:  "should parse filesystem successfully when some of the manifests contain comments",
+			parser:  New(metaScheme, objScheme),
+			backend: NewFsBackend(commentedFs, FsDir("."), FsFilters(SkipDirs(), SkipNotYAML())),
+			pkg: &Package{
+				meta:    nil,
+				objects: []runtime.Object{crd},
+			},
+		},
 		"FsBackendAll": {
 			reason:  "should parse filesystem successfully with multiple objects in single file",
 			parser:  New(metaScheme, objScheme),