Skip to content

Commit e1eebae

Browse files
joelanfordjoelanford
authored
Add WalkMetas* functions and meta.Name field (#1089)
* add WalkMetasFS and WalkMetasReader functions These function are like WalkFS except that callers provide a function that handles a Meta object at a time, rather than an entire file's worth of FBC objects. For callers that can operate on a single meta at a time, these functions will provide significant reductions in memory usage. Signed-off-by: Joe Lanford <[email protected]> * address PR feedback Signed-off-by: Joe Lanford <[email protected]> --------- Signed-off-by: Joe Lanford <[email protected]>
1 parent 41fa00f commit e1eebae

File tree

5 files changed

+208
-76
lines changed

5 files changed

+208
-76
lines changed

‎alpha/declcfg/declcfg.go‎

Lines changed: 59 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,13 @@
11
package declcfg
22

33
import (
4+
"bytes"
45
"encoding/json"
6+
"errors"
7+
"fmt"
8+
"strings"
9+
10+
"go4.org/bytereplacer"
511

612
"github.com/operator-framework/operator-registry/alpha/property"
713
)
@@ -84,6 +90,7 @@ type RelatedImage struct {
8490
type Meta struct {
8591
Schema string
8692
Package string
93+
Name string
8794

8895
Blob json.RawMessage
8996
}
@@ -93,17 +100,68 @@ func (m Meta) MarshalJSON() ([]byte, error) {
93100
}
94101

95102
func (m *Meta) UnmarshalJSON(blob []byte) error {
103+
blob = bytereplacer.New(`\u003c`, "<", `\u003e`, ">", `\u0026`, "&").Replace(blob)
104+
96105
type tmp struct {
97106
Schema string `json:"schema"`
98107
Package string `json:"package,omitempty"`
108+
Name string `json:"name,omitempty"`
99109
Properties []property.Property `json:"properties,omitempty"`
100110
}
101111
var t tmp
102112
if err := json.Unmarshal(blob, &t); err != nil {
103-
return err
113+
// TODO: return an error that includes the the full JSON message,
114+
// the offset of the error, and the error message. Let callers
115+
// decide how to format it.
116+
return errors.New(resolveUnmarshalErr(blob, err))
104117
}
105118
m.Schema = t.Schema
106119
m.Package = t.Package
120+
m.Name = t.Name
107121
m.Blob = blob
108122
return nil
109123
}
124+
125+
func resolveUnmarshalErr(data []byte, err error) string {
126+
var te *json.UnmarshalTypeError
127+
if errors.As(err, &te) {
128+
return formatUnmarshallErrorString(data, te.Error(), te.Offset)
129+
}
130+
var se *json.SyntaxError
131+
if errors.As(err, &se) {
132+
return formatUnmarshallErrorString(data, se.Error(), se.Offset)
133+
}
134+
return err.Error()
135+
}
136+
137+
func formatUnmarshallErrorString(data []byte, errmsg string, offset int64) string {
138+
sb := new(strings.Builder)
139+
_, _ = sb.WriteString(fmt.Sprintf("%s at offset %d (indicated by <==)\n ", errmsg, offset))
140+
// attempt to present the erroneous JSON in indented, human-readable format
141+
// errors result in presenting the original, unformatted output
142+
var pretty bytes.Buffer
143+
err := json.Indent(&pretty, data, "", " ")
144+
if err == nil {
145+
pString := pretty.String()
146+
// calc the prettified string offset which correlates to the original string offset
147+
var pOffset, origOffset int64
148+
origOffset = 0
149+
for origOffset = 0; origOffset < offset; {
150+
if pString[pOffset] != '\n' && pString[pOffset] != ' ' {
151+
origOffset++
152+
}
153+
pOffset++
154+
}
155+
_, _ = sb.WriteString(pString[:pOffset])
156+
_, _ = sb.WriteString(" <== ")
157+
_, _ = sb.WriteString(pString[pOffset:])
158+
} else {
159+
for i := int64(0); i < offset; i++ {
160+
_ = sb.WriteByte(data[i])
161+
}
162+
_, _ = sb.WriteString(" <== ")
163+
_, _ = sb.Write(data[offset:])
164+
}
165+
166+
return sb.String()
167+
}

‎alpha/declcfg/load.go‎

Lines changed: 71 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,12 @@
11
package declcfg
22

33
import (
4-
"bytes"
54
"encoding/json"
65
"errors"
76
"fmt"
87
"io"
98
"io/fs"
109
"path/filepath"
11-
"strings"
1210

1311
"github.com/joelanford/ignore"
1412
"github.com/operator-framework/api/pkg/operators"
@@ -22,13 +20,68 @@ const (
2220
indexIgnoreFilename = ".indexignore"
2321
)
2422

23+
type WalkMetasFSFunc func(path string, meta *Meta, err error) error
24+
25+
func WalkMetasFS(root fs.FS, walkFn WalkMetasFSFunc) error {
26+
return walkFiles(root, func(root fs.FS, path string, err error) error {
27+
if err != nil {
28+
return walkFn(path, nil, err)
29+
}
30+
31+
f, err := root.Open(path)
32+
if err != nil {
33+
return walkFn(path, nil, err)
34+
}
35+
defer f.Close()
36+
37+
return WalkMetasReader(f, func(meta *Meta, err error) error {
38+
return walkFn(path, meta, err)
39+
})
40+
})
41+
}
42+
43+
type WalkMetasReaderFunc func(meta *Meta, err error) error
44+
45+
func WalkMetasReader(r io.Reader, walkFn WalkMetasReaderFunc) error {
46+
dec := yaml.NewYAMLOrJSONDecoder(r, 4096)
47+
for {
48+
var in Meta
49+
if err := dec.Decode(&in); err != nil {
50+
if errors.Is(err, io.EOF) {
51+
break
52+
}
53+
return walkFn(nil, err)
54+
}
55+
56+
if err := walkFn(&in, nil); err != nil {
57+
return err
58+
}
59+
}
60+
return nil
61+
}
62+
2563
type WalkFunc func(path string, cfg *DeclarativeConfig, err error) error
2664

2765
// WalkFS walks root using a gitignore-style filename matcher to skip files
2866
// that match patterns found in .indexignore files found throughout the filesystem.
2967
// It calls walkFn for each declarative config file it finds. If WalkFS encounters
3068
// an error loading or parsing any file, the error will be immediately returned.
3169
func WalkFS(root fs.FS, walkFn WalkFunc) error {
70+
return walkFiles(root, func(root fs.FS, path string, err error) error {
71+
if err != nil {
72+
return walkFn(path, nil, err)
73+
}
74+
75+
cfg, err := LoadFile(root, path)
76+
if err != nil {
77+
return walkFn(path, cfg, err)
78+
}
79+
80+
return walkFn(path, cfg, nil)
81+
})
82+
}
83+
84+
func walkFiles(root fs.FS, fn func(root fs.FS, path string, err error) error) error {
3285
if root == nil {
3386
return fmt.Errorf("no declarative config filesystem provided")
3487
}
@@ -40,20 +93,15 @@ func WalkFS(root fs.FS, walkFn WalkFunc) error {
4093

4194
return fs.WalkDir(root, ".", func(path string, info fs.DirEntry, err error) error {
4295
if err != nil {
43-
return walkFn(path, nil, err)
96+
return fn(root, path, err)
4497
}
4598
// avoid validating a directory, an .indexignore file, or any file that matches
4699
// an ignore pattern outlined in a .indexignore file.
47100
if info.IsDir() || info.Name() == indexIgnoreFilename || matcher.Match(path, false) {
48101
return nil
49102
}
50103

51-
cfg, err := LoadFile(root, path)
52-
if err != nil {
53-
return walkFn(path, cfg, err)
54-
}
55-
56-
return walkFn(path, cfg, err)
104+
return fn(root, path, nil)
57105
})
58106
}
59107

@@ -123,46 +171,38 @@ func extractCSV(objs []string) string {
123171
// Path references will not be de-referenced so callers are responsible for de-referencing if necessary.
124172
func LoadReader(r io.Reader) (*DeclarativeConfig, error) {
125173
cfg := &DeclarativeConfig{}
126-
dec := yaml.NewYAMLOrJSONDecoder(r, 4096)
127-
for {
128-
doc := json.RawMessage{}
129-
if err := dec.Decode(&doc); err != nil {
130-
if errors.Is(err, io.EOF) {
131-
break
132-
}
133-
return nil, err
134-
}
135-
doc = []byte(strings.NewReplacer(`\u003c`, "<", `\u003e`, ">", `\u0026`, "&").Replace(string(doc)))
136174

137-
var in Meta
138-
if err := json.Unmarshal(doc, &in); err != nil {
139-
return nil, fmt.Errorf("unmarshal error: %s", resolveUnmarshalErr(doc, err))
175+
if err := WalkMetasReader(r, func(in *Meta, err error) error {
176+
if err != nil {
177+
return err
140178
}
141-
142179
switch in.Schema {
143180
case SchemaPackage:
144181
var p Package
145-
if err := json.Unmarshal(doc, &p); err != nil {
146-
return nil, fmt.Errorf("parse package: %v", err)
182+
if err := json.Unmarshal(in.Blob, &p); err != nil {
183+
return fmt.Errorf("parse package: %v", err)
147184
}
148185
cfg.Packages = append(cfg.Packages, p)
149186
case SchemaChannel:
150187
var c Channel
151-
if err := json.Unmarshal(doc, &c); err != nil {
152-
return nil, fmt.Errorf("parse channel: %v", err)
188+
if err := json.Unmarshal(in.Blob, &c); err != nil {
189+
return fmt.Errorf("parse channel: %v", err)
153190
}
154191
cfg.Channels = append(cfg.Channels, c)
155192
case SchemaBundle:
156193
var b Bundle
157-
if err := json.Unmarshal(doc, &b); err != nil {
158-
return nil, fmt.Errorf("parse bundle: %v", err)
194+
if err := json.Unmarshal(in.Blob, &b); err != nil {
195+
return fmt.Errorf("parse bundle: %v", err)
159196
}
160197
cfg.Bundles = append(cfg.Bundles, b)
161198
case "":
162-
return nil, fmt.Errorf("object '%s' is missing root schema field", string(doc))
199+
return fmt.Errorf("object '%s' is missing root schema field", string(in.Blob))
163200
default:
164-
cfg.Others = append(cfg.Others, in)
201+
cfg.Others = append(cfg.Others, *in)
165202
}
203+
return nil
204+
}); err != nil {
205+
return nil, err
166206
}
167207
return cfg, nil
168208
}
@@ -187,47 +227,3 @@ func LoadFile(root fs.FS, path string) (*DeclarativeConfig, error) {
187227

188228
return cfg, nil
189229
}
190-
191-
func resolveUnmarshalErr(data []byte, err error) string {
192-
var te *json.UnmarshalTypeError
193-
if errors.As(err, &te) {
194-
return formatUnmarshallErrorString(data, te.Error(), te.Offset)
195-
}
196-
var se *json.SyntaxError
197-
if errors.As(err, &se) {
198-
return formatUnmarshallErrorString(data, se.Error(), se.Offset)
199-
}
200-
return err.Error()
201-
}
202-
203-
func formatUnmarshallErrorString(data []byte, errmsg string, offset int64) string {
204-
sb := new(strings.Builder)
205-
_, _ = sb.WriteString(fmt.Sprintf("%s at offset %d (indicated by <==)\n ", errmsg, offset))
206-
// attempt to present the erroneous JSON in indented, human-readable format
207-
// errors result in presenting the original, unformatted output
208-
var pretty bytes.Buffer
209-
err := json.Indent(&pretty, data, "", " ")
210-
if err == nil {
211-
pString := pretty.String()
212-
// calc the prettified string offset which correlates to the original string offset
213-
var pOffset, origOffset int64
214-
origOffset = 0
215-
for origOffset = 0; origOffset < offset; {
216-
pOffset++
217-
if pString[pOffset] != '\n' && pString[pOffset] != ' ' {
218-
origOffset++
219-
}
220-
}
221-
_, _ = sb.WriteString(pString[:pOffset])
222-
_, _ = sb.WriteString(" <== ")
223-
_, _ = sb.WriteString(pString[pOffset:])
224-
} else {
225-
for i := int64(0); i < offset; i++ {
226-
_ = sb.WriteByte(data[i])
227-
}
228-
_, _ = sb.WriteString(" <== ")
229-
_, _ = sb.Write(data[offset:])
230-
}
231-
232-
return sb.String()
233-
}

‎alpha/declcfg/load_test.go‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,73 @@ func TestLoadReader(t *testing.T) {
9292
}
9393
}
9494

95+
func TestWalkMetasFS(t *testing.T) {
96+
type spec struct {
97+
name string
98+
fsys fs.FS
99+
assertion require.ErrorAssertionFunc
100+
expectNumPackages int
101+
expectNumChannels int
102+
expectNumBundles int
103+
expectNumOthers int
104+
}
105+
specs := []spec{
106+
{
107+
name: "Error/NilFS",
108+
fsys: nil,
109+
assertion: require.Error,
110+
},
111+
{
112+
name: "Error/NonExistentDir",
113+
fsys: os.DirFS("non/existent/dir/"),
114+
assertion: require.Error,
115+
},
116+
{
117+
name: "Error/Invalid",
118+
fsys: invalidFS,
119+
assertion: require.Error,
120+
},
121+
{
122+
name: "Success/ValidDir",
123+
fsys: validFS,
124+
assertion: require.NoError,
125+
expectNumPackages: 3,
126+
expectNumChannels: 0,
127+
expectNumBundles: 12,
128+
expectNumOthers: 1,
129+
},
130+
}
131+
132+
for _, s := range specs {
133+
t.Run(s.name, func(t *testing.T) {
134+
numPackages, numChannels, numBundles, numOthers := 0, 0, 0, 0
135+
err := WalkMetasFS(s.fsys, func(path string, meta *Meta, err error) error {
136+
if err != nil {
137+
return err
138+
}
139+
switch meta.Schema {
140+
case SchemaPackage:
141+
numPackages++
142+
case SchemaChannel:
143+
numChannels++
144+
case SchemaBundle:
145+
numBundles++
146+
default:
147+
numOthers++
148+
}
149+
return nil
150+
})
151+
s.assertion(t, err)
152+
if err == nil {
153+
assert.Equal(t, s.expectNumPackages, numPackages, "unexpected package count")
154+
assert.Equal(t, s.expectNumChannels, numChannels, "unexpected channel count")
155+
assert.Equal(t, s.expectNumBundles, numBundles, "unexpected bundle count")
156+
assert.Equal(t, s.expectNumOthers, numOthers, "unexpected others count")
157+
}
158+
})
159+
}
160+
}
161+
95162
func TestLoadFS(t *testing.T) {
96163
type spec struct {
97164
name string

‎go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ require (
3131
github.com/spf13/cobra v1.6.0
3232
github.com/stretchr/testify v1.8.0
3333
go.etcd.io/bbolt v1.3.6
34+
go4.org v0.0.0-20230225012048-214862532bf5
3435
golang.org/x/mod v0.6.0
3536
golang.org/x/net v0.7.0
3637
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4

0 commit comments

Comments
 (0)