Skip to content

Commit bce843d

Browse files
committed
(feat) Add semver insert mode
This adds a new flag to `opm index add` and `opm registry add` which defines two new insert modes to add bundles to the existing update graph Rather than rely on parsing the CSV for the replaces field to define the channel update graph explicitly, in these new modes `opm` implicitly infers the update graph based on the version attached to the bundle and semantic versioning rules (https://semver.org/#summary)
1 parent 1a514b0 commit bce843d

File tree

22 files changed

+1004
-65
lines changed

22 files changed

+1004
-65
lines changed

cmd/opm/index/add.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"k8s.io/kubectl/pkg/util/templates"
99

1010
"github.com/operator-framework/operator-registry/pkg/lib/indexer"
11+
"github.com/operator-framework/operator-registry/pkg/registry"
1112
)
1213

1314
var (
@@ -55,6 +56,7 @@ func addIndexAddCmd(parent *cobra.Command) {
5556
indexCmd.Flags().StringP("container-tool", "c", "podman", "tool to interact with container images (save, build, etc.). One of: [docker, podman]")
5657
indexCmd.Flags().StringP("tag", "t", "", "custom tag for container image being built")
5758
indexCmd.Flags().Bool("permissive", false, "allow registry load errors")
59+
indexCmd.Flags().StringP("mode", "", "replaces", "graph update mode that defines how channel graphs are updated. One of: [replaces, semver, semver-skippatch]")
5860

5961
if err := indexCmd.Flags().MarkHidden("debug"); err != nil {
6062
logrus.Panic(err.Error())
@@ -107,6 +109,16 @@ func runIndexAddCmdFunc(cmd *cobra.Command, args []string) error {
107109
return err
108110
}
109111

112+
mode, err := cmd.Flags().GetString("mode")
113+
if err != nil {
114+
return err
115+
}
116+
117+
modeEnum, err := registry.GetModeFromString(mode)
118+
if err != nil {
119+
return err
120+
}
121+
110122
logger := logrus.WithFields(logrus.Fields{"bundles": bundles})
111123

112124
logger.Info("building the index")
@@ -121,6 +133,7 @@ func runIndexAddCmdFunc(cmd *cobra.Command, args []string) error {
121133
Tag: tag,
122134
Bundles: bundles,
123135
Permissive: permissive,
136+
Mode: modeEnum,
124137
}
125138

126139
err = indexAdder.AddToIndex(request)

cmd/opm/registry/add.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"github.com/spf13/cobra"
66

77
"github.com/operator-framework/operator-registry/pkg/lib/registry"
8+
reg "github.com/operator-framework/operator-registry/pkg/registry"
89
)
910

1011
func newRegistryAddCmd() *cobra.Command {
@@ -28,6 +29,7 @@ func newRegistryAddCmd() *cobra.Command {
2829
rootCmd.Flags().StringSliceP("bundle-images", "b", []string{}, "comma separated list of links to bundle image")
2930
rootCmd.Flags().Bool("permissive", false, "allow registry load errors")
3031
rootCmd.Flags().Bool("skip-tls", false, "skip TLS certificate verification for container image registries while pulling bundles")
32+
rootCmd.Flags().StringP("mode", "", "replaces", "graph update mode that defines how channel graphs are updated. One of: [replaces, semver, semver-skippatch]")
3133

3234
rootCmd.Flags().StringP("container-tool", "c", "", "")
3335
if err := rootCmd.Flags().MarkDeprecated("container-tool", "ignored in favor of standalone image manipulation"); err != nil {
@@ -55,11 +57,22 @@ func addFunc(cmd *cobra.Command, args []string) error {
5557
return err
5658
}
5759

60+
mode, err := cmd.Flags().GetString("mode")
61+
if err != nil {
62+
return err
63+
}
64+
65+
modeEnum, err := reg.GetModeFromString(mode)
66+
if err != nil {
67+
return err
68+
}
69+
5870
request := registry.AddToRegistryRequest{
5971
Permissive: permissive,
6072
SkipTLS: skipTLS,
6173
InputDatabase: fromFilename,
6274
Bundles: bundleImages,
75+
Mode: modeEnum,
6376
}
6477

6578
logger := logrus.WithFields(logrus.Fields{"bundles": bundleImages})

pkg/lib/bundle/validate.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -272,7 +272,7 @@ func (i imageValidator) ValidateBundleContent(manifestDir string) error {
272272

273273
// Validate the bundle object
274274
if len(unstObjs) > 0 {
275-
bundle := registry.NewBundle(csvName, "", "", unstObjs...)
275+
bundle := registry.NewBundle(csvName, "", nil, unstObjs...)
276276
bundleValidator := v.BundleValidator
277277
results := bundleValidator.Validate(bundle)
278278
if len(results) > 0 {

pkg/lib/indexer/indexer.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ type AddToIndexRequest struct {
4949
OutDockerfile string
5050
Bundles []string
5151
Tag string
52+
Mode pregistry.Mode
5253
}
5354

5455
// AddToIndex is an aggregate API used to generate a registry index image with additional bundles
@@ -89,6 +90,7 @@ func (i ImageIndexer) AddToIndex(request AddToIndexRequest) error {
8990
Bundles: request.Bundles,
9091
InputDatabase: databaseFile,
9192
Permissive: request.Permissive,
93+
Mode: request.Mode,
9294
}
9395

9496
// Add the bundles to the registry

pkg/lib/registry/registry.go

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ type AddToRegistryRequest struct {
2525
SkipTLS bool
2626
InputDatabase string
2727
Bundles []string
28+
Mode registry.Mode
2829
}
2930

3031
func (r RegistryUpdater) AddToRegistry(request AddToRegistryRequest) error {
@@ -44,6 +45,11 @@ func (r RegistryUpdater) AddToRegistry(request AddToRegistryRequest) error {
4445
return err
4546
}
4647

48+
graphLoader, err := sqlite.NewSQLGraphLoaderFromDB(db)
49+
if err != nil {
50+
return err
51+
}
52+
4753
// TODO: Dependency inject the registry if we want to swap it out.
4854
reg, destroy, err := containerdregistry.NewRegistry(
4955
containerdregistry.SkipTLS(request.SkipTLS),
@@ -59,7 +65,7 @@ func (r RegistryUpdater) AddToRegistry(request AddToRegistryRequest) error {
5965

6066
// TODO(njhale): Parallelize this once bundle add is commutative
6167
for _, ref := range request.Bundles {
62-
if err := populate(context.TODO(), dbLoader, reg, image.SimpleReference(ref)); err != nil {
68+
if err := populate(context.TODO(), dbLoader, graphLoader, reg, image.SimpleReference(ref), request.Mode); err != nil {
6369
err = fmt.Errorf("error loading bundle from image: %s", err)
6470
if !request.Permissive {
6571
r.Logger.WithError(err).Error("permissive mode disabled")
@@ -73,7 +79,7 @@ func (r RegistryUpdater) AddToRegistry(request AddToRegistryRequest) error {
7379
return utilerrors.NewAggregate(errs) // nil if no errors
7480
}
7581

76-
func populate(ctx context.Context, loader registry.Load, reg image.Registry, ref image.Reference) error {
82+
func populate(ctx context.Context, loader registry.Load, graphLoader registry.GraphLoader, reg image.Registry, ref image.Reference, mode registry.Mode) error {
7783
workingDir, err := ioutil.TempDir("./", "bundle_tmp")
7884
if err != nil {
7985
return err
@@ -88,9 +94,9 @@ func populate(ctx context.Context, loader registry.Load, reg image.Registry, ref
8894
return err
8995
}
9096

91-
populator := registry.NewDirectoryPopulator(loader, ref, workingDir)
97+
populator := registry.NewDirectoryPopulator(loader, graphLoader, ref, workingDir)
9298

93-
return populator.Populate()
99+
return populator.Populate(mode)
94100
}
95101

96102
type DeleteFromRegistryRequest struct {

pkg/registry/bundle.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,22 +31,22 @@ type Bundle struct {
3131
Name string
3232
Objects []*unstructured.Unstructured
3333
Package string
34-
Channel string
34+
Channels []string
3535
BundleImage string
3636
csv *ClusterServiceVersion
3737
crds []*v1beta1.CustomResourceDefinition
3838
cacheStale bool
3939
}
4040

41-
func NewBundle(name, pkgName, channelName string, objs ...*unstructured.Unstructured) *Bundle {
42-
bundle := &Bundle{Name: name, Package: pkgName, Channel: channelName, cacheStale: false}
41+
func NewBundle(name, pkgName string, channels []string, objs ...*unstructured.Unstructured) *Bundle {
42+
bundle := &Bundle{Name: name, Package: pkgName, Channels: channels, cacheStale: false}
4343
for _, o := range objs {
4444
bundle.Add(o)
4545
}
4646
return bundle
4747
}
4848

49-
func NewBundleFromStrings(name, pkgName, channelName string, objs []string) (*Bundle, error) {
49+
func NewBundleFromStrings(name, pkgName string, channels []string, objs []string) (*Bundle, error) {
5050
unstObjs := []*unstructured.Unstructured{}
5151
for _, o := range objs {
5252
dec := yaml.NewYAMLOrJSONDecoder(strings.NewReader(o), 10)
@@ -56,7 +56,7 @@ func NewBundleFromStrings(name, pkgName, channelName string, objs []string) (*Bu
5656
}
5757
unstObjs = append(unstObjs, unst)
5858
}
59-
return NewBundle(name, pkgName, channelName, unstObjs...), nil
59+
return NewBundle(name, pkgName, channels, unstObjs...), nil
6060
}
6161

6262
func (b *Bundle) Size() int {

pkg/registry/bundlegraphloader.go

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
package registry
2+
3+
import (
4+
"fmt"
5+
6+
"github.com/blang/semver"
7+
)
8+
9+
// BundleGraphLoader generates updated graphs by adding bundles to them, updating
10+
// the graph implicitly via semantic version of each bundle
11+
type BundleGraphLoader struct {
12+
}
13+
14+
// AddBundleToGraph takes a bundle and an existing graph and updates the graph to insert the new bundle
15+
// into each channel it is included in
16+
func (g *BundleGraphLoader) AddBundleToGraph(bundle *Bundle, graph *Package, newDefaultChannel string, skippatch bool) (*Package, error) {
17+
bundleVersion, err := bundle.Version()
18+
if err != nil {
19+
return nil, fmt.Errorf("Unable to extract bundle version from bundle %s, can't insert in semver mode", bundle.BundleImage)
20+
}
21+
22+
versionToAdd, err := semver.Make(bundleVersion)
23+
if err != nil {
24+
return nil, fmt.Errorf("Bundle version %s is not valid", bundleVersion)
25+
}
26+
27+
newBundleKey := BundleKey{
28+
CsvName: bundle.Name,
29+
Version: versionToAdd.String(),
30+
BundlePath: bundle.BundleImage,
31+
}
32+
33+
// initialize the graph if it started empty
34+
if graph.Name == "" {
35+
graph.Name = bundle.Package
36+
}
37+
if newDefaultChannel != "" {
38+
graph.DefaultChannel = newDefaultChannel
39+
}
40+
41+
// generate the DAG for each channel the new bundle is being insert into
42+
for _, channel := range bundle.Channels {
43+
replaces := make(map[BundleKey]struct{}, 0)
44+
45+
// If the channel doesn't exist yet, initialize it
46+
if !graph.HasChannel(channel) {
47+
// create the channel and add a single node
48+
newChannelGraph := Channel{
49+
Head: newBundleKey,
50+
Nodes: map[BundleKey]map[BundleKey]struct{}{
51+
newBundleKey: nil,
52+
},
53+
}
54+
if graph.Channels == nil {
55+
graph.Channels = make(map[string]Channel, 1)
56+
}
57+
graph.Channels[channel] = newChannelGraph
58+
continue
59+
}
60+
61+
// find the version(s) it should sit between
62+
channelGraph := graph.Channels[channel]
63+
if channelGraph.Nodes == nil {
64+
channelGraph.Nodes = make(map[BundleKey]map[BundleKey]struct{}, 1)
65+
}
66+
67+
lowestAhead := BundleKey{}
68+
greatestBehind := BundleKey{}
69+
skipPatchCandidates := []BundleKey{}
70+
71+
// Iterate over existing nodes and compare the new node's version to find the
72+
// lowest version above it and highest version below it (to insert between these nodes)
73+
for node := range channelGraph.Nodes {
74+
nodeVersion, err := semver.Make(node.Version)
75+
if err != nil {
76+
return nil, fmt.Errorf("Unable to parse existing bundle version stored in index %s %s %s",
77+
node.CsvName, node.Version, node.BundlePath)
78+
}
79+
80+
switch comparison := nodeVersion.Compare(versionToAdd); comparison {
81+
case 0:
82+
return nil, fmt.Errorf("Bundle version %s already added to index", bundleVersion)
83+
case 1:
84+
if lowestAhead.IsEmpty() {
85+
lowestAhead = node
86+
} else {
87+
lowestAheadSemver, _ := semver.Make(lowestAhead.Version)
88+
if nodeVersion.LT(lowestAheadSemver) {
89+
lowestAhead = node
90+
}
91+
}
92+
case -1:
93+
if greatestBehind.IsEmpty() {
94+
greatestBehind = node
95+
} else {
96+
greatestBehindSemver, _ := semver.Make(greatestBehind.Version)
97+
if nodeVersion.GT(greatestBehindSemver) {
98+
greatestBehind = node
99+
}
100+
}
101+
}
102+
103+
// if skippatch mode is enabled, check each node to determine if z-updates should
104+
// be replaced as well. Keep track of them to delete those nodes from the graph itself,
105+
// just be aware of them for replacements
106+
if skippatch {
107+
if isSkipPatchCandidate(versionToAdd, nodeVersion) {
108+
skipPatchCandidates = append(skipPatchCandidates, node)
109+
replaces[node] = struct{}{}
110+
}
111+
}
112+
}
113+
114+
// If we found a node behind the one we're adding, make the new node replace it
115+
if !greatestBehind.IsEmpty() {
116+
replaces[greatestBehind] = struct{}{}
117+
}
118+
119+
// If we found a node ahead of the one we're adding, make the lowest to replace
120+
// the new node. If we didn't find a node semantically ahead, the new node is
121+
// the new channel head
122+
if !lowestAhead.IsEmpty() {
123+
channelGraph.Nodes[lowestAhead] = map[BundleKey]struct{}{
124+
newBundleKey: struct{}{},
125+
}
126+
} else {
127+
channelGraph.Head = newBundleKey
128+
}
129+
130+
if skippatch {
131+
// Remove the nodes that are now being skipped by a new patch version update
132+
for _, candidate := range skipPatchCandidates {
133+
delete(channelGraph.Nodes, candidate)
134+
}
135+
}
136+
137+
// add the node and update the graph
138+
channelGraph.Nodes[newBundleKey] = replaces
139+
graph.Channels[channel] = channelGraph
140+
}
141+
142+
return graph, nil
143+
}
144+
145+
func isSkipPatchCandidate(version, toCompare semver.Version) bool {
146+
return (version.Major == toCompare.Major) && (version.Minor == toCompare.Minor) && (version.Patch > toCompare.Patch)
147+
}

0 commit comments

Comments
 (0)