Skip to content

Commit cac3168

Browse files
Merge pull request #247 from kevinrizza/semver-mode
Semver index insert
2 parents 09daf8d + bce843d commit cac3168

File tree

22 files changed

+1004
-65
lines changed

22 files changed

+1004
-65
lines changed

cmd/opm/index/add.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"k8s.io/kubectl/pkg/util/templates"
99

1010
"github.com/operator-framework/operator-registry/pkg/lib/indexer"
11+
"github.com/operator-framework/operator-registry/pkg/registry"
1112
)
1213

1314
var (
@@ -55,6 +56,7 @@ func addIndexAddCmd(parent *cobra.Command) {
5556
indexCmd.Flags().StringP("container-tool", "c", "podman", "tool to interact with container images (save, build, etc.). One of: [docker, podman]")
5657
indexCmd.Flags().StringP("tag", "t", "", "custom tag for container image being built")
5758
indexCmd.Flags().Bool("permissive", false, "allow registry load errors")
59+
indexCmd.Flags().StringP("mode", "", "replaces", "graph update mode that defines how channel graphs are updated. One of: [replaces, semver, semver-skippatch]")
5860

5961
if err := indexCmd.Flags().MarkHidden("debug"); err != nil {
6062
logrus.Panic(err.Error())
@@ -107,6 +109,16 @@ func runIndexAddCmdFunc(cmd *cobra.Command, args []string) error {
107109
return err
108110
}
109111

112+
mode, err := cmd.Flags().GetString("mode")
113+
if err != nil {
114+
return err
115+
}
116+
117+
modeEnum, err := registry.GetModeFromString(mode)
118+
if err != nil {
119+
return err
120+
}
121+
110122
logger := logrus.WithFields(logrus.Fields{"bundles": bundles})
111123

112124
logger.Info("building the index")
@@ -121,6 +133,7 @@ func runIndexAddCmdFunc(cmd *cobra.Command, args []string) error {
121133
Tag: tag,
122134
Bundles: bundles,
123135
Permissive: permissive,
136+
Mode: modeEnum,
124137
}
125138

126139
err = indexAdder.AddToIndex(request)

cmd/opm/registry/add.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"github.com/spf13/cobra"
66

77
"github.com/operator-framework/operator-registry/pkg/lib/registry"
8+
reg "github.com/operator-framework/operator-registry/pkg/registry"
89
)
910

1011
func newRegistryAddCmd() *cobra.Command {
@@ -28,6 +29,7 @@ func newRegistryAddCmd() *cobra.Command {
2829
rootCmd.Flags().StringSliceP("bundle-images", "b", []string{}, "comma separated list of links to bundle image")
2930
rootCmd.Flags().Bool("permissive", false, "allow registry load errors")
3031
rootCmd.Flags().Bool("skip-tls", false, "skip TLS certificate verification for container image registries while pulling bundles")
32+
rootCmd.Flags().StringP("mode", "", "replaces", "graph update mode that defines how channel graphs are updated. One of: [replaces, semver, semver-skippatch]")
3133

3234
rootCmd.Flags().StringP("container-tool", "c", "", "")
3335
if err := rootCmd.Flags().MarkDeprecated("container-tool", "ignored in favor of standalone image manipulation"); err != nil {
@@ -55,11 +57,22 @@ func addFunc(cmd *cobra.Command, args []string) error {
5557
return err
5658
}
5759

60+
mode, err := cmd.Flags().GetString("mode")
61+
if err != nil {
62+
return err
63+
}
64+
65+
modeEnum, err := reg.GetModeFromString(mode)
66+
if err != nil {
67+
return err
68+
}
69+
5870
request := registry.AddToRegistryRequest{
5971
Permissive: permissive,
6072
SkipTLS: skipTLS,
6173
InputDatabase: fromFilename,
6274
Bundles: bundleImages,
75+
Mode: modeEnum,
6376
}
6477

6578
logger := logrus.WithFields(logrus.Fields{"bundles": bundleImages})

pkg/lib/bundle/validate.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -272,7 +272,7 @@ func (i imageValidator) ValidateBundleContent(manifestDir string) error {
272272

273273
// Validate the bundle object
274274
if len(unstObjs) > 0 {
275-
bundle := registry.NewBundle(csvName, "", "", unstObjs...)
275+
bundle := registry.NewBundle(csvName, "", nil, unstObjs...)
276276
bundleValidator := v.BundleValidator
277277
results := bundleValidator.Validate(bundle)
278278
if len(results) > 0 {

pkg/lib/indexer/indexer.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ type AddToIndexRequest struct {
4949
OutDockerfile string
5050
Bundles []string
5151
Tag string
52+
Mode pregistry.Mode
5253
}
5354

5455
// AddToIndex is an aggregate API used to generate a registry index image with additional bundles
@@ -89,6 +90,7 @@ func (i ImageIndexer) AddToIndex(request AddToIndexRequest) error {
8990
Bundles: request.Bundles,
9091
InputDatabase: databaseFile,
9192
Permissive: request.Permissive,
93+
Mode: request.Mode,
9294
}
9395

9496
// Add the bundles to the registry

pkg/lib/registry/registry.go

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ type AddToRegistryRequest struct {
2525
SkipTLS bool
2626
InputDatabase string
2727
Bundles []string
28+
Mode registry.Mode
2829
}
2930

3031
func (r RegistryUpdater) AddToRegistry(request AddToRegistryRequest) error {
@@ -44,6 +45,11 @@ func (r RegistryUpdater) AddToRegistry(request AddToRegistryRequest) error {
4445
return err
4546
}
4647

48+
graphLoader, err := sqlite.NewSQLGraphLoaderFromDB(db)
49+
if err != nil {
50+
return err
51+
}
52+
4753
// TODO: Dependency inject the registry if we want to swap it out.
4854
reg, destroy, err := containerdregistry.NewRegistry(
4955
containerdregistry.SkipTLS(request.SkipTLS),
@@ -59,7 +65,7 @@ func (r RegistryUpdater) AddToRegistry(request AddToRegistryRequest) error {
5965

6066
// TODO(njhale): Parallelize this once bundle add is commutative
6167
for _, ref := range request.Bundles {
62-
if err := populate(context.TODO(), dbLoader, reg, image.SimpleReference(ref)); err != nil {
68+
if err := populate(context.TODO(), dbLoader, graphLoader, reg, image.SimpleReference(ref), request.Mode); err != nil {
6369
err = fmt.Errorf("error loading bundle from image: %s", err)
6470
if !request.Permissive {
6571
r.Logger.WithError(err).Error("permissive mode disabled")
@@ -73,7 +79,7 @@ func (r RegistryUpdater) AddToRegistry(request AddToRegistryRequest) error {
7379
return utilerrors.NewAggregate(errs) // nil if no errors
7480
}
7581

76-
func populate(ctx context.Context, loader registry.Load, reg image.Registry, ref image.Reference) error {
82+
func populate(ctx context.Context, loader registry.Load, graphLoader registry.GraphLoader, reg image.Registry, ref image.Reference, mode registry.Mode) error {
7783
workingDir, err := ioutil.TempDir("./", "bundle_tmp")
7884
if err != nil {
7985
return err
@@ -88,9 +94,9 @@ func populate(ctx context.Context, loader registry.Load, reg image.Registry, ref
8894
return err
8995
}
9096

91-
populator := registry.NewDirectoryPopulator(loader, ref, workingDir)
97+
populator := registry.NewDirectoryPopulator(loader, graphLoader, ref, workingDir)
9298

93-
return populator.Populate()
99+
return populator.Populate(mode)
94100
}
95101

96102
type DeleteFromRegistryRequest struct {

pkg/registry/bundle.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,22 +31,22 @@ type Bundle struct {
3131
Name string
3232
Objects []*unstructured.Unstructured
3333
Package string
34-
Channel string
34+
Channels []string
3535
BundleImage string
3636
csv *ClusterServiceVersion
3737
crds []*v1beta1.CustomResourceDefinition
3838
cacheStale bool
3939
}
4040

41-
func NewBundle(name, pkgName, channelName string, objs ...*unstructured.Unstructured) *Bundle {
42-
bundle := &Bundle{Name: name, Package: pkgName, Channel: channelName, cacheStale: false}
41+
func NewBundle(name, pkgName string, channels []string, objs ...*unstructured.Unstructured) *Bundle {
42+
bundle := &Bundle{Name: name, Package: pkgName, Channels: channels, cacheStale: false}
4343
for _, o := range objs {
4444
bundle.Add(o)
4545
}
4646
return bundle
4747
}
4848

49-
func NewBundleFromStrings(name, pkgName, channelName string, objs []string) (*Bundle, error) {
49+
func NewBundleFromStrings(name, pkgName string, channels []string, objs []string) (*Bundle, error) {
5050
unstObjs := []*unstructured.Unstructured{}
5151
for _, o := range objs {
5252
dec := yaml.NewYAMLOrJSONDecoder(strings.NewReader(o), 10)
@@ -56,7 +56,7 @@ func NewBundleFromStrings(name, pkgName, channelName string, objs []string) (*Bu
5656
}
5757
unstObjs = append(unstObjs, unst)
5858
}
59-
return NewBundle(name, pkgName, channelName, unstObjs...), nil
59+
return NewBundle(name, pkgName, channels, unstObjs...), nil
6060
}
6161

6262
func (b *Bundle) Size() int {

pkg/registry/bundlegraphloader.go

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
package registry
2+
3+
import (
4+
"fmt"
5+
6+
"github.com/blang/semver"
7+
)
8+
9+
// BundleGraphLoader generates updated graphs by adding bundles to them, updating
10+
// the graph implicitly via semantic version of each bundle
11+
type BundleGraphLoader struct {
12+
}
13+
14+
// AddBundleToGraph takes a bundle and an existing graph and updates the graph to insert the new bundle
15+
// into each channel it is included in
16+
func (g *BundleGraphLoader) AddBundleToGraph(bundle *Bundle, graph *Package, newDefaultChannel string, skippatch bool) (*Package, error) {
17+
bundleVersion, err := bundle.Version()
18+
if err != nil {
19+
return nil, fmt.Errorf("Unable to extract bundle version from bundle %s, can't insert in semver mode", bundle.BundleImage)
20+
}
21+
22+
versionToAdd, err := semver.Make(bundleVersion)
23+
if err != nil {
24+
return nil, fmt.Errorf("Bundle version %s is not valid", bundleVersion)
25+
}
26+
27+
newBundleKey := BundleKey{
28+
CsvName: bundle.Name,
29+
Version: versionToAdd.String(),
30+
BundlePath: bundle.BundleImage,
31+
}
32+
33+
// initialize the graph if it started empty
34+
if graph.Name == "" {
35+
graph.Name = bundle.Package
36+
}
37+
if newDefaultChannel != "" {
38+
graph.DefaultChannel = newDefaultChannel
39+
}
40+
41+
// generate the DAG for each channel the new bundle is being insert into
42+
for _, channel := range bundle.Channels {
43+
replaces := make(map[BundleKey]struct{}, 0)
44+
45+
// If the channel doesn't exist yet, initialize it
46+
if !graph.HasChannel(channel) {
47+
// create the channel and add a single node
48+
newChannelGraph := Channel{
49+
Head: newBundleKey,
50+
Nodes: map[BundleKey]map[BundleKey]struct{}{
51+
newBundleKey: nil,
52+
},
53+
}
54+
if graph.Channels == nil {
55+
graph.Channels = make(map[string]Channel, 1)
56+
}
57+
graph.Channels[channel] = newChannelGraph
58+
continue
59+
}
60+
61+
// find the version(s) it should sit between
62+
channelGraph := graph.Channels[channel]
63+
if channelGraph.Nodes == nil {
64+
channelGraph.Nodes = make(map[BundleKey]map[BundleKey]struct{}, 1)
65+
}
66+
67+
lowestAhead := BundleKey{}
68+
greatestBehind := BundleKey{}
69+
skipPatchCandidates := []BundleKey{}
70+
71+
// Iterate over existing nodes and compare the new node's version to find the
72+
// lowest version above it and highest version below it (to insert between these nodes)
73+
for node := range channelGraph.Nodes {
74+
nodeVersion, err := semver.Make(node.Version)
75+
if err != nil {
76+
return nil, fmt.Errorf("Unable to parse existing bundle version stored in index %s %s %s",
77+
node.CsvName, node.Version, node.BundlePath)
78+
}
79+
80+
switch comparison := nodeVersion.Compare(versionToAdd); comparison {
81+
case 0:
82+
return nil, fmt.Errorf("Bundle version %s already added to index", bundleVersion)
83+
case 1:
84+
if lowestAhead.IsEmpty() {
85+
lowestAhead = node
86+
} else {
87+
lowestAheadSemver, _ := semver.Make(lowestAhead.Version)
88+
if nodeVersion.LT(lowestAheadSemver) {
89+
lowestAhead = node
90+
}
91+
}
92+
case -1:
93+
if greatestBehind.IsEmpty() {
94+
greatestBehind = node
95+
} else {
96+
greatestBehindSemver, _ := semver.Make(greatestBehind.Version)
97+
if nodeVersion.GT(greatestBehindSemver) {
98+
greatestBehind = node
99+
}
100+
}
101+
}
102+
103+
// if skippatch mode is enabled, check each node to determine if z-updates should
104+
// be replaced as well. Keep track of them to delete those nodes from the graph itself,
105+
// just be aware of them for replacements
106+
if skippatch {
107+
if isSkipPatchCandidate(versionToAdd, nodeVersion) {
108+
skipPatchCandidates = append(skipPatchCandidates, node)
109+
replaces[node] = struct{}{}
110+
}
111+
}
112+
}
113+
114+
// If we found a node behind the one we're adding, make the new node replace it
115+
if !greatestBehind.IsEmpty() {
116+
replaces[greatestBehind] = struct{}{}
117+
}
118+
119+
// If we found a node ahead of the one we're adding, make the lowest to replace
120+
// the new node. If we didn't find a node semantically ahead, the new node is
121+
// the new channel head
122+
if !lowestAhead.IsEmpty() {
123+
channelGraph.Nodes[lowestAhead] = map[BundleKey]struct{}{
124+
newBundleKey: struct{}{},
125+
}
126+
} else {
127+
channelGraph.Head = newBundleKey
128+
}
129+
130+
if skippatch {
131+
// Remove the nodes that are now being skipped by a new patch version update
132+
for _, candidate := range skipPatchCandidates {
133+
delete(channelGraph.Nodes, candidate)
134+
}
135+
}
136+
137+
// add the node and update the graph
138+
channelGraph.Nodes[newBundleKey] = replaces
139+
graph.Channels[channel] = channelGraph
140+
}
141+
142+
return graph, nil
143+
}
144+
145+
func isSkipPatchCandidate(version, toCompare semver.Version) bool {
146+
return (version.Major == toCompare.Major) && (version.Minor == toCompare.Minor) && (version.Patch > toCompare.Patch)
147+
}

0 commit comments

Comments
 (0)