CLOUD-861: Automate operator release

Makefile

@@ -5,11 +5,7 @@ GIT_COMMIT:=$(shell git rev-parse --short HEAD)
 IMG ?= perconalab/version-service:$(GIT_BRANCH)-$(GIT_COMMIT)
 
 init:
-	go build -modfile=tools/go.mod -o bin/yq github.com/mikefarah/yq/v3
-	go build -modfile=tools/go.mod -o bin/protoc-gen-go google.golang.org/protobuf/cmd/protoc-gen-go
-	go build -modfile=tools/go.mod -o bin/protoc-gen-go-grpc google.golang.org/grpc/cmd/protoc-gen-go-grpc
-	go build -modfile=tools/go.mod -o bin/protoc-gen-grpc-gateway github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway
-	go build -modfile=tools/go.mod -o bin/protoc-gen-openapiv2 github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2
+	cd tools && go generate -x -tags=tools
 
 	curl -L "https://github.com/bufbuild/buf/releases/download/v1.34.0/buf-$(shell uname -s)-$(shell uname -m)" -o "./bin/buf"
 	chmod +x ./bin/buf

tools/operator-tool/README.md

@@ -0,0 +1,43 @@
+# operator-tool
+
+`operator-tool` is designed to generate a source file for a version service. It retrieves a list of product versions from the [Percona Downloads](https://www.percona.com/downloads) API (`https://www.percona.com/products-api.php`) and searches for the corresponding images in the [Docker Hub repository](https://hub.docker.com/u/percona). If an image is not specified in the API, the latest tag of that image will be used.
+
+Build it using `make init`.
+
+## Usage
+
+### Help
+
+```
+$ ./bin/operator-tool --help
+Usage of ./bin/operator-tool:
+  -file string
+        Specify an older source file. The operator-tool will exclude any versions that are older than those listed in this file.
+  -operator string
+        Operator name. Available values: [psmdb-operator pxc-operator ps-operator pg-operator]
+  -verbose
+        Show logs
+  -version string
+        Operator version
+
+```
+
+### Generating source file from zero
+
+```
+$ ./bin/operator-tool --operator "psmdb-operator" --version "1.17.0" # outputs source file for psmdb-operator
+...
+$ ./bin/operator-tool --operator "pg-operator" --version "2.5.0"     # outputs source file for pg-operator
+...
+$ ./bin/operator-tool --operator "ps-operator" --version "0.8.0"     # outputs source file for ps-operator
+...
+$ ./bin/operator-tool --operator "pxc-operator" --version "1.15.1"   # outputs source file for pxc-operator
+...
+```
+
+### Generating source file based on older file
+
+```
+$ ./bin/operator-tool --file ./sources/operator.2.5.0.pg-operator.json --version "1.17.0" # outputs source file for pg-operator, excluding older versions specified in the file
+...
+```

tools/operator-tool/cmd/filler.go

@@ -0,0 +1,212 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"regexp"
+	"slices"
+	"strings"
+
+	vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api"
+
+	"operator-tool/registry"
+)
+
+// VersionMapFiller is a helper type for creating a map[string]*vsAPI.Version
+// using information retrieved from Docker Hub.
+type VersionMapFiller struct {
+	RegistryClient *registry.RegistryClient
+	errs           []error
+}
+
+func (f *VersionMapFiller) exec(vm map[string]*vsAPI.Version, err error) map[string]*vsAPI.Version {
+	if err != nil {
+		f.errs = append(f.errs, err)
+		return nil
+	}
+
+	f.setRecommended(vm)
+
+	return vm
+}
+
+// setRecommended sets a recommended status to the latest version.
+func (f *VersionMapFiller) setRecommended(vm map[string]*vsAPI.Version) {
+	maxVer := ""
+	for k := range vm {
+		if maxVer == "" {
+			maxVer = k
+			continue
+		}
+
+		if goversion(k).Compare(goversion(maxVer)) > 0 {
+			maxVer = k
+		}
+	}
+
+	if _, ok := vm[maxVer]; ok {
+		vm[maxVer].Status = vsAPI.Status_recommended
+	}
+}
+
+// Normal returns a map[string]*Version for the specified image by filtering tags
+// with the given list of versions.
+//
+// The map may include image tags with the following suffixes: "", "-amd64", "-arm64", and "-multi".
+func (f *VersionMapFiller) Normal(image string, versions []string) map[string]*vsAPI.Version {
+	return f.exec(getVersionMap(f.RegistryClient, image, versions))
+}
+
+// Regex returns a map[string]*Version for the specified image by filtering tags
+// with the given list of versions and a regular expression.
+//
+// The regex argument must contain at least one matching group, which will be used
+// to filter the necessary images. For example, given the regex "(^.*)(?:-logcollector)"
+// and versions []string{"1.2.1"}, the tag "1.2.1-logcollector" will be included,
+// while "1.3.1-logcollector", "1.2.1-some-string", and "1.2.1" will not be included.
+//
+// The map may include image tags with the following suffixes: "", "-amd64", "-arm64", and "-multi".
+func (f *VersionMapFiller) Regex(image string, regex string, versions []string) map[string]*vsAPI.Version {
+	return f.exec(getVersionMapRegex(f.RegistryClient, image, regex, versions))
+}
+
+// Latest returns a map[string]*Version with latest version tag of the specified image.
+//
+// The map may include image tags with the following suffixes: "", "-amd64", "-arm64", and "-multi".
+func (f *VersionMapFiller) Latest(image string) map[string]*vsAPI.Version {
+	return f.exec(getVersionMapLatestVer(f.RegistryClient, image))
+}
+
+func (f *VersionMapFiller) Error() error {
+	return errors.Join(f.errs...)
+}
+
+func getVersionMapRegex(rc *registry.RegistryClient, image string, regex string, versions []string) (map[string]*vsAPI.Version, error) {
+	m := make(map[string]*vsAPI.Version)
+	r := regexp.MustCompile(regex)
+	for _, v := range versions {
+		images, err := rc.GetImages(image, func(tag string) bool {
+			matches := r.FindStringSubmatch(tag)
+			if len(matches) <= 1 {
+				return false
+			}
+			if matches[1] != v {
+				return false
+			}
+			return true
+		})
+		if err != nil {
+			return nil, err
+		}
+		if len(images) == 0 {
+			log.Printf("DEBUG: tag %s for image %s with regexp %s was not found\n", v, image, regex)
+			continue
+		}
+
+		vm, err := versionMapFromImages(v, images)
+		if err != nil {
+			return nil, err
+		}
+		m[v] = vm
+	}
+	return m, nil
+}
+
+func getVersionMap(rc *registry.RegistryClient, image string, versions []string) (map[string]*vsAPI.Version, error) {
+	m := make(map[string]*vsAPI.Version)
+	for _, v := range versions {
+		images, err := rc.GetImages(image, func(tag string) bool {
+			allowedSuffixes := []string{"", "-amd64", "-arm64", "-multi"}
+			for _, s := range allowedSuffixes {
+				if tag+s == v {
+					return true
+				}
+			}
+			return false
+		})
+		if err != nil {
+			return nil, err
+		}
+		if len(images) == 0 {
+			log.Printf("DEBUG: tag %s for image %s was not found\n", v, image)
+			continue
+		}
+		vm, err := versionMapFromImages(v, images)
+		if err != nil {
+			return nil, err
+		}
+		m[v] = vm
+	}
+	return m, nil
+}
+
+func getVersionMapLatestVer(rc *registry.RegistryClient, imageName string) (map[string]*vsAPI.Version, error) {
+	image, err := rc.GetLatestImage(imageName)
+	if err != nil {
+		return nil, err
+	}
+	vm, err := versionMapFromImages(image.Tag, []registry.Image{image})
+	if err != nil {
+		return nil, err
+	}
+	return map[string]*vsAPI.Version{
+		image.Tag: vm,
+	}, nil
+}
+
+// versionMapFromImages returns a Version for a given list of images and a base tag without any suffixes.
+//
+// Some images on Docker Hub are tagged like <name>, <name>-arm64, <name>-amd64, and <name>-multi.
+// This function attempts to use information from images with both amd64 and arm64 builds. If both are not available, it defaults to amd64.
+//
+// If multiple provided images share the same suffix, the function returns a Version with information for the latest image.
+func versionMapFromImages(baseTag string, images []registry.Image) (*vsAPI.Version, error) {
+	slices.SortFunc(images, func(a, b registry.Image) int {
+		return goversion(b.Tag).Compare(goversion(a.Tag))
+	})
+	imageName := images[0].Name
+	var multiImage, amd64Image, arm64Image *registry.Image
+	for _, image := range images {
+		if strings.HasSuffix(image.Tag, "-arm64") {
+			arm64Image = &image
+			continue
+		}
+		if multiImage == nil {
+			if (image.DigestAMD64 != "" && image.DigestARM64 != "") || strings.HasSuffix(image.Tag, "-multi") {
+				multiImage = &image
+				continue
+			}
+		}
+		if image.Tag == baseTag || amd64Image == nil {
+			amd64Image = &image
+			continue
+		}
+	}
+	var imagePath, imageHash, imageHashArm64 string
+
+	switch {
+	case multiImage != nil:
+		imagePath = multiImage.FullName()
+		imageHash = multiImage.DigestAMD64
+		imageHashArm64 = multiImage.DigestARM64
+	case amd64Image != nil && arm64Image != nil:
+		log.Printf("WARNING: Image %s has both %s and %s tags, but doesn't have \"-multi\" tag. Using %s\n", imageName, amd64Image, arm64Image, amd64Image)
+		fallthrough
+	case amd64Image != nil:
+		imagePath = amd64Image.FullName()
+		imageHash = amd64Image.DigestAMD64
+	case arm64Image != nil:
+		imagePath = arm64Image.FullName()
+		imageHashArm64 = arm64Image.DigestARM64
+	default:
+		return nil, fmt.Errorf("necessary tags for %s image were not found", imageName)
+	}
+
+	return &vsAPI.Version{
+		ImagePath:      imagePath,
+		ImageHash:      imageHash,
+		ImageHashArm64: imageHashArm64,
+		Status:         vsAPI.Status_available,
+	}, nil
+}

tools/operator-tool/cmd/main.go

@@ -0,0 +1,126 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"slices"
+
+	vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api"
+
+	"operator-tool/registry"
+)
+
+const (
+	operatorNamePSMDB = "psmdb-operator"
+	operatorNamePXC   = "pxc-operator"
+	operatorNamePS    = "ps-operator"
+	operatorNamePG    = "pg-operator"
+)
+
+var validOperatorNames = []string{
+	operatorNamePSMDB,
+	operatorNamePXC,
+	operatorNamePS,
+	operatorNamePG,
+}
+
+var (
+	operatorName = flag.String("operator", "", fmt.Sprintf("Operator name. Available values: %v", validOperatorNames))
+	version      = flag.String("version", "", "Operator version")
+	filePath     = flag.String("file", "", "Specify an older source file. The operator-tool will exclude any versions that are older than those listed in this file.")
+	verbose      = flag.Bool("verbose", false, "Show logs")
+)
+
+func main() {
+	flag.Parse()
+
+	if *version == "" {
+		log.Println("ERROR: --version should be provided")
+		os.Exit(1)
+	}
+
+	if *filePath != "" {
+		product, err := readBaseFile(*filePath)
+		if err != nil {
+			log.Println("ERROR: failed to read base file:", err.Error())
+			os.Exit(1)
+		}
+		*operatorName = product.Versions[0].Product
+	} else {
+		if *operatorName == "" {
+			log.Println("ERROR: --operator or --file should be provided")
+			os.Exit(1)
+		}
+	}
+
+	switch {
+	case slices.Contains(validOperatorNames, *operatorName):
+		if !*verbose {
+			log.SetOutput(io.Discard)
+		}
+
+		if err := printSourceFile(*operatorName, *version, *filePath); err != nil {
+			log.Println("ERROR: failed to generate source file: ", err.Error())
+			os.Exit(1)
+		}
+	default:
+		log.Printf("ERROR: Unknown operator name: %s. Available values: %v\n", *operatorName, validOperatorNames)
+		os.Exit(1)
+	}
+}
+
+func printSourceFile(operatorName, version, file string) error {
+	r, err := getProductResponse(operatorName, version)
+	if err != nil {
+		return fmt.Errorf("failed to get product response: %w", err)
+	}
+	if file != "" {
+		if err := deleteOldVersions(file, r.Versions[0].Matrix); err != nil {
+			return fmt.Errorf("failed to delete old verisons from version matrix: %w", err)
+		}
+	}
+
+	content, err := marshal(r)
+	if err != nil {
+		return fmt.Errorf("failed to marshal product response: %w", err)
+	}
+
+	fmt.Println(string(content))
+	return nil
+}
+
+func getProductResponse(operatorName, version string) (*vsAPI.ProductResponse, error) {
+	var versionMatrix *vsAPI.VersionMatrix
+	var err error
+
+	f := &VersionMapFiller{
+		RegistryClient: registry.NewClient(),
+	}
+	switch operatorName {
+	case operatorNamePG:
+		versionMatrix, err = pgVersionMatrix(f, operatorName, version)
+	case operatorNamePS:
+		versionMatrix, err = psVersionMatrix(f, operatorName, version)
+	case operatorNamePSMDB:
+		versionMatrix, err = psmdbVersionMatrix(f, operatorName, version)
+	case operatorNamePXC:
+		versionMatrix, err = pxcVersionMatrix(f, operatorName, version)
+	default:
+		panic("problems with validation. unknown operator name " + operatorName)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to get version matrix: %w", err)
+	}
+	return &vsAPI.ProductResponse{
+		Versions: []*vsAPI.OperatorVersion{
+			{
+				Product:  operatorName,
+				Operator: version,
+				Matrix:   versionMatrix,
+			},
+		},
+	}, nil
+}