feat: adds previous versions to self hosted helm registry

.github/workflows/checks.yml

@@ -162,6 +162,8 @@ jobs:
 
   pre-commit:
     runs-on: ubuntu-22.04
+    env:
+      GH_TOKEN: ${{ github.token }}
     steps:
       - name: Check out code
         uses: actions/checkout@v4

hack/addons/generate-mindthegap-repofile.sh

@@ -12,9 +12,21 @@ ASSETS_DIR="$(mktemp -d -p "${TMPDIR:-/tmp}")"
 
 cp "${GIT_REPO_ROOT}/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml" "${ASSETS_DIR}"
 
+gh release download "${PREVIOUS_CAREN_CHARTS_VERSION}" \
+  -p "runtime-extension-components.yaml" -O "${ASSETS_DIR}/runtime-extension-components-n-1.yaml"
+
+gh release download "${CAREN_CHARTS_VERSION_N_MINUS_2}" \
+  -p "runtime-extension-components.yaml" -O "${ASSETS_DIR}/runtime-extension-components-n-2.yaml"
+
+yq e '. | select(.metadata.name == "default-helm-addons-config")' >>"${ASSETS_DIR}/previous-charts.yaml" <"${ASSETS_DIR}/runtime-extension-components-n-1.yaml"
+
+yq e '. | select(.metadata.name == "default-helm-addons-config")' >>"${ASSETS_DIR}/n-2-charts.yaml" <"${ASSETS_DIR}/runtime-extension-components-n-2.yaml"
+
 # this sed line is needed because the go library is unable to parse yaml with a template string.
 sed -i s/"{{ .Values.helmAddonsConfigMap }}"/placeholder/g "${ASSETS_DIR}/helm-config.yaml"
-go run "${GIT_REPO_ROOT}/hack/tools/mindthegap-helm-reg/main.go" --input-configmap-file="${ASSETS_DIR}/helm-config.yaml" --output-file="${ASSETS_DIR}/repos.yaml"
+go run "${GIT_REPO_ROOT}/hack/tools/mindthegap-helm-reg/main.go" --input-configmap-file="${ASSETS_DIR}/helm-config.yaml" --output-file="${ASSETS_DIR}/repos.yaml" \
+  --previous-configmap-file="${ASSETS_DIR}/previous-charts.yaml" \
+  --n-minus-2-configmap-file="${ASSETS_DIR}/n-2-charts.yaml"
 
 # add warning not to edit file directly
 cat <<EOF >"${GIT_REPO_ROOT}/hack/addons/mindthegap-helm-registry/repos.yaml"

hack/addons/mindthegap-helm-registry/repos.yaml

@@ -16,16 +16,19 @@ repositories:
     charts:
       aws-ebs-csi-driver:
       - 2.35.1
+      - 2.32.0
   cilium:
     repoURL: https://helm.cilium.io/
     charts:
       cilium:
       - 1.16.2
+      - 1.15.6
   cluster-autoscaler:
     repoURL: https://kubernetes.github.io/autoscaler
     charts:
       cluster-autoscaler:
       - 9.40.0
+      - 9.37.0
   local-path-provisioner:
     repoURL: https://charts.containeroo.ch
     charts:
@@ -36,16 +39,19 @@ repositories:
     charts:
       metallb:
       - 0.14.8
+      - 0.14.5
   node-feature-discovery:
     repoURL: https://kubernetes-sigs.github.io/node-feature-discovery/charts
     charts:
       node-feature-discovery:
       - 0.16.4
+      - 0.16.1
   nutanix-cloud-provider:
     repoURL: https://nutanix.github.io/helm/
     charts:
       nutanix-cloud-provider:
       - 0.4.1
+      - 0.4.0
   nutanix-csi-storage:
     repoURL: https://nutanix.github.io/helm-releases/
     charts:
@@ -56,8 +62,10 @@ repositories:
     charts:
       snapshot-controller:
       - 3.0.6
+      - 3.0.5
   tigera-operator:
     repoURL: https://docs.tigera.io/calico/charts
     charts:
       tigera-operator:
       - v3.28.2
+      - v3.28.0

hack/tools/mindthegap-helm-reg/main.go

@@ -9,6 +9,8 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"path/filepath"
+	"slices"
 
 	"gopkg.in/yaml.v2"
 	corev1 "k8s.io/api/core/v1"
@@ -36,57 +38,58 @@ var log = ctrl.LoggerFrom(context.Background())
 func main() {
 	args := os.Args
 	var (
-		outputFile         string
-		inputConfigMapFile string
+		outputFile            string
+		inputConfigMapFile    string
+		previousConfigMapFile string
+		nminus2ConfigMapFile  string
 	)
 	flagSet := flag.NewFlagSet("mindthegap-helm-registry", flag.ExitOnError)
-	flagSet.StringVar(&outputFile, "output-file", "",
-		"output file name to write config map to.")
-	flagSet.StringVar(&inputConfigMapFile, "input-configmap-file", "",
-		"input configmap file to create the mindthegap repo file from")
+	flagSet.StringVar(
+		&outputFile,
+		"output-file",
+		"",
+		"output file name to write config map to.",
+	)
+	flagSet.StringVar(
+		&inputConfigMapFile,
+		"input-configmap-file",
+		"",
+		"input configmap file to create the mindthegap repo file from",
+	)
+	flagSet.StringVar(
+		&previousConfigMapFile,
+		"previous-configmap-file",
+		"",
+		"input configmap file to create the mindthegap repo file from",
+	)
+	flagSet.StringVar(
+		&nminus2ConfigMapFile,
+		"n-minus-2-configmap-file",
+		"",
+		"input configmap file to create the mindthegap repo file from",
+	)
 	err := flagSet.Parse(args[1:])
 	if err != nil {
 		log.Error(err, "failed to parse args")
 	}
-	fullPath := inputConfigMapFile
-	if !path.IsAbs(fullPath) {
-		wd, err := os.Getwd()
-		if err != nil {
-			log.Error(err, "failed to get wd")
-			return
-		}
-		fullPath = path.Join(wd, inputConfigMapFile)
-	}
-	f, err := os.Open(fullPath)
+	inputCm, err := getConfigMapFromFile(inputConfigMapFile)
 	if err != nil {
-		log.Error(err, "failed to open file")
-		return
-	}
-	defer f.Close()
-	cm := &corev1.ConfigMap{}
-	err = yamlDecode.NewYAMLOrJSONDecoder(f, 1024).Decode(cm)
-	if err != nil {
-		log.Error(err, fmt.Sprintf("failed to unmarshal file %s", fullPath))
+		log.Error(err, fmt.Sprintf("failed to get configmap from file %s %w", inputConfigMapFile, err))
 	}
 	out := HelmChartsConfig{
 		map[string]Repository{},
 	}
-	for _, info := range cm.Data {
-		var settings HelmChartFromConfigMap
-		err = yaml.Unmarshal([]byte(info), &settings)
-		if err != nil {
-			log.Error(err, "failed unmarshl settings")
-			return
-		}
-		out.Repositories[settings.Name] = Repository{
-			RepoURL: settings.Repository,
-			Charts: map[string][]string{
-				settings.Name: {
-					settings.Version,
-				},
-			},
-		}
+	ConfigMapToHelmChartConfig(&out, inputCm)
+	previousCm, err := getConfigMapFromFile(previousConfigMapFile)
+	if err != nil {
+		log.Error(err, fmt.Sprintf("failed to get configmap from file %s %w", inputConfigMapFile, err))
 	}
+	ConfigMapToHelmChartConfig(&out, previousCm)
+	nMinus2Cm, err := getConfigMapFromFile(nminus2ConfigMapFile)
+	if err != nil {
+		log.Error(err, fmt.Sprintf("failed to get configmap from file %s %w", inputConfigMapFile, err))
+	}
+	ConfigMapToHelmChartConfig(&out, nMinus2Cm)
 	b, err := yaml.Marshal(out)
 	if err != nil {
 		log.Error(err, fmt.Sprintf("failed to marshal obj %v", out))
@@ -99,7 +102,7 @@ func main() {
 		}
 		fullOutputfilePath = path.Join(wd, outputFile)
 	}
-	f, err = os.OpenFile(fullOutputfilePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o666)
+	f, err := os.OpenFile(fullOutputfilePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o666)
 	if err != nil {
 		log.Error(err, "failed to create file")
 	}
@@ -109,3 +112,58 @@ func main() {
 		log.Error(err, "failed to write to file")
 	}
 }
+
+func ConfigMapToHelmChartConfig(out *HelmChartsConfig, cm *corev1.ConfigMap) {
+	for _, info := range cm.Data {
+		var settings HelmChartFromConfigMap
+		err := yaml.Unmarshal([]byte(info), &settings)
+		if err != nil {
+			log.Error(err, "failed unmarshl settings")
+			return
+		}
+		repo, ok := out.Repositories[settings.Name]
+		// if this is the first time we saw this add a new entry
+		if !ok {
+			out.Repositories[settings.Name] = Repository{
+				RepoURL: settings.Repository,
+				Charts: map[string][]string{
+					settings.Name: {
+						settings.Version,
+					},
+				},
+			}
+			continue
+		}
+		// we've seen it already only add a new chart if the versions are different
+		if !slices.Contains(repo.Charts[settings.Name], settings.Version) {
+			repo.Charts[settings.Name] = append(repo.Charts[settings.Name], settings.Version)
+		}
+	}
+}
+
+func getConfigMapFromFile(configMapFile string) (*corev1.ConfigMap, error) {
+	fullPath, err := EnsureFullPath(configMapFile)
+	if err != nil {
+		return nil, err
+	}
+	f, err := os.Open(fullPath)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	cm := &corev1.ConfigMap{}
+	err = yamlDecode.NewYAMLOrJSONDecoder(f, 1024).Decode(cm)
+	return cm, err
+}
+
+func EnsureFullPath(filename string) (string, error) {
+	fullPath, err := filepath.Abs(filename)
+	if err != nil {
+		return "", err
+	}
+	_, err = os.Stat(fullPath)
+	if err != nil {
+		return "", err
+	}
+	return fullPath, nil
+}

make/addons.mk

@@ -25,10 +25,13 @@ export KUBE_VIP_VERSION := v0.8.3
 
 export METALLB_CHART_VERSION := 0.14.8
 
+# these versions correspond to n-1 and n-2 shipped releases.
+export PREVIOUS_CAREN_CHARTS_VERSION := v0.14.6
+export CAREN_CHARTS_VERSION_N_MINUS_2 := v0.13.7
+
 .PHONY: addons.sync
 addons.sync: $(addprefix update-addon.,calico cilium nfd cluster-autoscaler snapshot-controller local-path-provisioner-csi aws-ebs-csi kube-vip)
 addons.sync: $(addprefix update-addon.aws-ccm.,127 128 129 130 131)
-addons.sync: template-helm-repository
 
 .PHONY: update-addon.calico
 update-addon.calico: ; $(info $(M) updating calico manifests)