fix: handle rate limits and resource leaks in Artifact Hub sync

generators/artifacthub/package.go

@@ -3,17 +3,17 @@ package artifacthub
 import (
 	"encoding/json"
 	"fmt"
-	"net/http"
-	"strings"
-	"time"
-
 	"github.com/meshery/meshkit/utils"
 	"github.com/meshery/meshkit/utils/component"
 	"github.com/meshery/meshkit/utils/manifests"
 	"github.com/meshery/schemas/models/v1beta1/category"
 	_component "github.com/meshery/schemas/models/v1beta1/component"
 	"github.com/meshery/schemas/models/v1beta1/model"
+	"github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v2"
+	"net/http"
+	"strings"
+	"time"
 )
 
 const ArtifactHubAPIEndpoint = "https://artifacthub.io/api/v1"
@@ -133,7 +133,7 @@ func (pkg *AhPackage) UpdatePackageData() error {
 		if !strings.HasSuffix(pkg.RepoUrl, "/") {
 			pkg.RepoUrl = pkg.RepoUrl + "/"
 		}
-		chartUrl = fmt.Sprintf("%s%s", pkg.RepoUrl, chartUrl)
+		pkg.ChartUrl = fmt.Sprintf("%s%s", pkg.RepoUrl, chartUrl)
 	}
 	pkg.ChartUrl = chartUrl
 	return nil
@@ -143,7 +143,7 @@ func (pkg *AhPackage) Validator() {
 
 }
 
-// GetAllAhHelmPackages returns a list of all AhPackages and is super slow to avoid rate limits.
+// GetAllAhHelmPackages returns a list of all AhPackages, using exponential backoff to handle rate limits.
 func GetAllAhHelmPackages() ([]AhPackage, error) {
 	pkgs := make([]AhPackage, 0)
 	resp, err := http.Get(AhHelmExporterEndpoint)
@@ -155,35 +155,75 @@ func GetAllAhHelmPackages() ([]AhPackage, error) {
 		return nil, ErrGetAllHelmPackages(err)
 	}
 	defer resp.Body.Close()
+
 	var res []map[string]interface{}
 	err = json.NewDecoder(resp.Body).Decode(&res)
 	if err != nil {
 		return nil, err
 	}
+
+	const maxRetries = 5
+
 	for _, p := range res {
-		name := p["name"].(string)
-		repo := p["repository"].(map[string]interface{})["name"].(string)
+		name, ok := p["name"].(string)
+		if !ok {
+			logrus.WithFields(logrus.Fields{"package_data": p}).Warn("Skipping package due to missing or invalid name")
+			continue
+		}
+		repoMap, ok := p["repository"].(map[string]interface{})
+		if !ok {
+			logrus.WithFields(logrus.Fields{"package_data": p}).Warn("Skipping package due to missing or invalid repository")
+			continue
+		}
+
+		repo, ok := repoMap["name"].(string)
+		if !ok {
+			logrus.WithFields(logrus.Fields{"package_data": p}).Warn("Skipping package due to missing or invalid repository name")
+			continue
+		}
 		url := fmt.Sprintf("https://artifacthub.io/api/v1/packages/helm/%s/%s", repo, name)
-		resp, err := http.Get(url)
+
+		var resp *http.Response
+		var err error
+
+		for i := 0; i < maxRetries; i++ {
+			resp, err = http.Get(url)
+			if err != nil {
+				break
+			}
+			if resp.StatusCode == 429 {
+				resp.Body.Close()
+				time.Sleep(time.Duration(1<<i) * time.Second)
+				continue
+			}
+			break
+		}
+
 		if err != nil {
-			fmt.Println(err)
+			logrus.WithError(err).WithField("url", url).Error("Failed to fetch package")
 			continue
 		}
+
 		if resp.StatusCode != 200 {
 			err = fmt.Errorf("status code %d for %s", resp.StatusCode, url)
-			fmt.Println(err)
+			logrus.WithError(err).WithField("url", url).Error("Failed to fetch package")
+			resp.Body.Close()
 			continue
 		}
-		defer resp.Body.Close()
-		var res map[string]interface{}
-		err = json.NewDecoder(resp.Body).Decode(&res)
+
+		var pkgRes map[string]interface{}
+		err = json.NewDecoder(resp.Body).Decode(&pkgRes)
+		resp.Body.Close()
+
 		if err != nil {
-			fmt.Println(err)
+			logrus.WithError(err).WithField("url", url).Error("Failed to decode package response")
 			continue
 		}
 
-		pkgs = append(pkgs, *parseArtifacthubResponse(res))
-		time.Sleep(500 * time.Millisecond)
+		parsedPkg := parseArtifacthubResponse(pkgRes)
+		if parsedPkg != nil {
+			pkgs = append(pkgs, *parsedPkg)
+		}
 	}
 	return pkgs, nil
 }
@@ -226,3 +266,4 @@ func parseArtifacthubResponse(response map[string]interface{}) *AhPackage {
 		Official:          official,
 	}
 }
+