diff --git a/generators/artifacthub/package.go b/generators/artifacthub/package.go index a57d3605..a288739c 100644 --- a/generators/artifacthub/package.go +++ b/generators/artifacthub/package.go @@ -3,17 +3,17 @@ package artifacthub import ( "encoding/json" "fmt" - "net/http" - "strings" - "time" - "github.com/meshery/meshkit/utils" "github.com/meshery/meshkit/utils/component" "github.com/meshery/meshkit/utils/manifests" "github.com/meshery/schemas/models/v1beta1/category" _component "github.com/meshery/schemas/models/v1beta1/component" "github.com/meshery/schemas/models/v1beta1/model" + "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" + "net/http" + "strings" + "time" ) const ArtifactHubAPIEndpoint = "https://artifacthub.io/api/v1" @@ -133,7 +133,7 @@ func (pkg *AhPackage) UpdatePackageData() error { if !strings.HasSuffix(pkg.RepoUrl, "/") { pkg.RepoUrl = pkg.RepoUrl + "/" } - chartUrl = fmt.Sprintf("%s%s", pkg.RepoUrl, chartUrl) + pkg.ChartUrl = fmt.Sprintf("%s%s", pkg.RepoUrl, chartUrl) } pkg.ChartUrl = chartUrl return nil @@ -143,7 +143,7 @@ func (pkg *AhPackage) Validator() { } -// GetAllAhHelmPackages returns a list of all AhPackages and is super slow to avoid rate limits. +// GetAllAhHelmPackages returns a list of all AhPackages, using exponential backoff to handle rate limits. func GetAllAhHelmPackages() ([]AhPackage, error) { pkgs := make([]AhPackage, 0) resp, err := http.Get(AhHelmExporterEndpoint) @@ -155,35 +155,75 @@ func GetAllAhHelmPackages() ([]AhPackage, error) { return nil, ErrGetAllHelmPackages(err) } defer resp.Body.Close() + var res []map[string]interface{} err = json.NewDecoder(resp.Body).Decode(&res) if err != nil { return nil, err } + + const maxRetries = 5 + for _, p := range res { - name := p["name"].(string) - repo := p["repository"].(map[string]interface{})["name"].(string) + name, ok := p["name"].(string) + if !ok { + logrus.WithFields(logrus.Fields{"package_data": p}).Warn("Skipping package due to missing or invalid name") + continue + } + repoMap, ok := p["repository"].(map[string]interface{}) + if !ok { + logrus.WithFields(logrus.Fields{"package_data": p}).Warn("Skipping package due to missing or invalid repository") + continue + } + + repo, ok := repoMap["name"].(string) + if !ok { + logrus.WithFields(logrus.Fields{"package_data": p}).Warn("Skipping package due to missing or invalid repository name") + continue + } url := fmt.Sprintf("https://artifacthub.io/api/v1/packages/helm/%s/%s", repo, name) - resp, err := http.Get(url) + + var resp *http.Response + var err error + + for i := 0; i < maxRetries; i++ { + resp, err = http.Get(url) + if err != nil { + break + } + if resp.StatusCode == 429 { + resp.Body.Close() + time.Sleep(time.Duration(1<