chore(metrics): properly detect rate limits (#3499)

* chore: detect more error codes

* chore: add delay between successful calls

* chore: set max timeout for the workflow

* fix: modified modules was incorrect for usage-metrics

Author: mdelapenya

.github/workflows/usage-metrics.yml

@@ -20,6 +20,7 @@ env:
 jobs:
   collect-metrics:
     runs-on: ubuntu-latest
+    timeout-minutes: 120
     steps:
       - name: Checkout repository
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

scripts/changed-modules.sh

@@ -170,6 +170,8 @@ for file in $modified_files; do
         fi
     elif [[ $file == modulegen/* ]]; then
         modified_modules+=("\"modulegen\"")
+    elif [[ $file == usage-metrics/* ]]; then
+        modified_modules+=("\"usage-metrics\"")
     else
         # a file from the core module is modified, so include all modules in the list and stop the loop
         # check if the file is in one of the excluded modules

usage-metrics/collect-metrics.go

@@ -78,6 +78,12 @@ func collectMetrics(versions []string, csvPath string) error {
 
 		metrics = append(metrics, metric)
 		fmt.Printf("Successfully queried: %s has %d usages on %s\n", version, count, metric.Date)
+
+		// Add delay to avoid rate limiting (30 requests/minute = 2 seconds between requests)
+		if len(metrics) < len(versions) {
+			log.Printf("Waiting 2 seconds before next query to avoid rate limiting...")
+			time.Sleep(2 * time.Second)
+		}
 	}
 
 	// Sort metrics by version
@@ -127,7 +133,9 @@ func queryGitHubUsageWithRetry(version string) (int, error) {
 		lastErr = err
 
 		// Check if it's a rate limit error
-		if strings.Contains(err.Error(), "rate limit") || strings.Contains(err.Error(), "403") {
+		if strings.Contains(err.Error(), "rate limit") ||
+			strings.Contains(err.Error(), "403") ||
+			strings.Contains(err.Error(), "429") {
 			log.Printf("Rate limit hit for version %s, will retry with backoff", version)
 			continue
 		}