Fix contributors

Author: benjeffery

_plugins/github_api.rb

@@ -10,21 +10,43 @@ class Generator < Jekyll::Generator
     safe true
     priority :highest
 
+    def fetch_all_pages(url, headers)
+      all_results = []
+      page = 1
+      per_page = 100  # Maximum allowed by GitHub API
+      
+      loop do
+        response = HTTParty.get("#{url}?page=#{page}&per_page=#{per_page}", headers: headers)
+        page_results = JSON.load(response.body)
+        
+        # Break if we get an empty array or if it's not an array (error case)
+        break if !page_results.is_a?(Array) || page_results.empty?
+        
+        all_results.concat(page_results)
+        page += 1
+        
+        # Break if we got fewer results than per_page (last page)
+        break if page_results.length < per_page
+      end
+      
+      all_results
+    end
+
     def generate(site)
         headers = {
           Authorization: 'token '+ ENV['GITHUB_TOKEN'],
         }
       site.data['contributors'] = {}
       site.collections['software'].docs.each do |d|
         d.data['github'] = {}
-        d.data['github']['releases'] = JSON.load(
-          HTTParty.get("https://api.github.com/repos/#{d['gh_org']}/#{d['name']}/releases", headers: headers).body
+        d.data['github']['releases'] = fetch_all_pages(
+          "https://api.github.com/repos/#{d['gh_org']}/#{d['name']}/releases", headers
         )
         d.data['github']['repo'] = JSON.load(
           HTTParty.get("https://api.github.com/repos/#{d['gh_org']}/#{d['name']}", headers: headers).body
         )
-        d.data['github']['contributors'] = JSON.load(
-          HTTParty.get("https://api.github.com/repos/#{d['gh_org']}/#{d['name']}/contributors", headers: headers).body
+        d.data['github']['contributors'] = fetch_all_pages(
+          "https://api.github.com/repos/#{d['gh_org']}/#{d['name']}/contributors", headers
         )
         site.config['extra_contributors'].each do |login, repo|
             if repo == d['name']
@@ -51,22 +73,39 @@ def generate(site)
       site.data['contributors'].each do |c|
         fetched = false
         sleep_time = 1
-        while not fetched do
+        retry_count = 0
+        max_retries = 5
+        
+        while not fetched and retry_count < max_retries do
             begin
-              fetched = JSON.load(HTTParty.get("https://api.github.com/users/#{c['login']}", headers: headers).body)
-              sleep_time = 1
+              response = HTTParty.get("https://api.github.com/users/#{c['login']}", headers: headers)
+              user_data = JSON.load(response.body)
+              
+              # Check if we got valid user data
+              if user_data.is_a?(Hash) && user_data.key?('login')
+                c.merge!(user_data)
+                fetched = true
+                sleep_time = 1
+              else
+                puts "Invalid user data for #{c['login']}: #{user_data}"
+                retry_count += 1
+              end
             rescue => e
-              puts e.message, sleep_time
-              if e.message.include?("rate limit exceeded")
-                puts("Sleeping for #{sleep_time} seconds")
+              puts "Error fetching user #{c['login']}: #{e.message}"
+              retry_count += 1
+              
+              if e.message.include?("rate limit exceeded") || e.message.include?("403")
+                puts("Rate limit hit, sleeping for #{sleep_time} seconds")
                 sleep(sleep_time)
-                sleep_time = sleep_time * 2
+                sleep_time = [sleep_time * 2, 60].min  # Cap at 60 seconds
+              elsif retry_count >= max_retries
+                puts("Max retries reached for user #{c['login']}, skipping")
+                break
               else
-                raise # re-raise the last exception
+                sleep(1)  # Brief pause before retry
               end
             end
         end
-        c.merge!(fetched)
       end
     end
   end