fix: add retries for gtihub api calls in prebuild step

scripts/authors/generate-contributors.js

@@ -27,6 +27,48 @@ const cache = new Map();
 // Cache for email to GitHub username mapping
 const emailToUsernameCache = new Map();
 
+// Retry configuration
+const RETRY_CONFIG = {
+    maxRetries: 3,
+    baseDelay: 1000, // 1 second base delay
+    maxDelay: 30000, // 30 seconds max delay
+    backoffMultiplier: 2
+};
+
+// Sleep utility function
+const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
+
+// Retry wrapper with exponential backoff
+async function withRetry(fn, description, endpoint) {
+    let lastError;
+
+    for (let attempt = 1; attempt <= RETRY_CONFIG.maxRetries; attempt++) {
+        try {
+            return await fn();
+        } catch (error) {
+            lastError = error;
+
+            if (attempt === RETRY_CONFIG.maxRetries) {
+                console.warn(`GitHub API failed for ${endpoint} after ${RETRY_CONFIG.maxRetries} attempts: ${error.message}`);
+                return null;
+            }
+
+            // Calculate delay with exponential backoff
+            const delay = Math.min(
+                RETRY_CONFIG.baseDelay * Math.pow(RETRY_CONFIG.backoffMultiplier, attempt - 1),
+                RETRY_CONFIG.maxDelay
+            );
+
+            console.warn(`GitHub API failed for ${endpoint} (attempt ${attempt}/${RETRY_CONFIG.maxRetries}): ${error.message}`);
+            console.warn(`Retrying in ${delay}ms...`);
+
+            await sleep(delay);
+        }
+    }
+
+    return null;
+}
+
 // GitHub API helper
 async function fetchFromGitHub(endpoint) {
     if (cache.has(endpoint)) return cache.get(endpoint);
@@ -40,21 +82,21 @@ async function fetchFromGitHub(endpoint) {
         headers['Authorization'] = `token ${process.env.GITHUB_ACCESS_TOKEN}`;
     }
 
-    try {
+    const data = await withRetry(async () => {
         const response = await fetch(`${GITHUB_CONFIG.apiBase}${endpoint}`, { headers });
 
         if (!response.ok) {
-            if (response.status === 404) return null;
             throw new Error(`GitHub API: ${response.status} ${response.statusText}`);
         }
 
-        const data = await response.json();
+        return await response.json();
+    }, 'GitHub API fetch', endpoint);
+
+    if (data) {
         cache.set(endpoint, data);
-        return data;
-    } catch (error) {
-        console.warn(`GitHub API failed for ${endpoint}:`, error.message);
-        return null;
     }
+
+    return data;
 }
 
 // Contributor resolution helpers

scripts/load_github_discussions.py

@@ -2,9 +2,45 @@
 import json
 import os
 import csv
+import time
 from datetime import datetime, timezone
 import xml.dom.minidom as md
 
+# Retry configuration
+RETRY_CONFIG = {
+    'max_retries': 3,
+    'base_delay': 1.0,  # 1 second base delay
+    'max_delay': 30.0,  # 30 seconds max delay
+    'backoff_multiplier': 2
+}
+
+def with_retry(func, description, *args, **kwargs):
+    """Retry wrapper with exponential backoff"""
+    last_error = None
+
+    for attempt in range(1, RETRY_CONFIG['max_retries'] + 1):
+        try:
+            return func(*args, **kwargs)
+        except Exception as error:
+            last_error = error
+
+            if attempt == RETRY_CONFIG['max_retries']:
+                print(f"❌ {description} failed after {RETRY_CONFIG['max_retries']} attempts")
+                raise error
+
+            # Calculate delay with exponential backoff
+            delay = min(
+                RETRY_CONFIG['base_delay'] * (RETRY_CONFIG['backoff_multiplier'] ** (attempt - 1)),
+                RETRY_CONFIG['max_delay']
+            )
+
+            print(f"⚠️  {description} failed (attempt {attempt}/{RETRY_CONFIG['max_retries']}): {error}")
+            print(f"🔄 Retrying in {delay}s...")
+
+            time.sleep(delay)
+
+    raise last_error
+
 def run_query(query, variables):
     url = 'https://api.github.com/graphql'
     token = os.environ.get("GITHUB_ACCESS_TOKEN")
@@ -15,13 +51,13 @@ def run_query(query, variables):
         'Authorization': f'Bearer {token}',
         'Content-Type': 'application/json',
     }
-    response = requests.post(url, json={'query': query, 'variables': variables}, headers=headers)
 
-    if response.status_code == 401:
-        raise ValueError("GitHub API returned 401 Unauthorized. Please check if your GITHUB_TOKEN is valid and has the necessary permissions.")
+    def _make_request():
+        response = requests.post(url, json={'query': query, 'variables': variables}, headers=headers)
+        response.raise_for_status()
+        return response.json()
 
-    response.raise_for_status()
-    return response.json()
+    return with_retry(_make_request, f"GitHub GraphQL API query")
 
 def load_github_discussions():
     query = """
@@ -216,9 +252,13 @@ def generate_sitemap(discussions, filename="public/github-discussions-sitemap.xm
         discussions = load_github_discussions()
         save_discussions_to_json(discussions)
         generate_sitemap(discussions)
+        print("✅ Successfully loaded GitHub discussions, saved JSON, and generated sitemap")
     except ValueError as e:
-        print(f"Error: {e}")
+        print(f"❌ Configuration error: {e}")
+        exit(1)
     except requests.exceptions.RequestException as e:
-        print(f"Network error occurred: {e}")
+        print(f"❌ Network error occurred (after retries): {e}")
+        exit(1)
     except Exception as e:
-        print(f"An unexpected error occurred: {e}")
+        print(f"❌ An unexpected error occurred: {e}")
+        exit(1)

scripts/update-github-stars.js

@@ -6,10 +6,52 @@ const path = require('path');
 const GITHUB_REPO_API_URL = "https://api.github.com/repos/langfuse/langfuse";
 const STARS_FILE_PATH = path.join(__dirname, '..', 'src', 'github-stars.ts');
 
+// Retry configuration
+const RETRY_CONFIG = {
+  maxRetries: 3,
+  baseDelay: 1000, // 1 second base delay
+  maxDelay: 30000, // 30 seconds max delay
+  backoffMultiplier: 2
+};
+
+// Sleep utility function
+const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
+
+// Retry wrapper with exponential backoff
+async function withRetry(fn, description) {
+  let lastError;
+
+  for (let attempt = 1; attempt <= RETRY_CONFIG.maxRetries; attempt++) {
+    try {
+      return await fn();
+    } catch (error) {
+      lastError = error;
+
+      if (attempt === RETRY_CONFIG.maxRetries) {
+        console.error(`❌ ${description} failed after ${RETRY_CONFIG.maxRetries} attempts`);
+        throw error;
+      }
+
+      // Calculate delay with exponential backoff
+      const delay = Math.min(
+        RETRY_CONFIG.baseDelay * Math.pow(RETRY_CONFIG.backoffMultiplier, attempt - 1),
+        RETRY_CONFIG.maxDelay
+      );
+
+      console.warn(`⚠️  ${description} failed (attempt ${attempt}/${RETRY_CONFIG.maxRetries}): ${error.message}`);
+      console.warn(`🔄 Retrying in ${delay}ms...`);
+
+      await sleep(delay);
+    }
+  }
+
+  throw lastError;
+}
+
 async function updateGitHubStars() {
   try {
     console.log('Fetching GitHub stars...');
-    
+
     // Prepare headers for the GitHub API request
     const headers = {
       Accept: "application/vnd.github.v3+json",
@@ -23,25 +65,29 @@ async function updateGitHubStars() {
       console.warn('Warning: GITHUB_ACCESS_TOKEN not set. API requests may be rate limited.');
     }
 
-    // Fetch repository information from GitHub API
-    const response = await fetch(GITHUB_REPO_API_URL, { headers });
-
-    if (!response.ok) {
-      throw new Error(`GitHub API request failed: ${response.status} ${response.statusText}`);
-    }
+    const { starCount } = await withRetry(async () => {
+      // Fetch repository information from GitHub API
+      const response = await fetch(GITHUB_REPO_API_URL, { headers });
 
-    const repoData = await response.json();
-    const starCount = repoData.stargazers_count;
+      if (!response.ok) {
+        throw new Error(`GitHub API request failed: ${response.status} ${response.statusText}`);
+      }
 
-    if (typeof starCount !== 'number') {
-      throw new Error('Invalid star count received from GitHub API');
-    }
+      const repoData = await response.json();
+      const starCount = repoData.stargazers_count;
+
+      if (typeof starCount !== 'number') {
+        throw new Error('Invalid star count received from GitHub API');
+      }
+
+      return { starCount };
+    }, 'GitHub API fetch');
 
     console.log(`✅ Fetched ${starCount} GitHub stars`);
 
     // Update the TypeScript file
     const newContent = `export const GITHUB_STARS = ${starCount};`;
-    
+
     fs.writeFileSync(STARS_FILE_PATH, newContent, 'utf8');
     console.log(`✅ Updated ${STARS_FILE_PATH} with ${starCount} stars`);