Enhance GitHub stats action and background function

Andre-Diamond · Andre-Diamond · commit 8981b856ac57 · 2025-08-08T06:53:09.000+02:00
- Refactored `getStats.js` to include hardcoded endpoints for BASE_URL and Netlify function URL, improving configuration clarity.
- Implemented retry logic for GitHub API calls to handle rate limits and errors more gracefully.
- Added functions to fetch missing commits, pull requests, and issues, streamlining data collection from GitHub.
- Updated the Netlify background function to process incoming data more efficiently, including upserting contributors, commits, pull requests, and issues into Supabase.
- Improved error handling and logging for better debugging and monitoring of the data processing flow.
diff --git a/apps/mesh-gov/src/pages/api/github/existing-ids.ts b/apps/mesh-gov/src/pages/api/github/existing-ids.ts
@@ -0,0 +1,77 @@
+import type { NextApiRequest, NextApiResponse } from 'next'
+import { supabase } from '../../../lib/supabase'
+
+type ExistingIdsResponse = {
+    repoId: number | null
+    commitShas: string[]
+    prNumbers: number[]
+    issueNumbers: number[]
+}
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+    if (req.method !== 'GET') {
+        return res.status(405).json({ error: 'Method not allowed' })
+    }
+
+    try {
+        const orgLogin = (req.query.org as string | undefined)?.trim()
+        const repoName = (req.query.repo as string | undefined)?.trim()
+
+        if (!orgLogin || !repoName) {
+            return res.status(400).json({ error: 'Missing required query params: org, repo' })
+        }
+
+        // Get org id
+        const { data: orgs, error: orgError } = await supabase
+            .from('github_orgs')
+            .select('id')
+            .eq('login', orgLogin)
+            .limit(1)
+
+        if (orgError) throw new Error(orgError.message)
+        if (!orgs || orgs.length === 0) {
+            return res.status(404).json({ error: 'Organization not found' })
+        }
+
+        const orgId = orgs[0].id as number
+
+        // Get repo id by org_id + name
+        const { data: repos, error: repoError } = await supabase
+            .from('github_repos')
+            .select('id, name')
+            .eq('org_id', orgId)
+            .eq('name', repoName)
+            .limit(1)
+
+        if (repoError) throw new Error(repoError.message)
+        if (!repos || repos.length === 0) {
+            const empty: ExistingIdsResponse = { repoId: null, commitShas: [], prNumbers: [], issueNumbers: [] }
+            return res.status(200).json(empty)
+        }
+
+        const repoId = repos[0].id as number
+
+        // Fetch existing commit SHAs, PR numbers, Issue numbers in parallel
+        const [commitsResp, prsResp, issuesResp] = await Promise.all([
+            supabase.from('commits').select('sha').eq('repo_id', repoId),
+            supabase.from('pull_requests').select('number').eq('repo_id', repoId),
+            supabase.from('issues').select('number').eq('repo_id', repoId),
+        ])
+
+        if (commitsResp.error) throw new Error(commitsResp.error.message)
+        if (prsResp.error) throw new Error(prsResp.error.message)
+        if (issuesResp.error) throw new Error(issuesResp.error.message)
+
+        const commitShas = (commitsResp.data ?? []).map((c: any) => c.sha as string)
+        const prNumbers = (prsResp.data ?? []).map((p: any) => p.number as number)
+        const issueNumbers = (issuesResp.data ?? []).map((i: any) => i.number as number)
+
+        const payload: ExistingIdsResponse = { repoId, commitShas, prNumbers, issueNumbers }
+        return res.status(200).json(payload)
+    } catch (err: any) {
+        console.error('existing-ids API error:', err?.message || err)
+        return res.status(500).json({ error: 'Internal server error' })
+    }
+}
+
+
diff --git a/apps/shared-backend/github-actions/github-stats-action/getStats.js b/apps/shared-backend/github-actions/github-stats-action/getStats.js
@@ -6,11 +6,16 @@ import fetch from 'node-fetch'
 const ORG = process.env.ORG
 const REPO = process.env.REPO
 const GITHUB_TOKEN = process.env.GITHUB_TOKEN
+// Hardcoded endpoints
+const BASE_URL = 'https://gov.meshjs.dev/'
+const NETLIFY_FUNCTION_URL = 'https://glittering-chebakia-09bd42.netlify.app/.netlify/functions/github-stats-background'
 
 // Debug logging
 console.log('🔍 Environment variables:')
 console.log(`  ORG: ${process.env.ORG}`)
 console.log(`  REPO: ${process.env.REPO}`)
+console.log(`  BASE_URL: ${BASE_URL}`)
+console.log(`  NETLIFY_FUNCTION_URL: ${NETLIFY_FUNCTION_URL}`)
 console.log(`  GITHUB_TOKEN: ${GITHUB_TOKEN ? '[REDACTED]' : '(not provided)'}`)
 
 // Input validation function
@@ -42,6 +47,8 @@ function validateInputs() {
         errors.push('GITHUB_TOKEN must be a non-empty string')
     }
 
+    // BASE_URL and NETLIFY_FUNCTION_URL are hardcoded
+
     return errors
 }
 
@@ -58,13 +65,13 @@ if (validationErrors.length > 0) {
 const parsedOrg = ORG.trim()
 const parsedRepo = REPO.trim()
 const parsedGithubToken = GITHUB_TOKEN.trim()
+const parsedBaseUrl = BASE_URL.trim().replace(/\/$/, '')
+const parsedFunctionUrl = NETLIFY_FUNCTION_URL.trim()
 
 console.log('📊 Resolved values:')
 console.log(`  ORG: ${parsedOrg}`)
 console.log(`  REPO: ${parsedRepo}`)
-
-// Netlify function configuration - hardcoded URL
-const NETLIFY_FUNCTION_URL = 'https://glittering-chebakia-09bd42.netlify.app/.netlify/functions/github-stats-background'
+console.log(`  BASE_URL: ${parsedBaseUrl}`)
 
 // Helper function to make HTTP requests
 async function makeRequest(url, options = {}) {
@@ -103,41 +110,167 @@ async function makeRequest(url, options = {}) {
     }
 }
 
+// Retry helper for GitHub API calls
+async function retryWithBackoff(fn, maxRetries = 5, initialDelayMs = 1000) {
+    let attempt = 0
+    let delay = initialDelayMs
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        try {
+            return await fn()
+        } catch (error) {
+            const status = error.status || error.code || error?.response?.status
+            if (attempt < maxRetries && (status === 403 || status === 429 || (typeof status === 'number' && status >= 500))) {
+                attempt += 1
+                console.warn(`Retrying after error ${status} (attempt ${attempt}/${maxRetries})...`)
+                await new Promise((r) => setTimeout(r, delay))
+                delay *= 2
+                continue
+            }
+            throw error
+        }
+    }
+}
+
+function ghHeaders() {
+    return {
+        'Accept': 'application/vnd.github.v3+json',
+        'Authorization': `token ${parsedGithubToken}`
+    }
+}
+
+async function fetchExistingIds() {
+    const url = `${parsedBaseUrl}/api/github/existing-ids?org=${encodeURIComponent(parsedOrg)}&repo=${encodeURIComponent(parsedRepo)}`
+    return await makeRequest(url, { method: 'GET' })
+}
+
+async function fetchAllMissingCommits(existingShasSet) {
+    const results = []
+    let page = 1
+    while (true) {
+        const data = await retryWithBackoff(async () => {
+            const resp = await fetch(`https://api.github.com/repos/${parsedOrg}/${parsedRepo}/commits?per_page=100&page=${page}`, { headers: ghHeaders() })
+            if (!resp.ok) throw { status: resp.status, message: await resp.text() }
+            return resp.json()
+        })
+        if (!Array.isArray(data) || data.length === 0) break
+        for (const item of data) {
+            if (!existingShasSet.has(item.sha)) {
+                // Ensure we have full commit details
+                const details = await retryWithBackoff(async () => {
+                    const resp = await fetch(`https://api.github.com/repos/${parsedOrg}/${parsedRepo}/commits/${item.sha}`, { headers: ghHeaders() })
+                    if (!resp.ok) throw { status: resp.status, message: await resp.text() }
+                    return resp.json()
+                })
+                results.push(details)
+            }
+        }
+        page += 1
+    }
+    return results
+}
+
+async function fetchAllMissingPulls(existingNumbersSet) {
+    const results = []
+    let page = 1
+    while (true) {
+        const data = await retryWithBackoff(async () => {
+            const resp = await fetch(`https://api.github.com/repos/${parsedOrg}/${parsedRepo}/pulls?state=all&per_page=100&page=${page}`, { headers: ghHeaders() })
+            if (!resp.ok) throw { status: resp.status, message: await resp.text() }
+            return resp.json()
+        })
+        if (!Array.isArray(data) || data.length === 0) break
+        for (const pr of data) {
+            if (!existingNumbersSet.has(pr.number)) {
+                const prDetails = await retryWithBackoff(async () => {
+                    const resp = await fetch(`https://api.github.com/repos/${parsedOrg}/${parsedRepo}/pulls/${pr.number}`, { headers: ghHeaders() })
+                    if (!resp.ok) throw { status: resp.status, message: await resp.text() }
+                    return resp.json()
+                })
+                const prCommits = await retryWithBackoff(async () => {
+                    const resp = await fetch(`https://api.github.com/repos/${parsedOrg}/${parsedRepo}/pulls/${pr.number}/commits`, { headers: ghHeaders() })
+                    if (!resp.ok) throw { status: resp.status, message: await resp.text() }
+                    return resp.json()
+                })
+                results.push({ details: prDetails, commits: prCommits })
+            }
+        }
+        page += 1
+    }
+    return results
+}
+
+async function fetchAllMissingIssues(existingNumbersSet) {
+    const results = []
+    let page = 1
+    while (true) {
+        const data = await retryWithBackoff(async () => {
+            const resp = await fetch(`https://api.github.com/repos/${parsedOrg}/${parsedRepo}/issues?state=all&per_page=100&page=${page}`, { headers: ghHeaders() })
+            if (!resp.ok) throw { status: resp.status, message: await resp.text() }
+            return resp.json()
+        })
+        if (!Array.isArray(data) || data.length === 0) break
+        for (const issue of data) {
+            if (issue.pull_request) continue // skip PRs
+            if (!existingNumbersSet.has(issue.number)) {
+                results.push(issue)
+            }
+        }
+        page += 1
+    }
+    return results
+}
+
 // Main function
 async function main() {
     console.log(`🚀 Starting GitHub stats collection for repository: ${parsedOrg}/${parsedRepo}`)
 
     // Trigger the Netlify background function
-    console.log('📡 Triggering Netlify background function...')
+    console.log('📥 Fetching existing IDs from mesh-gov API...')
+    const existing = await fetchExistingIds()
+    const existingCommitShas = new Set(existing.commitShas || [])
+    const existingPrNumbers = new Set(existing.prNumbers || [])
+    const existingIssueNumbers = new Set(existing.issueNumbers || [])
 
-    const functionUrl = new URL(NETLIFY_FUNCTION_URL)
-    functionUrl.searchParams.set('org', parsedOrg)
-    functionUrl.searchParams.set('repo', parsedRepo)
-    functionUrl.searchParams.set('githubToken', parsedGithubToken)
+    console.log(`🔎 Found existing: commits=${existingCommitShas.size}, PRs=${existingPrNumbers.size}, issues=${existingIssueNumbers.size}`)
 
-    console.log(`🌐 Calling URL: ${functionUrl.toString().replace(parsedGithubToken, '[REDACTED]')}`)
+    console.log('⬇️  Fetching missing commits from GitHub...')
+    const missingCommits = await fetchAllMissingCommits(existingCommitShas)
+    console.log(`🧮 Missing commits: ${missingCommits.length}`)
 
-    try {
-        const functionResponse = await makeRequest(functionUrl.toString(), {
-            method: 'GET'
-        })
+    console.log('⬇️  Fetching missing pull requests from GitHub...')
+    const missingPulls = await fetchAllMissingPulls(existingPrNumbers)
+    console.log(`🧮 Missing PRs: ${missingPulls.length}`)
 
-        console.log('✅ Background function triggered successfully')
-        console.log(`📊 Response: ${JSON.stringify(functionResponse, null, 2)}`)
-
-        // If the response indicates success (even if it's not JSON), we're done
-        if (functionResponse.success !== false) {
-            console.log('✅ Background function appears to have been triggered successfully')
-            console.log('🎉 GitHub stats collection initiated - the background function will handle the rest')
-            process.exit(0)
-        } else {
-            console.error('❌ Background function returned an error')
-            process.exit(1)
-        }
-    } catch (error) {
-        console.error('❌ Failed to trigger background function:', error.message)
-        process.exit(1)
+    console.log('⬇️  Fetching missing issues from GitHub...')
+    const missingIssues = await fetchAllMissingIssues(existingIssueNumbers)
+    console.log(`🧮 Missing issues: ${missingIssues.length}`)
+
+    const payload = {
+        org: parsedOrg,
+        repo: parsedRepo,
+        commits: missingCommits,
+        pulls: missingPulls,
+        issues: missingIssues,
     }
+
+    if (
+        (missingCommits && missingCommits.length) ||
+        (missingPulls && missingPulls.length) ||
+        (missingIssues && missingIssues.length)
+    ) {
+        console.log('📡 Sending new data to Netlify background function...')
+        const response = await makeRequest(parsedFunctionUrl, {
+            method: 'POST',
+            body: JSON.stringify(payload),
+        })
+        console.log(`✅ Background function response: ${JSON.stringify(response, null, 2)}`)
+    } else {
+        console.log('✅ No new data to send')
+    }
+
+    console.log('🎉 Done')
+    process.exit(0)
 }
 
 // Run the main function
diff --git a/apps/shared-backend/netlify/functions/github-stats-background.mts b/apps/shared-backend/netlify/functions/github-stats-background.mts
