diff --git a/.claude/plans/github-contributors-data-layer.md b/.claude/plans/github-contributors-data-layer.md new file mode 100644 index 00000000000..9eba10cd5d0 --- /dev/null +++ b/.claude/plans/github-contributors-data-layer.md @@ -0,0 +1,172 @@ +# Plan: Migrate GitHub Contributors to Data Layer + +## Summary + +Replace the current per-request GitHub API fetching with pre-computed data stored in Netlify Blobs via the existing data-layer infrastructure. This eliminates ~173K-347K API calls per build. + +## Files to Delete (Previous Implementation) + +- `src/scripts/github/getGitHubContributors.ts` +- `src/data/github/contributors.json` +- `src/data/github/app-contributors.json` +- `.github/workflows/get-github-contributors.yml` + +## Files to Create + +### 1. `src/data-layer/fetchers/fetchGitHubContributors.ts` + +New fetcher that: +- Fetches contributors for all content files from GitHub API +- Fetches contributors for all app pages +- Returns `GitHubContributorsData` type +- Follows existing fetcher patterns (logging, error handling, rate limiting) + +```typescript +export const FETCH_GITHUB_CONTRIBUTORS_TASK_ID = "fetch-github-contributors" + +export async function fetchGitHubContributors(): Promise { + // Fetch all content file contributors + // Fetch all app page contributors + // Return combined data +} +``` + +### 2. `src/data-layer/mocks/fetch-github-contributors.json` + +Mock data for local development with `USE_MOCK_DATA=true`. + +## Files to Modify + +### 1. `src/lib/types.ts` + +Add type definition: +```typescript +export type GitHubContributorsData = { + content: Record // slug -> contributors + appPages: Record // pagePath -> contributors + generatedAt: string +} +``` + +### 2. `src/data-layer/tasks.ts` + +- Add import for `fetchGitHubContributors` +- Add key: `GITHUB_CONTRIBUTORS: "fetch-github-contributors"` +- Add to `DAILY` array: `[KEYS.GITHUB_CONTRIBUTORS, fetchGitHubContributors]` + +### 3. `src/data-layer/index.ts` + +Add getter: +```typescript +export const getGitHubContributors = () => + get(KEYS.GITHUB_CONTRIBUTORS) +``` + +### 4. `src/lib/data/index.ts` + +Add cached wrapper: +```typescript +export const getGitHubContributors = createCachedGetter( + dataLayer.getGitHubContributors, + ["github-contributors"], + CACHE_REVALIDATE_DAY +) +``` + +### 5. `src/lib/utils/gh.ts` + +- Remove the static JSON imports I added earlier +- Remove `getStaticContentContributors` and `getStaticAppContributors` +- Keep `fetchAndCacheGitHubContributors` as fallback for dev/new files + +### 6. `src/lib/utils/contributors.ts` + +Update to use data-layer: +```typescript +import { getGitHubContributors } from "@/lib/data" + +export const getMarkdownFileContributorInfo = async (...) => { + const contributorsData = await getGitHubContributors() + let gitHubContributors = contributorsData?.content[slug] || null + + // Fallback to API if not in data layer (new files during dev) + if (!gitHubContributors) { + gitHubContributors = await fetchAndCacheGitHubContributors(...) + } + // ... rest unchanged +} + +export const getAppPageContributorInfo = async (...) => { + const contributorsData = await getGitHubContributors() + let uniqueGitHubContributors = contributorsData?.appPages[pagePath] || null + + // Fallback to API if not in data layer + if (!uniqueGitHubContributors) { + // ... existing API fetch logic + } + // ... rest unchanged +} +``` + +## Data Flow + +``` +Trigger.dev (daily) + ↓ +fetchGitHubContributors() - fetches from GitHub API + ↓ +set(KEYS.GITHUB_CONTRIBUTORS, data) - stores in Netlify Blobs + ↓ +Page render calls getGitHubContributors() + ↓ +unstable_cache + React cache (request dedup) + ↓ +storage.get() - retrieves from Netlify Blobs + ↓ +contributors.ts uses data (zero API calls) +``` + +## Implementation Order + +1. Delete previous implementation files +2. Add `GitHubContributorsData` type to `src/lib/types.ts` +3. Create `src/data-layer/fetchers/fetchGitHubContributors.ts` +4. Create `src/data-layer/mocks/fetch-github-contributors.json` +5. Update `src/data-layer/tasks.ts` (key + import + DAILY registration) +6. Update `src/data-layer/index.ts` (add getter) +7. Update `src/lib/data/index.ts` (add cached wrapper) +8. Update `src/lib/utils/gh.ts` (remove static imports/functions) +9. Update `src/lib/utils/contributors.ts` (use data-layer) +10. Run `pnpm lint:fix` and `npx tsc --noEmit` + +## Notes + +- **No filesystem access** in Trigger.dev - use GitHub Contents API to list files +- Rate limiting: Use delays between requests (100-500ms) +- App pages list: Predefined static list (changes infrequently) +- Content files: Use GitHub API `GET /repos/{owner}/{repo}/contents/{path}` to recursively list `public/content/` + +## GitHub API for File Discovery + +```typescript +// List directory contents recursively +async function listContentFiles(path = "public/content"): Promise { + const url = `https://api.github.com/repos/ethereum/ethereum-org-website/contents/${path}` + const response = await fetch(url, { + headers: { Authorization: `token ${token}` } + }) + const items = await response.json() + + const slugs: string[] = [] + for (const item of items) { + if (item.type === "dir" && item.name !== "translations") { + // Recursively list subdirectories + slugs.push(...await listContentFiles(item.path)) + } else if (item.name === "index.md") { + // Found a content file, extract slug + slugs.push(path.replace("public/content/", "")) + } + } + return slugs +} +``` diff --git a/app/[locale]/10years/page.tsx b/app/[locale]/10years/page.tsx index cf7d587f8db..cb823770134 100644 --- a/app/[locale]/10years/page.tsx +++ b/app/[locale]/10years/page.tsx @@ -5,7 +5,7 @@ import { setRequestLocale, } from "next-intl/server" -import type { CommitHistory, Lang, PageParams } from "@/lib/types" +import type { Lang, PageParams } from "@/lib/types" import Emoji from "@/components/Emoji" import I18nProvider from "@/components/I18nProvider" @@ -64,11 +64,9 @@ const Page = async ({ params }: { params: PageParams }) => { const innovationCards = await getInnovationCards() const adoptionCards = await getAdoptionCards() - const commitHistoryCache: CommitHistory = {} const { contributors } = await getAppPageContributorInfo( "10years", - locale as Lang, - commitHistoryCache + locale as Lang ) return ( diff --git a/app/[locale]/apps/[application]/page.tsx b/app/[locale]/apps/[application]/page.tsx index b99a7548800..03a83d394d4 100644 --- a/app/[locale]/apps/[application]/page.tsx +++ b/app/[locale]/apps/[application]/page.tsx @@ -6,7 +6,7 @@ import { setRequestLocale, } from "next-intl/server" -import type { ChainName, CommitHistory, Lang, PageParams } from "@/lib/types" +import type { ChainName, Lang, PageParams } from "@/lib/types" import AppCard from "@/components/AppCard" import ChainImages from "@/components/ChainImages" @@ -131,11 +131,9 @@ const Page = async ({ return new Date(app.dateOfLaunch).getFullYear() } - const commitHistoryCache: CommitHistory = {} const { contributors } = await getAppPageContributorInfo( "apps/[application]", - locale as Lang, - commitHistoryCache + locale as Lang ) return ( diff --git a/app/[locale]/apps/categories/[catetgoryName]/page.tsx b/app/[locale]/apps/categories/[catetgoryName]/page.tsx index 9b67093d057..0656403daae 100644 --- a/app/[locale]/apps/categories/[catetgoryName]/page.tsx +++ b/app/[locale]/apps/categories/[catetgoryName]/page.tsx @@ -8,7 +8,6 @@ import { import { AppCategoryEnum, - type CommitHistory, type Lang, type PageParams, type SectionNavDetails, @@ -104,11 +103,9 @@ const Page = async ({ }) ) - const commitHistoryCache: CommitHistory = {} const { contributors } = await getAppPageContributorInfo( "apps/categories/[catetgoryName]", - locale as Lang, - commitHistoryCache + locale as Lang ) return ( diff --git a/app/[locale]/apps/page.tsx b/app/[locale]/apps/page.tsx index 44a07b8f30e..df0fa462990 100644 --- a/app/[locale]/apps/page.tsx +++ b/app/[locale]/apps/page.tsx @@ -5,7 +5,7 @@ import { setRequestLocale, } from "next-intl/server" -import { CommitHistory, Lang, PageParams } from "@/lib/types" +import { Lang, PageParams } from "@/lib/types" import AppCard from "@/components/AppCard" import Breadcrumbs from "@/components/Breadcrumbs" @@ -67,11 +67,9 @@ const Page = async ({ params }: { params: PageParams }) => { const requiredNamespaces = getRequiredNamespacesForPage("/apps") const messages = pick(allMessages, requiredNamespaces) - const commitHistoryCache: CommitHistory = {} const { contributors } = await getAppPageContributorInfo( "apps", - locale as Lang, - commitHistoryCache + locale as Lang ) return ( diff --git a/app/[locale]/assets/page.tsx b/app/[locale]/assets/page.tsx index e37966ae161..e36927fb261 100644 --- a/app/[locale]/assets/page.tsx +++ b/app/[locale]/assets/page.tsx @@ -5,7 +5,7 @@ import { setRequestLocale, } from "next-intl/server" -import type { CommitHistory, Lang, PageParams } from "@/lib/types" +import type { Lang, PageParams } from "@/lib/types" import I18nProvider from "@/components/I18nProvider" @@ -26,11 +26,9 @@ export default async function Page({ params }: { params: PageParams }) { const requiredNamespaces = getRequiredNamespacesForPage("/assets") const messages = pick(allMessages, requiredNamespaces) - const commitHistoryCache: CommitHistory = {} const { contributors } = await getAppPageContributorInfo( "assets", - locale as Lang, - commitHistoryCache + locale as Lang ) return ( diff --git a/app/[locale]/bug-bounty/page.tsx b/app/[locale]/bug-bounty/page.tsx index 764cad904d4..4b7c350119e 100644 --- a/app/[locale]/bug-bounty/page.tsx +++ b/app/[locale]/bug-bounty/page.tsx @@ -1,7 +1,7 @@ import { getTranslations } from "next-intl/server" import type { ComponentProps } from "react" -import type { ChildOnlyProp, CommitHistory, Lang, Params } from "@/lib/types" +import type { ChildOnlyProp, Lang, Params } from "@/lib/types" /* Uncomment for Bug Bounty Banner: */ import Breadcrumbs from "@/components/Breadcrumbs" @@ -115,13 +115,8 @@ export default async function Page({ params }: { params: Promise }) { const t = await getTranslations({ namespace: "page-bug-bounty" }) const tCommon = await getTranslations({ namespace: "common" }) - const commitHistoryCache: CommitHistory = {} const { contributors, lastEditLocaleTimestamp } = - await getAppPageContributorInfo( - "bug-bounty", - locale as Lang, - commitHistoryCache - ) + await getAppPageContributorInfo("bug-bounty", locale as Lang) const consensusBountyHunters: Node[] = consensusData.sort(sortBountyHuntersFn) const executionBountyHunters: Node[] = executionData.sort(sortBountyHuntersFn) diff --git a/app/[locale]/collectibles/page.tsx b/app/[locale]/collectibles/page.tsx index 0957f1f31d8..2f0b4f2d6ad 100644 --- a/app/[locale]/collectibles/page.tsx +++ b/app/[locale]/collectibles/page.tsx @@ -5,7 +5,7 @@ import { setRequestLocale, } from "next-intl/server" -import type { CommitHistory, Lang, PageParams } from "@/lib/types" +import type { Lang, PageParams } from "@/lib/types" import { HubHero } from "@/components/Hero" import I18nProvider from "@/components/I18nProvider" @@ -56,11 +56,9 @@ export default async function Page({ params }: { params: PageParams }) { const requiredNamespaces = getRequiredNamespacesForPage("/collectibles/") const pickedMessages = pick(allMessages, requiredNamespaces) - const commitHistoryCache: CommitHistory = {} const { contributors } = await getAppPageContributorInfo( "collectibles", - locale as Lang, - commitHistoryCache + locale as Lang ) return ( diff --git a/app/[locale]/community/events/page.tsx b/app/[locale]/community/events/page.tsx index ba60016670c..f821ee07f76 100644 --- a/app/[locale]/community/events/page.tsx +++ b/app/[locale]/community/events/page.tsx @@ -8,12 +8,7 @@ import { } from "lucide-react" import { getMessages, getTranslations } from "next-intl/server" -import type { - CommitHistory, - Lang, - PageParams, - SectionNavDetails, -} from "@/lib/types" +import type { Lang, PageParams, SectionNavDetails } from "@/lib/types" import ContentHero from "@/components/Hero/ContentHero" import I18nProvider from "@/components/I18nProvider" @@ -63,11 +58,9 @@ const Page = async ({ params }: { params: PageParams }) => { const requiredNamespaces = getRequiredNamespacesForPage("/community/events") const messages = pick(allMessages, requiredNamespaces) - const commitHistoryCache: CommitHistory = {} const { contributors } = await getAppPageContributorInfo( "community/events", - locale as Lang, - commitHistoryCache + locale as Lang ) const events = mapEventTranslations(_events, t) diff --git a/app/[locale]/community/page.tsx b/app/[locale]/community/page.tsx index 092e6cfa28b..7be5b193dbe 100644 --- a/app/[locale]/community/page.tsx +++ b/app/[locale]/community/page.tsx @@ -5,7 +5,7 @@ import { setRequestLocale, } from "next-intl/server" -import type { CommitHistory, Lang, PageParams } from "@/lib/types" +import type { Lang, PageParams } from "@/lib/types" import I18nProvider from "@/components/I18nProvider" @@ -26,11 +26,9 @@ export default async function Page({ params }: { params: PageParams }) { const requiredNamespaces = getRequiredNamespacesForPage("/community") const pickedMessages = pick(allMessages, requiredNamespaces) - const commitHistoryCache: CommitHistory = {} const { contributors } = await getAppPageContributorInfo( "community", - locale as Lang, - commitHistoryCache + locale as Lang ) return ( diff --git a/app/[locale]/contributing/translation-program/acknowledgements/page.tsx b/app/[locale]/contributing/translation-program/acknowledgements/page.tsx index abd537d06c0..9ef5c4b16a4 100644 --- a/app/[locale]/contributing/translation-program/acknowledgements/page.tsx +++ b/app/[locale]/contributing/translation-program/acknowledgements/page.tsx @@ -5,7 +5,7 @@ import { setRequestLocale, } from "next-intl/server" -import type { CommitHistory, Lang, PageParams } from "@/lib/types" +import type { Lang, PageParams } from "@/lib/types" import I18nProvider from "@/components/I18nProvider" @@ -28,11 +28,9 @@ const Page = async ({ params }: { params: PageParams }) => { ) const messages = pick(allMessages, requiredNamespaces) - const commitHistoryCache: CommitHistory = {} const { contributors } = await getAppPageContributorInfo( "contributing/translation-program/acknowledgements", - locale as Lang, - commitHistoryCache + locale as Lang ) return ( diff --git a/app/[locale]/contributing/translation-program/contributors/page.tsx b/app/[locale]/contributing/translation-program/contributors/page.tsx index eea58f1515c..34209469159 100644 --- a/app/[locale]/contributing/translation-program/contributors/page.tsx +++ b/app/[locale]/contributing/translation-program/contributors/page.tsx @@ -5,7 +5,7 @@ import { setRequestLocale, } from "next-intl/server" -import type { CommitHistory, Lang, PageParams } from "@/lib/types" +import type { Lang, PageParams } from "@/lib/types" import I18nProvider from "@/components/I18nProvider" @@ -21,11 +21,9 @@ const Page = async ({ params }: { params: PageParams }) => { setRequestLocale(locale) - const commitHistoryCache: CommitHistory = {} const { contributors } = await getAppPageContributorInfo( "contributing/translation-program/contributors", - locale as Lang, - commitHistoryCache + locale as Lang ) // Get i18n messages diff --git a/app/[locale]/contributing/translation-program/translatathon/leaderboard/page.tsx b/app/[locale]/contributing/translation-program/translatathon/leaderboard/page.tsx index 0c59ce6c02d..49459c20a89 100644 --- a/app/[locale]/contributing/translation-program/translatathon/leaderboard/page.tsx +++ b/app/[locale]/contributing/translation-program/translatathon/leaderboard/page.tsx @@ -1,6 +1,6 @@ import { setRequestLocale } from "next-intl/server" -import type { CommitHistory, Lang } from "@/lib/types" +import type { Lang } from "@/lib/types" import { List as ButtonDropdownList } from "@/components/ButtonDropdown" import ContentHero, { ContentHeroProps } from "@/components/Hero/ContentHero" @@ -102,11 +102,9 @@ const Page = async ({ params }: { params: Promise<{ locale: string }> }) => { }) } - const commitHistoryCache: CommitHistory = {} const { contributors } = await getAppPageContributorInfo( "contributing/translation-program/translatathon/leaderboard", - locale as Lang, - commitHistoryCache + locale as Lang ) return ( diff --git a/app/[locale]/developers/page.tsx b/app/[locale]/developers/page.tsx index 2f69b11e16e..2842c74ada0 100644 --- a/app/[locale]/developers/page.tsx +++ b/app/[locale]/developers/page.tsx @@ -1,6 +1,6 @@ import { getTranslations } from "next-intl/server" -import type { CommitHistory, Lang, PageParams } from "@/lib/types" +import type { Lang, PageParams } from "@/lib/types" import { ChildOnlyProp } from "@/lib/types" import BigNumber from "@/components/BigNumber" @@ -141,11 +141,9 @@ const DevelopersPage = async ({ params }: { params: PageParams }) => { const hackathons = (await getHackathons()).slice(0, 5) - const commitHistoryCache: CommitHistory = {} const { contributors } = await getAppPageContributorInfo( "developers", - locale as Lang, - commitHistoryCache + locale as Lang ) return ( diff --git a/app/[locale]/developers/tools/[category]/page.tsx b/app/[locale]/developers/tools/[category]/page.tsx index 09538fbacdc..5e2c167ce5d 100644 --- a/app/[locale]/developers/tools/[category]/page.tsx +++ b/app/[locale]/developers/tools/[category]/page.tsx @@ -1,7 +1,7 @@ import { notFound, redirect } from "next/navigation" import { getTranslations, setRequestLocale } from "next-intl/server" -import type { CommitHistory, Lang, PageParams } from "@/lib/types" +import type { Lang, PageParams } from "@/lib/types" import { ContentHero } from "@/components/Hero" import MainArticle from "@/components/MainArticle" @@ -81,11 +81,9 @@ const Page = async ({ .filter(Boolean) // Get contributor info for JSON-LD - const commitHistoryCache: CommitHistory = {} const { contributors } = await getAppPageContributorInfo( `developers/tools/${category}`, - locale as Lang, - commitHistoryCache + locale as Lang ) return ( diff --git a/app/[locale]/developers/tools/page.tsx b/app/[locale]/developers/tools/page.tsx index 922555b4ed9..f2962cdb36b 100644 --- a/app/[locale]/developers/tools/page.tsx +++ b/app/[locale]/developers/tools/page.tsx @@ -1,7 +1,7 @@ import { redirect } from "next/navigation" import { getTranslations, setRequestLocale } from "next-intl/server" -import type { CommitHistory, Lang, PageParams } from "@/lib/types" +import type { Lang, PageParams } from "@/lib/types" import AppCard from "@/components/AppCard" import { ContentHero } from "@/components/Hero" @@ -72,11 +72,9 @@ const Page = async ({ ) as DeveloperToolsByCategory // Get contributor info for JSON-LD - const commitHistoryCache: CommitHistory = {} const { contributors } = await getAppPageContributorInfo( "developers/tools", - locale as Lang, - commitHistoryCache + locale as Lang ) return ( diff --git a/app/[locale]/developers/tutorials/page.tsx b/app/[locale]/developers/tutorials/page.tsx index 4049c35358b..9523f9a2400 100644 --- a/app/[locale]/developers/tutorials/page.tsx +++ b/app/[locale]/developers/tutorials/page.tsx @@ -6,7 +6,7 @@ import { setRequestLocale, } from "next-intl/server" -import type { CommitHistory, Lang, PageParams } from "@/lib/types" +import type { Lang, PageParams } from "@/lib/types" import FeedbackCard from "@/components/FeedbackCard" import ContentHero, { ContentHeroProps } from "@/components/Hero/ContentHero" @@ -78,11 +78,9 @@ const Page = async ({ params }: { params: PageParams }) => { const internalTutorials = await getTutorialsData(locale) - const commitHistoryCache: CommitHistory = {} const { contributors } = await getAppPageContributorInfo( "developers/tutorials", - locale as Lang, - commitHistoryCache + locale as Lang ) const heroProps: ContentHeroProps = { diff --git a/app/[locale]/ethereum-history-founder-and-ownership/page.tsx b/app/[locale]/ethereum-history-founder-and-ownership/page.tsx index 9aaa5a7b102..e690349124f 100644 --- a/app/[locale]/ethereum-history-founder-and-ownership/page.tsx +++ b/app/[locale]/ethereum-history-founder-and-ownership/page.tsx @@ -1,6 +1,6 @@ import { getTranslations, setRequestLocale } from "next-intl/server" -import type { CommitHistory, Lang, ToCItem } from "@/lib/types" +import type { Lang, ToCItem } from "@/lib/types" import CommentCard from "@/components/CommentCard" import FileContributors from "@/components/FileContributors" @@ -30,12 +30,10 @@ const Page = async ({ params }: { params: Promise<{ locale: Lang }> }) => { namespace: "page-ethereum-history-founder-and-ownership", }) - const commitHistoryCache: CommitHistory = {} const { contributors, lastEditLocaleTimestamp } = await getAppPageContributorInfo( "ethereum-history-founder-and-ownership", - locale as Lang, - commitHistoryCache + locale as Lang ) const tocItems: ToCItem[] = [ diff --git a/app/[locale]/ethereum-vs-bitcoin/page.tsx b/app/[locale]/ethereum-vs-bitcoin/page.tsx index 367785179f9..63a45a36f38 100644 --- a/app/[locale]/ethereum-vs-bitcoin/page.tsx +++ b/app/[locale]/ethereum-vs-bitcoin/page.tsx @@ -1,6 +1,6 @@ import { getTranslations, setRequestLocale } from "next-intl/server" -import type { CommitHistory, Lang, ToCItem } from "@/lib/types" +import type { Lang, ToCItem } from "@/lib/types" import FileContributors from "@/components/FileContributors" import ContentHero, { ContentHeroProps } from "@/components/Hero/ContentHero" @@ -35,13 +35,8 @@ const Page = async ({ params }: { params: Promise<{ locale: Lang }> }) => { namespace: "page-ethereum-vs-bitcoin", }) - const commitHistoryCache: CommitHistory = {} const { contributors, lastEditLocaleTimestamp } = - await getAppPageContributorInfo( - "ethereum-vs-bitcoin", - locale as Lang, - commitHistoryCache - ) + await getAppPageContributorInfo("ethereum-vs-bitcoin", locale as Lang) const tocItems: ToCItem[] = [ { diff --git a/app/[locale]/founders/page.tsx b/app/[locale]/founders/page.tsx index aba82190610..de611d3794e 100644 --- a/app/[locale]/founders/page.tsx +++ b/app/[locale]/founders/page.tsx @@ -2,12 +2,7 @@ import React from "react" import { Banknote, ChartNoAxesCombined, Handshake } from "lucide-react" import { getTranslations } from "next-intl/server" -import type { - CommitHistory, - Lang, - PageParams, - SectionNavDetails, -} from "@/lib/types" +import type { Lang, PageParams, SectionNavDetails } from "@/lib/types" import ContentHero from "@/components/Hero/ContentHero" import { CheckCircle } from "@/components/icons/CheckCircle" @@ -342,11 +337,9 @@ const Page = async ({ params }: { params: PageParams }) => { }, ] - const commitHistoryCache: CommitHistory = {} const { contributors } = await getAppPageContributorInfo( "founders", - locale as Lang, - commitHistoryCache + locale as Lang ) return ( diff --git a/app/[locale]/gas/page.tsx b/app/[locale]/gas/page.tsx index 4caca134ded..5132638b61c 100644 --- a/app/[locale]/gas/page.tsx +++ b/app/[locale]/gas/page.tsx @@ -5,7 +5,7 @@ import { setRequestLocale, } from "next-intl/server" -import type { CommitHistory, Lang, PageParams } from "@/lib/types" +import type { Lang, PageParams } from "@/lib/types" import I18nProvider from "@/components/I18nProvider" @@ -26,9 +26,8 @@ const Page = async ({ params }: { params: PageParams }) => { const requiredNamespaces = getRequiredNamespacesForPage("/gas") const messages = pick(allMessages, requiredNamespaces) - const commitHistoryCache: CommitHistory = {} const { contributors, lastEditLocaleTimestamp } = - await getAppPageContributorInfo("gas", locale as Lang, commitHistoryCache) + await getAppPageContributorInfo("gas", locale as Lang) return ( diff --git a/app/[locale]/get-eth/page.tsx b/app/[locale]/get-eth/page.tsx index 6bf5930aaf1..c0875b204b1 100644 --- a/app/[locale]/get-eth/page.tsx +++ b/app/[locale]/get-eth/page.tsx @@ -7,12 +7,7 @@ import { } from "next-intl/server" import type { ReactNode } from "react" -import type { - ChildOnlyProp, - CommitHistory, - Lang, - PageParams, -} from "@/lib/types" +import type { ChildOnlyProp, Lang, PageParams } from "@/lib/types" import CalloutBanner from "@/components/CalloutBanner" import CardList, { @@ -42,11 +37,12 @@ import { Skeleton } from "@/components/ui/skeleton" import { cn } from "@/lib/utils/cn" import { getAppPageContributorInfo } from "@/lib/utils/contributors" -import { getLastGitCommitDateByPath } from "@/lib/utils/gh" import { getMetadata } from "@/lib/utils/metadata" import { screens } from "@/lib/utils/screen" import { getRequiredNamespacesForPage } from "@/lib/utils/translations" +import { exchangesByCountryLastUpdated } from "@/data/exchangesByCountry" + import GetEthPageJsonLD from "./page-jsonld" import uniswap from "@/public/images/dapps/uni.png" @@ -131,10 +127,6 @@ export default async function Page({ params }: { params: PageParams }) { }, ] - const lastDataUpdateDate = getLastGitCommitDateByPath( - "src/data/exchangesByCountry.ts" - ) - setRequestLocale(locale) // Get i18n messages @@ -142,13 +134,8 @@ export default async function Page({ params }: { params: PageParams }) { const requiredNamespaces = getRequiredNamespacesForPage("/get-eth") const messages = pick(allMessages, requiredNamespaces) - const commitHistoryCache: CommitHistory = {} const { contributors, lastEditLocaleTimestamp } = - await getAppPageContributorInfo( - "get-eth", - locale as Lang, - commitHistoryCache - ) + await getAppPageContributorInfo("get-eth", locale as Lang) return ( <> @@ -283,7 +270,9 @@ export default async function Page({ params }: { params: PageParams }) {

{/* CLIENT SIDE */} - + diff --git a/app/[locale]/layer-2/learn/page.tsx b/app/[locale]/layer-2/learn/page.tsx index da8820dd8b2..d7382cfc9b3 100644 --- a/app/[locale]/layer-2/learn/page.tsx +++ b/app/[locale]/layer-2/learn/page.tsx @@ -5,7 +5,7 @@ import { setRequestLocale, } from "next-intl/server" -import type { CommitHistory, Lang, PageParams } from "@/lib/types" +import type { Lang, PageParams } from "@/lib/types" import I18nProvider from "@/components/I18nProvider" @@ -26,13 +26,8 @@ const Page = async ({ params }: { params: PageParams }) => { const requiredNamespaces = getRequiredNamespacesForPage("/layer-2/learn") const messages = pick(allMessages, requiredNamespaces) - const commitHistoryCache: CommitHistory = {} const { contributors, lastEditLocaleTimestamp } = - await getAppPageContributorInfo( - "layer-2/learn", - locale as Lang, - commitHistoryCache - ) + await getAppPageContributorInfo("layer-2/learn", locale as Lang) return ( diff --git a/app/[locale]/layer-2/networks/page.tsx b/app/[locale]/layer-2/networks/page.tsx index 7bbf47e17b6..ee771507e56 100644 --- a/app/[locale]/layer-2/networks/page.tsx +++ b/app/[locale]/layer-2/networks/page.tsx @@ -6,7 +6,7 @@ import { setRequestLocale, } from "next-intl/server" -import type { CommitHistory, Lang, PageParams } from "@/lib/types" +import type { Lang, PageParams } from "@/lib/types" import I18nProvider from "@/components/I18nProvider" @@ -131,11 +131,9 @@ const Page = async ({ params }: { params: PageParams }) => { }, } - const commitHistoryCache: CommitHistory = {} const { contributors } = await getAppPageContributorInfo( "layer-2/networks", - locale as Lang, - commitHistoryCache + locale as Lang ) return ( diff --git a/app/[locale]/layer-2/page.tsx b/app/[locale]/layer-2/page.tsx index 3d895be5e52..7492f27439a 100644 --- a/app/[locale]/layer-2/page.tsx +++ b/app/[locale]/layer-2/page.tsx @@ -5,7 +5,7 @@ import { setRequestLocale, } from "next-intl/server" -import type { CommitHistory, Lang, PageParams } from "@/lib/types" +import type { Lang, PageParams } from "@/lib/types" import I18nProvider from "@/components/I18nProvider" @@ -66,11 +66,9 @@ const Page = async ({ params }: { params: PageParams }) => { const requiredNamespaces = getRequiredNamespacesForPage("/layer-2") const messages = pick(allMessages, requiredNamespaces) - const commitHistoryCache: CommitHistory = {} const { contributors } = await getAppPageContributorInfo( "layer-2", - locale as Lang, - commitHistoryCache + locale as Lang ) return ( diff --git a/app/[locale]/learn/page.tsx b/app/[locale]/learn/page.tsx index 445246f4a8d..cb0f6cef201 100644 --- a/app/[locale]/learn/page.tsx +++ b/app/[locale]/learn/page.tsx @@ -2,7 +2,7 @@ import { HTMLAttributes, ReactNode } from "react" import { getTranslations } from "next-intl/server" import type { ChildOnlyProp, PageParams, ToCItem } from "@/lib/types" -import type { CommitHistory, Lang } from "@/lib/types" +import type { Lang } from "@/lib/types" import OriginalCard, { type CardProps as OriginalCardProps, @@ -119,9 +119,8 @@ export default async function Page({ params }: { params: PageParams }) { const t = await getTranslations({ locale, namespace: "page-learn" }) const tCommon = await getTranslations({ locale, namespace: "common" }) - const commitHistoryCache: CommitHistory = {} const { contributors, lastEditLocaleTimestamp } = - await getAppPageContributorInfo("learn", locale as Lang, commitHistoryCache) + await getAppPageContributorInfo("learn", locale as Lang) const tocItems = [ { diff --git a/app/[locale]/quizzes/page.tsx b/app/[locale]/quizzes/page.tsx index 62526a0fabe..cd798498909 100644 --- a/app/[locale]/quizzes/page.tsx +++ b/app/[locale]/quizzes/page.tsx @@ -5,7 +5,7 @@ import { setRequestLocale, } from "next-intl/server" -import type { CommitHistory, Lang, PageParams } from "@/lib/types" +import type { Lang, PageParams } from "@/lib/types" import I18nProvider from "@/components/I18nProvider" @@ -26,11 +26,9 @@ const Page = async ({ params }: { params: PageParams }) => { const requiredNamespaces = getRequiredNamespacesForPage("/quizzes") const messages = pick(allMessages, requiredNamespaces) - const commitHistoryCache: CommitHistory = {} const { contributors } = await getAppPageContributorInfo( "quizzes", - locale as Lang, - commitHistoryCache + locale as Lang ) return ( diff --git a/app/[locale]/resources/page.tsx b/app/[locale]/resources/page.tsx index d3341f4aedd..24af2f0f6d2 100644 --- a/app/[locale]/resources/page.tsx +++ b/app/[locale]/resources/page.tsx @@ -1,6 +1,6 @@ import { getTranslations } from "next-intl/server" -import type { CommitHistory, Lang, PageParams } from "@/lib/types" +import type { Lang, PageParams } from "@/lib/types" import BannerNotification from "@/components/Banners/BannerNotification" import { HubHero } from "@/components/Hero" @@ -66,11 +66,9 @@ const Page = async ({ params }: { params: PageParams }) => { ...blobStats, }) - const commitHistoryCache: CommitHistory = {} const { contributors } = await getAppPageContributorInfo( "resources", - locale as Lang, - commitHistoryCache + locale as Lang ) return ( diff --git a/app/[locale]/roadmap/_vision/page.tsx b/app/[locale]/roadmap/_vision/page.tsx index a9a59085544..af0568e4b30 100644 --- a/app/[locale]/roadmap/_vision/page.tsx +++ b/app/[locale]/roadmap/_vision/page.tsx @@ -5,7 +5,7 @@ import { setRequestLocale, } from "next-intl/server" -import type { CommitHistory, Lang, PageParams } from "@/lib/types" +import type { Lang, PageParams } from "@/lib/types" import I18nProvider from "@/components/I18nProvider" @@ -26,13 +26,8 @@ const Page = async ({ params }: { params: PageParams }) => { const requiredNamespaces = getRequiredNamespacesForPage("/roadmap/vision") const messages = pick(allMessages, requiredNamespaces) - const commitHistoryCache: CommitHistory = {} const { contributors, lastEditLocaleTimestamp } = - await getAppPageContributorInfo( - "roadmap/vision", - locale as Lang, - commitHistoryCache - ) + await getAppPageContributorInfo("roadmap/vision", locale as Lang) return ( diff --git a/app/[locale]/roadmap/page.tsx b/app/[locale]/roadmap/page.tsx index 0ee5c9fe479..f306c589dc5 100644 --- a/app/[locale]/roadmap/page.tsx +++ b/app/[locale]/roadmap/page.tsx @@ -5,7 +5,7 @@ import { setRequestLocale, } from "next-intl/server" -import type { CommitHistory, Lang, PageParams } from "@/lib/types" +import type { Lang, PageParams } from "@/lib/types" import I18nProvider from "@/components/I18nProvider" @@ -26,11 +26,9 @@ const Page = async ({ params }: { params: PageParams }) => { const requiredNamespaces = getRequiredNamespacesForPage("/roadmap") const messages = pick(allMessages, requiredNamespaces) - const commitHistoryCache: CommitHistory = {} const { contributors } = await getAppPageContributorInfo( "roadmap", - locale as Lang, - commitHistoryCache + locale as Lang ) return ( diff --git a/app/[locale]/run-a-node/page.tsx b/app/[locale]/run-a-node/page.tsx index 1f78bc371d2..c968f98c43f 100644 --- a/app/[locale]/run-a-node/page.tsx +++ b/app/[locale]/run-a-node/page.tsx @@ -5,7 +5,7 @@ import { setRequestLocale, } from "next-intl/server" -import type { CommitHistory, Lang, PageParams } from "@/lib/types" +import type { Lang, PageParams } from "@/lib/types" import I18nProvider from "@/components/I18nProvider" @@ -26,13 +26,8 @@ const Page = async ({ params }: { params: PageParams }) => { const requiredNamespaces = getRequiredNamespacesForPage("/run-a-node") const messages = pick(allMessages, requiredNamespaces) - const commitHistoryCache: CommitHistory = {} const { contributors, lastEditLocaleTimestamp } = - await getAppPageContributorInfo( - "run-a-node", - locale as Lang, - commitHistoryCache - ) + await getAppPageContributorInfo("run-a-node", locale as Lang) return ( diff --git a/app/[locale]/stablecoins/page.tsx b/app/[locale]/stablecoins/page.tsx index f7765d1c24d..1bb1dc9c00c 100644 --- a/app/[locale]/stablecoins/page.tsx +++ b/app/[locale]/stablecoins/page.tsx @@ -6,7 +6,7 @@ import { setRequestLocale, } from "next-intl/server" -import type { CommitHistory, Lang, PageParams } from "@/lib/types" +import type { Lang, PageParams } from "@/lib/types" import CalloutBannerSSR from "@/components/CalloutBannerSSR" import DataProductCard from "@/components/DataProductCard" @@ -410,11 +410,9 @@ async function Page({ params }: { params: PageParams }) { }, ] - const commitHistoryCache: CommitHistory = {} const { contributors } = await getAppPageContributorInfo( "stablecoins", - locale as Lang, - commitHistoryCache + locale as Lang ) return ( diff --git a/app/[locale]/staking/page.tsx b/app/[locale]/staking/page.tsx index ea1ecf93108..079c3dd7f24 100644 --- a/app/[locale]/staking/page.tsx +++ b/app/[locale]/staking/page.tsx @@ -5,7 +5,7 @@ import { setRequestLocale, } from "next-intl/server" -import { CommitHistory, Lang, PageParams, StakingStatsData } from "@/lib/types" +import { Lang, PageParams, StakingStatsData } from "@/lib/types" import I18nProvider from "@/components/I18nProvider" @@ -45,13 +45,8 @@ const Page = async ({ params }: { params: PageParams }) => { const requiredNamespaces = getRequiredNamespacesForPage("/staking") const messages = pick(allMessages, requiredNamespaces) - const commitHistoryCache: CommitHistory = {} const { contributors, lastEditLocaleTimestamp } = - await getAppPageContributorInfo( - "staking", - locale as Lang, - commitHistoryCache - ) + await getAppPageContributorInfo("staking", locale as Lang) return ( diff --git a/app/[locale]/start/page.tsx b/app/[locale]/start/page.tsx index d8f90d68420..a8e28efe447 100644 --- a/app/[locale]/start/page.tsx +++ b/app/[locale]/start/page.tsx @@ -5,7 +5,7 @@ import { setRequestLocale, } from "next-intl/server" -import type { CommitHistory, Lang, PageParams } from "@/lib/types" +import type { Lang, PageParams } from "@/lib/types" import I18nProvider from "@/components/I18nProvider" import { Image } from "@/components/Image" @@ -40,11 +40,9 @@ const Page = async ({ params }: { params: PageParams }) => { supportedLanguages: [], })) - const commitHistoryCache: CommitHistory = {} const { contributors } = await getAppPageContributorInfo( "start", - locale as Lang, - commitHistoryCache + locale as Lang ) return ( diff --git a/app/[locale]/trillion-dollar-security/page.tsx b/app/[locale]/trillion-dollar-security/page.tsx index 138789747d2..ff37e8f15ba 100644 --- a/app/[locale]/trillion-dollar-security/page.tsx +++ b/app/[locale]/trillion-dollar-security/page.tsx @@ -2,7 +2,7 @@ import React from "react" import Image from "next/image" import { getTranslations, setRequestLocale } from "next-intl/server" -import type { CommitHistory, Lang, PageParams } from "@/lib/types" +import type { Lang, PageParams } from "@/lib/types" import MainArticle from "@/components/MainArticle" import { ButtonLink } from "@/components/ui/buttons/Button" @@ -53,11 +53,9 @@ const TdsPage = async ({ params }: { params: PageParams }) => { namespace: "page-trillion-dollar-security", }) - const commitHistoryCache: CommitHistory = {} const { contributors } = await getAppPageContributorInfo( "trillion-dollar-security", - locale as Lang, - commitHistoryCache + locale as Lang ) return ( diff --git a/app/[locale]/wallets/find-wallet/page.tsx b/app/[locale]/wallets/find-wallet/page.tsx index acfbe4f1cfb..5cc16069edf 100644 --- a/app/[locale]/wallets/find-wallet/page.tsx +++ b/app/[locale]/wallets/find-wallet/page.tsx @@ -5,7 +5,7 @@ import { setRequestLocale, } from "next-intl/server" -import type { CommitHistory, Lang, PageParams } from "@/lib/types" +import type { Lang, PageParams } from "@/lib/types" import Breadcrumbs from "@/components/Breadcrumbs" import FindWalletProductTable from "@/components/FindWalletProductTable/lazy" @@ -52,11 +52,9 @@ const Page = async ({ params }: { params: PageParams }) => { ) const messages = pick(allMessages, requiredNamespaces) - const commitHistoryCache: CommitHistory = {} const { contributors } = await getAppPageContributorInfo( "wallets/find-wallet", - locale as Lang, - commitHistoryCache + locale as Lang ) return ( diff --git a/app/[locale]/wallets/page.tsx b/app/[locale]/wallets/page.tsx index d7f490d828b..121569ff372 100644 --- a/app/[locale]/wallets/page.tsx +++ b/app/[locale]/wallets/page.tsx @@ -6,7 +6,7 @@ import { setRequestLocale, } from "next-intl/server" -import type { CommitHistory, Lang, PageParams } from "@/lib/types" +import type { Lang, PageParams } from "@/lib/types" import Callout from "@/components/Callout" import Card from "@/components/Card" @@ -57,13 +57,8 @@ const Page = async ({ params }: { params: PageParams }) => { const requiredNamespaces = getRequiredNamespacesForPage("/wallets") const messages = pick(allMessages, requiredNamespaces) - const commitHistoryCache: CommitHistory = {} const { contributors, lastEditLocaleTimestamp } = - await getAppPageContributorInfo( - "wallets", - locale as Lang, - commitHistoryCache - ) + await getAppPageContributorInfo("wallets", locale as Lang) const heroContent = { title: t("page-wallets-title"), diff --git a/app/[locale]/what-is-ether/page.tsx b/app/[locale]/what-is-ether/page.tsx index 0604f6237c4..d3dd151eae5 100644 --- a/app/[locale]/what-is-ether/page.tsx +++ b/app/[locale]/what-is-ether/page.tsx @@ -1,7 +1,7 @@ import { Landmark, SquareCode, User } from "lucide-react" import { getTranslations } from "next-intl/server" -import type { CommitHistory, Lang, ToCItem } from "@/lib/types" +import type { Lang, ToCItem } from "@/lib/types" import FileContributors from "@/components/FileContributors" import ContentHero, { ContentHeroProps } from "@/components/Hero/ContentHero" @@ -40,13 +40,8 @@ const Page = async ({ params }: { params: { locale: Lang } }) => { namespace: "page-what-is-ether", }) - const commitHistoryCache: CommitHistory = {} const { contributors, lastEditLocaleTimestamp } = - await getAppPageContributorInfo( - "what-is-ether", - locale as Lang, - commitHistoryCache - ) + await getAppPageContributorInfo("what-is-ether", locale as Lang) const heroProps: ContentHeroProps = { breadcrumbs: { diff --git a/app/[locale]/what-is-ethereum/page.tsx b/app/[locale]/what-is-ethereum/page.tsx index e541351e4ca..b108645e257 100644 --- a/app/[locale]/what-is-ethereum/page.tsx +++ b/app/[locale]/what-is-ethereum/page.tsx @@ -8,7 +8,7 @@ import { } from "lucide-react" import { getTranslations } from "next-intl/server" -import type { CommitHistory, Lang, PageParams, ToCItem } from "@/lib/types" +import type { Lang, PageParams, ToCItem } from "@/lib/types" import DocLink from "@/components/DocLink" import FeedbackCard from "@/components/FeedbackCard" @@ -55,13 +55,8 @@ const Page = async ({ params }: { params: PageParams }) => { namespace: "page-what-is-ethereum", }) - const commitHistoryCache: CommitHistory = {} const { contributors, lastEditLocaleTimestamp } = - await getAppPageContributorInfo( - "what-is-ethereum", - locale as Lang, - commitHistoryCache - ) + await getAppPageContributorInfo("what-is-ethereum", locale as Lang) const tocItems: ToCItem[] = [ { title: t("page-what-is-ethereum-toc-ethereum"), url: "#ethereum" }, diff --git a/app/[locale]/what-is-the-ethereum-network/page.tsx b/app/[locale]/what-is-the-ethereum-network/page.tsx index 1460ff2721d..c19a2e7fb8a 100644 --- a/app/[locale]/what-is-the-ethereum-network/page.tsx +++ b/app/[locale]/what-is-the-ethereum-network/page.tsx @@ -1,6 +1,6 @@ import { getTranslations } from "next-intl/server" -import type { CommitHistory, Lang, ToCItem } from "@/lib/types" +import type { Lang, ToCItem } from "@/lib/types" import CommentCard from "@/components/CommentCard" import DocLink from "@/components/DocLink" @@ -34,12 +34,10 @@ const Page = async ({ params }: { params: Promise<{ locale: Lang }> }) => { namespace: "page-what-is-the-ethereum-network", }) - const commitHistoryCache: CommitHistory = {} const { contributors, lastEditLocaleTimestamp } = await getAppPageContributorInfo( "what-is-the-ethereum-network", - locale as Lang, - commitHistoryCache + locale as Lang ) const heroProps: ContentHeroProps = { diff --git a/docs/solutions/performance-issues/github-contributors-data-layer-migration.md b/docs/solutions/performance-issues/github-contributors-data-layer-migration.md new file mode 100644 index 00000000000..4c0fd10047f --- /dev/null +++ b/docs/solutions/performance-issues/github-contributors-data-layer-migration.md @@ -0,0 +1,175 @@ +--- +title: "Migrate GitHub Contributors from Build-time API Calls to Scheduled Data-Layer Tasks" +slug: "github-contributors-data-layer-migration" +category: "performance-issues" +severity: "high" +symptoms: + - "26,900 GitHub API calls per build (97% redundancy)" + - "Build failures due to rate limit exceeded (5,000 calls/hour)" + - "~30 minute build time on Netlify" + - "No cross-worker cache sharing in Next.js parallel builds" +components: + - "src/lib/utils/contributors.ts" + - "src/lib/utils/gh.ts" + - "src/data-layer/" + - "Trigger.dev scheduled tasks" + - "Netlify Blobs storage" +tags: + - "data-layer" + - "trigger.dev" + - "github-api" + - "caching" + - "rate-limiting" + - "next.js" +resolved_at: "2026-01-27" +--- + +# GitHub Contributors Data-Layer Migration + +## Problem + +The site made **26,900 GitHub API calls** during every production build to fetch contributor information, when only **~734 unique paths** needed to be queried. + +### Symptoms + +- Build failures when GitHub rate limits exceeded (5,000 req/hour) +- ~30 minute build times on Netlify +- Logs showing same paths fetched multiple times across workers + +### Root Causes + +1. **No cross-worker cache sharing**: Next.js uses 13 worker processes during build, each with isolated memory +2. **Fresh cache per page render**: Each app page created a new empty `commitHistoryCache` +3. **Duplicate legacy path calls**: API called twice for identical URLs (current + legacy path) +4. **Multiple historical paths per app page**: 6 paths checked per page (most don't exist) +5. **Multiple renders per locale**: 25 locales × 2 passes = 50 renders per page + +### Call Multiplication + +| Factor | Multiplier | +|--------|------------| +| App pages | 39 | +| × Historical paths per page | × 6 | +| × Locales | × 25 | +| × Renders per locale | × 2 | +| × Legacy duplicate call | × 2 | +| **Subtotal (app pages)** | **23,400** | +| + Markdown pages | + ~3,500 | +| **Total** | **~26,900** | + +## Solution + +Migrate to data-layer infrastructure: fetch contributor data via scheduled Trigger.dev task, store in Netlify Blobs, read during builds with zero API calls. + +### Architecture + +``` +Trigger.dev (weekly, Sundays midnight UTC) + ↓ +fetchGitHubContributors() - fetches from GitHub API (~734 calls) + ↓ +set(KEYS.GITHUB_CONTRIBUTORS, data) - stores in Netlify Blobs + ↓ +Build time: getGitHubContributors() + ↓ +unstable_cache + React cache (request dedup) + ↓ +storage.get() - retrieves from Netlify Blobs + ↓ +contributors.ts uses pre-computed data (zero API calls) +``` + +### Files Changed + +| File | Change | +|------|--------| +| `src/lib/types.ts` | Added `GitHubContributorsData` type | +| `src/data-layer/fetchers/fetchGitHubContributors.ts` | **New** - Scheduled fetcher | +| `src/data-layer/mocks/fetch-github-contributors.json` | **New** - Mock data for local dev | +| `src/data-layer/tasks.ts` | Added `WEEKLY` array, registered task | +| `src/data-layer/index.ts` | Added `getGitHubContributors()` getter | +| `src/lib/data/index.ts` | Added cached wrapper with daily TTL | +| `src/lib/utils/contributors.ts` | Now uses data-layer instead of direct API | + +### Key Implementation Details + +**Type Definition** (`src/lib/types.ts`): +```typescript +export type GitHubContributorsData = { + content: Record // slug → contributors + appPages: Record // pagePath → contributors + generatedAt: string +} +``` + +**Consumer Code** (`src/lib/utils/contributors.ts`): +```typescript +import { getGitHubContributors } from "@/lib/data" + +export const getMarkdownFileContributorInfo = async (slug, locale, fileLang, _) => { + const contributorsData = await getGitHubContributors() + const gitHubContributors = contributorsData?.content[slug] ?? [] + // ... rest unchanged +} + +export const getAppPageContributorInfo = async (pagePath, locale, _) => { + const contributorsData = await getGitHubContributors() + const gitHubContributors = contributorsData?.appPages[pagePath] ?? [] + // ... rest unchanged +} +``` + +**Task Registration** (`src/data-layer/tasks.ts`): +```typescript +const WEEKLY: Task[] = [ + [KEYS.GITHUB_CONTRIBUTORS, fetchGitHubContributors], +] + +export const weeklyTask = schedules.task({ + id: "weekly-data-fetch", + cron: "0 0 * * 0", // Sundays at midnight UTC + run: () => runTasks(WEEKLY), +}) +``` + +## Results + +| Metric | Before | After | Improvement | +|--------|--------|-------|-------------| +| API calls per build | 26,900 | 0 | 100% reduction | +| API calls per week | 26,900 × n builds | ~734 | 97% reduction | +| Rate limit risk | Critical | Safe | Eliminated | + +## Prevention + +### When to Use Data-Layer vs Direct API + +| Scenario | Approach | +|----------|----------| +| Data changes rarely (daily/weekly) | Data-layer with scheduled task | +| Data needed across many pages | Data-layer (shared storage) | +| Real-time data required | Direct API with proper caching | +| Single page needs data | Direct API acceptable | + +### Warning Signs + +- Same API endpoint called multiple times in build logs +- API calls proportional to `pages × locales × workers` +- Rate limit warnings during builds +- Build times increasing with page count + +### Testing + +```bash +# Local development - use mock data +USE_MOCK_DATA=true pnpm dev + +# Test the fetcher manually +pnpm trigger:dev +``` + +## Related Documentation + +- [GitHub API Calls Diagnostic](../../github-api-calls-diagnostic.md) - Original problem analysis +- [Data Layer Documentation](../../src/data-layer/docs.md) - Architecture guide +- [API Keys](../../api-keys.md) - Environment variable setup diff --git a/package.json b/package.json index 4e0ec380898..d3865615a5a 100644 --- a/package.json +++ b/package.json @@ -170,4 +170,4 @@ "sha.js": ">=2.4.12" } } -} +} \ No newline at end of file diff --git a/src/data-layer/fetchers/fetchGitHubContributors.ts b/src/data-layer/fetchers/fetchGitHubContributors.ts new file mode 100644 index 00000000000..98386935b04 --- /dev/null +++ b/src/data-layer/fetchers/fetchGitHubContributors.ts @@ -0,0 +1,352 @@ +import type { FileContributor, GitHubContributorsData } from "@/lib/types" + +import { CONTENT_DIR, OLD_CONTENT_DIR } from "@/lib/constants" + +const GITHUB_API_BASE = + "https://api.github.com/repos/ethereum/ethereum-org-website" + +// Optimized settings for parallel fetching +const BATCH_SIZE = 20 // Concurrent requests per batch +const BATCH_DELAY_MS = 50 // Small delay between batches to avoid rate limiting + +const APP_PAGES_PREFIX = "app/[locale]/" + +/** + * Generate all historical paths for an app page. + * Used to aggregate git history across directory structure migrations. + * + * For app router paths, also includes underscore-prefixed variants of each + * segment (e.g., roadmap/vision → roadmap/_vision) since Next.js private + * folders use the _ prefix but represent the same page. + */ +function getAllHistoricalPaths(pagePath: string): string[] { + const paths = [ + `src/pages/${pagePath}.tsx`, + `src/pages/${pagePath}/index.tsx`, + `src/pages/[locale]/${pagePath}.tsx`, + `src/pages/[locale]/${pagePath}/index.tsx`, + `${APP_PAGES_PREFIX}${pagePath}/page.tsx`, + `${APP_PAGES_PREFIX}${pagePath}/_components/${pagePath}.tsx`, + ] + + // Add underscore-prefixed variants for each segment + const segments = pagePath.split("/") + for (let i = 0; i < segments.length; i++) { + const variant = [...segments] + variant[i] = `_${variant[i]}` + paths.push(`${APP_PAGES_PREFIX}${variant.join("/")}/page.tsx`) + } + + return paths +} + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + +/** + * Process items in parallel batches. + * Executes `fn` for each item, with at most `batchSize` concurrent operations. + */ +async function parallelBatch( + items: T[], + fn: (item: T) => Promise, + batchSize: number = BATCH_SIZE +): Promise { + const results: R[] = [] + + for (let i = 0; i < items.length; i += batchSize) { + const batch = items.slice(i, i + batchSize) + const batchResults = await Promise.all(batch.map(fn)) + results.push(...batchResults) + + // Small delay between batches to avoid rate limiting + if (i + batchSize < items.length) { + await delay(BATCH_DELAY_MS) + } + } + + return results +} + +/** + * Fetch commits for a file path from GitHub API. + * Returns contributors in FileContributor format. + */ +async function fetchCommitsForPath( + filepath: string, + token: string +): Promise { + const url = new URL(`${GITHUB_API_BASE}/commits`) + url.searchParams.set("path", filepath) + url.searchParams.set("sha", "master") + + const response = await fetch(url.href, { + headers: { + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github.v3+json", + }, + }) + + // Handle rate limiting + if ( + response.status === 403 && + response.headers.get("X-RateLimit-Remaining") === "0" + ) { + const resetTime = response.headers.get("X-RateLimit-Reset") + if (resetTime) { + const waitTime = +resetTime - Math.floor(Date.now() / 1000) + console.log(`Rate limit exceeded, waiting ${waitTime}s...`) + await delay(waitTime * 1000) + return fetchCommitsForPath(filepath, token) // Retry + } + } + + if (!response.ok) { + // 404 is expected for paths that don't exist + if (response.status !== 404) { + console.warn( + `Failed to fetch commits for ${filepath}: ${response.status}` + ) + } + return [] + } + + const commits = await response.json() + if (!Array.isArray(commits)) { + return [] + } + + // Transform to FileContributor format and deduplicate + const contributors = commits + .filter((commit: { author?: unknown }) => !!commit.author) + .map( + (commit: { + author: { login: string; avatar_url: string; html_url: string } + commit: { author: { date: string } } + }) => ({ + login: commit.author.login, + avatar_url: commit.author.avatar_url, + html_url: commit.author.html_url, + date: commit.commit.author.date, + }) + ) + + // Remove duplicates by login (keep first = most recent) + const seen = new Set() + return contributors.filter((c: FileContributor) => { + if (seen.has(c.login)) return false + seen.add(c.login) + return true + }) +} + +/** + * Fetch contributors for multiple paths and merge/deduplicate results. + */ +async function fetchContributorsForPaths( + paths: string[], + token: string +): Promise { + const results = await parallelBatch(paths, (path) => + fetchCommitsForPath(path, token) + ) + + const allContributors = results.flat() + + // Deduplicate by login (keep first = most recent) + const seen = new Set() + return allContributors.filter((c) => { + if (seen.has(c.login)) return false + seen.add(c.login) + return true + }) +} + +interface GitTreeItem { + path: string + type: "blob" | "tree" + sha: string +} + +/** + * Fetch the full repo tree in ONE API call. + * Returns both content slugs and app page paths discovered from the tree. + */ +async function discoverPathsFromTree(token: string): Promise<{ + contentSlugs: string[] + appPagePaths: string[] +}> { + const url = `${GITHUB_API_BASE}/git/trees/master?recursive=1` + + const response = await fetch(url, { + headers: { + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github.v3+json", + }, + }) + + if (!response.ok) { + throw new Error(`Failed to fetch repo tree: ${response.status}`) + } + + const data = await response.json() + const tree: GitTreeItem[] = data.tree + + const contentSlugs: string[] = [] + const appPagePaths: string[] = [] + + const contentPrefix = `${CONTENT_DIR}/` + const translationsSegment = "/translations/" + + for (const item of tree) { + if (item.type !== "blob") continue + + // Content files: public/content/{slug}/index.md (excluding translations) + if ( + item.path.startsWith(contentPrefix) && + item.path.endsWith("/index.md") && + !item.path.includes(translationsSegment) + ) { + const relativePath = item.path.slice(contentPrefix.length) + const slug = relativePath.replace(/\/index\.md$/, "") + contentSlugs.push(slug) + } + + // App pages: app/[locale]/{pagePath}/page.tsx + // Strip the prefix and /page.tsx suffix to get the pagePath key. + // Next.js private folders (prefixed with _) are excluded from routing, + // so strip leading underscores from segments to match the route key + // that page components pass to getAppPageContributorInfo(). + if ( + item.path.startsWith(APP_PAGES_PREFIX) && + item.path.endsWith("/page.tsx") + ) { + const pagePath = item.path + .slice(APP_PAGES_PREFIX.length) + .replace(/\/page\.tsx$/, "") + + // Normalize private folder prefixes: _vision → vision + const normalized = pagePath + .split("/") + .map((seg) => (seg.startsWith("_") ? seg.slice(1) : seg)) + .join("/") + + appPagePaths.push(normalized) + } + } + + return { contentSlugs, appPagePaths } +} + +/** + * Fetch GitHub contributors data for all content files and app pages. + * This runs as a scheduled task (daily) and stores results in Netlify Blobs. + * + * Optimizations: + * - Uses git/trees API to list all content files in ONE request + * - Fetches commits in parallel batches (20 concurrent requests) + * - Minimal delays between batches (50ms) + */ +export async function fetchGitHubContributors(): Promise { + const token = process.env.GITHUB_TOKEN_READ_ONLY + + if (!token) { + throw new Error("GitHub token not set (GITHUB_TOKEN_READ_ONLY)") + } + + console.log("Starting GitHub contributors fetch...") + const startTime = Date.now() + + const result: GitHubContributorsData = { + content: {}, + appPages: {}, + generatedAt: new Date().toISOString(), + } + + // 1. Discover all paths from repo tree (single API call) + console.log("Discovering paths from git/trees API...") + const { contentSlugs, appPagePaths } = await discoverPathsFromTree(token) + console.log( + `Found ${contentSlugs.length} content slugs and ${appPagePaths.length} app pages in ${Date.now() - startTime}ms` + ) + + // Prepare all paths to fetch (current + legacy for each slug) + const contentPathPairs = contentSlugs.map((slug) => ({ + slug, + paths: [ + `${CONTENT_DIR}/${slug}/index.md`, + `${OLD_CONTENT_DIR}/${slug}/index.md`, + ], + })) + + console.log( + `Fetching contributors for ${contentSlugs.length} content files (parallel batches of ${BATCH_SIZE})...` + ) + const contentStartTime = Date.now() + + // Fetch all content file contributors in parallel batches + const contentResults = await parallelBatch( + contentPathPairs, + async ({ slug, paths }) => { + const contributors = await fetchContributorsForPaths(paths, token) + return { slug, contributors } + } + ) + + // Populate result + for (const { slug, contributors } of contentResults) { + if (contributors.length > 0) { + result.content[slug] = contributors + } + } + + console.log( + `Fetched contributors for ${Object.keys(result.content).length} content files in ${Date.now() - contentStartTime}ms` + ) + + // 2. Fetch app page contributors + console.log( + `Fetching contributors for ${appPagePaths.length} app pages (parallel batches of ${BATCH_SIZE})...` + ) + const appPagesStartTime = Date.now() + + // Prepare all paths for each app page + const appPagePathPairs = appPagePaths.map((pagePath) => ({ + pagePath, + paths: getAllHistoricalPaths(pagePath), + })) + + // Fetch all app page contributors in parallel batches + const appPageResults = await parallelBatch( + appPagePathPairs, + async ({ pagePath, paths }) => { + const contributors = await fetchContributorsForPaths(paths, token) + return { pagePath, contributors } + } + ) + + // Populate result + for (const { pagePath, contributors } of appPageResults) { + if (contributors.length > 0) { + result.appPages[pagePath] = contributors + } + } + + console.log( + `Fetched contributors for ${Object.keys(result.appPages).length} app pages in ${Date.now() - appPagesStartTime}ms` + ) + + const totalContributors = + Object.values(result.content).flat().length + + Object.values(result.appPages).flat().length + + const totalTime = Date.now() - startTime + console.log("GitHub contributors fetch complete", { + contentFiles: Object.keys(result.content).length, + appPages: Object.keys(result.appPages).length, + totalContributors, + totalTimeMs: totalTime, + generatedAt: result.generatedAt, + }) + + return result +} diff --git a/src/data-layer/index.ts b/src/data-layer/index.ts index 938ede215c9..29466a0dd19 100644 --- a/src/data-layer/index.ts +++ b/src/data-layer/index.ts @@ -5,6 +5,7 @@ import type { CommunityPick, EventItem, GHIssue, + GitHubContributorsData, GithubRepoData, GrowThePieData, GrowThePieMasterData, @@ -48,3 +49,4 @@ export const getEventsData = () => get(KEYS.EVENTS) export const getDeveloperToolsData = () => get(KEYS.DEVELOPER_TOOLS) export const getAccountHolders = () => get(KEYS.ACCOUNT_HOLDERS) export const getTranslationGlossary = () => get(KEYS.TRANSLATION_GLOSSARY) +export const getGitHubContributors = () => get(KEYS.GITHUB_CONTRIBUTORS) diff --git a/src/data-layer/mocks/fetch-github-contributors.json b/src/data-layer/mocks/fetch-github-contributors.json new file mode 100644 index 00000000000..a13db3adb31 --- /dev/null +++ b/src/data-layer/mocks/fetch-github-contributors.json @@ -0,0 +1,23 @@ +{ + "content": { + "about": [ + { + "login": "example-contributor", + "avatar_url": "https://avatars.githubusercontent.com/u/1?v=4", + "html_url": "https://github.com/example-contributor", + "date": "2025-01-01T00:00:00Z" + } + ] + }, + "appPages": { + "staking": [ + { + "login": "example-contributor", + "avatar_url": "https://avatars.githubusercontent.com/u/1?v=4", + "html_url": "https://github.com/example-contributor", + "date": "2025-01-01T00:00:00Z" + } + ] + }, + "generatedAt": "2025-01-01T00:00:00Z" +} diff --git a/src/data-layer/tasks.ts b/src/data-layer/tasks.ts index d18c4afefdf..fc73bc97a13 100644 --- a/src/data-layer/tasks.ts +++ b/src/data-layer/tasks.ts @@ -1,6 +1,7 @@ /** * Trigger.dev scheduled tasks for data fetching. * + * Weekly tasks run on Sundays at midnight UTC. * Daily tasks run at midnight UTC. * Hourly tasks run every hour. */ @@ -20,6 +21,7 @@ import { fetchEthPrice } from "./fetchers/fetchEthPrice" import { fetchEvents } from "./fetchers/fetchEvents" import { fetchGFIs } from "./fetchers/fetchGFIs" import { fetchGitHistory } from "./fetchers/fetchGitHistory" +import { fetchGitHubContributors } from "./fetchers/fetchGitHubContributors" import { fetchGithubRepoData } from "./fetchers/fetchGithubRepoData" import { fetchGrowThePie } from "./fetchers/fetchGrowThePie" import { fetchGrowThePieBlockspace } from "./fetchers/fetchGrowThePieBlockspace" @@ -36,6 +38,7 @@ import { set } from "./storage" export const KEYS = { APPS: "fetch-apps", CALENDAR_EVENTS: "fetch-calendar-events", + GITHUB_CONTRIBUTORS: "fetch-github-contributors", COMMUNITY_PICKS: "fetch-community-picks", DEVELOPER_TOOLS: "fetch-developer-tools", GFIS: "fetch-gfis", @@ -63,6 +66,8 @@ export const KEYS = { // Task definition: storage key + fetch function type TaskDef = [string, () => Promise] +const WEEKLY: TaskDef[] = [[KEYS.GITHUB_CONTRIBUTORS, fetchGitHubContributors]] + const DAILY: TaskDef[] = [ [KEYS.ACCOUNT_HOLDERS, fetchAccountHolders], [KEYS.APPS, fetchApps], @@ -113,13 +118,24 @@ function createDataTask([key, fetchFn]: TaskDef) { }) } +const weeklyFetchTasks = WEEKLY.map(createDataTask) const dailyFetchTasks = DAILY.map(createDataTask) const hourlyFetchTasks = HOURLY.map(createDataTask) // Must export for trigger.dev to discover -export const allFetchTasks = [...dailyFetchTasks, ...hourlyFetchTasks] +export const allFetchTasks = [ + ...weeklyFetchTasks, + ...dailyFetchTasks, + ...hourlyFetchTasks, +] // ─── Scheduled orchestrators ─── +export const weeklyTask = schedules.task({ + id: "weekly-data-fetch", + cron: "0 0 * * 0", // Sundays at midnight UTC + run: () => Promise.all(weeklyFetchTasks.map((t) => t.trigger())), +}) + export const dailyTask = schedules.task({ id: "daily-data-fetch", cron: "0 0 * * *", diff --git a/src/data/exchangesByCountry.ts b/src/data/exchangesByCountry.ts index 8dabfe59620..034688dd0ae 100644 --- a/src/data/exchangesByCountry.ts +++ b/src/data/exchangesByCountry.ts @@ -1,3 +1,6 @@ +// Last updated date for the exchanges data (update when modifying this file) +export const exchangesByCountryLastUpdated = "2025-10-21T13:11:58-07:00" + const exchangesByCountry = { // Afghanistan AF: [ diff --git a/src/lib/data/index.ts b/src/lib/data/index.ts index 6210b06e9b4..74c5fb66012 100644 --- a/src/lib/data/index.ts +++ b/src/lib/data/index.ts @@ -11,7 +11,7 @@ const CACHE_REVALIDATE_DAY = BASE_TIME_UNIT * 24 function createCachedGetter( fetcher: () => Promise, cacheKey: string[], - revalidate: number + revalidate: number | false ) { const persistentCache = unstable_cache(fetcher, cacheKey, { revalidate }) return cache(persistentCache) @@ -160,3 +160,21 @@ export const getTranslationGlossary = createCachedGetter( ["translation-glossary"], CACHE_REVALIDATE_DAY ) + +export const getGitHubContributors = createCachedGetter( + dataLayer.getGitHubContributors, + ["github-contributors"], + CACHE_REVALIDATE_DAY +) + +/** + * Static-cached version of getGitHubContributors — no revalidation. + * Use this in static pages (e.g., md content pages via [...slug]) to avoid + * opting them into ISR, which would cause 404s in the serverless environment + * where public/content is not available. + */ +export const getStaticGitHubContributors = createCachedGetter( + dataLayer.getGitHubContributors, + ["github-contributors-static"], + false +) diff --git a/src/lib/md/data.ts b/src/lib/md/data.ts index 74e4093ca58..b2ca9d77611 100644 --- a/src/lib/md/data.ts +++ b/src/lib/md/data.ts @@ -2,13 +2,7 @@ import { MDXRemoteProps } from "next-mdx-remote" import readingTime, { ReadTimeResults } from "reading-time" import type { Layout } from "@/lib/types" -import { - CommitHistory, - FileContributor, - Frontmatter, - Lang, - ToCItem, -} from "@/lib/types" +import { FileContributor, Frontmatter, Lang, ToCItem } from "@/lib/types" import { getMarkdownFileContributorInfo } from "@/lib/utils/contributors" import { getLocaleTimestamp } from "@/lib/utils/time" @@ -18,8 +12,6 @@ import { getLayoutFromSlug } from "../utils/layout" import { compile, extractLayoutFromMarkdown } from "./compile" import { importMd } from "./import" -const commitHistoryCache: CommitHistory = {} - interface GetPageDataParams { locale: string slug: string @@ -81,8 +73,7 @@ export async function getPageData({ await getMarkdownFileContributorInfo( slug, locale, - frontmatter.lang as string, - commitHistoryCache + frontmatter.lang as string ) // Format timestamp diff --git a/src/lib/types.ts b/src/lib/types.ts index bc35853c6f4..baba196d74e 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -427,8 +427,18 @@ export type FileContributor = { date: string } -type FilePath = string -export type CommitHistory = Record +/** + * GitHub contributors data stored in the data-layer. + * Keyed by file path, contains list of contributors for each file. + */ +export type GitHubContributorsData = { + /** Content files: slug (e.g., "eth", "wallets/find-wallet") → contributors */ + content: Record + /** App pages: pagePath (e.g., "staking", "developers") → contributors */ + appPages: Record + /** ISO timestamp when data was generated */ + generatedAt: string +} /** * Table of contents diff --git a/src/lib/utils/contributors.ts b/src/lib/utils/contributors.ts index bb8c35c3d3b..51768521e4a 100644 --- a/src/lib/utils/contributors.ts +++ b/src/lib/utils/contributors.ts @@ -1,37 +1,31 @@ import { join } from "path" -import type { CommitHistory, FileContributor, Lang } from "@/lib/types" +import type { FileContributor, Lang } from "@/lib/types" -import { CONTENT_DIR, CONTENT_PATH, DEFAULT_LOCALE } from "@/lib/constants" +import { CONTENT_PATH, DEFAULT_LOCALE } from "@/lib/constants" import { convertToFileContributorFromCrowdin, getCrowdinContributors, } from "./crowdin" -import { - fetchAndCacheGitHubContributors, - getAppPageLastCommitDate, - getMarkdownLastCommitDate, -} from "./gh" +import { getAppPageLastCommitDate } from "./gh" import { getLocaleTimestamp } from "./time" +import { getGitHubContributors, getStaticGitHubContributors } from "@/lib/data" + export const getMarkdownFileContributorInfo = async ( slug: string, locale: string, - fileLang: string, - cache: CommitHistory + fileLang: string ) => { const mdPath = join(CONTENT_PATH, slug) - const mdDir = join(CONTENT_DIR, slug) - const gitHubContributors = await fetchAndCacheGitHubContributors( - join("/", mdDir, "index.md"), - cache - ) + const contributorsData = await getStaticGitHubContributors() + const gitHubContributors = contributorsData?.content[slug] ?? [] - const latestCommitDate = getMarkdownLastCommitDate(slug, locale!) - const gitHubLastEdit = gitHubContributors[0]?.date - const lastUpdatedDate = gitHubLastEdit || latestCommitDate + // Use contributor date from data-layer, fallback to current date for new/missing content + const lastUpdatedDate = + gitHubContributors[0]?.date || new Date().toISOString() const crowdinContributors = convertToFileContributorFromCrowdin( getCrowdinContributors(mdPath, locale as Lang) @@ -47,44 +41,14 @@ export const getMarkdownFileContributorInfo = async ( return { contributors, lastUpdatedDate } } -/** - * Returns an array of possible historical file paths for a given page, - * accounting for different directory structures and migrations over time. - * - * @param pagePath - The relative path of the page (without extension). - * @returns An array of strings representing all historical file paths for the page. - * - * @remarks - * This function is used to track all possible locations a page may have existed in the repository, - * which is useful for aggregating git history and contributor information. - * - * @note - * If a page is migrated or its location changes, ensure the new path is added to this list. - * This maintains a complete historical record for accurate git history tracking. - */ -const getAllHistoricalPaths = (pagePath: string): string[] => [ - join("src/pages", `${pagePath}.tsx`), - join("src/pages", pagePath, "index.tsx"), - join("src/pages/[locale]", `${pagePath}.tsx`), - join("src/pages/[locale]", pagePath, "index.tsx"), - join("app/[locale]", pagePath, "page.tsx"), - join("app/[locale]", pagePath, "_components", `${pagePath}.tsx`), -] - export const getAppPageContributorInfo = async ( pagePath: string, - locale: Lang, - cache: CommitHistory + locale: Lang ) => { // TODO: Incorporate Crowdin contributor information - const gitHubContributors = await getAllHistoricalPaths(pagePath).reduce( - async (acc, path) => { - const contributors = await fetchAndCacheGitHubContributors(path, cache) - return [...(await acc), ...contributors] - }, - Promise.resolve([] as FileContributor[]) - ) + const contributorsData = await getGitHubContributors() + const gitHubContributors = contributorsData?.appPages[pagePath] ?? [] const uniqueGitHubContributors = gitHubContributors.filter( (contributor, index, self) => diff --git a/src/lib/utils/gh.ts b/src/lib/utils/gh.ts index 50a90d5d5b5..9588ec3e332 100644 --- a/src/lib/utils/gh.ts +++ b/src/lib/utils/gh.ts @@ -1,37 +1,4 @@ -import { execSync } from "child_process" -import fs from "fs" -import { join } from "path" - -import { - CONTENT_DIR, - DEFAULT_LOCALE, - GITHUB_COMMITS_URL, - OLD_CONTENT_DIR, - TRANSLATIONS_DIR, -} from "@/lib/constants" - -import type { Commit, CommitHistory, FileContributor } from "../types" - -const getGitLogFromPath = (path: string): string => { - // git command to show file last commit info - const gitCommand = `git log -1 -- ${path}` - // Execute git command and parse result to string - return execSync(gitCommand).toString() -} - -const extractDateFromGitLogInfo = (logInfo: string): string => { - // Filter commit date in log and return date using ISOString format (same that GH API uses) - try { - const lastCommitDate = logInfo - .split("\n") - .filter((x) => x.startsWith("Date: "))[0] - .slice("Date:".length) - .trim() - return new Date(lastCommitDate).toISOString() - } catch { - return new Date().toISOString() - } -} +import type { FileContributor } from "../types" export const getAppPageLastCommitDate = ( gitHubContributors: FileContributor[] @@ -43,32 +10,6 @@ export const getAppPageLastCommitDate = ( }, new Date(0)) .toString() -export const getLastGitCommitDateByPath = (path: string): string => { - if (!fs.existsSync(path)) throw new Error(`File not found: ${path}`) - const logInfo = getGitLogFromPath(path) - return extractDateFromGitLogInfo(logInfo) -} - -// This util filters the git log to get the file last commit info, and then the commit date (last update) -export const getMarkdownLastCommitDate = ( - slug: string, - locale: string -): string => { - const translatedContentPath = join(TRANSLATIONS_DIR, locale, slug, "index.md") - const contentIsNotTranslated = !fs.existsSync(translatedContentPath) - let filePath = "" - - if (locale === DEFAULT_LOCALE || contentIsNotTranslated) { - // Use git log data from english content - filePath = join(CONTENT_DIR, slug, "index.md") - } else { - // Use git log data from translated content - filePath = join(TRANSLATIONS_DIR, locale, slug, "index.md") - } - - return getLastGitCommitDateByPath(filePath) -} - const LABELS_TO_SEARCH = [ "content", "design", @@ -111,75 +52,3 @@ export const normalizeLabels = (labels: string[]) => { // remove duplicates return Array.from(new Set(labelsFound)) } - -async function fetchWithRateLimit(filepath: string): Promise { - const url = new URL(GITHUB_COMMITS_URL) - url.searchParams.set("path", filepath) - url.searchParams.set("sha", "master") - - const gitHubToken = process.env.GITHUB_TOKEN_READ_ONLY - - // If no token available, return empty array - if (!gitHubToken) return [] - - const response = await fetch(url.href, { - headers: { Authorization: `token ${gitHubToken}` }, - }) - - if ( - response.status === 403 && - response.headers.get("X-RateLimit-Remaining") === "0" - ) { - console.warn(`GitHub API rate limit exceeded for ${filepath}. Skipping.`) - return [] - } - - if (!response.ok) { - console.warn(`GitHub API error for ${filepath}: ${response.statusText}`) - return [] - } - - const json = await response.json() - if (!Array.isArray(json)) { - console.warn("Unexpected response from GitHub API", json) - return [] - } - return json -} - -// Fetch commit history and save it to a JSON file -export const fetchAndCacheGitHubContributors = async ( - filepath: string, - cache: CommitHistory -) => { - // First, check cache for existing commit history for English version (despite locale) - if (cache[filepath]) return cache[filepath] - - // Fetch and save commit history for file - const history = (await fetchWithRateLimit(filepath)) || [] - - const legacyHistory = - (await fetchWithRateLimit( - filepath.replace(CONTENT_DIR, OLD_CONTENT_DIR) - )) || [] - - // Transform commitHistory - const contributors = [...history, ...legacyHistory] - .filter(({ author }) => !!author) - .map((contribution) => { - const { login, avatar_url, html_url } = contribution.author - const { date } = contribution.commit.author - return { login, avatar_url, html_url, date } - }) - - // Remove duplicates from same login - const uniqueContributors = contributors.filter( - (contributor, index, self) => - index === self.findIndex((t) => t.login === contributor.login) - ) - - // Amend to cache - cache[filepath] = uniqueContributors - - return uniqueContributors -}