Skip to content

Commit a67673f

Browse files
sparkisonsparkison
committed
fix: Make sure dynamic counts are cached to prevent getting rate limited
1 parent 39cf113 commit a67673f

File tree

7 files changed

+313
-76
lines changed

7 files changed

+313
-76
lines changed

‎.github/workflows/refresh-data.yml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
name: Refresh Static Data
2+
3+
on:
4+
schedule:
5+
# Every 6 hours
6+
- cron: '0 */6 * * *'
7+
# Allow manual trigger from the Actions tab
8+
workflow_dispatch:
9+
10+
permissions:
11+
contents: write
12+
13+
jobs:
14+
refresh:
15+
runs-on: ubuntu-latest
16+
steps:
17+
- name: Checkout
18+
uses: actions/checkout@v4
19+
20+
- name: Setup Node.js
21+
uses: actions/setup-node@v4
22+
with:
23+
node-version: 20
24+
25+
- name: Fetch downloads and contributors
26+
run: node ./scripts/fetch-static-data.js
27+
28+
- name: Commit updated data files
29+
run: |
30+
git config user.name "github-actions[bot]"
31+
git config user.email "github-actions[bot]@users.noreply.github.com"
32+
git add static/data/downloads.json static/data/contributors.json
33+
# Only commit if there are actual changes
34+
git diff --cached --quiet || git commit -m "chore: refresh static data"
35+
git push

‎package.json‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,9 @@
1515
"dev": "docusaurus start",
1616
"generate-screenshots": "node ./scripts/generate-screenshots.js",
1717
"watch-screenshots": "node ./scripts/generate-screenshots.js --watch",
18-
"prestart": "npm run generate-screenshots",
19-
"prebuild": "npm run generate-screenshots",
18+
"fetch-static-data": "node ./scripts/fetch-static-data.js",
19+
"prestart": "npm run fetch-static-data && npm run generate-screenshots",
20+
"prebuild": "npm run fetch-static-data && npm run generate-screenshots",
2021
"update-release": "node update-release.js"
2122
},
2223
"dependencies": {
@@ -56,4 +57,4 @@
5657
"node": ">=20.0"
5758
},
5859
"description": "Docusaurus example project"
59-
}
60+
}

‎scripts/fetch-static-data.js‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
#!/usr/bin/env node
2+
3+
/**
4+
* Fetch static data at build time to avoid runtime API rate limits.
5+
* Writes results to static/data/ so they are served as static assets.
6+
*
7+
* Data fetched:
8+
* - Docker Hub pulls (via shields.io) → static/data/downloads.json
9+
* - GitHub contributors across all repos → static/data/contributors.json
10+
*/
11+
12+
const https = require('https');
13+
const fs = require('fs');
14+
const path = require('path');
15+
16+
const DATA_DIR = path.join(__dirname, '..', 'static', 'data');
17+
18+
const REPOS = [
19+
'm3ue/m3u-editor',
20+
'm3ue/m3u-proxy',
21+
'm3ue/m3u-editor-docs-v2',
22+
];
23+
24+
/**
25+
* Perform a GET request and resolve with the parsed JSON body.
26+
*/
27+
function fetchJson(url, headers = {}) {
28+
return new Promise((resolve, reject) => {
29+
const options = { headers: { 'User-Agent': 'docusaurus-prebuild', ...headers } };
30+
https.get(url, options, (res) => {
31+
let raw = '';
32+
res.on('data', (chunk) => { raw += chunk; });
33+
res.on('end', () => {
34+
if (res.statusCode >= 200 && res.statusCode < 300) {
35+
try {
36+
resolve(JSON.parse(raw));
37+
} catch (e) {
38+
reject(new Error(`JSON parse error for ${url}: ${e.message}`));
39+
}
40+
} else {
41+
reject(new Error(`HTTP ${res.statusCode} for ${url}`));
42+
}
43+
});
44+
}).on('error', reject);
45+
});
46+
}
47+
48+
/**
49+
* Fetch Docker Hub pull count and write to static/data/downloads.json
50+
*/
51+
async function fetchDownloads() {
52+
const data = await fetchJson(
53+
'https://img.shields.io/docker/pulls/sparkison/m3u-editor.json'
54+
);
55+
const formatted = data.value || '0';
56+
const result = { formatted: `${formatted}+`, fetchedAt: new Date().toISOString() };
57+
fs.writeFileSync(
58+
path.join(DATA_DIR, 'downloads.json'),
59+
JSON.stringify(result, null, 2)
60+
);
61+
console.log(` ✓ Downloads cached: ${result.formatted}`);
62+
}
63+
64+
/**
65+
* Fetch contributors from all repos and write to static/data/contributors.json
66+
*/
67+
async function fetchContributors() {
68+
const allContributors = new Map();
69+
70+
for (const repo of REPOS) {
71+
console.log(` → Fetching contributors for ${repo}...`);
72+
const data = await fetchJson(
73+
`https://api.github.com/repos/${repo}/contributors?per_page=100`,
74+
{ Accept: 'application/vnd.github.v3+json' }
75+
);
76+
77+
data.forEach((contributor) => {
78+
if (contributor.login.endsWith('[bot]') || contributor.login === 'Copilot') {
79+
return;
80+
}
81+
if (allContributors.has(contributor.login)) {
82+
allContributors.get(contributor.login).contributions += contributor.contributions;
83+
} else {
84+
allContributors.set(contributor.login, {
85+
login: contributor.login,
86+
avatar_url: contributor.avatar_url,
87+
html_url: contributor.html_url,
88+
contributions: contributor.contributions,
89+
});
90+
}
91+
});
92+
}
93+
94+
const sorted = Array.from(allContributors.values())
95+
.sort((a, b) => b.contributions - a.contributions);
96+
97+
const result = { contributors: sorted, fetchedAt: new Date().toISOString() };
98+
fs.writeFileSync(
99+
path.join(DATA_DIR, 'contributors.json'),
100+
JSON.stringify(result, null, 2)
101+
);
102+
console.log(` ✓ Contributors cached: ${sorted.length} unique contributors`);
103+
}
104+
105+
(async () => {
106+
console.log('Fetching static data...');
107+
fs.mkdirSync(DATA_DIR, { recursive: true });
108+
109+
const results = await Promise.allSettled([fetchDownloads(), fetchContributors()]);
110+
111+
results.forEach((r) => {
112+
if (r.status === 'rejected') {
113+
console.warn(` ⚠ Warning: ${r.reason.message}`);
114+
}
115+
});
116+
117+
console.log('Static data fetch complete.');
118+
})();

‎src/components/Contributors/index.js‎

Lines changed: 14 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -1,78 +1,28 @@
11
import React, { useState, useEffect } from 'react';
22
import styles from './styles.module.css';
33

4-
const REPOS = [
5-
'm3ue/m3u-editor',
6-
'm3ue/m3u-proxy',
7-
'm3ue/m3u-editor-docs-v2'
8-
];
9-
104
export default function Contributors() {
115
const [contributors, setContributors] = useState([]);
126
const [loading, setLoading] = useState(true);
137
const [error, setError] = useState(null);
148

159
useEffect(() => {
16-
fetchContributors();
10+
fetch('/data/contributors.json')
11+
.then((r) => {
12+
if (!r.ok) throw new Error('Failed to fetch contributors data');
13+
return r.json();
14+
})
15+
.then((data) => {
16+
setContributors(data.contributors || []);
17+
setLoading(false);
18+
})
19+
.catch((err) => {
20+
console.error('Error loading contributors:', err);
21+
setError(err.message);
22+
setLoading(false);
23+
});
1724
}, []);
1825

19-
const fetchContributors = async () => {
20-
try {
21-
const allContributors = new Map();
22-
23-
// Fetch contributors from all repos
24-
for (const repo of REPOS) {
25-
const response = await fetch(
26-
`https://api.github.com/repos/${repo}/contributors?per_page=100`,
27-
{
28-
headers: {
29-
'Accept': 'application/vnd.github.v3+json'
30-
}
31-
}
32-
);
33-
34-
if (!response.ok) {
35-
throw new Error(`Failed to fetch contributors for ${repo}`);
36-
}
37-
38-
const data = await response.json();
39-
40-
// Merge contributors, filtering out bots
41-
data.forEach(contributor => {
42-
// Skip bots (any login ending with [bot])
43-
if (contributor.login.endsWith('[bot]') || contributor.login === 'Copilot') {
44-
return;
45-
}
46-
47-
if (allContributors.has(contributor.login)) {
48-
// Add contributions from this repo to existing contributor
49-
const existing = allContributors.get(contributor.login);
50-
existing.contributions += contributor.contributions;
51-
} else {
52-
// Add new contributor
53-
allContributors.set(contributor.login, {
54-
login: contributor.login,
55-
avatar_url: contributor.avatar_url,
56-
html_url: contributor.html_url,
57-
contributions: contributor.contributions
58-
});
59-
}
60-
});
61-
}
62-
63-
// Convert to array and sort by contributions
64-
const sortedContributors = Array.from(allContributors.values())
65-
.sort((a, b) => b.contributions - a.contributions);
66-
67-
setContributors(sortedContributors);
68-
setLoading(false);
69-
} catch (err) {
70-
console.error('Error fetching contributors:', err);
71-
setError(err.message);
72-
setLoading(false);
73-
}
74-
};
75-
7626
if (loading) {
7727
return (
7828
<section className={styles.contributorsSection}>

‎src/components/DownloadBadge/index.js‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,21 +5,17 @@ export default function DownloadBadge() {
55
const [downloadsText, setDownloadsText] = useState('Loading...');
66

77
useEffect(() => {
8-
// Fetch from shields.io JSON endpoint (no CORS issues)
9-
fetch('https://img.shields.io/docker/pulls/sparkison/m3u-editor.json')
8+
fetch('/data/downloads.json')
109
.then((r) => {
1110
if (!r.ok) throw new Error('Failed to fetch');
1211
return r.json();
1312
})
1413
.then((data) => {
15-
if (data && data.value) {
16-
// shields.io returns the value already formatted (e.g., "178k")
17-
// If you want to show the exact number with "+", you can parse it
18-
setDownloadsText(`${data.value}+`);
14+
if (data && data.formatted) {
15+
setDownloadsText(data.formatted);
1916
}
2017
})
2118
.catch(() => {
22-
// Fallback to hardcoded value (update periodically)
2319
setDownloadsText('100,000+');
2420
});
2521
}, []);

0 commit comments

Comments
 (0)