Skip to content

Commit 110911c

Browse files
dontcallmedomdontcallmedom
authored
Merge pull request #79 from w3c/graphql-revamp
Use @octokit/core + @octokit/plugin-throttling for GraphQL
2 parents f16db58 + 56cc1f7 commit 110911c

File tree

8 files changed

+433
-335
lines changed

8 files changed

+433
-335
lines changed

‎lib/github.js

Lines changed: 68 additions & 102 deletions
Original file line numberDiff line numberDiff line change
@@ -9,63 +9,25 @@ const graphql = require("./graphql.js");
99
const Octokat = require("octokat");
1010
const octo = new Octokat({token: config.ghToken});
1111

12-
async function fetchLabelPage(org, repo, acc = {edges: []}, cursor = null) {
13-
console.warn("Fetching labels for " + repo);
14-
let res;
15-
try {
16-
res = await graphql(`
17-
query {
18-
repository(owner:"${org}",name:"${repo}") {
19-
labels(first:10 ${cursor ? 'after:"' + cursor + '"' : ''}) {
20-
edges {
21-
node {
22-
name
23-
color
24-
}
25-
}
26-
pageInfo {
27-
endCursor
28-
hasNextPage
29-
}
30-
}
31-
}
32-
33-
}`);
34-
} catch (err) {
35-
// istanbul ignore next
36-
console.error("query failed " + JSON.stringify(err));
37-
}
38-
// istanbul ignore else
39-
if (res && res.repository) {
40-
const labels = {edges: acc.edges.concat(res.repository.labels.edges)};
41-
if (res.repository.labels.pageInfo.hasNextPage) {
42-
return fetchLabelPage(org, repo, labels, res.repository.labels.pageInfo.endCursor);
43-
} else {
44-
return {"repo": {"owner": org, "name": repo}, labels};
45-
}
46-
} else {
47-
console.error("Fetching label for " + repo + " at cursor " + cursor + " failed with " + JSON.stringify(res) + ", not retrying");
48-
return {"repo": {"owner": org, "name": repo}, "labels": acc};
49-
//return fetchLabelPage(org, repo, acc, cursor);
50-
}
51-
}
52-
53-
async function fetchRepoPage(org, acc = [], cursor = null) {
54-
let res;
55-
try {
56-
res = await graphql(`
57-
query {
58-
organization(login:"${org}") {
59-
repositories(first:10 ${cursor ? 'after:"' + cursor + '"' : ''}) {
60-
edges {
61-
node {
62-
id, name, owner { login } , isArchived, homepageUrl, description, isPrivate, createdAt
63-
labels(first:10) {
64-
edges {
65-
node {
66-
name
67-
color
68-
}
12+
const repoQuery = `
13+
query ($org: String!, $cursor: String) {
14+
organization(login: $org) {
15+
repositories(first: 10, after: $cursor) {
16+
nodes {
17+
id
18+
name
19+
owner {
20+
login
21+
}
22+
isArchived
23+
homepageUrl
24+
description
25+
isPrivate
26+
createdAt
27+
labels(first: 100) {
28+
nodes {
29+
name
30+
color
6931
}
7032
pageInfo {
7133
endCursor
@@ -103,66 +65,70 @@ async function fetchRepoPage(org, acc = [], cursor = null) {
10365
text
10466
}
10567
}
106-
codeOfConduct { body }
68+
codeOfConduct {
69+
body
70+
}
10771
readme: object(expression: "HEAD:README.md") {
10872
... on Blob {
10973
text
11074
}
11175
}
11276
}
113-
cursor
114-
}
115-
pageInfo {
116-
endCursor
117-
hasNextPage
77+
pageInfo {
78+
endCursor
79+
hasNextPage
80+
}
11881
}
11982
}
12083
}
121-
rateLimit {
122-
limit
123-
cost
124-
remaining
125-
resetAt
126-
}
127-
}`);
128-
} catch (err) {
129-
// istanbul ignore next
130-
console.error(err);
131-
}
132-
// Fetch labels if they are paginated
133-
// istanbul ignore else
134-
if (res && res.organization) {
135-
console.error("GitHub rate limit: " + JSON.stringify(res.rateLimit));
136-
return Promise.all(
137-
res.organization.repositories.edges
138-
.filter(e => e.node.labels.pageInfo.hasNextPage)
139-
.map(e => fetchLabelPage(e.node.owner.login, e.node.name, e.node.labels, e.node.labels.pageInfo.endCursor))
140-
).then((labelsPerRepos) => {
141-
const data = acc.concat(res.organization.repositories.edges.map(e => e.node));
142-
labelsPerRepos.forEach(({repo, labels}) => {
143-
data.find(r => r.owner.login == repo.owner && r.name == repo.name).labels = labels;
144-
});
145-
// Clean up labels data structure
146-
data.forEach(r => {
147-
if (r.labels && r.labels.edges) {
148-
r.labels = r.labels.edges.map(e => e.node);
84+
`;
85+
86+
const labelQuery = `
87+
query ($org: String!, $repo: String!, $cursor: String!) {
88+
repository(owner: $org, name: $repo) {
89+
labels(first: 100, after: $cursor) {
90+
nodes {
91+
name
92+
color
93+
}
94+
pageInfo {
95+
endCursor
96+
hasNextPage
14997
}
150-
});
151-
if (res.organization.repositories.pageInfo.hasNextPage) {
152-
return fetchRepoPage(org, data, res.organization.repositories.pageInfo.endCursor);
153-
} else {
154-
return data;
15598
}
156-
});
157-
} else {
158-
console.error("Fetching repo results at cursor " + cursor + " failed, retrying");
159-
return fetchRepoPage(org, acc, cursor);
99+
}
100+
}
101+
`;
102+
103+
async function *listRepos(org) {
104+
for (let cursor = null; ;) {
105+
const res = await graphql(repoQuery, {org, cursor});
106+
for (const repo of res.organization.repositories.nodes) {
107+
const labels = repo.labels.nodes;
108+
// Fetch more labels if they are paginated
109+
for (let pageInfo = repo.labels.pageInfo; pageInfo.hasNextPage;) {
110+
const res = await graphql(labelQuery, {
111+
org,
112+
repo: repo.name,
113+
cursor: pageInfo.endCursor
114+
});
115+
labels.push(...res.repository.labels.nodes);
116+
pageInfo = res.repository.labels.pageInfo;
117+
}
118+
repo.labels = labels;
119+
yield repo;
120+
}
121+
if (res.organization.repositories.pageInfo.hasNextPage) {
122+
cursor = res.organization.repositories.pageInfo.endCursor;
123+
} else {
124+
break;
125+
}
160126
}
161127
}
162128

163-
async function fetchRepoHooks(org, repo) {
129+
async function listRepoHooks(org, repo) {
164130
const hooks = await octo.repos(`${org}/${repo}`).hooks.fetch();
165131
return hooks.items;
166132
}
167133

168-
module.exports = {fetchRepoPage, fetchRepoHooks};
134+
module.exports = {listRepos, listRepoHooks};

‎lib/graphql.js

Lines changed: 28 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,36 @@
11
/* eslint-env node */
2+
/* istanbul ignore file */
23

34
"use strict";
45

56
const config = require("../config.json");
6-
const fetch = require("node-fetch");
7-
8-
const GH_API = "https://api.github.com/graphql";
9-
10-
// use https://developer.github.com/v4/explorer/ to debug queries
11-
12-
const GH_HEADERS = {
13-
"Accept": "application/vnd.github.v4.idl",
14-
"User-Agent": "graphql-github/0.1",
15-
"Content-Type": "application/json",
16-
"Authorization": "bearer " + config.ghToken
17-
};
18-
19-
async function graphql(query) {
20-
const options = {
21-
method: 'POST',
22-
headers: GH_HEADERS,
23-
body: JSON.stringify({query})
24-
};
25-
26-
const obj = await fetch(GH_API, options).then(res => res.json());
27-
28-
if (obj.errors) {
29-
const ghErr = obj.errors[0]; // just return the first error
30-
const err = new Error(ghErr.message, "unknown", -1);
31-
if (ghErr.type) {
32-
err.type = ghErr.type;
7+
const Octokit = require("@octokit/core").Octokit
8+
.plugin(require("@octokit/plugin-throttling"));
9+
10+
const MAX_RETRIES = 3;
11+
12+
const octokit = new Octokit({
13+
auth: config.ghToken,
14+
throttle: {
15+
onRateLimit: (retryAfter, options) => {
16+
if (options.request.retryCount < MAX_RETRIES) {
17+
console.warn(`Rate limit exceeded, retrying after ${retryAfter} seconds`)
18+
return true;
19+
} else {
20+
console.error(`Rate limit exceeded, giving up after ${MAX_RETRIES} retries`);
21+
return false;
22+
}
23+
},
24+
onAbuseLimit: (retryAfter, options) => {
25+
if (options.request.retryCount < MAX_RETRIES) {
26+
console.warn(`Abuse detected triggered, retrying after ${retryAfter} seconds`)
27+
return true;
28+
} else {
29+
console.error(`Abuse detected triggered, giving up after ${MAX_RETRIES} retries`);
30+
return false;
31+
}
3332
}
34-
err.all = obj.errors;
35-
throw err;
3633
}
37-
return obj.data;
38-
}
34+
});
3935

40-
module.exports = graphql;
36+
module.exports = octokit.graphql;

‎lib/w3cLicenses.js

Lines changed: 18 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -8,36 +8,30 @@ const graphql = require("./graphql.js");
88
// Set up the config of this repository
99
async function licenses() {
1010
let res = await graphql(`
11-
query {
12-
repository(owner:"w3c",name:"licenses") {
13-
contributing: object(expression: "HEAD:WG-CONTRIBUTING.md") {
14-
... on Blob {
15-
text
11+
query {
12+
repository(owner: "w3c", name: "licenses") {
13+
contributing: object(expression: "HEAD:WG-CONTRIBUTING.md") {
14+
... on Blob {
15+
text
16+
}
1617
}
17-
}
18-
contributingSw: object(expression: "HEAD:WG-CONTRIBUTING-SW.md") {
19-
... on Blob {
20-
text
18+
contributingSw: object(expression: "HEAD:WG-CONTRIBUTING-SW.md") {
19+
... on Blob {
20+
text
21+
}
2122
}
22-
}
23-
license: object(expression: "HEAD:WG-LICENSE.md") {
24-
... on Blob {
25-
text
23+
license: object(expression: "HEAD:WG-LICENSE.md") {
24+
... on Blob {
25+
text
26+
}
2627
}
27-
}
28-
licenseSw: object(expression: "HEAD:WG-LICENSE-SW.md") {
29-
... on Blob {
30-
text
28+
licenseSw: object(expression: "HEAD:WG-LICENSE-SW.md") {
29+
... on Blob {
30+
text
31+
}
3132
}
3233
}
3334
}
34-
rateLimit {
35-
limit
36-
cost
37-
remaining
38-
resetAt
39-
}
40-
}
4135
`);
4236
res = res.repository;
4337

0 commit comments

Comments
 (0)