Skip to content

Commit 56cc1f7

Browse files
foolipfoolip
committed
Use @octokit/core + @octokit/plugin-throttling for GraphQL
Automatic throttling and retries should make this more reliable, and failing queries are no longer caught and logged and instead allowed to be fatal. The use of GraphQL variables for cursor makes some of the pagination code a bit more straightforward. All queries are serialized, but by requesting 100 labels the overall time is about the same as this avoid a lot of extra label queries. Also normalize the indentation of the GraphQL query blocks.
1 parent 2ec91d3 commit 56cc1f7

File tree

8 files changed

+433
-335
lines changed

8 files changed

+433
-335
lines changed

‎lib/github.js

Lines changed: 68 additions & 102 deletions
Original file line numberDiff line numberDiff line change
@@ -9,63 +9,25 @@ const graphql = require("./graphql.js");
99
const Octokat = require("octokat");
1010
const octo = new Octokat({token: config.ghToken});
1111

12-
async function fetchLabelPage(org, repo, acc = {edges: []}, cursor = null) {
13-
console.warn("Fetching labels for " + repo);
14-
let res;
15-
try {
16-
res = await graphql(`
17-
query {
18-
repository(owner:"${org}",name:"${repo}") {
19-
labels(first:10 ${cursor ? 'after:"' + cursor + '"' : ''}) {
20-
edges {
21-
node {
22-
name
23-
color
24-
}
25-
}
26-
pageInfo {
27-
endCursor
28-
hasNextPage
29-
}
30-
}
31-
}
32-
33-
}`);
34-
} catch (err) {
35-
// istanbul ignore next
36-
console.error("query failed " + JSON.stringify(err));
37-
}
38-
// istanbul ignore else
39-
if (res && res.repository) {
40-
const labels = {edges: acc.edges.concat(res.repository.labels.edges)};
41-
if (res.repository.labels.pageInfo.hasNextPage) {
42-
return fetchLabelPage(org, repo, labels, res.repository.labels.pageInfo.endCursor);
43-
} else {
44-
return {"repo": {"owner": org, "name": repo}, labels};
45-
}
46-
} else {
47-
console.error("Fetching label for " + repo + " at cursor " + cursor + " failed with " + JSON.stringify(res) + ", not retrying");
48-
return {"repo": {"owner": org, "name": repo}, "labels": acc};
49-
//return fetchLabelPage(org, repo, acc, cursor);
50-
}
51-
}
52-
53-
async function fetchRepoPage(org, acc = [], cursor = null) {
54-
let res;
55-
try {
56-
res = await graphql(`
57-
query {
58-
organization(login:"${org}") {
59-
repositories(first:10 ${cursor ? 'after:"' + cursor + '"' : ''}) {
60-
edges {
61-
node {
62-
id, name, owner { login } , isArchived, homepageUrl, description, isPrivate, createdAt
63-
labels(first:10) {
64-
edges {
65-
node {
66-
name
67-
color
68-
}
12+
const repoQuery = `
13+
query ($org: String!, $cursor: String) {
14+
organization(login: $org) {
15+
repositories(first: 10, after: $cursor) {
16+
nodes {
17+
id
18+
name
19+
owner {
20+
login
21+
}
22+
isArchived
23+
homepageUrl
24+
description
25+
isPrivate
26+
createdAt
27+
labels(first: 100) {
28+
nodes {
29+
name
30+
color
6931
}
7032
pageInfo {
7133
endCursor
@@ -103,66 +65,70 @@ async function fetchRepoPage(org, acc = [], cursor = null) {
10365
text
10466
}
10567
}
106-
codeOfConduct { body }
68+
codeOfConduct {
69+
body
70+
}
10771
readme: object(expression: "HEAD:README.md") {
10872
... on Blob {
10973
text
11074
}
11175
}
11276
}
113-
cursor
114-
}
115-
pageInfo {
116-
endCursor
117-
hasNextPage
77+
pageInfo {
78+
endCursor
79+
hasNextPage
80+
}
11881
}
11982
}
12083
}
121-
rateLimit {
122-
limit
123-
cost
124-
remaining
125-
resetAt
126-
}
127-
}`);
128-
} catch (err) {
129-
// istanbul ignore next
130-
console.error(err);
131-
}
132-
// Fetch labels if they are paginated
133-
// istanbul ignore else
134-
if (res && res.organization) {
135-
console.error("GitHub rate limit: " + JSON.stringify(res.rateLimit));
136-
return Promise.all(
137-
res.organization.repositories.edges
138-
.filter(e => e.node.labels.pageInfo.hasNextPage)
139-
.map(e => fetchLabelPage(e.node.owner.login, e.node.name, e.node.labels, e.node.labels.pageInfo.endCursor))
140-
).then((labelsPerRepos) => {
141-
const data = acc.concat(res.organization.repositories.edges.map(e => e.node));
142-
labelsPerRepos.forEach(({repo, labels}) => {
143-
data.find(r => r.owner.login == repo.owner && r.name == repo.name).labels = labels;
144-
});
145-
// Clean up labels data structure
146-
data.forEach(r => {
147-
if (r.labels && r.labels.edges) {
148-
r.labels = r.labels.edges.map(e => e.node);
84+
`;
85+
86+
const labelQuery = `
87+
query ($org: String!, $repo: String!, $cursor: String!) {
88+
repository(owner: $org, name: $repo) {
89+
labels(first: 100, after: $cursor) {
90+
nodes {
91+
name
92+
color
93+
}
94+
pageInfo {
95+
endCursor
96+
hasNextPage
14997
}
150-
});
151-
if (res.organization.repositories.pageInfo.hasNextPage) {
152-
return fetchRepoPage(org, data, res.organization.repositories.pageInfo.endCursor);
153-
} else {
154-
return data;
15598
}
156-
});
157-
} else {
158-
console.error("Fetching repo results at cursor " + cursor + " failed, retrying");
159-
return fetchRepoPage(org, acc, cursor);
99+
}
100+
}
101+
`;
102+
103+
async function *listRepos(org) {
104+
for (let cursor = null; ;) {
105+
const res = await graphql(repoQuery, {org, cursor});
106+
for (const repo of res.organization.repositories.nodes) {
107+
const labels = repo.labels.nodes;
108+
// Fetch more labels if they are paginated
109+
for (let pageInfo = repo.labels.pageInfo; pageInfo.hasNextPage;) {
110+
const res = await graphql(labelQuery, {
111+
org,
112+
repo: repo.name,
113+
cursor: pageInfo.endCursor
114+
});
115+
labels.push(...res.repository.labels.nodes);
116+
pageInfo = res.repository.labels.pageInfo;
117+
}
118+
repo.labels = labels;
119+
yield repo;
120+
}
121+
if (res.organization.repositories.pageInfo.hasNextPage) {
122+
cursor = res.organization.repositories.pageInfo.endCursor;
123+
} else {
124+
break;
125+
}
160126
}
161127
}
162128

163-
async function fetchRepoHooks(org, repo) {
129+
async function listRepoHooks(org, repo) {
164130
const hooks = await octo.repos(`${org}/${repo}`).hooks.fetch();
165131
return hooks.items;
166132
}
167133

168-
module.exports = {fetchRepoPage, fetchRepoHooks};
134+
module.exports = {listRepos, listRepoHooks};

‎lib/graphql.js

Lines changed: 28 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,36 @@
11
/* eslint-env node */
2+
/* istanbul ignore file */
23

34
"use strict";
45

56
const config = require("../config.json");
6-
const fetch = require("node-fetch");
7-
8-
const GH_API = "https://api.github.com/graphql";
9-
10-
// use https://developer.github.com/v4/explorer/ to debug queries
11-
12-
const GH_HEADERS = {
13-
"Accept": "application/vnd.github.v4.idl",
14-
"User-Agent": "graphql-github/0.1",
15-
"Content-Type": "application/json",
16-
"Authorization": "bearer " + config.ghToken
17-
};
18-
19-
async function graphql(query) {
20-
const options = {
21-
method: 'POST',
22-
headers: GH_HEADERS,
23-
body: JSON.stringify({query})
24-
};
25-
26-
const obj = await fetch(GH_API, options).then(res => res.json());
27-
28-
if (obj.errors) {
29-
const ghErr = obj.errors[0]; // just return the first error
30-
const err = new Error(ghErr.message, "unknown", -1);
31-
if (ghErr.type) {
32-
err.type = ghErr.type;
7+
const Octokit = require("@octokit/core").Octokit
8+
.plugin(require("@octokit/plugin-throttling"));
9+
10+
const MAX_RETRIES = 3;
11+
12+
const octokit = new Octokit({
13+
auth: config.ghToken,
14+
throttle: {
15+
onRateLimit: (retryAfter, options) => {
16+
if (options.request.retryCount < MAX_RETRIES) {
17+
console.warn(`Rate limit exceeded, retrying after ${retryAfter} seconds`)
18+
return true;
19+
} else {
20+
console.error(`Rate limit exceeded, giving up after ${MAX_RETRIES} retries`);
21+
return false;
22+
}
23+
},
24+
onAbuseLimit: (retryAfter, options) => {
25+
if (options.request.retryCount < MAX_RETRIES) {
26+
console.warn(`Abuse detected triggered, retrying after ${retryAfter} seconds`)
27+
return true;
28+
} else {
29+
console.error(`Abuse detected triggered, giving up after ${MAX_RETRIES} retries`);
30+
return false;
31+
}
3332
}
34-
err.all = obj.errors;
35-
throw err;
3633
}
37-
return obj.data;
38-
}
34+
});
3935

40-
module.exports = graphql;
36+
module.exports = octokit.graphql;

‎lib/w3cLicenses.js

Lines changed: 18 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -8,36 +8,30 @@ const graphql = require("./graphql.js");
88
// Set up the config of this repository
99
async function licenses() {
1010
let res = await graphql(`
11-
query {
12-
repository(owner:"w3c",name:"licenses") {
13-
contributing: object(expression: "HEAD:WG-CONTRIBUTING.md") {
14-
... on Blob {
15-
text
11+
query {
12+
repository(owner: "w3c", name: "licenses") {
13+
contributing: object(expression: "HEAD:WG-CONTRIBUTING.md") {
14+
... on Blob {
15+
text
16+
}
1617
}
17-
}
18-
contributingSw: object(expression: "HEAD:WG-CONTRIBUTING-SW.md") {
19-
... on Blob {
20-
text
18+
contributingSw: object(expression: "HEAD:WG-CONTRIBUTING-SW.md") {
19+
... on Blob {
20+
text
21+
}
2122
}
22-
}
23-
license: object(expression: "HEAD:WG-LICENSE.md") {
24-
... on Blob {
25-
text
23+
license: object(expression: "HEAD:WG-LICENSE.md") {
24+
... on Blob {
25+
text
26+
}
2627
}
27-
}
28-
licenseSw: object(expression: "HEAD:WG-LICENSE-SW.md") {
29-
... on Blob {
30-
text
28+
licenseSw: object(expression: "HEAD:WG-LICENSE-SW.md") {
29+
... on Blob {
30+
text
31+
}
3132
}
3233
}
3334
}
34-
rateLimit {
35-
limit
36-
cost
37-
remaining
38-
resetAt
39-
}
40-
}
4135
`);
4236
res = res.repository;
4337

0 commit comments

Comments
 (0)