Skip to content

Commit 82821a0

Browse files
maltejurmaltejur
committed
Add octokit throttling
1 parent 12e631a commit 82821a0

File tree

3 files changed

+94
-103
lines changed

3 files changed

+94
-103
lines changed

‎package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@
2323
"@material-ui/lab": "^4.0.0-alpha.60",
2424
"@material-ui/styles": "^4.11.5",
2525
"@monaco-editor/react": "^4.4.5",
26-
"@octokit/core": "^3.6.0",
26+
"@octokit/core": "^5.0.2",
27+
"@octokit/plugin-throttling": "^8.1.3",
2728
"ansi_up": "^5.1.0",
2829
"atob": "^2.1.2",
2930
"aws-sdk": "^2.1223.0",

‎scripts/fetch-algorithms.ts

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { exec } from "child_process";
66
import fs from "fs";
77
import path from "path";
88
import { Octokit } from "@octokit/core";
9+
import { throttling } from "@octokit/plugin-throttling";
910
import dotenv from "dotenv";
1011
import chalk from "chalk";
1112
import AWS from "aws-sdk";
@@ -25,9 +26,35 @@ import renderMarkdown from "../lib/markdown";
2526
import renderNotebook from "../lib/notebookjs";
2627

2728
dotenv.config();
28-
const octokit = new Octokit(
29-
process.env.GH_TOKEN ? { auth: process.env.GH_TOKEN } : {}
30-
);
29+
const octokit = new (Octokit.plugin(throttling))({
30+
...(process.env.GH_TOKEN ? { auth: process.env.GH_TOKEN } : {}),
31+
throttle: {
32+
onRateLimit: (retryAfter, options, _, retryCount) => {
33+
console.warn(
34+
`Request quota exhausted for request ${options.method} ${options.url}`
35+
);
36+
37+
if (retryCount < 2) {
38+
console.info(`Retrying after ${retryAfter} seconds!`);
39+
return true;
40+
}
41+
42+
return false;
43+
},
44+
onSecondaryRateLimit: (retryAfter, options, _, retryCount) => {
45+
console.warn(
46+
`SecondaryRateLimit detected for request ${options.method} ${options.url}`
47+
);
48+
49+
if (retryCount < 2) {
50+
console.info(`Retrying after ${retryAfter} seconds!`);
51+
return true;
52+
}
53+
54+
return false;
55+
},
56+
},
57+
});
3158

3259
let algorithms: { [key: string]: Algorithm } = {};
3360
let categories: { [category: string]: string[] } = {};
@@ -279,9 +306,8 @@ const categoriesToSkip = ["main", "src", "algorithms", "problems"];
279306
if (match) {
280307
const algorithm = algorithms[normalizeAlgorithm(match[1])];
281308
if (algorithm) {
282-
algorithm.explanationUrl[
283-
locale.code
284-
] = `https://github.com/TheAlgorithms/Algorithms-Explanation/tree/master/${dir}`;
309+
algorithm.explanationUrl[locale.code] =
310+
`https://github.com/TheAlgorithms/Algorithms-Explanation/tree/master/${dir}`;
285311
algorithm.body[locale.code] = await renderMarkdown(
286312
(await fs.promises.readFile(dir))
287313
.toString()

‎yarn.lock

Lines changed: 60 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -531,88 +531,79 @@
531531
"@nodelib/fs.scandir" "2.1.5"
532532
fastq "^1.6.0"
533533

534-
"@octokit/auth-token@^2.4.4":
535-
version "2.5.0"
536-
resolved "https://registry.yarnpkg.com/@octokit/auth-token/-/auth-token-2.5.0.tgz#27c37ea26c205f28443402477ffd261311f21e36"
537-
integrity sha512-r5FVUJCOLl19AxiuZD2VRZ/ORjp/4IN98Of6YJoJOkY75CIBuYfmiNHGrDwXr+aLGG55igl9QrxX3hbiXlLb+g==
538-
dependencies:
539-
"@octokit/types" "^6.0.3"
540-
541-
"@octokit/core@^3.6.0":
542-
version "3.6.0"
543-
resolved "https://registry.yarnpkg.com/@octokit/core/-/core-3.6.0.tgz#3376cb9f3008d9b3d110370d90e0a1fcd5fe6085"
544-
integrity sha512-7RKRKuA4xTjMhY+eG3jthb3hlZCsOwg3rztWh75Xc+ShDWOfDDATWbeZpAHBNRpm4Tv9WgBMOy1zEJYXG6NJ7Q==
545-
dependencies:
546-
"@octokit/auth-token" "^2.4.4"
547-
"@octokit/graphql" "^4.5.8"
548-
"@octokit/request" "^5.6.3"
549-
"@octokit/request-error" "^2.0.5"
550-
"@octokit/types" "^6.0.3"
534+
"@octokit/auth-token@^4.0.0":
535+
version "4.0.0"
536+
resolved "https://registry.yarnpkg.com/@octokit/auth-token/-/auth-token-4.0.0.tgz#40d203ea827b9f17f42a29c6afb93b7745ef80c7"
537+
integrity sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA==
538+
539+
"@octokit/core@^5.0.2":
540+
version "5.0.2"
541+
resolved "https://registry.yarnpkg.com/@octokit/core/-/core-5.0.2.tgz#ae7c5d61fdd98ba348a27c3cc510879a130b1234"
542+
integrity sha512-cZUy1gUvd4vttMic7C0lwPed8IYXWYp8kHIMatyhY8t8n3Cpw2ILczkV5pGMPqef7v0bLo0pOHrEHarsau2Ydg==
543+
dependencies:
544+
"@octokit/auth-token" "^4.0.0"
545+
"@octokit/graphql" "^7.0.0"
546+
"@octokit/request" "^8.0.2"
547+
"@octokit/request-error" "^5.0.0"
548+
"@octokit/types" "^12.0.0"
551549
before-after-hook "^2.2.0"
552550
universal-user-agent "^6.0.0"
553551

554-
"@octokit/endpoint@^6.0.1":
555-
version "6.0.12"
556-
resolved "https://registry.yarnpkg.com/@octokit/endpoint/-/endpoint-6.0.12.tgz#3b4d47a4b0e79b1027fb8d75d4221928b2d05658"
557-
integrity sha512-lF3puPwkQWGfkMClXb4k/eUT/nZKQfxinRWJrdZaJO85Dqwo/G0yOC434Jr2ojwafWJMYqFGFa5ms4jJUgujdA==
552+
"@octokit/endpoint@^9.0.0":
553+
version "9.0.4"
554+
resolved "https://registry.yarnpkg.com/@octokit/endpoint/-/endpoint-9.0.4.tgz#8afda5ad1ffc3073d08f2b450964c610b821d1ea"
555+
integrity sha512-DWPLtr1Kz3tv8L0UvXTDP1fNwM0S+z6EJpRcvH66orY6Eld4XBMCSYsaWp4xIm61jTWxK68BrR7ibO+vSDnZqw==
558556
dependencies:
559-
"@octokit/types" "^6.0.3"
560-
is-plain-object "^5.0.0"
557+
"@octokit/types" "^12.0.0"
561558
universal-user-agent "^6.0.0"
562559

563-
"@octokit/graphql@^4.5.8":
564-
version "4.8.0"
565-
resolved "https://registry.yarnpkg.com/@octokit/graphql/-/graphql-4.8.0.tgz#664d9b11c0e12112cbf78e10f49a05959aa22cc3"
566-
integrity sha512-0gv+qLSBLKF0z8TKaSKTsS39scVKF9dbMxJpj3U0vC7wjNWFuIpL/z76Qe2fiuCbDRcJSavkXsVtMS6/dtQQsg==
560+
"@octokit/graphql@^7.0.0":
561+
version "7.0.2"
562+
resolved "https://registry.yarnpkg.com/@octokit/graphql/-/graphql-7.0.2.tgz#3df14b9968192f9060d94ed9e3aa9780a76e7f99"
563+
integrity sha512-OJ2iGMtj5Tg3s6RaXH22cJcxXRi7Y3EBqbHTBRq+PQAqfaS8f/236fUrWhfSn8P4jovyzqucxme7/vWSSZBX2Q==
567564
dependencies:
568-
"@octokit/request" "^5.6.0"
569-
"@octokit/types" "^6.0.3"
565+
"@octokit/request" "^8.0.1"
566+
"@octokit/types" "^12.0.0"
570567
universal-user-agent "^6.0.0"
571568

572-
"@octokit/openapi-types@^11.2.0":
573-
version "11.2.0"
574-
resolved "https://registry.yarnpkg.com/@octokit/openapi-types/-/openapi-types-11.2.0.tgz#b38d7fc3736d52a1e96b230c1ccd4a58a2f400a6"
575-
integrity sha512-PBsVO+15KSlGmiI8QAzaqvsNlZlrDlyAJYcrXBCvVUxCp7VnXjkwPoFHgjEJXx3WF9BAwkA6nfCUA7i9sODzKA==
569+
"@octokit/openapi-types@^19.0.2":
570+
version "19.1.0"
571+
resolved "https://registry.yarnpkg.com/@octokit/openapi-types/-/openapi-types-19.1.0.tgz#75ec7e64743870fc73e1ab4bc6ec252ecdd624dc"
572+
integrity sha512-6G+ywGClliGQwRsjvqVYpklIfa7oRPA0vyhPQG/1Feh+B+wU0vGH1JiJ5T25d3g1JZYBHzR2qefLi9x8Gt+cpw==
576573

577-
"@octokit/request-error@^2.0.5", "@octokit/request-error@^2.1.0":
578-
version "2.1.0"
579-
resolved "https://registry.yarnpkg.com/@octokit/request-error/-/request-error-2.1.0.tgz#9e150357831bfc788d13a4fd4b1913d60c74d677"
580-
integrity sha512-1VIvgXxs9WHSjicsRwq8PlR2LR2x6DwsJAaFgzdi0JfJoGSO8mYI/cHJQ+9FbN21aa+DrgNLnwObmyeSC8Rmpg==
574+
"@octokit/plugin-throttling@^8.1.3":
575+
version "8.1.3"
576+
resolved "https://registry.yarnpkg.com/@octokit/plugin-throttling/-/plugin-throttling-8.1.3.tgz#7fb0e001c0cb9383c6be07740b8ec326ed990f6b"
577+
integrity sha512-pfyqaqpc0EXh5Cn4HX9lWYsZ4gGbjnSmUILeu4u2gnuM50K/wIk9s1Pxt3lVeVwekmITgN/nJdoh43Ka+vye8A==
581578
dependencies:
582-
"@octokit/types" "^6.0.3"
579+
"@octokit/types" "^12.2.0"
580+
bottleneck "^2.15.3"
581+
582+
"@octokit/request-error@^5.0.0":
583+
version "5.0.1"
584+
resolved "https://registry.yarnpkg.com/@octokit/request-error/-/request-error-5.0.1.tgz#277e3ce3b540b41525e07ba24c5ef5e868a72db9"
585+
integrity sha512-X7pnyTMV7MgtGmiXBwmO6M5kIPrntOXdyKZLigNfQWSEQzVxR4a4vo49vJjTWX70mPndj8KhfT4Dx+2Ng3vnBQ==
586+
dependencies:
587+
"@octokit/types" "^12.0.0"
583588
deprecation "^2.0.0"
584589
once "^1.4.0"
585590

586-
"@octokit/request@^5.6.0":
587-
version "5.6.2"
588-
resolved "https://registry.yarnpkg.com/@octokit/request/-/request-5.6.2.tgz#1aa74d5da7b9e04ac60ef232edd9a7438dcf32d8"
589-
integrity sha512-je66CvSEVf0jCpRISxkUcCa0UkxmFs6eGDRSbfJtAVwbLH5ceqF+YEyC8lj8ystKyZTy8adWr0qmkY52EfOeLA==
591+
"@octokit/request@^8.0.1", "@octokit/request@^8.0.2":
592+
version "8.1.6"
593+
resolved "https://registry.yarnpkg.com/@octokit/request/-/request-8.1.6.tgz#a76a859c30421737a3918b40973c2ff369009571"
594+
integrity sha512-YhPaGml3ncZC1NfXpP3WZ7iliL1ap6tLkAp6MvbK2fTTPytzVUyUesBBogcdMm86uRYO5rHaM1xIWxigWZ17MQ==
590595
dependencies:
591-
"@octokit/endpoint" "^6.0.1"
592-
"@octokit/request-error" "^2.1.0"
593-
"@octokit/types" "^6.16.1"
594-
is-plain-object "^5.0.0"
595-
node-fetch "^2.6.1"
596+
"@octokit/endpoint" "^9.0.0"
597+
"@octokit/request-error" "^5.0.0"
598+
"@octokit/types" "^12.0.0"
596599
universal-user-agent "^6.0.0"
597600

598-
"@octokit/request@^5.6.3":
599-
version "5.6.3"
600-
resolved "https://registry.yarnpkg.com/@octokit/request/-/request-5.6.3.tgz#19a022515a5bba965ac06c9d1334514eb50c48b0"
601-
integrity sha512-bFJl0I1KVc9jYTe9tdGGpAMPy32dLBXXo1dS/YwSCTL/2nd9XeHsY616RE3HPXDVk+a+dBuzyz5YdlXwcDTr2A==
601+
"@octokit/types@^12.0.0", "@octokit/types@^12.2.0":
602+
version "12.3.0"
603+
resolved "https://registry.yarnpkg.com/@octokit/types/-/types-12.3.0.tgz#e3f8bc53f65ef551e19cc1a0fea15adadec17d2d"
604+
integrity sha512-nJ8X2HRr234q3w/FcovDlA+ttUU4m1eJAourvfUUtwAWeqL8AsyRqfnLvVnYn3NFbUnsmzQCzLNdFerPwdmcDQ==
602605
dependencies:
603-
"@octokit/endpoint" "^6.0.1"
604-
"@octokit/request-error" "^2.1.0"
605-
"@octokit/types" "^6.16.1"
606-
is-plain-object "^5.0.0"
607-
node-fetch "^2.6.7"
608-
universal-user-agent "^6.0.0"
609-
610-
"@octokit/types@^6.0.3", "@octokit/types@^6.16.1":
611-
version "6.34.0"
612-
resolved "https://registry.yarnpkg.com/@octokit/types/-/types-6.34.0.tgz#c6021333334d1ecfb5d370a8798162ddf1ae8218"
613-
integrity sha512-s1zLBjWhdEI2zwaoSgyOFoKSl109CUcVBCc7biPJ3aAf6LGLU6szDvi31JPU7bxfla2lqfhjbbg/5DdFNxOwHw==
614-
dependencies:
615-
"@octokit/openapi-types" "^11.2.0"
606+
"@octokit/openapi-types" "^19.0.2"
616607

617608
"@rushstack/eslint-patch@^1.1.3":
618609
version "1.1.3"
@@ -1126,6 +1117,11 @@ bl@^4.1.0:
11261117
inherits "^2.0.4"
11271118
readable-stream "^3.4.0"
11281119

1120+
bottleneck@^2.15.3:
1121+
version "2.19.5"
1122+
resolved "https://registry.yarnpkg.com/bottleneck/-/bottleneck-2.19.5.tgz#5df0b90f59fd47656ebe63c78a98419205cadd91"
1123+
integrity sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==
1124+
11291125
brace-expansion@^1.1.7:
11301126
version "1.1.11"
11311127
resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd"
@@ -3349,20 +3345,6 @@ node-fetch@*:
33493345
data-uri-to-buffer "^3.0.1"
33503346
fetch-blob "^3.1.2"
33513347

3352-
node-fetch@^2.6.1:
3353-
version "2.6.5"
3354-
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.5.tgz#42735537d7f080a7e5f78b6c549b7146be1742fd"
3355-
integrity sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ==
3356-
dependencies:
3357-
whatwg-url "^5.0.0"
3358-
3359-
node-fetch@^2.6.7:
3360-
version "2.6.7"
3361-
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
3362-
integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
3363-
dependencies:
3364-
whatwg-url "^5.0.0"
3365-
33663348
node-fetch@^3.2.4:
33673349
version "3.2.4"
33683350
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-3.2.4.tgz#3fbca2d8838111048232de54cb532bd3cf134947"
@@ -4302,11 +4284,6 @@ tr46@^2.1.0:
43024284
dependencies:
43034285
punycode "^2.1.1"
43044286

4305-
tr46@~0.0.3:
4306-
version "0.0.3"
4307-
resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
4308-
integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=
4309-
43104287
trough@^1.0.0:
43114288
version "1.0.5"
43124289
resolved "https://registry.yarnpkg.com/trough/-/trough-1.0.5.tgz#b8b639cefad7d0bb2abd37d433ff8293efa5f406"
@@ -4600,11 +4577,6 @@ web-streams-polyfill@^3.0.3:
46004577
resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.1.1.tgz#1516f2d4ea8f1bdbfed15eb65cb2df87098c8364"
46014578
integrity sha512-Czi3fG883e96T4DLEPRvufrF2ydhOOW1+1a6c3gNjH2aIh50DNFBdfwh2AKoOf1rXvpvavAoA11Qdq9+BKjE0Q==
46024579

4603-
webidl-conversions@^3.0.0:
4604-
version "3.0.1"
4605-
resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871"
4606-
integrity sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=
4607-
46084580
webidl-conversions@^5.0.0:
46094581
version "5.0.0"
46104582
resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-5.0.0.tgz#ae59c8a00b121543a2acc65c0434f57b0fc11aff"
@@ -4627,14 +4599,6 @@ whatwg-mimetype@^2.3.0:
46274599
resolved "https://registry.yarnpkg.com/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz#3d4b1e0312d2079879f826aff18dbeeca5960fbf"
46284600
integrity sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g==
46294601

4630-
whatwg-url@^5.0.0:
4631-
version "5.0.0"
4632-
resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d"
4633-
integrity sha1-lmRU6HZUYuN2RNNib2dCzotwll0=
4634-
dependencies:
4635-
tr46 "~0.0.3"
4636-
webidl-conversions "^3.0.0"
4637-
46384602
whatwg-url@^8.0.0, whatwg-url@^8.5.0:
46394603
version "8.7.0"
46404604
resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-8.7.0.tgz#656a78e510ff8f3937bc0bcbe9f5c0ac35941b77"

0 commit comments

Comments
 (0)