feat: rate limit requests using aperture

Author: gitcommitshow

contributors.js

@@ -10,7 +10,7 @@ exports.getRepoContributors = getRepoContributors;
 
 const path = require('path');
 
-const { makeRequest } = require('./network.js');
+const { makeRequestWithRateLimit } = require('./network.js');
 
 // Configurations (Optional)
 // Repo owner that you want to analyze
@@ -43,7 +43,7 @@ async function getAllRepos(owner=REPO_OWNER, options) {
         GITHUB_REQUEST_OPTIONS.headers["Authorization"] = "token "+options.GITHUB_PERSONAL_TOKEN;
     }
     let url = `https://api.github.com/orgs/${owner}/repos?per_page=100&page=${pageNo}`;
-    const { res, data } = await makeRequest('GET', url, Object.assign({},GITHUB_REQUEST_OPTIONS));
+    const { res, data } = await makeRequestWithRateLimit('GET', url, Object.assign({},GITHUB_REQUEST_OPTIONS));
     console.log("Repo list request finished");
     console.log('HTTP status: ', res.statusCode);
     // console.log(data)
@@ -71,7 +71,7 @@ async function getAllRepos(owner=REPO_OWNER, options) {
 async function getRepoContributors(fullRepoName, pageNo = 1) {
     let url = `https://api.github.com/repos/${fullRepoName}/contributors?per_page=100&page=${pageNo}`;
     console.log(url);
-    const { res, data } = await makeRequest('GET', url, Object.assign({},GITHUB_REQUEST_OPTIONS));
+    const { res, data } = await makeRequestWithRateLimit('GET', url, Object.assign({},GITHUB_REQUEST_OPTIONS));
     console.log("Contributors request finished for " + fullRepoName)
     // console.log(data)
     let dataJsonArray = JSON.parse(data);

network.js

@@ -1,13 +1,29 @@
 const https = require('https');
+const { ApertureClient } = require("@fluxninja/aperture-js");
 
 exports.makeRequest = makeRequest;
+exports.makeRequestWithRateLimit = makeRequestWithRateLimit;
+
+var apertureClient;
+
+function getApertureClient(){
+    if(!apertureClient) {
+        apertureClient = new ApertureClient({
+            address: process.env.APERTURE_SERVICE_ADDRESS,
+            apiKey: process.env.APERTURE_API_KEY,
+        });
+    }
+    return apertureClient;
+}
 
 /**
  * Sends an HTTPS request based on the specified method and options
  * This function returns a Promise that resolves with the response and data received from the server
  * @param {string} method - The HTTP method to use (e.g., 'GET', 'POST').
  * @param {string} url - The URL to which the request is sent.
- * @param {Object} [options] - The options for the request. This includes headers, request body (for POST/PUT), etc.
+ * @param {Object} [requestOptions] - The options for the request. This includes headers, request body (for POST/PUT), etc.
+ * @param {Object} [requestOptions.body] - The data that needs to be sent with the request, used for POST/PUT requests
+ * @param {Object} [requestOptions.headers] - The headers that needs to be sent with the request
  * @returns {Promise<{res: https.IncomingMessage, data: string}>} A Promise that resolves with the response object and the body data as a string.
  * @throws {Error} Throws an error if the request cannot be completed
  *
@@ -23,13 +39,13 @@ exports.makeRequest = makeRequest;
  *   }
  * }
  * */
-async function makeRequest(method, url, options) {
+async function makeRequest(method, url, requestOptions) {
     return new Promise((resolve, reject) => {
         // Ensure options is an object and set the method
-        options = typeof options === 'object' ? options : {};
-        options.method = method;
+        requestOptions = typeof requestOptions === 'object' ? requestOptions : {};
+        requestOptions.method = method;
 
-        const req = https.request(url, options, res => {
+        const req = https.request(url, requestOptions, res => {
             // Handle HTTP response stream
             let data = '';
             res.on('data', chunk => data += chunk);
@@ -42,10 +58,63 @@ async function makeRequest(method, url, options) {
         });
 
         // Handle POST/PUT data if provided
-        if (options.data) {
-            req.write(options.data);
+        if (requestOptions.data) {
+            req.write(requestOptions.data);
         }
 
         req.end();
     });
-}
+}
+
+
+/**
+ * Sends an HTTPS request with rate limiting. The function checks if the request is allowed by the rate limiter,
+ * and if so, sends an HTTPS request based on the specified method and options. This function returns a Promise
+ * that resolves with the response and data received from the server or rejects if an error occurs or the rate limit is exceeded.
+ *
+ * @param {string} method - The HTTP method to use (e.g., 'GET', 'POST').
+ * @param {string} url - The URL to which the request is sent.
+ * @param {Object} [options] - The options for the request. This includes headers, request body (for POST/PUT), etc.
+ * @param {Object} [options.rateLimitOptions] - Options related to rate limits
+ * @param {Object} [options.body] - The data that needs to be sent with the request, used for POST/PUT requests
+ * @param {Object} [options.headers] - The headers that needs to be sent with the request
+ * @returns {Promise<{res: https.IncomingMessage, data: string}>} A Promise that resolves with the response object and the body data as a string.
+ * @throws {Error} Throws an error if the rate limit is exceeded or if an invalid argument is passed.
+ *
+ * @example
+ * // Example usage for a GET request within an async function
+ * async function getExample() {
+ *   try {
+ *     const { res, data } = await makeRequest('GET', 'https://example.com');
+ *     console.log('Status Code:', res.statusCode);
+ *     console.log('Body:', data);
+ *   } catch (error) {
+ *     console.error('Error:', error.message);
+ *   }
+ * }
+ * */
+async function makeRequestWithRateLimit(method, url, requestOptions){
+    const flow = await apertureClient.startFlow("external-api-request", {
+      labels: {
+        url: url,
+      },
+      grpcCallOptions: {
+        deadline: Date.now() + 300, // ms
+      },
+    });
+  
+    if (flow.shouldRun()) {
+      // Add business logic to process incoming request
+      console.log("Request accepted. Processing...");
+      const {res, data} = await makeRequest(...arguments)
+      return { res, data}
+    } else {
+      console.log("Request rate-limited. Try again later.");
+      // Handle flow rejection
+      flow.setStatus(FlowStatus.Error);
+    }
+  
+    if (flow) {
+      flow.end();
+    }
+  }