use env var GITHUB_TOKEN for api calls (if set)

Author: khassel

scripts/updateGitHubApiData.js

@@ -1,5 +1,6 @@
 import fs from "node:fs";
 import {getJson} from "./utils.js";
+import process from "node:process";
 
 let queryCount = 0;
 let maxQueryCount = 58;
@@ -12,8 +13,8 @@ function printProgress (count, total) {
 // Function to check whether new data should be retrieved.
 function shouldFetch (repository) {
   let retrieve = false;
-  if (repository.url.includes("github.com")) {
-    if (queryCount < maxQueryCount) {
+  if (repository.url.includes("github.com") && maxQueryCount > 0) {
+    if (queryCount < maxQueryCount || process.env.GITHUB_TOKEN) {
       retrieve = true;
     }
   }
@@ -75,6 +76,11 @@ async function updateData () {
 
     sortModuleListByLastUpdate(previousData, moduleList);
 
+    const headers = {};
+    if (process.env.GITHUB_TOKEN) {
+      headers.Authorization = `Bearer ${process.env.GITHUB_TOKEN}`;
+    }
+
     for (const module of moduleList) {
       const repositoryId = module.id;
       const repositoryApiUrl = `https://api.github.com/repos/${repositoryId}`;
@@ -85,12 +91,12 @@ async function updateData () {
 
       if (shouldFetchData) {
         printProgress(moduleCount, moduleListLength);
-        const response = await fetch(repositoryApiUrl);
+        const response = await fetch(repositoryApiUrl, {headers});
         const data = await response.json();
         queryCount += 1;
 
         const branchUrl = `https://api.github.com/repos/${repositoryId}/commits/${data.default_branch}`;
-        const branchResponse = await fetch(branchUrl);
+        const branchResponse = await fetch(branchUrl, {headers});
         const branchData = await branchResponse.json();
         queryCount += 1;
 
@@ -129,40 +135,7 @@ async function updateData () {
         useHistoricalData(previousData, repositoryId, module, results);
       }
 
-      // Quick-and-dirty way to include the number of stars for non-GitHub repositories.
-      if (!module.url.includes("github.com")) {
-        switch (module.name) {
-          case "MMM-bergfex":
-            module.stars = 1;
-            break;
-          case "MMM-Flights":
-            module.stars = 2;
-            break;
-          case "MMM-InstagramView":
-            module.stars = 1;
-            break;
-          case "mmm-ratp":
-            module.stars = 2;
-            break;
-          case "MMM-NCTtimes":
-            module.stars = 1;
-            break;
-          case "MMM-RecyclingCalendar":
-            module.stars = 1;
-            break;
-          case "MMM-RepoStats":
-            module.stars = 2;
-            break;
-          case "MMM-YouTubeWebView":
-            module.stars = 1;
-            break;
-          default:
-            module.stars = 1;
-            break;
-        }
-        // Since far fewer users have accounts with non-GitHub hosts, repos get a small star boost.
-        module.stars *= 3;
-      }
+      setNonGithubStars(module);
     }
 
     results.sort((a, b) => a.id.localeCompare(b.id));
@@ -176,12 +149,52 @@ async function updateData () {
 
     fs.writeFileSync(localFilePath, JSON.stringify(updateInfo, null, 2));
     fs.writeFileSync("docs/data/modules.stage.2.json", JSON.stringify(sortedModuleList, null, 2));
+    if (maxQueryCount < queryCount) {
+      maxQueryCount = 0;
+    }
     console.info("\nGitHub data update completed. queryCount:", queryCount, "maxQueryCount:", maxQueryCount, "results:", results.length, "modules:", moduleListLength);
   } catch (error) {
     console.error("Error fetching GitHub API data:", error);
   }
 }
 
+function setNonGithubStars (module) {
+  // Quick-and-dirty way to include the number of stars for non-GitHub repositories.
+  if (!module.url.includes("github.com")) {
+    switch (module.name) {
+      case "MMM-bergfex":
+        module.stars = 1;
+        break;
+      case "MMM-Flights":
+        module.stars = 2;
+        break;
+      case "MMM-InstagramView":
+        module.stars = 1;
+        break;
+      case "mmm-ratp":
+        module.stars = 2;
+        break;
+      case "MMM-NCTtimes":
+        module.stars = 1;
+        break;
+      case "MMM-RecyclingCalendar":
+        module.stars = 1;
+        break;
+      case "MMM-RepoStats":
+        module.stars = 2;
+        break;
+      case "MMM-YouTubeWebView":
+        module.stars = 1;
+        break;
+      default:
+        module.stars = 1;
+        break;
+    }
+    // Since far fewer users have accounts with non-GitHub hosts, repos get a small star boost.
+    module.stars *= 3;
+  }
+}
+
 function useHistoricalData (previousData, repositoryId, module, results) {
   // Add the existing data without updating it
   const existingRepository = previousData.repositories?.find((repo) => repo.id === repositoryId);