feat: github collectStarsWorker and tests

Author: abose

src/constants.js

@@ -6,6 +6,10 @@ export const secretAccessKey = getConfigs().secretAccessKey;
 export const githubAPIToken = getConfigs().githubAPIToken;
 export const cocoEndPoint = getConfigs().cocoEndPoint;
 export const cocoAuthKey = getConfigs().cocoAuthKey;
+// GitHub API rate limit is 5000 for authenticated users. Be conservative and set this to 2000.
+export const githubHourlyRateLimit = getConfigs().githubHourlyRateLimit;
+// the repository where public ops related issues can be created. Eg. "phcode-dev/extensionService"
+export const opsRepo = getConfigs().opsRepo;
 export const DATABASE_NAME = `phcode_extensions_${stage}`;
 export const EXTENSIONS_DETAILS_TABLE = `${DATABASE_NAME}.extensionDetails`;
 export const RELEASE_DETAILS_TABLE = `${DATABASE_NAME}.releaseDetails`;

src/github.js

@@ -1,12 +1,49 @@
 import { Octokit } from "@octokit/rest";
-import {githubAPIToken} from "./constants.js";
+import {githubAPIToken, githubHourlyRateLimit, opsRepo, stage} from "./constants.js";
 
 // github api docs: https://docs.github.com/en/rest/issues/issues?apiVersion=2022-11-28#create-an-issue
 export const _gitHub = {
     Octokit
 };
 
 let octokit;
+const ONE_HOUR = 1000*60*60,
+    ONE_DAY = ONE_HOUR * 24;
+let requestsInThisHour = 0,
+    githubOpsIssueForTheDay, issueUpdatedInThisHour = false;
+
+/* c8 ignore start */
+// not testing this as no time and is manually tested. If you are touching this code, manual test thoroughly
+function _resetTPH() {
+    requestsInThisHour = 0;
+    issueUpdatedInThisHour = false;
+}
+function _resetGithubOpsIssue() {
+    githubOpsIssueForTheDay = null;
+}
+export function setupGitHubOpsMonitoring() {
+    setInterval(_resetTPH, ONE_HOUR);
+    setInterval(_resetGithubOpsIssue, ONE_DAY);
+}
+/* c8 ignore stop */
+
+async function _newRequestMetric() {
+    requestsInThisHour++;
+    if(requestsInThisHour > githubHourlyRateLimit/2) {
+        let opsRepoSplit = opsRepo.split("/"); //Eg. "phcode-dev/extensionService"
+        if(issueUpdatedInThisHour){
+            return;
+        }
+        issueUpdatedInThisHour = true;
+        const message = `Github API requests for the hour is at ${requestsInThisHour}, Max allowed is ${githubHourlyRateLimit}`;
+        if(!githubOpsIssueForTheDay) {
+            githubOpsIssueForTheDay = await createIssue(opsRepoSplit[0], opsRepoSplit[1],
+                `[OPS-${stage}] Github Rate Limit above threshold.`, message);
+        } else {
+            await commentOnIssue(opsRepoSplit[0], opsRepoSplit[1], githubOpsIssueForTheDay.number, message);
+        }
+    }
+}
 
 export function initGitHubClient() {
     if(octokit){
@@ -37,6 +74,7 @@ export async function createIssue(owner, repo, title, body) {
     // https://docs.github.com/en/rest/issues/issues?apiVersion=2022-11-28#create-an-issue
     // {... "html_url": "https://github.com/octocat/Hello-World/issues/1347", ...}
     console.log("create Github issue: ", arguments);
+    _newRequestMetric();
     let response = await octokit.request(`POST /repos/${owner}/${repo}/issues`, {
         owner,
         repo,
@@ -62,6 +100,7 @@ export async function createIssue(owner, repo, title, body) {
 export async function commentOnIssue(owner, repo, issueNumber, commentString) {
     // https://docs.github.com/en/rest/issues/comments?apiVersion=2022-11-28
     console.log("Comment on Github issue: ", arguments);
+    _newRequestMetric();
     let response = await octokit.request(`POST /repos/${owner}/${repo}/issues/${issueNumber}/comments`, {
         owner,
         repo,
@@ -86,6 +125,7 @@ export async function getOrgDetails(org) {
     // https://docs.github.com/en/rest/orgs/orgs?apiVersion=2022-11-28#get-an-organization
     console.log("Get Org details: ", arguments);
     try{
+        _newRequestMetric();
         let response = await octokit.request(`GET /orgs/${org}`, {
             org
         });
@@ -117,10 +157,11 @@ export async function getOrgDetails(org) {
  * @param {string} repo
  * @return {Promise<{html_url:string, stargazers_count:number}> | null}
  */
-export async function getRepoDetails(owner, repo) {
+export async function getRepoDetails(owner, repo, log = true) {
     // https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#get-a-repository
-    console.log("Get Repo details: ", arguments);
+    log && console.log("Get Repo details: ", arguments);
     try{
+        _newRequestMetric();
         let response = await octokit.request(`GET /repos/${owner}/${repo}`, {
             owner, repo
         });
@@ -130,7 +171,7 @@ export async function getRepoDetails(owner, repo) {
             stargazers_count: response.data.stargazers_count
         };
 
-        console.log("GitHub repo details: ", repoDetails);
+        log && console.log("GitHub repo details: ", repoDetails);
         return repoDetails;
     } catch (e) {
         if(e.status === 404){
@@ -155,6 +196,7 @@ export async function getReleaseDetails(owner, repo, tag) {
     // https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#get-a-repository
     console.log("Get Release details: ", arguments);
     try{
+        _newRequestMetric();
         let response = await octokit.request(`GET /repos/${owner}/${repo}/releases/tags/${tag}`, {
             owner, repo, tag
         });

src/server.js

@@ -29,9 +29,10 @@ import {getHelloSchema, hello} from "./api/hello.js";
 import {getPublishGithubReleaseSchema, publishGithubRelease} from "./api/publishGithubRelease.js";
 import path from 'path';
 import { fileURLToPath } from 'url';
-import {initGitHubClient} from "./github.js";
+import {initGitHubClient, setupGitHubOpsMonitoring} from "./github.js";
 import {getGetGithubReleaseStatusSchema, getGithubReleaseStatus} from "./api/getGithubReleaseStatus.js";
 import {getCountDownloadSchema, countDownload} from "./api/countDownload.js";
+import {startCollectStarsWorker} from "./utils/sync.js";
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -124,6 +125,8 @@ export async function startServer() {
     console.log("awaiting connection to coco db");
     await db.init(cocoEndPoint, cocoAuthKey);
     initGitHubClient();
+    setupGitHubOpsMonitoring();
+    startCollectStarsWorker();
     console.log("connected to coco db");
     setupTasks();
     const configs = getConfigs();

src/utils/configs.js

@@ -3,7 +3,8 @@ import * as fs from "fs";
 let APP_CONFIG = null;
 
 function _checkRequiredConfigs(config) {
-    const requiredConfigVars = ["cocoEndPoint", "cocoAuthKey", "stage", "githubAPIToken", "baseURL"];
+    const requiredConfigVars = ["cocoEndPoint", "cocoAuthKey", "stage", "githubAPIToken", "baseURL",
+        "githubHourlyRateLimit", "opsRepo"];
     let missingEnvVars = [];
     for (let envName of requiredConfigVars){
         if(!config[envName]){

src/utils/sync.js

@@ -3,10 +3,12 @@ import {
     EXTENSIONS_DETAILS_TABLE,
     POPULARITY_FILE,
     REGISTRY_FILE,
-    REGISTRY_VERSION_FILE
+    REGISTRY_VERSION_FILE,
+    FIELD_EXTENSION_ID
 } from "../constants.js";
 import db from "../db.js";
 import {S3} from "../s3.js";
+import {getRepoDetails} from "../github.js";
 
 export async function syncRegistryDBToS3JSON() {
     console.log("syncing non synced extension data in db to s3 extension.json");
@@ -58,3 +60,92 @@ export async function syncRegistryDBToS3JSON() {
     }
     console.log("syncPending status updated in db for: ", await Promise.all(updatePromises));
 }
+
+
+const ONE_HOUR = 1000*60*60, HOURS_IN_DAY = 24, ONE_DAY = ONE_HOUR * HOURS_IN_DAY;
+let extensionsStarsCollectedToday = []; // will be reset every day, collect stars from GitHub once daily
+
+async function _updateStargazerCount(owner, repo, extensionId) {
+    let repoDetails = await getRepoDetails(owner, repo, false);
+    if(repoDetails) {
+        const queryObj = {};
+        queryObj[FIELD_EXTENSION_ID] = extensionId;
+        let registryPKGJSON = await db.getFromIndex(EXTENSIONS_DETAILS_TABLE, queryObj);
+        if(!registryPKGJSON.isSuccess){
+            console.error("Error getting extensionPKG details from db: " + extensionId);
+            // dont fail, continue with next repo
+            return;
+        }
+        if(registryPKGJSON.documents.length === 1){
+            const document = registryPKGJSON.documents[0];
+            const documentId = registryPKGJSON.documents[0].documentId;
+            document.gihubStars = repoDetails.stargazers_count;
+            let status = await db.update(EXTENSIONS_DETAILS_TABLE, documentId, document,
+                `$.metadata.version='${document.metadata.version}'`);
+            if(!status.isSuccess) {
+                console.error("Error updating stars for extension in db: " + extensionId);
+                // dont fail, continue with next repo
+                return;
+            }
+        }
+    }
+}
+
+/**
+ * Collects github star count every hour in batches considering GitHub throttles at 2000 GitHub Api requests per hour.
+ */
+export async function _collectStarsWorker() { // exported for tests only
+    console.log("Number of extensions whose stars collected today: ", extensionsStarsCollectedToday.length);
+    let registry = JSON.parse(await S3.getObject(EXTENSIONS_BUCKET, REGISTRY_FILE));
+    let extensionIDs = Object.keys(registry);
+    const numExtensionsToCollect = (extensionIDs.length/HOURS_IN_DAY) * 2; // so that the task completes in half day
+    let extensionsToCollect = []; // all extensions whose stars have not been collected today
+    for(let extensionID of extensionIDs) {
+        if(extensionsStarsCollectedToday.includes(extensionID)){
+            // already collected
+            continue;
+        }
+        if(!registry[extensionID].ownerRepo){
+            // no repo, so nothing to see here
+            extensionsStarsCollectedToday.push(extensionID);
+            continue;
+        }
+        extensionsToCollect.push(extensionID);
+    }
+    let collectedStarsForExtensions = [];
+    for(let i=0; i < numExtensionsToCollect && i < extensionsToCollect.length; i++){
+        let extensionID = extensionsToCollect[i];
+        let repoSplit = registry[extensionID].ownerRepo.split("/");//"https://github.com/Brackets-Themes/808"
+        const repo = repoSplit[repoSplit.length-1],
+            owner = repoSplit[repoSplit.length-2];
+        // this is purposefully serial
+        await _updateStargazerCount(owner, repo, extensionID);
+        extensionsStarsCollectedToday.push(extensionID);
+        collectedStarsForExtensions.push(extensionID);
+    }
+    console.log(`collecting stars for ${collectedStarsForExtensions.length} extensions of MAX allowed ${numExtensionsToCollect}`);
+    return {collectedStarsForExtensions, extensionsStarsCollectedToday};
+}
+
+/* c8 ignore start */
+// not testing this as no time and is manually tested. If you are touching this code, manual test thoroughly
+let worker;
+export function startCollectStarsWorker() {
+    if(worker){
+        return;
+    }
+    worker = setInterval(_collectStarsWorker, ONE_HOUR);
+    setInterval(()=>{
+        extensionsStarsCollectedToday = [];
+    }, ONE_DAY);
+}
+/* c8 ignore end */
+/**
+ * sets download count of extensions.
+ * publishes registry.json and popularity.json into s3
+ * does not increase registry_version.json
+ * @private
+ */
+function _syncPopularityHourly() {
+
+}

test/integration/hello.spec.js

@@ -47,11 +47,11 @@ describe('Integration Tests for hello api', function () {
         let output = await fetch("http://localhost:5000/www", { method: 'GET'});
         expect(output.status).eql(200);
         output = await output.text();
-        expect(output.includes("Hello HTML")).eql(true);
+        expect(output.includes("<!DOCTYPE html>")).eql(true);
         output = await fetch("http://localhost:5000/www/", { method: 'GET'});
         expect(output.status).eql(200);
         output = await output.text();
-        expect(output.includes("Hello HTML")).eql(true);
+        expect(output.includes("<!DOCTYPE html>")).eql(true);
     });
 
     it('should get 404 if static web page doesnt exist', async function () {

test/integration/setupTestConfig.js

@@ -12,7 +12,9 @@ const defaultTestConfig = {
     "accessKeyId": "update in testConfig.json file to run integ tests",
     "secretAccessKey": "update in testConfig.json file to run integ tests",
     "githubAPIToken": "update in testConfig.json file to run integ tests",
-    "baseURL": "update in testConfig.json file to run integ tests"
+    "baseURL": "update in testConfig.json file to run integ tests",
+    "githubHourlyRateLimit": "update in testConfig.json file to run integ tests",
+    "opsRepo": "update in testConfig.json file to run integ tests"
 };
 
 

test/unit/setupMocks.js

@@ -135,11 +135,11 @@ export function setS3Mock(bucket, key, contents) {
  * @param org
  * @param repo
  */
-export function getRepoDetails(org, repo) {
+export function getRepoDetails(org, repo, starGazers = 3) {
     mockedFunctions.githubRequestFnMock = githubRequestFnMock;
     getRepoDetailsResponses[`${org}/${repo}`] = {
         data: {
-            stargazers_count: 3,
+            stargazers_count: starGazers,
             html_url: `https://github.com/${org}/${repo}`
         }
     };

test/unit/utils/.app.json

@@ -7,6 +7,8 @@
   "stage": "test",
   "githubAPIToken": "githubToken",
   "baseURL": "http://localhost:5000",
+  "githubHourlyRateLimit": 2000,
+  "opsRepo": "phcode-dev/extensionService",
   "mysql": {
     "host": "localhost",
     "port": "3306",