Updated Survey and removed dependency on cloud

mageroni · mageroni · commit f374fabddc30 · 2023-11-18T01:03:28.000-06:00
diff --git a/app.yml b/app.yml
@@ -62,7 +62,7 @@ default_permissions:
 
   # Repository contents, commits, branches, downloads, releases, and merges.
   # https://developer.github.com/v3/apps/permissions/#permission-on-contents
-  # contents: read
+  contents: write
 
   # Deployments and deployment statuses.
   # https://developer.github.com/v3/apps/permissions/#permission-on-deployments
diff --git a/index.js b/index.js
@@ -5,8 +5,8 @@
 
 const dedent = require("dedent");
 const fs = require("fs");
-const parse = require("csv-parse/lib/sync");
-const sql = require("mssql");
+const csvParser = require("csv-parser");
+const stream = require('stream');
 require("dotenv").config();
 
 module.exports = (app) => {
@@ -16,9 +16,9 @@ module.exports = (app) => {
   app.on("pull_request.closed", async (context) => {
 
     let pr_number = context.payload.pull_request.number;
-    let detectedLanguage = "en";
     let pr_author = context.payload.pull_request.user.login;
-    let organization_name = context.payload.repository.owner.login
+    let organization_name = context.payload.repository.owner.login;
+    let detectedLanguage = "en";
 
     // read file that aligns with detected language
     const issue_body = fs.readFileSync(
@@ -64,7 +64,10 @@ module.exports = (app) => {
 
   async function GetSurveyData(context) {
     let issue_body = context.payload.issue.body;
-    let issue_id = context.payload.issue.id;
+    let pctSelected = false;
+    let pctValue = new Array();
+    let freqSelected = false;
+    let freqValue = new Array();
 
     // save comment body if present
     let comment = null;
@@ -88,17 +91,9 @@ module.exports = (app) => {
       );
     });
 
-    // if there's a comment, insert it into the DB regardless of whether the user answered the survey or not
-    if (comment) {
-      await insertIntoDB(context, issue_id, pr_number, isCopilotUsed, null, null, comment);
-    }
-
     if (isCopilotUsed) {
-      await insertIntoDB(context, issue_id, pr_number, isCopilotUsed, null, null, comment);
-
       // loop through checkboxes and find the one that contains %
-      let pctSelected = false;
-      let pctValue = new Array();
+      
       for (const checkbox of checkboxes) {
         if (checkbox.includes("%")) {
           pctSelected = true;
@@ -108,14 +103,8 @@ module.exports = (app) => {
           app.log.info(copilotPercentage);
         }
       }
-      if (pctSelected) {
-        //if percentage is selected, insert into DB
-        await insertIntoDB(context, issue_id, pr_number, isCopilotUsed, pctValue, null, comment);
-      }
 
       // loop through checkboxes and find the ones that do not contain % and are not Yes or No
-      let freqSelected = false;
-      let freqValue = new Array();
       for (const checkbox of checkboxes) {
         if (
           !checkbox.includes("%") &&
@@ -125,7 +114,8 @@ module.exports = (app) => {
           !checkbox.includes("Oui") &&
           !checkbox.includes("Não") &&
           !checkbox.includes("No") &&
-          !checkbox.includes("Non")
+          !checkbox.includes("Non") || 
+          checkbox.includes("Not very much")
         ) {
           freqSelected = true;
           frequencyValue = checkbox;
@@ -135,11 +125,6 @@ module.exports = (app) => {
         }
       }
 
-      if (freqSelected) {
-        //if frequency is selected, insert into DB
-        await insertIntoDB(context, issue_id, pr_number, isCopilotUsed, null, freqValue, comment);
-      }
-
       if( pctSelected && freqSelected ){
         // close the issue
         try {
@@ -163,7 +148,6 @@ module.exports = (app) => {
           );
         })
       ) {
-        await insertIntoDB(context, issue_id, pr_number, isCopilotUsed, null, null, comment);
 
         if (comment) {
           try {
@@ -181,91 +165,118 @@ module.exports = (app) => {
       }
     }
 
-    return {
+    let data = {
       enterprise_name: context.payload.enterprise ? context.payload.enterprise.name : '',
       organization_name: context.payload.organization ? context.payload.organization.login : '',
       repository_name: context.payload.repository ? context.payload.repository.name : '',
       issue_id: context.payload.issue ? context.payload.issue.id : '',
       issue_number: context.payload.issue ? context.payload.issue.number : '',
-      PR_number: context.payload.pull_request ? context.payload.pull_request.number : '',
+      PR_number: pr_number || '',
       assignee_name: context.payload.issue.assignee ? context.payload.issue.assignee.login : '',
       is_copilot_used: isCopilotUsed ? 1 : 0,
       saving_percentage: pctValue || '',
       usage_frequency: freqValue || '',
       comment: comment || '',
       created_at: context.payload.issue ? context.payload.issue.created_at : '',
       completed_at: context.payload.issue ? context.payload.issue.updated_at : ''
-    }
+    };
+
+    return data;
 
   }
 
   async function insertIntoFile(context) {
+      let fileContent = "";
+      let results = [];
+      let resultString = "";
+
       try {
+
+        fileContent = await GetSurveyData(context);
+
         // Try to get the file
         let file = await context.octokit.repos.getContent({
           owner: context.payload.repository.owner.login,
           repo: context.payload.repository.name,
           path: "results.csv",
         });
+
         // If the file exists, get its contents
         let fileContents = Buffer.from(file.data.content, "base64").toString();
         // If the file contents are not empty, parse the CSV
         if (fileContents.length > 0) {
-          // parse a csv into an array
-          let result = parse(fileContents, {
-            columns: true,
-            skip_empty_lines: true,
+          // create a readable stream
+          let readableStream = new stream.Readable();
+          readableStream.push(fileContents);
+          readableStream.push(null); 
+
+          await new Promise((resolve, reject) => {
+            readableStream
+              .pipe(csvParser())
+              .on('data', (data) => results.push(data))
+              .on('end', () => {
+                app.log.info(results);
+                resolve();
+              })
+              .on('error', reject);
           });
-          app.log.info(result);
-        }
 
-        //check if the issue_id already exists in the array
-        let issue_id_index = result.findIndex((row) => {
-          return row.issue_id == issue_id;
-        });
 
-        if(issue_id_index != -1){
-          // if the issue_id exists, update the row in the array
-          result[issue_id_index] = GetSurveyData(context);
-        }else{
-          // if the issue_id does not exist, push the row into the array
-          result.push(GetSurveyData(context));
-        }
+          //check if the issue_id already exists in the array
+          let issue_id_index = results.findIndex((row) => {
+            return parseInt(row.issue_id) == parseInt(context.payload.issue.id);
+          });
 
-        // convert the array back into a csv
-        let csv = parse(result, { header: true });
-        // convert the csv into a string
-        let csvString = csv.join("\n");
-        // convert the string into a buffer
-        let csvBuffer = Buffer.from(csvString, "utf8");
-        // encode the buffer into base64
-        let csvEncoded = csvBuffer.toString("base64");
-
-        // commit the file to the repo
-        await context.octokit.repos.createOrUpdateFileContents({
-          owner: context.payload.repository.owner.login,
-          repo: context.payload.repository.name,
-          path: "results.csv",
-          message: 'update',
-          content: csvEncoded,
-          sha: file.data.sha,
-        });
+          if(issue_id_index != -1){
+            // save previous comments
+            if (results[issue_id_index].comment) {
+              fileContent.comment = results[issue_id_index].comment + ' || ' + fileContent.comment;
+            }
+
+            // if the issue_id exists, update the row in the array results
+            results[issue_id_index] = fileContent;
+          }else{
+            // if the issue_id does not exist, push the row into the array
+            results.push(fileContent);
+          }
+
+          resultString = Object.keys(results[0]).join(',') + '\n'; 
+          results.forEach((row) => {
+            resultString += Object.values(row).join(',') + '\n'; 
+          });
+
+          app.log.info("CSV String:\n" + resultString);
 
+          // commit the file to the repo
+          await context.octokit.repos.createOrUpdateFileContents({
+            owner: context.payload.repository.owner.login,
+            repo: context.payload.repository.name,
+            path: "results.csv",
+            message: 'Update results.csv',
+            content:  Buffer.from(resultString).toString('base64'),
+            sha: file.data.sha,
+          });
+
+          app.log.info("File updated successfully\n " + Buffer.from(resultString).toString('base64'));
+
+        }
       } catch (error) {
         // If the file does not exist, create it
         if (error.status === 404) {
+          let completeData = 'enterprise_name,organization_name,repository_name,issue_id,issue_number,PR_number,assignee_name,is_copilot_used,saving_percentage,usage_frequency,comment,created_at,completed_at\n' 
+                              + Object.values(fileContent).join(',');
           await context.octokit.repos.createOrUpdateFileContents({
             owner: context.payload.repository.owner.login,
             repo: context.payload.repository.name,
             path: "results.csv",
             message: 'initial commit',
-            content: 'enterprise_name, organization_name, repository_name, issue_id, issue_number, PR_number, assignee_name, is_copilot_used, saving_percentage, usage_frequency, comment, created_at, completed_at\n' + GetSurveyData(context)
+            content: Buffer.from(completeData).toString('base64'),
           });
-        } else {
-          // If the error is not a 404 (not found), log it
+        }else{
           app.log.error(error);
         }
       }
+      
 
   }
 
diff --git a/issue_template/copilot-usage-en.md b/issue_template/copilot-usage-en.md
@@ -6,9 +6,7 @@ For Pull Request XXX:
 - [ ] No
 - [ ] Yes
 
-2. Compared to your previous experience coding WITHOUT using Copilot (This number represents 100%)
-
-***How much less time did the coding take during this PR with Copilot?***
+2. Compared to your previous experience coding WITHOUT using Copilot (This number represents 100%) ***How much less time did the coding take during this PR with Copilot?***
 
 (Example: The PR would normally take 5 days, but only took 4 days with Copilot then the answer is 20%)
 - [ ] 0%
diff --git a/issue_template/copilot-usage-es.md b/issue_template/copilot-usage-es.md
@@ -6,9 +6,7 @@ Para el Pull Request XXX:
 - [ ] No
 - [ ] Si
 
-2. En comparación con su experiencia anterior codificando sin usar Copilot (este número representa el 100%)
-
-***¿Cuánto menos tiempo tomó la codificación durante este PR con Copilot?***
+2. En comparación con su experiencia anterior codificando sin usar Copilot (este número representa el 100%) ***¿Cuánto menos tiempo tomó la codificación durante este PR con Copilot?***
 
 (Ejemplo: el PR normalmente tomaría 5 días, pero solo tomó 4 días con Copilot, entonces la respuesta es 20%)
 - [ ] 0%
diff --git a/issue_template/copilot-usage-fr.md b/issue_template/copilot-usage-fr.md
@@ -6,9 +6,7 @@ Pour le Pull Request XXX :
 - [ ] Non
 - [ ] Oui
 
-2. Par rapport à votre expérience précédente de codage SANS utiliser Copilot (ce nombre représente 100 %)
-
-***Combien de temps le codage a-t-il pris pendant ce PR avec Copilot ?***
+2. Par rapport à votre expérience précédente de codage SANS utiliser Copilot (ce nombre représente 100 %) ***Combien de temps le codage a-t-il pris pendant ce PR avec Copilot ?***
    
 (Exemple : le PR prendrait normalement 5 jours, mais n'a pris que 4 jours avec Copilot alors la réponse est 20 %)
 - [ ] 0%
diff --git a/issue_template/copilot-usage-pt.md b/issue_template/copilot-usage-pt.md
@@ -6,9 +6,7 @@ Para o Pull Request XXX:
 - [ ] Não
 - [ ] Sim
 
-2. Em comparação com sua experiência anterior de codificação SEM usar o Copilot (este número representa 100%)
-
-***Quanto menos tempo demorou a codificação durante este PR com o Copilot?***
+2. Em comparação com sua experiência anterior de codificação SEM usar o Copilot (este número representa 100%) ***Quanto menos tempo demorou a codificação durante este PR com o Copilot?***
    
 (Exemplo: o PR normalmente levaria 5 dias, mas demorou apenas 4 dias com o Copilot, então a resposta é 20%)
 - [ ] 0%
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -16,20 +16,14 @@
     "test": "jest --detectOpenHandles"
   },
   "dependencies": {
-    "@azure/ai-language-text": "^1.1.0",
-    "@azure/ai-text-analytics": "^5.1.0",
-    "ai-text-analytics": "^0.0.1-security",
-    "applicationinsights": "^2.7.3",
-    "csv-parse": "^5.5.2",
+    "csv-parser": "^3.0.0",
     "dedent": "^1.5.1",
     "dotenv": "^16.3.1",
-    "mssql": "^9.2.0",
     "probot": "^12.3.1"
   },
   "devDependencies": {
     "jest": "^29.6.4",
     "nock": "^13.3.3",
-    "semver": ">= 7.5.4",
     "smee-client": "^1.2.2"
   },
   "engines": {
diff --git a/test/fixtures/issue_body.md b/test/fixtures/issue_body.md
@@ -6,9 +6,7 @@ For Pull Request #44:
 - [ ] No
 - [ ] Yes
 
-2. Compared to your previous experience coding WITHOUT using Copilot (This number represents 100%)
-
-***How much less time did the coding take during this PR with Copilot?***
+2. Compared to your previous experience coding WITHOUT using Copilot (This number represents 100%) ***How much less time did the coding take during this PR with Copilot?***
 
 (Example: The PR would normally take 5 days, but only took 4 days with Copilot then the answer is 20%)
 - [ ] 0%
diff --git a/test/index.test.js b/test/index.test.js
