reafactor: gitleaks rules update and general code clean up

Psingle20 · Psingle20 · commit cca6713bd95c · 2024-11-28T10:59:21.000+05:30
diff --git a/gitleaks.toml b/gitleaks.toml
diff --git a/proxy.config.json b/proxy.config.json
@@ -11,6 +11,12 @@
       "project": "finos",
       "name": "git-proxy",
       "url": "https://github.com/finos/git-proxy.git"
+    },
+    {
+      "project": "project name",
+      "name": "repo name",
+      "url": "repo url",
+      "LocalRepoRoot": "specify you local repository path"
     }
   ],
   "sink": [
diff --git a/src/proxy/chain.js b/src/proxy/chain.js
@@ -12,7 +12,7 @@ const pushActionChain = [
   proc.push.getDiff,               
   proc.push.checkSensitiveData,     // checkSensitiveData added
   proc.push.checkExifJpeg,
-  proc.push.checkForAiMlUsage,
+  proc.push.checkForAiMlusage,
   proc.push.clearBareClone,
   proc.push.checkCryptoImplementation,
   proc.push.scanDiff,
diff --git a/src/proxy/processors/push-action/checkCommitMessages.js b/src/proxy/processors/push-action/checkCommitMessages.js
@@ -1,6 +1,6 @@
 const Step = require('../../actions').Step;
 const config = require('../../../config');
-const { exec: eexec } = require('./checkForSecrets');
+const { exec: eexec } = require('./checkForSecrets.js');
 console.log(eexec);
 const commitConfig = config.getCommitConfig();
 
diff --git a/src/proxy/processors/push-action/checkCryptoImplementation.js b/src/proxy/processors/push-action/checkCryptoImplementation.js
@@ -135,6 +135,6 @@ const exec = async (req, action) => {
   }
 };
 
-// exec.displayName = 'checkCryptoImplementation.exec';
+exec.displayName = 'checkCryptoImplementation.exec';
 exports.exec = exec;
 exports.analyzeCodeForCrypto = analyzeCodeForCrypto;
diff --git a/src/proxy/processors/push-action/checkExifJpeg.js b/src/proxy/processors/push-action/checkExifJpeg.js
@@ -3,6 +3,8 @@ const { Step } = require('../../actions');
 const config = require('../../../config');
 
 const commitConfig = config.getCommitConfig();
+const authorizedlist = config.getAuthorisedList();
+
 const validExtensions = ['.jpeg', '.png', '.jpg', '.tiff'];
 // Make sure you have modified the proxy.config.json;
 // Function to check sensitive EXIF data
@@ -27,8 +29,9 @@ const checkSensitiveExifData = (metadata) => {
 };
 
 // Function to retrieve EXIF data using ExifTool
-const getExifData = async (filePath) => {
+const getExifData = async (relativePath,repoRoot) => {
     const exifTool = new ExifTool();
+    const filePath = path.join(repoRoot, relativePath);
     try {
         const metadata = await exifTool.read(filePath);
         return metadata ? checkSensitiveExifData(metadata) : true;
@@ -67,7 +70,14 @@ const exec = async (req, action, log = console.log) => {
         const filteredPaths = filePaths.filter(path => validExtensions.some(ext => path.endsWith(ext) && allowedFileType.includes(ext)));
 
         if (filteredPaths.length > 0) {
-            const exifResults = await Promise.all(filteredPaths.map(filePath => getExifData(filePath)));
+            
+            const exifResults = await Promise.all(
+                filteredPaths.map((Path) => {
+                  const repo = action.url;
+                  const repoRoot = authorizedlist.find((item) => item.url === repo).LocalRepoRoot;
+                  getExifData(Path, repoRoot);
+                }),
+              );
             const isBlocked = exifResults.some(result => !result);
 
             if (isBlocked) {
diff --git a/src/proxy/processors/push-action/checkForAiMlUsage.js b/src/proxy/processors/push-action/checkForAiMlUsage.js
@@ -1,6 +1,7 @@
 const { Step } = require('../../actions');
 const config = require('../../../config');
 const commitConfig = config.getCommitConfig();
+const authorizedlist = config.getAuthorisedList();
 
 const fs = require('fs');
 
@@ -58,21 +59,23 @@ const isAiMlFileByContent = (fileContent) => {
 
 
 // Main function to detect AI/ML usage in an array of file paths
-const detectAiMlUsageFiles = async (filePaths) => {
+const detectAiMlUsageFiles = async (filePaths,repoRoot) => {
     const results = [];
     // console.log("filePaths!", filePaths);    
-    for (const filePath of filePaths) {
+    for (let filePath of filePaths) {
         try {
             const fileName = filePath.split('/').pop();
             // console.log(fileName, "!!!");
             // Check if the file name itself indicates AI/ML usage
             if (isAiMlFileByExtension(fileName)) {
-                // console.log("FOUND EXTENSION for ", fileName);
+                console.log("FOUND EXTENSION for ", fileName);
                 results.push(false); continue; 
                 // Skip content check if the file name is a match
             }
             // Check for AI/ML indicators within the file content
             // console.log("testing content for ", fileName);
+            filePath = path.join(repoRoot, filePath);
+
             const content = await fs.promises.readFile(filePath, 'utf8');
             if (isAiMlFileByContent(content)) {
                 results.push(false); continue;
@@ -118,7 +121,9 @@ const exec = async (req, action, log = console.log) => {
         // console.log(filePaths);
 
         if (filePaths.length) {
-            const aiMlDetected = await detectAiMlUsageFiles(filePaths);
+            const repoRoot = authorizedlist.find((item) => item.url === action.url).LocalRepoRoot;
+
+            const aiMlDetected = await detectAiMlUsageFiles(filePaths,repoRoot);
             // console.log(aiMlDetected);
             const isBlocked = aiMlDetected.some(found => !found);
             // const isBlocked = false;
@@ -139,5 +144,5 @@ const exec = async (req, action, log = console.log) => {
     return action;
 };
 
-exec.displayName = 'logFileChanges.exec';
+exec.displayName = 'checkForAiMlUsage.exec';
 module.exports = { exec };
diff --git a/src/proxy/processors/push-action/checkForSecrets.js b/src/proxy/processors/push-action/checkForSecrets.js
@@ -4,6 +4,8 @@ const { exec: cexec } = require('child_process');
 const path = require('path');
 const config = require('../../../config');
 const commitConfig = config.getCommitConfig();
+const authorizedlist = config.getAuthorisedList();
+
 
 // Function to extract relevant file paths from Git diff content
 // go to proxyconfig.json and enable the feature
@@ -37,53 +39,39 @@ function extractRelevantDirectories(diffContent) {
 }
 
 // Function to run Gitleaks with directory paths
-function runGitleaks(filePaths) {
+function runGitleaks(filePaths,repoRoot) {
   return new Promise((resolve, reject) => {
     const filesToCheck = filePaths
-      .map((filePath) => `"${path.resolve(filePath).replace(/\\/g, '/')}"`)
+      .map((filePath) => `"${path.resolve(repoRoot,filePath).replace(/\\/g, '/')}"`)
       .join(' ');
+    console.log("filesToCheck:", filesToCheck);
 
     const configPath = path.resolve(__dirname, '../../../../gitleaks.toml').replace(/\\/g, '/');
-    const reportPath = path
-      .resolve(__dirname, '../../../../gitleaks_report.json')
-      .replace(/\\/g, '/');
+    const reportPath = repoRoot + '/gitleaks_report.json';
 
-    const command = `gitleaks dir ${filesToCheck}  --config="${configPath}" --report-format json --log-level error --report-path="${reportPath}"`;
+    const command = `gitleaks dir ${filesToCheck}  --config="${configPath}" --report-format json --log-level debug --report-path="${reportPath}"`;
     console.log(`Executing Gitleaks Command: ${command}`);
 
     cexec(command, (error, stdout, stderr) => {
       if (error) {
-        console.error(`Error executing gitleaks: ${error.message}`);
-        reject(new Error(`Error executing gitleaks: ${error.message}`));
-      } else if (stderr) {
-        console.error(`stderr: ${stderr}`);
-        reject(new Error(`stderr: ${stderr}`));
+        // If leaks are found, handle the warning gracefully
+        console.log("stderrrrr:",stderr);
+        if (stderr.includes("leaks found")) {
+          console.warn("Leaks were found, but execution succeeded.");
+          resolve(true); // Consider this a successful run
+        } else {
+          console.error(`Error executing gitleaks: ${error.message}`);
+          reject(new Error(`Error executing gitleaks: ${error.message}`));
+        }
       } else {
-        resolve(stdout);
+        resolve(false);
       }
     });
   });
 }
 
-// Function to check for sensitive secrets in the Gitleaks output
-function checkForSensitiveSecrets(output) {
-  try {
-    const findings = JSON.parse(output);
-
-    if (findings.length > 0) {
-      findings.forEach((finding) => {
-        console.log(`Secret found in file: ${finding.file}`);
-        console.log(`  Rule: ${finding.rule_id}`);
-        console.log(`  Secret: ${finding.secret}`);
-      });
-      return true;
-    }
-    return false;
-  } catch (error) {
-    console.error('Error parsing Gitleaks output:', error);
-    return false;
-  }
-}
+
+
 
 // Example usage in exec function
 const exec = async (req, action) => {
@@ -98,13 +86,14 @@ const exec = async (req, action) => {
 
   if (diffStep && diffStep.content) {
     const dirPaths = extractRelevantDirectories(diffStep.content);
-
+    const repoRoot = authorizedlist.find((item) => item.url === action.url).LocalRepoRoot;
+    
     if (dirPaths.length > 0) {
       try {
-        const result = await runGitleaks(dirPaths);
-        const hasSensitiveSecrets = checkForSensitiveSecrets(result);
+        const res = await runGitleaks(dirPaths,repoRoot);
+        
 
-        if (hasSensitiveSecrets) {
+        if (res) {
           step.blocked = true;
           step.blockedMessage = 'Sensitive secrets detected in the diff.';
           console.log('Sensitive secrets detected! Push blocked.');
@@ -126,7 +115,7 @@ const exec = async (req, action) => {
 };
 
 exec.displayName = 'checkforSecrets.exec';
-
+module.exports = { exec };
 
 
 
diff --git a/src/proxy/processors/push-action/checkSensitiveData.js b/src/proxy/processors/push-action/checkSensitiveData.js
@@ -8,6 +8,8 @@ const config = require('../../../config');
 // const { exec: getDiffExec } = require('./getDiff');
 // Function to check for sensitive data patterns
 const commitConfig = config.getCommitConfig();
+const authorizedlist = config.getAuthorisedList();
+
 const checkForSensitiveData = (cell) => {
     const sensitivePatterns = [
         /\d{3}-\d{2}-\d{4}/, // Social Security Number (SSN)
@@ -94,8 +96,9 @@ const checkLogJsonFiles = async (filePath) => {
     });
 };
 // Function to parse the file based on its extension
-const parseFile = async (filePath) => {
-  
+const parseFile = async (repoRoot, relativePath) => {
+  const filePath = path.join(repoRoot, relativePath);
+    
     const ext = path.extname(filePath).toLowerCase();
     const FilestoCheck = commitConfig.diff.block.proxyFileTypes;
     if(!FilestoCheck.includes(ext)){
@@ -145,16 +148,27 @@ const exec = async (req, action) => {
         const filePaths = extractFilePathsFromDiff(diffStep.content);
 
         if (filePaths.length > 0) {
-            // Check for sensitive data in all files
-            const sensitiveDataFound = await Promise.all(filePaths.map(parseFile));
-            const anySensitiveDataDetected = sensitiveDataFound.some(found => found);
-
-            if (anySensitiveDataDetected) {
-                step.blocked= true;
-                step.error = true;
-                step.errorMessage = 'Your push has been blocked due to sensitive data detection.';
-                console.log(step.errorMessage);
-            }
+            try {
+                const repoUrl = action.url;
+                const repo = authorizedlist.find((item) => item.url === repoUrl);
+                // console.log(repo);
+                const repoRoot = repo.LocalRepoRoot;
+                // console.log('my reporoot is ' + repoRoot);
+        
+                const sensitiveDataFound = await Promise.all(
+                  filePaths.map((filePath) => parseFile(repoRoot, filePath)),
+                );
+                const anySensitiveDataDetected = sensitiveDataFound.some((found) => found);
+        
+                if (anySensitiveDataDetected) {
+                  step.blocked = true;
+                  step.error = true;
+                  step.errorMessage = 'Your push has been blocked due to sensitive data detection.';
+                  console.log(step.errorMessage);
+                }
+              } catch (error) {
+                console.error(`Error processing files: ${error.message}`);
+              }
         } else {
             console.log('No file paths provided in the diff step.');
         }
diff --git a/src/proxy/processors/push-action/index.js b/src/proxy/processors/push-action/index.js
@@ -8,11 +8,11 @@ exports.scanDiff = require('./scanDiff').exec;
 exports.blockForAuth = require('./blockForAuth').exec;
 exports.checkIfWaitingAuth = require('./checkIfWaitingAuth').exec;
 exports.checkCommitMessages = require('./checkCommitMessages').exec;
-console.log(__dirname);
 exports.checkAuthorEmails = require('./checkAuthorEmails').exec;
 exports.checkUserPushPermission = require('./checkUserPushPermission').exec;
 exports.clearBareClone = require('./clearBareClone').exec;
 exports.checkSensitiveData = require('./checkSensitiveData').exec;
 exports.checkExifJpeg = require('./checkExifJpeg').exec;
 exports.checkForAiMlusage = require('./checkForAiMlUsage').exec;
+exports.checkForSecrets = require('./checkForSecrets').exec;
 exports.checkCryptoImplementation = require('./checkCryptoImplementation').exec;
diff --git a/test/test_data/GitleaksSampleData/sensitive_data.js b/test/test_data/GitleaksSampleData/sensitive_data.js
@@ -1,3 +1,3 @@
 // File containing sensitive AWS Access Key
-const secret = 'AKIAIOSFODNN8EXAMPLE';  // Example AWS access key
+const secret = 'AKIAIOSFODNN9EXPLEAM';  // Example AWS access key
 console.log(secret);
diff --git a/test/test_data/sensitveDatalikecsv/sensitive_data.csv b/test/test_data/sensitveDatalikecsv/sensitive_data.csv
diff --git a/test/test_data/sensitveDatalikecsv/sensitive_data2.xlsx b/test/test_data/sensitveDatalikecsv/sensitive_data2.xlsx
diff --git a/test/test_data/sensitveDatalikecsv/sensitive_data4.json b/test/test_data/sensitveDatalikecsv/sensitive_data4.json

Original file line number	Diff line number	Diff line change
`@@ -135,6 +135,6 @@ const exec = async (req, action) => {`
`135`	`135`	`}`
`136`	`136`	`};`
`137`	`137`
`138`		`-// exec.displayName = 'checkCryptoImplementation.exec';`
	`138`	`+exec.displayName = 'checkCryptoImplementation.exec';`
`139`	`139`	`exports.exec = exec;`
`140`	`140`	`exports.analyzeCodeForCrypto = analyzeCodeForCrypto;`