Merge pull request #300 from Checkmarx/other/benalvo/add-coverage-report

Optimize tests runtime and add coverage report for integration tests (AST-79390)

Author: cx-ben-alvo

.github/workflows/ci.yml

@@ -1,7 +1,5 @@
 name: Checkmarx One Jetbrains Plugin CI
-
 on: [ pull_request, workflow_dispatch ]
-
 env:
   CX_BASE_URI: ${{ secrets.CX_BASE_URI }}
   CX_APIKEY: ${{ secrets.CX_APIKEY }}
@@ -14,12 +12,9 @@ env:
   CX_NOT_MATCH_TEST_PROJECT: ${{ secrets.CX_NOT_MATCH_TEST_PROJECT }}
   CX_NOT_MATCH_TEST_BRANCH: ${{ secrets.CX_NOT_MATCH_TEST_BRANCH }}
   CX_NOT_MATCH_TEST_SCAN_ID: ${{ secrets.CX_NOT_MATCH_TEST_SCAN_ID }}
-
-
 concurrency:
   group: ${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
-
 jobs:
   testUI:
     needs: [ testIntegration ]
@@ -35,7 +30,7 @@ jobs:
           distribution: zulu
           java-version: 11
       - name: Setup FFmpeg
-        uses: FedericoCarboni/setup-ffmpeg@v2
+        uses: FedericoCarboni/setup-ffmpeg@583042d32dd1cabb8bd09df03bde06080da5c87c #v2.0.0
         with:
           # Not strictly necessary, but it may prevent rate limit
           # errors especially on GitHub-hosted macos machines.
@@ -94,13 +89,21 @@ jobs:
       - name: Clean
         run: ./gradlew clean
       # Run tests
-      - name: Tests
-        run: ./gradlew test -i --tests com.checkmarx.intellij.standard*
+      - name: Run Tests with coverage report
+        id: test_results
+        run: |
+          ./gradlew test --tests "com.checkmarx.intellij.standard*" jacocoTestReport
+      # Save coverage report as an artifact
+      - name: Upload Coverage Report
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-report
+          path: build/reports/jacoco/test/html/
       # Save report if tests fail
       - name: Save fails report
         if: ${{ failure() }}
         uses: actions/upload-artifact@v4
         with:
           name: test-fails-report-integration
           path: |
-            build/reports
+            build/reports

build.gradle

@@ -2,6 +2,7 @@ plugins {
     id 'io.freefair.lombok' version '8.6'
     id 'org.jetbrains.intellij' version '1.17.4'
     id 'java'
+    id 'jacoco'
 }
 
 group 'com.checkmarx'
@@ -20,6 +21,18 @@ repositories {
     }
 }
 
+jacoco {
+    toolVersion = "0.8.7"
+}
+
+tasks.jacocoTestReport {
+    dependsOn test // Ensure tests run before generating the report
+    reports {
+        xml.required.set(true)
+        html.required.set(true)
+    }
+}
+
 dependencies {
     testImplementation 'com.intellij.remoterobot:remote-robot:' + remoteRobotVersion
     testImplementation('com.intellij.remoterobot:remote-fixtures:' + remoteRobotVersion) {
@@ -85,6 +98,15 @@ runIdeForUiTests {
     systemProperty 'jb.consents.confirmation.enabled', 'false'
 }
 
+jacocoTestReport {
+    additionalSourceDirs.setFrom(files(sourceSets.main.allSource.srcDirs))
+    classDirectories.setFrom(
+            fileTree(dir: "$buildDir/classes/java/main", excludes: [])
+    )
+    executionData.setFrom(fileTree(dir: "$buildDir", includes: ["jacoco/test.exec"]))
+}
+
+
 publishPlugin {
     token.set System.getenv("PUBLISH_TOKEN")
     if (project.hasProperty("rchannels")) {

src/test/java/com/checkmarx/intellij/standard/commands/TestScan.java

@@ -27,11 +27,9 @@ public void testGetList() {
         String msg = String.format("project: %s branch: %s scans: %d", project.getId(), Environment.BRANCH_NAME, scans.size());
         Assertions.assertTrue(scans.size() > 0, msg);
         Assertions.assertTrue(scans.size() <= 10000, msg);
-        for (com.checkmarx.ast.scan.Scan scan : scans) {
-            Assertions.assertEquals("Completed", scan.getStatus());
-            Assertions.assertEquals(Environment.BRANCH_NAME, scan.getBranch());
-            Assertions.assertEquals(getEnvProject().getId(), scan.getProjectId());
-        }
+
+        // Check that all scans are for the correct project and branch and have completed
+        Assertions.assertTrue(scans.stream().allMatch(scan -> scan.getProjectId().equals(project.getId()) && scan.getBranch().equals(Environment.BRANCH_NAME) && scan.getStatus().equals("Completed")));
     }
 
     @Test

src/test/java/com/checkmarx/intellij/standard/commands/TestScanAsca.java

@@ -13,6 +13,7 @@
 import com.intellij.psi.PsiFile;
 import com.intellij.psi.PsiManager;
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
 
 public class TestScanAsca extends BaseTest {
     AscaService ascaService = new AscaService();
@@ -43,6 +44,7 @@ private PsiFile createPsiFileFromPath(String filePath) {
         return psiFile;
     }
 
+    @Test
     public void testRunAscaScan_FileWithVulnerabilities_Success() {
         PsiFile psiFile = createPsiFileFromPath("src/test/java/com/checkmarx/intellij/standard/data/python-vul-file.py");
         Project project = ProjectManager.getInstance().getDefaultProject();
@@ -55,6 +57,7 @@ public void testRunAscaScan_FileWithVulnerabilities_Success() {
         });
     }
 
+    @Test
     public void testRunAscaScan_FileWithNoVulnerabilities_Success() {
         PsiFile psiFile = createPsiFileFromPath("src/test/java/com/checkmarx/intellij/standard/data/csharp-no-vul.cs");
         Project project = ProjectManager.getInstance().getDefaultProject();
@@ -66,6 +69,7 @@ public void testRunAscaScan_FileWithNoVulnerabilities_Success() {
         });
     }
 
+    @Test
     public void testRunAscaScan_FileWithoutExtension_Fail() {
         PsiFile psiFile = createPsiFileFromPath("src/test/java/com/checkmarx/intellij/standard/data/file");
         Project project = ProjectManager.getInstance().getDefaultProject();