feat: add example github action workflow to display mutaiton coverage

jungs1 · jungs1 · commit 0c03051e3bb4 · 2024-07-06T22:58:55.000-04:00
diff --git a/README.md b/README.md
@@ -3,11 +3,8 @@
 
   Open-Source Language Agnostic LLM-based Mutation Testing for Automated Software Testing
   
-  Maintained by [CodeIntegrity](https://www.codeintegrity.ai). Anyone is welcome to contribute. 🌟
-
   [![GitHub license](https://img.shields.io/badge/License-AGPL_3.0-blue.svg)](https://github.com/yourcompany/mutahunter/blob/main/LICENSE)
   [![Discord](https://badgen.net/badge/icon/discord?icon=discord&label&color=purple)](https://discord.gg/S5u3RDMq)
-  [![Twitter](https://img.shields.io/twitter/follow/CodeIntegrity)](https://twitter.com/CodeIntegrity)
   [![Unit Tests](https://github.com/codeintegrity-ai/mutahunter/actions/workflows/test.yaml/badge.svg)](https://github.com/codeintegrity-ai/mutahunter/actions/workflows/test.yaml)
   <a href="https://github.com/codeintegrity-ai/mutahunter/commits/main">
   <img alt="GitHub" src="https://img.shields.io/github/last-commit/codeintegrity-ai/mutahunter/main?style=for-the-badge" height="20">
@@ -19,6 +16,7 @@
 - [Overview](#overview)
 - [Features](#features)
 - [Getting Started](#getting-started)
+- [CI/CD Integration](#cicd-integration)
 - [Roadmap](#roadmap)
 - [Cash Bounty Program](#cash-bounty-program)
 
@@ -102,12 +100,63 @@ Feel free to add more examples! ✨
 Check the logs directory to view the report:
 
 - `mutants.json` - Contains the list of mutants generated.
-- `mutation_coverage.json` - Contains the mutation coverage percentage.
-- `mutation_coverage_detail.json` - Contains detailed information per source file.
-
-## Cash Bounty Program
-
-Help us improve Mutahunter and get rewarded! We have a cash bounty program to incentivize contributions to the project. Check out the [bounty board](https://docs.google.com/spreadsheets/d/1cT2_O55m5txrUgZV81g1gtqE_ZDu9LlzgbpNa_HIisc/edit?gid=0#gid=0) to see the available bounties and claim one today!
+- `coverage.txt` - Contains information about mutation coverage.
+
+## CI/CD Integration
+
+You can integrate Mutahunter into your CI/CD pipeline to automate mutation testing. Here is an example GitHub Actions workflow file:
+
+![CI/CD](/images/github-bot.png)
+
+```yaml
+name: Mutahunter CI/CD 
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  mutahunter:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 2 # needed for git diff
+
+      - name: Set up Python 
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+
+      - name: Install Mutahunter
+        run: pip install git+https://github.com/codeintegrity-ai/mutahunter.git
+
+      - name: Set up Java for your project
+        uses: actions/setup-java@v2
+        with:
+          distribution: "adopt"
+          java-version: "17"
+
+      - name: Install dependencies and run tests
+        run: mvn test
+
+      - name: Run Mutahunter
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        run: |
+          mutahunter run --test-command "mvn test" --code-coverage-report-path "target/site/jacoco/jacoco.xml" --coverage-type jacoco --model "gpt-4o" --modified-files-only 
+
+      - name: PR comment the mutation coverage
+        uses: thollander/actions-comment-pull-request@v2.5.0
+        with:
+          filePath: logs/_latest/coverage.txt
+```
 
 ## Roadmap
 
@@ -116,9 +165,13 @@ Help us improve Mutahunter and get rewarded! We have a cash bounty program to in
 - [x] **Support for Other Coverage Report Formats:** Add compatibility for various coverage report formats.
 - [x] **Change-Based Testing:** Implement mutation testing on modified files based on the latest commit or pull request changes.
 - [x] **Extreme Mutation Testing:** Apply mutations to the codebase without using LLMs to detect pseudo-tested methods with significantly lower computational cost.
-- [ ] **Mutant Analysis:** Automatically analyze survived mutants to identify potential weaknesses in the test suite. Any suggestions are welcome!
-- [ ] **CI/CD Integration:** Develop connectors for popular CI/CD platforms like GitHub Actions.
-- [ ] **Automatic PR Bot:** Create a bot that automatically identifies bugs from the survived mutants list and provides fix suggestions.
+- [x] **CI/CD Integration:** Display mutation coverage in pull requests and automate mutation testing using GitHub Actions.
+- [ ] **Mutant Analysis:** Automatically analyze survived mutants to identify potential weaknesses in the test suite.
+
+## Cash Bounty Program
+
+Help us improve Mutahunter and get rewarded! We have a cash bounty program to incentivize contributions to the project. Check out the [bounty board](https://docs.google.com/spreadsheets/d/1cT2_O55m5txrUgZV81g1gtqE_ZDu9LlzgbpNa_HIisc/edit?gid=0#gid=0) to see the available bounties and claim one today!
+
 
 ## Acknowledgements
 
diff --git a/images/github-bot.png b/images/github-bot.png
diff --git a/src/mutahunter/core/report.py b/src/mutahunter/core/report.py
@@ -80,18 +80,19 @@ def generate_mutant_report(
         else:
             logger.info("💰 Expected Cost: $%.5f USD 💰", total_cost)
 
-        mutation_coverage = {
-            "total_mutants": len(mutants),
-            "killed_mutants": len(killed_mutants),
-            "survived_mutants": len(survived_mutants),
-            "timeout_mutants": len(timeout_mutants),
-            "compile_error_mutants": len(compile_error_mutants),
-            "mutation_coverage": total_mutation_coverage,
-            "line_coverage": line_coverage,
-            "expected_cost": total_cost,
-        }
-
-        self.save_report("logs/_latest/mutation_coverage.json", mutation_coverage)
+        with open("logs/_latest/coverage.txt", "a") as file:
+            file.write("Mutation Coverage:\n")
+            file.write(f"📊 Line Coverage: {line_coverage} 📊\n")
+            file.write(f"🎯 Mutation Coverage: {total_mutation_coverage} 🎯\n")
+            file.write(f"🦠 Total Mutants: {len(mutants)} 🦠\n")
+            file.write(f"🛡️ Survived Mutants: {len(survived_mutants)} 🛡️\n")
+            file.write(f"🗡️ Killed Mutants: {len(killed_mutants)} 🗡️\n")
+            file.write(f"🕒 Timeout Mutants: {len(timeout_mutants)} 🕒\n")
+            file.write(f"🔥 Compile Error Mutants: {len(compile_error_mutants)} 🔥\n")
+            if self.config.extreme:
+                file.write("💰 No Cost for extreme mutation testing 💰\n")
+            else:
+                file.write("💰 Expected Cost: $%.5f USD 💰\n", total_cost)
 
     def generate_mutant_report_detail(self, mutants: List[Mutant]) -> None:
         """
@@ -118,6 +119,7 @@ def generate_mutant_report_detail(self, mutants: List[Mutant]) -> None:
                 report_detail[source_path]["survived_mutants"] += 1
             elif mutant["status"] == "TIMEOUT":
                 report_detail[source_path]["timeout_mutants"] += 1
+
             elif mutant["status"] == "COMPILE_ERROR":
                 report_detail[source_path]["compile_error_mutants"] += 1
 
@@ -134,7 +136,19 @@ def generate_mutant_report_detail(self, mutants: List[Mutant]) -> None:
             )
             detail["mutation_coverage"] = mutation_coverage
 
-        self.save_report("logs/_latest/mutation_coverage_detail.json", report_detail)
+        with open("logs/_latest/coverage.txt", "a") as file:
+            file.write("\nDetailed Mutation Coverage:\n")
+            for source_path, detail in report_detail.items():
+                file.write(f"📂 Source File: {source_path} 📂\n")
+                file.write(f"🎯  Mutation Coverage: {detail['mutation_coverage']}🎯\n")
+                file.write(f"🦠  Total Mutants: {detail['total_mutants']} 🦠\n")
+                file.write(f"🛡️  Survived Mutants: {detail['survived_mutants']} 🛡️\n")
+                file.write(f"🗡️  Killed Mutants: {detail['killed_mutants']} 🗡️\n")
+                file.write(f"🕒  Timeout Mutants: {detail['timeout_mutants']} 🕒\n")
+                file.write(
+                    f"🔥  Compile Error Mutants: {detail['compile_error_mutants']}🔥\n"
+                )
+                file.write("\n")
 
     def save_report(self, filepath: str, data: Any) -> None:
         """
diff --git a/tests/test_report.py b/tests/test_report.py
@@ -70,103 +70,6 @@ def mutants():
     ]
 
 
-def test_generate_mutant_report_detail_all_statuses(config):
-    mutants = [
-        Mutant(
-            id="1",
-            source_path="app.go",
-            mutant_path="mutant1.py",
-            status="KILLED",
-            error_msg="",
-            mutant_code="",
-            type="",
-            description="",
-        ),
-        Mutant(
-            id="2",
-            source_path="app.go",
-            mutant_path="mutant2.py",
-            status="SURVIVED",
-            error_msg="",
-            mutant_code="",
-            type="",
-            description="",
-        ),
-        Mutant(
-            id="3",
-            source_path="app.go",
-            mutant_path="mutant3.py",
-            status="TIMEOUT",
-            error_msg="",
-            mutant_code="",
-            type="",
-            description="",
-        ),
-        Mutant(
-            id="4",
-            source_path="app.go",
-            mutant_path="mutant4.py",
-            status="COMPILE_ERROR",
-            error_msg="",
-            mutant_code="",
-            type="",
-            description="",
-        ),
-    ]
-    report = MutantReport(config)
-    mutants = [asdict(mutant) for mutant in mutants]
-
-    with patch.object(report, "save_report") as mock_save_report:
-        report.generate_mutant_report_detail(mutants)
-
-        mock_save_report.assert_called_once_with(
-            "logs/_latest/mutation_coverage_detail.json",
-            {
-                "app.go": {
-                    "total_mutants": 4,
-                    "killed_mutants": 1,
-                    "survived_mutants": 1,
-                    "timeout_mutants": 1,
-                    "compile_error_mutants": 1,
-                    "mutation_coverage": "50.00%",
-                }
-            },
-        )
-
-
-def test_generate_mutant_report(mutants, config):
-    report = MutantReport(config)
-    mutants = [asdict(mutant) for mutant in mutants]
-
-    with (
-        patch.object(report, "save_report") as mock_save_report,
-        patch("mutahunter.core.logger.logger.info") as mock_logger_info,
-    ):
-        report.generate_mutant_report(mutants, 0.0, 0.0)
-        mock_logger_info.assert_any_call("📊 Line Coverage: %s 📊", "0.00%")
-        mock_logger_info.assert_any_call("🎯 Mutation Coverage: %s 🎯", "50.00%")
-        mock_logger_info.assert_any_call("🦠 Total Mutants: %d 🦠", len(mutants))
-        mock_logger_info.assert_any_call("🛡️ Survived Mutants: %d 🛡️", 2)
-        mock_logger_info.assert_any_call("🗡️ Killed Mutants: %d 🗡️", 2)
-        mock_logger_info.assert_any_call("🕒 Timeout Mutants: %d 🕒", 0)
-        mock_logger_info.assert_any_call("🔥 Compile Error Mutants: %d 🔥", 0)
-        mock_logger_info.assert_any_call("💰 Expected Cost: $%.5f USD 💰", 0.0)
-
-        mock_save_report.assert_called_once_with(
-            "logs/_latest/mutation_coverage.json",
-            {
-                "total_mutants": 4,
-                "killed_mutants": 2,
-                "survived_mutants": 2,
-                "timeout_mutants": 0,
-                "compile_error_mutants": 0,
-                "mutation_coverage": "50.00%",
-                "line_coverage": "0.00%",
-                "expected_cost": 0.0,
-            },
-        )
-
-
 def test_save_report(config):
     report = MutantReport(config)
     data = {"key": "value"}
