DGS-16859: Enable SonarQube code coverage reporting (confluentinc#1887)

Author: GunalKupta

.gitignore

@@ -30,3 +30,6 @@ tmp-KafkaCluster
 venv_test
 venv_examples
 *Zone.Identifier
+.coverage
+**/coverage.xml
+**/test-report.xml

.semaphore/semaphore.yml

@@ -14,6 +14,22 @@ global_job_config:
       - checkout
       - mkdir artifacts
 blocks:
+  - name: Test
+    dependencies: []
+    run:
+      # don't run the tests on non-functional changes...
+      when: "change_in('/', {exclude: ['.github/']})"
+    task:
+      jobs:
+        - name: Test
+          commands:
+            - sem-version python 3.9
+            - pip install tox
+            - tox -e cover
+            - mkdir test-output
+            - cp test-report.xml test-output
+            - test-results publish test-output
+            - artifact push workflow coverage.xml
   - name: "Wheels: OSX x64 - Python 3.6-3.12"
     run:
       when: "tag =~ '.*'"
@@ -285,3 +301,16 @@ blocks:
             - cd ..
             - artifact push project artifacts/confluent-kafka-python-wheels-${SEMAPHORE_GIT_TAG_NAME}-${SEMAPHORE_WORKFLOW_ID}.tgz --destination confluent-kafka-python-wheels-${SEMAPHORE_GIT_TAG_NAME}-${SEMAPHORE_WORKFLOW_ID}.tgz
             - echo Thank you
+
+after_pipeline:
+  task:
+    agent:
+      machine:
+        type: s1-prod-ubuntu20-04-amd64-1
+    jobs:
+      - name: SonarQube
+        commands:
+          - checkout
+          - sem-version java 11
+          - artifact pull workflow coverage.xml
+          - emit-sonarqube-data --run_only_sonar_scan

requirements/requirements-tests.txt

@@ -6,3 +6,4 @@ pytest
 pytest-timeout
 requests-mock
 respx
+pytest_cov

service.yml

@@ -6,6 +6,12 @@ git:
 github:
   enable: true
   repo_name: confluentinc/confluent-kafka-python
+sonarqube:
+  enable: true
+  coverage_exclusions:
+    - "**/*.pb.*"
+    - "**/mk-include/**/*"
+    - "examples/**"
 semaphore:
   enable: true
   pipeline_enable: false

sonar-project.properties

@@ -1,7 +1,7 @@
 ### service-bot sonarqube plugin managed file
 sonar.coverage.exclusions=**/test/**/*,**/tests/**/*,**/mock/**/*,**/mocks/**/*,**/*mock*,**/*test*
 sonar.cpd.exclusions=**/test/**/*,**/tests/**/*,**/mock/**/*,**/mocks/**/*,**/*mock*,**/*test*
-sonar.exclusions=**/*.pb.*,**/mk-include/**/*
+sonar.exclusions=**/*.pb.*,**/mk-include/**/*,examples/**
 sonar.language=python
 sonar.projectKey=confluent-kafka-python
 sonar.python.coverage.reportPaths=**/coverage.xml

tests/README.md

@@ -123,3 +123,10 @@ Tox can be used to test against various supported Python versions (py27, py36, p
 
     ```$ ./tests/run.sh tox```
 
+
+### Running with test coverage
+From top-level directory:
+```
+pip install tox
+./tests/run.sh tox -e cover
+```

tox.ini

@@ -2,13 +2,13 @@
 envlist = flake8,py37,py38,py39,py310,py311,py312,py313
 
 [testenv]
+deps =
+    -r requirements/requirements-tests-install.txt
+    .
 passenv =
     #http://tox.readthedocs.io/en/latest/config.html#confval-passenv=SPACE-SEPARATED-GLOBNAMES
     *
 commands =
-    # Install main package and all sub-packages
-    pip install -r requirements/requirements-tests-install.txt
-    pip install .
     # Early verification that module is loadable
     python -c 'import confluent_kafka ; print(confluent_kafka.version())'
     # Run tests (large timeout to allow docker image downloads)
@@ -20,6 +20,16 @@ commands =
 deps = flake8
 commands = flake8
 
+[testenv:cover]
+commands =
+    # Install source with editable flag (-e) so that SonarQube can parse the coverage report.
+    # Otherwise, the report shows source files located in site-packages, which SonarQube cannot find.
+    # Example: ".tox/cover/lib/python3.11/site-packages/confluent_kafka/__init__.py"
+    #   instead of "src/confluent_kafka/__init__.py"
+    pip install -e .
+    pytest {env:PYTESTARGS:} --cov confluent_kafka --cov-report term --cov-report html --cov-report xml \
+        --cov-branch --junitxml=test-report.xml tests/ {posargs}
+
 [pytest]
 python_files = test_*
 testpaths = tests