Report Coverage Number in CI (Azure#39639)

* report a single overall coverage % per package

Author: scbedd

.gitignore

@@ -18,7 +18,7 @@ pythonenv*/
 .ptvs/
 
 # Coverage report
-**/.coverage
+**/.coverage*
 _coverage
 
 # Build results

eng/pipelines/templates/steps/build-test.yml

@@ -155,7 +155,7 @@ steps:
 
       python -m pip install coverage==7.2.5
       python scripts/devops_tasks/create_coverage.py
-    displayName: Create Coverage Report
+    displayName: Report Coverage
     condition: and(succeeded(), ${{ parameters.RunCoverage }})
 
   - ${{ if eq('true', parameters.UseFederatedAuth) }}:

eng/tox/run_coverage.py

@@ -0,0 +1,51 @@
+import coverage
+import argparse
+import os
+import json
+import datetime
+
+from ci_tools.parsing import ParsedSetup
+from ci_tools.variables import in_ci
+from ci_tools.environment_exclusions import is_check_enabled
+
+def get_total_coverage(coverage_file: str) -> float:
+    cov = coverage.Coverage(data_file=coverage_file)
+    cov.load()
+    report = cov.report()
+
+    return report
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Check coverage for a package."
+    )
+
+    parser.add_argument(
+        "-t",
+        "--target",
+        dest="target_package",
+        help="The target package directory on disk. This script looks for a .coverage file under <target_package> directory.",
+        required=True,
+    )
+
+    args = parser.parse_args()
+    pkg_details = ParsedSetup.from_path(args.target_package)
+
+    possible_coverage_file = os.path.join(args.target_package, ".coverage")
+
+    if os.path.exists(possible_coverage_file):
+        total_coverage = get_total_coverage(possible_coverage_file)
+        print(f"Total coverage for {pkg_details.name} is {total_coverage:.2f}%")
+
+        if in_ci():
+            metric_obj = {}
+            metric_obj["value"] = total_coverage / 100
+            metric_obj["name"] = "test_coverage_ratio"
+            metric_obj["labels"] = { "package": pkg_details.name }
+            metric_obj["timestamp"] = datetime.datetime.now(datetime.timezone.utc).isoformat()
+            print(f'logmetric: {json.dumps(metric_obj)}')
+
+    if is_check_enabled(args.target_package, "cov_enforcement", False):
+        print("Coverage enforcement is enabled for this package.")
+

eng/tox/tox.ini

@@ -73,6 +73,7 @@ commands =
     python {repository_root}/eng/tox/create_package_and_install.py -d {envtmpdir} -p {tox_root} -w {envtmpdir}
     python -m pip freeze
     pytest {[pytest]default_args} {posargs} {tox_root}
+    python {repository_root}/eng/tox/run_coverage.py -t {tox_root}
 
 
 [testenv:pylint]

scripts/devops_tasks/create_coverage.py

@@ -53,7 +53,7 @@ def generate_coverage_xml():
     if os.path.exists(coverage_dir):
         logging.info("Generating coverage XML")
         commands = ["coverage", "xml", "-i"]
-        run_check_call(commands, root_dir, always_exit = False)
+        run_check_call(commands, root_dir, always_exit=False)
     else:
         logging.error("Coverage file is not available in {} to generate coverage XML".format(coverage_dir))
 
@@ -72,11 +72,12 @@ def fix_coverage_xml(coverage_file):
         out = re.sub("\.?\.tox[\s\S\.\d]*?\.site-packages", "", out)
 
     if out:
-        with open(coverage_file, 'w') as cov_file:
+        with open(coverage_file, "w") as cov_file:
             cov_file.write(out)
 
+
 if __name__ == "__main__":
-    coverage_xml = os.path.join(root_dir, 'coverage.xml')
+    coverage_xml = os.path.join(root_dir, "coverage.xml")
 
     collect_tox_coverage_files()
     generate_coverage_xml()