contest: vm: capture code coverage

Code coverage is a valuable info to get to know how much we can trust a test suite, and easily find out what needs to be improved.

It is quite easy to get such info with the kernel:

- The kernel needs to be compiled with GCOV_KERNEL=y, and have either
  GCOV_PROFILE_ALL=y, or GCOV_PROFILE := y set in the Makefiles. The
  recommended way is to add 'GCOV_PROFILE' in net/Makefile and
  drivers/net/Makefile.

- Before stopping the VM, the LCOV file can be captured using the 'lcov'
  tool, version >= 2.0 is recommended.

- 'genhtml' from the LCOV project can be used later to generate an HTML
  version using all the .lcov files. It could be done per LCOV file, but
  that will then only show the coverage per VM, not the global one.

This GCOV support is disabled by default. It can be enabled via
'vm.gcov=on'. I suggest to keep it off by default, and switch it to on
later when everything is in place.

Signed-off-by: Matthieu Baerts (NGI0) <[email protected]>

Author: matttbe

contest/remote/lib/vm.py

@@ -9,6 +9,7 @@
 import os
 import psutil
 import re
+import shutil
 import signal
 from .crash import has_crash, extract_crash
 
@@ -38,6 +39,7 @@
 default_timeout=15
 boot_timeout=45
 slowdown=2.5 # mark the machine as slow and multiply the ksft timeout by 2.5
+gcov=off / on
 """
 
 
@@ -66,6 +68,7 @@ def __init__(self, config, vm_name=""):
 
         self.filter_data = None
         self.has_kmemleak = None
+        self.has_gcov = self.config.getboolean('vm', 'gcov', fallback=False)
         self.log_out = ""
         self.log_err = ""
 
@@ -101,11 +104,13 @@ def build(self, extra_configs, override_configs=None):
         if extra_configs:
             configs += extra_configs
 
+        gcov = " --configitem GCOV_KERNEL=y" if self.has_gcov else ""
+
         print(f"INFO{self.print_pfx} building kernel")
         # Make sure we rebuild, config and module deps can be stale otherwise
         self.tree_cmd("make mrproper")
 
-        rc = self.tree_cmd("vng -v -b" + " -f ".join([""] + configs))
+        rc = self.tree_cmd("vng -v -b" + " -f ".join([""] + configs) + gcov)
         if rc != 0:
             print(f"INFO{self.print_pfx} kernel build failed")
             return False
@@ -203,6 +208,7 @@ def start(self, cwd=None):
         self.cmd("ls /sys/kernel/debug/")
         self.drain_to_prompt()
         self.has_kmemleak = "kmemleak" in self.log_out[off:]
+        self.has_gcov = self.has_gcov and "gcov" in self.log_out[off:]
 
         self._set_env()
 
@@ -387,6 +393,18 @@ def check_health(self):
             self.cmd("echo scan > /sys/kernel/debug/kmemleak && cat /sys/kernel/debug/kmemleak")
             self.drain_to_prompt()
 
+    def capture_gcov(self, dest):
+        if not self.has_gcov:
+            return
+
+        lcov = "kernel.lcov"
+        self.cmd(f"lcov --capture --keep-going --rc geninfo_unexecuted_blocks=1 --function-coverage --branch-coverage -j $(nproc) -o {lcov}")
+        self.drain_to_prompt()
+
+        lcov = os.path.join(self.tree_path, lcov)
+        if os.path.isfile(lcov):
+            shutil.copy(lcov, dest)
+
     def bash_prev_retcode(self):
         self.cmd("echo $?")
         stdout, stderr = self.drain_to_prompt()

contest/remote/vmksft-p.py

@@ -218,6 +218,7 @@ def _vm_thread(config, results_path, thr_id, hard_stop, in_queue, out_queue):
             vm = None
 
     if vm is not None:
+        vm.capture_gcov(results_path + f'/kernel-thr{thr_id}-{vm_id}.lcov')
         vm.stop()
         vm.dump_log(results_path + f'/vm-stop-thr{thr_id}-{vm_id}')
     return