Workaround eBPF errors (#1350)

The kernel is raising errors related to the variable `@every`. We
workaround this problem by replacing it with an integer literal
(expanded from Python template variable `$EVERY`) instead.

The USDT `process_root_nodes` may not exist in the compiled .so file of
the VM binding because the methods that contain this USDT are generic
methods that are instantiated on demand. VMs that do not use
object-enqueuing tracing (such as OpenJDK) may not have this USDT. We
added the `--no-root-nodes` option to skip that USDT.

Added some debugging options to `capture.py`, including `-d` and `-v`.

Author: wks

tools/tracing/timeline/README.md

@@ -19,7 +19,7 @@ Suite.
 Run the following command with a **normal** user (*not* as `root` or using `sudo`):
 
 ```shell
-./capture.py -e 50 -m /path/to/libmmtk_openjdk.so
+./capture.py -e 50 -m /path/to/libmmtk_openjdk.so --no-root-nodes
 ```
 
 `-e 50` means we only capture one GC in every 50 GCs because otherwise it will have to print too
@@ -30,6 +30,8 @@ post-processing.  If one single GC still produces too much log and overruns the
 should consider setting the `BPFTRACE_PERF_RB_PAGES` environment variable.  See the man page of
 `bpftrace`.)
 
+`--no-root-nodes` skips the `process_root_nodes` USDT which does not exist in `libmmtk_openjdk.so`.
+
 Replace `/path/to/libmmtk_openjdk.so` with the actual path to the `.so` that contains MMTk and its
 binding.
 
@@ -84,7 +86,7 @@ This means things are working properly.  Now re-run `./capture.py` again, but pi
 file.
 
 ```
-./capture.py -m /path/to/libmmtk_openjdk.so > mybenchmark.log
+./capture.py -m /path/to/libmmtk_openjdk.so --no-root-nodes > mybenchmark.log
 ```
 
 Type the root password if prompted.

tools/tracing/timeline/capture.bt

@@ -1,5 +1,4 @@
 BEGIN {
-    @every = $EVERY;
     @harness = $HARNESS;
 
     @gc_count = 0;
@@ -33,7 +32,7 @@ usdt:$MMTK:mmtk:gc_start {
     printf("GC,B,%d,%lu\n", tid, nsecs);
     @gc_count += 1;
     // bpftrace warns that signed `%` operator may have undefiend behavior.
-    if ((uint64)@gc_count % (uint64)@every == 0 && @stats_enabled) {
+    if ((uint64)@gc_count % $EVERY == 0 && @stats_enabled) {
         @enable_print = 1;
     } else {
         @enable_print = 0;
@@ -87,11 +86,13 @@ usdt:$MMTK:mmtk:roots {
     }
 }
 
+//////// BEGIN:PROCESS_ROOT_NODES
 usdt:$MMTK:mmtk:process_root_nodes {
     if (@enable_print) {
         printf("process_root_nodes,meta,%d,%lu,%lu,%lu\n", tid, nsecs, arg0, arg1);
     }
 }
+//////// END:PROCESS_ROOT_NODES
 
 usdt:$MMTK:mmtk:process_slots {
     if (@enable_print) {

tools/tracing/timeline/capture.py

@@ -27,14 +27,32 @@ def get_args():
                         help="Path of the MMTk binary")
     parser.add_argument("-H", "--harness", action="store_true",
                         help="Only collect data for the timing iteration (harness_begin/harness_end)")
-    parser.add_argument("-p", "--print-script", action="store_true",
-                        help="Print the content of the bpftrace script")
     parser.add_argument("-e", "--every", metavar="N", type=int, default=1,
                         help="Only capture every N-th GC"),
     parser.add_argument("-x", "--extra", metavar="S", type=str, action="append",
                         help="Append script S after 'capture.bt'.  Use this option multiple times to append multiple scripts."),
+    parser.add_argument("--no-root-nodes", action="store_true",
+                        help="Do not add 'process_root_nodes' probe." +
+                        "  This USDT is conditionally generated by Rust generics, and may not exist in some VM bindings.")
+    parser.add_argument("-p", "--print-script", action="store_true",
+                        help="Print the content of the bpftrace script")
+    parser.add_argument("-d", "--dry-run", action="store_true",
+                        help="Print the bpftrace command to be executed, but does not actually execute the command.  Useful with -p")
+    parser.add_argument("-v", "--verbose", action="store_true",
+                        help="Add -v to the bpftrace command.  Useful for debugging"),
     return parser.parse_args()
 
+def delete_lines_between(lines, begin, end):
+    begin_index, end_index = None, None
+    for i, line in enumerate(lines):
+        if begin in line and begin_index is None:
+            begin_index = i
+        if begin_index is not None and end in line:
+            end_index = i
+            break
+    else:
+        raise Exception(f"Cannot find {begin} and {end} in the script. {begin_index} {end_index}")
+    del lines[begin_index:end_index + 1]
 
 def main():
     args = get_args()
@@ -48,12 +66,15 @@ def main():
     if args.extra is not None:
         for extra_script in args.extra:
             script_paths.append(Path(extra_script))
-    script_texts = []
+    scripts_lines = []
     for script_path in script_paths:
-        script_text = script_path.read_text()
-        script_texts.append(script_text)
+        script_lines = script_path.read_text().splitlines()
+        scripts_lines.extend(script_lines)
 
-    merged_script = "\n".join(script_texts)
+    if args.no_root_nodes:
+        delete_lines_between(scripts_lines, "BEGIN:PROCESS_ROOT_NODES", "END:PROCESS_ROOT_NODES")
+
+    merged_script = "\n".join(scripts_lines)
 
     template = Template(merged_script)
     with tempfile.NamedTemporaryFile(mode="w+t") as tmp:
@@ -66,17 +87,28 @@ def main():
             print(content)
         tmp.write(content)
         tmp.flush()
-        # We use execvp to replace the current process instead of creating
-        # a subprocess (or sh -c). This is so that when users invoke this from
-        # the command line, Ctrl-C will be captured by bpftrace instead of the
-        # outer Python script. The temporary file can then be cleaned up by
-        # the END probe in bpftrace.
-        #
-        # In theory, you can implement this via pty, but it is very finicky
-        # and doesn't work reliably.
-        # See also https://github.com/anupli/running-ng/commit/b74e3a13f56dd97f73432d8a391e1d6cd9db8663
-        os.execvp("sudo", ["sudo", args.bpftrace,
-                           "--unsafe", tmp.name])
+
+        extra_options = []
+        if args.verbose:
+            extra_options.append("-v")
+
+        command_line = ["sudo", args.bpftrace] + extra_options + ["--unsafe", tmp.name]
+
+        if args.dry_run:
+            print("Dry run.  Command to execute:")
+            print(" ".join(f"'{c}'" for c in command_line))
+            # tempfile will be deleted at the end of `with`.
+        else:
+            # We use execvp to replace the current process instead of creating
+            # a subprocess (or sh -c). This is so that when users invoke this from
+            # the command line, Ctrl-C will be captured by bpftrace instead of the
+            # outer Python script. The temporary file can then be cleaned up by
+            # the END probe in bpftrace.
+            #
+            # In theory, you can implement this via pty, but it is very finicky
+            # and doesn't work reliably.
+            # See also https://github.com/anupli/running-ng/commit/b74e3a13f56dd97f73432d8a391e1d6cd9db8663
+            os.execvp("sudo", command_line)
 
 
 if __name__ == "__main__":