feat: add binary trace support

upgrade runtime_tracing to 0.12 and allow flushing traces as json or binary; benchmark script now measures native binary format

Author: zah

.agents/codex-setup

@@ -4,7 +4,11 @@ AGENTS_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 cd $AGENTS_DIR
 
 sudo apt-get update
-sudo apt-get install -y --no-install-recommends just ruby-full libclang-dev
+sudo apt-get install -y --no-install-recommends \
+  just \
+  ruby-full \
+  libclang-dev \
+  capnproto libcapnp-dev
 
 pushd ..
   just build-extension

.agents/tasks/2025/06/28-2114-upgrade-runtime-tracing

@@ -0,0 +1,5 @@
+There is a new version of the runtime_tracing create used by the `gems/codetracer-ruby-recorder/ext/native_tracer/src/lib.rs` module.
+
+The primary new feature is that the create support a new binary format for the trace files and requires the user to choose between json and binary when flushing the trace to a file.
+
+Please migrate the project to the new version of runtime_tracing and enhance the benchmark setup to compare the pure ruby implementation vs json in the native implementation vs the new binary format in the native implementation.

.github/workflows/ci.yml

@@ -16,10 +16,38 @@ jobs:
       - uses: actions/checkout@v4
         with:
           lfs: true
+      - name: Install system dependencies
+        shell: bash
+        run: |
+          if [[ "$RUNNER_OS" == "Linux" ]]; then
+            sudo apt-get update
+            sudo apt-get install -y --no-install-recommends libclang-dev capnproto libcapnp-dev pkg-config
+            # Set environment variables for Rust/bindgen
+            LLVM_VERSION=$(ls /usr/lib/ | grep llvm | sort -V | tail -1)
+            echo "LIBCLANG_PATH=/usr/lib/$LLVM_VERSION/lib" >> $GITHUB_ENV
+            echo "CLANG_PATH=/usr/bin/clang" >> $GITHUB_ENV
+          elif [[ "$RUNNER_OS" == "macOS" ]]; then
+            brew install capnp pkg-config
+          elif [[ "$RUNNER_OS" == "Windows" ]]; then
+            # Install LLVM/Clang for Windows
+            choco install llvm -y
+            # Install vcpkg for Cap'n Proto on Windows
+            git clone https://github.com/Microsoft/vcpkg.git
+            cd vcpkg
+            ./bootstrap-vcpkg.bat
+            ./vcpkg install capnproto:x64-windows
+            echo "VCPKG_ROOT=$(pwd)" >> $GITHUB_ENV
+            echo "CMAKE_TOOLCHAIN_FILE=$(pwd)/scripts/buildsystems/vcpkg.cmake" >> $GITHUB_ENV
+            cd ..
+          fi
       - name: Setup Ruby
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: '3.2'
+      - name: Setup Rust
+        uses: dtolnay/rust-toolchain@stable
+      - name: Install Ruby dependencies
+        run: bundle install
       - name: Setup just
         uses: extractions/setup-just@v1
       - name: Build extension
@@ -33,10 +61,23 @@ jobs:
       - uses: actions/checkout@v4
         with:
           lfs: true
+      - name: Install system dependencies
+        shell: bash
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends libclang-dev capnproto libcapnp-dev pkg-config
+          # Set environment variables for Rust/bindgen
+          LLVM_VERSION=$(ls /usr/lib/ | grep llvm | sort -V | tail -1)
+          echo "LIBCLANG_PATH=/usr/lib/$LLVM_VERSION/lib" >> $GITHUB_ENV
+          echo "CLANG_PATH=/usr/bin/clang" >> $GITHUB_ENV
       - name: Setup Ruby
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: '3.2'
+      - name: Setup Rust
+        uses: dtolnay/rust-toolchain@stable
+      - name: Install Ruby dependencies
+        run: bundle install
       - name: Setup just
         uses: extractions/setup-just@v1
       - name: Build extension

flake.lock

@@ -1,7 +1,7 @@
 {
   description = "Development environment for codetracer-ruby-recorder";
 
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05";
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05";
 
   outputs = {
     self,
@@ -33,6 +33,8 @@
           # For build automation
           just
           git-lfs
+
+          capnproto # Required for the native tracer's Cap'n Proto serialization
         ] ++ pkgs.lib.optionals isLinux [
           # C standard library headers required for Ruby C extension compilation on Linux
           # Without this, build fails with "stdarg.h file not found" error

flake.nix

@@ -11,7 +11,7 @@ crate-type = ["cdylib"]
 
 [dependencies]
 rb-sys = "0.9"
-runtime_tracing = "0.10.0"
+runtime_tracing = "0.12.1"
 
 [build-dependencies]
 rb-sys-env = "0.2"

gems/codetracer-ruby-recorder/ext/native_tracer/Cargo.lock

@@ -133,6 +133,7 @@ fn value_type_id(val: &ValueRecord) -> runtime_tracing::TypeId {
         | Reference { type_id, .. }
         | Raw { type_id, .. }
         | Error { type_id, .. }
+        | BigInt { type_id, .. }
         | None { type_id } => *type_id,
         Cell { .. } => runtime_tracing::NONE_TYPE_ID,
     }
@@ -266,12 +267,15 @@ unsafe extern "C" fn disable_tracing(self_val: VALUE) -> VALUE {
     Qnil.into()
 }
 
-fn flush_to_dir(tracer: &Tracer, dir: &Path) -> Result<(), Box<dyn std::error::Error>> {
+fn flush_to_dir(tracer: &Tracer, dir: &Path, format: runtime_tracing::TraceEventsFileFormat) -> Result<(), Box<dyn std::error::Error>> {
     std::fs::create_dir_all(dir)?;
-    let events = dir.join("trace.json");
+    let events = match format {
+        runtime_tracing::TraceEventsFileFormat::Json => dir.join("trace.json"),
+        runtime_tracing::TraceEventsFileFormat::Binary => dir.join("trace.bin"),
+    };
     let metadata = dir.join("trace_metadata.json");
     let paths = dir.join("trace_paths.json");
-    tracer.store_trace_events(&events)?;
+    tracer.store_trace_events(&events, format)?;
     tracer.store_trace_metadata(&metadata)?;
     tracer.store_trace_paths(&paths)?;
     Ok(())
@@ -551,16 +555,32 @@ unsafe fn record_event(tracer: &mut Tracer, path: &str, line: i64, content: &str
     }));
 }
 
-unsafe extern "C" fn flush_trace(self_val: VALUE, out_dir: VALUE) -> VALUE {
+unsafe extern "C" fn flush_trace(self_val: VALUE, out_dir: VALUE, format: VALUE) -> VALUE {
     let recorder_ptr = get_recorder(self_val);
     let recorder = &mut *recorder_ptr;
     let ptr = RSTRING_PTR(out_dir) as *const u8;
     let len = RSTRING_LEN(out_dir) as usize;
     let slice = std::slice::from_raw_parts(ptr, len);
 
+    let fmt = if NIL_P(format) {
+        runtime_tracing::TraceEventsFileFormat::Json
+    } else if RB_SYMBOL_P(format) {
+        let id = rb_sym2id(format);
+        match CStr::from_ptr(rb_id2name(id)).to_str().unwrap_or("") {
+            "binary" | "bin" => runtime_tracing::TraceEventsFileFormat::Binary,
+            "json" => runtime_tracing::TraceEventsFileFormat::Json,
+            _ => {
+                rb_raise(rb_eIOError, b"Unknown format\0".as_ptr() as *const c_char);
+                runtime_tracing::TraceEventsFileFormat::Json
+            }
+        }
+    } else {
+        runtime_tracing::TraceEventsFileFormat::Json
+    };
+
     match std::str::from_utf8(slice) {
         Ok(path_str) => {
-            if let Err(e) = flush_to_dir(&recorder.tracer, Path::new(path_str)) {
+            if let Err(e) = flush_to_dir(&recorder.tracer, Path::new(path_str), fmt) {
                 rb_raise(rb_eIOError, b"Failed to flush trace: %s\0".as_ptr() as *const c_char, e.to_string().as_ptr() as *const c_char);
             }
         }
@@ -704,10 +724,10 @@ pub extern "C" fn Init_codetracer_ruby_recorder() {
             0
         );
         rb_define_method(
-            class, 
-            b"flush_trace\0".as_ptr() as *const c_char, 
-            Some(std::mem::transmute(flush_trace as *const ())), 
-            1
+            class,
+            b"flush_trace\0".as_ptr() as *const c_char,
+            Some(std::mem::transmute(flush_trace as *const ())),
+            2
         );
         rb_define_method(
             class, 

gems/codetracer-ruby-recorder/ext/native_tracer/Cargo.toml

@@ -15,6 +15,9 @@ def self.parse_argv_and_trace_ruby_file(argv)
         opts.on('-o DIR', '--out-dir DIR', 'Directory to write trace files') do |dir|
           options[:out_dir] = dir
         end
+        opts.on('-f FORMAT', '--format FORMAT', 'trace format: json or binary') do |fmt|
+          options[:format] = fmt
+        end
         opts.on('-h', '--help', 'Print this help') do
           puts opts
           exit
@@ -32,11 +35,12 @@ def self.parse_argv_and_trace_ruby_file(argv)
       program_args = argv.dup
 
       out_dir = options[:out_dir] || ENV['CODETRACER_RUBY_RECORDER_OUT_DIR'] || Dir.pwd
-      trace_ruby_file(program, out_dir, program_args)
+      format = (options[:format] || 'json').to_sym
+      trace_ruby_file(program, out_dir, program_args, format)
       0
     end
 
-    def self.trace_ruby_file(program, out_dir, program_args = [])
+    def self.trace_ruby_file(program, out_dir, program_args = [], format = :json)
       recorder = RubyRecorder.new
       return 1 unless recorder.available?
 
@@ -56,7 +60,7 @@ def self.trace_ruby_file(program, out_dir, program_args = [])
         ARGV.concat(original_argv)
 
         recorder.stop
-        recorder.flush_trace(out_dir)
+        recorder.flush_trace(out_dir, format)
       end
       0
     end
@@ -96,8 +100,8 @@ def record_event(path, line, content)
     end
 
     # Flush trace to output directory
-    def flush_trace(out_dir)
-      @recorder.flush_trace(out_dir) if @recorder
+    def flush_trace(out_dir, format = :json)
+      @recorder.flush_trace(out_dir, format) if @recorder
     end
 
     # Check if recorder is available