Value capture (#40)

* We snapshot all variables at each step of the script
* We don't step through internal Python code which is not associated
with a real Python file, thus reducing significantly the size of the
trace.

Author: tzanko-matev

.config/nextest.toml

@@ -0,0 +1,3 @@
+[profile.default]
+# Hard cap on wall-clock time for each individual test.
+test-timeout = "60s"

Justfile

@@ -38,7 +38,7 @@ test: cargo-test py-test
 
 # Run Rust unit tests without default features to link Python C library
 cargo-test:
-    uv run cargo test --manifest-path codetracer-python-recorder/Cargo.toml --no-default-features
+    uv run cargo nextest run --manifest-path codetracer-python-recorder/Cargo.toml --no-default-features
 
 py-test:
     uv run --group dev --group test pytest

codetracer-python-recorder/Cargo.lock

@@ -25,3 +25,4 @@ env_logger = "0.11"
 
 [dev-dependencies]
 pyo3 = { version = "0.25.1", features = ["auto-initialize"] }
+tempfile = "3.10"

codetracer-python-recorder/Cargo.toml

@@ -1,7 +1,7 @@
+use dashmap::DashMap;
 use once_cell::sync::OnceCell;
 use pyo3::prelude::*;
 use pyo3::types::PyCode;
-use dashmap::DashMap;
 use std::sync::Arc;
 
 /// A wrapper around Python `code` objects providing cached access to
@@ -50,73 +50,73 @@ impl CodeObjectWrapper {
     }
 
     pub fn filename<'py>(&'py self, py: Python<'py>) -> PyResult<&'py str> {
-        let value = self.cache.filename.get_or_try_init(|| -> PyResult<String> {
-            let s: String = self
-                .as_bound(py)
-                .getattr("co_filename")?
-                .extract()?;
-            Ok(s)
-        })?;
+        let value = self
+            .cache
+            .filename
+            .get_or_try_init(|| -> PyResult<String> {
+                let s: String = self.as_bound(py).getattr("co_filename")?.extract()?;
+                Ok(s)
+            })?;
         Ok(value.as_str())
     }
 
     pub fn qualname<'py>(&'py self, py: Python<'py>) -> PyResult<&'py str> {
-        let value = self.cache.qualname.get_or_try_init(|| -> PyResult<String> {
-            let s: String = self
-                .as_bound(py)
-                .getattr("co_qualname")?
-                .extract()?;
-            Ok(s)
-        })?;
+        let value = self
+            .cache
+            .qualname
+            .get_or_try_init(|| -> PyResult<String> {
+                let s: String = self.as_bound(py).getattr("co_qualname")?.extract()?;
+                Ok(s)
+            })?;
         Ok(value.as_str())
     }
 
     pub fn first_line(&self, py: Python<'_>) -> PyResult<u32> {
-        let value = *self.cache.firstlineno.get_or_try_init(|| -> PyResult<u32> {
-            let v: u32 = self
-                .as_bound(py)
-                .getattr("co_firstlineno")?
-                .extract()?;
-            Ok(v)
-        })?;
+        let value = *self
+            .cache
+            .firstlineno
+            .get_or_try_init(|| -> PyResult<u32> {
+                let v: u32 = self.as_bound(py).getattr("co_firstlineno")?.extract()?;
+                Ok(v)
+            })?;
         Ok(value)
     }
 
     pub fn arg_count(&self, py: Python<'_>) -> PyResult<u16> {
         let value = *self.cache.argcount.get_or_try_init(|| -> PyResult<u16> {
-            let v: u16 = self
-                .as_bound(py)
-                .getattr("co_argcount")?
-                .extract()?;
+            let v: u16 = self.as_bound(py).getattr("co_argcount")?.extract()?;
             Ok(v)
         })?;
         Ok(value)
     }
 
     pub fn flags(&self, py: Python<'_>) -> PyResult<u32> {
         let value = *self.cache.flags.get_or_try_init(|| -> PyResult<u32> {
-            let v: u32 = self
-                .as_bound(py)
-                .getattr("co_flags")?
-                .extract()?;
+            let v: u32 = self.as_bound(py).getattr("co_flags")?.extract()?;
             Ok(v)
         })?;
         Ok(value)
     }
 
     fn lines<'py>(&'py self, py: Python<'py>) -> PyResult<&'py [LineEntry]> {
-        let vec = self.cache.lines.get_or_try_init(|| -> PyResult<Vec<LineEntry>> {
-            let mut entries = Vec::new();
-            let iter = self.as_bound(py).call_method0("co_lines")?;
-            let iter = iter.try_iter()?;
-            for item in iter {
-                let (start, _end, line): (u32, u32, Option<u32>) = item?.extract()?;
-                if let Some(line) = line {
-                    entries.push(LineEntry { offset: start, line });
+        let vec = self
+            .cache
+            .lines
+            .get_or_try_init(|| -> PyResult<Vec<LineEntry>> {
+                let mut entries = Vec::new();
+                let iter = self.as_bound(py).call_method0("co_lines")?;
+                let iter = iter.try_iter()?;
+                for item in iter {
+                    let (start, _end, line): (u32, u32, Option<u32>) = item?.extract()?;
+                    if let Some(line) = line {
+                        entries.push(LineEntry {
+                            offset: start,
+                            line,
+                        });
+                    }
                 }
-            }
-            Ok(entries)
-        })?;
+                Ok(entries)
+            })?;
         Ok(vec.as_slice())
     }
 
@@ -161,4 +161,3 @@ impl CodeObjectRegistry {
         self.map.clear();
     }
 }
-

codetracer-python-recorder/src/code_object.rs

@@ -1,17 +1,23 @@
+//! Runtime tracing module backed by PyO3.
+//!
+//! Tracer implementations must return `CallbackResult` from every callback so they can
+//! signal when CPython should disable further monitoring for a location by propagating
+//! the `sys.monitoring.DISABLE` sentinel.
+
 use std::fs;
-use std::path::{PathBuf, Path};
+use std::path::Path;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Once;
 
 use pyo3::exceptions::PyRuntimeError;
 use pyo3::prelude::*;
-use std::fmt;
-
 pub mod code_object;
-pub mod tracer;
 mod runtime_tracer;
+pub mod tracer;
 pub use crate::code_object::{CodeObjectRegistry, CodeObjectWrapper};
-pub use crate::tracer::{install_tracer, uninstall_tracer, EventSet, Tracer};
+pub use crate::tracer::{
+    install_tracer, uninstall_tracer, CallbackOutcome, CallbackResult, EventSet, Tracer,
+};
 
 /// Global flag tracking whether tracing is active.
 static ACTIVE: AtomicBool = AtomicBool::new(false);
@@ -25,20 +31,14 @@ fn init_rust_logging_with_default(default_filter: &str) {
         let env = env_logger::Env::default().default_filter_or(default_filter);
         // Use a compact format with timestamps and targets to aid debugging.
         let mut builder = env_logger::Builder::from_env(env);
-        builder
-            .format_timestamp_micros()
-            .format_target(true);
+        builder.format_timestamp_micros().format_target(true);
         let _ = builder.try_init();
     });
 }
 
 /// Start tracing using sys.monitoring and runtime_tracing writer.
 #[pyfunction]
-fn start_tracing(
-    path: &str,
-    format: &str,
-    activation_path: Option<&str>,
-) -> PyResult<()> {
+fn start_tracing(path: &str, format: &str, activation_path: Option<&str>) -> PyResult<()> {
     // Ensure logging is ready before any tracer logs might be emitted.
     // Default only our crate to debug to avoid excessive verbosity from deps.
     init_rust_logging_with_default("codetracer_python_recorder=debug");
@@ -49,26 +49,31 @@ fn start_tracing(
     // Interpret `path` as a directory where trace files will be written.
     let out_dir = Path::new(path);
     if out_dir.exists() && !out_dir.is_dir() {
-        return Err(PyRuntimeError::new_err("trace path exists and is not a directory"));
+        return Err(PyRuntimeError::new_err(
+            "trace path exists and is not a directory",
+        ));
     }
     if !out_dir.exists() {
         // Best-effort create the directory tree
-        fs::create_dir_all(&out_dir)
-            .map_err(|e| PyRuntimeError::new_err(format!("failed to create trace directory: {}", e)))?;
+        fs::create_dir_all(&out_dir).map_err(|e| {
+            PyRuntimeError::new_err(format!("failed to create trace directory: {}", e))
+        })?;
     }
 
     // Map format string to enum
     let fmt = match format.to_lowercase().as_str() {
         "json" => runtime_tracing::TraceEventsFileFormat::Json,
         // Use BinaryV0 for "binary" to avoid streaming writer here.
-        "binary" | "binaryv0" | "binary_v0" | "b0" => runtime_tracing::TraceEventsFileFormat::BinaryV0,
+        "binary" | "binaryv0" | "binary_v0" | "b0" => {
+            runtime_tracing::TraceEventsFileFormat::BinaryV0
+        }
         //TODO AI! We need to assert! that the format is among the known values.
         other => {
             eprintln!("Unknown format '{}', defaulting to binary (v0)", other);
             runtime_tracing::TraceEventsFileFormat::BinaryV0
         }
     };
-    
+
     // Build output file paths inside the directory.
     let (events_path, meta_path, paths_path) = match fmt {
         runtime_tracing::TraceEventsFileFormat::Json => (
@@ -90,17 +95,10 @@ fn start_tracing(
         // Program and args: keep minimal; Python-side API stores full session info if needed
         let sys = py.import("sys")?;
         let argv = sys.getattr("argv")?;
-        let program: String = argv
-            .get_item(0)?
-            .extract::<String>()?;
+        let program: String = argv.get_item(0)?.extract::<String>()?;
         //TODO: Error-handling. What to do if argv is empty? Does this ever happen?
 
-        let mut tracer = runtime_tracer::RuntimeTracer::new(
-            &program,
-            &[],
-            fmt,
-            activation_path,
-        );
+        let mut tracer = runtime_tracer::RuntimeTracer::new(&program, &[], fmt, activation_path);
 
         // Start location: prefer activation path, otherwise best-effort argv[0]
         let start_path: &Path = activation_path.unwrap_or(Path::new(&program));