diff --git a/.agents/tasks/2025/08/18-1208-tracer-trait b/.agents/tasks/2025/08/18-1208-tracer-trait new file mode 100644 index 0000000..c1717fa --- /dev/null +++ b/.agents/tasks/2025/08/18-1208-tracer-trait @@ -0,0 +1,80 @@ +1. Create a trait for a tracer implemented using the Python monitoring API. The methods of the trait should correspond to the events that one can subscribe to via the API. + +Here's a sketch of the design of the trait. We want to support tracers which implement only some of the methods. + +``rs +use bitflags::bitflags; + +bitflags! { + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub struct EventMask: u32 { + const CALL = 1 << 0; + //CODEX: write this + ... + } +} + +#[non_exhaustive] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Event { + Call, + //CODEX: write this + ... + // Non-exhaustive for forward compatibility. +} + +pub trait Tracer: Send { + /// Tell the dispatcher what you want to receive. Default: nothing (fast path). + fn interest(&self) -> EventMask { EventMask::empty() } + + // Default no-ops let implementers pick only what they need. + fn on_call(&mut self, ...) {} + //CODEX: write this +} + +// Example tracer: only cares about CALL. +struct CallsOnly; +impl Tracer for CallsOnly { + fn interest(&self) -> EventMask { EventMask::CALL | ... } + fn on_call(&mut self, ...) { println!("call: {:?}", ...); } + ... +} + +// Dispatcher checks the mask to avoid vtable calls it doesn’t need. +pub struct Dispatcher { + tracer: Box, + mask: EventMask, +} + +impl Dispatcher { + pub fn new(tracer: Box) -> Self { + let mask = tracer.interest(); + Self { tracer, mask } + } + + pub fn dispatch_call(&mut self, ...) { + if self.mask.contains(EventMask::CALL) { + self.tracer.on_call(...); + } + + } + // CODEX: ... same for other events +} +`` + +2. Create code which takes a trait implementation and hooks it to the global tracing. Follow the design-docs for the specific API that needs to be implemented. + +3. Create a test implementation of the trait which prints text to stdout. Run the test implementation. Note that the test implementation should not be included in the final build +artefact, it will be used only for testing. + +4. Update the testing framework to be able to use `just test` also for Rust tests. Specifically we want to run our test implementation from point 2. using `just test` + +Refer to the design-docs folder for the current planned design. Add/update files in the folder to match what was implemented in this task. + + +--- FOLLOW UP TASK --- +1. In codetracer-python-recorder/Cargo.toml, move pyo3’s extension-module feature to an optional crate feature and enable it only for release builds. +2. Update Justfile (and CI scripts) to run cargo test --no-default-features so the test binary links with the Python C library. +3. Add pyo3 with auto-initialize under [dev-dependencies] if tests require the interpreter to be initialized automatically. +--- FOLLOW UP TASK --- +thread 'tracer_prints_on_call' panicked at tests/print_tracer.rs:21:51:\ncalled `Result::unwrap()` on an `Err` value: PyErr { type: , value: AttributeError("module 'sys' has no attribute 'monitoring'"), traceback: None }\nnote: run with `RUST_BACKTRACE=1` environment variable to display a backtrace\n\n\nfailures:\n tracer_prints_on_call\n\ntest result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s \ No newline at end of file diff --git a/Justfile b/Justfile index ce4a73b..2e67f20 100644 --- a/Justfile +++ b/Justfile @@ -32,9 +32,15 @@ dev: uv run --directory codetracer-python-recorder maturin develop --uv # Run unit tests of dev build -test: - uv run --group dev --group test pytest +test: cargo-test py-test + +# Run Rust unit tests without default features to link Python C library +cargo-test: + uv run cargo test --manifest-path codetracer-python-recorder/Cargo.toml --no-default-features +py-test: + uv run --group dev --group test pytest + # Run tests only on the pure recorder test-pure: uv run --group dev --group test pytest codetracer-pure-python-recorder diff --git a/codetracer-python-recorder/Cargo.lock b/codetracer-python-recorder/Cargo.lock index da12a63..e0c1fa0 100644 --- a/codetracer-python-recorder/Cargo.lock +++ b/codetracer-python-recorder/Cargo.lock @@ -68,6 +68,7 @@ checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" name = "codetracer-python-recorder" version = "0.1.0" dependencies = [ + "bitflags", "pyo3", "runtime_tracing", ] diff --git a/codetracer-python-recorder/Cargo.toml b/codetracer-python-recorder/Cargo.toml index e3c31c3..b25b86c 100644 --- a/codetracer-python-recorder/Cargo.toml +++ b/codetracer-python-recorder/Cargo.toml @@ -8,8 +8,16 @@ repository = "https://github.com/metacraft-labs/codetracer-python-recorder" [lib] name = "codetracer_python_recorder" -crate-type = ["cdylib"] +crate-type = ["cdylib", "rlib"] + +[features] +extension-module = ["pyo3/extension-module"] +default = ["extension-module"] [dependencies] -pyo3 = { version = "0.25.1", features = ["extension-module"] } -runtime_tracing = "0.14.0" \ No newline at end of file +pyo3 = { version = "0.25.1" } +runtime_tracing = "0.14.0" +bitflags = "2.4" + +[dev-dependencies] +pyo3 = { version = "0.25.1", features = ["auto-initialize"] } diff --git a/codetracer-python-recorder/src/lib.rs b/codetracer-python-recorder/src/lib.rs index 92782b5..5db531e 100644 --- a/codetracer-python-recorder/src/lib.rs +++ b/codetracer-python-recorder/src/lib.rs @@ -3,6 +3,9 @@ use std::sync::atomic::{AtomicBool, Ordering}; use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::*; +pub mod tracer; +pub use crate::tracer::{install_tracer, uninstall_tracer, EventMask, Tracer}; + /// Global flag tracking whether tracing is active. static ACTIVE: AtomicBool = AtomicBool::new(false); diff --git a/codetracer-python-recorder/src/tracer.rs b/codetracer-python-recorder/src/tracer.rs new file mode 100644 index 0000000..c4d66ec --- /dev/null +++ b/codetracer-python-recorder/src/tracer.rs @@ -0,0 +1,186 @@ +use std::sync::Mutex; + +use bitflags::bitflags; +use pyo3::{ + exceptions::{PyRuntimeError, PyTypeError}, + ffi, + prelude::*, + types::{PyModule, PyTuple}, +}; + +bitflags! { + /// Bitmask of monitoring events a tracer can subscribe to. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub struct EventMask: u32 { + /// Python function calls. + const CALL = 1 << 0; + /// Line execution events. + const LINE = 1 << 1; + } +} + +#[non_exhaustive] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Event { + Call, + Line, +} + +/// Trait implemented by tracing backends. +/// +/// Each method corresponds to an event from `sys.monitoring`. Default +/// implementations allow implementers to only handle the events they care +/// about. +pub trait Tracer: Send { + /// Return the set of events the tracer wants to receive. + fn interest(&self) -> EventMask { + EventMask::empty() + } + + /// Called on Python function calls. + fn on_call(&mut self, _py: Python<'_>, _frame: *mut ffi::PyFrameObject) {} + + /// Called on line execution. + fn on_line(&mut self, _py: Python<'_>, _frame: *mut ffi::PyFrameObject, _lineno: u32) {} +} + +/// Dispatcher routing events based on the tracer's interest mask. +pub struct Dispatcher { + tracer: Box, + mask: EventMask, +} + +impl Dispatcher { + pub fn new(tracer: Box) -> Self { + let mask = tracer.interest(); + Self { tracer, mask } + } + + pub fn dispatch_call(&mut self, py: Python<'_>, frame: *mut ffi::PyFrameObject) { + if self.mask.contains(EventMask::CALL) { + self.tracer.on_call(py, frame); + } + } + + pub fn dispatch_line(&mut self, py: Python<'_>, frame: *mut ffi::PyFrameObject, lineno: u32) { + if self.mask.contains(EventMask::LINE) { + self.tracer.on_line(py, frame, lineno); + } + } +} + +struct Global { + dispatcher: Dispatcher, + tool_id: u8, + callbacks: Vec>, +} + +static GLOBAL: Mutex> = Mutex::new(None); + +/// Install a tracer and hook it into Python's `sys.monitoring`. +pub fn install_tracer(py: Python<'_>, tracer: Box) -> PyResult<()> { + let mask = tracer.interest(); + let mut guard = GLOBAL.lock().unwrap(); + if guard.is_some() { + return Err(PyRuntimeError::new_err("tracer already installed")); + } + let monitoring = py.import("sys")?.getattr("monitoring")?; + // `use_tool_id` changed its signature between Python versions. + // Try calling it with the newer single-argument form first and fall back to + // the older two-argument variant if that fails with a `TypeError`. + const FALLBACK_ID: u8 = 5; + let tool_id: u8 = match monitoring.call_method1("use_tool_id", ("codetracer",)) { + Ok(obj) => obj.extract()?, + Err(err) => { + if err.is_instance_of::(py) { + monitoring.call_method1("use_tool_id", (FALLBACK_ID, "codetracer"))?; + FALLBACK_ID + } else { + return Err(err); + } + } + }; + let events = monitoring.getattr("events")?; + let module = PyModule::new(py, "_codetracer_callbacks")?; + + let mut callbacks = Vec::new(); + if mask.contains(EventMask::CALL) { + module.add_function(wrap_pyfunction!(callback_call, &module)?)?; + let cb = module.getattr("callback_call")?; + let ev = events.getattr("CALL")?; + monitoring.call_method("register_callback", (tool_id, ev, &cb), None)?; + callbacks.push(cb.unbind()); + } + if mask.contains(EventMask::LINE) { + module.add_function(wrap_pyfunction!(callback_line, &module)?)?; + let cb = module.getattr("callback_line")?; + let ev = events.getattr("LINE")?; + monitoring.call_method("register_callback", (tool_id, ev, &cb), None)?; + callbacks.push(cb.unbind()); + } + if let Err(err) = + monitoring.call_method("set_events", (tool_id, mask.bits(), mask.bits()), None) + { + if err.is_instance_of::(py) { + monitoring.call_method1("set_events", (tool_id, mask.bits()))?; + } else { + return Err(err); + } + } + + *guard = Some(Global { + dispatcher: Dispatcher::new(tracer), + tool_id, + callbacks, + }); + Ok(()) +} + +/// Remove the installed tracer if any. +pub fn uninstall_tracer(py: Python<'_>) -> PyResult<()> { + let mut guard = GLOBAL.lock().unwrap(); + if let Some(global) = guard.take() { + let monitoring = py.import("sys")?.getattr("monitoring")?; + let events = monitoring.getattr("events")?; + if global.dispatcher.mask.contains(EventMask::CALL) { + let ev = events.getattr("CALL")?; + monitoring.call_method("register_callback", (global.tool_id, ev, py.None()), None)?; + } + if global.dispatcher.mask.contains(EventMask::LINE) { + let ev = events.getattr("LINE")?; + monitoring.call_method("register_callback", (global.tool_id, ev, py.None()), None)?; + } + if let Err(err) = monitoring.call_method( + "set_events", + (global.tool_id, 0u32, global.dispatcher.mask.bits()), + None, + ) { + if err.is_instance_of::(py) { + monitoring.call_method1("set_events", (global.tool_id, 0u32))?; + } else { + return Err(err); + } + } + monitoring.call_method1("free_tool_id", (global.tool_id,))?; + } + Ok(()) +} + +#[pyfunction] +fn callback_call(py: Python<'_>, args: &Bound<'_, PyTuple>) -> PyResult<()> { + let frame = args.get_item(0)?.as_ptr() as *mut ffi::PyFrameObject; + if let Some(global) = GLOBAL.lock().unwrap().as_mut() { + global.dispatcher.dispatch_call(py, frame); + } + Ok(()) +} + +#[pyfunction] +fn callback_line(py: Python<'_>, args: &Bound<'_, PyTuple>) -> PyResult<()> { + let frame = args.get_item(0)?.as_ptr() as *mut ffi::PyFrameObject; + let lineno: u32 = args.get_item(1)?.extract()?; + if let Some(global) = GLOBAL.lock().unwrap().as_mut() { + global.dispatcher.dispatch_line(py, frame, lineno); + } + Ok(()) +} diff --git a/codetracer-python-recorder/tests/print_tracer.rs b/codetracer-python-recorder/tests/print_tracer.rs new file mode 100644 index 0000000..2e6bf8a --- /dev/null +++ b/codetracer-python-recorder/tests/print_tracer.rs @@ -0,0 +1,27 @@ +use codetracer_python_recorder::{install_tracer, uninstall_tracer, EventMask, Tracer}; +use pyo3::prelude::*; +use pyo3::ffi::PyFrameObject; +use std::ffi::CString; + +struct PrintTracer; + +impl Tracer for PrintTracer { + fn interest(&self) -> EventMask { + EventMask::CALL + } + + fn on_call(&mut self, _py: Python<'_>, _frame: *mut PyFrameObject) { + println!("call event"); + } +} + +#[test] +fn tracer_prints_on_call() { + Python::with_gil(|py| { + install_tracer(py, Box::new(PrintTracer)).unwrap(); + let code = CString::new("def foo():\n return 1\nfoo()").unwrap(); + py.run(&code, None, None).unwrap(); + uninstall_tracer(py).unwrap(); + }); +} + diff --git a/design-docs/design-001.md b/design-docs/design-001.md index 1945608..063e668 100644 --- a/design-docs/design-001.md +++ b/design-docs/design-001.md @@ -8,6 +8,13 @@ The tracer collects `sys.monitoring` events, converts them to `runtime_tracing` ## Architecture +### Tracer Abstraction +Rust code exposes a `Tracer` trait representing callbacks for Python +`sys.monitoring` events. Implementations advertise their desired events via an +`EventMask` bit flag returned from `interest`. A `Dispatcher` wraps a trait +object and forwards events only when the mask contains the corresponding flag, +allowing tracers to implement just the methods they care about. + ### Tool Initialization - Acquire a tool identifier via `sys.monitoring.use_tool_id`; store it for the lifetime of the tracer. ```rs