diff --git a/.envrc b/.envrc index 2a33d7a..ed864dd 100644 --- a/.envrc +++ b/.envrc @@ -1,3 +1,6 @@ watch_file nix/flake.nix watch_file nix/flake.lock +watch_file .venv/pyvenv.cfg use flake ./nix + +. .venv/bin/activate diff --git a/codetracer-python-recorder/src/logging.rs b/codetracer-python-recorder/src/logging.rs index ce28117..4808354 100644 --- a/codetracer-python-recorder/src/logging.rs +++ b/codetracer-python-recorder/src/logging.rs @@ -1,183 +1,33 @@ //! Diagnostics utilities: structured logging, metrics sinks, and error trailers. -use std::cell::Cell; -use std::collections::BTreeMap; -use std::fs::{File, OpenOptions}; -use std::io::{self, Write}; -use std::path::Path; -use std::str::FromStr; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Mutex, Once, RwLock}; -use std::time::{SystemTime, UNIX_EPOCH}; +mod logger; +mod metrics; +mod trailer; -use log::{LevelFilter, Log, Metadata, Record}; -use once_cell::sync::OnceCell; -use pyo3::prelude::*; -use recorder_errors::{ErrorCode, RecorderError}; -use serde::Serialize; -use uuid::Uuid; - -use crate::policy::RecorderPolicy; - -thread_local! { - static ERROR_CODE_OVERRIDE: Cell> = Cell::new(None); -} - -static LOGGER_INSTANCE: OnceCell<&'static RecorderLogger> = OnceCell::new(); -static INIT_LOGGER: Once = Once::new(); -static JSON_ERRORS_ENABLED: AtomicBool = AtomicBool::new(false); -static ERROR_TRAILER_WRITER: OnceCell>> = OnceCell::new(); -static METRICS_SINK: OnceCell> = OnceCell::new(); - -/// Structured logging initialisation applied on module import and during tracing start. -/// -/// The first caller installs a process-wide logger that emits JSON records containing -/// `run_id`, `trace_id`, and optional `error_code` fields. Subsequent calls are no-ops. -pub fn init_rust_logging_with_default(default_filter: &str) { - INIT_LOGGER.call_once(|| { - let default_spec = FilterSpec::parse(default_filter, LevelFilter::Warn) - .unwrap_or_else(|_| FilterSpec::new(LevelFilter::Warn)); +#[cfg(test)] +pub(crate) use logger::snapshot_run_and_trace; +pub use logger::{ + init_rust_logging_with_default, log_recorder_error, set_active_trace_id, with_error_code, + with_error_code_opt, +}; +#[allow(unused_imports)] +pub use metrics::{ + install_metrics, record_detach, record_dropped_event, record_panic, RecorderMetrics, +}; +pub use trailer::emit_error_trailer; - let initial_spec = std::env::var("RUST_LOG") - .ok() - .and_then(|spec| FilterSpec::parse(&spec, default_spec.global).ok()) - .unwrap_or_else(|| default_spec.clone()); +#[cfg(test)] +pub use metrics::test_support; +#[cfg(test)] +pub use trailer::set_error_trailer_writer_for_tests; - let logger = RecorderLogger::new(default_spec, initial_spec); - let leaked: &'static RecorderLogger = Box::leak(Box::new(logger)); - log::set_logger(leaked).expect("recorder logger already initialised"); - log::set_max_level(leaked.filter.read().expect("filter lock").max_level()); - let _ = LOGGER_INSTANCE.set(leaked); - }); -} +use crate::policy::RecorderPolicy; +use pyo3::types::PyAnyMethods; +use recorder_errors::ErrorCode; -/// Apply the current policy to logging and diagnostics outputs. pub fn apply_policy(policy: &RecorderPolicy) { - if let Some(logger) = LOGGER_INSTANCE.get() { - logger.apply_policy(policy); - } - JSON_ERRORS_ENABLED.store(policy.json_errors, Ordering::SeqCst); -} - -/// Scope a log emission with an explicit `ErrorCode` so the structured logger can attach it. -pub fn with_error_code(code: ErrorCode, op: F) -> R -where - F: FnOnce() -> R, -{ - ERROR_CODE_OVERRIDE.with(|cell| { - let previous = cell.replace(Some(code)); - let result = op(); - cell.set(previous); - result - }) -} - -/// Scope a log emission with an optional `ErrorCode` (falls back to `ERR_UNKNOWN`). -pub fn with_error_code_opt(code: Option, op: F) -> R -where - F: FnOnce() -> R, -{ - match code { - Some(code) => with_error_code(code, op), - None => with_error_code(ErrorCode::Unknown, op), - } -} - -/// Update the active trace identifier associated with subsequent log records. -pub fn set_active_trace_id(trace_id: Option) { - if let Some(logger) = LOGGER_INSTANCE.get() { - let mut guard = logger.trace_id.write().expect("trace id lock"); - *guard = trace_id; - } -} - -/// Log a structured representation of `err` for observability pipelines. -pub fn log_recorder_error(label: &str, err: &RecorderError) { - let message = build_error_text(err, Some(label)); - with_error_code(err.code, || { - log::error!(target: "codetracer_python_recorder::errors", "{}", message); - }); -} - -/// Emit a JSON error trailer on stderr when the policy requests it. -pub fn emit_error_trailer(err: &RecorderError) { - if !JSON_ERRORS_ENABLED.load(Ordering::SeqCst) { - return; - } - - let Some(logger) = LOGGER_INSTANCE.get() else { - return; - }; - - let trace_id = logger.trace_id.read().expect("trace id lock").clone(); - - let mut context = serde_json::Map::new(); - for (key, value) in &err.context { - context.insert((*key).to_string(), serde_json::Value::String(value.clone())); - } - - let payload = serde_json::json!({ - "run_id": logger.run_id, - "trace_id": trace_id, - "error_code": err.code.as_str(), - "error_kind": format!("{:?}", err.kind), - "message": err.message(), - "context": context, - }); - - if let Ok(mut bytes) = serde_json::to_vec(&payload) { - bytes.push(b'\n'); - if let Some(writer) = ERROR_TRAILER_WRITER.get() { - let mut guard = writer.lock().expect("error trailer writer lock"); - let _ = guard.write_all(&bytes); - let _ = guard.flush(); - } else { - let mut stderr = io::stderr().lock(); - let _ = stderr.write_all(&bytes); - let _ = stderr.flush(); - } - } -} - -/// Metrics interface allowing pluggable sinks (default: no-op). -pub trait RecorderMetrics: Send + Sync { - /// Record that an event stream was dropped for the provided reason. - fn record_dropped_event(&self, _reason: &'static str) {} - /// Record that tracing detached, optionally linked to an error code. - fn record_detach(&self, _reason: &'static str, _error_code: Option<&str>) {} - /// Record that a panic was caught and converted into an error. - fn record_panic(&self, _label: &'static str) {} -} - -struct NoopMetrics; - -impl RecorderMetrics for NoopMetrics {} - -fn metrics_sink() -> &'static dyn RecorderMetrics { - METRICS_SINK - .get_or_init(|| Box::new(NoopMetrics) as Box) - .as_ref() -} - -/// Install a custom metrics sink. Intended for embedding or tests. -#[cfg_attr(not(test), allow(dead_code))] -pub fn install_metrics(metrics: Box) -> Result<(), Box> { - METRICS_SINK.set(metrics) -} - -/// Record that we abandoned a monitoring location (e.g., synthetic filename). -pub fn record_dropped_event(reason: &'static str) { - metrics_sink().record_dropped_event(reason); -} - -/// Record that we detached per-policy or due to unrecoverable failure. -pub fn record_detach(reason: &'static str, error_code: Option<&str>) { - metrics_sink().record_detach(reason, error_code); -} - -/// Record that we caught a panic at the FFI boundary. -pub fn record_panic(label: &'static str) { - metrics_sink().record_panic(label); + logger::apply_logger_policy(policy); + trailer::set_json_errors_enabled(policy.json_errors); } /// Attempt to read an `ErrorCode` attribute from a Python exception value. @@ -188,369 +38,13 @@ pub fn error_code_from_pyerr(py: pyo3::Python<'_>, err: &pyo3::PyErr) -> Option< ErrorCode::parse(&code_str) } -/// Provide a helper for tests to override the error trailer destination. -#[cfg(test)] -pub fn set_error_trailer_writer_for_tests(writer: Box) { - let _ = ERROR_TRAILER_WRITER.set(Mutex::new(writer)); -} - -struct RecorderLogger { - run_id: String, - trace_id: RwLock>, - default_filter: FilterSpec, - filter: RwLock, - writer: Mutex, -} - -impl RecorderLogger { - fn new(default_filter: FilterSpec, initial: FilterSpec) -> Self { - Self { - run_id: Uuid::new_v4().to_string(), - trace_id: RwLock::new(None), - writer: Mutex::new(Destination::Stderr), - filter: RwLock::new(initial), - default_filter, - } - } - - fn apply_policy(&self, policy: &RecorderPolicy) { - let new_filter = match policy.log_level.as_deref() { - Some(spec) if !spec.trim().is_empty() => { - match FilterSpec::parse(spec, self.default_filter.global) { - Ok(parsed) => parsed, - Err(_) => { - with_error_code(ErrorCode::InvalidPolicyValue, || { - log::warn!( - target: "codetracer_python_recorder::logging", - "invalid log level filter '{}'; reverting to default", - spec - ); - }); - self.default_filter.clone() - } - } - } - _ => self.default_filter.clone(), - }; - - { - let mut guard = self.filter.write().expect("filter lock"); - *guard = new_filter.clone(); - } - log::set_max_level(new_filter.max_level()); - - match policy.log_file.as_ref() { - Some(path) => match open_log_file(path) { - Ok(file) => { - *self.writer.lock().expect("writer lock") = Destination::File(file); - } - Err(err) => { - with_error_code(ErrorCode::Io, || { - log::warn!( - target: "codetracer_python_recorder::logging", - "failed to open log file '{}': {}", - path.display(), - err - ); - }); - *self.writer.lock().expect("writer lock") = Destination::Stderr; - } - }, - None => { - *self.writer.lock().expect("writer lock") = Destination::Stderr; - } - } - } - - fn enabled(&self, metadata: &Metadata<'_>) -> bool { - self.filter.read().expect("filter lock").allows(metadata) - } - - fn write_entry(&self, entry: &LogEntry<'_>) { - match serde_json::to_vec(entry) { - Ok(mut bytes) => { - bytes.push(b'\n'); - if let Err(err) = self.writer.lock().expect("writer lock").write_all(&bytes) { - let mut stderr = io::stderr().lock(); - let _ = stderr.write_all(&bytes); - let _ = writeln!( - stderr, - "{{\"run_id\":\"{}\",\"message\":\"logger write failure: {}\"}}", - self.run_id, err - ); - } - } - Err(_) => { - // Fallback to plain message if serialization fails - let mut stderr = io::stderr().lock(); - let _ = writeln!( - stderr, - "{{\"run_id\":\"{}\",\"message\":\"failed to encode log entry\"}}", - self.run_id - ); - } - } - } -} - -impl Log for RecorderLogger { - fn enabled(&self, metadata: &Metadata<'_>) -> bool { - self.enabled(metadata) - } - - fn log(&self, record: &Record<'_>) { - if !self.enabled(record.metadata()) { - return; - } - - let thread_code = ERROR_CODE_OVERRIDE.with(|cell| cell.get()); - let error_code = thread_code.map(|code| code.as_str().to_string()); - let mut fields = BTreeMap::new(); - if let Some(code) = error_code.as_ref() { - fields.insert( - "error_code".to_string(), - serde_json::Value::String(code.clone()), - ); - } - - let trace_id = self.trace_id.read().expect("trace id lock").clone(); - - let entry = LogEntry { - ts_micros: current_timestamp_micros(), - level: record.level().as_str(), - target: record.target(), - run_id: &self.run_id, - trace_id: trace_id.as_deref(), - message: record.args().to_string(), - error_code, - module_path: record.module_path(), - file: record.file(), - line: record.line(), - fields, - }; - - self.write_entry(&entry); - } - - fn flush(&self) { - let _ = self.writer.lock().expect("writer lock").flush(); - } -} - -#[derive(Clone)] -struct FilterSpec { - global: LevelFilter, - targets: Vec<(String, LevelFilter)>, -} - -impl FilterSpec { - fn new(global: LevelFilter) -> Self { - Self { - global, - targets: Vec::new(), - } - } - - fn parse(spec: &str, default_global: LevelFilter) -> Result { - let mut filter = Self::new(default_global); - for part in spec.split(',') { - let trimmed = part.trim(); - if trimmed.is_empty() { - continue; - } - if let Some((target, level)) = trimmed.split_once('=') { - let lvl = LevelFilter::from_str(level.trim()).map_err(|_| ())?; - filter.targets.push((target.trim().to_string(), lvl)); - } else { - filter.global = LevelFilter::from_str(trimmed).map_err(|_| ())?; - } - } - Ok(filter) - } - - fn allows(&self, metadata: &Metadata<'_>) -> bool { - let mut allowed = self.global; - let mut matched_len = 0usize; - let target = metadata.target(); - for (pattern, level) in &self.targets { - if target == pattern - || target.starts_with(pattern) && target.chars().nth(pattern.len()) == Some(':') - { - if pattern.len() > matched_len { - matched_len = pattern.len(); - allowed = *level; - } - } - } - allowed >= metadata.level().to_level_filter() - } - - fn max_level(&self) -> LevelFilter { - self.targets - .iter() - .fold(self.global, |acc, (_, lvl)| acc.max(*lvl)) - } -} - -#[derive(Serialize)] -struct LogEntry<'a> { - ts_micros: i128, - level: &'a str, - target: &'a str, - run_id: &'a str, - #[serde(skip_serializing_if = "Option::is_none")] - trace_id: Option<&'a str>, - message: String, - #[serde(skip_serializing_if = "Option::is_none")] - error_code: Option, - #[serde(skip_serializing_if = "Option::is_none")] - module_path: Option<&'a str>, - #[serde(skip_serializing_if = "Option::is_none")] - file: Option<&'a str>, - #[serde(skip_serializing_if = "Option::is_none")] - line: Option, - #[serde(skip_serializing_if = "BTreeMap::is_empty")] - fields: BTreeMap, -} - -fn current_timestamp_micros() -> i128 { - match SystemTime::now().duration_since(UNIX_EPOCH) { - Ok(duration) => { - let secs = duration.as_secs() as i128; - let micros = duration.subsec_micros() as i128; - secs * 1_000_000 + micros - } - Err(_) => 0, - } -} - -enum Destination { - Stderr, - File(File), -} - -impl Destination { - fn write_all(&mut self, bytes: &[u8]) -> io::Result<()> { - match self { - Destination::Stderr => { - let mut stderr = io::stderr().lock(); - stderr.write_all(bytes)?; - stderr.flush() - } - Destination::File(file) => { - file.write_all(bytes)?; - file.flush() - } - } - } - - fn flush(&mut self) -> io::Result<()> { - match self { - Destination::Stderr => io::stderr().lock().flush(), - Destination::File(file) => file.flush(), - } - } -} - -fn open_log_file(path: &Path) -> io::Result { - OpenOptions::new().create(true).append(true).open(path) -} - -fn build_error_text(err: &RecorderError, label: Option<&str>) -> String { - let mut text = String::new(); - if let Some(label) = label { - text.push_str(label); - text.push_str(": "); - } - text.push_str(err.message()); - if !err.context.is_empty() { - text.push_str(" ("); - let mut first = true; - for (key, value) in &err.context { - if !first { - text.push_str(", "); - } - first = false; - text.push_str(key); - text.push('='); - text.push_str(value); - } - text.push(')'); - } - text -} - -#[cfg(test)] -pub mod test_support { - use super::*; - use once_cell::sync::OnceCell; - use std::sync::{Arc, Mutex}; - - #[derive(Clone, Default)] - pub struct CapturingMetrics { - events: Arc>>, - } - - #[derive(Clone, Debug, PartialEq, Eq)] - pub enum MetricEvent { - Dropped(&'static str), - Detach(&'static str, Option), - Panic(&'static str), - } - - impl CapturingMetrics { - pub fn take(&self) -> Vec { - let mut guard = self.events.lock().expect("metrics events lock"); - let events = guard.clone(); - guard.clear(); - events - } - } - - impl RecorderMetrics for CapturingMetrics { - fn record_dropped_event(&self, reason: &'static str) { - self.events - .lock() - .expect("metrics events lock") - .push(MetricEvent::Dropped(reason)); - } - - fn record_detach(&self, reason: &'static str, error_code: Option<&str>) { - self.events - .lock() - .expect("metrics events lock") - .push(MetricEvent::Detach( - reason, - error_code.map(|s| s.to_string()), - )); - } - - fn record_panic(&self, label: &'static str) { - self.events - .lock() - .expect("metrics events lock") - .push(MetricEvent::Panic(label)); - } - } - - static CAPTURING: OnceCell = OnceCell::new(); - - pub fn install() -> &'static CapturingMetrics { - CAPTURING.get_or_init(|| { - let metrics = CapturingMetrics::default(); - let _ = super::install_metrics(Box::new(metrics.clone())); - metrics - }) - } -} - #[cfg(test)] mod tests { use super::*; - use crate::policy::RecorderPolicy; use once_cell::sync::OnceCell; - use recorder_errors::{ErrorCode, ErrorKind}; + use recorder_errors::{ErrorCode, ErrorKind, RecorderError}; use serde_json::Value; + use std::io::{self, Write}; use std::sync::{Arc, Mutex}; use tempfile::tempdir; @@ -558,8 +52,8 @@ mod tests { init_rust_logging_with_default("codetracer_python_recorder=debug"); } - fn build_policy() -> RecorderPolicy { - RecorderPolicy::default() + fn build_policy() -> crate::policy::RecorderPolicy { + crate::policy::RecorderPolicy::default() } struct VecWriter { @@ -615,7 +109,7 @@ mod tests { Some("sample message") ); - apply_policy(&RecorderPolicy::default()); + apply_policy(&crate::policy::RecorderPolicy::default()); } #[test] diff --git a/codetracer-python-recorder/src/logging/logger.rs b/codetracer-python-recorder/src/logging/logger.rs new file mode 100644 index 0000000..17cdc9e --- /dev/null +++ b/codetracer-python-recorder/src/logging/logger.rs @@ -0,0 +1,379 @@ +use std::cell::Cell; +use std::collections::BTreeMap; +use std::fs::{File, OpenOptions}; +use std::io::{self, Write}; +use std::path::Path; +use std::str::FromStr; +use std::sync::{Mutex, Once, RwLock}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use log::{LevelFilter, Log, Metadata, Record}; +use once_cell::sync::OnceCell; +use recorder_errors::{ErrorCode, RecorderError}; +use serde::Serialize; +use serde_json::Value; +use uuid::Uuid; + +use crate::policy::RecorderPolicy; + +thread_local! { + static ERROR_CODE_OVERRIDE: Cell> = Cell::new(None); +} + +static LOGGER_INSTANCE: OnceCell<&'static RecorderLogger> = OnceCell::new(); +static INIT_LOGGER: Once = Once::new(); + +pub fn init_rust_logging_with_default(default_filter: &str) { + INIT_LOGGER.call_once(|| { + let default_spec = FilterSpec::parse(default_filter, LevelFilter::Warn) + .unwrap_or_else(|_| FilterSpec::new(LevelFilter::Warn)); + + let initial_spec = std::env::var("RUST_LOG") + .ok() + .and_then(|spec| FilterSpec::parse(&spec, default_spec.global).ok()) + .unwrap_or_else(|| default_spec.clone()); + + let logger = RecorderLogger::new(default_spec, initial_spec); + let leaked: &'static RecorderLogger = Box::leak(Box::new(logger)); + log::set_logger(leaked).expect("recorder logger already initialised"); + log::set_max_level(leaked.filter.read().expect("filter lock").max_level()); + let _ = LOGGER_INSTANCE.set(leaked); + }); +} + +pub(crate) fn apply_logger_policy(policy: &RecorderPolicy) { + if let Some(logger) = LOGGER_INSTANCE.get() { + logger.apply_policy(policy); + } +} + +pub fn with_error_code(code: ErrorCode, op: F) -> R +where + F: FnOnce() -> R, +{ + ERROR_CODE_OVERRIDE.with(|cell| { + let previous = cell.replace(Some(code)); + let result = op(); + cell.set(previous); + result + }) +} + +pub fn with_error_code_opt(code: Option, op: F) -> R +where + F: FnOnce() -> R, +{ + match code { + Some(code) => with_error_code(code, op), + None => with_error_code(ErrorCode::Unknown, op), + } +} + +pub fn set_active_trace_id(trace_id: Option) { + if let Some(logger) = LOGGER_INSTANCE.get() { + let mut guard = logger.trace_id.write().expect("trace id lock"); + *guard = trace_id; + } +} + +pub fn log_recorder_error(label: &str, err: &RecorderError) { + let message = build_error_text(err, Some(label)); + with_error_code(err.code, || { + log::error!(target: "codetracer_python_recorder::errors", "{}", message); + }); +} + +pub(crate) fn snapshot_run_and_trace() -> Option<(String, Option)> { + LOGGER_INSTANCE + .get() + .map(|logger| (logger.run_id.clone(), logger.snapshot_trace_id())) +} + +struct RecorderLogger { + run_id: String, + trace_id: RwLock>, + default_filter: FilterSpec, + filter: RwLock, + writer: Mutex, +} + +impl RecorderLogger { + fn new(default_filter: FilterSpec, initial: FilterSpec) -> Self { + Self { + run_id: Uuid::new_v4().to_string(), + trace_id: RwLock::new(None), + writer: Mutex::new(Destination::Stderr), + filter: RwLock::new(initial), + default_filter, + } + } + + fn apply_policy(&self, policy: &RecorderPolicy) { + let new_filter = match policy.log_level.as_deref() { + Some(spec) if !spec.trim().is_empty() => { + match FilterSpec::parse(spec, self.default_filter.global) { + Ok(parsed) => parsed, + Err(_) => { + with_error_code(ErrorCode::InvalidPolicyValue, || { + log::warn!( + target: "codetracer_python_recorder::logging", + "invalid log level filter '{}'; reverting to default", + spec + ); + }); + self.default_filter.clone() + } + } + } + _ => self.default_filter.clone(), + }; + + { + let mut guard = self.filter.write().expect("filter lock"); + *guard = new_filter.clone(); + } + log::set_max_level(new_filter.max_level()); + + match policy.log_file.as_ref() { + Some(path) => match open_log_file(path) { + Ok(file) => { + *self.writer.lock().expect("writer lock") = Destination::File(file); + } + Err(err) => { + with_error_code(ErrorCode::Io, || { + log::warn!( + target: "codetracer_python_recorder::logging", + "failed to open log file '{}': {}", + path.display(), + err + ); + }); + *self.writer.lock().expect("writer lock") = Destination::Stderr; + } + }, + None => { + *self.writer.lock().expect("writer lock") = Destination::Stderr; + } + } + } + + fn enabled(&self, metadata: &Metadata<'_>) -> bool { + self.filter.read().expect("filter lock").allows(metadata) + } + + fn write_entry(&self, entry: &LogEntry<'_>) { + match serde_json::to_vec(entry) { + Ok(mut bytes) => { + bytes.push(b'\n'); + if let Err(err) = self.writer.lock().expect("writer lock").write_all(&bytes) { + let mut stderr = io::stderr().lock(); + let _ = stderr.write_all(&bytes); + let _ = writeln!( + stderr, + "{{\"run_id\":\"{}\",\"message\":\"logger write failure: {}\"}}", + self.run_id, err + ); + } + } + Err(_) => { + let mut stderr = io::stderr().lock(); + let _ = writeln!( + stderr, + "{{\"run_id\":\"{}\",\"message\":\"failed to encode log entry\"}}", + self.run_id + ); + } + } + } + + fn snapshot_trace_id(&self) -> Option { + self.trace_id.read().expect("trace id lock").clone() + } +} + +impl Log for RecorderLogger { + fn enabled(&self, metadata: &Metadata<'_>) -> bool { + self.enabled(metadata) + } + + fn log(&self, record: &Record<'_>) { + if !self.enabled(record.metadata()) { + return; + } + + let thread_code = ERROR_CODE_OVERRIDE.with(|cell| cell.get()); + let error_code = thread_code.map(|code| code.as_str().to_string()); + let mut fields = BTreeMap::new(); + if let Some(code) = error_code.as_ref() { + fields.insert( + "error_code".to_string(), + serde_json::Value::String(code.clone()), + ); + } + + let trace_id = self.trace_id.read().expect("trace id lock").clone(); + + let entry = LogEntry { + ts_micros: current_timestamp_micros(), + level: record.level().as_str(), + target: record.target(), + run_id: &self.run_id, + trace_id: trace_id.as_deref(), + message: record.args().to_string(), + error_code, + module_path: record.module_path(), + file: record.file(), + line: record.line(), + fields, + }; + + self.write_entry(&entry); + } + + fn flush(&self) { + let _ = self.writer.lock().expect("writer lock").flush(); + } +} + +#[derive(Clone)] +struct FilterSpec { + global: LevelFilter, + targets: Vec<(String, LevelFilter)>, +} + +impl FilterSpec { + fn new(global: LevelFilter) -> Self { + Self { + global, + targets: Vec::new(), + } + } + + fn parse(spec: &str, default_global: LevelFilter) -> Result { + let mut filter = Self::new(default_global); + for part in spec.split(',') { + let trimmed = part.trim(); + if trimmed.is_empty() { + continue; + } + if let Some((target, level)) = trimmed.split_once('=') { + let lvl = LevelFilter::from_str(level.trim()).map_err(|_| ())?; + filter.targets.push((target.trim().to_string(), lvl)); + } else { + filter.global = LevelFilter::from_str(trimmed).map_err(|_| ())?; + } + } + Ok(filter) + } + + fn allows(&self, metadata: &Metadata<'_>) -> bool { + let mut allowed = self.global; + let mut matched_len = 0usize; + let target = metadata.target(); + for (pattern, level) in &self.targets { + if target == pattern + || target.starts_with(pattern) && target.chars().nth(pattern.len()) == Some(':') + { + if pattern.len() > matched_len { + matched_len = pattern.len(); + allowed = *level; + } + } + } + allowed >= metadata.level().to_level_filter() + } + + fn max_level(&self) -> LevelFilter { + self.targets + .iter() + .fold(self.global, |acc, (_, lvl)| acc.max(*lvl)) + } +} + +#[derive(Serialize)] +struct LogEntry<'a> { + ts_micros: i128, + level: &'a str, + target: &'a str, + run_id: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + trace_id: Option<&'a str>, + message: String, + #[serde(skip_serializing_if = "Option::is_none")] + error_code: Option, + #[serde(skip_serializing_if = "Option::is_none")] + module_path: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + file: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + line: Option, + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + fields: BTreeMap, +} + +fn current_timestamp_micros() -> i128 { + match SystemTime::now().duration_since(UNIX_EPOCH) { + Ok(duration) => { + let secs = duration.as_secs() as i128; + let micros = duration.subsec_micros() as i128; + secs * 1_000_000 + micros + } + Err(_) => 0, + } +} + +enum Destination { + Stderr, + File(File), +} + +impl Destination { + fn write_all(&mut self, bytes: &[u8]) -> io::Result<()> { + match self { + Destination::Stderr => { + let mut stderr = io::stderr().lock(); + stderr.write_all(bytes)?; + stderr.flush() + } + Destination::File(file) => { + file.write_all(bytes)?; + file.flush() + } + } + } + + fn flush(&mut self) -> io::Result<()> { + match self { + Destination::Stderr => io::stderr().lock().flush(), + Destination::File(file) => file.flush(), + } + } +} + +fn open_log_file(path: &Path) -> io::Result { + OpenOptions::new().create(true).append(true).open(path) +} + +fn build_error_text(err: &RecorderError, label: Option<&str>) -> String { + let mut text = String::new(); + if let Some(label) = label { + text.push_str(label); + text.push_str(": "); + } + text.push_str(err.message()); + if !err.context.is_empty() { + text.push_str(" ("); + let mut first = true; + for (key, value) in &err.context { + if !first { + text.push_str(", "); + } + first = false; + text.push_str(key); + text.push('='); + text.push_str(value); + } + text.push(')'); + } + text +} diff --git a/codetracer-python-recorder/src/logging/metrics.rs b/codetracer-python-recorder/src/logging/metrics.rs new file mode 100644 index 0000000..a70fa0f --- /dev/null +++ b/codetracer-python-recorder/src/logging/metrics.rs @@ -0,0 +1,108 @@ +use once_cell::sync::OnceCell; + +/// Metrics interface allowing pluggable sinks (default: no-op). +pub trait RecorderMetrics: Send + Sync { + /// Record that an event stream was dropped for the provided reason. + fn record_dropped_event(&self, _reason: &'static str) {} + /// Record that tracing detached, optionally linked to an error code. + fn record_detach(&self, _reason: &'static str, _error_code: Option<&str>) {} + /// Record that a panic was caught and converted into an error. + fn record_panic(&self, _label: &'static str) {} +} + +struct NoopMetrics; + +impl RecorderMetrics for NoopMetrics {} + +static METRICS_SINK: OnceCell> = OnceCell::new(); + +fn metrics_sink() -> &'static dyn RecorderMetrics { + METRICS_SINK + .get_or_init(|| Box::new(NoopMetrics) as Box) + .as_ref() +} + +/// Install a custom metrics sink. Intended for embedding or tests. +#[cfg_attr(not(test), allow(dead_code))] +pub fn install_metrics(metrics: Box) -> Result<(), Box> { + METRICS_SINK.set(metrics) +} + +/// Record that we abandoned a monitoring location (e.g., synthetic filename). +pub fn record_dropped_event(reason: &'static str) { + metrics_sink().record_dropped_event(reason); +} + +/// Record that we detached per-policy or due to unrecoverable failure. +pub fn record_detach(reason: &'static str, error_code: Option<&str>) { + metrics_sink().record_detach(reason, error_code); +} + +/// Record that we caught a panic at the FFI boundary. +pub fn record_panic(label: &'static str) { + metrics_sink().record_panic(label); +} + +#[cfg(test)] +pub mod test_support { + use super::*; + use once_cell::sync::OnceCell; + use std::sync::{Arc, Mutex}; + + #[derive(Clone, Default)] + pub struct CapturingMetrics { + events: Arc>>, + } + + #[derive(Clone, Debug, PartialEq, Eq)] + pub enum MetricEvent { + Dropped(&'static str), + Detach(&'static str, Option), + Panic(&'static str), + } + + impl CapturingMetrics { + pub fn take(&self) -> Vec { + let mut guard = self.events.lock().expect("metrics events lock"); + let events = guard.clone(); + guard.clear(); + events + } + } + + impl RecorderMetrics for CapturingMetrics { + fn record_dropped_event(&self, reason: &'static str) { + self.events + .lock() + .expect("metrics events lock") + .push(MetricEvent::Dropped(reason)); + } + + fn record_detach(&self, reason: &'static str, error_code: Option<&str>) { + self.events + .lock() + .expect("metrics events lock") + .push(MetricEvent::Detach( + reason, + error_code.map(|s| s.to_string()), + )); + } + + fn record_panic(&self, label: &'static str) { + self.events + .lock() + .expect("metrics events lock") + .push(MetricEvent::Panic(label)); + } + } + + static CAPTURING: OnceCell = OnceCell::new(); + + pub fn install() -> &'static CapturingMetrics { + CAPTURING.get_or_init(|| { + let metrics = CapturingMetrics::default(); + let _ = super::install_metrics(Box::new(metrics.clone())); + metrics + }) + } +} diff --git a/codetracer-python-recorder/src/logging/trailer.rs b/codetracer-python-recorder/src/logging/trailer.rs new file mode 100644 index 0000000..bed3210 --- /dev/null +++ b/codetracer-python-recorder/src/logging/trailer.rs @@ -0,0 +1,57 @@ +use std::io::{self, Write}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Mutex; + +use once_cell::sync::OnceCell; +use recorder_errors::RecorderError; + +use super::logger; + +static JSON_ERRORS_ENABLED: AtomicBool = AtomicBool::new(false); +static ERROR_TRAILER_WRITER: OnceCell>> = OnceCell::new(); + +pub(crate) fn set_json_errors_enabled(enabled: bool) { + JSON_ERRORS_ENABLED.store(enabled, Ordering::SeqCst); +} + +pub fn emit_error_trailer(err: &RecorderError) { + if !JSON_ERRORS_ENABLED.load(Ordering::SeqCst) { + return; + } + + let Some((run_id, trace_id)) = logger::snapshot_run_and_trace() else { + return; + }; + + let mut context = serde_json::Map::new(); + for (key, value) in &err.context { + context.insert((*key).to_string(), serde_json::Value::String(value.clone())); + } + + let payload = serde_json::json!({ + "run_id": run_id, + "trace_id": trace_id, + "error_code": err.code.as_str(), + "error_kind": format!("{:?}", err.kind), + "message": err.message(), + "context": context, + }); + + if let Ok(mut bytes) = serde_json::to_vec(&payload) { + bytes.push(b'\n'); + if let Some(writer) = ERROR_TRAILER_WRITER.get() { + let mut guard = writer.lock().expect("error trailer writer lock"); + let _ = guard.write_all(&bytes); + let _ = guard.flush(); + } else { + let mut stderr = io::stderr().lock(); + let _ = stderr.write_all(&bytes); + let _ = stderr.flush(); + } + } +} + +#[cfg(test)] +pub fn set_error_trailer_writer_for_tests(writer: Box) { + let _ = ERROR_TRAILER_WRITER.set(Mutex::new(writer)); +} diff --git a/codetracer-python-recorder/src/monitoring/api.rs b/codetracer-python-recorder/src/monitoring/api.rs new file mode 100644 index 0000000..0bc02c2 --- /dev/null +++ b/codetracer-python-recorder/src/monitoring/api.rs @@ -0,0 +1,230 @@ +//! Monitoring API abstractions. + +use std::any::Any; + +use crate::code_object::CodeObjectWrapper; +use pyo3::prelude::*; +use pyo3::types::PyAny; + +use super::{CallbackOutcome, CallbackResult, EventSet, MonitoringEvents, NO_EVENTS}; + +/// Trait implemented by tracing backends. +/// +/// Each method corresponds to an event from `sys.monitoring`. Default +/// implementations allow implementers to only handle the events they care +/// about. +/// +/// Every callback returns a `CallbackResult` so implementations can propagate +/// Python exceptions or request that CPython disables future events for a +/// location by yielding the `CallbackOutcome::DisableLocation` sentinel. +pub trait Tracer: Send + Any { + /// Downcast support for implementations that need to be accessed + /// behind a `Box` (e.g., for flushing/finishing). + fn as_any(&mut self) -> &mut dyn Any + where + Self: 'static, + Self: Sized, + { + self + } + + /// Return the set of events the tracer wants to receive. + fn interest(&self, _events: &MonitoringEvents) -> EventSet { + NO_EVENTS + } + + /// Called on Python function calls. + fn on_call( + &mut self, + _py: Python<'_>, + _code: &CodeObjectWrapper, + _offset: i32, + _callable: &Bound<'_, PyAny>, + _arg0: Option<&Bound<'_, PyAny>>, + ) -> CallbackResult { + Ok(CallbackOutcome::Continue) + } + + /// Called on line execution. + fn on_line( + &mut self, + _py: Python<'_>, + _code: &CodeObjectWrapper, + _lineno: u32, + ) -> CallbackResult { + Ok(CallbackOutcome::Continue) + } + + /// Called when an instruction is about to be executed (by offset). + fn on_instruction( + &mut self, + _py: Python<'_>, + _code: &CodeObjectWrapper, + _offset: i32, + ) -> CallbackResult { + Ok(CallbackOutcome::Continue) + } + + /// Called when a jump in the control flow graph is made. + fn on_jump( + &mut self, + _py: Python<'_>, + _code: &CodeObjectWrapper, + _offset: i32, + _destination_offset: i32, + ) -> CallbackResult { + Ok(CallbackOutcome::Continue) + } + + /// Called when a conditional branch is considered. + fn on_branch( + &mut self, + _py: Python<'_>, + _code: &CodeObjectWrapper, + _offset: i32, + _destination_offset: i32, + ) -> CallbackResult { + Ok(CallbackOutcome::Continue) + } + + /// Called at start of a Python function (frame on stack). + /// + /// Implementations should fail fast on irrecoverable conditions + /// (e.g., inability to access the current frame/locals) by + /// returning an error. + fn on_py_start( + &mut self, + _py: Python<'_>, + _code: &CodeObjectWrapper, + _offset: i32, + ) -> CallbackResult { + Ok(CallbackOutcome::Continue) + } + + /// Notify the tracer that an unrecoverable error occurred and the runtime + /// is transitioning into a detach/disable flow. + fn notify_failure(&mut self, _py: Python<'_>) -> PyResult<()> { + Ok(()) + } + + /// Called on resumption of a generator/coroutine (not via throw()). + fn on_py_resume( + &mut self, + _py: Python<'_>, + _code: &CodeObjectWrapper, + _offset: i32, + ) -> CallbackResult { + Ok(CallbackOutcome::Continue) + } + + /// Called immediately before a Python function returns. + fn on_py_return( + &mut self, + _py: Python<'_>, + _code: &CodeObjectWrapper, + _offset: i32, + _retval: &Bound<'_, PyAny>, + ) -> CallbackResult { + Ok(CallbackOutcome::Continue) + } + + /// Called immediately before a Python function yields. + fn on_py_yield( + &mut self, + _py: Python<'_>, + _code: &CodeObjectWrapper, + _offset: i32, + _retval: &Bound<'_, PyAny>, + ) -> CallbackResult { + Ok(CallbackOutcome::Continue) + } + + /// Called when a Python function is resumed by throw(). + fn on_py_throw( + &mut self, + _py: Python<'_>, + _code: &CodeObjectWrapper, + _offset: i32, + _exception: &Bound<'_, PyAny>, + ) -> CallbackResult { + Ok(CallbackOutcome::Continue) + } + + /// Called when exiting a Python function during exception unwinding. + fn on_py_unwind( + &mut self, + _py: Python<'_>, + _code: &CodeObjectWrapper, + _offset: i32, + _exception: &Bound<'_, PyAny>, + ) -> CallbackResult { + Ok(CallbackOutcome::Continue) + } + + /// Called when an exception is raised (excluding STOP_ITERATION). + fn on_raise( + &mut self, + _py: Python<'_>, + _code: &CodeObjectWrapper, + _offset: i32, + _exception: &Bound<'_, PyAny>, + ) -> CallbackResult { + Ok(CallbackOutcome::Continue) + } + + /// Called when an exception is re-raised. + fn on_reraise( + &mut self, + _py: Python<'_>, + _code: &CodeObjectWrapper, + _offset: i32, + _exception: &Bound<'_, PyAny>, + ) -> CallbackResult { + Ok(CallbackOutcome::Continue) + } + + /// Called when an exception is handled. + fn on_exception_handled( + &mut self, + _py: Python<'_>, + _code: &CodeObjectWrapper, + _offset: i32, + _exception: &Bound<'_, PyAny>, + ) -> CallbackResult { + Ok(CallbackOutcome::Continue) + } + + /// Called on return from any non-Python callable. + fn on_c_return( + &mut self, + _py: Python<'_>, + _code: &CodeObjectWrapper, + _offset: i32, + _callable: &Bound<'_, PyAny>, + _arg0: Option<&Bound<'_, PyAny>>, + ) -> CallbackResult { + Ok(CallbackOutcome::Continue) + } + + /// Called when an exception is raised from any non-Python callable. + fn on_c_raise( + &mut self, + _py: Python<'_>, + _code: &CodeObjectWrapper, + _offset: i32, + _callable: &Bound<'_, PyAny>, + _arg0: Option<&Bound<'_, PyAny>>, + ) -> CallbackResult { + Ok(CallbackOutcome::Continue) + } + + /// Flush any buffered state to storage. Default is a no-op. + fn flush(&mut self, _py: Python<'_>) -> PyResult<()> { + Ok(()) + } + + /// Finish and close any underlying writers. Default is a no-op. + fn finish(&mut self, _py: Python<'_>) -> PyResult<()> { + Ok(()) + } +} diff --git a/codetracer-python-recorder/src/monitoring/callbacks.rs b/codetracer-python-recorder/src/monitoring/callbacks.rs new file mode 100644 index 0000000..2477087 --- /dev/null +++ b/codetracer-python-recorder/src/monitoring/callbacks.rs @@ -0,0 +1,694 @@ +//! sys.monitoring callback metadata and helpers. + +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::Mutex; + +use crate::code_object::{CodeObjectRegistry, CodeObjectWrapper}; +use crate::ffi; +use crate::logging; +use crate::policy::{self, OnRecorderError}; +use log::{error, trace, warn}; +use pyo3::prelude::*; +use pyo3::types::{PyAny, PyCode, PyModule}; +use pyo3::wrap_pyfunction; +use recorder_errors::ErrorCode; + +use super::api::Tracer; +use super::{register_callback, EventId, EventSet, MonitoringEvents, ToolId}; + +pub use super::{CallbackFn, CallbackOutcome, CallbackResult}; + +/// Global tracer state shared between callback invocations and installer. +pub(super) struct Global { + pub(super) registry: CodeObjectRegistry, + pub(super) tracer: Box, + pub(super) mask: EventSet, + pub(super) tool: ToolId, + pub(super) disable_sentinel: Py, +} + +pub(super) static GLOBAL: Mutex> = Mutex::new(None); + +fn catch_callback(label: &'static str, callback: F) -> CallbackResult +where + F: FnOnce() -> CallbackResult, +{ + match catch_unwind(AssertUnwindSafe(callback)) { + Ok(result) => result, + Err(payload) => Err(ffi::panic_to_pyerr(label, payload)), + } +} + +fn call_tracer_with_code<'py, F>( + py: Python<'py>, + guard: &mut Option, + code: &Bound<'py, PyCode>, + label: &'static str, + callback: F, +) -> CallbackResult +where + F: FnOnce(&mut dyn Tracer, &CodeObjectWrapper) -> CallbackResult, +{ + let global = guard.as_mut().expect("tracer installed"); + let wrapper = global.registry.get_or_insert(py, code); + let tracer = global.tracer.as_mut(); + catch_callback(label, || callback(tracer, &wrapper)) +} + +fn handle_callback_result( + py: Python<'_>, + guard: &mut Option, + result: CallbackResult, +) -> PyResult> { + match result { + Ok(CallbackOutcome::Continue) => Ok(py.None()), + Ok(CallbackOutcome::DisableLocation) => Ok(guard + .as_ref() + .map(|global| global.disable_sentinel.clone_ref(py)) + .unwrap_or_else(|| py.None())), + Err(err) => handle_callback_error(py, guard, err), + } +} + +fn handle_callback_error( + py: Python<'_>, + guard: &mut Option, + err: PyErr, +) -> PyResult> { + let policy = policy::policy_snapshot(); + match policy.on_recorder_error { + OnRecorderError::Abort => Err(err), + OnRecorderError::Disable => { + let message = err.to_string(); + let code = logging::error_code_from_pyerr(py, &err); + logging::record_detach("policy_disable", code.map(|code| code.as_str())); + logging::with_error_code_opt(code, || { + error!( + "recorder callback error; disabling tracer per policy: {}", + message + ); + }); + if let Some(global) = guard.as_mut() { + if let Err(notify_err) = global.tracer.notify_failure(py) { + logging::with_error_code(ErrorCode::TraceIncomplete, || { + warn!( + "failed to notify tracer about disable transition: {}", + notify_err + ); + }); + } + } + super::install::uninstall_locked(py, guard)?; + Ok(py.None()) + } + } +} + +#[pyfunction] +pub(super) fn callback_call( + py: Python<'_>, + code: Bound<'_, PyCode>, + offset: i32, + callable: Bound<'_, PyAny>, + arg0: Option>, +) -> PyResult> { + ffi::wrap_pyfunction("callback_call", || { + let mut guard = GLOBAL.lock().unwrap(); + if guard.is_none() { + return Ok(py.None()); + } + let result = + call_tracer_with_code(py, &mut guard, &code, "callback_call", |tracer, wrapper| { + tracer.on_call(py, wrapper, offset, &callable, arg0.as_ref()) + }); + handle_callback_result(py, &mut guard, result) + }) +} + +#[pyfunction] +pub(super) fn callback_line( + py: Python<'_>, + code: Bound<'_, PyCode>, + lineno: u32, +) -> PyResult> { + ffi::wrap_pyfunction("callback_line", || { + let mut guard = GLOBAL.lock().unwrap(); + if guard.is_none() { + return Ok(py.None()); + } + let result = + call_tracer_with_code(py, &mut guard, &code, "callback_line", |tracer, wrapper| { + tracer.on_line(py, wrapper, lineno) + }); + handle_callback_result(py, &mut guard, result) + }) +} + +#[pyfunction] +pub(super) fn callback_instruction( + py: Python<'_>, + code: Bound<'_, PyCode>, + instruction_offset: i32, +) -> PyResult> { + ffi::wrap_pyfunction("callback_instruction", || { + let mut guard = GLOBAL.lock().unwrap(); + if guard.is_none() { + return Ok(py.None()); + } + let result = call_tracer_with_code( + py, + &mut guard, + &code, + "callback_instruction", + |tracer, wrapper| tracer.on_instruction(py, wrapper, instruction_offset), + ); + handle_callback_result(py, &mut guard, result) + }) +} + +#[pyfunction] +pub(super) fn callback_jump( + py: Python<'_>, + code: Bound<'_, PyCode>, + instruction_offset: i32, + destination_offset: i32, +) -> PyResult> { + ffi::wrap_pyfunction("callback_jump", || { + let mut guard = GLOBAL.lock().unwrap(); + if guard.is_none() { + return Ok(py.None()); + } + let result = + call_tracer_with_code(py, &mut guard, &code, "callback_jump", |tracer, wrapper| { + tracer.on_jump(py, wrapper, instruction_offset, destination_offset) + }); + handle_callback_result(py, &mut guard, result) + }) +} + +#[pyfunction] +pub(super) fn callback_branch( + py: Python<'_>, + code: Bound<'_, PyCode>, + instruction_offset: i32, + destination_offset: i32, +) -> PyResult> { + ffi::wrap_pyfunction("callback_branch", || { + let mut guard = GLOBAL.lock().unwrap(); + if guard.is_none() { + return Ok(py.None()); + } + let result = call_tracer_with_code( + py, + &mut guard, + &code, + "callback_branch", + |tracer, wrapper| tracer.on_branch(py, wrapper, instruction_offset, destination_offset), + ); + handle_callback_result(py, &mut guard, result) + }) +} + +#[pyfunction] +pub(super) fn callback_py_start( + py: Python<'_>, + code: Bound<'_, PyCode>, + instruction_offset: i32, +) -> PyResult> { + ffi::wrap_pyfunction("callback_py_start", || { + let mut guard = GLOBAL.lock().unwrap(); + if guard.is_none() { + return Ok(py.None()); + } + let result = call_tracer_with_code( + py, + &mut guard, + &code, + "callback_py_start", + |tracer, wrapper| tracer.on_py_start(py, wrapper, instruction_offset), + ); + handle_callback_result(py, &mut guard, result) + }) +} + +#[pyfunction] +pub(super) fn callback_py_resume( + py: Python<'_>, + code: Bound<'_, PyCode>, + instruction_offset: i32, +) -> PyResult> { + ffi::wrap_pyfunction("callback_py_resume", || { + let mut guard = GLOBAL.lock().unwrap(); + if guard.is_none() { + return Ok(py.None()); + } + let result = call_tracer_with_code( + py, + &mut guard, + &code, + "callback_py_resume", + |tracer, wrapper| tracer.on_py_resume(py, wrapper, instruction_offset), + ); + handle_callback_result(py, &mut guard, result) + }) +} + +#[pyfunction] +pub(super) fn callback_py_return( + py: Python<'_>, + code: Bound<'_, PyCode>, + instruction_offset: i32, + retval: Bound<'_, PyAny>, +) -> PyResult> { + ffi::wrap_pyfunction("callback_py_return", || { + let mut guard = GLOBAL.lock().unwrap(); + if guard.is_none() { + return Ok(py.None()); + } + let result = call_tracer_with_code( + py, + &mut guard, + &code, + "callback_py_return", + |tracer, wrapper| tracer.on_py_return(py, wrapper, instruction_offset, &retval), + ); + handle_callback_result(py, &mut guard, result) + }) +} + +#[pyfunction] +pub(super) fn callback_py_yield( + py: Python<'_>, + code: Bound<'_, PyCode>, + instruction_offset: i32, + retval: Bound<'_, PyAny>, +) -> PyResult> { + ffi::wrap_pyfunction("callback_py_yield", || { + let mut guard = GLOBAL.lock().unwrap(); + if guard.is_none() { + return Ok(py.None()); + } + let result = call_tracer_with_code( + py, + &mut guard, + &code, + "callback_py_yield", + |tracer, wrapper| tracer.on_py_yield(py, wrapper, instruction_offset, &retval), + ); + handle_callback_result(py, &mut guard, result) + }) +} + +#[pyfunction] +pub(super) fn callback_py_throw( + py: Python<'_>, + code: Bound<'_, PyCode>, + instruction_offset: i32, + exception: Bound<'_, PyAny>, +) -> PyResult> { + ffi::wrap_pyfunction("callback_py_throw", || { + let mut guard = GLOBAL.lock().unwrap(); + if guard.is_none() { + return Ok(py.None()); + } + let result = call_tracer_with_code( + py, + &mut guard, + &code, + "callback_py_throw", + |tracer, wrapper| tracer.on_py_throw(py, wrapper, instruction_offset, &exception), + ); + handle_callback_result(py, &mut guard, result) + }) +} + +#[pyfunction] +pub(super) fn callback_py_unwind( + py: Python<'_>, + code: Bound<'_, PyCode>, + instruction_offset: i32, + exception: Bound<'_, PyAny>, +) -> PyResult> { + ffi::wrap_pyfunction("callback_py_unwind", || { + let mut guard = GLOBAL.lock().unwrap(); + if guard.is_none() { + return Ok(py.None()); + } + let result = call_tracer_with_code( + py, + &mut guard, + &code, + "callback_py_unwind", + |tracer, wrapper| tracer.on_py_unwind(py, wrapper, instruction_offset, &exception), + ); + handle_callback_result(py, &mut guard, result) + }) +} + +#[pyfunction] +pub(super) fn callback_raise( + py: Python<'_>, + code: Bound<'_, PyCode>, + instruction_offset: i32, + exception: Bound<'_, PyAny>, +) -> PyResult> { + ffi::wrap_pyfunction("callback_raise", || { + let mut guard = GLOBAL.lock().unwrap(); + if guard.is_none() { + return Ok(py.None()); + } + let result = call_tracer_with_code( + py, + &mut guard, + &code, + "callback_raise", + |tracer, wrapper| tracer.on_raise(py, wrapper, instruction_offset, &exception), + ); + handle_callback_result(py, &mut guard, result) + }) +} + +#[pyfunction] +pub(super) fn callback_reraise( + py: Python<'_>, + code: Bound<'_, PyCode>, + instruction_offset: i32, + exception: Bound<'_, PyAny>, +) -> PyResult> { + ffi::wrap_pyfunction("callback_reraise", || { + let mut guard = GLOBAL.lock().unwrap(); + if guard.is_none() { + return Ok(py.None()); + } + let result = call_tracer_with_code( + py, + &mut guard, + &code, + "callback_reraise", + |tracer, wrapper| tracer.on_reraise(py, wrapper, instruction_offset, &exception), + ); + handle_callback_result(py, &mut guard, result) + }) +} + +#[pyfunction] +pub(super) fn callback_exception_handled( + py: Python<'_>, + code: Bound<'_, PyCode>, + instruction_offset: i32, + exception: Bound<'_, PyAny>, +) -> PyResult> { + ffi::wrap_pyfunction("callback_exception_handled", || { + let mut guard = GLOBAL.lock().unwrap(); + if guard.is_none() { + return Ok(py.None()); + } + let result = call_tracer_with_code( + py, + &mut guard, + &code, + "callback_exception_handled", + |tracer, wrapper| { + tracer.on_exception_handled(py, wrapper, instruction_offset, &exception) + }, + ); + handle_callback_result(py, &mut guard, result) + }) +} + +// See comment in Tracer trait +// #[pyfunction] +// pub(super) fn callback_stop_iteration( +// py: Python<'_>, +// code: Bound<'_, PyAny>, +// instruction_offset: i32, +// exception: Bound<'_, PyAny>, +// ) -> PyResult<()> { +// if let Some(global) = GLOBAL.lock().unwrap().as_mut() { +// global +// .tracer +// .on_stop_iteration(py, &code, instruction_offset, &exception); +// } +// Ok(()) +// } + +#[pyfunction] +pub(super) fn callback_c_return( + py: Python<'_>, + code: Bound<'_, PyCode>, + offset: i32, + callable: Bound<'_, PyAny>, + arg0: Option>, +) -> PyResult> { + ffi::wrap_pyfunction("callback_c_return", || { + let mut guard = GLOBAL.lock().unwrap(); + if guard.is_none() { + return Ok(py.None()); + } + let result = call_tracer_with_code( + py, + &mut guard, + &code, + "callback_c_return", + |tracer, wrapper| tracer.on_c_return(py, wrapper, offset, &callable, arg0.as_ref()), + ); + handle_callback_result(py, &mut guard, result) + }) +} + +#[pyfunction] +pub(super) fn callback_c_raise( + py: Python<'_>, + code: Bound<'_, PyCode>, + offset: i32, + callable: Bound<'_, PyAny>, + arg0: Option>, +) -> PyResult> { + ffi::wrap_pyfunction("callback_c_raise", || { + let mut guard = GLOBAL.lock().unwrap(); + if guard.is_none() { + return Ok(py.None()); + } + let result = call_tracer_with_code( + py, + &mut guard, + &code, + "callback_c_raise", + |tracer, wrapper| tracer.on_c_raise(py, wrapper, offset, &callable, arg0.as_ref()), + ); + handle_callback_result(py, &mut guard, result) + }) +} + +/// Function pointer used to instantiate a PyO3 callback. +type CallbackFactory = for<'py> fn(&Bound<'py, PyModule>) -> PyResult>; + +/// Metadata describing how to register a sys.monitoring callback. +pub struct CallbackSpec { + /// Debug label (mirrors the PyO3 function name). + pub name: &'static str, + event: fn(&MonitoringEvents) -> EventId, + factory: CallbackFactory, +} + +impl CallbackSpec { + pub const fn new( + name: &'static str, + event: fn(&MonitoringEvents) -> EventId, + factory: CallbackFactory, + ) -> Self { + Self { + name, + event, + factory, + } + } + + /// Resolve the CPython event identifier for this callback. + pub fn event(&self, events: &MonitoringEvents) -> EventId { + (self.event)(events) + } + + /// Instantiate and bind the PyO3 callback into the provided module. + pub fn make<'py>(&self, module: &Bound<'py, PyModule>) -> PyResult> { + (self.factory)(module) + } + + /// Return true when the callback should be active for the supplied mask. + pub fn enabled(&self, mask: &EventSet, events: &MonitoringEvents) -> bool { + mask.contains(&self.event(events)) + } +} + +/// Declarative list describing all recorder callbacks. +pub static CALLBACK_SPECS: &[CallbackSpec] = &[ + CallbackSpec::new("callback_call", |ev| ev.CALL, wrap_callback_call), + CallbackSpec::new("callback_line", |ev| ev.LINE, wrap_callback_line), + CallbackSpec::new( + "callback_instruction", + |ev| ev.INSTRUCTION, + wrap_callback_instruction, + ), + CallbackSpec::new("callback_jump", |ev| ev.JUMP, wrap_callback_jump), + CallbackSpec::new("callback_branch", |ev| ev.BRANCH, wrap_callback_branch), + CallbackSpec::new( + "callback_py_start", + |ev| ev.PY_START, + wrap_callback_py_start, + ), + CallbackSpec::new( + "callback_py_resume", + |ev| ev.PY_RESUME, + wrap_callback_py_resume, + ), + CallbackSpec::new( + "callback_py_return", + |ev| ev.PY_RETURN, + wrap_callback_py_return, + ), + CallbackSpec::new( + "callback_py_yield", + |ev| ev.PY_YIELD, + wrap_callback_py_yield, + ), + CallbackSpec::new( + "callback_py_throw", + |ev| ev.PY_THROW, + wrap_callback_py_throw, + ), + CallbackSpec::new( + "callback_py_unwind", + |ev| ev.PY_UNWIND, + wrap_callback_py_unwind, + ), + CallbackSpec::new("callback_raise", |ev| ev.RAISE, wrap_callback_raise), + CallbackSpec::new("callback_reraise", |ev| ev.RERAISE, wrap_callback_reraise), + CallbackSpec::new( + "callback_exception_handled", + |ev| ev.EXCEPTION_HANDLED, + wrap_callback_exception_handled, + ), + // See comment in Tracer trait: STOP_ITERATION intentionally omitted. + CallbackSpec::new( + "callback_c_return", + |ev| ev.C_RETURN, + wrap_callback_c_return, + ), + CallbackSpec::new("callback_c_raise", |ev| ev.C_RAISE, wrap_callback_c_raise), +]; + +/// Iterate over the callbacks enabled for the provided mask. +pub fn enabled_specs<'a>( + mask: &'a EventSet, + events: &'a MonitoringEvents, +) -> impl Iterator + 'a { + CALLBACK_SPECS + .iter() + .filter(move |spec| spec.enabled(mask, events)) +} + +/// Register all callbacks enabled by the supplied mask. +pub fn register_enabled_callbacks<'py>( + py: Python<'py>, + module: &Bound<'py, PyModule>, + tool: &ToolId, + mask: &EventSet, + events: &MonitoringEvents, +) -> PyResult<()> { + for spec in enabled_specs(mask, events) { + let event = spec.event(events); + trace!( + "[monitoring] registering callback `{}` for event id {}", + spec.name, + event.0 + ); + let cb = spec.make(module)?; + register_callback(py, tool, &event, Some(&cb))?; + } + Ok(()) +} + +/// Unregister previously installed callbacks that were enabled by the mask. +pub fn unregister_enabled_callbacks( + py: Python<'_>, + tool: &ToolId, + mask: &EventSet, + events: &MonitoringEvents, +) -> PyResult<()> { + for spec in enabled_specs(mask, events) { + let event = spec.event(events); + trace!( + "[monitoring] unregistering callback `{}` for event id {}", + spec.name, + event.0 + ); + register_callback(py, tool, &event, None)?; + } + Ok(()) +} + +fn wrap_callback_call<'py>(module: &Bound<'py, PyModule>) -> PyResult> { + wrap_pyfunction!(callback_call, module) +} + +fn wrap_callback_line<'py>(module: &Bound<'py, PyModule>) -> PyResult> { + wrap_pyfunction!(callback_line, module) +} + +fn wrap_callback_instruction<'py>(module: &Bound<'py, PyModule>) -> PyResult> { + wrap_pyfunction!(callback_instruction, module) +} + +fn wrap_callback_jump<'py>(module: &Bound<'py, PyModule>) -> PyResult> { + wrap_pyfunction!(callback_jump, module) +} + +fn wrap_callback_branch<'py>(module: &Bound<'py, PyModule>) -> PyResult> { + wrap_pyfunction!(callback_branch, module) +} + +fn wrap_callback_py_start<'py>(module: &Bound<'py, PyModule>) -> PyResult> { + wrap_pyfunction!(callback_py_start, module) +} + +fn wrap_callback_py_resume<'py>(module: &Bound<'py, PyModule>) -> PyResult> { + wrap_pyfunction!(callback_py_resume, module) +} + +fn wrap_callback_py_return<'py>(module: &Bound<'py, PyModule>) -> PyResult> { + wrap_pyfunction!(callback_py_return, module) +} + +fn wrap_callback_py_yield<'py>(module: &Bound<'py, PyModule>) -> PyResult> { + wrap_pyfunction!(callback_py_yield, module) +} + +fn wrap_callback_py_throw<'py>(module: &Bound<'py, PyModule>) -> PyResult> { + wrap_pyfunction!(callback_py_throw, module) +} + +fn wrap_callback_py_unwind<'py>(module: &Bound<'py, PyModule>) -> PyResult> { + wrap_pyfunction!(callback_py_unwind, module) +} + +fn wrap_callback_raise<'py>(module: &Bound<'py, PyModule>) -> PyResult> { + wrap_pyfunction!(callback_raise, module) +} + +fn wrap_callback_reraise<'py>(module: &Bound<'py, PyModule>) -> PyResult> { + wrap_pyfunction!(callback_reraise, module) +} + +fn wrap_callback_exception_handled<'py>( + module: &Bound<'py, PyModule>, +) -> PyResult> { + wrap_pyfunction!(callback_exception_handled, module) +} + +fn wrap_callback_c_return<'py>(module: &Bound<'py, PyModule>) -> PyResult> { + wrap_pyfunction!(callback_c_return, module) +} + +fn wrap_callback_c_raise<'py>(module: &Bound<'py, PyModule>) -> PyResult> { + wrap_pyfunction!(callback_c_raise, module) +} diff --git a/codetracer-python-recorder/src/monitoring/install.rs b/codetracer-python-recorder/src/monitoring/install.rs new file mode 100644 index 0000000..22afd4f --- /dev/null +++ b/codetracer-python-recorder/src/monitoring/install.rs @@ -0,0 +1,84 @@ +//! Tracer installation plumbing backed by the callbacks module. + +use crate::code_object::CodeObjectRegistry; +use crate::ffi; +use log::warn; +use pyo3::{prelude::*, types::PyModule}; +use recorder_errors::{usage, ErrorCode}; + +use super::api::Tracer; +use super::callbacks::{self, Global, GLOBAL}; +use super::{acquire_tool_id, free_tool_id, monitoring_events, set_events, NO_EVENTS}; + +pub(super) fn uninstall_locked(py: Python<'_>, guard: &mut Option) -> PyResult<()> { + if let Some(mut global) = guard.take() { + let finish_result = global.tracer.finish(py); + + let cleanup_result = (|| -> PyResult<()> { + let events = monitoring_events(py)?; + callbacks::unregister_enabled_callbacks(py, &global.tool, &global.mask, events)?; + set_events(py, &global.tool, NO_EVENTS)?; + free_tool_id(py, &global.tool)?; + Ok(()) + })(); + + if let Err(err) = finish_result { + if let Err(cleanup_err) = cleanup_result { + warn!( + "failed to reset monitoring callbacks after finish error: {}", + cleanup_err + ); + } + return Err(err); + } + + cleanup_result?; + } + Ok(()) +} + +/// Install a tracer and hook it into Python's `sys.monitoring`. +pub fn install_tracer(py: Python<'_>, tracer: Box) -> PyResult<()> { + let mut guard = GLOBAL.lock().unwrap(); + if guard.is_some() { + return Err(ffi::map_recorder_error(usage!( + ErrorCode::TracerInstallConflict, + "tracer already installed" + ))); + } + + let tool = acquire_tool_id(py)?; + let events = monitoring_events(py)?; + let monitoring = py.import("sys")?.getattr("monitoring")?; + let disable_sentinel = monitoring.getattr("DISABLE")?.unbind(); + + let module = PyModule::new(py, "_codetracer_callbacks")?; + + let mask = tracer.interest(events); + callbacks::register_enabled_callbacks(py, &module, &tool, &mask, events)?; + + set_events(py, &tool, mask)?; + + *guard = Some(Global { + registry: CodeObjectRegistry::default(), + tracer, + mask, + tool, + disable_sentinel, + }); + Ok(()) +} + +/// Remove the installed tracer if any. +pub fn uninstall_tracer(py: Python<'_>) -> PyResult<()> { + let mut guard = GLOBAL.lock().unwrap(); + uninstall_locked(py, &mut guard) +} + +/// Flush the currently installed tracer if any. +pub fn flush_installed_tracer(py: Python<'_>) -> PyResult<()> { + if let Some(global) = GLOBAL.lock().unwrap().as_mut() { + global.tracer.flush(py)?; + } + Ok(()) +} diff --git a/codetracer-python-recorder/src/monitoring/mod.rs b/codetracer-python-recorder/src/monitoring/mod.rs index 29b5107..dffdd92 100644 --- a/codetracer-python-recorder/src/monitoring/mod.rs +++ b/codetracer-python-recorder/src/monitoring/mod.rs @@ -4,9 +4,13 @@ use pyo3::prelude::*; use pyo3::types::PyCFunction; use std::sync::OnceLock; -mod tracer; +pub(crate) mod api; +pub(crate) mod callbacks; +pub(crate) mod install; +pub mod tracer; -pub use tracer::{flush_installed_tracer, install_tracer, uninstall_tracer, Tracer}; +pub use api::Tracer; +pub use install::{flush_installed_tracer, install_tracer, uninstall_tracer}; const MONITORING_TOOL_NAME: &str = "codetracer"; diff --git a/codetracer-python-recorder/src/monitoring/tracer.rs b/codetracer-python-recorder/src/monitoring/tracer.rs index ff63aee..f0d0b53 100644 --- a/codetracer-python-recorder/src/monitoring/tracer.rs +++ b/codetracer-python-recorder/src/monitoring/tracer.rs @@ -1,906 +1,3 @@ -//! Tracer trait and sys.monitoring callback plumbing. +//! Legacy tracer facade kept for backwards compatibility with existing imports. -use std::any::Any; -use std::panic::{catch_unwind, AssertUnwindSafe}; -use std::sync::Mutex; - -use crate::code_object::{CodeObjectRegistry, CodeObjectWrapper}; -use crate::ffi; -use crate::logging; -use crate::policy::{self, OnRecorderError}; -use log::{error, warn}; -use pyo3::{ - prelude::*, - types::{PyAny, PyCode, PyModule}, -}; -use recorder_errors::{usage, ErrorCode}; - -use super::{ - acquire_tool_id, free_tool_id, monitoring_events, register_callback, set_events, - CallbackOutcome, CallbackResult, EventSet, MonitoringEvents, ToolId, NO_EVENTS, -}; - -/// Trait implemented by tracing backends. -/// -/// Each method corresponds to an event from `sys.monitoring`. Default -/// implementations allow implementers to only handle the events they care -/// about. -/// -/// Every callback returns a `CallbackResult` so implementations can propagate -/// Python exceptions or request that CPython disables future events for a -/// location by yielding the `CallbackOutcome::DisableLocation` sentinel. -pub trait Tracer: Send + Any { - /// Downcast support for implementations that need to be accessed - /// behind a `Box` (e.g., for flushing/finishing). - fn as_any(&mut self) -> &mut dyn Any - where - Self: 'static, - Self: Sized, - { - self - } - - /// Return the set of events the tracer wants to receive. - fn interest(&self, _events: &MonitoringEvents) -> EventSet { - NO_EVENTS - } - - /// Called on Python function calls. - fn on_call( - &mut self, - _py: Python<'_>, - _code: &CodeObjectWrapper, - _offset: i32, - _callable: &Bound<'_, PyAny>, - _arg0: Option<&Bound<'_, PyAny>>, - ) -> CallbackResult { - Ok(CallbackOutcome::Continue) - } - - /// Called on line execution. - fn on_line( - &mut self, - _py: Python<'_>, - _code: &CodeObjectWrapper, - _lineno: u32, - ) -> CallbackResult { - Ok(CallbackOutcome::Continue) - } - - /// Called when an instruction is about to be executed (by offset). - fn on_instruction( - &mut self, - _py: Python<'_>, - _code: &CodeObjectWrapper, - _offset: i32, - ) -> CallbackResult { - Ok(CallbackOutcome::Continue) - } - - /// Called when a jump in the control flow graph is made. - fn on_jump( - &mut self, - _py: Python<'_>, - _code: &CodeObjectWrapper, - _offset: i32, - _destination_offset: i32, - ) -> CallbackResult { - Ok(CallbackOutcome::Continue) - } - - /// Called when a conditional branch is considered. - fn on_branch( - &mut self, - _py: Python<'_>, - _code: &CodeObjectWrapper, - _offset: i32, - _destination_offset: i32, - ) -> CallbackResult { - Ok(CallbackOutcome::Continue) - } - - /// Called at start of a Python function (frame on stack). - /// - /// Implementations should fail fast on irrecoverable conditions - /// (e.g., inability to access the current frame/locals) by - /// returning an error. - fn on_py_start( - &mut self, - _py: Python<'_>, - _code: &CodeObjectWrapper, - _offset: i32, - ) -> CallbackResult { - Ok(CallbackOutcome::Continue) - } - - /// Notify the tracer that an unrecoverable error occurred and the runtime - /// is transitioning into a detach/disable flow. - fn notify_failure(&mut self, _py: Python<'_>) -> PyResult<()> { - Ok(()) - } - - /// Called on resumption of a generator/coroutine (not via throw()). - fn on_py_resume( - &mut self, - _py: Python<'_>, - _code: &CodeObjectWrapper, - _offset: i32, - ) -> CallbackResult { - Ok(CallbackOutcome::Continue) - } - - /// Called immediately before a Python function returns. - fn on_py_return( - &mut self, - _py: Python<'_>, - _code: &CodeObjectWrapper, - _offset: i32, - _retval: &Bound<'_, PyAny>, - ) -> CallbackResult { - Ok(CallbackOutcome::Continue) - } - - /// Called immediately before a Python function yields. - fn on_py_yield( - &mut self, - _py: Python<'_>, - _code: &CodeObjectWrapper, - _offset: i32, - _retval: &Bound<'_, PyAny>, - ) -> CallbackResult { - Ok(CallbackOutcome::Continue) - } - - /// Called when a Python function is resumed by throw(). - fn on_py_throw( - &mut self, - _py: Python<'_>, - _code: &CodeObjectWrapper, - _offset: i32, - _exception: &Bound<'_, PyAny>, - ) -> CallbackResult { - Ok(CallbackOutcome::Continue) - } - - /// Called when exiting a Python function during exception unwinding. - fn on_py_unwind( - &mut self, - _py: Python<'_>, - _code: &CodeObjectWrapper, - _offset: i32, - _exception: &Bound<'_, PyAny>, - ) -> CallbackResult { - Ok(CallbackOutcome::Continue) - } - - /// Called when an exception is raised (excluding STOP_ITERATION). - fn on_raise( - &mut self, - _py: Python<'_>, - _code: &CodeObjectWrapper, - _offset: i32, - _exception: &Bound<'_, PyAny>, - ) -> CallbackResult { - Ok(CallbackOutcome::Continue) - } - - /// Called when an exception is re-raised. - fn on_reraise( - &mut self, - _py: Python<'_>, - _code: &CodeObjectWrapper, - _offset: i32, - _exception: &Bound<'_, PyAny>, - ) -> CallbackResult { - Ok(CallbackOutcome::Continue) - } - - /// Called when an exception is handled. - fn on_exception_handled( - &mut self, - _py: Python<'_>, - _code: &CodeObjectWrapper, - _offset: i32, - _exception: &Bound<'_, PyAny>, - ) -> CallbackResult { - Ok(CallbackOutcome::Continue) - } - - /// Called when an artificial StopIteration is raised. - // Tzanko: I have been unable to write Python code that emits this event. This happens both in Python 3.12, 3.13 - // Here are some relevant discussions which might explain why, I haven't investigated the issue fully - // https://github.com/python/cpython/issues/116090, - // https://github.com/python/cpython/issues/118692 - // fn on_stop_iteration( - // &mut self, - // _py: Python<'_>, - // _code: &CodeObjectWrapper, - // _offset: i32, - // _exception: &Bound<'_, PyAny>, - // ) { - // } - - /// Called on return from any non-Python callable. - fn on_c_return( - &mut self, - _py: Python<'_>, - _code: &CodeObjectWrapper, - _offset: i32, - _callable: &Bound<'_, PyAny>, - _arg0: Option<&Bound<'_, PyAny>>, - ) -> CallbackResult { - Ok(CallbackOutcome::Continue) - } - - /// Called when an exception is raised from any non-Python callable. - fn on_c_raise( - &mut self, - _py: Python<'_>, - _code: &CodeObjectWrapper, - _offset: i32, - _callable: &Bound<'_, PyAny>, - _arg0: Option<&Bound<'_, PyAny>>, - ) -> CallbackResult { - Ok(CallbackOutcome::Continue) - } - - /// Flush any buffered state to storage. Default is a no-op. - fn flush(&mut self, _py: Python<'_>) -> PyResult<()> { - Ok(()) - } - - /// Finish and close any underlying writers. Default is a no-op. - fn finish(&mut self, _py: Python<'_>) -> PyResult<()> { - Ok(()) - } -} - -struct Global { - registry: CodeObjectRegistry, - tracer: Box, - mask: EventSet, - tool: ToolId, - disable_sentinel: Py, -} - -static GLOBAL: Mutex> = Mutex::new(None); - -fn catch_callback(label: &'static str, callback: F) -> CallbackResult -where - F: FnOnce() -> CallbackResult, -{ - match catch_unwind(AssertUnwindSafe(callback)) { - Ok(result) => result, - Err(payload) => Err(ffi::panic_to_pyerr(label, payload)), - } -} - -fn call_tracer_with_code<'py, F>( - py: Python<'py>, - guard: &mut Option, - code: &Bound<'py, PyCode>, - label: &'static str, - callback: F, -) -> CallbackResult -where - F: FnOnce(&mut dyn Tracer, &CodeObjectWrapper) -> CallbackResult, -{ - let global = guard.as_mut().expect("tracer installed"); - let wrapper = global.registry.get_or_insert(py, code); - let tracer = global.tracer.as_mut(); - catch_callback(label, || callback(tracer, &wrapper)) -} - -fn handle_callback_result( - py: Python<'_>, - guard: &mut Option, - result: CallbackResult, -) -> PyResult> { - match result { - Ok(CallbackOutcome::Continue) => Ok(py.None()), - Ok(CallbackOutcome::DisableLocation) => Ok(guard - .as_ref() - .map(|global| global.disable_sentinel.clone_ref(py)) - .unwrap_or_else(|| py.None())), - Err(err) => handle_callback_error(py, guard, err), - } -} - -fn handle_callback_error( - py: Python<'_>, - guard: &mut Option, - err: PyErr, -) -> PyResult> { - let policy = policy::policy_snapshot(); - match policy.on_recorder_error { - OnRecorderError::Abort => Err(err), - OnRecorderError::Disable => { - let message = err.to_string(); - let code = logging::error_code_from_pyerr(py, &err); - logging::record_detach("policy_disable", code.map(|code| code.as_str())); - logging::with_error_code_opt(code, || { - error!( - "recorder callback error; disabling tracer per policy: {}", - message - ); - }); - if let Some(global) = guard.as_mut() { - if let Err(notify_err) = global.tracer.notify_failure(py) { - logging::with_error_code(ErrorCode::TraceIncomplete, || { - warn!( - "failed to notify tracer about disable transition: {}", - notify_err - ); - }); - } - } - uninstall_locked(py, guard)?; - Ok(py.None()) - } - } -} - -fn uninstall_locked(py: Python<'_>, guard: &mut Option) -> PyResult<()> { - if let Some(mut global) = guard.take() { - let finish_result = global.tracer.finish(py); - - let cleanup_result = (|| -> PyResult<()> { - let events = monitoring_events(py)?; - if global.mask.contains(&events.CALL) { - register_callback(py, &global.tool, &events.CALL, None)?; - } - if global.mask.contains(&events.LINE) { - register_callback(py, &global.tool, &events.LINE, None)?; - } - if global.mask.contains(&events.INSTRUCTION) { - register_callback(py, &global.tool, &events.INSTRUCTION, None)?; - } - if global.mask.contains(&events.JUMP) { - register_callback(py, &global.tool, &events.JUMP, None)?; - } - if global.mask.contains(&events.BRANCH) { - register_callback(py, &global.tool, &events.BRANCH, None)?; - } - if global.mask.contains(&events.PY_START) { - register_callback(py, &global.tool, &events.PY_START, None)?; - } - if global.mask.contains(&events.PY_RESUME) { - register_callback(py, &global.tool, &events.PY_RESUME, None)?; - } - if global.mask.contains(&events.PY_RETURN) { - register_callback(py, &global.tool, &events.PY_RETURN, None)?; - } - if global.mask.contains(&events.PY_YIELD) { - register_callback(py, &global.tool, &events.PY_YIELD, None)?; - } - if global.mask.contains(&events.PY_THROW) { - register_callback(py, &global.tool, &events.PY_THROW, None)?; - } - if global.mask.contains(&events.PY_UNWIND) { - register_callback(py, &global.tool, &events.PY_UNWIND, None)?; - } - if global.mask.contains(&events.RAISE) { - register_callback(py, &global.tool, &events.RAISE, None)?; - } - if global.mask.contains(&events.RERAISE) { - register_callback(py, &global.tool, &events.RERAISE, None)?; - } - if global.mask.contains(&events.EXCEPTION_HANDLED) { - register_callback(py, &global.tool, &events.EXCEPTION_HANDLED, None)?; - } - // if global.mask.contains(&events.STOP_ITERATION) { - // register_callback(py, &global.tool, &events.STOP_ITERATION, None)?; - // } - if global.mask.contains(&events.C_RETURN) { - register_callback(py, &global.tool, &events.C_RETURN, None)?; - } - if global.mask.contains(&events.C_RAISE) { - register_callback(py, &global.tool, &events.C_RAISE, None)?; - } - - set_events(py, &global.tool, NO_EVENTS)?; - free_tool_id(py, &global.tool)?; - Ok(()) - })(); - - if let Err(err) = finish_result { - if let Err(cleanup_err) = cleanup_result { - warn!( - "failed to reset monitoring callbacks after finish error: {}", - cleanup_err - ); - } - return Err(err); - } - - cleanup_result?; - } - Ok(()) -} - -/// Install a tracer and hook it into Python's `sys.monitoring`. -pub fn install_tracer(py: Python<'_>, tracer: Box) -> PyResult<()> { - let mut guard = GLOBAL.lock().unwrap(); - if guard.is_some() { - return Err(ffi::map_recorder_error(usage!( - ErrorCode::TracerInstallConflict, - "tracer already installed" - ))); - } - - let tool = acquire_tool_id(py)?; - let events = monitoring_events(py)?; - let monitoring = py.import("sys")?.getattr("monitoring")?; - let disable_sentinel = monitoring.getattr("DISABLE")?.unbind(); - - let module = PyModule::new(py, "_codetracer_callbacks")?; - - let mask = tracer.interest(events); - - if mask.contains(&events.CALL) { - let cb = wrap_pyfunction!(callback_call, &module)?; - register_callback(py, &tool, &events.CALL, Some(&cb))?; - } - if mask.contains(&events.LINE) { - let cb = wrap_pyfunction!(callback_line, &module)?; - register_callback(py, &tool, &events.LINE, Some(&cb))?; - } - if mask.contains(&events.INSTRUCTION) { - let cb = wrap_pyfunction!(callback_instruction, &module)?; - register_callback(py, &tool, &events.INSTRUCTION, Some(&cb))?; - } - if mask.contains(&events.JUMP) { - let cb = wrap_pyfunction!(callback_jump, &module)?; - register_callback(py, &tool, &events.JUMP, Some(&cb))?; - } - if mask.contains(&events.BRANCH) { - let cb = wrap_pyfunction!(callback_branch, &module)?; - register_callback(py, &tool, &events.BRANCH, Some(&cb))?; - } - if mask.contains(&events.PY_START) { - let cb = wrap_pyfunction!(callback_py_start, &module)?; - register_callback(py, &tool, &events.PY_START, Some(&cb))?; - } - if mask.contains(&events.PY_RESUME) { - let cb = wrap_pyfunction!(callback_py_resume, &module)?; - register_callback(py, &tool, &events.PY_RESUME, Some(&cb))?; - } - if mask.contains(&events.PY_RETURN) { - let cb = wrap_pyfunction!(callback_py_return, &module)?; - register_callback(py, &tool, &events.PY_RETURN, Some(&cb))?; - } - if mask.contains(&events.PY_YIELD) { - let cb = wrap_pyfunction!(callback_py_yield, &module)?; - register_callback(py, &tool, &events.PY_YIELD, Some(&cb))?; - } - if mask.contains(&events.PY_THROW) { - let cb = wrap_pyfunction!(callback_py_throw, &module)?; - register_callback(py, &tool, &events.PY_THROW, Some(&cb))?; - } - if mask.contains(&events.PY_UNWIND) { - let cb = wrap_pyfunction!(callback_py_unwind, &module)?; - register_callback(py, &tool, &events.PY_UNWIND, Some(&cb))?; - } - if mask.contains(&events.RAISE) { - let cb = wrap_pyfunction!(callback_raise, &module)?; - register_callback(py, &tool, &events.RAISE, Some(&cb))?; - } - if mask.contains(&events.RERAISE) { - let cb = wrap_pyfunction!(callback_reraise, &module)?; - register_callback(py, &tool, &events.RERAISE, Some(&cb))?; - } - if mask.contains(&events.EXCEPTION_HANDLED) { - let cb = wrap_pyfunction!(callback_exception_handled, &module)?; - register_callback(py, &tool, &events.EXCEPTION_HANDLED, Some(&cb))?; - } - // See comment in Tracer trait - // if mask.contains(&events.STOP_ITERATION) { - // let cb = wrap_pyfunction!(callback_stop_iteration, &module)?; - // register_callback(py, &tool, &events.STOP_ITERATION, Some(&cb))?; - // } - if mask.contains(&events.C_RETURN) { - let cb = wrap_pyfunction!(callback_c_return, &module)?; - register_callback(py, &tool, &events.C_RETURN, Some(&cb))?; - } - if mask.contains(&events.C_RAISE) { - let cb = wrap_pyfunction!(callback_c_raise, &module)?; - register_callback(py, &tool, &events.C_RAISE, Some(&cb))?; - } - - set_events(py, &tool, mask)?; - - *guard = Some(Global { - registry: CodeObjectRegistry::default(), - tracer, - mask, - tool, - disable_sentinel, - }); - Ok(()) -} - -/// Remove the installed tracer if any. -pub fn uninstall_tracer(py: Python<'_>) -> PyResult<()> { - let mut guard = GLOBAL.lock().unwrap(); - uninstall_locked(py, &mut guard) -} - -/// Flush the currently installed tracer if any. -pub fn flush_installed_tracer(py: Python<'_>) -> PyResult<()> { - if let Some(global) = GLOBAL.lock().unwrap().as_mut() { - global.tracer.flush(py)?; - } - Ok(()) -} - -#[pyfunction] -fn callback_call( - py: Python<'_>, - code: Bound<'_, PyCode>, - offset: i32, - callable: Bound<'_, PyAny>, - arg0: Option>, -) -> PyResult> { - ffi::wrap_pyfunction("callback_call", || { - let mut guard = GLOBAL.lock().unwrap(); - if guard.is_none() { - return Ok(py.None()); - } - let result = - call_tracer_with_code(py, &mut guard, &code, "callback_call", |tracer, wrapper| { - tracer.on_call(py, wrapper, offset, &callable, arg0.as_ref()) - }); - handle_callback_result(py, &mut guard, result) - }) -} - -#[pyfunction] -fn callback_line(py: Python<'_>, code: Bound<'_, PyCode>, lineno: u32) -> PyResult> { - ffi::wrap_pyfunction("callback_line", || { - let mut guard = GLOBAL.lock().unwrap(); - if guard.is_none() { - return Ok(py.None()); - } - let result = - call_tracer_with_code(py, &mut guard, &code, "callback_line", |tracer, wrapper| { - tracer.on_line(py, wrapper, lineno) - }); - handle_callback_result(py, &mut guard, result) - }) -} - -#[pyfunction] -fn callback_instruction( - py: Python<'_>, - code: Bound<'_, PyCode>, - instruction_offset: i32, -) -> PyResult> { - ffi::wrap_pyfunction("callback_instruction", || { - let mut guard = GLOBAL.lock().unwrap(); - if guard.is_none() { - return Ok(py.None()); - } - let result = call_tracer_with_code( - py, - &mut guard, - &code, - "callback_instruction", - |tracer, wrapper| tracer.on_instruction(py, wrapper, instruction_offset), - ); - handle_callback_result(py, &mut guard, result) - }) -} - -#[pyfunction] -fn callback_jump( - py: Python<'_>, - code: Bound<'_, PyCode>, - instruction_offset: i32, - destination_offset: i32, -) -> PyResult> { - ffi::wrap_pyfunction("callback_jump", || { - let mut guard = GLOBAL.lock().unwrap(); - if guard.is_none() { - return Ok(py.None()); - } - let result = - call_tracer_with_code(py, &mut guard, &code, "callback_jump", |tracer, wrapper| { - tracer.on_jump(py, wrapper, instruction_offset, destination_offset) - }); - handle_callback_result(py, &mut guard, result) - }) -} - -#[pyfunction] -fn callback_branch( - py: Python<'_>, - code: Bound<'_, PyCode>, - instruction_offset: i32, - destination_offset: i32, -) -> PyResult> { - ffi::wrap_pyfunction("callback_branch", || { - let mut guard = GLOBAL.lock().unwrap(); - if guard.is_none() { - return Ok(py.None()); - } - let result = call_tracer_with_code( - py, - &mut guard, - &code, - "callback_branch", - |tracer, wrapper| tracer.on_branch(py, wrapper, instruction_offset, destination_offset), - ); - handle_callback_result(py, &mut guard, result) - }) -} - -#[pyfunction] -fn callback_py_start( - py: Python<'_>, - code: Bound<'_, PyCode>, - instruction_offset: i32, -) -> PyResult> { - ffi::wrap_pyfunction("callback_py_start", || { - let mut guard = GLOBAL.lock().unwrap(); - if guard.is_none() { - return Ok(py.None()); - } - let result = call_tracer_with_code( - py, - &mut guard, - &code, - "callback_py_start", - |tracer, wrapper| tracer.on_py_start(py, wrapper, instruction_offset), - ); - handle_callback_result(py, &mut guard, result) - }) -} - -#[pyfunction] -fn callback_py_resume( - py: Python<'_>, - code: Bound<'_, PyCode>, - instruction_offset: i32, -) -> PyResult> { - ffi::wrap_pyfunction("callback_py_resume", || { - let mut guard = GLOBAL.lock().unwrap(); - if guard.is_none() { - return Ok(py.None()); - } - let result = call_tracer_with_code( - py, - &mut guard, - &code, - "callback_py_resume", - |tracer, wrapper| tracer.on_py_resume(py, wrapper, instruction_offset), - ); - handle_callback_result(py, &mut guard, result) - }) -} - -#[pyfunction] -fn callback_py_return( - py: Python<'_>, - code: Bound<'_, PyCode>, - instruction_offset: i32, - retval: Bound<'_, PyAny>, -) -> PyResult> { - ffi::wrap_pyfunction("callback_py_return", || { - let mut guard = GLOBAL.lock().unwrap(); - if guard.is_none() { - return Ok(py.None()); - } - let result = call_tracer_with_code( - py, - &mut guard, - &code, - "callback_py_return", - |tracer, wrapper| tracer.on_py_return(py, wrapper, instruction_offset, &retval), - ); - handle_callback_result(py, &mut guard, result) - }) -} - -#[pyfunction] -fn callback_py_yield( - py: Python<'_>, - code: Bound<'_, PyCode>, - instruction_offset: i32, - retval: Bound<'_, PyAny>, -) -> PyResult> { - ffi::wrap_pyfunction("callback_py_yield", || { - let mut guard = GLOBAL.lock().unwrap(); - if guard.is_none() { - return Ok(py.None()); - } - let result = call_tracer_with_code( - py, - &mut guard, - &code, - "callback_py_yield", - |tracer, wrapper| tracer.on_py_yield(py, wrapper, instruction_offset, &retval), - ); - handle_callback_result(py, &mut guard, result) - }) -} - -#[pyfunction] -fn callback_py_throw( - py: Python<'_>, - code: Bound<'_, PyCode>, - instruction_offset: i32, - exception: Bound<'_, PyAny>, -) -> PyResult> { - ffi::wrap_pyfunction("callback_py_throw", || { - let mut guard = GLOBAL.lock().unwrap(); - if guard.is_none() { - return Ok(py.None()); - } - let result = call_tracer_with_code( - py, - &mut guard, - &code, - "callback_py_throw", - |tracer, wrapper| tracer.on_py_throw(py, wrapper, instruction_offset, &exception), - ); - handle_callback_result(py, &mut guard, result) - }) -} - -#[pyfunction] -fn callback_py_unwind( - py: Python<'_>, - code: Bound<'_, PyCode>, - instruction_offset: i32, - exception: Bound<'_, PyAny>, -) -> PyResult> { - ffi::wrap_pyfunction("callback_py_unwind", || { - let mut guard = GLOBAL.lock().unwrap(); - if guard.is_none() { - return Ok(py.None()); - } - let result = call_tracer_with_code( - py, - &mut guard, - &code, - "callback_py_unwind", - |tracer, wrapper| tracer.on_py_unwind(py, wrapper, instruction_offset, &exception), - ); - handle_callback_result(py, &mut guard, result) - }) -} - -#[pyfunction] -fn callback_raise( - py: Python<'_>, - code: Bound<'_, PyCode>, - instruction_offset: i32, - exception: Bound<'_, PyAny>, -) -> PyResult> { - ffi::wrap_pyfunction("callback_raise", || { - let mut guard = GLOBAL.lock().unwrap(); - if guard.is_none() { - return Ok(py.None()); - } - let result = call_tracer_with_code( - py, - &mut guard, - &code, - "callback_raise", - |tracer, wrapper| tracer.on_raise(py, wrapper, instruction_offset, &exception), - ); - handle_callback_result(py, &mut guard, result) - }) -} - -#[pyfunction] -fn callback_reraise( - py: Python<'_>, - code: Bound<'_, PyCode>, - instruction_offset: i32, - exception: Bound<'_, PyAny>, -) -> PyResult> { - ffi::wrap_pyfunction("callback_reraise", || { - let mut guard = GLOBAL.lock().unwrap(); - if guard.is_none() { - return Ok(py.None()); - } - let result = call_tracer_with_code( - py, - &mut guard, - &code, - "callback_reraise", - |tracer, wrapper| tracer.on_reraise(py, wrapper, instruction_offset, &exception), - ); - handle_callback_result(py, &mut guard, result) - }) -} - -#[pyfunction] -fn callback_exception_handled( - py: Python<'_>, - code: Bound<'_, PyCode>, - instruction_offset: i32, - exception: Bound<'_, PyAny>, -) -> PyResult> { - ffi::wrap_pyfunction("callback_exception_handled", || { - let mut guard = GLOBAL.lock().unwrap(); - if guard.is_none() { - return Ok(py.None()); - } - let result = call_tracer_with_code( - py, - &mut guard, - &code, - "callback_exception_handled", - |tracer, wrapper| { - tracer.on_exception_handled(py, wrapper, instruction_offset, &exception) - }, - ); - handle_callback_result(py, &mut guard, result) - }) -} - -// See comment in Tracer trait -// #[pyfunction] -// fn callback_stop_iteration( -// py: Python<'_>, -// code: Bound<'_, PyAny>, -// instruction_offset: i32, -// exception: Bound<'_, PyAny>, -// ) -> PyResult<()> { -// if let Some(global) = GLOBAL.lock().unwrap().as_mut() { -// global -// .tracer -// .on_stop_iteration(py, &code, instruction_offset, &exception); -// } -// Ok(()) -// } - -#[pyfunction] -fn callback_c_return( - py: Python<'_>, - code: Bound<'_, PyCode>, - offset: i32, - callable: Bound<'_, PyAny>, - arg0: Option>, -) -> PyResult> { - ffi::wrap_pyfunction("callback_c_return", || { - let mut guard = GLOBAL.lock().unwrap(); - if guard.is_none() { - return Ok(py.None()); - } - let result = call_tracer_with_code( - py, - &mut guard, - &code, - "callback_c_return", - |tracer, wrapper| tracer.on_c_return(py, wrapper, offset, &callable, arg0.as_ref()), - ); - handle_callback_result(py, &mut guard, result) - }) -} - -#[pyfunction] -fn callback_c_raise( - py: Python<'_>, - code: Bound<'_, PyCode>, - offset: i32, - callable: Bound<'_, PyAny>, - arg0: Option>, -) -> PyResult> { - ffi::wrap_pyfunction("callback_c_raise", || { - let mut guard = GLOBAL.lock().unwrap(); - if guard.is_none() { - return Ok(py.None()); - } - let result = call_tracer_with_code( - py, - &mut guard, - &code, - "callback_c_raise", - |tracer, wrapper| tracer.on_c_raise(py, wrapper, offset, &callable, arg0.as_ref()), - ); - handle_callback_result(py, &mut guard, result) - }) -} +pub use super::install::{flush_installed_tracer, install_tracer, uninstall_tracer}; diff --git a/codetracer-python-recorder/src/policy.rs b/codetracer-python-recorder/src/policy.rs index 4826bf5..c26f81b 100644 --- a/codetracer-python-recorder/src/policy.rs +++ b/codetracer-python-recorder/src/policy.rs @@ -1,394 +1,32 @@ //! Runtime configuration policy for the recorder. -use std::env; -use std::path::PathBuf; -use std::str::FromStr; -use std::sync::RwLock; - -use once_cell::sync::OnceCell; -use recorder_errors::{usage, ErrorCode, RecorderError, RecorderResult}; - -/// Environment variable configuring how the recorder reacts to internal errors. -pub const ENV_ON_RECORDER_ERROR: &str = "CODETRACER_ON_RECORDER_ERROR"; -/// Environment variable enforcing that a trace file must be produced. -pub const ENV_REQUIRE_TRACE: &str = "CODETRACER_REQUIRE_TRACE"; -/// Environment variable toggling whether partial trace files are kept. -pub const ENV_KEEP_PARTIAL_TRACE: &str = "CODETRACER_KEEP_PARTIAL_TRACE"; -/// Environment variable controlling log level for the recorder crate. -pub const ENV_LOG_LEVEL: &str = "CODETRACER_LOG_LEVEL"; -/// Environment variable pointing to a log destination file. -pub const ENV_LOG_FILE: &str = "CODETRACER_LOG_FILE"; -/// Environment variable enabling JSON error trailers on stderr. -pub const ENV_JSON_ERRORS: &str = "CODETRACER_JSON_ERRORS"; -/// Environment variable toggling IO capture strategies. -pub const ENV_CAPTURE_IO: &str = "CODETRACER_CAPTURE_IO"; - -static POLICY: OnceCell> = OnceCell::new(); - -fn policy_cell() -> &'static RwLock { - POLICY.get_or_init(|| RwLock::new(RecorderPolicy::default())) -} - -/// Behaviour when the recorder encounters an error. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum OnRecorderError { - /// Propagate the error to callers; tracing stops with a non-zero exit. - Abort, - /// Disable tracing but allow the host process to continue running. - Disable, -} - -impl Default for OnRecorderError { - fn default() -> Self { - OnRecorderError::Abort - } -} - -#[derive(Debug)] -pub struct PolicyParseError(pub RecorderError); - -impl FromStr for OnRecorderError { - type Err = PolicyParseError; - - fn from_str(value: &str) -> Result { - match value.trim().to_ascii_lowercase().as_str() { - "abort" => Ok(OnRecorderError::Abort), - "disable" => Ok(OnRecorderError::Disable), - other => Err(PolicyParseError(usage!( - ErrorCode::InvalidPolicyValue, - "invalid on_recorder_error value '{}' (expected 'abort' or 'disable')", - other - ))), - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct IoCapturePolicy { - pub line_proxies: bool, - pub fd_fallback: bool, -} - -impl Default for IoCapturePolicy { - fn default() -> Self { - Self { - line_proxies: true, - fd_fallback: false, - } - } -} - -/// Recorder-wide runtime configuration. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct RecorderPolicy { - pub on_recorder_error: OnRecorderError, - pub require_trace: bool, - pub keep_partial_trace: bool, - pub log_level: Option, - pub log_file: Option, - pub json_errors: bool, - pub io_capture: IoCapturePolicy, -} - -impl Default for RecorderPolicy { - fn default() -> Self { - Self { - on_recorder_error: OnRecorderError::Abort, - require_trace: false, - keep_partial_trace: false, - log_level: None, - log_file: None, - json_errors: false, - io_capture: IoCapturePolicy::default(), - } - } -} - -impl RecorderPolicy { - fn apply_update(&mut self, update: PolicyUpdate) { - if let Some(on_err) = update.on_recorder_error { - self.on_recorder_error = on_err; - } - if let Some(require_trace) = update.require_trace { - self.require_trace = require_trace; - } - if let Some(keep_partial) = update.keep_partial_trace { - self.keep_partial_trace = keep_partial; - } - if let Some(level) = update.log_level { - self.log_level = match level.trim() { - "" => None, - other => Some(other.to_string()), - }; - } - if let Some(path) = update.log_file { - self.log_file = match path { - PolicyPath::Clear => None, - PolicyPath::Value(pb) => Some(pb), - }; - } - if let Some(json_errors) = update.json_errors { - self.json_errors = json_errors; - } - if let Some(line_proxies) = update.io_capture_line_proxies { - self.io_capture.line_proxies = line_proxies; - if !self.io_capture.line_proxies { - self.io_capture.fd_fallback = false; - } - } - if let Some(fd_fallback) = update.io_capture_fd_fallback { - // fd fallback requires proxies to be on. - self.io_capture.fd_fallback = fd_fallback && self.io_capture.line_proxies; - } - } -} - -/// Internal helper representing path updates. -#[derive(Debug, Clone)] -enum PolicyPath { - Clear, - Value(PathBuf), -} - -/// Mutation record for the policy. -#[derive(Debug, Default, Clone)] -struct PolicyUpdate { - on_recorder_error: Option, - require_trace: Option, - keep_partial_trace: Option, - log_level: Option, - log_file: Option, - json_errors: Option, - io_capture_line_proxies: Option, - io_capture_fd_fallback: Option, -} - -/// Snapshot the current policy. -pub fn policy_snapshot() -> RecorderPolicy { - policy_cell().read().expect("policy lock poisoned").clone() -} - -/// Apply the provided update to the global policy. -fn apply_policy_update(update: PolicyUpdate) { - let mut guard = policy_cell().write().expect("policy lock poisoned"); - guard.apply_update(update); - crate::logging::apply_policy(&guard); -} - -/// Load policy overrides from environment variables. -pub fn configure_policy_from_env() -> RecorderResult<()> { - let mut update = PolicyUpdate::default(); - - if let Ok(value) = env::var(ENV_ON_RECORDER_ERROR) { - let on_err = OnRecorderError::from_str(&value).map_err(|err| err.0)?; - update.on_recorder_error = Some(on_err); - } - - if let Ok(value) = env::var(ENV_REQUIRE_TRACE) { - update.require_trace = Some(parse_bool(&value)?); - } - - if let Ok(value) = env::var(ENV_KEEP_PARTIAL_TRACE) { - update.keep_partial_trace = Some(parse_bool(&value)?); - } - - if let Ok(value) = env::var(ENV_LOG_LEVEL) { - update.log_level = Some(value); - } - - if let Ok(value) = env::var(ENV_LOG_FILE) { - let path = if value.trim().is_empty() { - PolicyPath::Clear - } else { - PolicyPath::Value(PathBuf::from(value)) - }; - update.log_file = Some(path); - } - - if let Ok(value) = env::var(ENV_JSON_ERRORS) { - update.json_errors = Some(parse_bool(&value)?); - } - - if let Ok(value) = env::var(ENV_CAPTURE_IO) { - let (line_proxies, fd_fallback) = parse_capture_io(&value)?; - update.io_capture_line_proxies = Some(line_proxies); - update.io_capture_fd_fallback = Some(fd_fallback); - } - - apply_policy_update(update); - Ok(()) -} - -fn parse_bool(value: &str) -> RecorderResult { - match value.trim().to_ascii_lowercase().as_str() { - "1" | "true" | "t" | "yes" | "y" => Ok(true), - "0" | "false" | "f" | "no" | "n" => Ok(false), - other => Err(usage!( - ErrorCode::InvalidPolicyValue, - "invalid boolean value '{}' (expected true/false)", - other - )), - } -} - -fn parse_capture_io(value: &str) -> RecorderResult<(bool, bool)> { - let trimmed = value.trim(); - if trimmed.is_empty() { - let default = IoCapturePolicy::default(); - return Ok((default.line_proxies, default.fd_fallback)); - } - - let lower = trimmed.to_ascii_lowercase(); - if matches!( - lower.as_str(), - "0" | "off" | "false" | "disable" | "disabled" | "none" - ) { - return Ok((false, false)); - } - if matches!(lower.as_str(), "1" | "on" | "true" | "enable" | "enabled") { - return Ok((true, false)); - } - - let mut line_proxies = false; - let mut fd_fallback = false; - for token in lower.split(|c| matches!(c, ',' | '+')) { - match token.trim() { - "" => {} - "proxies" | "proxy" => line_proxies = true, - "fd" | "mirror" | "fallback" => { - line_proxies = true; - fd_fallback = true; - } - other => { - return Err(usage!( - ErrorCode::InvalidPolicyValue, - "invalid CODETRACER_CAPTURE_IO value '{}'", - other - )); - } - } - } - - if !line_proxies && !fd_fallback { - return Err(usage!( - ErrorCode::InvalidPolicyValue, - "CODETRACER_CAPTURE_IO must enable at least 'proxies' or 'fd'" - )); - } - - Ok((line_proxies, fd_fallback)) -} - -// === PyO3 helpers === - -use pyo3::prelude::*; -use pyo3::types::PyDict; - -use crate::ffi; - -#[pyfunction(name = "configure_policy")] -#[pyo3(signature = (on_recorder_error=None, require_trace=None, keep_partial_trace=None, log_level=None, log_file=None, json_errors=None, io_capture_line_proxies=None, io_capture_fd_fallback=None))] -pub fn configure_policy_py( - on_recorder_error: Option<&str>, - require_trace: Option, - keep_partial_trace: Option, - log_level: Option<&str>, - log_file: Option<&str>, - json_errors: Option, - io_capture_line_proxies: Option, - io_capture_fd_fallback: Option, -) -> PyResult<()> { - let mut update = PolicyUpdate::default(); - - if let Some(value) = on_recorder_error { - match OnRecorderError::from_str(value) { - Ok(parsed) => update.on_recorder_error = Some(parsed), - Err(err) => return Err(ffi::map_recorder_error(err.0)), - } - } - - if let Some(value) = require_trace { - update.require_trace = Some(value); - } - - if let Some(value) = keep_partial_trace { - update.keep_partial_trace = Some(value); - } - - if let Some(value) = log_level { - update.log_level = Some(value.to_string()); - } - - if let Some(value) = log_file { - let path = if value.trim().is_empty() { - PolicyPath::Clear - } else { - PolicyPath::Value(PathBuf::from(value)) - }; - update.log_file = Some(path); - } - - if let Some(value) = json_errors { - update.json_errors = Some(value); - } - - if let Some(value) = io_capture_line_proxies { - update.io_capture_line_proxies = Some(value); - } - - if let Some(value) = io_capture_fd_fallback { - update.io_capture_fd_fallback = Some(value); - } - - apply_policy_update(update); - Ok(()) -} - -#[pyfunction(name = "configure_policy_from_env")] -pub fn py_configure_policy_from_env() -> PyResult<()> { - configure_policy_from_env().map_err(ffi::map_recorder_error) -} - -#[pyfunction(name = "policy_snapshot")] -pub fn py_policy_snapshot(py: Python<'_>) -> PyResult { - let snapshot = policy_snapshot(); - let dict = PyDict::new(py); - dict.set_item( - "on_recorder_error", - match snapshot.on_recorder_error { - OnRecorderError::Abort => "abort", - OnRecorderError::Disable => "disable", - }, - )?; - dict.set_item("require_trace", snapshot.require_trace)?; - dict.set_item("keep_partial_trace", snapshot.keep_partial_trace)?; - if let Some(level) = snapshot.log_level.as_deref() { - dict.set_item("log_level", level)?; - } else { - dict.set_item("log_level", py.None())?; - } - if let Some(path) = snapshot.log_file.as_ref() { - dict.set_item("log_file", path.display().to_string())?; - } else { - dict.set_item("log_file", py.None())?; - } - dict.set_item("json_errors", snapshot.json_errors)?; - - let io_dict = PyDict::new(py); - io_dict.set_item("line_proxies", snapshot.io_capture.line_proxies)?; - io_dict.set_item("fd_fallback", snapshot.io_capture.fd_fallback)?; - dict.set_item("io_capture", io_dict)?; - Ok(dict.into()) -} +mod env; +mod ffi; +mod model; + +#[allow(unused_imports)] +pub use env::{ + configure_policy_from_env, ENV_CAPTURE_IO, ENV_JSON_ERRORS, ENV_KEEP_PARTIAL_TRACE, + ENV_LOG_FILE, ENV_LOG_LEVEL, ENV_ON_RECORDER_ERROR, ENV_REQUIRE_TRACE, +}; +#[allow(unused_imports)] +pub use ffi::{configure_policy_py, py_configure_policy_from_env, py_policy_snapshot}; +#[allow(unused_imports)] +pub use model::PolicyParseError; +#[allow(unused_imports)] +pub use model::{policy_snapshot, IoCapturePolicy, OnRecorderError, RecorderPolicy}; #[cfg(test)] mod tests { use super::*; - use std::path::Path; + use crate::policy::model::{ + apply_policy_update, reset_policy_for_tests, PolicyPath, PolicyUpdate, + }; + use recorder_errors::ErrorCode; + use std::path::{Path, PathBuf}; fn reset_policy() { - let mut guard = super::policy_cell().write().expect("policy lock poisoned"); - *guard = RecorderPolicy::default(); + reset_policy_for_tests(); } #[test] @@ -436,13 +74,13 @@ mod tests { fn configure_policy_from_env_parses_values() { reset_policy(); let env_guard = env_lock(); - env::set_var(ENV_ON_RECORDER_ERROR, "disable"); - env::set_var(ENV_REQUIRE_TRACE, "true"); - env::set_var(ENV_KEEP_PARTIAL_TRACE, "1"); - env::set_var(ENV_LOG_LEVEL, "info"); - env::set_var(ENV_LOG_FILE, "/tmp/out.log"); - env::set_var(ENV_JSON_ERRORS, "yes"); - env::set_var(ENV_CAPTURE_IO, "proxies,fd"); + std::env::set_var(ENV_ON_RECORDER_ERROR, "disable"); + std::env::set_var(ENV_REQUIRE_TRACE, "true"); + std::env::set_var(ENV_KEEP_PARTIAL_TRACE, "1"); + std::env::set_var(ENV_LOG_LEVEL, "info"); + std::env::set_var(ENV_LOG_FILE, "/tmp/out.log"); + std::env::set_var(ENV_JSON_ERRORS, "yes"); + std::env::set_var(ENV_CAPTURE_IO, "proxies,fd"); configure_policy_from_env().expect("configure from env"); @@ -464,7 +102,7 @@ mod tests { fn configure_policy_from_env_accepts_plus_separator() { reset_policy(); let env_guard = env_lock(); - env::set_var(ENV_CAPTURE_IO, "proxies+fd"); + std::env::set_var(ENV_CAPTURE_IO, "proxies+fd"); configure_policy_from_env().expect("configure from env with plus separator"); @@ -480,7 +118,7 @@ mod tests { fn configure_policy_from_env_rejects_invalid_boolean() { reset_policy(); let env_guard = env_lock(); - env::set_var(ENV_REQUIRE_TRACE, "sometimes"); + std::env::set_var(ENV_REQUIRE_TRACE, "sometimes"); let err = configure_policy_from_env().expect_err("invalid bool should error"); assert_eq!(err.code, ErrorCode::InvalidPolicyValue); @@ -493,7 +131,7 @@ mod tests { fn configure_policy_from_env_rejects_invalid_capture_io() { reset_policy(); let env_guard = env_lock(); - env::set_var(ENV_CAPTURE_IO, "invalid-token"); + std::env::set_var(ENV_CAPTURE_IO, "invalid-token"); let err = configure_policy_from_env().expect_err("invalid capture io should error"); assert_eq!(err.code, ErrorCode::InvalidPolicyValue); @@ -519,7 +157,7 @@ mod tests { ENV_JSON_ERRORS, ENV_CAPTURE_IO, ] { - env::remove_var(key); + std::env::remove_var(key); } } } diff --git a/codetracer-python-recorder/src/policy/env.rs b/codetracer-python-recorder/src/policy/env.rs new file mode 100644 index 0000000..2392b7a --- /dev/null +++ b/codetracer-python-recorder/src/policy/env.rs @@ -0,0 +1,190 @@ +//! Environment variable parsing for recorder policy overrides. + +use crate::policy::model::{apply_policy_update, OnRecorderError, PolicyPath, PolicyUpdate}; +use recorder_errors::{usage, ErrorCode, RecorderResult}; +use std::env; +use std::str::FromStr; + +/// Environment variable configuring how the recorder reacts to internal errors. +pub const ENV_ON_RECORDER_ERROR: &str = "CODETRACER_ON_RECORDER_ERROR"; +/// Environment variable enforcing that a trace file must be produced. +pub const ENV_REQUIRE_TRACE: &str = "CODETRACER_REQUIRE_TRACE"; +/// Environment variable toggling whether partial trace files are kept. +pub const ENV_KEEP_PARTIAL_TRACE: &str = "CODETRACER_KEEP_PARTIAL_TRACE"; +/// Environment variable controlling log level for the recorder crate. +pub const ENV_LOG_LEVEL: &str = "CODETRACER_LOG_LEVEL"; +/// Environment variable pointing to a log destination file. +pub const ENV_LOG_FILE: &str = "CODETRACER_LOG_FILE"; +/// Environment variable enabling JSON error trailers on stderr. +pub const ENV_JSON_ERRORS: &str = "CODETRACER_JSON_ERRORS"; +/// Environment variable toggling IO capture strategies. +pub const ENV_CAPTURE_IO: &str = "CODETRACER_CAPTURE_IO"; + +/// Load policy overrides from environment variables. +pub fn configure_policy_from_env() -> RecorderResult<()> { + let mut update = PolicyUpdate::default(); + + if let Ok(value) = env::var(ENV_ON_RECORDER_ERROR) { + let on_err = OnRecorderError::from_str(&value).map_err(|err| err.0)?; + update.on_recorder_error = Some(on_err); + } + + if let Ok(value) = env::var(ENV_REQUIRE_TRACE) { + update.require_trace = Some(parse_bool(&value)?); + } + + if let Ok(value) = env::var(ENV_KEEP_PARTIAL_TRACE) { + update.keep_partial_trace = Some(parse_bool(&value)?); + } + + if let Ok(value) = env::var(ENV_LOG_LEVEL) { + update.log_level = Some(value); + } + + if let Ok(value) = env::var(ENV_LOG_FILE) { + let path = if value.trim().is_empty() { + PolicyPath::Clear + } else { + PolicyPath::Value(value.into()) + }; + update.log_file = Some(path); + } + + if let Ok(value) = env::var(ENV_JSON_ERRORS) { + update.json_errors = Some(parse_bool(&value)?); + } + + if let Ok(value) = env::var(ENV_CAPTURE_IO) { + let (line_proxies, fd_fallback) = parse_capture_io(&value)?; + update.io_capture_line_proxies = Some(line_proxies); + update.io_capture_fd_fallback = Some(fd_fallback); + } + + apply_policy_update(update); + Ok(()) +} + +fn parse_bool(value: &str) -> RecorderResult { + match value.trim().to_ascii_lowercase().as_str() { + "1" | "true" | "t" | "yes" | "y" => Ok(true), + "0" | "false" | "f" | "no" | "n" => Ok(false), + other => Err(usage!( + ErrorCode::InvalidPolicyValue, + "invalid boolean value '{}' (expected true/false)", + other + )), + } +} + +fn parse_capture_io(value: &str) -> RecorderResult<(bool, bool)> { + let trimmed = value.trim(); + if trimmed.is_empty() { + let default = crate::policy::model::IoCapturePolicy::default(); + return Ok((default.line_proxies, default.fd_fallback)); + } + + let lower = trimmed.to_ascii_lowercase(); + if matches!( + lower.as_str(), + "0" | "off" | "false" | "disable" | "disabled" | "none" + ) { + return Ok((false, false)); + } + if matches!(lower.as_str(), "1" | "on" | "true" | "enable" | "enabled") { + return Ok((true, false)); + } + + let mut line_proxies = false; + let mut fd_fallback = false; + for token in lower.split(|c| matches!(c, ',' | '+')) { + match token.trim() { + "" => {} + "proxies" | "proxy" => line_proxies = true, + "fd" | "mirror" | "fallback" => { + line_proxies = true; + fd_fallback = true; + } + other => { + return Err(usage!( + ErrorCode::InvalidPolicyValue, + "invalid CODETRACER_CAPTURE_IO value '{}'", + other + )); + } + } + } + + if !line_proxies && !fd_fallback { + return Err(usage!( + ErrorCode::InvalidPolicyValue, + "CODETRACER_CAPTURE_IO must enable at least 'proxies' or 'fd'" + )); + } + + Ok((line_proxies, fd_fallback)) +} + +#[cfg(test)] +mod tests { + #[cfg(test)] + use super::*; + use crate::policy::model::{policy_snapshot, reset_policy_for_tests}; + + #[test] + fn configure_policy_from_env_updates_fields() { + let _guard = EnvGuard; + reset_policy_for_tests(); + std::env::set_var(ENV_ON_RECORDER_ERROR, "disable"); + std::env::set_var(ENV_REQUIRE_TRACE, "true"); + std::env::set_var(ENV_KEEP_PARTIAL_TRACE, "1"); + std::env::set_var(ENV_LOG_LEVEL, "info"); + std::env::set_var(ENV_LOG_FILE, "/tmp/out.log"); + std::env::set_var(ENV_JSON_ERRORS, "yes"); + std::env::set_var(ENV_CAPTURE_IO, "proxies,fd"); + + configure_policy_from_env().expect("configure from env"); + let snap = policy_snapshot(); + assert_eq!(snap.on_recorder_error, OnRecorderError::Disable); + assert!(snap.require_trace); + assert!(snap.keep_partial_trace); + assert_eq!(snap.log_level.as_deref(), Some("info")); + assert_eq!( + snap.log_file.as_ref().map(|p| p.display().to_string()), + Some("/tmp/out.log".to_string()) + ); + assert!(snap.json_errors); + assert!(snap.io_capture.line_proxies); + assert!(snap.io_capture.fd_fallback); + } + + #[test] + fn parse_capture_io_handles_aliases() { + assert_eq!(parse_capture_io("proxies+fd").unwrap(), (true, true)); + assert_eq!(parse_capture_io("proxies").unwrap(), (true, false)); + + assert!(parse_capture_io("invalid-token").is_err()); + } + + #[test] + fn parse_bool_rejects_invalid() { + assert!(parse_bool("sometimes").is_err()); + } + + struct EnvGuard; + + impl Drop for EnvGuard { + fn drop(&mut self) { + for key in [ + ENV_ON_RECORDER_ERROR, + ENV_REQUIRE_TRACE, + ENV_KEEP_PARTIAL_TRACE, + ENV_LOG_LEVEL, + ENV_LOG_FILE, + ENV_JSON_ERRORS, + ENV_CAPTURE_IO, + ] { + std::env::remove_var(key); + } + } + } +} diff --git a/codetracer-python-recorder/src/policy/ffi.rs b/codetracer-python-recorder/src/policy/ffi.rs new file mode 100644 index 0000000..f2588df --- /dev/null +++ b/codetracer-python-recorder/src/policy/ffi.rs @@ -0,0 +1,226 @@ +//! PyO3 bindings exposing policy configuration to Python callers. + +use super::env::configure_policy_from_env; +use super::model::{ + apply_policy_update, policy_snapshot, OnRecorderError, PolicyPath, PolicyUpdate, +}; +use crate::ffi; +use pyo3::prelude::*; +use pyo3::types::PyDict; +use std::path::PathBuf; +use std::str::FromStr; + +#[pyfunction(name = "configure_policy")] +#[pyo3(signature = (on_recorder_error=None, require_trace=None, keep_partial_trace=None, log_level=None, log_file=None, json_errors=None, io_capture_line_proxies=None, io_capture_fd_fallback=None))] +pub fn configure_policy_py( + on_recorder_error: Option<&str>, + require_trace: Option, + keep_partial_trace: Option, + log_level: Option<&str>, + log_file: Option<&str>, + json_errors: Option, + io_capture_line_proxies: Option, + io_capture_fd_fallback: Option, +) -> PyResult<()> { + let mut update = PolicyUpdate::default(); + + if let Some(value) = on_recorder_error { + match OnRecorderError::from_str(value) { + Ok(parsed) => update.on_recorder_error = Some(parsed), + Err(err) => return Err(ffi::map_recorder_error(err.0)), + } + } + + if let Some(value) = require_trace { + update.require_trace = Some(value); + } + + if let Some(value) = keep_partial_trace { + update.keep_partial_trace = Some(value); + } + + if let Some(value) = log_level { + update.log_level = Some(value.to_string()); + } + + if let Some(value) = log_file { + let path = if value.trim().is_empty() { + PolicyPath::Clear + } else { + PolicyPath::Value(PathBuf::from(value)) + }; + update.log_file = Some(path); + } + + if let Some(value) = json_errors { + update.json_errors = Some(value); + } + + if let Some(value) = io_capture_line_proxies { + update.io_capture_line_proxies = Some(value); + } + + if let Some(value) = io_capture_fd_fallback { + update.io_capture_fd_fallback = Some(value); + } + + apply_policy_update(update); + Ok(()) +} + +#[pyfunction(name = "configure_policy_from_env")] +pub fn py_configure_policy_from_env() -> PyResult<()> { + configure_policy_from_env().map_err(ffi::map_recorder_error) +} + +#[pyfunction(name = "policy_snapshot")] +pub fn py_policy_snapshot(py: Python<'_>) -> PyResult { + let snapshot = policy_snapshot(); + let dict = PyDict::new(py); + dict.set_item( + "on_recorder_error", + match snapshot.on_recorder_error { + OnRecorderError::Abort => "abort", + OnRecorderError::Disable => "disable", + }, + )?; + dict.set_item("require_trace", snapshot.require_trace)?; + dict.set_item("keep_partial_trace", snapshot.keep_partial_trace)?; + if let Some(level) = snapshot.log_level.as_deref() { + dict.set_item("log_level", level)?; + } else { + dict.set_item("log_level", py.None())?; + } + if let Some(path) = snapshot.log_file.as_ref() { + dict.set_item("log_file", path.display().to_string())?; + } else { + dict.set_item("log_file", py.None())?; + } + dict.set_item("json_errors", snapshot.json_errors)?; + + let io_dict = PyDict::new(py); + io_dict.set_item("line_proxies", snapshot.io_capture.line_proxies)?; + io_dict.set_item("fd_fallback", snapshot.io_capture.fd_fallback)?; + dict.set_item("io_capture", io_dict)?; + Ok(dict.into()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::policy::model::{policy_snapshot, reset_policy_for_tests}; + use pyo3::Python; + + #[test] + fn configure_policy_py_updates_policy() { + reset_policy_for_tests(); + configure_policy_py( + Some("disable"), + Some(true), + Some(true), + Some("debug"), + Some("/tmp/log.txt"), + Some(true), + Some(true), + Some(true), + ) + .expect("configure policy via PyO3 facade"); + + let snap = policy_snapshot(); + assert_eq!(snap.on_recorder_error, OnRecorderError::Disable); + assert!(snap.require_trace); + assert!(snap.keep_partial_trace); + assert_eq!(snap.log_level.as_deref(), Some("debug")); + assert_eq!( + snap.log_file + .as_ref() + .map(|p| p.display().to_string()) + .as_deref(), + Some("/tmp/log.txt") + ); + assert!(snap.json_errors); + assert!(snap.io_capture.line_proxies); + assert!(snap.io_capture.fd_fallback); + reset_policy_for_tests(); + } + + #[test] + fn configure_policy_py_rejects_invalid_on_recorder_error() { + reset_policy_for_tests(); + let err = configure_policy_py(Some("unknown"), None, None, None, None, None, None, None) + .expect_err("invalid variant should error"); + // Ensure the error maps through map_recorder_error by checking the display text. + let message = Python::with_gil(|py| err.value(py).to_string()); + assert!( + message.contains("invalid on_recorder_error value"), + "unexpected error message: {message}" + ); + reset_policy_for_tests(); + } + + #[test] + fn py_configure_policy_from_env_propagates_error() { + reset_policy_for_tests(); + let _guard = EnvGuard; + std::env::set_var(super::super::env::ENV_REQUIRE_TRACE, "maybe"); + Python::with_gil(|py| { + let err = py_configure_policy_from_env().expect_err("invalid env should error"); + let message = err.value(py).to_string(); + assert!( + message.contains("invalid boolean value"), + "unexpected error message: {message}" + ); + }); + reset_policy_for_tests(); + } + + #[test] + fn py_policy_snapshot_matches_model() { + reset_policy_for_tests(); + configure_policy_py( + Some("disable"), + Some(true), + Some(true), + Some("info"), + Some(""), + Some(true), + Some(false), + Some(false), + ) + .expect("configure policy"); + + Python::with_gil(|py| { + let obj = py_policy_snapshot(py).expect("snapshot dict"); + let dict = obj.bind(py).downcast::().expect("dict"); + + assert!( + dict.contains("on_recorder_error") + .expect("check on_recorder_error key"), + "expected on_recorder_error in snapshot" + ); + assert!( + dict.contains("io_capture").expect("check io_capture key"), + "expected io_capture in snapshot" + ); + }); + reset_policy_for_tests(); + } + + struct EnvGuard; + + impl Drop for EnvGuard { + fn drop(&mut self) { + for key in [ + super::super::env::ENV_ON_RECORDER_ERROR, + super::super::env::ENV_REQUIRE_TRACE, + super::super::env::ENV_KEEP_PARTIAL_TRACE, + super::super::env::ENV_LOG_LEVEL, + super::super::env::ENV_LOG_FILE, + super::super::env::ENV_JSON_ERRORS, + super::super::env::ENV_CAPTURE_IO, + ] { + std::env::remove_var(key); + } + } + } +} diff --git a/codetracer-python-recorder/src/policy/model.rs b/codetracer-python-recorder/src/policy/model.rs new file mode 100644 index 0000000..6b296b0 --- /dev/null +++ b/codetracer-python-recorder/src/policy/model.rs @@ -0,0 +1,165 @@ +//! Policy data structures and in-memory management. + +use once_cell::sync::OnceCell; +use recorder_errors::{usage, ErrorCode, RecorderError}; +use std::path::PathBuf; +use std::str::FromStr; +use std::sync::RwLock; + +static POLICY: OnceCell> = OnceCell::new(); + +fn policy_cell() -> &'static RwLock { + POLICY.get_or_init(|| RwLock::new(RecorderPolicy::default())) +} + +/// Behaviour when the recorder encounters an error. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OnRecorderError { + /// Propagate the error to callers; tracing stops with a non-zero exit. + Abort, + /// Disable tracing but allow the host process to continue running. + Disable, +} + +impl Default for OnRecorderError { + fn default() -> Self { + OnRecorderError::Abort + } +} + +#[derive(Debug)] +pub struct PolicyParseError(pub RecorderError); + +impl FromStr for OnRecorderError { + type Err = PolicyParseError; + + fn from_str(value: &str) -> Result { + match value.trim().to_ascii_lowercase().as_str() { + "abort" => Ok(OnRecorderError::Abort), + "disable" => Ok(OnRecorderError::Disable), + other => Err(PolicyParseError(usage!( + ErrorCode::InvalidPolicyValue, + "invalid on_recorder_error value '{}' (expected 'abort' or 'disable')", + other + ))), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IoCapturePolicy { + pub line_proxies: bool, + pub fd_fallback: bool, +} + +impl Default for IoCapturePolicy { + fn default() -> Self { + Self { + line_proxies: true, + fd_fallback: false, + } + } +} + +/// Recorder-wide runtime configuration. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RecorderPolicy { + pub on_recorder_error: OnRecorderError, + pub require_trace: bool, + pub keep_partial_trace: bool, + pub log_level: Option, + pub log_file: Option, + pub json_errors: bool, + pub io_capture: IoCapturePolicy, +} + +impl Default for RecorderPolicy { + fn default() -> Self { + Self { + on_recorder_error: OnRecorderError::Abort, + require_trace: false, + keep_partial_trace: false, + log_level: None, + log_file: None, + json_errors: false, + io_capture: IoCapturePolicy::default(), + } + } +} + +impl RecorderPolicy { + pub(crate) fn apply_update(&mut self, update: PolicyUpdate) { + if let Some(on_err) = update.on_recorder_error { + self.on_recorder_error = on_err; + } + if let Some(require_trace) = update.require_trace { + self.require_trace = require_trace; + } + if let Some(keep_partial) = update.keep_partial_trace { + self.keep_partial_trace = keep_partial; + } + if let Some(level) = update.log_level { + self.log_level = match level.trim() { + "" => None, + other => Some(other.to_string()), + }; + } + if let Some(path) = update.log_file { + self.log_file = match path { + PolicyPath::Clear => None, + PolicyPath::Value(pb) => Some(pb), + }; + } + if let Some(json_errors) = update.json_errors { + self.json_errors = json_errors; + } + if let Some(line_proxies) = update.io_capture_line_proxies { + self.io_capture.line_proxies = line_proxies; + if !self.io_capture.line_proxies { + self.io_capture.fd_fallback = false; + } + } + if let Some(fd_fallback) = update.io_capture_fd_fallback { + // fd fallback requires proxies to be on. + self.io_capture.fd_fallback = fd_fallback && self.io_capture.line_proxies; + } + } +} + +/// Internal helper representing path updates. +#[derive(Debug, Clone)] +pub(crate) enum PolicyPath { + Clear, + Value(PathBuf), +} + +/// Mutation record for the policy. +#[derive(Debug, Default, Clone)] +pub(crate) struct PolicyUpdate { + pub(crate) on_recorder_error: Option, + pub(crate) require_trace: Option, + pub(crate) keep_partial_trace: Option, + pub(crate) log_level: Option, + pub(crate) log_file: Option, + pub(crate) json_errors: Option, + pub(crate) io_capture_line_proxies: Option, + pub(crate) io_capture_fd_fallback: Option, +} + +/// Snapshot the current policy. +pub fn policy_snapshot() -> RecorderPolicy { + policy_cell().read().expect("policy lock poisoned").clone() +} + +/// Apply the provided update to the global policy and propagate logging changes. +pub(crate) fn apply_policy_update(update: PolicyUpdate) { + let mut guard = policy_cell().write().expect("policy lock poisoned"); + guard.apply_update(update); + crate::logging::apply_policy(&guard); +} + +#[cfg(test)] +pub(crate) fn reset_policy_for_tests() { + let mut guard = policy_cell().write().expect("policy lock poisoned"); + *guard = RecorderPolicy::default(); +} diff --git a/codetracer-python-recorder/src/runtime/mod.rs b/codetracer-python-recorder/src/runtime/mod.rs index c17c1ba..976a29c 100644 --- a/codetracer-python-recorder/src/runtime/mod.rs +++ b/codetracer-python-recorder/src/runtime/mod.rs @@ -1,4 +1,7 @@ -//! Runtime tracer facade translating sys.monitoring callbacks into `runtime_tracing` records. +//! Runtime tracing facade wiring sys.monitoring callbacks into dedicated collaborators. +//! +//! The [`tracer`] module hosts lifecycle, IO, filtering, and event pipelines and re-exports +//! [`RuntimeTracer`] so callers can keep importing it from `crate::runtime`. mod activation; mod frame_inspector; @@ -6,2600 +9,9 @@ pub mod io_capture; mod line_snapshots; mod logging; mod output_paths; +pub mod tracer; mod value_capture; mod value_encoder; -pub use line_snapshots::{FrameId, LineSnapshotStore}; pub use output_paths::TraceOutputPaths; - -use activation::ActivationController; -use frame_inspector::capture_frame; -use logging::log_event; -use value_capture::{ - capture_call_arguments, record_return_value, record_visible_scope, ValueFilterStats, -}; - -use std::collections::{hash_map::Entry, HashMap, HashSet}; -use std::fs; -use std::path::{Path, PathBuf}; -#[cfg(feature = "integration-test")] -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; -#[cfg(feature = "integration-test")] -use std::sync::OnceLock; -use std::thread::{self, ThreadId}; - -use pyo3::prelude::*; -use pyo3::types::PyAny; - -use recorder_errors::{bug, enverr, target, usage, ErrorCode, RecorderResult}; -use runtime_tracing::NonStreamingTraceWriter; -use runtime_tracing::{ - EventLogKind, Line, PathId, RecordEvent, TraceEventsFileFormat, TraceLowLevelEvent, TraceWriter, -}; - -use crate::code_object::CodeObjectWrapper; -use crate::ffi; -use crate::logging::{record_dropped_event, set_active_trace_id, with_error_code}; -use crate::monitoring::{ - events_union, CallbackOutcome, CallbackResult, EventSet, MonitoringEvents, Tracer, -}; -use crate::policy::{policy_snapshot, RecorderPolicy}; -use crate::runtime::io_capture::{ - IoCapturePipeline, IoCaptureSettings, IoChunk, IoChunkFlags, IoStream, ScopedMuteIoCapture, -}; -use crate::trace_filter::engine::{ExecDecision, ScopeResolution, TraceFilterEngine, ValueKind}; -use serde::Serialize; -use serde_json::{self, json}; - -use uuid::Uuid; - -struct TraceIdResetGuard; - -impl TraceIdResetGuard { - fn new() -> Self { - TraceIdResetGuard - } -} - -impl Drop for TraceIdResetGuard { - fn drop(&mut self) { - set_active_trace_id(None); - } -} - -fn io_flag_labels(flags: IoChunkFlags) -> Vec<&'static str> { - let mut labels = Vec::new(); - if flags.contains(IoChunkFlags::NEWLINE_TERMINATED) { - labels.push("newline"); - } - if flags.contains(IoChunkFlags::EXPLICIT_FLUSH) { - labels.push("flush"); - } - if flags.contains(IoChunkFlags::STEP_BOUNDARY) { - labels.push("step_boundary"); - } - if flags.contains(IoChunkFlags::TIME_SPLIT) { - labels.push("time_split"); - } - if flags.contains(IoChunkFlags::INPUT_CHUNK) { - labels.push("input"); - } - if flags.contains(IoChunkFlags::FD_MIRROR) { - labels.push("mirror"); - } - labels -} - -/// Minimal runtime tracer that maps Python sys.monitoring events to -/// runtime_tracing writer operations. -pub struct RuntimeTracer { - writer: NonStreamingTraceWriter, - format: TraceEventsFileFormat, - activation: ActivationController, - program_path: PathBuf, - ignored_code_ids: HashSet, - function_ids: HashMap, - output_paths: Option, - events_recorded: bool, - encountered_failure: bool, - trace_id: String, - line_snapshots: Arc, - io_capture: Option, - trace_filter: Option>, - scope_cache: HashMap>, - filter_stats: FilterStats, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum ShouldTrace { - Trace, - SkipAndDisable, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum FailureStage { - PyStart, - Line, - Finish, -} - -impl FailureStage { - fn as_str(self) -> &'static str { - match self { - FailureStage::PyStart => "py_start", - FailureStage::Line => "line", - FailureStage::Finish => "finish", - } - } -} - -#[derive(Debug, Default)] -struct FilterStats { - skipped_scopes: u64, - values: ValueFilterStats, -} - -impl FilterStats { - fn record_skip(&mut self) { - self.skipped_scopes += 1; - } - - fn values_mut(&mut self) -> &mut ValueFilterStats { - &mut self.values - } - - fn reset(&mut self) { - self.skipped_scopes = 0; - self.values = ValueFilterStats::default(); - } - - fn summary_json(&self) -> serde_json::Value { - let mut redactions = serde_json::Map::new(); - let mut drops = serde_json::Map::new(); - for kind in ValueKind::ALL { - redactions.insert( - kind.label().to_string(), - json!(self.values.redacted_count(kind)), - ); - drops.insert( - kind.label().to_string(), - json!(self.values.dropped_count(kind)), - ); - } - json!({ - "scopes_skipped": self.skipped_scopes, - "value_redactions": serde_json::Value::Object(redactions), - "value_drops": serde_json::Value::Object(drops), - }) - } -} - -// Failure injection helpers are only compiled for integration tests. -#[cfg_attr(not(feature = "integration-test"), allow(dead_code))] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum FailureMode { - Stage(FailureStage), - SuppressEvents, - TargetArgs, - Panic, -} - -#[cfg(feature = "integration-test")] -static FAILURE_MODE: OnceLock> = OnceLock::new(); -#[cfg(feature = "integration-test")] -static FAILURE_TRIGGERED: AtomicBool = AtomicBool::new(false); - -#[cfg(feature = "integration-test")] -fn configured_failure_mode() -> Option { - *FAILURE_MODE.get_or_init(|| { - let raw = std::env::var("CODETRACER_TEST_INJECT_FAILURE").ok(); - if let Some(value) = raw.as_deref() { - let _mute = ScopedMuteIoCapture::new(); - log::debug!("[RuntimeTracer] test failure injection mode: {}", value); - } - raw.and_then(|raw| match raw.trim().to_ascii_lowercase().as_str() { - "py_start" | "py-start" => Some(FailureMode::Stage(FailureStage::PyStart)), - "line" => Some(FailureMode::Stage(FailureStage::Line)), - "finish" => Some(FailureMode::Stage(FailureStage::Finish)), - "suppress-events" | "suppress_events" | "suppress" => Some(FailureMode::SuppressEvents), - "target" | "target-args" | "target_args" => Some(FailureMode::TargetArgs), - "panic" | "panic-callback" | "panic_callback" => Some(FailureMode::Panic), - _ => None, - }) - }) -} - -#[cfg(feature = "integration-test")] -fn should_inject_failure(stage: FailureStage) -> bool { - matches!(configured_failure_mode(), Some(FailureMode::Stage(mode)) if mode == stage) - && mark_failure_triggered() -} - -#[cfg(not(feature = "integration-test"))] -fn should_inject_failure(_stage: FailureStage) -> bool { - false -} - -#[cfg(feature = "integration-test")] -fn should_inject_target_error() -> bool { - matches!(configured_failure_mode(), Some(FailureMode::TargetArgs)) && mark_failure_triggered() -} - -#[cfg(not(feature = "integration-test"))] -fn should_inject_target_error() -> bool { - false -} - -#[cfg(feature = "integration-test")] -fn should_panic_in_callback() -> bool { - matches!(configured_failure_mode(), Some(FailureMode::Panic)) && mark_failure_triggered() -} - -#[cfg(not(feature = "integration-test"))] -#[allow(dead_code)] -fn should_panic_in_callback() -> bool { - false -} - -#[cfg(feature = "integration-test")] -fn suppress_events() -> bool { - matches!(configured_failure_mode(), Some(FailureMode::SuppressEvents)) -} - -#[cfg(not(feature = "integration-test"))] -fn suppress_events() -> bool { - false -} - -#[cfg(feature = "integration-test")] -fn mark_failure_triggered() -> bool { - !FAILURE_TRIGGERED.swap(true, Ordering::SeqCst) -} - -#[cfg(not(feature = "integration-test"))] -#[allow(dead_code)] -fn mark_failure_triggered() -> bool { - false -} - -#[cfg(feature = "integration-test")] -fn injected_failure_err(stage: FailureStage) -> PyErr { - let err = bug!( - ErrorCode::TraceIncomplete, - "test-injected failure at {}", - stage.as_str() - ) - .with_context("injection_stage", stage.as_str().to_string()); - ffi::map_recorder_error(err) -} - -#[cfg(not(feature = "integration-test"))] -fn injected_failure_err(stage: FailureStage) -> PyErr { - let err = bug!( - ErrorCode::TraceIncomplete, - "failure injection requested at {} without fail-injection feature", - stage.as_str() - ) - .with_context("injection_stage", stage.as_str().to_string()); - ffi::map_recorder_error(err) -} - -fn is_real_filename(filename: &str) -> bool { - let trimmed = filename.trim(); - !(trimmed.starts_with('<') && trimmed.ends_with('>')) -} - -impl RuntimeTracer { - pub fn new( - program: &str, - args: &[String], - format: TraceEventsFileFormat, - activation_path: Option<&Path>, - trace_filter: Option>, - ) -> Self { - let mut writer = NonStreamingTraceWriter::new(program, args); - writer.set_format(format); - let activation = ActivationController::new(activation_path); - let program_path = PathBuf::from(program); - Self { - writer, - format, - activation, - program_path, - ignored_code_ids: HashSet::new(), - function_ids: HashMap::new(), - output_paths: None, - events_recorded: false, - encountered_failure: false, - trace_id: Uuid::new_v4().to_string(), - line_snapshots: Arc::new(LineSnapshotStore::new()), - io_capture: None, - trace_filter, - scope_cache: HashMap::new(), - filter_stats: FilterStats::default(), - } - } - - /// Share the snapshot store with collaborators (IO capture, tests). - #[cfg_attr(not(test), allow(dead_code))] - pub fn line_snapshot_store(&self) -> Arc { - Arc::clone(&self.line_snapshots) - } - - pub fn install_io_capture(&mut self, py: Python<'_>, policy: &RecorderPolicy) -> PyResult<()> { - let settings = IoCaptureSettings { - line_proxies: policy.io_capture.line_proxies, - fd_mirror: policy.io_capture.fd_fallback, - }; - let pipeline = IoCapturePipeline::install(py, Arc::clone(&self.line_snapshots), settings)?; - self.io_capture = pipeline; - Ok(()) - } - - fn flush_io_before_step(&mut self, thread_id: ThreadId) { - if let Some(pipeline) = self.io_capture.as_ref() { - pipeline.flush_before_step(thread_id); - } - self.drain_io_chunks(); - } - - fn flush_pending_io(&mut self) { - if let Some(pipeline) = self.io_capture.as_ref() { - pipeline.flush_all(); - } - self.drain_io_chunks(); - } - - fn drain_io_chunks(&mut self) { - if let Some(pipeline) = self.io_capture.as_ref() { - let chunks = pipeline.drain_chunks(); - for chunk in chunks { - self.record_io_chunk(chunk); - } - } - } - - fn record_io_chunk(&mut self, mut chunk: IoChunk) { - if chunk.path_id.is_none() { - if let Some(path) = chunk.path.as_deref() { - let path_id = TraceWriter::ensure_path_id(&mut self.writer, Path::new(path)); - chunk.path_id = Some(path_id); - } - } - - let kind = match chunk.stream { - IoStream::Stdout => EventLogKind::Write, - IoStream::Stderr => EventLogKind::WriteOther, - IoStream::Stdin => EventLogKind::Read, - }; - - let metadata = self.build_io_metadata(&chunk); - let content = String::from_utf8_lossy(&chunk.payload).into_owned(); - - TraceWriter::add_event( - &mut self.writer, - TraceLowLevelEvent::Event(RecordEvent { - kind, - metadata, - content, - }), - ); - self.mark_event(); - } - - fn scope_resolution( - &mut self, - py: Python<'_>, - code: &CodeObjectWrapper, - ) -> Option> { - let engine = self.trace_filter.as_ref()?; - let code_id = code.id(); - - if let Some(existing) = self.scope_cache.get(&code_id) { - return Some(existing.clone()); - } - - match engine.resolve(py, code) { - Ok(resolution) => { - if resolution.exec() == ExecDecision::Trace { - self.scope_cache.insert(code_id, Arc::clone(&resolution)); - } else { - self.scope_cache.remove(&code_id); - } - Some(resolution) - } - Err(err) => { - let message = err.to_string(); - let error_code = err.code; - with_error_code(error_code, || { - let _mute = ScopedMuteIoCapture::new(); - log::error!( - "[RuntimeTracer] trace filter resolution failed for code id {}: {}", - code_id, - message - ); - }); - record_dropped_event("filter_resolution_error"); - None - } - } - } - - fn build_io_metadata(&self, chunk: &IoChunk) -> String { - #[derive(Serialize)] - struct IoEventMetadata<'a> { - stream: &'a str, - thread: String, - path_id: Option, - line: Option, - frame_id: Option, - flags: Vec<&'a str>, - } - - let snapshot = self.line_snapshots.snapshot_for_thread(chunk.thread_id); - let path_id = chunk - .path_id - .map(|id| id.0) - .or_else(|| snapshot.as_ref().map(|snap| snap.path_id().0)); - let line = chunk - .line - .map(|line| line.0) - .or_else(|| snapshot.as_ref().map(|snap| snap.line().0)); - let frame_id = chunk - .frame_id - .or_else(|| snapshot.as_ref().map(|snap| snap.frame_id())); - - let metadata = IoEventMetadata { - stream: match chunk.stream { - IoStream::Stdout => "stdout", - IoStream::Stderr => "stderr", - IoStream::Stdin => "stdin", - }, - thread: format!("{:?}", chunk.thread_id), - path_id, - line, - frame_id: frame_id.map(|id| id.as_raw()), - flags: io_flag_labels(chunk.flags), - }; - - match serde_json::to_string(&metadata) { - Ok(json) => json, - Err(err) => { - let _mute = ScopedMuteIoCapture::new(); - log::error!("failed to serialise IO metadata: {err}"); - "{}".to_string() - } - } - } - - fn teardown_io_capture(&mut self, py: Python<'_>) { - if let Some(mut pipeline) = self.io_capture.take() { - pipeline.flush_all(); - let chunks = pipeline.drain_chunks(); - for chunk in chunks { - self.record_io_chunk(chunk); - } - pipeline.uninstall(py); - let trailing = pipeline.drain_chunks(); - for chunk in trailing { - self.record_io_chunk(chunk); - } - } - } - - /// Configure output files and write initial metadata records. - pub fn begin(&mut self, outputs: &TraceOutputPaths, start_line: u32) -> PyResult<()> { - let start_path = self.activation.start_path(&self.program_path); - { - let _mute = ScopedMuteIoCapture::new(); - log::debug!("{}", start_path.display()); - } - outputs - .configure_writer(&mut self.writer, start_path, start_line) - .map_err(ffi::map_recorder_error)?; - self.output_paths = Some(outputs.clone()); - self.events_recorded = false; - self.encountered_failure = false; - set_active_trace_id(Some(self.trace_id.clone())); - Ok(()) - } - - fn mark_event(&mut self) { - if suppress_events() { - let _mute = ScopedMuteIoCapture::new(); - log::debug!("[RuntimeTracer] skipping event mark due to test injection"); - return; - } - self.events_recorded = true; - } - - fn mark_failure(&mut self) { - self.encountered_failure = true; - } - - fn cleanup_partial_outputs(&self) -> RecorderResult<()> { - if let Some(outputs) = &self.output_paths { - for path in [outputs.events(), outputs.metadata(), outputs.paths()] { - if path.exists() { - fs::remove_file(path).map_err(|err| { - enverr!(ErrorCode::Io, "failed to remove partial trace file") - .with_context("path", path.display().to_string()) - .with_context("io", err.to_string()) - })?; - } - } - } - Ok(()) - } - - fn require_trace_or_fail(&self, policy: &RecorderPolicy) -> RecorderResult<()> { - if policy.require_trace && !self.events_recorded { - return Err(usage!( - ErrorCode::TraceMissing, - "recorder policy requires a trace but no events were recorded" - )); - } - Ok(()) - } - - fn finalise_writer(&mut self) -> RecorderResult<()> { - TraceWriter::finish_writing_trace_metadata(&mut self.writer).map_err(|err| { - enverr!(ErrorCode::Io, "failed to finalise trace metadata") - .with_context("source", err.to_string()) - })?; - self.append_filter_metadata()?; - TraceWriter::finish_writing_trace_paths(&mut self.writer).map_err(|err| { - enverr!(ErrorCode::Io, "failed to finalise trace paths") - .with_context("source", err.to_string()) - })?; - TraceWriter::finish_writing_trace_events(&mut self.writer).map_err(|err| { - enverr!(ErrorCode::Io, "failed to finalise trace events") - .with_context("source", err.to_string()) - })?; - Ok(()) - } - - fn append_filter_metadata(&self) -> RecorderResult<()> { - let Some(outputs) = &self.output_paths else { - return Ok(()); - }; - let Some(engine) = self.trace_filter.as_ref() else { - return Ok(()); - }; - - let path = outputs.metadata(); - let original = fs::read_to_string(path).map_err(|err| { - enverr!(ErrorCode::Io, "failed to read trace metadata") - .with_context("path", path.display().to_string()) - .with_context("source", err.to_string()) - })?; - - let mut metadata: serde_json::Value = serde_json::from_str(&original).map_err(|err| { - enverr!(ErrorCode::Io, "failed to parse trace metadata JSON") - .with_context("path", path.display().to_string()) - .with_context("source", err.to_string()) - })?; - - let filters = engine.summary(); - let filters_json: Vec = filters - .entries - .iter() - .map(|entry| { - json!({ - "path": entry.path.to_string_lossy(), - "sha256": entry.sha256, - "name": entry.name, - "version": entry.version, - }) - }) - .collect(); - - if let serde_json::Value::Object(ref mut obj) = metadata { - obj.insert( - "trace_filter".to_string(), - json!({ - "filters": filters_json, - "stats": self.filter_stats.summary_json(), - }), - ); - let serialised = serde_json::to_string(&metadata).map_err(|err| { - enverr!(ErrorCode::Io, "failed to serialise trace metadata") - .with_context("path", path.display().to_string()) - .with_context("source", err.to_string()) - })?; - fs::write(path, serialised).map_err(|err| { - enverr!(ErrorCode::Io, "failed to write trace metadata") - .with_context("path", path.display().to_string()) - .with_context("source", err.to_string()) - })?; - Ok(()) - } else { - Err( - enverr!(ErrorCode::Io, "trace metadata must be a JSON object") - .with_context("path", path.display().to_string()), - ) - } - } - - fn ensure_function_id( - &mut self, - py: Python<'_>, - code: &CodeObjectWrapper, - ) -> PyResult { - match self.function_ids.entry(code.id()) { - Entry::Occupied(entry) => Ok(*entry.get()), - Entry::Vacant(slot) => { - let name = code.qualname(py)?; - let filename = code.filename(py)?; - let first_line = code.first_line(py)?; - let function_id = TraceWriter::ensure_function_id( - &mut self.writer, - name, - Path::new(filename), - Line(first_line as i64), - ); - Ok(*slot.insert(function_id)) - } - } - } - - fn should_trace_code(&mut self, py: Python<'_>, code: &CodeObjectWrapper) -> ShouldTrace { - let code_id = code.id(); - if self.ignored_code_ids.contains(&code_id) { - return ShouldTrace::SkipAndDisable; - } - - if let Some(resolution) = self.scope_resolution(py, code) { - match resolution.exec() { - ExecDecision::Skip => { - self.scope_cache.remove(&code_id); - self.filter_stats.record_skip(); - self.ignored_code_ids.insert(code_id); - record_dropped_event("filter_scope_skip"); - return ShouldTrace::SkipAndDisable; - } - ExecDecision::Trace => { - // already cached for future use - } - } - } - - let filename = match code.filename(py) { - Ok(name) => name, - Err(err) => { - with_error_code(ErrorCode::Io, || { - let _mute = ScopedMuteIoCapture::new(); - log::error!("failed to resolve code filename: {err}"); - }); - record_dropped_event("filename_lookup_failed"); - self.scope_cache.remove(&code_id); - self.ignored_code_ids.insert(code_id); - return ShouldTrace::SkipAndDisable; - } - }; - if is_real_filename(filename) { - ShouldTrace::Trace - } else { - self.scope_cache.remove(&code_id); - self.ignored_code_ids.insert(code_id); - record_dropped_event("synthetic_filename"); - ShouldTrace::SkipAndDisable - } - } -} - -impl Tracer for RuntimeTracer { - fn interest(&self, events: &MonitoringEvents) -> EventSet { - // Minimal set: function start, step lines, and returns - events_union(&[events.PY_START, events.LINE, events.PY_RETURN]) - } - - fn on_py_start( - &mut self, - py: Python<'_>, - code: &CodeObjectWrapper, - _offset: i32, - ) -> CallbackResult { - let is_active = self.activation.should_process_event(py, code); - if matches!( - self.should_trace_code(py, code), - ShouldTrace::SkipAndDisable - ) { - return Ok(CallbackOutcome::DisableLocation); - } - if !is_active { - return Ok(CallbackOutcome::Continue); - } - - if should_inject_failure(FailureStage::PyStart) { - return Err(injected_failure_err(FailureStage::PyStart)); - } - - if should_inject_target_error() { - return Err(ffi::map_recorder_error( - target!( - ErrorCode::TraceIncomplete, - "test-injected target error from capture_call_arguments" - ) - .with_context("injection_stage", "capture_call_arguments"), - )); - } - - log_event(py, code, "on_py_start", None); - - let scope_resolution = self.scope_cache.get(&code.id()).cloned(); - let value_policy = scope_resolution.as_ref().map(|res| res.value_policy()); - let wants_telemetry = value_policy.is_some(); - - if let Ok(fid) = self.ensure_function_id(py, code) { - let mut telemetry_holder = if wants_telemetry { - Some(self.filter_stats.values_mut()) - } else { - None - }; - let telemetry = telemetry_holder.as_deref_mut(); - match capture_call_arguments(py, &mut self.writer, code, value_policy, telemetry) { - Ok(args) => TraceWriter::register_call(&mut self.writer, fid, args), - Err(err) => { - let details = err.to_string(); - with_error_code(ErrorCode::FrameIntrospectionFailed, || { - let _mute = ScopedMuteIoCapture::new(); - log::error!("on_py_start: failed to capture args: {details}"); - }); - return Err(ffi::map_recorder_error( - enverr!( - ErrorCode::FrameIntrospectionFailed, - "failed to capture call arguments" - ) - .with_context("details", details), - )); - } - } - self.mark_event(); - } - - Ok(CallbackOutcome::Continue) - } - - fn on_line(&mut self, py: Python<'_>, code: &CodeObjectWrapper, lineno: u32) -> CallbackResult { - let is_active = self.activation.should_process_event(py, code); - if matches!( - self.should_trace_code(py, code), - ShouldTrace::SkipAndDisable - ) { - return Ok(CallbackOutcome::DisableLocation); - } - if !is_active { - return Ok(CallbackOutcome::Continue); - } - - if should_inject_failure(FailureStage::Line) { - return Err(injected_failure_err(FailureStage::Line)); - } - - #[cfg(feature = "integration-test")] - { - if should_panic_in_callback() { - panic!("test-injected panic in on_line"); - } - } - - log_event(py, code, "on_line", Some(lineno)); - - self.flush_io_before_step(thread::current().id()); - - let scope_resolution = self.scope_cache.get(&code.id()).cloned(); - let value_policy = scope_resolution.as_ref().map(|res| res.value_policy()); - let wants_telemetry = value_policy.is_some(); - - let line_value = Line(lineno as i64); - let mut recorded_path: Option<(PathId, Line)> = None; - - if let Ok(filename) = code.filename(py) { - let path = Path::new(filename); - let path_id = TraceWriter::ensure_path_id(&mut self.writer, path); - TraceWriter::register_step(&mut self.writer, path, line_value); - self.mark_event(); - recorded_path = Some((path_id, line_value)); - } - - let snapshot = capture_frame(py, code)?; - - if let Some((path_id, line)) = recorded_path { - let frame_id = FrameId::from_raw(snapshot.frame_ptr() as usize as u64); - self.line_snapshots - .record(thread::current().id(), path_id, line, frame_id); - } - - let mut recorded: HashSet = HashSet::new(); - let mut telemetry_holder = if wants_telemetry { - Some(self.filter_stats.values_mut()) - } else { - None - }; - let telemetry = telemetry_holder.as_deref_mut(); - record_visible_scope( - py, - &mut self.writer, - &snapshot, - &mut recorded, - value_policy, - telemetry, - ); - - Ok(CallbackOutcome::Continue) - } - - fn on_py_return( - &mut self, - py: Python<'_>, - code: &CodeObjectWrapper, - _offset: i32, - retval: &Bound<'_, PyAny>, - ) -> CallbackResult { - let is_active = self.activation.should_process_event(py, code); - if matches!( - self.should_trace_code(py, code), - ShouldTrace::SkipAndDisable - ) { - return Ok(CallbackOutcome::DisableLocation); - } - if !is_active { - return Ok(CallbackOutcome::Continue); - } - - log_event(py, code, "on_py_return", None); - - self.flush_pending_io(); - - let scope_resolution = self.scope_cache.get(&code.id()).cloned(); - let value_policy = scope_resolution.as_ref().map(|res| res.value_policy()); - let wants_telemetry = value_policy.is_some(); - let object_name = scope_resolution.as_ref().and_then(|res| res.object_name()); - - let mut telemetry_holder = if wants_telemetry { - Some(self.filter_stats.values_mut()) - } else { - None - }; - let telemetry = telemetry_holder.as_deref_mut(); - - record_return_value( - py, - &mut self.writer, - retval, - value_policy, - telemetry, - object_name, - ); - self.mark_event(); - if self.activation.handle_return_event(code.id()) { - let _mute = ScopedMuteIoCapture::new(); - log::debug!("[RuntimeTracer] deactivated on activation return"); - } - - Ok(CallbackOutcome::Continue) - } - - fn notify_failure(&mut self, _py: Python<'_>) -> PyResult<()> { - self.mark_failure(); - Ok(()) - } - - fn flush(&mut self, _py: Python<'_>) -> PyResult<()> { - // Trace event entry - let _mute = ScopedMuteIoCapture::new(); - log::debug!("[RuntimeTracer] flush"); - drop(_mute); - self.flush_pending_io(); - // For non-streaming formats we can update the events file. - match self.format { - TraceEventsFileFormat::Json | TraceEventsFileFormat::BinaryV0 => { - TraceWriter::finish_writing_trace_events(&mut self.writer).map_err(|err| { - ffi::map_recorder_error( - enverr!(ErrorCode::Io, "failed to finalise trace events") - .with_context("source", err.to_string()), - ) - })?; - } - TraceEventsFileFormat::Binary => { - // Streaming writer: no partial flush to avoid closing the stream. - } - } - self.ignored_code_ids.clear(); - self.scope_cache.clear(); - Ok(()) - } - - fn finish(&mut self, py: Python<'_>) -> PyResult<()> { - // Trace event entry - let _mute_finish = ScopedMuteIoCapture::new(); - log::debug!("[RuntimeTracer] finish"); - - if should_inject_failure(FailureStage::Finish) { - return Err(injected_failure_err(FailureStage::Finish)); - } - - set_active_trace_id(Some(self.trace_id.clone())); - let _reset = TraceIdResetGuard::new(); - let policy = policy_snapshot(); - - self.teardown_io_capture(py); - - if self.encountered_failure { - if policy.keep_partial_trace { - if let Err(err) = self.finalise_writer() { - with_error_code(ErrorCode::TraceIncomplete, || { - log::warn!( - "failed to finalise partial trace after disable: {}", - err.message() - ); - }); - } - if let Some(outputs) = &self.output_paths { - with_error_code(ErrorCode::TraceIncomplete, || { - log::warn!( - "recorder detached after failure; keeping partial trace at {}", - outputs.events().display() - ); - }); - } - } else { - self.cleanup_partial_outputs() - .map_err(ffi::map_recorder_error)?; - } - self.ignored_code_ids.clear(); - self.function_ids.clear(); - self.scope_cache.clear(); - self.line_snapshots.clear(); - self.filter_stats.reset(); - return Ok(()); - } - - self.require_trace_or_fail(&policy) - .map_err(ffi::map_recorder_error)?; - self.finalise_writer().map_err(ffi::map_recorder_error)?; - self.ignored_code_ids.clear(); - self.function_ids.clear(); - self.scope_cache.clear(); - self.filter_stats.reset(); - self.line_snapshots.clear(); - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::monitoring::CallbackOutcome; - use crate::policy; - use crate::trace_filter::config::TraceFilterConfig; - use pyo3::types::{PyAny, PyCode, PyModule}; - use pyo3::wrap_pyfunction; - use runtime_tracing::{FullValueRecord, StepRecord, TraceLowLevelEvent, ValueRecord}; - use serde::Deserialize; - use std::cell::Cell; - use std::collections::BTreeMap; - use std::ffi::CString; - use std::fs; - use std::path::Path; - use std::sync::Arc; - use std::thread; - - thread_local! { - static ACTIVE_TRACER: Cell<*mut RuntimeTracer> = Cell::new(std::ptr::null_mut()); - static LAST_OUTCOME: Cell> = Cell::new(None); - } - - struct ScopedTracer; - - impl ScopedTracer { - fn new(tracer: &mut RuntimeTracer) -> Self { - let ptr = tracer as *mut _; - ACTIVE_TRACER.with(|cell| cell.set(ptr)); - ScopedTracer - } - } - - impl Drop for ScopedTracer { - fn drop(&mut self) { - ACTIVE_TRACER.with(|cell| cell.set(std::ptr::null_mut())); - } - } - - fn last_outcome() -> Option { - LAST_OUTCOME.with(|cell| cell.get()) - } - - fn reset_policy(_py: Python<'_>) { - policy::configure_policy_py( - Some("abort"), - Some(false), - Some(false), - None, - None, - Some(false), - None, - None, - ) - .expect("reset recorder policy"); - } - - #[test] - fn detects_real_filenames() { - assert!(is_real_filename("example.py")); - assert!(is_real_filename(" /tmp/module.py ")); - assert!(is_real_filename("src/.py")); - assert!(!is_real_filename("")); - assert!(!is_real_filename(" ")); - assert!(!is_real_filename("")); - } - - #[test] - fn skips_synthetic_filename_events() { - Python::with_gil(|py| { - let mut tracer = - RuntimeTracer::new("test.py", &[], TraceEventsFileFormat::Json, None, None); - ensure_test_module(py); - let script = format!("{PRELUDE}\nsnapshot()\n"); - { - let _guard = ScopedTracer::new(&mut tracer); - LAST_OUTCOME.with(|cell| cell.set(None)); - let script_c = CString::new(script).expect("script contains nul byte"); - py.run(script_c.as_c_str(), None, None) - .expect("execute synthetic script"); - } - assert!( - tracer.writer.events.is_empty(), - "expected no events for synthetic filename" - ); - assert_eq!(last_outcome(), Some(CallbackOutcome::DisableLocation)); - - let compile_fn = py - .import("builtins") - .expect("import builtins") - .getattr("compile") - .expect("fetch compile"); - let binding = compile_fn - .call1(("pass", "", "exec")) - .expect("compile code object"); - let code_obj = binding.downcast::().expect("downcast code object"); - let wrapper = CodeObjectWrapper::new(py, &code_obj); - assert_eq!( - tracer.should_trace_code(py, &wrapper), - ShouldTrace::SkipAndDisable - ); - }); - } - - #[test] - fn traces_real_file_events() { - let snapshots = run_traced_script("snapshot()\n"); - assert!( - !snapshots.is_empty(), - "expected snapshots for real file execution" - ); - assert_eq!(last_outcome(), Some(CallbackOutcome::Continue)); - } - - #[test] - fn callbacks_do_not_import_sys_monitoring() { - let body = r#" -import builtins -_orig_import = builtins.__import__ - -def guard(name, *args, **kwargs): - if name == "sys.monitoring": - raise RuntimeError("callback imported sys.monitoring") - return _orig_import(name, *args, **kwargs) - -builtins.__import__ = guard -try: - snapshot() -finally: - builtins.__import__ = _orig_import -"#; - let snapshots = run_traced_script(body); - assert!( - !snapshots.is_empty(), - "expected snapshots when import guard active" - ); - assert_eq!(last_outcome(), Some(CallbackOutcome::Continue)); - } - - #[test] - fn records_return_values_and_deactivates_activation() { - Python::with_gil(|py| { - ensure_test_module(py); - let tmp = tempfile::tempdir().expect("create temp dir"); - let script_path = tmp.path().join("activation_script.py"); - let script = format!( - "{PRELUDE}\n\n\ -def compute():\n emit_return(\"tail\")\n return \"tail\"\n\n\ -result = compute()\n" - ); - std::fs::write(&script_path, &script).expect("write script"); - - let program = script_path.to_string_lossy().into_owned(); - let mut tracer = RuntimeTracer::new( - &program, - &[], - TraceEventsFileFormat::Json, - Some(script_path.as_path()), - None, - ); - - { - let _guard = ScopedTracer::new(&mut tracer); - LAST_OUTCOME.with(|cell| cell.set(None)); - let run_code = format!( - "import runpy\nrunpy.run_path(r\"{}\")", - script_path.display() - ); - let run_code_c = CString::new(run_code).expect("script contains nul byte"); - py.run(run_code_c.as_c_str(), None, None) - .expect("execute test script"); - } - - let returns: Vec = tracer - .writer - .events - .iter() - .filter_map(|event| match event { - TraceLowLevelEvent::Return(record) => { - Some(SimpleValue::from_value(&record.return_value)) - } - _ => None, - }) - .collect(); - - assert!( - returns.contains(&SimpleValue::String("tail".to_string())), - "expected recorded string return, got {:?}", - returns - ); - assert_eq!(last_outcome(), Some(CallbackOutcome::Continue)); - assert!(!tracer.activation.is_active()); - }); - } - - #[test] - fn line_snapshot_store_tracks_last_step() { - Python::with_gil(|py| { - ensure_test_module(py); - let tmp = tempfile::tempdir().expect("create temp dir"); - let script_path = tmp.path().join("snapshot_script.py"); - let script = format!("{PRELUDE}\n\nsnapshot()\n"); - std::fs::write(&script_path, &script).expect("write script"); - - let mut tracer = RuntimeTracer::new( - "snapshot_script.py", - &[], - TraceEventsFileFormat::Json, - None, - None, - ); - let store = tracer.line_snapshot_store(); - - { - let _guard = ScopedTracer::new(&mut tracer); - LAST_OUTCOME.with(|cell| cell.set(None)); - let run_code = format!( - "import runpy\nrunpy.run_path(r\"{}\")", - script_path.display() - ); - let run_code_c = CString::new(run_code).expect("script contains nul byte"); - py.run(run_code_c.as_c_str(), None, None) - .expect("execute snapshot script"); - } - - let last_step: StepRecord = tracer - .writer - .events - .iter() - .rev() - .find_map(|event| match event { - TraceLowLevelEvent::Step(step) => Some(step.clone()), - _ => None, - }) - .expect("expected one step event"); - - let thread_id = thread::current().id(); - let snapshot = store - .snapshot_for_thread(thread_id) - .expect("snapshot should be recorded"); - - assert_eq!(snapshot.line(), last_step.line); - assert_eq!(snapshot.path_id(), last_step.path_id); - assert!(snapshot.captured_at().elapsed().as_secs_f64() >= 0.0); - }); - } - - #[derive(Debug, Deserialize)] - struct IoMetadata { - stream: String, - path_id: Option, - line: Option, - flags: Vec, - } - - #[test] - fn io_capture_records_python_and_native_output() { - Python::with_gil(|py| { - reset_policy(py); - policy::configure_policy_py( - Some("abort"), - Some(false), - Some(false), - None, - None, - Some(false), - Some(true), - Some(false), - ) - .expect("enable io capture proxies"); - - ensure_test_module(py); - let tmp = tempfile::tempdir().expect("create temp dir"); - let script_path = tmp.path().join("io_script.py"); - let script = format!( - "{PRELUDE}\n\nprint('python out')\nfrom ctypes import pythonapi, c_char_p\npythonapi.PySys_WriteStdout(c_char_p(b'native out\\n'))\n" - ); - std::fs::write(&script_path, &script).expect("write script"); - - let mut tracer = RuntimeTracer::new( - script_path.to_string_lossy().as_ref(), - &[], - TraceEventsFileFormat::Json, - None, - None, - ); - let outputs = TraceOutputPaths::new(tmp.path(), TraceEventsFileFormat::Json); - tracer.begin(&outputs, 1).expect("begin tracer"); - tracer - .install_io_capture(py, &policy::policy_snapshot()) - .expect("install io capture"); - - { - let _guard = ScopedTracer::new(&mut tracer); - LAST_OUTCOME.with(|cell| cell.set(None)); - let run_code = format!( - "import runpy\nrunpy.run_path(r\"{}\")", - script_path.display() - ); - let run_code_c = CString::new(run_code).expect("script contains nul byte"); - py.run(run_code_c.as_c_str(), None, None) - .expect("execute io script"); - } - - tracer.finish(py).expect("finish tracer"); - - let io_events: Vec<(IoMetadata, Vec)> = tracer - .writer - .events - .iter() - .filter_map(|event| match event { - TraceLowLevelEvent::Event(record) => { - let metadata: IoMetadata = serde_json::from_str(&record.metadata).ok()?; - Some((metadata, record.content.as_bytes().to_vec())) - } - _ => None, - }) - .collect(); - - assert!(io_events - .iter() - .any(|(meta, payload)| meta.stream == "stdout" - && String::from_utf8_lossy(payload).contains("python out"))); - assert!(io_events - .iter() - .any(|(meta, payload)| meta.stream == "stdout" - && String::from_utf8_lossy(payload).contains("native out"))); - assert!(io_events.iter().all(|(meta, _)| { - if meta.stream == "stdout" { - meta.path_id.is_some() && meta.line.is_some() - } else { - true - } - })); - assert!(io_events - .iter() - .filter(|(meta, _)| meta.stream == "stdout") - .any(|(meta, _)| meta.flags.iter().any(|flag| flag == "newline"))); - - reset_policy(py); - }); - } - - #[cfg(unix)] - #[test] - fn fd_mirror_captures_os_write_payloads() { - Python::with_gil(|py| { - reset_policy(py); - policy::configure_policy_py( - Some("abort"), - Some(false), - Some(false), - None, - None, - Some(false), - Some(true), - Some(true), - ) - .expect("enable io capture with fd fallback"); - - ensure_test_module(py); - let tmp = tempfile::tempdir().expect("tempdir"); - let script_path = tmp.path().join("fd_mirror.py"); - std::fs::write( - &script_path, - format!( - "{PRELUDE}\nimport os\nprint('proxy line')\nos.write(1, b'fd stdout\\n')\nos.write(2, b'fd stderr\\n')\n" - ), - ) - .expect("write script"); - - let mut tracer = RuntimeTracer::new( - script_path.to_string_lossy().as_ref(), - &[], - TraceEventsFileFormat::Json, - None, - None, - ); - let outputs = TraceOutputPaths::new(tmp.path(), TraceEventsFileFormat::Json); - tracer.begin(&outputs, 1).expect("begin tracer"); - tracer - .install_io_capture(py, &policy::policy_snapshot()) - .expect("install io capture"); - - { - let _guard = ScopedTracer::new(&mut tracer); - LAST_OUTCOME.with(|cell| cell.set(None)); - let run_code = format!( - "import runpy\nrunpy.run_path(r\"{}\")", - script_path.display() - ); - let run_code_c = CString::new(run_code).expect("script contains nul byte"); - py.run(run_code_c.as_c_str(), None, None) - .expect("execute fd script"); - } - - tracer.finish(py).expect("finish tracer"); - - let io_events: Vec<(IoMetadata, Vec)> = tracer - .writer - .events - .iter() - .filter_map(|event| match event { - TraceLowLevelEvent::Event(record) => { - let metadata: IoMetadata = serde_json::from_str(&record.metadata).ok()?; - Some((metadata, record.content.as_bytes().to_vec())) - } - _ => None, - }) - .collect(); - - let stdout_mirror = io_events.iter().find(|(meta, _)| { - meta.stream == "stdout" && meta.flags.iter().any(|flag| flag == "mirror") - }); - assert!( - stdout_mirror.is_some(), - "expected mirror event for stdout: {:?}", - io_events - ); - let stdout_payload = &stdout_mirror.unwrap().1; - assert!( - String::from_utf8_lossy(stdout_payload).contains("fd stdout"), - "mirror stdout payload missing expected text" - ); - - let stderr_mirror = io_events.iter().find(|(meta, _)| { - meta.stream == "stderr" && meta.flags.iter().any(|flag| flag == "mirror") - }); - assert!( - stderr_mirror.is_some(), - "expected mirror event for stderr: {:?}", - io_events - ); - let stderr_payload = &stderr_mirror.unwrap().1; - assert!( - String::from_utf8_lossy(stderr_payload).contains("fd stderr"), - "mirror stderr payload missing expected text" - ); - - assert!(io_events.iter().any(|(meta, payload)| { - meta.stream == "stdout" - && !meta.flags.iter().any(|flag| flag == "mirror") - && String::from_utf8_lossy(payload).contains("proxy line") - })); - - reset_policy(py); - }); - } - - #[cfg(unix)] - #[test] - fn fd_mirror_disabled_does_not_capture_os_write() { - Python::with_gil(|py| { - reset_policy(py); - policy::configure_policy_py( - Some("abort"), - Some(false), - Some(false), - None, - None, - Some(false), - Some(true), - Some(false), - ) - .expect("enable proxies without fd fallback"); - - ensure_test_module(py); - let tmp = tempfile::tempdir().expect("tempdir"); - let script_path = tmp.path().join("fd_disabled.py"); - std::fs::write( - &script_path, - format!( - "{PRELUDE}\nimport os\nprint('proxy line')\nos.write(1, b'fd stdout\\n')\nos.write(2, b'fd stderr\\n')\n" - ), - ) - .expect("write script"); - - let mut tracer = RuntimeTracer::new( - script_path.to_string_lossy().as_ref(), - &[], - TraceEventsFileFormat::Json, - None, - None, - ); - let outputs = TraceOutputPaths::new(tmp.path(), TraceEventsFileFormat::Json); - tracer.begin(&outputs, 1).expect("begin tracer"); - tracer - .install_io_capture(py, &policy::policy_snapshot()) - .expect("install io capture"); - - { - let _guard = ScopedTracer::new(&mut tracer); - LAST_OUTCOME.with(|cell| cell.set(None)); - let run_code = format!( - "import runpy\nrunpy.run_path(r\"{}\")", - script_path.display() - ); - let run_code_c = CString::new(run_code).expect("script contains nul byte"); - py.run(run_code_c.as_c_str(), None, None) - .expect("execute fd script"); - } - - tracer.finish(py).expect("finish tracer"); - - let io_events: Vec<(IoMetadata, Vec)> = tracer - .writer - .events - .iter() - .filter_map(|event| match event { - TraceLowLevelEvent::Event(record) => { - let metadata: IoMetadata = serde_json::from_str(&record.metadata).ok()?; - Some((metadata, record.content.as_bytes().to_vec())) - } - _ => None, - }) - .collect(); - - assert!( - !io_events - .iter() - .any(|(meta, _)| meta.flags.iter().any(|flag| flag == "mirror")), - "mirror events should not be present when fallback disabled" - ); - - assert!( - !io_events.iter().any(|(_, payload)| { - String::from_utf8_lossy(payload).contains("fd stdout") - || String::from_utf8_lossy(payload).contains("fd stderr") - }), - "native os.write payload unexpectedly captured without fallback" - ); - - assert!(io_events.iter().any(|(meta, payload)| { - meta.stream == "stdout" && String::from_utf8_lossy(payload).contains("proxy line") - })); - - reset_policy(py); - }); - } - - #[pyfunction] - fn capture_line(py: Python<'_>, code: Bound<'_, PyCode>, lineno: u32) -> PyResult<()> { - ffi::wrap_pyfunction("test_capture_line", || { - ACTIVE_TRACER.with(|cell| -> PyResult<()> { - let ptr = cell.get(); - if ptr.is_null() { - panic!("No active RuntimeTracer for capture_line"); - } - unsafe { - let tracer = &mut *ptr; - let wrapper = CodeObjectWrapper::new(py, &code); - match tracer.on_line(py, &wrapper, lineno) { - Ok(outcome) => { - LAST_OUTCOME.with(|cell| cell.set(Some(outcome))); - Ok(()) - } - Err(err) => Err(err), - } - } - })?; - Ok(()) - }) - } - - #[pyfunction] - fn capture_return_event( - py: Python<'_>, - code: Bound<'_, PyCode>, - value: Bound<'_, PyAny>, - ) -> PyResult<()> { - ffi::wrap_pyfunction("test_capture_return_event", || { - ACTIVE_TRACER.with(|cell| -> PyResult<()> { - let ptr = cell.get(); - if ptr.is_null() { - panic!("No active RuntimeTracer for capture_return_event"); - } - unsafe { - let tracer = &mut *ptr; - let wrapper = CodeObjectWrapper::new(py, &code); - match tracer.on_py_return(py, &wrapper, 0, &value) { - Ok(outcome) => { - LAST_OUTCOME.with(|cell| cell.set(Some(outcome))); - Ok(()) - } - Err(err) => Err(err), - } - } - })?; - Ok(()) - }) - } - - const PRELUDE: &str = r#" -import inspect -from test_tracer import capture_line, capture_return_event - -def snapshot(line=None): - frame = inspect.currentframe().f_back - lineno = frame.f_lineno if line is None else line - capture_line(frame.f_code, lineno) - -def snap(value): - frame = inspect.currentframe().f_back - capture_line(frame.f_code, frame.f_lineno) - return value - -def emit_return(value): - frame = inspect.currentframe().f_back - capture_return_event(frame.f_code, value) - return value -"#; - - #[derive(Debug, Clone, PartialEq)] - enum SimpleValue { - None, - Bool(bool), - Int(i64), - String(String), - Tuple(Vec), - Sequence(Vec), - Raw(String), - } - - impl SimpleValue { - fn from_value(value: &ValueRecord) -> Self { - match value { - ValueRecord::None { .. } => SimpleValue::None, - ValueRecord::Bool { b, .. } => SimpleValue::Bool(*b), - ValueRecord::Int { i, .. } => SimpleValue::Int(*i), - ValueRecord::String { text, .. } => SimpleValue::String(text.clone()), - ValueRecord::Tuple { elements, .. } => { - SimpleValue::Tuple(elements.iter().map(SimpleValue::from_value).collect()) - } - ValueRecord::Sequence { elements, .. } => { - SimpleValue::Sequence(elements.iter().map(SimpleValue::from_value).collect()) - } - ValueRecord::Raw { r, .. } => SimpleValue::Raw(r.clone()), - ValueRecord::Error { msg, .. } => SimpleValue::Raw(msg.clone()), - other => SimpleValue::Raw(format!("{other:?}")), - } - } - } - - #[derive(Debug)] - struct Snapshot { - line: i64, - vars: BTreeMap, - } - - fn collect_snapshots(events: &[TraceLowLevelEvent]) -> Vec { - let mut names: Vec = Vec::new(); - let mut snapshots: Vec = Vec::new(); - let mut current: Option = None; - for event in events { - match event { - TraceLowLevelEvent::VariableName(name) => names.push(name.clone()), - TraceLowLevelEvent::Step(step) => { - if let Some(snapshot) = current.take() { - snapshots.push(snapshot); - } - current = Some(Snapshot { - line: step.line.0, - vars: BTreeMap::new(), - }); - } - TraceLowLevelEvent::Value(FullValueRecord { variable_id, value }) => { - if let Some(snapshot) = current.as_mut() { - let index = variable_id.0; - let name = names - .get(index) - .cloned() - .unwrap_or_else(|| panic!("Missing variable name for id {}", index)); - snapshot.vars.insert(name, SimpleValue::from_value(value)); - } - } - _ => {} - } - } - if let Some(snapshot) = current.take() { - snapshots.push(snapshot); - } - snapshots - } - - fn ensure_test_module(py: Python<'_>) { - let module = PyModule::new(py, "test_tracer").expect("create module"); - module - .add_function(wrap_pyfunction!(capture_line, &module).expect("wrap capture_line")) - .expect("add function"); - module - .add_function( - wrap_pyfunction!(capture_return_event, &module).expect("wrap capture_return_event"), - ) - .expect("add return capture function"); - py.import("sys") - .expect("import sys") - .getattr("modules") - .expect("modules attr") - .set_item("test_tracer", module) - .expect("insert module"); - } - - fn run_traced_script(body: &str) -> Vec { - Python::with_gil(|py| { - let mut tracer = - RuntimeTracer::new("test.py", &[], TraceEventsFileFormat::Json, None, None); - ensure_test_module(py); - let tmp = tempfile::tempdir().expect("create temp dir"); - let script_path = tmp.path().join("script.py"); - let script = format!("{PRELUDE}\n{body}"); - std::fs::write(&script_path, &script).expect("write script"); - { - let _guard = ScopedTracer::new(&mut tracer); - LAST_OUTCOME.with(|cell| cell.set(None)); - let run_code = format!( - "import runpy\nrunpy.run_path(r\"{}\")", - script_path.display() - ); - let run_code_c = CString::new(run_code).expect("script contains nul byte"); - py.run(run_code_c.as_c_str(), None, None) - .expect("execute test script"); - } - collect_snapshots(&tracer.writer.events) - }) - } - - fn write_filter(path: &Path, contents: &str) { - fs::write(path, contents.trim_start()).expect("write filter"); - } - - #[test] - fn trace_filter_redacts_values() { - Python::with_gil(|py| { - ensure_test_module(py); - - let project = tempfile::tempdir().expect("project dir"); - let project_root = project.path(); - let filters_dir = project_root.join(".codetracer"); - fs::create_dir(&filters_dir).expect("create .codetracer"); - let filter_path = filters_dir.join("filters.toml"); - write_filter( - &filter_path, - r#" - [meta] - name = "redact" - version = 1 - - [scope] - default_exec = "trace" - default_value_action = "allow" - - [[scope.rules]] - selector = "pkg:app.sec" - exec = "trace" - value_default = "allow" - - [[scope.rules.value_patterns]] - selector = "arg:password" - action = "redact" - - [[scope.rules.value_patterns]] - selector = "local:password" - action = "redact" - - [[scope.rules.value_patterns]] - selector = "local:secret" - action = "redact" - - [[scope.rules.value_patterns]] - selector = "global:shared_secret" - action = "redact" - - [[scope.rules.value_patterns]] - selector = "ret:literal:app.sec.sensitive" - action = "redact" - - [[scope.rules.value_patterns]] - selector = "local:internal" - action = "drop" - "#, - ); - let config = TraceFilterConfig::from_paths(&[filter_path]).expect("load filter"); - let engine = Arc::new(TraceFilterEngine::new(config)); - - let app_dir = project_root.join("app"); - fs::create_dir_all(&app_dir).expect("create app dir"); - let script_path = app_dir.join("sec.py"); - let body = r#" -shared_secret = "initial" - -def sensitive(password): - secret = "token" - internal = "hidden" - public = "visible" - globals()['shared_secret'] = password - snapshot() - emit_return(password) - return password - -sensitive("s3cr3t") -"#; - let script = format!("{PRELUDE}\n{body}", PRELUDE = PRELUDE, body = body); - fs::write(&script_path, script).expect("write script"); - - let mut tracer = RuntimeTracer::new( - script_path.to_string_lossy().as_ref(), - &[], - TraceEventsFileFormat::Json, - None, - Some(engine), - ); - - { - let _guard = ScopedTracer::new(&mut tracer); - LAST_OUTCOME.with(|cell| cell.set(None)); - let run_code = format!( - "import runpy, sys\nsys.path.insert(0, r\"{}\")\nrunpy.run_path(r\"{}\")", - project_root.display(), - script_path.display() - ); - let run_code_c = CString::new(run_code).expect("script contains nul byte"); - py.run(run_code_c.as_c_str(), None, None) - .expect("execute filtered script"); - } - - let mut variable_names: Vec = Vec::new(); - for event in &tracer.writer.events { - if let TraceLowLevelEvent::VariableName(name) = event { - variable_names.push(name.clone()); - } - } - assert!( - !variable_names.iter().any(|name| name == "internal"), - "internal variable should not be recorded" - ); - - let password_index = variable_names - .iter() - .position(|name| name == "password") - .expect("password variable recorded"); - let password_value = tracer - .writer - .events - .iter() - .find_map(|event| match event { - TraceLowLevelEvent::Value(record) if record.variable_id.0 == password_index => { - Some(record.value.clone()) - } - _ => None, - }) - .expect("password value recorded"); - match password_value { - ValueRecord::Error { ref msg, .. } => assert_eq!(msg, ""), - ref other => panic!("expected password argument redacted, got {other:?}"), - } - - let snapshots = collect_snapshots(&tracer.writer.events); - let snapshot = find_snapshot_with_vars( - &snapshots, - &["secret", "public", "shared_secret", "password"], - ); - assert_var( - snapshot, - "secret", - SimpleValue::Raw("".to_string()), - ); - assert_var( - snapshot, - "public", - SimpleValue::String("visible".to_string()), - ); - assert_var( - snapshot, - "shared_secret", - SimpleValue::Raw("".to_string()), - ); - assert_var( - snapshot, - "password", - SimpleValue::Raw("".to_string()), - ); - assert_no_variable(&snapshots, "internal"); - - let return_record = tracer - .writer - .events - .iter() - .find_map(|event| match event { - TraceLowLevelEvent::Return(record) => Some(record.clone()), - _ => None, - }) - .expect("return record"); - - match return_record.return_value { - ValueRecord::Error { ref msg, .. } => assert_eq!(msg, ""), - ref other => panic!("expected redacted return value, got {other:?}"), - } - }); - } - - #[test] - fn trace_filter_metadata_includes_summary() { - Python::with_gil(|py| { - reset_policy(py); - ensure_test_module(py); - - let project = tempfile::tempdir().expect("project dir"); - let project_root = project.path(); - let filters_dir = project_root.join(".codetracer"); - fs::create_dir(&filters_dir).expect("create .codetracer"); - let filter_path = filters_dir.join("filters.toml"); - write_filter( - &filter_path, - r#" - [meta] - name = "redact" - version = 1 - - [scope] - default_exec = "trace" - default_value_action = "allow" - - [[scope.rules]] - selector = "pkg:app.sec" - exec = "trace" - value_default = "allow" - - [[scope.rules.value_patterns]] - selector = "arg:password" - action = "redact" - - [[scope.rules.value_patterns]] - selector = "local:password" - action = "redact" - - [[scope.rules.value_patterns]] - selector = "local:secret" - action = "redact" - - [[scope.rules.value_patterns]] - selector = "global:shared_secret" - action = "redact" - - [[scope.rules.value_patterns]] - selector = "ret:literal:app.sec.sensitive" - action = "redact" - - [[scope.rules.value_patterns]] - selector = "local:internal" - action = "drop" - "#, - ); - let config = TraceFilterConfig::from_paths(&[filter_path]).expect("load filter"); - let engine = Arc::new(TraceFilterEngine::new(config)); - - let app_dir = project_root.join("app"); - fs::create_dir_all(&app_dir).expect("create app dir"); - let script_path = app_dir.join("sec.py"); - let body = r#" -shared_secret = "initial" - -def sensitive(password): - secret = "token" - internal = "hidden" - public = "visible" - globals()['shared_secret'] = password - snapshot() - emit_return(password) - return password - -sensitive("s3cr3t") -"#; - let script = format!("{PRELUDE}\n{body}", PRELUDE = PRELUDE, body = body); - fs::write(&script_path, script).expect("write script"); - - let outputs_dir = tempfile::tempdir().expect("outputs dir"); - let outputs = TraceOutputPaths::new(outputs_dir.path(), TraceEventsFileFormat::Json); - - let program = script_path.to_string_lossy().into_owned(); - let mut tracer = RuntimeTracer::new( - &program, - &[], - TraceEventsFileFormat::Json, - None, - Some(engine), - ); - tracer.begin(&outputs, 1).expect("begin tracer"); - - { - let _guard = ScopedTracer::new(&mut tracer); - LAST_OUTCOME.with(|cell| cell.set(None)); - let run_code = format!( - "import runpy, sys\nsys.path.insert(0, r\"{}\")\nrunpy.run_path(r\"{}\")", - project_root.display(), - script_path.display() - ); - let run_code_c = CString::new(run_code).expect("script contains nul byte"); - py.run(run_code_c.as_c_str(), None, None) - .expect("execute script"); - } - - tracer.finish(py).expect("finish tracer"); - - let metadata_str = fs::read_to_string(outputs.metadata()).expect("read metadata"); - let metadata: serde_json::Value = - serde_json::from_str(&metadata_str).expect("parse metadata"); - let trace_filter = metadata - .get("trace_filter") - .and_then(|value| value.as_object()) - .expect("trace_filter metadata"); - - let filters = trace_filter - .get("filters") - .and_then(|value| value.as_array()) - .expect("filters array"); - assert_eq!(filters.len(), 1); - let filter_entry = filters[0].as_object().expect("filter entry"); - assert_eq!( - filter_entry.get("name").and_then(|v| v.as_str()), - Some("redact") - ); - - let stats = trace_filter - .get("stats") - .and_then(|value| value.as_object()) - .expect("stats object"); - assert_eq!( - stats.get("scopes_skipped").and_then(|v| v.as_u64()), - Some(0) - ); - let value_redactions = stats - .get("value_redactions") - .and_then(|value| value.as_object()) - .expect("value_redactions object"); - assert_eq!( - value_redactions.get("argument").and_then(|v| v.as_u64()), - Some(0) - ); - // Argument values currently surface through local snapshots; once call-record redaction wiring lands this count should rise above zero. - assert_eq!( - value_redactions.get("local").and_then(|v| v.as_u64()), - Some(2) - ); - assert_eq!( - value_redactions.get("global").and_then(|v| v.as_u64()), - Some(1) - ); - assert_eq!( - value_redactions.get("return").and_then(|v| v.as_u64()), - Some(1) - ); - assert_eq!( - value_redactions.get("attribute").and_then(|v| v.as_u64()), - Some(0) - ); - let value_drops = stats - .get("value_drops") - .and_then(|value| value.as_object()) - .expect("value_drops object"); - assert_eq!( - value_drops.get("argument").and_then(|v| v.as_u64()), - Some(0) - ); - assert_eq!(value_drops.get("local").and_then(|v| v.as_u64()), Some(1)); - assert_eq!(value_drops.get("global").and_then(|v| v.as_u64()), Some(0)); - assert_eq!(value_drops.get("return").and_then(|v| v.as_u64()), Some(0)); - assert_eq!( - value_drops.get("attribute").and_then(|v| v.as_u64()), - Some(0) - ); - }); - } - - fn assert_var(snapshot: &Snapshot, name: &str, expected: SimpleValue) { - let actual = snapshot - .vars - .get(name) - .unwrap_or_else(|| panic!("{name} missing at line {}", snapshot.line)); - assert_eq!( - actual, &expected, - "Unexpected value for {name} at line {}", - snapshot.line - ); - } - - fn find_snapshot_with_vars<'a>(snapshots: &'a [Snapshot], names: &[&str]) -> &'a Snapshot { - snapshots - .iter() - .find(|snap| names.iter().all(|n| snap.vars.contains_key(*n))) - .unwrap_or_else(|| panic!("No snapshot containing variables {:?}", names)) - } - - fn assert_no_variable(snapshots: &[Snapshot], name: &str) { - if snapshots.iter().any(|snap| snap.vars.contains_key(name)) { - panic!("Variable {name} unexpectedly captured"); - } - } - - #[test] - fn captures_simple_function_locals() { - let snapshots = run_traced_script( - r#" -def simple_function(x): - snapshot() - a = 1 - snapshot() - b = a + x - snapshot() - return a, b - -simple_function(5) -"#, - ); - - assert_var(&snapshots[0], "x", SimpleValue::Int(5)); - assert!(!snapshots[0].vars.contains_key("a")); - assert_var(&snapshots[1], "a", SimpleValue::Int(1)); - assert_var(&snapshots[2], "b", SimpleValue::Int(6)); - } - - #[test] - fn captures_closure_variables() { - let snapshots = run_traced_script( - r#" -def outer_func(x): - snapshot() - y = 1 - snapshot() - def inner_func(z): - nonlocal y - snapshot() - w = x + y + z - snapshot() - y = w - snapshot() - return w - total = inner_func(5) - snapshot() - return y, total - -result = outer_func(2) -"#, - ); - - let inner_entry = find_snapshot_with_vars(&snapshots, &["x", "y", "z"]); - assert_var(inner_entry, "x", SimpleValue::Int(2)); - assert_var(inner_entry, "y", SimpleValue::Int(1)); - - let w_snapshot = find_snapshot_with_vars(&snapshots, &["w", "x", "y", "z"]); - assert_var(w_snapshot, "w", SimpleValue::Int(8)); - - let outer_after = find_snapshot_with_vars(&snapshots, &["total", "y"]); - assert_var(outer_after, "total", SimpleValue::Int(8)); - assert_var(outer_after, "y", SimpleValue::Int(8)); - } - - #[test] - fn captures_globals() { - let snapshots = run_traced_script( - r#" -GLOBAL_VAL = 10 -counter = 0 -snapshot() - -def global_test(): - snapshot() - local_copy = GLOBAL_VAL - snapshot() - global counter - counter += 1 - snapshot() - return local_copy, counter - -before = counter -snapshot() -result = global_test() -snapshot() -after = counter -snapshot() -"#, - ); - - let access_global = find_snapshot_with_vars(&snapshots, &["local_copy", "GLOBAL_VAL"]); - assert_var(access_global, "GLOBAL_VAL", SimpleValue::Int(10)); - assert_var(access_global, "local_copy", SimpleValue::Int(10)); - - let last_counter = snapshots - .iter() - .rev() - .find(|snap| snap.vars.contains_key("counter")) - .expect("Expected at least one counter snapshot"); - assert_var(last_counter, "counter", SimpleValue::Int(1)); - } - - #[test] - fn captures_class_scope() { - let snapshots = run_traced_script( - r#" -CONSTANT = 42 -snapshot() - -class MetaCounter(type): - count = 0 - snapshot() - def __init__(cls, name, bases, attrs): - snapshot() - MetaCounter.count += 1 - super().__init__(name, bases, attrs) - -class Sample(metaclass=MetaCounter): - snapshot() - a = 10 - snapshot() - b = a + 5 - snapshot() - print(a, b, CONSTANT) - snapshot() - def method(self): - snapshot() - return self.a + self.b - -instance = Sample() -snapshot() -instances = MetaCounter.count -snapshot() -_ = instance.method() -snapshot() -"#, - ); - - let meta_init = find_snapshot_with_vars(&snapshots, &["cls", "name", "attrs"]); - assert_var(meta_init, "name", SimpleValue::String("Sample".to_string())); - - let class_body = find_snapshot_with_vars(&snapshots, &["a", "b"]); - assert_var(class_body, "a", SimpleValue::Int(10)); - assert_var(class_body, "b", SimpleValue::Int(15)); - - let method_snapshot = find_snapshot_with_vars(&snapshots, &["self"]); - assert!(method_snapshot.vars.contains_key("self")); - } - - #[test] - fn captures_lambda_and_comprehensions() { - let snapshots = run_traced_script( - r#" -factor = 2 -snapshot() -double = lambda y: snap(y * factor) -snapshot() -lambda_value = double(5) -snapshot() -squares = [snap(n ** 2) for n in range(3)] -snapshot() -scaled_set = {snap(n * factor) for n in range(3)} -snapshot() -mapping = {n: snap(n * factor) for n in range(3)} -snapshot() -gen_exp = (snap(n * factor) for n in range(3)) -snapshot() -result_list = list(gen_exp) -snapshot() -"#, - ); - - let lambda_snapshot = find_snapshot_with_vars(&snapshots, &["y", "factor"]); - assert_var(lambda_snapshot, "y", SimpleValue::Int(5)); - assert_var(lambda_snapshot, "factor", SimpleValue::Int(2)); - - let list_comp = find_snapshot_with_vars(&snapshots, &["n", "factor"]); - assert!(matches!(list_comp.vars.get("n"), Some(SimpleValue::Int(_)))); - - let result_snapshot = find_snapshot_with_vars(&snapshots, &["result_list"]); - assert!(matches!( - result_snapshot.vars.get("result_list"), - Some(SimpleValue::Sequence(_)) - )); - } - - #[test] - fn captures_generators_and_coroutines() { - let snapshots = run_traced_script( - r#" -import asyncio -snapshot() - - -def counter_gen(n): - snapshot() - total = 0 - for i in range(n): - total += i - snapshot() - yield total - snapshot() - return total - -async def async_sum(data): - snapshot() - total = 0 - for x in data: - total += x - snapshot() - await asyncio.sleep(0) - snapshot() - return total - -gen = counter_gen(3) -gen_results = list(gen) -snapshot() -coroutine_result = asyncio.run(async_sum([1, 2, 3])) -snapshot() -"#, - ); - - let generator_step = find_snapshot_with_vars(&snapshots, &["i", "total"]); - assert!(matches!( - generator_step.vars.get("i"), - Some(SimpleValue::Int(_)) - )); - - let coroutine_steps: Vec<&Snapshot> = snapshots - .iter() - .filter(|snap| snap.vars.contains_key("x")) - .collect(); - assert!(!coroutine_steps.is_empty()); - let final_coroutine_step = coroutine_steps.last().unwrap(); - assert_var(final_coroutine_step, "total", SimpleValue::Int(6)); - - let coroutine_result_snapshot = find_snapshot_with_vars(&snapshots, &["coroutine_result"]); - assert!(coroutine_result_snapshot - .vars - .contains_key("coroutine_result")); - } - - #[test] - fn captures_exception_and_with_blocks() { - let snapshots = run_traced_script( - r#" -import io -__file__ = "test_script.py" - -def exception_and_with_demo(x): - snapshot() - try: - inv = 10 / x - snapshot() - except ZeroDivisionError as e: - snapshot() - error_msg = f"Error: {e}" - snapshot() - else: - snapshot() - inv += 1 - snapshot() - finally: - snapshot() - final_flag = True - snapshot() - with io.StringIO("dummy line") as f: - snapshot() - first_line = f.readline() - snapshot() - snapshot() - return locals() - -result1 = exception_and_with_demo(0) -snapshot() -result2 = exception_and_with_demo(5) -snapshot() -"#, - ); - - let except_snapshot = find_snapshot_with_vars(&snapshots, &["e", "error_msg"]); - assert!(matches!( - except_snapshot.vars.get("error_msg"), - Some(SimpleValue::String(_)) - )); - - let finally_snapshot = find_snapshot_with_vars(&snapshots, &["final_flag"]); - assert_var(finally_snapshot, "final_flag", SimpleValue::Bool(true)); - - let with_snapshot = find_snapshot_with_vars(&snapshots, &["f", "first_line"]); - assert!(with_snapshot.vars.contains_key("first_line")); - } - - #[test] - fn captures_decorators() { - let snapshots = run_traced_script( - r#" -setting = "Hello" -snapshot() - - -def my_decorator(func): - snapshot() - def wrapper(*args, **kwargs): - snapshot() - return func(*args, **kwargs) - return wrapper - -@my_decorator -def greet(name): - snapshot() - message = f"Hi, {name}" - snapshot() - return message - -output = greet("World") -snapshot() -"#, - ); - - let decorator_snapshot = find_snapshot_with_vars(&snapshots, &["func", "setting"]); - assert!(decorator_snapshot.vars.contains_key("func")); - - let wrapper_snapshot = find_snapshot_with_vars(&snapshots, &["args", "kwargs", "setting"]); - assert!(wrapper_snapshot.vars.contains_key("args")); - - let greet_snapshot = find_snapshot_with_vars(&snapshots, &["name", "message"]); - assert_var( - greet_snapshot, - "name", - SimpleValue::String("World".to_string()), - ); - } - - #[test] - fn captures_dynamic_execution() { - let snapshots = run_traced_script( - r#" -expr_code = "dynamic_var = 99" -snapshot() -exec(expr_code) -snapshot() -check = dynamic_var + 1 -snapshot() - -def eval_test(): - snapshot() - value = 10 - formula = "value * 2" - snapshot() - result = eval(formula) - snapshot() - return result - -out = eval_test() -snapshot() -"#, - ); - - let exec_snapshot = find_snapshot_with_vars(&snapshots, &["dynamic_var"]); - assert_var(exec_snapshot, "dynamic_var", SimpleValue::Int(99)); - - let eval_snapshot = find_snapshot_with_vars(&snapshots, &["value", "formula"]); - assert_var(eval_snapshot, "value", SimpleValue::Int(10)); - } - - #[test] - fn captures_imports() { - let snapshots = run_traced_script( - r#" -import math -snapshot() - -def import_test(): - snapshot() - import os - snapshot() - constant = math.pi - snapshot() - cwd = os.getcwd() - snapshot() - return constant, cwd - -val, path = import_test() -snapshot() -"#, - ); - - let global_import = find_snapshot_with_vars(&snapshots, &["math"]); - assert!(matches!( - global_import.vars.get("math"), - Some(SimpleValue::Raw(_)) - )); - - let local_import = find_snapshot_with_vars(&snapshots, &["os", "constant"]); - assert!(local_import.vars.contains_key("os")); - } - - #[test] - fn builtins_not_recorded() { - let snapshots = run_traced_script( - r#" -def builtins_test(seq): - snapshot() - n = len(seq) - snapshot() - m = max(seq) - snapshot() - return n, m - -result = builtins_test([5, 3, 7]) -snapshot() -"#, - ); - - let len_snapshot = find_snapshot_with_vars(&snapshots, &["n"]); - assert_var(len_snapshot, "n", SimpleValue::Int(3)); - assert_no_variable(&snapshots, "len"); - } - - #[test] - fn finish_enforces_require_trace_policy() { - Python::with_gil(|py| { - policy::configure_policy_py( - Some("abort"), - Some(true), - Some(false), - None, - None, - Some(false), - None, - None, - ) - .expect("enable require_trace policy"); - - let script_dir = tempfile::tempdir().expect("script dir"); - let program_path = script_dir.path().join("program.py"); - std::fs::write(&program_path, "print('hi')\n").expect("write program"); - - let outputs_dir = tempfile::tempdir().expect("outputs dir"); - let outputs = TraceOutputPaths::new(outputs_dir.path(), TraceEventsFileFormat::Json); - - let mut tracer = RuntimeTracer::new( - program_path.to_string_lossy().as_ref(), - &[], - TraceEventsFileFormat::Json, - None, - None, - ); - tracer.begin(&outputs, 1).expect("begin tracer"); - - let err = tracer - .finish(py) - .expect_err("finish should error when require_trace true"); - let message = err.to_string(); - assert!( - message.contains("ERR_TRACE_MISSING"), - "expected trace missing error, got {message}" - ); - - reset_policy(py); - }); - } - - #[test] - fn finish_removes_partial_outputs_when_policy_forbids_keep() { - Python::with_gil(|py| { - reset_policy(py); - - let script_dir = tempfile::tempdir().expect("script dir"); - let program_path = script_dir.path().join("program.py"); - std::fs::write(&program_path, "print('hi')\n").expect("write program"); - - let outputs_dir = tempfile::tempdir().expect("outputs dir"); - let outputs = TraceOutputPaths::new(outputs_dir.path(), TraceEventsFileFormat::Json); - - let mut tracer = RuntimeTracer::new( - program_path.to_string_lossy().as_ref(), - &[], - TraceEventsFileFormat::Json, - None, - None, - ); - tracer.begin(&outputs, 1).expect("begin tracer"); - tracer.mark_failure(); - - tracer.finish(py).expect("finish after failure"); - - assert!(!outputs.events().exists(), "expected events file removed"); - assert!( - !outputs.metadata().exists(), - "expected metadata file removed" - ); - assert!(!outputs.paths().exists(), "expected paths file removed"); - }); - } - - #[test] - fn finish_keeps_partial_outputs_when_policy_allows() { - Python::with_gil(|py| { - policy::configure_policy_py( - Some("abort"), - Some(false), - Some(true), - None, - None, - Some(false), - None, - None, - ) - .expect("enable keep_partial policy"); - - let script_dir = tempfile::tempdir().expect("script dir"); - let program_path = script_dir.path().join("program.py"); - std::fs::write(&program_path, "print('hi')\n").expect("write program"); - - let outputs_dir = tempfile::tempdir().expect("outputs dir"); - let outputs = TraceOutputPaths::new(outputs_dir.path(), TraceEventsFileFormat::Json); - - let mut tracer = RuntimeTracer::new( - program_path.to_string_lossy().as_ref(), - &[], - TraceEventsFileFormat::Json, - None, - None, - ); - tracer.begin(&outputs, 1).expect("begin tracer"); - tracer.mark_failure(); - - tracer.finish(py).expect("finish after failure"); - - assert!(outputs.events().exists(), "expected events file retained"); - assert!( - outputs.metadata().exists(), - "expected metadata file retained" - ); - assert!(outputs.paths().exists(), "expected paths file retained"); - - reset_policy(py); - }); - } -} +pub use tracer::RuntimeTracer; diff --git a/codetracer-python-recorder/src/runtime/tracer/events.rs b/codetracer-python-recorder/src/runtime/tracer/events.rs new file mode 100644 index 0000000..8b1db99 --- /dev/null +++ b/codetracer-python-recorder/src/runtime/tracer/events.rs @@ -0,0 +1,452 @@ +//! Event handling pipeline for `RuntimeTracer`. + +use super::filtering::TraceDecision; +use super::runtime_tracer::RuntimeTracer; +use crate::code_object::CodeObjectWrapper; +use crate::ffi; +use crate::logging::with_error_code; +use crate::monitoring::{ + events_union, CallbackOutcome, CallbackResult, EventSet, MonitoringEvents, Tracer, +}; +use crate::policy::policy_snapshot; +use crate::runtime::frame_inspector::capture_frame; +use crate::runtime::io_capture::ScopedMuteIoCapture; +use crate::runtime::line_snapshots::FrameId; +use crate::runtime::logging::log_event; +use crate::runtime::value_capture::{ + capture_call_arguments, record_return_value, record_visible_scope, +}; +use pyo3::prelude::*; +use pyo3::types::PyAny; +use recorder_errors::{bug, enverr, target, ErrorCode}; +use runtime_tracing::{Line, PathId, TraceEventsFileFormat, TraceWriter}; +use std::collections::HashSet; +use std::path::Path; +use std::thread; + +#[cfg(feature = "integration-test")] +use std::sync::atomic::{AtomicBool, Ordering}; +#[cfg(feature = "integration-test")] +use std::sync::OnceLock; + +#[cfg(feature = "integration-test")] +static FAILURE_MODE: OnceLock> = OnceLock::new(); +#[cfg(feature = "integration-test")] +static FAILURE_TRIGGERED: AtomicBool = AtomicBool::new(false); + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum FailureStage { + PyStart, + Line, + Finish, +} + +impl FailureStage { + fn as_str(self) -> &'static str { + match self { + FailureStage::PyStart => "py_start", + FailureStage::Line => "line", + FailureStage::Finish => "finish", + } + } +} + +// Failure injection helpers are only compiled for integration tests. +#[cfg_attr(not(feature = "integration-test"), allow(dead_code))] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum FailureMode { + Stage(FailureStage), + SuppressEvents, + TargetArgs, + Panic, +} + +#[cfg(feature = "integration-test")] +fn configured_failure_mode() -> Option { + *FAILURE_MODE.get_or_init(|| { + let raw = std::env::var("CODETRACER_TEST_INJECT_FAILURE").ok(); + if let Some(value) = raw.as_deref() { + let _mute = ScopedMuteIoCapture::new(); + log::debug!("[RuntimeTracer] test failure injection mode: {}", value); + } + raw.and_then(|raw| match raw.trim().to_ascii_lowercase().as_str() { + "py_start" | "py-start" => Some(FailureMode::Stage(FailureStage::PyStart)), + "line" => Some(FailureMode::Stage(FailureStage::Line)), + "finish" => Some(FailureMode::Stage(FailureStage::Finish)), + "suppress-events" | "suppress_events" | "suppress" => Some(FailureMode::SuppressEvents), + "target" | "target-args" | "target_args" => Some(FailureMode::TargetArgs), + "panic" | "panic-callback" | "panic_callback" => Some(FailureMode::Panic), + _ => None, + }) + }) +} + +#[cfg(feature = "integration-test")] +fn should_inject_failure(stage: FailureStage) -> bool { + matches!(configured_failure_mode(), Some(FailureMode::Stage(mode)) if mode == stage) + && mark_failure_triggered() +} + +#[cfg(not(feature = "integration-test"))] +fn should_inject_failure(_stage: FailureStage) -> bool { + false +} + +#[cfg(feature = "integration-test")] +fn should_inject_target_error() -> bool { + matches!(configured_failure_mode(), Some(FailureMode::TargetArgs)) && mark_failure_triggered() +} + +#[cfg(not(feature = "integration-test"))] +fn should_inject_target_error() -> bool { + false +} + +#[cfg(feature = "integration-test")] +fn should_panic_in_callback() -> bool { + matches!(configured_failure_mode(), Some(FailureMode::Panic)) && mark_failure_triggered() +} + +#[cfg(not(feature = "integration-test"))] +#[allow(dead_code)] +fn should_panic_in_callback() -> bool { + false +} + +#[cfg(feature = "integration-test")] +fn mark_failure_triggered() -> bool { + !FAILURE_TRIGGERED.swap(true, Ordering::SeqCst) +} + +#[cfg(not(feature = "integration-test"))] +#[allow(dead_code)] +fn mark_failure_triggered() -> bool { + false +} + +#[cfg(feature = "integration-test")] +fn injected_failure_err(stage: FailureStage) -> PyErr { + let err = bug!( + ErrorCode::TraceIncomplete, + "test-injected failure at {}", + stage.as_str() + ) + .with_context("injection_stage", stage.as_str().to_string()); + ffi::map_recorder_error(err) +} + +#[cfg(not(feature = "integration-test"))] +fn injected_failure_err(stage: FailureStage) -> PyErr { + let err = bug!( + ErrorCode::TraceIncomplete, + "failure injection requested at {} without fail-injection feature", + stage.as_str() + ) + .with_context("injection_stage", stage.as_str().to_string()); + ffi::map_recorder_error(err) +} + +#[cfg(feature = "integration-test")] +pub(super) fn suppress_events() -> bool { + matches!(configured_failure_mode(), Some(FailureMode::SuppressEvents)) +} + +#[cfg(not(feature = "integration-test"))] +pub(super) fn suppress_events() -> bool { + false +} + +impl Tracer for RuntimeTracer { + fn interest(&self, events: &MonitoringEvents) -> EventSet { + // Minimal set: function start, step lines, and returns + events_union(&[events.PY_START, events.LINE, events.PY_RETURN]) + } + + fn on_py_start( + &mut self, + py: Python<'_>, + code: &CodeObjectWrapper, + _offset: i32, + ) -> CallbackResult { + let is_active = self + .lifecycle + .activation_mut() + .should_process_event(py, code); + if matches!( + self.should_trace_code(py, code), + TraceDecision::SkipAndDisable + ) { + return Ok(CallbackOutcome::DisableLocation); + } + if !is_active { + return Ok(CallbackOutcome::Continue); + } + + if should_inject_failure(FailureStage::PyStart) { + return Err(injected_failure_err(FailureStage::PyStart)); + } + + if should_inject_target_error() { + return Err(ffi::map_recorder_error( + target!( + ErrorCode::TraceIncomplete, + "test-injected target error from capture_call_arguments" + ) + .with_context("injection_stage", "capture_call_arguments"), + )); + } + + log_event(py, code, "on_py_start", None); + + let scope_resolution = self.filter.cached_resolution(code.id()); + let value_policy = scope_resolution.as_ref().map(|res| res.value_policy()); + let wants_telemetry = value_policy.is_some(); + + if let Ok(fid) = self.ensure_function_id(py, code) { + let mut telemetry_holder = if wants_telemetry { + Some(self.filter.values_mut()) + } else { + None + }; + let telemetry = telemetry_holder.as_deref_mut(); + match capture_call_arguments(py, &mut self.writer, code, value_policy, telemetry) { + Ok(args) => TraceWriter::register_call(&mut self.writer, fid, args), + Err(err) => { + let details = err.to_string(); + with_error_code(ErrorCode::FrameIntrospectionFailed, || { + let _mute = ScopedMuteIoCapture::new(); + log::error!("on_py_start: failed to capture args: {details}"); + }); + return Err(ffi::map_recorder_error( + enverr!( + ErrorCode::FrameIntrospectionFailed, + "failed to capture call arguments" + ) + .with_context("details", details), + )); + } + } + self.mark_event(); + } + + Ok(CallbackOutcome::Continue) + } + + fn on_line(&mut self, py: Python<'_>, code: &CodeObjectWrapper, lineno: u32) -> CallbackResult { + let is_active = self + .lifecycle + .activation_mut() + .should_process_event(py, code); + if matches!( + self.should_trace_code(py, code), + TraceDecision::SkipAndDisable + ) { + return Ok(CallbackOutcome::DisableLocation); + } + if !is_active { + return Ok(CallbackOutcome::Continue); + } + + if should_inject_failure(FailureStage::Line) { + return Err(injected_failure_err(FailureStage::Line)); + } + + #[cfg(feature = "integration-test")] + { + if should_panic_in_callback() { + panic!("test-injected panic in on_line"); + } + } + + log_event(py, code, "on_line", Some(lineno)); + + self.flush_io_before_step(thread::current().id()); + + let scope_resolution = self.filter.cached_resolution(code.id()); + let value_policy = scope_resolution.as_ref().map(|res| res.value_policy()); + let wants_telemetry = value_policy.is_some(); + + let line_value = Line(lineno as i64); + let mut recorded_path: Option<(PathId, Line)> = None; + + if let Ok(filename) = code.filename(py) { + let path = Path::new(filename); + let path_id = TraceWriter::ensure_path_id(&mut self.writer, path); + TraceWriter::register_step(&mut self.writer, path, line_value); + self.mark_event(); + recorded_path = Some((path_id, line_value)); + } + + let snapshot = capture_frame(py, code)?; + + if let Some((path_id, line)) = recorded_path { + let frame_id = FrameId::from_raw(snapshot.frame_ptr() as usize as u64); + self.io + .record_snapshot(thread::current().id(), path_id, line, frame_id); + } + + let mut recorded: HashSet = HashSet::new(); + let mut telemetry_holder = if wants_telemetry { + Some(self.filter.values_mut()) + } else { + None + }; + let telemetry = telemetry_holder.as_deref_mut(); + record_visible_scope( + py, + &mut self.writer, + &snapshot, + &mut recorded, + value_policy, + telemetry, + ); + + Ok(CallbackOutcome::Continue) + } + + fn on_py_return( + &mut self, + py: Python<'_>, + code: &CodeObjectWrapper, + _offset: i32, + retval: &Bound<'_, PyAny>, + ) -> CallbackResult { + let is_active = self + .lifecycle + .activation_mut() + .should_process_event(py, code); + if matches!( + self.should_trace_code(py, code), + TraceDecision::SkipAndDisable + ) { + return Ok(CallbackOutcome::DisableLocation); + } + if !is_active { + return Ok(CallbackOutcome::Continue); + } + + log_event(py, code, "on_py_return", None); + + self.flush_pending_io(); + + let scope_resolution = self.filter.cached_resolution(code.id()); + let value_policy = scope_resolution.as_ref().map(|res| res.value_policy()); + let wants_telemetry = value_policy.is_some(); + let object_name = scope_resolution.as_ref().and_then(|res| res.object_name()); + + let mut telemetry_holder = if wants_telemetry { + Some(self.filter.values_mut()) + } else { + None + }; + let telemetry = telemetry_holder.as_deref_mut(); + + record_return_value( + py, + &mut self.writer, + retval, + value_policy, + telemetry, + object_name, + ); + self.mark_event(); + if self + .lifecycle + .activation_mut() + .handle_return_event(code.id()) + { + let _mute = ScopedMuteIoCapture::new(); + log::debug!("[RuntimeTracer] deactivated on activation return"); + } + + Ok(CallbackOutcome::Continue) + } + + fn notify_failure(&mut self, _py: Python<'_>) -> PyResult<()> { + self.mark_failure(); + Ok(()) + } + + fn flush(&mut self, _py: Python<'_>) -> PyResult<()> { + // Trace event entry + let _mute = ScopedMuteIoCapture::new(); + log::debug!("[RuntimeTracer] flush"); + drop(_mute); + self.flush_pending_io(); + // For non-streaming formats we can update the events file. + match self.format { + TraceEventsFileFormat::Json | TraceEventsFileFormat::BinaryV0 => { + TraceWriter::finish_writing_trace_events(&mut self.writer).map_err(|err| { + ffi::map_recorder_error( + enverr!(ErrorCode::Io, "failed to finalise trace events") + .with_context("source", err.to_string()), + ) + })?; + } + TraceEventsFileFormat::Binary => { + // Streaming writer: no partial flush to avoid closing the stream. + } + } + self.filter.clear_caches(); + Ok(()) + } + + fn finish(&mut self, py: Python<'_>) -> PyResult<()> { + // Trace event entry + let _mute_finish = ScopedMuteIoCapture::new(); + log::debug!("[RuntimeTracer] finish"); + + if should_inject_failure(FailureStage::Finish) { + return Err(injected_failure_err(FailureStage::Finish)); + } + + let _trace_scope = self.lifecycle.trace_id_scope(); + let policy = policy_snapshot(); + + if self.io.teardown(py, &mut self.writer) { + self.mark_event(); + } + + if self.lifecycle.encountered_failure() { + if policy.keep_partial_trace { + if let Err(err) = self.lifecycle.finalise(&mut self.writer, &self.filter) { + with_error_code(ErrorCode::TraceIncomplete, || { + log::warn!( + "failed to finalise partial trace after disable: {}", + err.message() + ); + }); + } + if let Some(outputs) = self.lifecycle.output_paths() { + with_error_code(ErrorCode::TraceIncomplete, || { + log::warn!( + "recorder detached after failure; keeping partial trace at {}", + outputs.events().display() + ); + }); + } + } else { + self.lifecycle + .cleanup_partial_outputs() + .map_err(ffi::map_recorder_error)?; + } + self.function_ids.clear(); + self.io.clear_snapshots(); + self.filter.reset(); + self.lifecycle.reset_event_state(); + return Ok(()); + } + + self.lifecycle + .require_trace_or_fail(&policy) + .map_err(ffi::map_recorder_error)?; + self.lifecycle + .finalise(&mut self.writer, &self.filter) + .map_err(ffi::map_recorder_error)?; + self.function_ids.clear(); + self.filter.reset(); + self.io.clear_snapshots(); + self.lifecycle.reset_event_state(); + Ok(()) + } +} diff --git a/codetracer-python-recorder/src/runtime/tracer/filtering.rs b/codetracer-python-recorder/src/runtime/tracer/filtering.rs new file mode 100644 index 0000000..c4fd804 --- /dev/null +++ b/codetracer-python-recorder/src/runtime/tracer/filtering.rs @@ -0,0 +1,191 @@ +//! Trace filter cache management for `RuntimeTracer`. + +use crate::code_object::CodeObjectWrapper; +use crate::logging::{record_dropped_event, with_error_code}; +use crate::runtime::io_capture::ScopedMuteIoCapture; +use crate::runtime::value_capture::ValueFilterStats; +use crate::trace_filter::engine::{ExecDecision, ScopeResolution, TraceFilterEngine, ValueKind}; +use pyo3::prelude::*; +use recorder_errors::ErrorCode; +use serde_json::{self, json}; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +/// Filtering outcome for a code object. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum TraceDecision { + Trace, + SkipAndDisable, +} + +/// Coordinates trace filter execution, caching, and telemetry. +pub(crate) struct FilterCoordinator { + engine: Option>, + ignored_code_ids: HashSet, + scope_cache: HashMap>, + stats: FilterStats, +} + +impl FilterCoordinator { + pub(crate) fn new(engine: Option>) -> Self { + Self { + engine, + ignored_code_ids: HashSet::new(), + scope_cache: HashMap::new(), + stats: FilterStats::default(), + } + } + + pub(crate) fn engine(&self) -> Option<&Arc> { + self.engine.as_ref() + } + + pub(crate) fn cached_resolution(&self, code_id: usize) -> Option> { + self.scope_cache.get(&code_id).cloned() + } + + pub(crate) fn summary_json(&self) -> serde_json::Value { + self.stats.summary_json() + } + + pub(crate) fn values_mut(&mut self) -> &mut ValueFilterStats { + self.stats.values_mut() + } + + pub(crate) fn clear_caches(&mut self) { + self.ignored_code_ids.clear(); + self.scope_cache.clear(); + } + + pub(crate) fn reset(&mut self) { + self.clear_caches(); + self.stats.reset(); + } + + pub(crate) fn decide(&mut self, py: Python<'_>, code: &CodeObjectWrapper) -> TraceDecision { + let code_id = code.id(); + if self.ignored_code_ids.contains(&code_id) { + return TraceDecision::SkipAndDisable; + } + + if let Some(resolution) = self.resolve(py, code) { + if resolution.exec() == ExecDecision::Skip { + self.mark_ignored(code_id); + self.stats.record_skip(); + record_dropped_event("filter_scope_skip"); + return TraceDecision::SkipAndDisable; + } + } + + let filename = match code.filename(py) { + Ok(name) => name, + Err(err) => { + with_error_code(ErrorCode::Io, || { + let _mute = ScopedMuteIoCapture::new(); + log::error!("failed to resolve code filename: {err}"); + }); + record_dropped_event("filename_lookup_failed"); + self.mark_ignored(code_id); + return TraceDecision::SkipAndDisable; + } + }; + + if is_real_filename(filename) { + TraceDecision::Trace + } else { + record_dropped_event("synthetic_filename"); + self.mark_ignored(code_id); + TraceDecision::SkipAndDisable + } + } + + fn resolve( + &mut self, + py: Python<'_>, + code: &CodeObjectWrapper, + ) -> Option> { + let engine = self.engine.as_ref()?; + let code_id = code.id(); + + if let Some(existing) = self.scope_cache.get(&code_id) { + return Some(existing.clone()); + } + + match engine.resolve(py, code) { + Ok(resolution) => { + if resolution.exec() == ExecDecision::Trace { + self.scope_cache.insert(code_id, Arc::clone(&resolution)); + } else { + self.scope_cache.remove(&code_id); + } + Some(resolution) + } + Err(err) => { + let message = err.to_string(); + let error_code = err.code; + with_error_code(error_code, || { + let _mute = ScopedMuteIoCapture::new(); + log::error!( + "[RuntimeTracer] trace filter resolution failed for code id {}: {}", + code_id, + message + ); + }); + record_dropped_event("filter_resolution_error"); + None + } + } + } + + fn mark_ignored(&mut self, code_id: usize) { + self.scope_cache.remove(&code_id); + self.ignored_code_ids.insert(code_id); + } +} + +/// Return true when the filename refers to a concrete source file. +pub(crate) fn is_real_filename(filename: &str) -> bool { + let trimmed = filename.trim(); + !(trimmed.starts_with('<') && trimmed.ends_with('>')) +} + +#[derive(Debug, Default)] +struct FilterStats { + skipped_scopes: u64, + values: ValueFilterStats, +} + +impl FilterStats { + fn record_skip(&mut self) { + self.skipped_scopes += 1; + } + + fn values_mut(&mut self) -> &mut ValueFilterStats { + &mut self.values + } + + fn reset(&mut self) { + self.skipped_scopes = 0; + self.values = ValueFilterStats::default(); + } + + fn summary_json(&self) -> serde_json::Value { + let mut redactions = serde_json::Map::new(); + let mut drops = serde_json::Map::new(); + for kind in ValueKind::ALL { + redactions.insert( + kind.label().to_string(), + json!(self.values.redacted_count(kind)), + ); + drops.insert( + kind.label().to_string(), + json!(self.values.dropped_count(kind)), + ); + } + json!({ + "scopes_skipped": self.skipped_scopes, + "value_redactions": serde_json::Value::Object(redactions), + "value_drops": serde_json::Value::Object(drops), + }) + } +} diff --git a/codetracer-python-recorder/src/runtime/tracer/io.rs b/codetracer-python-recorder/src/runtime/tracer/io.rs new file mode 100644 index 0000000..7e28ce2 --- /dev/null +++ b/codetracer-python-recorder/src/runtime/tracer/io.rs @@ -0,0 +1,224 @@ +//! IO capture coordination for `RuntimeTracer`. + +use crate::runtime::io_capture::{ + IoCapturePipeline, IoCaptureSettings, IoChunk, IoChunkFlags, IoStream, ScopedMuteIoCapture, +}; +use crate::runtime::line_snapshots::{FrameId, LineSnapshotStore}; +use pyo3::prelude::*; +use runtime_tracing::{ + EventLogKind, Line, NonStreamingTraceWriter, PathId, RecordEvent, TraceLowLevelEvent, + TraceWriter, +}; +use serde::Serialize; +use std::path::Path; +use std::sync::Arc; +use std::thread::ThreadId; + +/// Coordinates installation, flushing, and teardown of the IO capture pipeline. +pub(crate) struct IoCoordinator { + snapshots: Arc, + pipeline: Option, +} + +impl IoCoordinator { + /// Create a coordinator with a fresh snapshot store and no active pipeline. + pub(crate) fn new() -> Self { + Self { + snapshots: Arc::new(LineSnapshotStore::new()), + pipeline: None, + } + } + + /// Expose the shared snapshot store for collaborators (tests, IO capture). + pub(crate) fn snapshot_store(&self) -> Arc { + Arc::clone(&self.snapshots) + } + + /// Install the IO capture pipeline using the provided settings. + pub(crate) fn install( + &mut self, + py: Python<'_>, + settings: IoCaptureSettings, + ) -> PyResult<()> { + self.pipeline = IoCapturePipeline::install(py, Arc::clone(&self.snapshots), settings)?; + Ok(()) + } + + /// Flush buffered output for the active thread before emitting a step event. + pub(crate) fn flush_before_step( + &self, + thread_id: ThreadId, + writer: &mut NonStreamingTraceWriter, + ) -> bool { + let Some(pipeline) = self.pipeline.as_ref() else { + return false; + }; + + pipeline.flush_before_step(thread_id); + self.drain_chunks(pipeline, writer) + } + + /// Flush every buffered chunk regardless of thread affinity. + pub(crate) fn flush_all(&self, writer: &mut NonStreamingTraceWriter) -> bool { + let Some(pipeline) = self.pipeline.as_ref() else { + return false; + }; + + pipeline.flush_all(); + self.drain_chunks(pipeline, writer) + } + + /// Drain remaining chunks and uninstall the capture pipeline. + pub(crate) fn teardown( + &mut self, + py: Python<'_>, + writer: &mut NonStreamingTraceWriter, + ) -> bool { + let Some(mut pipeline) = self.pipeline.take() else { + return false; + }; + + pipeline.flush_all(); + let mut recorded = false; + + for chunk in pipeline.drain_chunks() { + recorded |= self.record_chunk(writer, chunk); + } + + pipeline.uninstall(py); + + for chunk in pipeline.drain_chunks() { + recorded |= self.record_chunk(writer, chunk); + } + + recorded + } + + /// Clear the snapshot cache once tracing concludes. + pub(crate) fn clear_snapshots(&self) { + self.snapshots.clear(); + } + + /// Record the latest frame snapshot for the active thread. + pub(crate) fn record_snapshot( + &self, + thread_id: ThreadId, + path_id: PathId, + line: Line, + frame_id: FrameId, + ) { + self.snapshots.record(thread_id, path_id, line, frame_id); + } + + fn drain_chunks( + &self, + pipeline: &IoCapturePipeline, + writer: &mut NonStreamingTraceWriter, + ) -> bool { + let mut recorded = false; + for chunk in pipeline.drain_chunks() { + recorded |= self.record_chunk(writer, chunk); + } + recorded + } + + fn record_chunk(&self, writer: &mut NonStreamingTraceWriter, mut chunk: IoChunk) -> bool { + if chunk.path_id.is_none() { + if let Some(path) = chunk.path.as_deref() { + let path_id = TraceWriter::ensure_path_id(writer, Path::new(path)); + chunk.path_id = Some(path_id); + } + } + + let kind = match chunk.stream { + IoStream::Stdout => EventLogKind::Write, + IoStream::Stderr => EventLogKind::WriteOther, + IoStream::Stdin => EventLogKind::Read, + }; + + let metadata = self.build_metadata(&chunk); + let content = String::from_utf8_lossy(&chunk.payload).into_owned(); + + TraceWriter::add_event( + writer, + TraceLowLevelEvent::Event(RecordEvent { + kind, + metadata, + content, + }), + ); + + true + } + + fn build_metadata(&self, chunk: &IoChunk) -> String { + #[derive(Serialize)] + struct IoEventMetadata<'a> { + stream: &'a str, + thread: String, + path_id: Option, + line: Option, + frame_id: Option, + flags: Vec<&'a str>, + } + + let snapshot = self.snapshots.snapshot_for_thread(chunk.thread_id); + let path_id = chunk + .path_id + .map(|id| id.0) + .or_else(|| snapshot.as_ref().map(|snap| snap.path_id().0)); + let line = chunk + .line + .map(|line| line.0) + .or_else(|| snapshot.as_ref().map(|snap| snap.line().0)); + let frame_id = chunk + .frame_id + .or_else(|| snapshot.as_ref().map(|snap| snap.frame_id())); + + let metadata = IoEventMetadata { + stream: match chunk.stream { + IoStream::Stdout => "stdout", + IoStream::Stderr => "stderr", + IoStream::Stdin => "stdin", + }, + thread: format!("{:?}", chunk.thread_id), + path_id, + line, + frame_id: frame_id.map(|id| id.as_raw()), + flags: flag_labels(chunk.flags), + }; + + match serde_json::to_string(&metadata) { + Ok(json) => json, + Err(err) => { + let _mute = ScopedMuteIoCapture::new(); + log::error!("failed to serialise IO metadata: {err}"); + "{}".to_string() + } + } + } +} + +/// Translate chunk flags into telemetry labels. +fn flag_labels(flags: IoChunkFlags) -> Vec<&'static str> { + let mut labels = Vec::new(); + if flags.contains(IoChunkFlags::NEWLINE_TERMINATED) { + labels.push("newline"); + } + if flags.contains(IoChunkFlags::EXPLICIT_FLUSH) { + labels.push("flush"); + } + if flags.contains(IoChunkFlags::STEP_BOUNDARY) { + labels.push("step_boundary"); + } + if flags.contains(IoChunkFlags::TIME_SPLIT) { + labels.push("time_split"); + } + if flags.contains(IoChunkFlags::INPUT_CHUNK) { + labels.push("input"); + } + if flags.contains(IoChunkFlags::FD_MIRROR) { + labels.push("mirror"); + } + labels +} diff --git a/codetracer-python-recorder/src/runtime/tracer/lifecycle.rs b/codetracer-python-recorder/src/runtime/tracer/lifecycle.rs new file mode 100644 index 0000000..24ca643 --- /dev/null +++ b/codetracer-python-recorder/src/runtime/tracer/lifecycle.rs @@ -0,0 +1,296 @@ +//! Lifecycle orchestration for `RuntimeTracer`. + +use crate::logging::set_active_trace_id; +use crate::policy::RecorderPolicy; +use crate::runtime::activation::ActivationController; +use crate::runtime::io_capture::ScopedMuteIoCapture; +use crate::runtime::output_paths::TraceOutputPaths; +use crate::runtime::tracer::filtering::FilterCoordinator; +use recorder_errors::{enverr, usage, ErrorCode, RecorderResult}; +use runtime_tracing::{NonStreamingTraceWriter, TraceWriter}; +use serde_json::{self, json}; +use std::fs; +use std::path::{Path, PathBuf}; +use uuid::Uuid; + +/// Coordinates writer setup, activation, and teardown flows. +#[derive(Debug)] +pub struct LifecycleController { + activation: ActivationController, + program_path: PathBuf, + output_paths: Option, + events_recorded: bool, + encountered_failure: bool, + trace_id: String, +} + +impl LifecycleController { + pub fn new(program: &str, activation_path: Option<&Path>) -> Self { + Self { + activation: ActivationController::new(activation_path), + program_path: PathBuf::from(program), + output_paths: None, + events_recorded: false, + encountered_failure: false, + trace_id: Uuid::new_v4().to_string(), + } + } + + #[cfg(test)] + pub fn activation(&self) -> &ActivationController { + &self.activation + } + + pub fn activation_mut(&mut self) -> &mut ActivationController { + &mut self.activation + } + + pub fn begin( + &mut self, + writer: &mut NonStreamingTraceWriter, + outputs: &TraceOutputPaths, + start_line: u32, + ) -> RecorderResult<()> { + let start_path = self.activation.start_path(&self.program_path); + { + let _mute = ScopedMuteIoCapture::new(); + log::debug!("{}", start_path.display()); + } + outputs.configure_writer(writer, start_path, start_line)?; + self.output_paths = Some(outputs.clone()); + self.events_recorded = false; + self.encountered_failure = false; + self.set_trace_id_active(); + Ok(()) + } + + pub fn mark_event(&mut self) { + self.events_recorded = true; + } + + pub fn mark_failure(&mut self) { + self.encountered_failure = true; + } + + pub fn encountered_failure(&self) -> bool { + self.encountered_failure + } + + pub fn require_trace_or_fail(&self, policy: &RecorderPolicy) -> RecorderResult<()> { + if policy.require_trace && !self.events_recorded { + return Err(usage!( + ErrorCode::TraceMissing, + "recorder policy requires a trace but no events were recorded" + )); + } + Ok(()) + } + + pub fn cleanup_partial_outputs(&self) -> RecorderResult<()> { + if let Some(outputs) = &self.output_paths { + for path in [outputs.events(), outputs.metadata(), outputs.paths()] { + if path.exists() { + fs::remove_file(path).map_err(|err| { + enverr!(ErrorCode::Io, "failed to remove partial trace file") + .with_context("path", path.display().to_string()) + .with_context("io", err.to_string()) + })?; + } + } + } + Ok(()) + } + + pub fn finalise( + &mut self, + writer: &mut NonStreamingTraceWriter, + filter: &FilterCoordinator, + ) -> RecorderResult<()> { + TraceWriter::finish_writing_trace_metadata(writer).map_err(|err| { + enverr!(ErrorCode::Io, "failed to finalise trace metadata") + .with_context("source", err.to_string()) + })?; + self.append_filter_metadata(filter)?; + TraceWriter::finish_writing_trace_paths(writer).map_err(|err| { + enverr!(ErrorCode::Io, "failed to finalise trace paths") + .with_context("source", err.to_string()) + })?; + TraceWriter::finish_writing_trace_events(writer).map_err(|err| { + enverr!(ErrorCode::Io, "failed to finalise trace events") + .with_context("source", err.to_string()) + })?; + Ok(()) + } + + pub fn output_paths(&self) -> Option<&TraceOutputPaths> { + self.output_paths.as_ref() + } + + pub fn reset_event_state(&mut self) { + self.output_paths = None; + self.events_recorded = false; + self.encountered_failure = false; + } + + fn append_filter_metadata(&self, filter: &FilterCoordinator) -> RecorderResult<()> { + let Some(outputs) = &self.output_paths else { + return Ok(()); + }; + let Some(engine) = filter.engine() else { + return Ok(()); + }; + + let path = outputs.metadata(); + let original = fs::read_to_string(path).map_err(|err| { + enverr!(ErrorCode::Io, "failed to read trace metadata") + .with_context("path", path.display().to_string()) + .with_context("source", err.to_string()) + })?; + + let mut metadata: serde_json::Value = serde_json::from_str(&original).map_err(|err| { + enverr!(ErrorCode::Io, "failed to parse trace metadata JSON") + .with_context("path", path.display().to_string()) + .with_context("source", err.to_string()) + })?; + + let filters = engine.summary(); + let filters_json: Vec = filters + .entries + .iter() + .map(|entry| { + json!({ + "path": entry.path.to_string_lossy(), + "sha256": entry.sha256, + "name": entry.name, + "version": entry.version, + }) + }) + .collect(); + + if let serde_json::Value::Object(ref mut obj) = metadata { + obj.insert( + "trace_filter".to_string(), + json!({ + "filters": filters_json, + "stats": filter.summary_json(), + }), + ); + let serialised = serde_json::to_string(&metadata).map_err(|err| { + enverr!(ErrorCode::Io, "failed to serialise trace metadata") + .with_context("path", path.display().to_string()) + .with_context("source", err.to_string()) + })?; + fs::write(path, serialised).map_err(|err| { + enverr!(ErrorCode::Io, "failed to write trace metadata") + .with_context("path", path.display().to_string()) + .with_context("source", err.to_string()) + })?; + Ok(()) + } else { + Err( + enverr!(ErrorCode::Io, "trace metadata must be a JSON object") + .with_context("path", path.display().to_string()), + ) + } + } + + fn set_trace_id_active(&self) { + set_active_trace_id(Some(self.trace_id.clone())); + } + + pub fn trace_id_scope(&self) -> TraceIdScope { + self.set_trace_id_active(); + TraceIdScope + } +} + +/// Guard that clears the active trace id when dropped. +pub(crate) struct TraceIdScope; + +impl Drop for TraceIdScope { + fn drop(&mut self) { + set_active_trace_id(None); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::logging::{init_rust_logging_with_default, snapshot_run_and_trace}; + use crate::policy::RecorderPolicy; + use crate::runtime::output_paths::TraceOutputPaths; + use recorder_errors::ErrorCode; + use runtime_tracing::{NonStreamingTraceWriter, TraceEventsFileFormat}; + + fn writer() -> NonStreamingTraceWriter { + NonStreamingTraceWriter::new("program.py", &[]) + } + + #[test] + fn policy_requiring_trace_fails_without_events() { + let controller = LifecycleController::new("program.py", None); + let mut policy = RecorderPolicy::default(); + policy.require_trace = true; + + let err = controller.require_trace_or_fail(&policy).unwrap_err(); + assert_eq!(err.code, ErrorCode::TraceMissing); + } + + #[test] + fn policy_requiring_trace_passes_after_event() { + let mut controller = LifecycleController::new("program.py", None); + let mut policy = RecorderPolicy::default(); + policy.require_trace = true; + controller.mark_event(); + + assert!(controller.require_trace_or_fail(&policy).is_ok()); + } + + #[test] + fn cleanup_removes_partial_outputs() { + let tmp = tempfile::tempdir().expect("tempdir"); + let outputs = TraceOutputPaths::new(tmp.path(), TraceEventsFileFormat::Json); + let mut controller = LifecycleController::new("program.py", None); + let mut writer = writer(); + + controller + .begin(&mut writer, &outputs, 1) + .expect("begin lifecycle"); + + std::fs::write(outputs.events(), "events").expect("write events"); + std::fs::write(outputs.metadata(), "{}").expect("write metadata"); + std::fs::write(outputs.paths(), "[]").expect("write paths"); + + controller + .cleanup_partial_outputs() + .expect("cleanup outputs"); + + assert!( + !outputs.events().exists(), + "expected events file removed after cleanup" + ); + assert!( + !outputs.metadata().exists(), + "expected metadata file removed after cleanup" + ); + assert!( + !outputs.paths().exists(), + "expected paths file removed after cleanup" + ); + } + + #[test] + fn trace_id_scope_sets_and_clears_active_id() { + init_rust_logging_with_default("codetracer_python_recorder=error"); + let controller = LifecycleController::new("program.py", None); + + { + let _scope = controller.trace_id_scope(); + let (_, active) = snapshot_run_and_trace().expect("logger initialised"); + assert!(matches!(active.as_deref(), Some(id) if !id.is_empty())); + } + + let (_, cleared) = snapshot_run_and_trace().expect("logger initialised"); + assert!(cleared.is_none(), "expected trace id cleared after scope"); + } +} diff --git a/codetracer-python-recorder/src/runtime/tracer/mod.rs b/codetracer-python-recorder/src/runtime/tracer/mod.rs new file mode 100644 index 0000000..86eb10e --- /dev/null +++ b/codetracer-python-recorder/src/runtime/tracer/mod.rs @@ -0,0 +1,13 @@ +//! Collaborators for the runtime tracer lifecycle, IO coordination, filtering, and event handling. +//! +//! Re-exports [`RuntimeTracer`] so downstream callers continue using `crate::runtime::RuntimeTracer` +//! without exposing the implementation modules outside the crate. + +pub(crate) mod events; +pub(crate) mod filtering; +pub(crate) mod io; +pub(crate) mod lifecycle; + +mod runtime_tracer; + +pub use runtime_tracer::RuntimeTracer; diff --git a/codetracer-python-recorder/src/runtime/tracer/runtime_tracer.rs b/codetracer-python-recorder/src/runtime/tracer/runtime_tracer.rs new file mode 100644 index 0000000..1bb37ed --- /dev/null +++ b/codetracer-python-recorder/src/runtime/tracer/runtime_tracer.rs @@ -0,0 +1,1764 @@ +use super::events::suppress_events; +use super::filtering::{FilterCoordinator, TraceDecision}; +use super::io::IoCoordinator; +use super::lifecycle::LifecycleController; +use crate::code_object::CodeObjectWrapper; +use crate::ffi; +use crate::policy::RecorderPolicy; +use crate::runtime::io_capture::{IoCaptureSettings, ScopedMuteIoCapture}; +use crate::runtime::line_snapshots::LineSnapshotStore; +use crate::runtime::output_paths::TraceOutputPaths; +use crate::trace_filter::engine::TraceFilterEngine; +use pyo3::prelude::*; +use runtime_tracing::NonStreamingTraceWriter; +use runtime_tracing::{Line, TraceEventsFileFormat, TraceWriter}; +use std::collections::{hash_map::Entry, HashMap}; +use std::path::Path; +use std::sync::Arc; +use std::thread::ThreadId; + +/// Minimal runtime tracer that maps Python sys.monitoring events to +/// runtime_tracing writer operations. +pub struct RuntimeTracer { + pub(super) writer: NonStreamingTraceWriter, + pub(super) format: TraceEventsFileFormat, + pub(super) lifecycle: LifecycleController, + pub(super) function_ids: HashMap, + pub(super) io: IoCoordinator, + pub(super) filter: FilterCoordinator, +} + +impl RuntimeTracer { + pub fn new( + program: &str, + args: &[String], + format: TraceEventsFileFormat, + activation_path: Option<&Path>, + trace_filter: Option>, + ) -> Self { + let mut writer = NonStreamingTraceWriter::new(program, args); + writer.set_format(format); + let lifecycle = LifecycleController::new(program, activation_path); + Self { + writer, + format, + lifecycle, + function_ids: HashMap::new(), + io: IoCoordinator::new(), + filter: FilterCoordinator::new(trace_filter), + } + } + + /// Share the snapshot store with collaborators (IO capture, tests). + #[cfg_attr(not(test), allow(dead_code))] + pub fn line_snapshot_store(&self) -> Arc { + self.io.snapshot_store() + } + + pub fn install_io_capture(&mut self, py: Python<'_>, policy: &RecorderPolicy) -> PyResult<()> { + let settings = IoCaptureSettings { + line_proxies: policy.io_capture.line_proxies, + fd_mirror: policy.io_capture.fd_fallback, + }; + self.io.install(py, settings) + } + + pub(super) fn flush_io_before_step(&mut self, thread_id: ThreadId) { + if self.io.flush_before_step(thread_id, &mut self.writer) { + self.mark_event(); + } + } + + pub(super) fn flush_pending_io(&mut self) { + if self.io.flush_all(&mut self.writer) { + self.mark_event(); + } + } + + /// Configure output files and write initial metadata records. + pub fn begin(&mut self, outputs: &TraceOutputPaths, start_line: u32) -> PyResult<()> { + self.lifecycle + .begin(&mut self.writer, outputs, start_line) + .map_err(ffi::map_recorder_error)?; + Ok(()) + } + + pub(super) fn mark_event(&mut self) { + if suppress_events() { + let _mute = ScopedMuteIoCapture::new(); + log::debug!("[RuntimeTracer] skipping event mark due to test injection"); + return; + } + self.lifecycle.mark_event(); + } + + pub(super) fn mark_failure(&mut self) { + self.lifecycle.mark_failure(); + } + + pub(super) fn ensure_function_id( + &mut self, + py: Python<'_>, + code: &CodeObjectWrapper, + ) -> PyResult { + match self.function_ids.entry(code.id()) { + Entry::Occupied(entry) => Ok(*entry.get()), + Entry::Vacant(slot) => { + let name = code.qualname(py)?; + let filename = code.filename(py)?; + let first_line = code.first_line(py)?; + let function_id = TraceWriter::ensure_function_id( + &mut self.writer, + name, + Path::new(filename), + Line(first_line as i64), + ); + Ok(*slot.insert(function_id)) + } + } + } + + pub(super) fn should_trace_code( + &mut self, + py: Python<'_>, + code: &CodeObjectWrapper, + ) -> TraceDecision { + self.filter.decide(py, code) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::monitoring::{CallbackOutcome, Tracer}; + use crate::policy; + use crate::runtime::tracer::filtering::is_real_filename; + use crate::trace_filter::config::TraceFilterConfig; + use pyo3::types::{PyAny, PyCode, PyModule}; + use pyo3::wrap_pyfunction; + use runtime_tracing::{FullValueRecord, StepRecord, TraceLowLevelEvent, ValueRecord}; + use serde::Deserialize; + use std::cell::Cell; + use std::collections::BTreeMap; + use std::ffi::CString; + use std::fs; + use std::path::Path; + use std::sync::Arc; + use std::thread; + + thread_local! { + static ACTIVE_TRACER: Cell<*mut RuntimeTracer> = Cell::new(std::ptr::null_mut()); + static LAST_OUTCOME: Cell> = Cell::new(None); + } + + struct ScopedTracer; + + impl ScopedTracer { + fn new(tracer: &mut RuntimeTracer) -> Self { + let ptr = tracer as *mut _; + ACTIVE_TRACER.with(|cell| cell.set(ptr)); + ScopedTracer + } + } + + impl Drop for ScopedTracer { + fn drop(&mut self) { + ACTIVE_TRACER.with(|cell| cell.set(std::ptr::null_mut())); + } + } + + fn last_outcome() -> Option { + LAST_OUTCOME.with(|cell| cell.get()) + } + + fn reset_policy(_py: Python<'_>) { + policy::configure_policy_py( + Some("abort"), + Some(false), + Some(false), + None, + None, + Some(false), + None, + None, + ) + .expect("reset recorder policy"); + } + + #[test] + fn detects_real_filenames() { + assert!(is_real_filename("example.py")); + assert!(is_real_filename(" /tmp/module.py ")); + assert!(is_real_filename("src/.py")); + assert!(!is_real_filename("")); + assert!(!is_real_filename(" ")); + assert!(!is_real_filename("")); + } + + #[test] + fn skips_synthetic_filename_events() { + Python::with_gil(|py| { + let mut tracer = + RuntimeTracer::new("test.py", &[], TraceEventsFileFormat::Json, None, None); + ensure_test_module(py); + let script = format!("{PRELUDE}\nsnapshot()\n"); + { + let _guard = ScopedTracer::new(&mut tracer); + LAST_OUTCOME.with(|cell| cell.set(None)); + let script_c = CString::new(script).expect("script contains nul byte"); + py.run(script_c.as_c_str(), None, None) + .expect("execute synthetic script"); + } + assert!( + tracer.writer.events.is_empty(), + "expected no events for synthetic filename" + ); + assert_eq!(last_outcome(), Some(CallbackOutcome::DisableLocation)); + + let compile_fn = py + .import("builtins") + .expect("import builtins") + .getattr("compile") + .expect("fetch compile"); + let binding = compile_fn + .call1(("pass", "", "exec")) + .expect("compile code object"); + let code_obj = binding.downcast::().expect("downcast code object"); + let wrapper = CodeObjectWrapper::new(py, &code_obj); + assert_eq!( + tracer.should_trace_code(py, &wrapper), + TraceDecision::SkipAndDisable + ); + }); + } + + #[test] + fn traces_real_file_events() { + let snapshots = run_traced_script("snapshot()\n"); + assert!( + !snapshots.is_empty(), + "expected snapshots for real file execution" + ); + assert_eq!(last_outcome(), Some(CallbackOutcome::Continue)); + } + + #[test] + fn callbacks_do_not_import_sys_monitoring() { + let body = r#" +import builtins +_orig_import = builtins.__import__ + +def guard(name, *args, **kwargs): + if name == "sys.monitoring": + raise RuntimeError("callback imported sys.monitoring") + return _orig_import(name, *args, **kwargs) + +builtins.__import__ = guard +try: + snapshot() +finally: + builtins.__import__ = _orig_import +"#; + let snapshots = run_traced_script(body); + assert!( + !snapshots.is_empty(), + "expected snapshots when import guard active" + ); + assert_eq!(last_outcome(), Some(CallbackOutcome::Continue)); + } + + #[test] + fn records_return_values_and_deactivates_activation() { + Python::with_gil(|py| { + ensure_test_module(py); + let tmp = tempfile::tempdir().expect("create temp dir"); + let script_path = tmp.path().join("activation_script.py"); + let script = format!( + "{PRELUDE}\n\n\ +def compute():\n emit_return(\"tail\")\n return \"tail\"\n\n\ +result = compute()\n" + ); + std::fs::write(&script_path, &script).expect("write script"); + + let program = script_path.to_string_lossy().into_owned(); + let mut tracer = RuntimeTracer::new( + &program, + &[], + TraceEventsFileFormat::Json, + Some(script_path.as_path()), + None, + ); + + { + let _guard = ScopedTracer::new(&mut tracer); + LAST_OUTCOME.with(|cell| cell.set(None)); + let run_code = format!( + "import runpy\nrunpy.run_path(r\"{}\")", + script_path.display() + ); + let run_code_c = CString::new(run_code).expect("script contains nul byte"); + py.run(run_code_c.as_c_str(), None, None) + .expect("execute test script"); + } + + let returns: Vec = tracer + .writer + .events + .iter() + .filter_map(|event| match event { + TraceLowLevelEvent::Return(record) => { + Some(SimpleValue::from_value(&record.return_value)) + } + _ => None, + }) + .collect(); + + assert!( + returns.contains(&SimpleValue::String("tail".to_string())), + "expected recorded string return, got {:?}", + returns + ); + assert_eq!(last_outcome(), Some(CallbackOutcome::Continue)); + assert!(!tracer.lifecycle.activation().is_active()); + }); + } + + #[test] + fn line_snapshot_store_tracks_last_step() { + Python::with_gil(|py| { + ensure_test_module(py); + let tmp = tempfile::tempdir().expect("create temp dir"); + let script_path = tmp.path().join("snapshot_script.py"); + let script = format!("{PRELUDE}\n\nsnapshot()\n"); + std::fs::write(&script_path, &script).expect("write script"); + + let mut tracer = RuntimeTracer::new( + "snapshot_script.py", + &[], + TraceEventsFileFormat::Json, + None, + None, + ); + let store = tracer.line_snapshot_store(); + + { + let _guard = ScopedTracer::new(&mut tracer); + LAST_OUTCOME.with(|cell| cell.set(None)); + let run_code = format!( + "import runpy\nrunpy.run_path(r\"{}\")", + script_path.display() + ); + let run_code_c = CString::new(run_code).expect("script contains nul byte"); + py.run(run_code_c.as_c_str(), None, None) + .expect("execute snapshot script"); + } + + let last_step: StepRecord = tracer + .writer + .events + .iter() + .rev() + .find_map(|event| match event { + TraceLowLevelEvent::Step(step) => Some(step.clone()), + _ => None, + }) + .expect("expected one step event"); + + let thread_id = thread::current().id(); + let snapshot = store + .snapshot_for_thread(thread_id) + .expect("snapshot should be recorded"); + + assert_eq!(snapshot.line(), last_step.line); + assert_eq!(snapshot.path_id(), last_step.path_id); + assert!(snapshot.captured_at().elapsed().as_secs_f64() >= 0.0); + }); + } + + #[derive(Debug, Deserialize)] + struct IoMetadata { + stream: String, + path_id: Option, + line: Option, + flags: Vec, + } + + #[test] + fn io_capture_records_python_and_native_output() { + Python::with_gil(|py| { + reset_policy(py); + policy::configure_policy_py( + Some("abort"), + Some(false), + Some(false), + None, + None, + Some(false), + Some(true), + Some(false), + ) + .expect("enable io capture proxies"); + + ensure_test_module(py); + let tmp = tempfile::tempdir().expect("create temp dir"); + let script_path = tmp.path().join("io_script.py"); + let script = format!( + "{PRELUDE}\n\nprint('python out')\nfrom ctypes import pythonapi, c_char_p\npythonapi.PySys_WriteStdout(c_char_p(b'native out\\n'))\n" + ); + std::fs::write(&script_path, &script).expect("write script"); + + let mut tracer = RuntimeTracer::new( + script_path.to_string_lossy().as_ref(), + &[], + TraceEventsFileFormat::Json, + None, + None, + ); + let outputs = TraceOutputPaths::new(tmp.path(), TraceEventsFileFormat::Json); + tracer.begin(&outputs, 1).expect("begin tracer"); + tracer + .install_io_capture(py, &policy::policy_snapshot()) + .expect("install io capture"); + + { + let _guard = ScopedTracer::new(&mut tracer); + LAST_OUTCOME.with(|cell| cell.set(None)); + let run_code = format!( + "import runpy\nrunpy.run_path(r\"{}\")", + script_path.display() + ); + let run_code_c = CString::new(run_code).expect("script contains nul byte"); + py.run(run_code_c.as_c_str(), None, None) + .expect("execute io script"); + } + + tracer.finish(py).expect("finish tracer"); + + let io_events: Vec<(IoMetadata, Vec)> = tracer + .writer + .events + .iter() + .filter_map(|event| match event { + TraceLowLevelEvent::Event(record) => { + let metadata: IoMetadata = serde_json::from_str(&record.metadata).ok()?; + Some((metadata, record.content.as_bytes().to_vec())) + } + _ => None, + }) + .collect(); + + assert!(io_events + .iter() + .any(|(meta, payload)| meta.stream == "stdout" + && String::from_utf8_lossy(payload).contains("python out"))); + assert!(io_events + .iter() + .any(|(meta, payload)| meta.stream == "stdout" + && String::from_utf8_lossy(payload).contains("native out"))); + assert!(io_events.iter().all(|(meta, _)| { + if meta.stream == "stdout" { + meta.path_id.is_some() && meta.line.is_some() + } else { + true + } + })); + assert!(io_events + .iter() + .filter(|(meta, _)| meta.stream == "stdout") + .any(|(meta, _)| meta.flags.iter().any(|flag| flag == "newline"))); + + reset_policy(py); + }); + } + + #[cfg(unix)] + #[test] + fn fd_mirror_captures_os_write_payloads() { + Python::with_gil(|py| { + reset_policy(py); + policy::configure_policy_py( + Some("abort"), + Some(false), + Some(false), + None, + None, + Some(false), + Some(true), + Some(true), + ) + .expect("enable io capture with fd fallback"); + + ensure_test_module(py); + let tmp = tempfile::tempdir().expect("tempdir"); + let script_path = tmp.path().join("fd_mirror.py"); + std::fs::write( + &script_path, + format!( + "{PRELUDE}\nimport os\nprint('proxy line')\nos.write(1, b'fd stdout\\n')\nos.write(2, b'fd stderr\\n')\n" + ), + ) + .expect("write script"); + + let mut tracer = RuntimeTracer::new( + script_path.to_string_lossy().as_ref(), + &[], + TraceEventsFileFormat::Json, + None, + None, + ); + let outputs = TraceOutputPaths::new(tmp.path(), TraceEventsFileFormat::Json); + tracer.begin(&outputs, 1).expect("begin tracer"); + tracer + .install_io_capture(py, &policy::policy_snapshot()) + .expect("install io capture"); + + { + let _guard = ScopedTracer::new(&mut tracer); + LAST_OUTCOME.with(|cell| cell.set(None)); + let run_code = format!( + "import runpy\nrunpy.run_path(r\"{}\")", + script_path.display() + ); + let run_code_c = CString::new(run_code).expect("script contains nul byte"); + py.run(run_code_c.as_c_str(), None, None) + .expect("execute fd script"); + } + + tracer.finish(py).expect("finish tracer"); + + let io_events: Vec<(IoMetadata, Vec)> = tracer + .writer + .events + .iter() + .filter_map(|event| match event { + TraceLowLevelEvent::Event(record) => { + let metadata: IoMetadata = serde_json::from_str(&record.metadata).ok()?; + Some((metadata, record.content.as_bytes().to_vec())) + } + _ => None, + }) + .collect(); + + let stdout_mirror = io_events.iter().find(|(meta, _)| { + meta.stream == "stdout" && meta.flags.iter().any(|flag| flag == "mirror") + }); + assert!( + stdout_mirror.is_some(), + "expected mirror event for stdout: {:?}", + io_events + ); + let stdout_payload = &stdout_mirror.unwrap().1; + assert!( + String::from_utf8_lossy(stdout_payload).contains("fd stdout"), + "mirror stdout payload missing expected text" + ); + + let stderr_mirror = io_events.iter().find(|(meta, _)| { + meta.stream == "stderr" && meta.flags.iter().any(|flag| flag == "mirror") + }); + assert!( + stderr_mirror.is_some(), + "expected mirror event for stderr: {:?}", + io_events + ); + let stderr_payload = &stderr_mirror.unwrap().1; + assert!( + String::from_utf8_lossy(stderr_payload).contains("fd stderr"), + "mirror stderr payload missing expected text" + ); + + assert!(io_events.iter().any(|(meta, payload)| { + meta.stream == "stdout" + && !meta.flags.iter().any(|flag| flag == "mirror") + && String::from_utf8_lossy(payload).contains("proxy line") + })); + + reset_policy(py); + }); + } + + #[cfg(unix)] + #[test] + fn fd_mirror_disabled_does_not_capture_os_write() { + Python::with_gil(|py| { + reset_policy(py); + policy::configure_policy_py( + Some("abort"), + Some(false), + Some(false), + None, + None, + Some(false), + Some(true), + Some(false), + ) + .expect("enable proxies without fd fallback"); + + ensure_test_module(py); + let tmp = tempfile::tempdir().expect("tempdir"); + let script_path = tmp.path().join("fd_disabled.py"); + std::fs::write( + &script_path, + format!( + "{PRELUDE}\nimport os\nprint('proxy line')\nos.write(1, b'fd stdout\\n')\nos.write(2, b'fd stderr\\n')\n" + ), + ) + .expect("write script"); + + let mut tracer = RuntimeTracer::new( + script_path.to_string_lossy().as_ref(), + &[], + TraceEventsFileFormat::Json, + None, + None, + ); + let outputs = TraceOutputPaths::new(tmp.path(), TraceEventsFileFormat::Json); + tracer.begin(&outputs, 1).expect("begin tracer"); + tracer + .install_io_capture(py, &policy::policy_snapshot()) + .expect("install io capture"); + + { + let _guard = ScopedTracer::new(&mut tracer); + LAST_OUTCOME.with(|cell| cell.set(None)); + let run_code = format!( + "import runpy\nrunpy.run_path(r\"{}\")", + script_path.display() + ); + let run_code_c = CString::new(run_code).expect("script contains nul byte"); + py.run(run_code_c.as_c_str(), None, None) + .expect("execute fd script"); + } + + tracer.finish(py).expect("finish tracer"); + + let io_events: Vec<(IoMetadata, Vec)> = tracer + .writer + .events + .iter() + .filter_map(|event| match event { + TraceLowLevelEvent::Event(record) => { + let metadata: IoMetadata = serde_json::from_str(&record.metadata).ok()?; + Some((metadata, record.content.as_bytes().to_vec())) + } + _ => None, + }) + .collect(); + + assert!( + !io_events + .iter() + .any(|(meta, _)| meta.flags.iter().any(|flag| flag == "mirror")), + "mirror events should not be present when fallback disabled" + ); + + assert!( + !io_events.iter().any(|(_, payload)| { + String::from_utf8_lossy(payload).contains("fd stdout") + || String::from_utf8_lossy(payload).contains("fd stderr") + }), + "native os.write payload unexpectedly captured without fallback" + ); + + assert!(io_events.iter().any(|(meta, payload)| { + meta.stream == "stdout" && String::from_utf8_lossy(payload).contains("proxy line") + })); + + reset_policy(py); + }); + } + + #[pyfunction] + fn capture_line(py: Python<'_>, code: Bound<'_, PyCode>, lineno: u32) -> PyResult<()> { + ffi::wrap_pyfunction("test_capture_line", || { + ACTIVE_TRACER.with(|cell| -> PyResult<()> { + let ptr = cell.get(); + if ptr.is_null() { + panic!("No active RuntimeTracer for capture_line"); + } + unsafe { + let tracer = &mut *ptr; + let wrapper = CodeObjectWrapper::new(py, &code); + match tracer.on_line(py, &wrapper, lineno) { + Ok(outcome) => { + LAST_OUTCOME.with(|cell| cell.set(Some(outcome))); + Ok(()) + } + Err(err) => Err(err), + } + } + })?; + Ok(()) + }) + } + + #[pyfunction] + fn capture_return_event( + py: Python<'_>, + code: Bound<'_, PyCode>, + value: Bound<'_, PyAny>, + ) -> PyResult<()> { + ffi::wrap_pyfunction("test_capture_return_event", || { + ACTIVE_TRACER.with(|cell| -> PyResult<()> { + let ptr = cell.get(); + if ptr.is_null() { + panic!("No active RuntimeTracer for capture_return_event"); + } + unsafe { + let tracer = &mut *ptr; + let wrapper = CodeObjectWrapper::new(py, &code); + match tracer.on_py_return(py, &wrapper, 0, &value) { + Ok(outcome) => { + LAST_OUTCOME.with(|cell| cell.set(Some(outcome))); + Ok(()) + } + Err(err) => Err(err), + } + } + })?; + Ok(()) + }) + } + + const PRELUDE: &str = r#" +import inspect +from test_tracer import capture_line, capture_return_event + +def snapshot(line=None): + frame = inspect.currentframe().f_back + lineno = frame.f_lineno if line is None else line + capture_line(frame.f_code, lineno) + +def snap(value): + frame = inspect.currentframe().f_back + capture_line(frame.f_code, frame.f_lineno) + return value + +def emit_return(value): + frame = inspect.currentframe().f_back + capture_return_event(frame.f_code, value) + return value +"#; + + #[derive(Debug, Clone, PartialEq)] + enum SimpleValue { + None, + Bool(bool), + Int(i64), + String(String), + Tuple(Vec), + Sequence(Vec), + Raw(String), + } + + impl SimpleValue { + fn from_value(value: &ValueRecord) -> Self { + match value { + ValueRecord::None { .. } => SimpleValue::None, + ValueRecord::Bool { b, .. } => SimpleValue::Bool(*b), + ValueRecord::Int { i, .. } => SimpleValue::Int(*i), + ValueRecord::String { text, .. } => SimpleValue::String(text.clone()), + ValueRecord::Tuple { elements, .. } => { + SimpleValue::Tuple(elements.iter().map(SimpleValue::from_value).collect()) + } + ValueRecord::Sequence { elements, .. } => { + SimpleValue::Sequence(elements.iter().map(SimpleValue::from_value).collect()) + } + ValueRecord::Raw { r, .. } => SimpleValue::Raw(r.clone()), + ValueRecord::Error { msg, .. } => SimpleValue::Raw(msg.clone()), + other => SimpleValue::Raw(format!("{other:?}")), + } + } + } + + #[derive(Debug)] + struct Snapshot { + line: i64, + vars: BTreeMap, + } + + fn collect_snapshots(events: &[TraceLowLevelEvent]) -> Vec { + let mut names: Vec = Vec::new(); + let mut snapshots: Vec = Vec::new(); + let mut current: Option = None; + for event in events { + match event { + TraceLowLevelEvent::VariableName(name) => names.push(name.clone()), + TraceLowLevelEvent::Step(step) => { + if let Some(snapshot) = current.take() { + snapshots.push(snapshot); + } + current = Some(Snapshot { + line: step.line.0, + vars: BTreeMap::new(), + }); + } + TraceLowLevelEvent::Value(FullValueRecord { variable_id, value }) => { + if let Some(snapshot) = current.as_mut() { + let index = variable_id.0; + let name = names + .get(index) + .cloned() + .unwrap_or_else(|| panic!("Missing variable name for id {}", index)); + snapshot.vars.insert(name, SimpleValue::from_value(value)); + } + } + _ => {} + } + } + if let Some(snapshot) = current.take() { + snapshots.push(snapshot); + } + snapshots + } + + fn ensure_test_module(py: Python<'_>) { + let module = PyModule::new(py, "test_tracer").expect("create module"); + module + .add_function(wrap_pyfunction!(capture_line, &module).expect("wrap capture_line")) + .expect("add function"); + module + .add_function( + wrap_pyfunction!(capture_return_event, &module).expect("wrap capture_return_event"), + ) + .expect("add return capture function"); + py.import("sys") + .expect("import sys") + .getattr("modules") + .expect("modules attr") + .set_item("test_tracer", module) + .expect("insert module"); + } + + fn run_traced_script(body: &str) -> Vec { + Python::with_gil(|py| { + let mut tracer = + RuntimeTracer::new("test.py", &[], TraceEventsFileFormat::Json, None, None); + ensure_test_module(py); + let tmp = tempfile::tempdir().expect("create temp dir"); + let script_path = tmp.path().join("script.py"); + let script = format!("{PRELUDE}\n{body}"); + std::fs::write(&script_path, &script).expect("write script"); + { + let _guard = ScopedTracer::new(&mut tracer); + LAST_OUTCOME.with(|cell| cell.set(None)); + let run_code = format!( + "import runpy\nrunpy.run_path(r\"{}\")", + script_path.display() + ); + let run_code_c = CString::new(run_code).expect("script contains nul byte"); + py.run(run_code_c.as_c_str(), None, None) + .expect("execute test script"); + } + collect_snapshots(&tracer.writer.events) + }) + } + + fn write_filter(path: &Path, contents: &str) { + fs::write(path, contents.trim_start()).expect("write filter"); + } + + #[test] + fn trace_filter_redacts_values() { + Python::with_gil(|py| { + ensure_test_module(py); + + let project = tempfile::tempdir().expect("project dir"); + let project_root = project.path(); + let filters_dir = project_root.join(".codetracer"); + fs::create_dir(&filters_dir).expect("create .codetracer"); + let filter_path = filters_dir.join("filters.toml"); + write_filter( + &filter_path, + r#" + [meta] + name = "redact" + version = 1 + + [scope] + default_exec = "trace" + default_value_action = "allow" + + [[scope.rules]] + selector = "pkg:app.sec" + exec = "trace" + value_default = "allow" + + [[scope.rules.value_patterns]] + selector = "arg:password" + action = "redact" + + [[scope.rules.value_patterns]] + selector = "local:password" + action = "redact" + + [[scope.rules.value_patterns]] + selector = "local:secret" + action = "redact" + + [[scope.rules.value_patterns]] + selector = "global:shared_secret" + action = "redact" + + [[scope.rules.value_patterns]] + selector = "ret:literal:app.sec.sensitive" + action = "redact" + + [[scope.rules.value_patterns]] + selector = "local:internal" + action = "drop" + "#, + ); + let config = TraceFilterConfig::from_paths(&[filter_path]).expect("load filter"); + let engine = Arc::new(TraceFilterEngine::new(config)); + + let app_dir = project_root.join("app"); + fs::create_dir_all(&app_dir).expect("create app dir"); + let script_path = app_dir.join("sec.py"); + let body = r#" +shared_secret = "initial" + +def sensitive(password): + secret = "token" + internal = "hidden" + public = "visible" + globals()['shared_secret'] = password + snapshot() + emit_return(password) + return password + +sensitive("s3cr3t") +"#; + let script = format!("{PRELUDE}\n{body}", PRELUDE = PRELUDE, body = body); + fs::write(&script_path, script).expect("write script"); + + let mut tracer = RuntimeTracer::new( + script_path.to_string_lossy().as_ref(), + &[], + TraceEventsFileFormat::Json, + None, + Some(engine), + ); + + { + let _guard = ScopedTracer::new(&mut tracer); + LAST_OUTCOME.with(|cell| cell.set(None)); + let run_code = format!( + "import runpy, sys\nsys.path.insert(0, r\"{}\")\nrunpy.run_path(r\"{}\")", + project_root.display(), + script_path.display() + ); + let run_code_c = CString::new(run_code).expect("script contains nul byte"); + py.run(run_code_c.as_c_str(), None, None) + .expect("execute filtered script"); + } + + let mut variable_names: Vec = Vec::new(); + for event in &tracer.writer.events { + if let TraceLowLevelEvent::VariableName(name) = event { + variable_names.push(name.clone()); + } + } + assert!( + !variable_names.iter().any(|name| name == "internal"), + "internal variable should not be recorded" + ); + + let password_index = variable_names + .iter() + .position(|name| name == "password") + .expect("password variable recorded"); + let password_value = tracer + .writer + .events + .iter() + .find_map(|event| match event { + TraceLowLevelEvent::Value(record) if record.variable_id.0 == password_index => { + Some(record.value.clone()) + } + _ => None, + }) + .expect("password value recorded"); + match password_value { + ValueRecord::Error { ref msg, .. } => assert_eq!(msg, ""), + ref other => panic!("expected password argument redacted, got {other:?}"), + } + + let snapshots = collect_snapshots(&tracer.writer.events); + let snapshot = find_snapshot_with_vars( + &snapshots, + &["secret", "public", "shared_secret", "password"], + ); + assert_var( + snapshot, + "secret", + SimpleValue::Raw("".to_string()), + ); + assert_var( + snapshot, + "public", + SimpleValue::String("visible".to_string()), + ); + assert_var( + snapshot, + "shared_secret", + SimpleValue::Raw("".to_string()), + ); + assert_var( + snapshot, + "password", + SimpleValue::Raw("".to_string()), + ); + assert_no_variable(&snapshots, "internal"); + + let return_record = tracer + .writer + .events + .iter() + .find_map(|event| match event { + TraceLowLevelEvent::Return(record) => Some(record.clone()), + _ => None, + }) + .expect("return record"); + + match return_record.return_value { + ValueRecord::Error { ref msg, .. } => assert_eq!(msg, ""), + ref other => panic!("expected redacted return value, got {other:?}"), + } + }); + } + + #[test] + fn trace_filter_metadata_includes_summary() { + Python::with_gil(|py| { + reset_policy(py); + ensure_test_module(py); + + let project = tempfile::tempdir().expect("project dir"); + let project_root = project.path(); + let filters_dir = project_root.join(".codetracer"); + fs::create_dir(&filters_dir).expect("create .codetracer"); + let filter_path = filters_dir.join("filters.toml"); + write_filter( + &filter_path, + r#" + [meta] + name = "redact" + version = 1 + + [scope] + default_exec = "trace" + default_value_action = "allow" + + [[scope.rules]] + selector = "pkg:app.sec" + exec = "trace" + value_default = "allow" + + [[scope.rules.value_patterns]] + selector = "arg:password" + action = "redact" + + [[scope.rules.value_patterns]] + selector = "local:password" + action = "redact" + + [[scope.rules.value_patterns]] + selector = "local:secret" + action = "redact" + + [[scope.rules.value_patterns]] + selector = "global:shared_secret" + action = "redact" + + [[scope.rules.value_patterns]] + selector = "ret:literal:app.sec.sensitive" + action = "redact" + + [[scope.rules.value_patterns]] + selector = "local:internal" + action = "drop" + "#, + ); + let config = TraceFilterConfig::from_paths(&[filter_path]).expect("load filter"); + let engine = Arc::new(TraceFilterEngine::new(config)); + + let app_dir = project_root.join("app"); + fs::create_dir_all(&app_dir).expect("create app dir"); + let script_path = app_dir.join("sec.py"); + let body = r#" +shared_secret = "initial" + +def sensitive(password): + secret = "token" + internal = "hidden" + public = "visible" + globals()['shared_secret'] = password + snapshot() + emit_return(password) + return password + +sensitive("s3cr3t") +"#; + let script = format!("{PRELUDE}\n{body}", PRELUDE = PRELUDE, body = body); + fs::write(&script_path, script).expect("write script"); + + let outputs_dir = tempfile::tempdir().expect("outputs dir"); + let outputs = TraceOutputPaths::new(outputs_dir.path(), TraceEventsFileFormat::Json); + + let program = script_path.to_string_lossy().into_owned(); + let mut tracer = RuntimeTracer::new( + &program, + &[], + TraceEventsFileFormat::Json, + None, + Some(engine), + ); + tracer.begin(&outputs, 1).expect("begin tracer"); + + { + let _guard = ScopedTracer::new(&mut tracer); + LAST_OUTCOME.with(|cell| cell.set(None)); + let run_code = format!( + "import runpy, sys\nsys.path.insert(0, r\"{}\")\nrunpy.run_path(r\"{}\")", + project_root.display(), + script_path.display() + ); + let run_code_c = CString::new(run_code).expect("script contains nul byte"); + py.run(run_code_c.as_c_str(), None, None) + .expect("execute script"); + } + + tracer.finish(py).expect("finish tracer"); + + let metadata_str = fs::read_to_string(outputs.metadata()).expect("read metadata"); + let metadata: serde_json::Value = + serde_json::from_str(&metadata_str).expect("parse metadata"); + let trace_filter = metadata + .get("trace_filter") + .and_then(|value| value.as_object()) + .expect("trace_filter metadata"); + + let filters = trace_filter + .get("filters") + .and_then(|value| value.as_array()) + .expect("filters array"); + assert_eq!(filters.len(), 1); + let filter_entry = filters[0].as_object().expect("filter entry"); + assert_eq!( + filter_entry.get("name").and_then(|v| v.as_str()), + Some("redact") + ); + + let stats = trace_filter + .get("stats") + .and_then(|value| value.as_object()) + .expect("stats object"); + assert_eq!( + stats.get("scopes_skipped").and_then(|v| v.as_u64()), + Some(0) + ); + let value_redactions = stats + .get("value_redactions") + .and_then(|value| value.as_object()) + .expect("value_redactions object"); + assert_eq!( + value_redactions.get("argument").and_then(|v| v.as_u64()), + Some(0) + ); + // Argument values currently surface through local snapshots; once call-record redaction wiring lands this count should rise above zero. + assert_eq!( + value_redactions.get("local").and_then(|v| v.as_u64()), + Some(2) + ); + assert_eq!( + value_redactions.get("global").and_then(|v| v.as_u64()), + Some(1) + ); + assert_eq!( + value_redactions.get("return").and_then(|v| v.as_u64()), + Some(1) + ); + assert_eq!( + value_redactions.get("attribute").and_then(|v| v.as_u64()), + Some(0) + ); + let value_drops = stats + .get("value_drops") + .and_then(|value| value.as_object()) + .expect("value_drops object"); + assert_eq!( + value_drops.get("argument").and_then(|v| v.as_u64()), + Some(0) + ); + assert_eq!(value_drops.get("local").and_then(|v| v.as_u64()), Some(1)); + assert_eq!(value_drops.get("global").and_then(|v| v.as_u64()), Some(0)); + assert_eq!(value_drops.get("return").and_then(|v| v.as_u64()), Some(0)); + assert_eq!( + value_drops.get("attribute").and_then(|v| v.as_u64()), + Some(0) + ); + }); + } + + fn assert_var(snapshot: &Snapshot, name: &str, expected: SimpleValue) { + let actual = snapshot + .vars + .get(name) + .unwrap_or_else(|| panic!("{name} missing at line {}", snapshot.line)); + assert_eq!( + actual, &expected, + "Unexpected value for {name} at line {}", + snapshot.line + ); + } + + fn find_snapshot_with_vars<'a>(snapshots: &'a [Snapshot], names: &[&str]) -> &'a Snapshot { + snapshots + .iter() + .find(|snap| names.iter().all(|n| snap.vars.contains_key(*n))) + .unwrap_or_else(|| panic!("No snapshot containing variables {:?}", names)) + } + + fn assert_no_variable(snapshots: &[Snapshot], name: &str) { + if snapshots.iter().any(|snap| snap.vars.contains_key(name)) { + panic!("Variable {name} unexpectedly captured"); + } + } + + #[test] + fn captures_simple_function_locals() { + let snapshots = run_traced_script( + r#" +def simple_function(x): + snapshot() + a = 1 + snapshot() + b = a + x + snapshot() + return a, b + +simple_function(5) +"#, + ); + + assert_var(&snapshots[0], "x", SimpleValue::Int(5)); + assert!(!snapshots[0].vars.contains_key("a")); + assert_var(&snapshots[1], "a", SimpleValue::Int(1)); + assert_var(&snapshots[2], "b", SimpleValue::Int(6)); + } + + #[test] + fn captures_closure_variables() { + let snapshots = run_traced_script( + r#" +def outer_func(x): + snapshot() + y = 1 + snapshot() + def inner_func(z): + nonlocal y + snapshot() + w = x + y + z + snapshot() + y = w + snapshot() + return w + total = inner_func(5) + snapshot() + return y, total + +result = outer_func(2) +"#, + ); + + let inner_entry = find_snapshot_with_vars(&snapshots, &["x", "y", "z"]); + assert_var(inner_entry, "x", SimpleValue::Int(2)); + assert_var(inner_entry, "y", SimpleValue::Int(1)); + + let w_snapshot = find_snapshot_with_vars(&snapshots, &["w", "x", "y", "z"]); + assert_var(w_snapshot, "w", SimpleValue::Int(8)); + + let outer_after = find_snapshot_with_vars(&snapshots, &["total", "y"]); + assert_var(outer_after, "total", SimpleValue::Int(8)); + assert_var(outer_after, "y", SimpleValue::Int(8)); + } + + #[test] + fn captures_globals() { + let snapshots = run_traced_script( + r#" +GLOBAL_VAL = 10 +counter = 0 +snapshot() + +def global_test(): + snapshot() + local_copy = GLOBAL_VAL + snapshot() + global counter + counter += 1 + snapshot() + return local_copy, counter + +before = counter +snapshot() +result = global_test() +snapshot() +after = counter +snapshot() +"#, + ); + + let access_global = find_snapshot_with_vars(&snapshots, &["local_copy", "GLOBAL_VAL"]); + assert_var(access_global, "GLOBAL_VAL", SimpleValue::Int(10)); + assert_var(access_global, "local_copy", SimpleValue::Int(10)); + + let last_counter = snapshots + .iter() + .rev() + .find(|snap| snap.vars.contains_key("counter")) + .expect("Expected at least one counter snapshot"); + assert_var(last_counter, "counter", SimpleValue::Int(1)); + } + + #[test] + fn captures_class_scope() { + let snapshots = run_traced_script( + r#" +CONSTANT = 42 +snapshot() + +class MetaCounter(type): + count = 0 + snapshot() + def __init__(cls, name, bases, attrs): + snapshot() + MetaCounter.count += 1 + super().__init__(name, bases, attrs) + +class Sample(metaclass=MetaCounter): + snapshot() + a = 10 + snapshot() + b = a + 5 + snapshot() + print(a, b, CONSTANT) + snapshot() + def method(self): + snapshot() + return self.a + self.b + +instance = Sample() +snapshot() +instances = MetaCounter.count +snapshot() +_ = instance.method() +snapshot() +"#, + ); + + let meta_init = find_snapshot_with_vars(&snapshots, &["cls", "name", "attrs"]); + assert_var(meta_init, "name", SimpleValue::String("Sample".to_string())); + + let class_body = find_snapshot_with_vars(&snapshots, &["a", "b"]); + assert_var(class_body, "a", SimpleValue::Int(10)); + assert_var(class_body, "b", SimpleValue::Int(15)); + + let method_snapshot = find_snapshot_with_vars(&snapshots, &["self"]); + assert!(method_snapshot.vars.contains_key("self")); + } + + #[test] + fn captures_lambda_and_comprehensions() { + let snapshots = run_traced_script( + r#" +factor = 2 +snapshot() +double = lambda y: snap(y * factor) +snapshot() +lambda_value = double(5) +snapshot() +squares = [snap(n ** 2) for n in range(3)] +snapshot() +scaled_set = {snap(n * factor) for n in range(3)} +snapshot() +mapping = {n: snap(n * factor) for n in range(3)} +snapshot() +gen_exp = (snap(n * factor) for n in range(3)) +snapshot() +result_list = list(gen_exp) +snapshot() +"#, + ); + + let lambda_snapshot = find_snapshot_with_vars(&snapshots, &["y", "factor"]); + assert_var(lambda_snapshot, "y", SimpleValue::Int(5)); + assert_var(lambda_snapshot, "factor", SimpleValue::Int(2)); + + let list_comp = find_snapshot_with_vars(&snapshots, &["n", "factor"]); + assert!(matches!(list_comp.vars.get("n"), Some(SimpleValue::Int(_)))); + + let result_snapshot = find_snapshot_with_vars(&snapshots, &["result_list"]); + assert!(matches!( + result_snapshot.vars.get("result_list"), + Some(SimpleValue::Sequence(_)) + )); + } + + #[test] + fn captures_generators_and_coroutines() { + let snapshots = run_traced_script( + r#" +import asyncio +snapshot() + + +def counter_gen(n): + snapshot() + total = 0 + for i in range(n): + total += i + snapshot() + yield total + snapshot() + return total + +async def async_sum(data): + snapshot() + total = 0 + for x in data: + total += x + snapshot() + await asyncio.sleep(0) + snapshot() + return total + +gen = counter_gen(3) +gen_results = list(gen) +snapshot() +coroutine_result = asyncio.run(async_sum([1, 2, 3])) +snapshot() +"#, + ); + + let generator_step = find_snapshot_with_vars(&snapshots, &["i", "total"]); + assert!(matches!( + generator_step.vars.get("i"), + Some(SimpleValue::Int(_)) + )); + + let coroutine_steps: Vec<&Snapshot> = snapshots + .iter() + .filter(|snap| snap.vars.contains_key("x")) + .collect(); + assert!(!coroutine_steps.is_empty()); + let final_coroutine_step = coroutine_steps.last().unwrap(); + assert_var(final_coroutine_step, "total", SimpleValue::Int(6)); + + let coroutine_result_snapshot = find_snapshot_with_vars(&snapshots, &["coroutine_result"]); + assert!(coroutine_result_snapshot + .vars + .contains_key("coroutine_result")); + } + + #[test] + fn captures_exception_and_with_blocks() { + let snapshots = run_traced_script( + r#" +import io +__file__ = "test_script.py" + +def exception_and_with_demo(x): + snapshot() + try: + inv = 10 / x + snapshot() + except ZeroDivisionError as e: + snapshot() + error_msg = f"Error: {e}" + snapshot() + else: + snapshot() + inv += 1 + snapshot() + finally: + snapshot() + final_flag = True + snapshot() + with io.StringIO("dummy line") as f: + snapshot() + first_line = f.readline() + snapshot() + snapshot() + return locals() + +result1 = exception_and_with_demo(0) +snapshot() +result2 = exception_and_with_demo(5) +snapshot() +"#, + ); + + let except_snapshot = find_snapshot_with_vars(&snapshots, &["e", "error_msg"]); + assert!(matches!( + except_snapshot.vars.get("error_msg"), + Some(SimpleValue::String(_)) + )); + + let finally_snapshot = find_snapshot_with_vars(&snapshots, &["final_flag"]); + assert_var(finally_snapshot, "final_flag", SimpleValue::Bool(true)); + + let with_snapshot = find_snapshot_with_vars(&snapshots, &["f", "first_line"]); + assert!(with_snapshot.vars.contains_key("first_line")); + } + + #[test] + fn captures_decorators() { + let snapshots = run_traced_script( + r#" +setting = "Hello" +snapshot() + + +def my_decorator(func): + snapshot() + def wrapper(*args, **kwargs): + snapshot() + return func(*args, **kwargs) + return wrapper + +@my_decorator +def greet(name): + snapshot() + message = f"Hi, {name}" + snapshot() + return message + +output = greet("World") +snapshot() +"#, + ); + + let decorator_snapshot = find_snapshot_with_vars(&snapshots, &["func", "setting"]); + assert!(decorator_snapshot.vars.contains_key("func")); + + let wrapper_snapshot = find_snapshot_with_vars(&snapshots, &["args", "kwargs", "setting"]); + assert!(wrapper_snapshot.vars.contains_key("args")); + + let greet_snapshot = find_snapshot_with_vars(&snapshots, &["name", "message"]); + assert_var( + greet_snapshot, + "name", + SimpleValue::String("World".to_string()), + ); + } + + #[test] + fn captures_dynamic_execution() { + let snapshots = run_traced_script( + r#" +expr_code = "dynamic_var = 99" +snapshot() +exec(expr_code) +snapshot() +check = dynamic_var + 1 +snapshot() + +def eval_test(): + snapshot() + value = 10 + formula = "value * 2" + snapshot() + result = eval(formula) + snapshot() + return result + +out = eval_test() +snapshot() +"#, + ); + + let exec_snapshot = find_snapshot_with_vars(&snapshots, &["dynamic_var"]); + assert_var(exec_snapshot, "dynamic_var", SimpleValue::Int(99)); + + let eval_snapshot = find_snapshot_with_vars(&snapshots, &["value", "formula"]); + assert_var(eval_snapshot, "value", SimpleValue::Int(10)); + } + + #[test] + fn captures_imports() { + let snapshots = run_traced_script( + r#" +import math +snapshot() + +def import_test(): + snapshot() + import os + snapshot() + constant = math.pi + snapshot() + cwd = os.getcwd() + snapshot() + return constant, cwd + +val, path = import_test() +snapshot() +"#, + ); + + let global_import = find_snapshot_with_vars(&snapshots, &["math"]); + assert!(matches!( + global_import.vars.get("math"), + Some(SimpleValue::Raw(_)) + )); + + let local_import = find_snapshot_with_vars(&snapshots, &["os", "constant"]); + assert!(local_import.vars.contains_key("os")); + } + + #[test] + fn builtins_not_recorded() { + let snapshots = run_traced_script( + r#" +def builtins_test(seq): + snapshot() + n = len(seq) + snapshot() + m = max(seq) + snapshot() + return n, m + +result = builtins_test([5, 3, 7]) +snapshot() +"#, + ); + + let len_snapshot = find_snapshot_with_vars(&snapshots, &["n"]); + assert_var(len_snapshot, "n", SimpleValue::Int(3)); + assert_no_variable(&snapshots, "len"); + } + + #[test] + fn finish_enforces_require_trace_policy() { + Python::with_gil(|py| { + policy::configure_policy_py( + Some("abort"), + Some(true), + Some(false), + None, + None, + Some(false), + None, + None, + ) + .expect("enable require_trace policy"); + + let script_dir = tempfile::tempdir().expect("script dir"); + let program_path = script_dir.path().join("program.py"); + std::fs::write(&program_path, "print('hi')\n").expect("write program"); + + let outputs_dir = tempfile::tempdir().expect("outputs dir"); + let outputs = TraceOutputPaths::new(outputs_dir.path(), TraceEventsFileFormat::Json); + + let mut tracer = RuntimeTracer::new( + program_path.to_string_lossy().as_ref(), + &[], + TraceEventsFileFormat::Json, + None, + None, + ); + tracer.begin(&outputs, 1).expect("begin tracer"); + + let err = tracer + .finish(py) + .expect_err("finish should error when require_trace true"); + let message = err.to_string(); + assert!( + message.contains("ERR_TRACE_MISSING"), + "expected trace missing error, got {message}" + ); + + reset_policy(py); + }); + } + + #[test] + fn finish_removes_partial_outputs_when_policy_forbids_keep() { + Python::with_gil(|py| { + reset_policy(py); + + let script_dir = tempfile::tempdir().expect("script dir"); + let program_path = script_dir.path().join("program.py"); + std::fs::write(&program_path, "print('hi')\n").expect("write program"); + + let outputs_dir = tempfile::tempdir().expect("outputs dir"); + let outputs = TraceOutputPaths::new(outputs_dir.path(), TraceEventsFileFormat::Json); + + let mut tracer = RuntimeTracer::new( + program_path.to_string_lossy().as_ref(), + &[], + TraceEventsFileFormat::Json, + None, + None, + ); + tracer.begin(&outputs, 1).expect("begin tracer"); + tracer.mark_failure(); + + tracer.finish(py).expect("finish after failure"); + + assert!(!outputs.events().exists(), "expected events file removed"); + assert!( + !outputs.metadata().exists(), + "expected metadata file removed" + ); + assert!(!outputs.paths().exists(), "expected paths file removed"); + }); + } + + #[test] + fn finish_keeps_partial_outputs_when_policy_allows() { + Python::with_gil(|py| { + policy::configure_policy_py( + Some("abort"), + Some(false), + Some(true), + None, + None, + Some(false), + None, + None, + ) + .expect("enable keep_partial policy"); + + let script_dir = tempfile::tempdir().expect("script dir"); + let program_path = script_dir.path().join("program.py"); + std::fs::write(&program_path, "print('hi')\n").expect("write program"); + + let outputs_dir = tempfile::tempdir().expect("outputs dir"); + let outputs = TraceOutputPaths::new(outputs_dir.path(), TraceEventsFileFormat::Json); + + let mut tracer = RuntimeTracer::new( + program_path.to_string_lossy().as_ref(), + &[], + TraceEventsFileFormat::Json, + None, + None, + ); + tracer.begin(&outputs, 1).expect("begin tracer"); + tracer.mark_failure(); + + tracer.finish(py).expect("finish after failure"); + + assert!(outputs.events().exists(), "expected events file retained"); + assert!( + outputs.metadata().exists(), + "expected metadata file retained" + ); + assert!(outputs.paths().exists(), "expected paths file retained"); + + reset_policy(py); + }); + } +} diff --git a/codetracer-python-recorder/src/session/bootstrap.rs b/codetracer-python-recorder/src/session/bootstrap.rs index a4697f1..4186ece 100644 --- a/codetracer-python-recorder/src/session/bootstrap.rs +++ b/codetracer-python-recorder/src/session/bootstrap.rs @@ -1,25 +1,24 @@ //! Helpers for preparing a tracing session before installing the runtime tracer. -use std::env; +mod filesystem; +mod filters; +mod metadata; + use std::fmt; -use std::fs; use std::path::{Path, PathBuf}; use std::sync::Arc; use pyo3::prelude::*; -use recorder_errors::{enverr, usage, ErrorCode}; use runtime_tracing::TraceEventsFileFormat; use crate::errors::Result; -use crate::trace_filter::config::TraceFilterConfig; use crate::trace_filter::engine::TraceFilterEngine; +use filesystem::{ensure_trace_directory, resolve_trace_format}; +use filters::load_trace_filter; +use metadata::collect_program_metadata; /// Basic metadata about the currently running Python program. -#[derive(Debug, Clone)] -pub struct ProgramMetadata { - pub program: String, - pub args: Vec, -} +pub use metadata::ProgramMetadata; /// Collected data required to start a tracing session. #[derive(Clone)] @@ -31,12 +30,6 @@ pub struct TraceSessionBootstrap { trace_filter: Option>, } -const TRACE_FILTER_DIR: &str = ".codetracer"; -const TRACE_FILTER_FILE: &str = "trace-filter.toml"; -const BUILTIN_FILTER_LABEL: &str = "builtin-default"; -const BUILTIN_TRACE_FILTER: &str = - include_str!("../../resources/trace_filters/builtin_default.toml"); - impl fmt::Debug for TraceSessionBootstrap { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("TraceSessionBootstrap") @@ -61,10 +54,7 @@ impl TraceSessionBootstrap { ) -> Result { ensure_trace_directory(trace_directory)?; let format = resolve_trace_format(format)?; - let metadata = collect_program_metadata(py).map_err(|err| { - enverr!(ErrorCode::Io, "failed to collect program metadata") - .with_context("details", err.to_string()) - })?; + let metadata = collect_program_metadata(py)?; let trace_filter = load_trace_filter(explicit_trace_filters, &metadata.program)?; Ok(Self { trace_directory: trace_directory.to_path_buf(), @@ -100,220 +90,13 @@ impl TraceSessionBootstrap { } } -/// Ensure the requested trace directory exists and is writable. -pub fn ensure_trace_directory(path: &Path) -> Result<()> { - if path.exists() { - if !path.is_dir() { - return Err(usage!( - ErrorCode::TraceDirectoryConflict, - "trace path exists and is not a directory" - ) - .with_context("path", path.display().to_string())); - } - return Ok(()); - } - - fs::create_dir_all(path).map_err(|e| { - enverr!( - ErrorCode::TraceDirectoryCreateFailed, - "failed to create trace directory" - ) - .with_context("path", path.display().to_string()) - .with_context("io", e.to_string()) - }) -} - -/// Convert a user-provided format string into the runtime representation. -pub fn resolve_trace_format(value: &str) -> Result { - match value.to_ascii_lowercase().as_str() { - "json" => Ok(TraceEventsFileFormat::Json), - // Accept historical aliases for the binary format. - "binary" | "binaryv0" | "binary_v0" | "b0" => Ok(TraceEventsFileFormat::BinaryV0), - other => Err(usage!( - ErrorCode::UnsupportedFormat, - "unsupported trace format '{}'. Expected one of: json, binary", - other - )), - } -} - -/// Capture program name and arguments from `sys.argv` for metadata records. -pub fn collect_program_metadata(py: Python<'_>) -> PyResult { - let sys = py.import("sys")?; - let argv = sys.getattr("argv")?; - - let program = match argv.get_item(0) { - Ok(obj) => obj.extract::()?, - Err(_) => String::from(""), - }; - - let args = match argv.len() { - Ok(len) if len > 1 => { - let mut items = Vec::with_capacity(len.saturating_sub(1)); - for idx in 1..len { - let value: String = argv.get_item(idx)?.extract()?; - items.push(value); - } - items - } - _ => Vec::new(), - }; - - Ok(ProgramMetadata { program, args }) -} - -fn load_trace_filter( - explicit: Option<&[PathBuf]>, - program: &str, -) -> Result>> { - let mut chain: Vec = Vec::new(); - - if let Some(default) = discover_default_trace_filter(program)? { - chain.push(default); - } - - if let Some(paths) = explicit { - chain.extend(paths.iter().cloned()); - } - - let config = TraceFilterConfig::from_inline_and_paths( - &[(BUILTIN_FILTER_LABEL, BUILTIN_TRACE_FILTER)], - &chain, - )?; - Ok(Some(Arc::new(TraceFilterEngine::new(config)))) -} - -fn discover_default_trace_filter(program: &str) -> Result> { - let start_dir = resolve_program_directory(program)?; - let mut current: Option<&Path> = Some(start_dir.as_path()); - while let Some(dir) = current { - let candidate = dir.join(TRACE_FILTER_DIR).join(TRACE_FILTER_FILE); - if matches!(fs::metadata(&candidate), Ok(metadata) if metadata.is_file()) { - return Ok(Some(candidate)); - } - current = dir.parent(); - } - Ok(None) -} - -fn resolve_program_directory(program: &str) -> Result { - let trimmed = program.trim(); - if trimmed.is_empty() || trimmed == "" { - return current_directory(); - } - - let path = Path::new(trimmed); - if path.is_absolute() { - if path.is_dir() { - return Ok(path.to_path_buf()); - } - if let Some(parent) = path.parent() { - return Ok(parent.to_path_buf()); - } - return current_directory(); - } - - let cwd = current_directory()?; - let joined = cwd.join(path); - if joined.is_dir() { - return Ok(joined); - } - if let Some(parent) = joined.parent() { - return Ok(parent.to_path_buf()); - } - Ok(cwd) -} - -fn current_directory() -> Result { - env::current_dir().map_err(|err| { - enverr!(ErrorCode::Io, "failed to resolve current directory") - .with_context("io", err.to_string()) - }) -} - #[cfg(test)] mod tests { use super::*; - use pyo3::types::PyList; - use recorder_errors::ErrorCode; + use metadata::tests::{with_sys_argv, ProgramArgs}; use std::path::PathBuf; use tempfile::tempdir; - #[test] - fn ensure_trace_directory_creates_missing_dir() { - let tmp = tempdir().expect("tempdir"); - let target = tmp.path().join("trace-out"); - ensure_trace_directory(&target).expect("create directory"); - assert!(target.is_dir()); - } - - #[test] - fn ensure_trace_directory_rejects_file_path() { - let tmp = tempdir().expect("tempdir"); - let file_path = tmp.path().join("trace.bin"); - std::fs::write(&file_path, b"stub").expect("write stub file"); - let err = ensure_trace_directory(&file_path).expect_err("should reject file path"); - assert_eq!(err.code, ErrorCode::TraceDirectoryConflict); - } - - #[test] - fn resolve_trace_format_accepts_supported_aliases() { - assert!(matches!( - resolve_trace_format("json").expect("json format"), - TraceEventsFileFormat::Json - )); - assert!(matches!( - resolve_trace_format("BiNaRy").expect("binary alias"), - TraceEventsFileFormat::BinaryV0 - )); - } - - #[test] - fn resolve_trace_format_rejects_unknown_values() { - let err = resolve_trace_format("yaml").expect_err("should reject yaml"); - assert_eq!(err.code, ErrorCode::UnsupportedFormat); - assert!(err.message().contains("unsupported trace format")); - } - - #[test] - fn collect_program_metadata_reads_sys_argv() { - Python::with_gil(|py| { - let sys = py.import("sys").expect("import sys"); - let original = sys.getattr("argv").expect("argv").unbind(); - let argv = PyList::new(py, ["/tmp/prog.py", "--flag", "value"]).expect("argv"); - sys.setattr("argv", argv).expect("set argv"); - - let result = collect_program_metadata(py); - sys.setattr("argv", original.bind(py)) - .expect("restore argv"); - - let metadata = result.expect("metadata"); - assert_eq!(metadata.program, "/tmp/prog.py"); - assert_eq!( - metadata.args, - vec!["--flag".to_string(), "value".to_string()] - ); - }); - } - - #[test] - fn collect_program_metadata_defaults_unknown_program() { - Python::with_gil(|py| { - let sys = py.import("sys").expect("import sys"); - let original = sys.getattr("argv").expect("argv").unbind(); - let empty = PyList::empty(py); - sys.setattr("argv", empty).expect("set empty argv"); - - let result = collect_program_metadata(py); - sys.setattr("argv", original.bind(py)) - .expect("restore argv"); - - let metadata = result.expect("metadata"); - assert_eq!(metadata.program, ""); - assert!(metadata.args.is_empty()); - }); - } - #[test] fn prepare_bootstrap_populates_fields_and_creates_directory() { Python::with_gil(|py| { @@ -322,21 +105,16 @@ mod tests { let activation = tmp.path().join("entry.py"); std::fs::write(&activation, "print('hi')\n").expect("write activation file"); - let sys = py.import("sys").expect("import sys"); - let original = sys.getattr("argv").expect("argv").unbind(); let program_str = activation.to_str().expect("utf8 path"); - let argv = PyList::new(py, [program_str, "--verbose"]).expect("argv"); - sys.setattr("argv", argv).expect("set argv"); - - let result = TraceSessionBootstrap::prepare( - py, - trace_dir.as_path(), - "json", - Some(activation.as_path()), - None, - ); - sys.setattr("argv", original.bind(py)) - .expect("restore argv"); + let result = with_sys_argv(py, ProgramArgs::new([program_str, "--verbose"]), || { + TraceSessionBootstrap::prepare( + py, + trace_dir.as_path(), + "json", + Some(activation.as_path()), + None, + ) + }); let bootstrap = result.expect("bootstrap"); assert!(trace_dir.is_dir()); @@ -357,15 +135,11 @@ mod tests { let script_path = tmp.path().join("app.py"); std::fs::write(&script_path, "print('hello')\n").expect("write script"); - let sys = py.import("sys").expect("import sys"); - let original = sys.getattr("argv").expect("argv").unbind(); - let argv = PyList::new(py, [script_path.to_str().expect("utf8 path")]).expect("argv"); - sys.setattr("argv", argv).expect("set argv"); - - let result = - TraceSessionBootstrap::prepare(py, trace_dir.as_path(), "json", None, None); - sys.setattr("argv", original.bind(py)) - .expect("restore argv"); + let result = with_sys_argv( + py, + ProgramArgs::new([script_path.to_str().expect("utf8 path")]), + || TraceSessionBootstrap::prepare(py, trace_dir.as_path(), "json", None, None), + ); let bootstrap = result.expect("bootstrap"); let engine = bootstrap.trace_filter().expect("builtin filter"); @@ -390,37 +164,13 @@ mod tests { let script_path = app_dir.join("main.py"); std::fs::write(&script_path, "print('run')\n").expect("write script"); - let filters_dir = project_root.join(TRACE_FILTER_DIR); - std::fs::create_dir(&filters_dir).expect("create filter dir"); - let filter_path = filters_dir.join(TRACE_FILTER_FILE); - std::fs::write( - &filter_path, - r#" - [meta] - name = "default" - version = 1 - - [scope] - default_exec = "trace" - default_value_action = "allow" + let filter_path = filters::tests::write_default_filter(project_root); - [[scope.rules]] - selector = "pkg:src" - exec = "trace" - value_default = "allow" - "#, - ) - .expect("write filter"); - - let sys = py.import("sys").expect("import sys"); - let original = sys.getattr("argv").expect("argv").unbind(); - let argv = PyList::new(py, [script_path.to_str().expect("utf8 path")]).expect("argv"); - sys.setattr("argv", argv).expect("set argv"); - - let result = - TraceSessionBootstrap::prepare(py, trace_dir.as_path(), "json", None, None); - sys.setattr("argv", original.bind(py)) - .expect("restore argv"); + let result = with_sys_argv( + py, + ProgramArgs::new([script_path.to_str().expect("utf8 path")]), + || TraceSessionBootstrap::prepare(py, trace_dir.as_path(), "json", None, None), + ); let bootstrap = result.expect("bootstrap"); let engine = bootstrap.trace_filter().expect("filter engine"); @@ -441,68 +191,24 @@ mod tests { let project_root = project.path(); let trace_dir = project_root.join("out"); - let app_dir = project_root.join("src"); - std::fs::create_dir_all(&app_dir).expect("create src dir"); - let script_path = app_dir.join("main.py"); - std::fs::write(&script_path, "print('run')\n").expect("write script"); - - let filters_dir = project_root.join(TRACE_FILTER_DIR); - std::fs::create_dir(&filters_dir).expect("create filter dir"); - let default_filter_path = filters_dir.join(TRACE_FILTER_FILE); - std::fs::write( - &default_filter_path, - r#" - [meta] - name = "default" - version = 1 - - [scope] - default_exec = "trace" - default_value_action = "allow" - - [[scope.rules]] - selector = "pkg:src" - exec = "trace" - value_default = "allow" - "#, - ) - .expect("write default filter"); - - let override_filter_path = project_root.join("override-filter.toml"); - std::fs::write( - &override_filter_path, - r#" - [meta] - name = "override" - version = 1 - - [scope] - default_exec = "trace" - default_value_action = "allow" - - [[scope.rules]] - selector = "pkg:src.special" - exec = "skip" - value_default = "redact" - "#, - ) - .expect("write override filter"); - - let sys = py.import("sys").expect("import sys"); - let original = sys.getattr("argv").expect("argv").unbind(); - let argv = PyList::new(py, [script_path.to_str().expect("utf8 path")]).expect("argv"); - sys.setattr("argv", argv).expect("set argv"); + let script_path = filters::tests::write_app(project_root); + let (default_filter_path, override_filter_path) = + filters::tests::write_default_and_override(project_root); let explicit = vec![override_filter_path.clone()]; - let result = TraceSessionBootstrap::prepare( + let result = with_sys_argv( py, - trace_dir.as_path(), - "json", - None, - Some(explicit.as_slice()), + ProgramArgs::new([script_path.to_str().expect("utf8 path")]), + || { + TraceSessionBootstrap::prepare( + py, + trace_dir.as_path(), + "json", + None, + Some(explicit.as_slice()), + ) + }, ); - sys.setattr("argv", original.bind(py)) - .expect("restore argv"); let bootstrap = result.expect("bootstrap"); let engine = bootstrap.trace_filter().expect("filter engine"); diff --git a/codetracer-python-recorder/src/session/bootstrap/filesystem.rs b/codetracer-python-recorder/src/session/bootstrap/filesystem.rs new file mode 100644 index 0000000..86a4c6f --- /dev/null +++ b/codetracer-python-recorder/src/session/bootstrap/filesystem.rs @@ -0,0 +1,121 @@ +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; + +use recorder_errors::{enverr, usage, ErrorCode}; +use runtime_tracing::TraceEventsFileFormat; + +use crate::errors::Result; + +/// Ensure the requested trace directory exists and is writable. +pub fn ensure_trace_directory(path: &Path) -> Result<()> { + if path.exists() { + if !path.is_dir() { + return Err(usage!( + ErrorCode::TraceDirectoryConflict, + "trace path exists and is not a directory" + ) + .with_context("path", path.display().to_string())); + } + return Ok(()); + } + + fs::create_dir_all(path).map_err(|e| { + enverr!( + ErrorCode::TraceDirectoryCreateFailed, + "failed to create trace directory" + ) + .with_context("path", path.display().to_string()) + .with_context("io", e.to_string()) + }) +} + +/// Convert a user-provided format string into the runtime representation. +pub fn resolve_trace_format(value: &str) -> Result { + match value.to_ascii_lowercase().as_str() { + "json" => Ok(TraceEventsFileFormat::Json), + // Accept historical aliases for the binary format. + "binary" | "binaryv0" | "binary_v0" | "b0" => Ok(TraceEventsFileFormat::BinaryV0), + other => Err(usage!( + ErrorCode::UnsupportedFormat, + "unsupported trace format '{}'. Expected one of: json, binary", + other + )), + } +} + +pub fn resolve_program_directory(program: &str) -> Result { + let trimmed = program.trim(); + if trimmed.is_empty() || trimmed == "" { + return current_directory(); + } + + let path = Path::new(trimmed); + if path.is_absolute() { + if path.is_dir() { + return Ok(path.to_path_buf()); + } + if let Some(parent) = path.parent() { + return Ok(parent.to_path_buf()); + } + return current_directory(); + } + + let cwd = current_directory()?; + let joined = cwd.join(path); + if joined.is_dir() { + return Ok(joined); + } + if let Some(parent) = joined.parent() { + return Ok(parent.to_path_buf()); + } + Ok(cwd) +} + +pub fn current_directory() -> Result { + env::current_dir().map_err(|err| { + enverr!(ErrorCode::Io, "failed to resolve current directory") + .with_context("io", err.to_string()) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + #[test] + fn creates_missing_directory() { + let tmp = tempdir().expect("tempdir"); + let target = tmp.path().join("trace-out"); + ensure_trace_directory(&target).expect("create directory"); + assert!(target.is_dir()); + } + + #[test] + fn rejects_existing_file_target() { + let tmp = tempdir().expect("tempdir"); + let file_path = tmp.path().join("trace.bin"); + std::fs::write(&file_path, b"stub").expect("write stub file"); + let err = ensure_trace_directory(&file_path).expect_err("should reject file path"); + assert_eq!(err.code, ErrorCode::TraceDirectoryConflict); + } + + #[test] + fn resolves_supported_formats() { + assert!(matches!( + resolve_trace_format("json").expect("json format"), + TraceEventsFileFormat::Json + )); + assert!(matches!( + resolve_trace_format("binary").expect("binary format"), + TraceEventsFileFormat::BinaryV0 + )); + } + + #[test] + fn rejects_unknown_format() { + let err = resolve_trace_format("yaml").expect_err("should reject yaml"); + assert_eq!(err.code, ErrorCode::UnsupportedFormat); + } +} diff --git a/codetracer-python-recorder/src/session/bootstrap/filters.rs b/codetracer-python-recorder/src/session/bootstrap/filters.rs new file mode 100644 index 0000000..3a17230 --- /dev/null +++ b/codetracer-python-recorder/src/session/bootstrap/filters.rs @@ -0,0 +1,164 @@ +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use crate::errors::Result; +use crate::trace_filter::config::TraceFilterConfig; +use crate::trace_filter::engine::TraceFilterEngine; + +use super::filesystem::resolve_program_directory; + +const TRACE_FILTER_DIR: &str = ".codetracer"; +const TRACE_FILTER_FILE: &str = "trace-filter.toml"; +const BUILTIN_FILTER_LABEL: &str = "builtin-default"; +const BUILTIN_TRACE_FILTER: &str = + include_str!("../../../resources/trace_filters/builtin_default.toml"); + +pub fn load_trace_filter( + explicit: Option<&[PathBuf]>, + program: &str, +) -> Result>> { + let mut chain: Vec = Vec::new(); + + if let Some(default) = discover_default_trace_filter(program)? { + chain.push(default); + } + + if let Some(paths) = explicit { + chain.extend(paths.iter().cloned()); + } + + let config = TraceFilterConfig::from_inline_and_paths( + &[(BUILTIN_FILTER_LABEL, BUILTIN_TRACE_FILTER)], + &chain, + )?; + Ok(Some(Arc::new(TraceFilterEngine::new(config)))) +} + +fn discover_default_trace_filter(program: &str) -> Result> { + let start_dir = resolve_program_directory(program)?; + let mut current: Option<&Path> = Some(start_dir.as_path()); + while let Some(dir) = current { + let candidate = dir.join(TRACE_FILTER_DIR).join(TRACE_FILTER_FILE); + if matches!(std::fs::metadata(&candidate), Ok(metadata) if metadata.is_file()) { + return Ok(Some(candidate)); + } + current = dir.parent(); + } + Ok(None) +} + +#[cfg(test)] +pub mod tests { + use super::*; + use std::fs; + use std::path::{Path, PathBuf}; + use tempfile::tempdir; + + pub fn write_default_filter(root: &Path) -> PathBuf { + let filters_dir = root.join(TRACE_FILTER_DIR); + fs::create_dir_all(&filters_dir).expect("create filter dir"); + let filter_path = filters_dir.join(TRACE_FILTER_FILE); + fs::write( + &filter_path, + r#" + [meta] + name = "default" + version = 1 + + [scope] + default_exec = "trace" + default_value_action = "allow" + + [[scope.rules]] + selector = "pkg:src" + exec = "trace" + value_default = "allow" + "#, + ) + .expect("write filter"); + filter_path + } + + pub fn write_app(root: &Path) -> PathBuf { + let app_dir = root.join("src"); + fs::create_dir_all(&app_dir).expect("create src dir"); + let script_path = app_dir.join("main.py"); + fs::write(&script_path, "print('run')\n").expect("write script"); + script_path + } + + pub fn write_default_and_override(root: &Path) -> (PathBuf, PathBuf) { + let default = write_default_filter(root); + let override_filter_path = root.join("override-filter.toml"); + fs::write( + &override_filter_path, + r#" + [meta] + name = "override" + version = 1 + + [scope] + default_exec = "trace" + default_value_action = "allow" + + [[scope.rules]] + selector = "pkg:src.special" + exec = "skip" + value_default = "redact" + "#, + ) + .expect("write override filter"); + (default, override_filter_path) + } + + #[test] + fn discover_filter_walks_directories() { + let temp = tempdir().expect("tempdir"); + let root = temp.path(); + write_default_filter(root); + let script = write_app(root); + let found = + discover_default_trace_filter(script.to_str().expect("utf8")).expect("discover"); + assert!(found.is_some()); + } + + #[test] + fn load_trace_filter_includes_builtin() { + let temp = tempdir().expect("tempdir"); + let root = temp.path(); + let script = write_app(root); + + let engine = load_trace_filter(None, script.to_str().expect("utf8")) + .expect("load") + .expect("engine"); + let summary = engine.summary(); + assert!(summary + .entries + .iter() + .any(|entry| entry.path == PathBuf::from(""))); + } + + #[test] + fn load_trace_filter_merges_default_and_override() { + let temp = tempdir().expect("tempdir"); + let root = temp.path(); + let script = write_app(root); + let (default_filter_path, override_filter_path) = write_default_and_override(root); + + let engine = load_trace_filter( + Some(&[override_filter_path.clone()]), + script.to_str().expect("utf8"), + ) + .expect("load") + .expect("engine"); + let paths: Vec = engine + .summary() + .entries + .iter() + .map(|entry| entry.path.clone()) + .collect(); + assert!(paths.contains(&PathBuf::from(""))); + assert!(paths.contains(&default_filter_path)); + assert!(paths.contains(&override_filter_path)); + } +} diff --git a/codetracer-python-recorder/src/session/bootstrap/metadata.rs b/codetracer-python-recorder/src/session/bootstrap/metadata.rs new file mode 100644 index 0000000..fc43840 --- /dev/null +++ b/codetracer-python-recorder/src/session/bootstrap/metadata.rs @@ -0,0 +1,110 @@ +use pyo3::prelude::*; +use recorder_errors::{enverr, ErrorCode, RecorderError}; + +use crate::errors::Result; + +/// Basic metadata about the currently running Python program. +#[derive(Debug, Clone)] +pub struct ProgramMetadata { + pub program: String, + pub args: Vec, +} + +fn metadata_error(err: pyo3::PyErr) -> RecorderError { + enverr!(ErrorCode::Io, "failed to collect program metadata") + .with_context("details", err.to_string()) +} + +/// Capture program name and arguments from `sys.argv` for metadata records. +pub fn collect_program_metadata(py: Python<'_>) -> Result { + let sys = py.import("sys").map_err(metadata_error)?; + let argv = sys.getattr("argv").map_err(metadata_error)?; + + let program = match argv.get_item(0) { + Ok(obj) => obj + .extract::() + .unwrap_or_else(|_| "".to_string()), + Err(_) => "".to_string(), + }; + + let args = match argv.len() { + Ok(len) if len > 1 => { + let mut items = Vec::with_capacity(len.saturating_sub(1)); + for idx in 1..len { + if let Ok(value) = argv.get_item(idx).and_then(|obj| obj.extract::()) { + items.push(value); + } + } + items + } + _ => Vec::new(), + }; + + Ok(ProgramMetadata { program, args }) +} + +#[cfg(test)] +pub mod tests { + use super::*; + use pyo3::types::PyList; + + /// Helper struct for building argv lists in tests. + pub struct ProgramArgs<'a> { + items: Vec<&'a str>, + } + + impl<'a> ProgramArgs<'a> { + pub fn new(items: [&'a str; N]) -> Self { + Self { + items: items.to_vec(), + } + } + + pub fn empty() -> Self { + Self { items: Vec::new() } + } + } + + pub fn with_sys_argv(py: Python<'_>, args: ProgramArgs<'_>, op: F) -> R + where + F: FnOnce() -> R, + { + let sys = py.import("sys").expect("import sys"); + let original = sys.getattr("argv").expect("argv").unbind(); + let argv = PyList::new(py, args.items).expect("argv"); + sys.setattr("argv", argv).expect("set argv"); + + let result = op(); + + sys.setattr("argv", original.bind(py)) + .expect("restore argv"); + result + } + + #[test] + fn collects_program_and_args() { + Python::with_gil(|py| { + let metadata = with_sys_argv( + py, + ProgramArgs::new(["/tmp/prog.py", "--flag", "value"]), + || collect_program_metadata(py), + ) + .expect("metadata"); + assert_eq!(metadata.program, "/tmp/prog.py"); + assert_eq!( + metadata.args, + vec!["--flag".to_string(), "value".to_string()] + ); + }); + } + + #[test] + fn defaults_to_unknown_program() { + Python::with_gil(|py| { + let metadata = with_sys_argv(py, ProgramArgs::empty(), || collect_program_metadata(py)) + .expect("metadata"); + assert_eq!(metadata.program, ""); + assert!(metadata.args.is_empty()); + }); + } +} diff --git a/codetracer-python-recorder/src/trace_filter/config.rs b/codetracer-python-recorder/src/trace_filter/config.rs index f1507ed..b241988 100644 --- a/codetracer-python-recorder/src/trace_filter/config.rs +++ b/codetracer-python-recorder/src/trace_filter/config.rs @@ -1,154 +1,17 @@ -//! Filter configuration loader that parses TOML files, resolves inheritance, and -//! prepares flattened scope/value rules for the runtime engine. +//! Filter configuration facade: composes inline and file-based sources into a +//! resolved [`TraceFilterConfig`](crate::trace_filter::model::TraceFilterConfig). //! //! The implementation follows the schema defined in //! `design-docs/US0028 - Configurable Python trace filters.md`. -use crate::trace_filter::selector::{MatchType, Selector, SelectorKind}; -use recorder_errors::{usage, ErrorCode, RecorderResult}; -use serde::Deserialize; -use sha2::{Digest, Sha256}; -use std::collections::HashSet; -use std::fs; -use std::path::{Component, Path, PathBuf}; - -/// Scope-level execution directive. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ExecDirective { - Trace, - Skip, -} - -impl ExecDirective { - fn parse(token: &str) -> Option { - match token { - "trace" => Some(ExecDirective::Trace), - "skip" => Some(ExecDirective::Skip), - _ => None, - } - } -} - -/// Value-level capture directive. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ValueAction { - Allow, - Redact, - Drop, -} - -impl ValueAction { - fn parse(token: &str) -> Option { - match token { - "allow" => Some(ValueAction::Allow), - "redact" => Some(ValueAction::Redact), - "drop" => Some(ValueAction::Drop), - // Backwards compatibility for deprecated `deny`. - "deny" => Some(ValueAction::Redact), - _ => None, - } - } -} +pub use crate::trace_filter::model::{ + ExecDirective, FilterMeta, FilterSource, FilterSummary, FilterSummaryEntry, IoConfig, IoStream, + ScopeRule, TraceFilterConfig, ValueAction, ValuePattern, +}; -/// IO streams that can be captured in addition to scope/value rules. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum IoStream { - Stdout, - Stderr, - Stdin, - Files, -} - -impl IoStream { - fn parse(token: &str) -> Option { - match token { - "stdout" => Some(IoStream::Stdout), - "stderr" => Some(IoStream::Stderr), - "stdin" => Some(IoStream::Stdin), - "files" => Some(IoStream::Files), - _ => None, - } - } -} - -/// Metadata describing the source filter file. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct FilterMeta { - pub name: String, - pub version: u32, - pub description: Option, - pub labels: Vec, -} - -/// IO capture configuration. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct IoConfig { - pub capture: bool, - pub streams: Vec, -} - -impl Default for IoConfig { - fn default() -> Self { - IoConfig { - capture: false, - streams: Vec::new(), - } - } -} - -/// Value pattern applied within a scope rule. -#[derive(Debug, Clone)] -pub struct ValuePattern { - pub selector: Selector, - pub action: ValueAction, - pub reason: Option, - pub source_id: usize, -} - -/// Scope rule constructed from the flattened configuration chain. -#[derive(Debug, Clone)] -pub struct ScopeRule { - pub selector: Selector, - pub exec: Option, - pub value_default: Option, - pub value_patterns: Vec, - pub reason: Option, - pub source_id: usize, -} - -/// Source information for each filter file participating in the chain. -#[derive(Debug, Clone)] -pub struct FilterSource { - pub path: PathBuf, - pub sha256: String, - pub project_root: PathBuf, - pub meta: FilterMeta, -} - -/// Summary used for embedding in trace metadata. -#[derive(Debug, Clone)] -pub struct FilterSummary { - pub entries: Vec, -} - -/// Single entry in the filter summary. -#[derive(Debug, Clone)] -pub struct FilterSummaryEntry { - pub path: PathBuf, - pub sha256: String, - pub name: String, - pub version: u32, -} - -/// Fully resolved filter configuration ready for runtime consumption. -#[derive(Debug, Clone)] -pub struct TraceFilterConfig { - default_exec: ExecDirective, - default_value_action: ValueAction, - io: IoConfig, - rules: Vec, - sources: Vec, -} +use crate::trace_filter::loader::ConfigAggregator; +use recorder_errors::{usage, ErrorCode, RecorderResult}; +use std::path::PathBuf; impl TraceFilterConfig { /// Load and compose filters from the provided paths. @@ -180,826 +43,4 @@ impl TraceFilterConfig { aggregator.finish() } - - /// Default execution directive applied before scope rules run. - pub fn default_exec(&self) -> ExecDirective { - self.default_exec - } - - /// Default value action applied before rule-specific overrides. - pub fn default_value_action(&self) -> ValueAction { - self.default_value_action - } - - /// IO capture configuration associated with the composed filter chain. - pub fn io(&self) -> &IoConfig { - &self.io - } - - /// Flattened scope rules in execution order. - pub fn rules(&self) -> &[ScopeRule] { - &self.rules - } - - /// Source filter metadata used for embedding in trace output. - pub fn sources(&self) -> &[FilterSource] { - &self.sources - } - - /// Helper producing a summary used by metadata writers. - pub fn summary(&self) -> FilterSummary { - let entries = self - .sources - .iter() - .map(|source| FilterSummaryEntry { - path: source.path.clone(), - sha256: source.sha256.clone(), - name: source.meta.name.clone(), - version: source.meta.version, - }) - .collect(); - FilterSummary { entries } - } -} - -#[derive(Default)] -struct ConfigAggregator { - default_exec: Option, - default_value_action: Option, - io: Option, - rules: Vec, - sources: Vec, -} - -impl ConfigAggregator { - fn ingest_file(&mut self, path: &Path) -> RecorderResult<()> { - let contents = fs::read_to_string(path).map_err(|err| { - usage!( - ErrorCode::InvalidPolicyValue, - "failed to read trace filter '{}': {}", - path.display(), - err - ) - })?; - - self.ingest_source(path, &contents) - } - - fn ingest_inline(&mut self, label: &str, contents: &str) -> RecorderResult<()> { - let pseudo_path = PathBuf::from(format!("")); - self.ingest_source(&pseudo_path, contents) - } - - fn ingest_source(&mut self, path: &Path, contents: &str) -> RecorderResult<()> { - let checksum = calculate_sha256(contents); - let raw: RawFilterFile = toml::from_str(contents).map_err(|err| { - usage!( - ErrorCode::InvalidPolicyValue, - "failed to parse trace filter '{}': {}", - path.display(), - err - ) - })?; - - let project_root = detect_project_root(path); - let source_index = self.sources.len(); - self.sources.push(FilterSource { - path: path.to_path_buf(), - sha256: checksum, - project_root: project_root.clone(), - meta: parse_meta(&raw.meta, path)?, - }); - - let defaults = resolve_defaults( - &raw.scope, - path, - self.default_exec, - self.default_value_action, - )?; - if let Some(exec) = defaults.exec { - self.default_exec = Some(exec); - } - if let Some(value_action) = defaults.value_action { - self.default_value_action = Some(value_action); - } - - if let Some(io) = parse_io(raw.io.as_ref(), path)? { - self.io = Some(io); - } - - let rules = parse_rules( - raw.scope.rules.as_deref().unwrap_or_default(), - path, - &project_root, - source_index, - )?; - self.rules.extend(rules); - - Ok(()) - } - - fn finish(self) -> RecorderResult { - let default_exec = self.default_exec.ok_or_else(|| { - usage!( - ErrorCode::InvalidPolicyValue, - "composed filters never set 'scope.default_exec'" - ) - })?; - let default_value_action = self.default_value_action.ok_or_else(|| { - usage!( - ErrorCode::InvalidPolicyValue, - "composed filters never set 'scope.default_value_action'" - ) - })?; - - let io = self.io.unwrap_or_default(); - - Ok(TraceFilterConfig { - default_exec, - default_value_action, - io, - rules: self.rules, - sources: self.sources, - }) - } -} - -fn calculate_sha256(contents: &str) -> String { - let mut hasher = Sha256::new(); - hasher.update(contents.as_bytes()); - let digest = hasher.finalize(); - format!("{:x}", digest) -} - -fn detect_project_root(path: &Path) -> PathBuf { - let mut current = path.parent(); - while let Some(dir) = current { - if dir.file_name().and_then(|name| name.to_str()) == Some(".codetracer") { - return dir - .parent() - .map(Path::to_path_buf) - .unwrap_or_else(|| dir.to_path_buf()); - } - current = dir.parent(); - } - path.parent() - .map(Path::to_path_buf) - .unwrap_or_else(|| PathBuf::from(".")) -} - -fn parse_meta(raw: &RawMeta, path: &Path) -> RecorderResult { - if raw.name.trim().is_empty() { - return Err(usage!( - ErrorCode::InvalidPolicyValue, - "'meta.name' cannot be empty in '{}'", - path.display() - )); - } - if raw.version < 1 { - return Err(usage!( - ErrorCode::InvalidPolicyValue, - "'meta.version' must be >= 1 in '{}'", - path.display() - )); - } - - let mut labels = Vec::new(); - let mut seen = HashSet::new(); - for label in &raw.labels { - if seen.insert(label) { - labels.push(label.clone()); - } - } - - Ok(FilterMeta { - name: raw.name.clone(), - version: raw.version as u32, - description: raw.description.clone(), - labels, - }) -} - -struct ResolvedDefaults { - exec: Option, - value_action: Option, -} - -fn resolve_defaults( - scope: &RawScope, - path: &Path, - current_exec: Option, - current_value_action: Option, -) -> RecorderResult { - let exec = parse_default_exec(&scope.default_exec, path, current_exec)?; - let value_action = - parse_default_value_action(&scope.default_value_action, path, current_value_action)?; - Ok(ResolvedDefaults { exec, value_action }) -} - -fn parse_default_exec( - token: &str, - path: &Path, - current_exec: Option, -) -> RecorderResult> { - match token { - "inherit" => { - if current_exec.is_none() { - return Err(usage!( - ErrorCode::InvalidPolicyValue, - "'scope.default_exec' in '{}' cannot inherit without a previous filter", - path.display() - )); - } - Ok(None) - } - _ => ExecDirective::parse(token) - .ok_or_else(|| { - usage!( - ErrorCode::InvalidPolicyValue, - "unsupported value '{}' for 'scope.default_exec' in '{}'", - token, - path.display() - ) - }) - .map(Some), - } -} - -fn parse_default_value_action( - token: &str, - path: &Path, - current_value_action: Option, -) -> RecorderResult> { - match token { - "inherit" => { - if current_value_action.is_none() { - return Err(usage!( - ErrorCode::InvalidPolicyValue, - "'scope.default_value_action' in '{}' cannot inherit without a previous filter", - path.display() - )); - } - Ok(None) - } - _ => ValueAction::parse(token) - .ok_or_else(|| { - usage!( - ErrorCode::InvalidPolicyValue, - "unsupported value '{}' for 'scope.default_value_action' in '{}'", - token, - path.display() - ) - }) - .map(Some), - } -} - -fn parse_io(raw: Option<&RawIo>, path: &Path) -> RecorderResult> { - let Some(raw) = raw else { - return Ok(None); - }; - - let capture = raw.capture.unwrap_or(false); - let streams = match raw.streams.as_ref() { - Some(values) => { - let mut parsed = Vec::new(); - let mut seen = HashSet::new(); - for value in values { - let stream = IoStream::parse(value).ok_or_else(|| { - usage!( - ErrorCode::InvalidPolicyValue, - "unsupported IO stream '{}' in '{}'", - value, - path.display() - ) - })?; - if seen.insert(stream) { - parsed.push(stream); - } - } - parsed - } - None => Vec::new(), - }; - - if capture && streams.is_empty() { - return Err(usage!( - ErrorCode::InvalidPolicyValue, - "'io.streams' must be provided when 'io.capture = true' in '{}'", - path.display() - )); - } - if let Some(modes) = raw.modes.as_ref() { - if !modes.is_empty() { - return Err(usage!( - ErrorCode::InvalidPolicyValue, - "'io.modes' is reserved and must be empty in '{}'", - path.display() - )); - } - } - - Ok(Some(IoConfig { capture, streams })) -} - -fn parse_rules( - raw_rules: &[RawScopeRule], - path: &Path, - project_root: &Path, - source_id: usize, -) -> RecorderResult> { - let mut rules = Vec::new(); - for (idx, raw_rule) in raw_rules.iter().enumerate() { - let location = format!("{} scope.rules[{}]", path.display(), idx); - let selector = - Selector::parse(&raw_rule.selector, &SCOPE_SELECTOR_KINDS).map_err(|err| { - usage!( - ErrorCode::InvalidPolicyValue, - "invalid scope selector in {}: {}", - location, - err - ) - })?; - let selector = normalize_scope_selector(selector, project_root, &location)?; - - let exec = match raw_rule.exec.as_deref() { - None | Some("inherit") => None, - Some(value) => Some(ExecDirective::parse(value).ok_or_else(|| { - usage!( - ErrorCode::InvalidPolicyValue, - "unsupported value '{}' for 'exec' in {}", - value, - location - ) - })?), - }; - - let value_default = match raw_rule.value_default.as_deref() { - None | Some("inherit") => None, - Some(value) => Some(ValueAction::parse(value).ok_or_else(|| { - usage!( - ErrorCode::InvalidPolicyValue, - "unsupported value '{}' for 'value_default' in {}", - value, - location - ) - })?), - }; - - let mut value_patterns = Vec::new(); - if let Some(patterns) = raw_rule.value_patterns.as_ref() { - for (pidx, pattern) in patterns.iter().enumerate() { - let pattern_location = format!("{} value_patterns[{}]", location, pidx); - let selector = - Selector::parse(&pattern.selector, &VALUE_SELECTOR_KINDS).map_err(|err| { - usage!( - ErrorCode::InvalidPolicyValue, - "invalid value selector in {}: {}", - pattern_location, - err - ) - })?; - let action = ValueAction::parse(pattern.action.as_str()).ok_or_else(|| { - usage!( - ErrorCode::InvalidPolicyValue, - "unsupported value '{}' for 'action' in {}", - pattern.action, - pattern_location - ) - })?; - - value_patterns.push(ValuePattern { - selector, - action, - reason: pattern.reason.clone(), - source_id, - }); - } - } - - rules.push(ScopeRule { - selector, - exec, - value_default, - value_patterns, - reason: raw_rule.reason.clone(), - source_id, - }); - } - Ok(rules) -} - -fn normalize_scope_selector( - selector: Selector, - project_root: &Path, - location: &str, -) -> RecorderResult { - if selector.kind() != SelectorKind::File { - return Ok(selector); - } - - let normalized_pattern = normalize_file_pattern( - selector.pattern(), - selector.match_type(), - project_root, - location, - )?; - if normalized_pattern == selector.pattern() { - return Ok(selector); - } - - let raw = match selector.match_type() { - MatchType::Glob => format!("file:{}", normalized_pattern), - MatchType::Literal => format!("file:literal:{}", normalized_pattern), - MatchType::Regex => format!("file:regex:{}", normalized_pattern), - }; - Selector::parse(&raw, &SCOPE_SELECTOR_KINDS).map_err(|err| { - usage!( - ErrorCode::InvalidPolicyValue, - "failed to normalise file selector in {}: {}", - location, - err - ) - }) -} - -fn normalize_file_pattern( - pattern: &str, - match_type: MatchType, - project_root: &Path, - location: &str, -) -> RecorderResult { - match match_type { - MatchType::Literal => normalize_literal_path(pattern, project_root, location), - MatchType::Glob => normalize_glob_pattern(pattern, project_root), - MatchType::Regex => Ok(pattern.to_string()), - } -} - -fn normalize_literal_path( - pattern: &str, - project_root: &Path, - location: &str, -) -> RecorderResult { - let path = Path::new(pattern); - let relative = if path.is_absolute() { - path.strip_prefix(project_root) - .map_err(|_| { - usage!( - ErrorCode::InvalidPolicyValue, - "file selector '{}' in {} must reside within project root '{}'", - pattern, - location, - project_root.display() - ) - })? - .to_path_buf() - } else { - path.to_path_buf() - }; - - let normalized = normalize_components(&relative, pattern, location)?; - Ok(pathbuf_to_posix(&normalized)) -} - -fn normalize_components(path: &Path, raw: &str, location: &str) -> RecorderResult { - let mut normalised = PathBuf::new(); - for component in path.components() { - match component { - Component::Prefix(_) | Component::RootDir => continue, - Component::CurDir => {} - Component::ParentDir => { - if !normalised.pop() { - return Err(usage!( - ErrorCode::InvalidPolicyValue, - "file selector '{}' in {} escapes the project root", - raw, - location - )); - } - } - Component::Normal(part) => normalised.push(part), - } - } - Ok(normalised) -} - -fn normalize_glob_pattern(pattern: &str, project_root: &Path) -> RecorderResult { - let mut replaced = pattern.replace('\\', "/"); - while replaced.starts_with("./") { - replaced = replaced[2..].to_string(); - } - - let trimmed = replaced.trim_start_matches('/'); - let root = pathbuf_to_posix(project_root); - if root.is_empty() { - return Ok(trimmed.to_string()); - } - - let root_with_slash = format!("{}/", root); - if trimmed.starts_with(&root_with_slash) { - Ok(trimmed[root_with_slash.len()..].to_string()) - } else if trimmed == root { - Ok(String::new()) - } else { - Ok(trimmed.to_string()) - } -} - -fn pathbuf_to_posix(path: &Path) -> String { - let mut parts = Vec::new(); - for component in path.components() { - if let Component::Normal(part) = component { - parts.push(part.to_string_lossy()); - } - } - parts.join("/") -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct RawFilterFile { - meta: RawMeta, - #[serde(default)] - io: Option, - scope: RawScope, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct RawMeta { - name: String, - version: u32, - #[serde(default)] - description: Option, - #[serde(default)] - labels: Vec, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct RawIo { - #[serde(default)] - capture: Option, - #[serde(default)] - streams: Option>, - #[serde(default)] - modes: Option>, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct RawScope { - default_exec: String, - default_value_action: String, - #[serde(default)] - rules: Option>, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct RawScopeRule { - selector: String, - #[serde(default)] - exec: Option, - #[serde(default)] - value_default: Option, - #[serde(default)] - reason: Option, - #[serde(default)] - value_patterns: Option>, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct RawValuePattern { - selector: String, - action: String, - #[serde(default)] - reason: Option, -} - -const SCOPE_SELECTOR_KINDS: [SelectorKind; 3] = [ - SelectorKind::Package, - SelectorKind::File, - SelectorKind::Object, -]; - -const VALUE_SELECTOR_KINDS: [SelectorKind; 5] = [ - SelectorKind::Local, - SelectorKind::Global, - SelectorKind::Arg, - SelectorKind::Return, - SelectorKind::Attr, -]; - -#[cfg(test)] -mod tests { - use super::*; - use std::io::Write; - use std::path::PathBuf; - use tempfile::tempdir; - - #[test] - fn composes_filters_and_resolves_inheritance() -> RecorderResult<()> { - let temp = tempdir().expect("temp dir"); - let project_root = temp.path(); - let filters_dir = project_root.join(".codetracer"); - fs::create_dir(&filters_dir).unwrap(); - fs::create_dir_all(project_root.join("app")).unwrap(); - - let base_path = filters_dir.join("base.toml"); - write_filter( - &base_path, - r#" - [meta] - name = "base" - version = 1 - - [scope] - default_exec = "trace" - default_value_action = "redact" - - [[scope.rules]] - selector = "pkg:my_app.core.*" - exec = "trace" - value_default = "allow" - - [[scope.rules.value_patterns]] - selector = "local:literal:user" - action = "allow" - - [io] - capture = false - "#, - ); - - let overrides_path = filters_dir.join("overrides.toml"); - let literal_path = project_root - .join(".codetracer") - .join("..") - .join("app") - .join("__init__.py"); - let overrides = format!( - r#" - [meta] - name = "overrides" - version = 1 - - [scope] - default_exec = "inherit" - default_value_action = "inherit" - - [[scope.rules]] - selector = "file:literal:{literal}" - exec = "inherit" - value_default = "redact" - - [[scope.rules.value_patterns]] - selector = "arg:password" - action = "redact" - - [io] - capture = true - streams = ["stdout", "stderr"] - "#, - literal = literal_path.to_string_lossy() - ); - write_filter(&overrides_path, overrides.as_str()); - - let config = TraceFilterConfig::from_paths(&[base_path.clone(), overrides_path.clone()])?; - - assert_eq!(config.default_exec(), ExecDirective::Trace); - assert_eq!(config.default_value_action(), ValueAction::Redact); - assert_eq!(config.io().capture, true); - assert_eq!( - config.io().streams, - vec![IoStream::Stdout, IoStream::Stderr] - ); - - assert_eq!(config.rules().len(), 2); - let file_rule = &config.rules()[1]; - assert!(matches!(file_rule.exec, None)); - assert_eq!(file_rule.value_default, Some(ValueAction::Redact)); - assert_eq!(file_rule.value_patterns.len(), 1); - assert_eq!(file_rule.value_patterns[0].selector.raw(), "arg:password"); - assert_eq!( - file_rule.selector.pattern(), - "app/__init__.py", - "absolute literal path normalised relative to project root" - ); - - let summary = config.summary(); - assert_eq!(summary.entries.len(), 2); - assert_eq!(summary.entries[0].name, "base"); - assert_eq!(summary.entries[1].name, "overrides"); - - Ok(()) - } - - #[test] - fn from_inline_and_paths_parses_inline_only() -> RecorderResult<()> { - let inline_filter = r#" - [meta] - name = "inline" - version = 1 - - [scope] - default_exec = "trace" - default_value_action = "allow" - "#; - - let config = TraceFilterConfig::from_inline_and_paths(&[("inline", inline_filter)], &[])?; - - assert_eq!(config.default_exec(), ExecDirective::Trace); - assert_eq!(config.default_value_action(), ValueAction::Allow); - assert_eq!(config.rules().len(), 0); - let summary = config.summary(); - assert_eq!(summary.entries.len(), 1); - assert_eq!(summary.entries[0].name, "inline"); - assert_eq!(summary.entries[0].path, PathBuf::from("")); - Ok(()) - } - - #[test] - fn rejects_unknown_keys() { - let temp = tempdir().expect("temp dir"); - let project_root = temp.path(); - let filters_dir = project_root.join(".codetracer"); - fs::create_dir(&filters_dir).unwrap(); - let path = filters_dir.join("invalid.toml"); - write_filter( - &path, - r#" - [meta] - name = "invalid" - version = 1 - extra = "nope" - - [scope] - default_exec = "trace" - default_value_action = "redact" - "#, - ); - - let err = TraceFilterConfig::from_paths(&[path]).expect_err("expected failure"); - assert_eq!(err.code, ErrorCode::InvalidPolicyValue); - } - - #[test] - fn rejects_inherit_without_base() { - let temp = tempdir().expect("temp dir"); - let project_root = temp.path(); - let filters_dir = project_root.join(".codetracer"); - fs::create_dir(&filters_dir).unwrap(); - let path = filters_dir.join("empty.toml"); - write_filter( - &path, - r#" - [meta] - name = "empty" - version = 1 - - [scope] - default_exec = "inherit" - default_value_action = "inherit" - "#, - ); - - let err = TraceFilterConfig::from_paths(&[path]).expect_err("expected failure"); - assert_eq!(err.code, ErrorCode::InvalidPolicyValue); - } - - #[test] - fn rejects_invalid_stream_value() { - let temp = tempdir().expect("temp dir"); - let project_root = temp.path(); - let filters_dir = project_root.join(".codetracer"); - fs::create_dir(&filters_dir).unwrap(); - let path = filters_dir.join("io.toml"); - write_filter( - &path, - r#" - [meta] - name = "io" - version = 1 - - [scope] - default_exec = "trace" - default_value_action = "allow" - - [io] - capture = true - streams = ["stdout", "invalid"] - "#, - ); - - let err = TraceFilterConfig::from_paths(&[path]).expect_err("expected failure"); - assert_eq!(err.code, ErrorCode::InvalidPolicyValue); - } - - fn write_filter(path: &Path, contents: &str) { - let mut file = fs::File::create(path).unwrap(); - file.write_all(contents.trim_start().as_bytes()).unwrap(); - } } diff --git a/codetracer-python-recorder/src/trace_filter/loader.rs b/codetracer-python-recorder/src/trace_filter/loader.rs new file mode 100644 index 0000000..b568533 --- /dev/null +++ b/codetracer-python-recorder/src/trace_filter/loader.rs @@ -0,0 +1,581 @@ +//! Trace filter configuration loader (TOML ingestion, aggregation). + +use crate::trace_filter::model::{ + ExecDirective, FilterMeta, FilterSource, IoConfig, IoStream, ScopeRule, TraceFilterConfig, + ValueAction, ValuePattern, +}; +use crate::trace_filter::selector::{MatchType, Selector, SelectorKind}; +use recorder_errors::{usage, ErrorCode, RecorderResult}; +use serde::Deserialize; +use sha2::{Digest, Sha256}; +use std::collections::HashSet; +use std::fs; +use std::path::{Component, Path, PathBuf}; + +/// Helper aggregating inline and file sources into a resolved configuration. +#[derive(Default)] +pub struct ConfigAggregator { + default_exec: Option, + default_value_action: Option, + io: Option, + rules: Vec, + sources: Vec, +} + +impl ConfigAggregator { + /// Ingest a filter from the filesystem. + pub fn ingest_file(&mut self, path: &Path) -> RecorderResult<()> { + let contents = fs::read_to_string(path).map_err(|err| { + usage!( + ErrorCode::InvalidPolicyValue, + "failed to read trace filter '{}': {}", + path.display(), + err + ) + })?; + + self.ingest_source(path, &contents) + } + + /// Ingest an inline filter (used for builtin defaults). + pub fn ingest_inline(&mut self, label: &str, contents: &str) -> RecorderResult<()> { + let pseudo_path = PathBuf::from(format!("")); + self.ingest_source(&pseudo_path, contents) + } + + /// Finalise the aggregation, producing a resolved configuration. + pub fn finish(self) -> RecorderResult { + let default_exec = self.default_exec.ok_or_else(|| { + usage!( + ErrorCode::InvalidPolicyValue, + "composed filters never set 'scope.default_exec'" + ) + })?; + let default_value_action = self.default_value_action.ok_or_else(|| { + usage!( + ErrorCode::InvalidPolicyValue, + "composed filters never set 'scope.default_value_action'" + ) + })?; + + let io = self.io.unwrap_or_default(); + + Ok(TraceFilterConfig { + default_exec, + default_value_action, + io, + rules: self.rules, + sources: self.sources, + }) + } + + fn ingest_source(&mut self, path: &Path, contents: &str) -> RecorderResult<()> { + let checksum = calculate_sha256(contents); + let raw: RawFilterFile = toml::from_str(contents).map_err(|err| { + usage!( + ErrorCode::InvalidPolicyValue, + "failed to parse trace filter '{}': {}", + path.display(), + err + ) + })?; + + let project_root = detect_project_root(path); + let source_index = self.sources.len(); + self.sources.push(FilterSource { + path: path.to_path_buf(), + sha256: checksum, + project_root: project_root.clone(), + meta: parse_meta(&raw.meta, path)?, + }); + + let defaults = resolve_defaults( + &raw.scope, + path, + self.default_exec, + self.default_value_action, + )?; + if let Some(exec) = defaults.exec { + self.default_exec = Some(exec); + } + if let Some(value_action) = defaults.value_action { + self.default_value_action = Some(value_action); + } + + if let Some(io) = parse_io(raw.io.as_ref(), path)? { + self.io = Some(io); + } + + let rules = parse_rules( + raw.scope.rules.as_deref().unwrap_or_default(), + path, + &project_root, + source_index, + )?; + self.rules.extend(rules); + + Ok(()) + } +} + +pub(crate) fn calculate_sha256(contents: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(contents.as_bytes()); + let digest = hasher.finalize(); + format!("{:x}", digest) +} + +pub(crate) fn detect_project_root(path: &Path) -> PathBuf { + let mut current = path.parent(); + while let Some(dir) = current { + if dir.file_name().and_then(|name| name.to_str()) == Some(".codetracer") { + return dir + .parent() + .map(Path::to_path_buf) + .unwrap_or_else(|| dir.to_path_buf()); + } + current = dir.parent(); + } + path.parent() + .map(Path::to_path_buf) + .unwrap_or_else(|| PathBuf::from(".")) +} + +pub(crate) fn parse_meta(raw: &RawMeta, path: &Path) -> RecorderResult { + if raw.name.trim().is_empty() { + return Err(usage!( + ErrorCode::InvalidPolicyValue, + "'meta.name' must not be empty in '{}'", + path.display() + )); + } + + if raw.version < 1 { + return Err(usage!( + ErrorCode::InvalidPolicyValue, + "'meta.version' must be >= 1 in '{}'", + path.display() + )); + } + + let mut labels = Vec::new(); + let mut seen = HashSet::new(); + for label in &raw.labels { + if seen.insert(label) { + labels.push(label.clone()); + } + } + + Ok(FilterMeta { + name: raw.name.clone(), + version: raw.version as u32, + description: raw.description.clone(), + labels, + }) +} + +pub(crate) struct ResolvedDefaults { + pub exec: Option, + pub value_action: Option, +} + +pub(crate) fn resolve_defaults( + scope: &RawScope, + path: &Path, + current_exec: Option, + current_value_action: Option, +) -> RecorderResult { + let exec = parse_default_exec(&scope.default_exec, path, current_exec)?; + let value_action = + parse_default_value_action(&scope.default_value_action, path, current_value_action)?; + Ok(ResolvedDefaults { exec, value_action }) +} + +pub(crate) fn parse_default_exec( + token: &str, + path: &Path, + current_exec: Option, +) -> RecorderResult> { + match token { + "inherit" => { + if current_exec.is_none() { + return Err(usage!( + ErrorCode::InvalidPolicyValue, + "'scope.default_exec' in '{}' cannot inherit without a previous filter", + path.display() + )); + } + Ok(None) + } + _ => ExecDirective::parse(token) + .ok_or_else(|| { + usage!( + ErrorCode::InvalidPolicyValue, + "unsupported value '{}' for 'scope.default_exec' in '{}'", + token, + path.display() + ) + }) + .map(Some), + } +} + +pub(crate) fn parse_default_value_action( + token: &str, + path: &Path, + current_value_action: Option, +) -> RecorderResult> { + match token { + "inherit" => { + if current_value_action.is_none() { + return Err(usage!( + ErrorCode::InvalidPolicyValue, + "'scope.default_value_action' in '{}' cannot inherit without a previous filter", + path.display() + )); + } + Ok(None) + } + _ => ValueAction::parse(token) + .ok_or_else(|| { + usage!( + ErrorCode::InvalidPolicyValue, + "unsupported value '{}' for 'scope.default_value_action' in '{}'", + token, + path.display() + ) + }) + .map(Some), + } +} + +pub(crate) fn parse_io(raw: Option<&RawIo>, path: &Path) -> RecorderResult> { + let Some(raw) = raw else { + return Ok(None); + }; + + let capture = raw.capture.unwrap_or(false); + let streams = match raw.streams.as_ref() { + Some(values) => { + let mut parsed = Vec::new(); + let mut seen = HashSet::new(); + for value in values { + let stream = IoStream::parse(value).ok_or_else(|| { + usage!( + ErrorCode::InvalidPolicyValue, + "unsupported IO stream '{}' in '{}'", + value, + path.display() + ) + })?; + if seen.insert(stream) { + parsed.push(stream); + } + } + parsed + } + None => Vec::new(), + }; + + if capture && streams.is_empty() { + return Err(usage!( + ErrorCode::InvalidPolicyValue, + "'io.streams' must be provided when 'io.capture = true' in '{}'", + path.display() + )); + } + if let Some(modes) = raw.modes.as_ref() { + if !modes.is_empty() { + return Err(usage!( + ErrorCode::InvalidPolicyValue, + "'io.modes' is reserved and must be empty in '{}'", + path.display() + )); + } + } + + Ok(Some(IoConfig { capture, streams })) +} + +pub(crate) fn parse_rules( + raw_rules: &[RawScopeRule], + path: &Path, + project_root: &Path, + source_id: usize, +) -> RecorderResult> { + let mut rules = Vec::new(); + for (idx, raw_rule) in raw_rules.iter().enumerate() { + let location = format!("{} scope.rules[{}]", path.display(), idx); + let selector = + Selector::parse(&raw_rule.selector, &SCOPE_SELECTOR_KINDS).map_err(|err| { + usage!( + ErrorCode::InvalidPolicyValue, + "invalid scope selector in {}: {}", + location, + err + ) + })?; + let selector = normalize_scope_selector(selector, project_root, &location)?; + + let exec = match raw_rule.exec.as_deref() { + None | Some("inherit") => None, + Some(value) => Some(ExecDirective::parse(value).ok_or_else(|| { + usage!( + ErrorCode::InvalidPolicyValue, + "unsupported value '{}' for 'exec' in {}", + value, + location + ) + })?), + }; + + let value_default = match raw_rule.value_default.as_deref() { + None | Some("inherit") => None, + Some(value) => Some(ValueAction::parse(value).ok_or_else(|| { + usage!( + ErrorCode::InvalidPolicyValue, + "unsupported value '{}' for 'value_default' in {}", + value, + location + ) + })?), + }; + + let mut value_patterns = Vec::new(); + if let Some(patterns) = raw_rule.value_patterns.as_ref() { + for (pidx, pattern) in patterns.iter().enumerate() { + let pattern_location = format!("{} value_patterns[{}]", location, pidx); + let selector = + Selector::parse(&pattern.selector, &VALUE_SELECTOR_KINDS).map_err(|err| { + usage!( + ErrorCode::InvalidPolicyValue, + "invalid value selector in {}: {}", + pattern_location, + err + ) + })?; + + let action = ValueAction::parse(&pattern.action).ok_or_else(|| { + usage!( + ErrorCode::InvalidPolicyValue, + "unsupported value '{}' for 'action' in {}", + pattern.action, + pattern_location + ) + })?; + + value_patterns.push(ValuePattern { + selector, + action, + reason: pattern.reason.clone(), + source_id, + }); + } + } + + rules.push(ScopeRule { + selector, + exec, + value_default, + value_patterns, + reason: raw_rule.reason.clone(), + source_id, + }); + } + Ok(rules) +} + +pub(crate) fn normalize_scope_selector( + selector: Selector, + project_root: &Path, + location: &str, +) -> RecorderResult { + match selector.kind() { + SelectorKind::File => { + let pattern = selector.pattern(); + if pattern.starts_with("glob:") { + let glob_pattern = &pattern["glob:".len()..]; + let normalized = normalize_glob_pattern(glob_pattern, project_root)?; + rebuild_selector(selector.kind(), selector.match_type(), &normalized) + } else { + let path = Path::new(pattern); + let normalized = normalize_file_selector(path, project_root, pattern, location)?; + rebuild_selector(selector.kind(), selector.match_type(), &normalized) + } + } + _ => Ok(selector), + } +} + +pub(crate) fn normalize_file_selector( + path: &Path, + project_root: &Path, + pattern: &str, + location: &str, +) -> RecorderResult { + let path = if path.is_absolute() { + path.strip_prefix(project_root) + .map_err(|_| { + usage!( + ErrorCode::InvalidPolicyValue, + "file selector '{}' in {} must reside within project root '{}'", + pattern, + location, + project_root.display() + ) + })? + .to_path_buf() + } else { + path.to_path_buf() + }; + + let normalized = normalize_components(&path, pattern, location)?; + Ok(pathbuf_to_posix(&normalized)) +} + +pub(crate) fn normalize_components( + path: &Path, + raw: &str, + location: &str, +) -> RecorderResult { + let mut normalised = PathBuf::new(); + for component in path.components() { + match component { + Component::Prefix(_) | Component::RootDir => continue, + Component::CurDir => {} + Component::ParentDir => { + if !normalised.pop() { + return Err(usage!( + ErrorCode::InvalidPolicyValue, + "file selector '{}' in {} escapes the project root", + raw, + location + )); + } + } + Component::Normal(part) => normalised.push(part), + } + } + Ok(normalised) +} + +pub(crate) fn normalize_glob_pattern(pattern: &str, project_root: &Path) -> RecorderResult { + let mut replaced = pattern.replace('\\', "/"); + while replaced.starts_with("./") { + replaced = replaced[2..].to_string(); + } + + let trimmed = replaced.trim_start_matches('/'); + let root = pathbuf_to_posix(project_root); + if root.is_empty() { + return Ok(trimmed.to_string()); + } + + let root_with_slash = format!("{}/", root); + if trimmed.starts_with(&root_with_slash) { + Ok(trimmed[root_with_slash.len()..].to_string()) + } else if trimmed == root { + Ok(String::new()) + } else { + Ok(trimmed.to_string()) + } +} + +pub(crate) fn pathbuf_to_posix(path: &Path) -> String { + let mut parts = Vec::new(); + for component in path.components() { + if let Component::Normal(part) = component { + parts.push(part.to_string_lossy()); + } + } + parts.join("/") +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RawFilterFile { + pub meta: RawMeta, + #[serde(default)] + pub io: Option, + pub scope: RawScope, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RawMeta { + pub name: String, + pub version: u32, + #[serde(default)] + pub description: Option, + #[serde(default)] + pub labels: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RawIo { + #[serde(default)] + pub capture: Option, + #[serde(default)] + pub streams: Option>, + #[serde(default)] + pub modes: Option>, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RawScope { + pub default_exec: String, + pub default_value_action: String, + #[serde(default)] + pub rules: Option>, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RawScopeRule { + pub selector: String, + #[serde(default)] + pub exec: Option, + #[serde(default)] + pub value_default: Option, + #[serde(default)] + pub reason: Option, + #[serde(default)] + pub value_patterns: Option>, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RawValuePattern { + pub selector: String, + pub action: String, + #[serde(default)] + pub reason: Option, +} + +const SCOPE_SELECTOR_KINDS: [SelectorKind; 3] = [ + SelectorKind::Package, + SelectorKind::File, + SelectorKind::Object, +]; +const VALUE_SELECTOR_KINDS: [SelectorKind; 5] = [ + SelectorKind::Local, + SelectorKind::Global, + SelectorKind::Arg, + SelectorKind::Return, + SelectorKind::Attr, +]; + +fn rebuild_selector( + kind: SelectorKind, + match_type: MatchType, + pattern: &str, +) -> RecorderResult { + let raw = match match_type { + MatchType::Glob => format!("{}:{}", kind.token(), pattern), + MatchType::Regex => format!("{}:regex:{}", kind.token(), pattern), + MatchType::Literal => format!("{}:literal:{}", kind.token(), pattern), + }; + Selector::parse(&raw, &[kind]) +} diff --git a/codetracer-python-recorder/src/trace_filter/mod.rs b/codetracer-python-recorder/src/trace_filter/mod.rs index 15af3fd..40d168c 100644 --- a/codetracer-python-recorder/src/trace_filter/mod.rs +++ b/codetracer-python-recorder/src/trace_filter/mod.rs @@ -2,4 +2,7 @@ pub mod config; pub mod engine; +pub mod loader; +pub mod model; pub mod selector; +pub mod summary; diff --git a/codetracer-python-recorder/src/trace_filter/model.rs b/codetracer-python-recorder/src/trace_filter/model.rs new file mode 100644 index 0000000..c685f15 --- /dev/null +++ b/codetracer-python-recorder/src/trace_filter/model.rs @@ -0,0 +1,175 @@ +//! Trace filter data models (directives, rules, summaries). + +use crate::trace_filter::selector::Selector; +use crate::trace_filter::summary; +use std::path::PathBuf; + +/// Scope-level execution directive. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ExecDirective { + Trace, + Skip, +} + +impl ExecDirective { + pub(crate) fn parse(token: &str) -> Option { + match token { + "trace" => Some(ExecDirective::Trace), + "skip" => Some(ExecDirective::Skip), + _ => None, + } + } +} + +/// Value-level capture directive. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ValueAction { + Allow, + Redact, + Drop, +} + +impl ValueAction { + pub(crate) fn parse(token: &str) -> Option { + match token { + "allow" => Some(ValueAction::Allow), + "redact" => Some(ValueAction::Redact), + "drop" => Some(ValueAction::Drop), + // Backwards compatibility for deprecated `deny`. + "deny" => Some(ValueAction::Redact), + _ => None, + } + } +} + +/// IO streams that can be captured in addition to scope/value rules. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum IoStream { + Stdout, + Stderr, + Stdin, + Files, +} + +impl IoStream { + pub(crate) fn parse(token: &str) -> Option { + match token { + "stdout" => Some(IoStream::Stdout), + "stderr" => Some(IoStream::Stderr), + "stdin" => Some(IoStream::Stdin), + "files" => Some(IoStream::Files), + _ => None, + } + } +} + +/// Metadata describing the source filter file. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FilterMeta { + pub name: String, + pub version: u32, + pub description: Option, + pub labels: Vec, +} + +/// IO capture configuration. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IoConfig { + pub capture: bool, + pub streams: Vec, +} + +impl Default for IoConfig { + fn default() -> Self { + IoConfig { + capture: false, + streams: Vec::new(), + } + } +} + +/// Value pattern applied within a scope rule. +#[derive(Debug, Clone)] +pub struct ValuePattern { + pub selector: Selector, + pub action: ValueAction, + pub reason: Option, + pub source_id: usize, +} + +/// Scope rule constructed from the flattened configuration chain. +#[derive(Debug, Clone)] +pub struct ScopeRule { + pub selector: Selector, + pub exec: Option, + pub value_default: Option, + pub value_patterns: Vec, + pub reason: Option, + pub source_id: usize, +} + +/// Source information for each filter file participating in the chain. +#[derive(Debug, Clone)] +pub struct FilterSource { + pub path: PathBuf, + pub sha256: String, + pub project_root: PathBuf, + pub meta: FilterMeta, +} + +/// Summary used for embedding in trace metadata. +#[derive(Debug, Clone)] +pub struct FilterSummary { + pub entries: Vec, +} + +/// Single entry in the filter summary. +#[derive(Debug, Clone)] +pub struct FilterSummaryEntry { + pub path: PathBuf, + pub sha256: String, + pub name: String, + pub version: u32, +} + +/// Fully resolved filter configuration ready for runtime consumption. +#[derive(Debug, Clone)] +pub struct TraceFilterConfig { + pub(crate) default_exec: ExecDirective, + pub(crate) default_value_action: ValueAction, + pub(crate) io: IoConfig, + pub(crate) rules: Vec, + pub(crate) sources: Vec, +} + +impl TraceFilterConfig { + /// Default execution directive applied before scope rules run. + pub fn default_exec(&self) -> ExecDirective { + self.default_exec + } + + /// Default value action applied before rule-specific overrides. + pub fn default_value_action(&self) -> ValueAction { + self.default_value_action + } + + /// IO capture configuration associated with the composed filter chain. + pub fn io(&self) -> &IoConfig { + &self.io + } + + /// Flattened scope rules in execution order. + pub fn rules(&self) -> &[ScopeRule] { + &self.rules + } + + /// Source filter metadata used for embedding in trace output. + pub fn sources(&self) -> &[FilterSource] { + &self.sources + } + + /// Helper producing a summary used by metadata writers. + pub fn summary(&self) -> FilterSummary { + summary::build_summary(&self.sources) + } +} diff --git a/codetracer-python-recorder/src/trace_filter/summary.rs b/codetracer-python-recorder/src/trace_filter/summary.rs new file mode 100644 index 0000000..bf57885 --- /dev/null +++ b/codetracer-python-recorder/src/trace_filter/summary.rs @@ -0,0 +1,17 @@ +//! Trace filter summaries used for metadata embedding. + +use crate::trace_filter::model::{FilterSource, FilterSummary, FilterSummaryEntry}; + +/// Build a summary object from the resolved filter sources list. +pub fn build_summary(sources: &[FilterSource]) -> FilterSummary { + let entries = sources + .iter() + .map(|source| FilterSummaryEntry { + path: source.path.clone(), + sha256: source.sha256.clone(), + name: source.meta.name.clone(), + version: source.meta.version, + }) + .collect(); + FilterSummary { entries } +} diff --git a/codetracer-python-recorder/tests/python/perf/test_trace_filter_perf.py b/codetracer-python-recorder/tests/python/perf/test_trace_filter_perf.py index a81fc2e..91c61c6 100644 --- a/codetracer-python-recorder/tests/python/perf/test_trace_filter_perf.py +++ b/codetracer-python-recorder/tests/python/perf/test_trace_filter_perf.py @@ -199,7 +199,7 @@ def test_trace_filter_perf_smoke(tmp_path: Path) -> None: assert glob.duration_seconds > 0 assert regex.duration_seconds > 0 - assert baseline.filter_names == ["bench-baseline"] + assert baseline.filter_names == ["builtin-default", "bench-baseline"] assert "bench-glob" in glob.filter_names assert "bench-regex" in regex.filter_names diff --git a/design-docs/adr/0001-file-level-single-responsibility.md b/design-docs/adr/0001-file-level-single-responsibility.md index 9252118..df45cbe 100644 --- a/design-docs/adr/0001-file-level-single-responsibility.md +++ b/design-docs/adr/0001-file-level-single-responsibility.md @@ -9,7 +9,7 @@ The codetracer Python recorder crate has evolved quickly and several source files now mix unrelated concerns: - [`src/lib.rs`](../../codetracer-python-recorder/src/lib.rs) hosts PyO3 module wiring, global logging setup, tracing session state, and filesystem validation in one place. -- [`src/runtime_tracer.rs`](../../codetracer-python-recorder/src/runtime_tracer.rs) interleaves activation gating, writer lifecycle control, PyFrame helpers, and Python value encoding logic, making it challenging to test or extend any portion independently. +- [`src/runtime/tracer/runtime_tracer.rs`](../../codetracer-python-recorder/src/runtime/tracer/runtime_tracer.rs) (formerly `src/runtime_tracer.rs`) interleaves activation gating, writer lifecycle control, PyFrame helpers, and Python value encoding logic, making it challenging to test or extend any portion independently. - [`src/tracer.rs`](../../codetracer-python-recorder/src/tracer.rs) combines sys.monitoring shim code with the `Tracer` trait, callback registration, and global caches. - [`codetracer_python_recorder/api.py`](../../codetracer-python-recorder/codetracer_python_recorder/api.py) mixes format constants, backend interaction, context manager ergonomics, and environment based auto-start side effects. @@ -43,7 +43,7 @@ These changes are mechanical reorganisations—no behavioural changes are expect 2. **Preserve APIs.** When moving functions, re-export them from their new module so that existing callers (Rust and Python) compile without modification in the same PR. 3. **Add Focused Tests.** Whenever a helper is extracted (e.g., value encoding), add or migrate unit tests that cover its edge cases. 4. **Document Moves.** Update doc comments and module-level docs to reflect the new structure. Remove outdated TODOs or convert them into follow-up issues. -5. **Coordinate on Shared Types.** When splitting `runtime_tracer.rs`, agree on ownership for shared structs (e.g., `RuntimeTracer` remains in `runtime/mod.rs`). Use `pub(crate)` to keep internals encapsulated. +5. **Coordinate on Shared Types.** When evolving the `runtime::tracer` modules, agree on ownership for shared structs (e.g., `RuntimeTracer` remains re-exported from `runtime/mod.rs`). Use `pub(crate)` to keep internals encapsulated. 6. **Python Imports.** After splitting the Python modules, ensure `__all__` in `__init__.py` continues to export the public API. Use relative imports to avoid accidental circular dependencies. 7. **Parallel Work.** Follow the sequencing from `design-docs/file-level-srp-refactor-plan.md` to know when tasks can proceed in parallel. diff --git a/design-docs/adr/0011-codetracer-architecture-refactor.md b/design-docs/adr/0011-codetracer-architecture-refactor.md new file mode 100644 index 0000000..238fdc8 --- /dev/null +++ b/design-docs/adr/0011-codetracer-architecture-refactor.md @@ -0,0 +1,72 @@ +# ADR 0011: Codetracer Python Recorder Architecture Refactor + +- **Status:** Accepted +- **Date:** 2025-02-14 +- **Deciders:** codetracer recorder maintainers +- **Consulted:** DX tooling crew, Runtime tracing stakeholders +- **Informed:** Replay consumers, Support engineering + +## Context +- `RuntimeTracer` (`codetracer-python-recorder/src/runtime/mod.rs`) has grown into a god object: it wires monitoring callbacks, trace file lifecycle, IO draining, policy enforcement, telemetry, and trace-filter integration inside a single 2 600+ line module. +- `trace_filter` files (`src/trace_filter/config.rs`, `engine.rs`) mix data modelling, on-disk parsing, default resolution, runtime caching, and PyO3-facing summaries, making it hard to isolate changes or test components individually. +- Monitoring plumbing (`src/monitoring/tracer.rs`) duplicates `sys.monitoring` registration boilerplate across >14 callback wrappers and stores the global tracer instance alongside policy decisions, reducing cohesion. +- Policy and diagnostics code (`src/policy.rs`, `src/logging.rs`) couple configuration models, env parsing, PyO3 bindings, metrics, file IO, and error-trailer logic in single modules. +- Python glue (`codetracer_python_recorder/cli.py`, `codetracer_python_recorder/session.py`) pulls details from the monolithic Rust modules, limiting our ability to present slimmer APIs or reuse bootstrapping logic elsewhere. +- The team wants stricter adherence to the single-responsibility principle and lower coupling so future features (e.g., new policy toggles, additional monitoring events, alternative telemetry sinks) can be added with minimal risk. + +## Problem +Large, multi-purpose modules make the recorder difficult to extend and review. Specific issues include: +- Testing isolated behaviours (e.g., trace-filter IO errors, policy inheritance) requires instantiating heavyweight structs because responsibilities are intertwined. +- Introducing new tracing behaviours often touches unrelated code, increasing the chance of regression (e.g., editing `RuntimeTracer` for filter tweaks while interfering with IO teardown). +- Reusing infrastructure (policy parsing, bootstrap metadata, logging) in other crates or integration tests is impractical because functionality is not encapsulated. +- The current code layout obscures high-level architecture, slowing onboarding and complicating code ownership boundaries. + +## Decision +We will modularise the recorder around cohesive responsibilities while preserving existing external APIs and without re-touching the recently refactored IO capture pipeline (`src/runtime/io_capture/`). + +1. **Trace Filter Layering** + - Extract configuration models, parsing/aggregation, and runtime compilation into distinct submodules (e.g., `trace_filter::model`, `::loader`, `::engine`, `::summary`). + - Keep file IO and TOML parsing contained in loader modules, letting runtime components depend only on pure data structures. + - Preserve the current public API (`TraceFilterConfig`, `TraceFilterEngine`) via a facade module to avoid churn for callers. + +2. **Policy & Diagnostics Separation** + - Split policy data structures from environment parsing and PyO3 bindings, yielding `policy::model`, `policy::env`, and `policy::ffi`. + - Partition logging into `logging::logger` (FilterSpec parsing, writer management), `logging::metrics`, and `logging::trailer`, with a top-level facade that applies policies. + - Ensure policy updates flow through a narrow interface consumed by both Rust and Python entry points. + +3. **Session Bootstrap Decomposition** + - Break `TraceSessionBootstrap` helpers into filesystem preparation, metadata capture, and filter loading modules. + - Provide a lightweight bootstrap service consumed by both Rust (`start_tracing`) and Python CLI/session wrappers, improving reuse and testability. + +4. **Monitoring Callback Plumbing** + - Move the `Tracer` trait and its helpers into dedicated modules (`monitoring::api`, `monitoring::install`). + - Replace duplicated callback registration functions with table-driven or macro-generated wrappers while keeping `install_tracer`, `uninstall_tracer`, and `flush_installed_tracer` signatures intact. + +5. **Runtime Tracer Orchestration** + - Factor `RuntimeTracer` responsibilities into focused collaborators handling lifecycle management, event handling, filter caching, and IO coordination. + - Maintain behavioural equivalence (activation gating, telemetry, failure injection) but reduce per-function responsibilities and clarify dependencies via constructor injection. + +## Consequences +- **Benefits:** + - Easier to reason about components and review changes; smaller files have clearer ownership and targeted unit tests. + - Reduced coupling between layers unlocks future features (e.g., alternative log sinks, additional monitoring events) without large-scale edits. + - Python-facing APIs rely on slimmer Rust facades, improving maintainability for CLI and embedding scenarios. +- **Costs:** + - Significant module churn requires careful coordination of imports, visibility, and re-exports. + - Temporary refactor scaffolding increases short-term complexity; we must keep commits small and well-tested. + - Documentation and developer onboarding material must be updated to reflect the new layout. +- **Risks:** + - Behavioural regressions if lifecycle or policy logic is incorrectly reassembled—mitigated by incremental changes with exhaustive unit/integration tests. + - Merge conflicts with parallel workstreams touching large files; we will stage the refactor and communicate timelines. + - Potential performance regressions if new abstractions add indirection; we will benchmark hot paths after each milestone. + +## Alternatives +- **Targeted cleanups only:** Fixing individual hotspots without broader modularisation would leave the overarching coupling unaddressed and perpetuate inconsistent boundaries. +- **Full rewrite:** Starting from scratch would be risky for existing integrations and offers little incremental value compared to methodical refactoring. + +## Rollout +1. Land the modularisation in staged PRs following the implementation plan, keeping behavioural changes isolated per milestone. +2. Maintain compatibility with current Python APIs and crate exports; adjust import paths gradually with deprecation windows if needed. +3. Update architectural documentation and developer guides once core milestones complete. +4. **Status snapshot (2025‑03‑12):** Milestones 1–6 are complete—the runtime tracer collaborators, monitoring plumbing, and integration cleanups all landed with `just test` (nextest + pytest) coverage, Python benchmark parity, and documentation updates. +5. Confirmed behaviour parity and flipped the ADR to **Accepted**; any follow-up regressions will be handled via normal maintenance triage. diff --git a/design-docs/codetracer-architecture-refactor-implementation-plan.md b/design-docs/codetracer-architecture-refactor-implementation-plan.md new file mode 100644 index 0000000..a03ae30 --- /dev/null +++ b/design-docs/codetracer-architecture-refactor-implementation-plan.md @@ -0,0 +1,105 @@ +# Codetracer Python Recorder Architecture Refactor – Implementation Plan + +## Overview +We will refactor the `codetracer-python-recorder` crate to reinforce single-responsibility boundaries and reduce coupling among runtime tracing, policy, monitoring, and diagnostics layers. The work follows ADR 0011 and deliberately excludes the recently refactored IO capture pipeline (`src/runtime/io_capture/`). + +## Goals +- Ensure large modules (`runtime/mod.rs`, `trace_filter/config.rs`, `trace_filter/engine.rs`, `monitoring/tracer.rs`, `logging.rs`, `policy.rs`, `session/bootstrap.rs`) each own a focused concern with cohesive helpers. +- Preserve existing public APIs (Rust crate exports and Python bindings) while internally re-organising responsibilities. +- Enable targeted unit testing by isolating IO, parsing, caching, and lifecycle logic. +- Maintain runtime performance and behaviour (activation gating, telemetry, failure injection, trace filtering). + +## Concept-to-file Mapping + +| Concept | Current location(s) | Target location(s) | +| --- | --- | --- | +| Trace filter configuration models & defaults | `codetracer-python-recorder/src/trace_filter/config.rs` | `codetracer-python-recorder/src/trace_filter/model.rs`, `.../loader.rs` | +| Trace filter file IO & aggregation | `codetracer-python-recorder/src/trace_filter/config.rs` | `codetracer-python-recorder/src/trace_filter/loader.rs` | +| Trace filter summaries for metadata | `codetracer-python-recorder/src/trace_filter/config.rs` | `codetracer-python-recorder/src/trace_filter/summary.rs` | +| Trace filter runtime engine & cache | `codetracer-python-recorder/src/trace_filter/engine.rs` | `codetracer-python-recorder/src/trace_filter/engine/mod.rs`, `.../engine/resolution.rs` | +| Policy data model & updates | `codetracer-python-recorder/src/policy.rs` | `codetracer-python-recorder/src/policy/model.rs` | +| Policy environment parsing | `codetracer-python-recorder/src/policy.rs` | `codetracer-python-recorder/src/policy/env.rs` | +| Policy PyO3 bindings | `codetracer-python-recorder/src/policy.rs` | `codetracer-python-recorder/src/policy/ffi.rs` | +| Logging: logger, filter specs, destinations | `codetracer-python-recorder/src/logging.rs` | `codetracer-python-recorder/src/logging/logger.rs` | +| Logging: metrics sink | `codetracer-python-recorder/src/logging.rs` | `codetracer-python-recorder/src/logging/metrics.rs` | +| Logging: error trailer emission | `codetracer-python-recorder/src/logging.rs` | `codetracer-python-recorder/src/logging/trailer.rs` | +| Session bootstrap filesystem prep | `codetracer-python-recorder/src/session/bootstrap.rs` | `codetracer-python-recorder/src/session/bootstrap/filesystem.rs` | +| Session bootstrap metadata capture | `codetracer-python-recorder/src/session/bootstrap.rs` | `codetracer-python-recorder/src/session/bootstrap/metadata.rs` | +| Session bootstrap filter loading | `codetracer-python-recorder/src/session/bootstrap.rs` | `codetracer-python-recorder/src/session/bootstrap/filters.rs` | +| Monitoring tracer trait & types | `codetracer-python-recorder/src/monitoring/tracer.rs` | `codetracer-python-recorder/src/monitoring/api.rs` | +| Monitoring install/uninstall plumbing | `codetracer-python-recorder/src/monitoring/tracer.rs` | `codetracer-python-recorder/src/monitoring/install.rs`, `.../callbacks.rs` | +| Runtime tracer lifecycle management | `codetracer-python-recorder/src/runtime/mod.rs` | `codetracer-python-recorder/src/runtime/tracer/lifecycle.rs` | +| Runtime tracer event handlers | `codetracer-python-recorder/src/runtime/mod.rs` | `codetracer-python-recorder/src/runtime/tracer/events.rs` | +| Runtime tracer IO coordination | `codetracer-python-recorder/src/runtime/mod.rs` | `codetracer-python-recorder/src/runtime/tracer/io.rs` | +| Runtime tracer filter cache & policy integration | `codetracer-python-recorder/src/runtime/mod.rs` | `codetracer-python-recorder/src/runtime/tracer/filtering.rs` | +| Python session orchestration | `codetracer-python-recorder/codetracer_python_recorder/session.py` | `codetracer-python-recorder/codetracer_python_recorder/session.py` (imports updated to new Rust facades) | +| Python CLI argument resolution | `codetracer-python-recorder/codetracer_python_recorder/cli.py` | `codetracer-python-recorder/codetracer_python_recorder/cli.py` (uses refactored bootstrap/service APIs) | + +## Scope +- Rust crate `codetracer-python-recorder`, excluding `src/runtime/io_capture/`. +- Python package glue (`codetracer_python_recorder/cli.py`, `codetracer_python_recorder/session.py`) only to the extent necessary to align imports with new Rust module facades. +- Existing unit/integration tests; add coverage as required by new abstractions. + +## Non-Goals +- Functional changes to tracing behaviour, policy semantics, or trace output formats. +- Revisiting IO capture mechanics or Python auto-start logic beyond import adjustments. +- Altering external CLI or Python API signatures (behavioural parity is mandatory). + +## Milestones + +### 1. Trace Filter Decomposition +- Introduce submodules: `trace_filter::model` (directives, value actions), `trace_filter::loader` (TOML parsing, source aggregation), `trace_filter::summary`, and move cache-independent helpers out of `engine.rs`. +- Refactor `TraceFilterEngine` to depend on compiled rule structs imported from new modules; keep resolver cache logic local. +- Update callers (`session/bootstrap.rs`, `runtime/mod.rs`) to use the facade module. +- Extend/adjust unit tests covering filter parsing and resolution. +- Run `just test`. + +### 2. Policy and Logging Separation +- Split `policy.rs` into `policy::model` (data structures, in-memory updates), `policy::env` (environment parsing), and `policy::ffi` (PyO3 functions). Create a top-level `policy/mod.rs` facade exporting current names. +- Extract logging responsibilities into `logging::logger` (FilterSpec, destination management), `logging::metrics`, `logging::trailer`, with a facade orchestrating policy application and structured logging helpers. +- Update call sites (`RuntimeTracer::finish`, `session::start_tracing`, tests) to use new facades. +- Refresh policy/logging unit tests; add coverage for failure cases in new modules. +- Run `just test`. + +### 3. Session Bootstrap Refactor +- Break `TraceSessionBootstrap` into submodules (`bootstrap::filesystem`, `bootstrap::metadata`, `bootstrap::filters`) maintaining a thin orchestrator struct. +- Provide dedicated unit tests for each submodule (e.g., metadata extraction, filter discovery). +- Update Python wrappers (`session.py`, `cli.py`) if import statements change; ensure behaviour remains identical. +- Verify `just test` and targeted CLI smoke tests (`just run` scenario if available). + +### 4. Monitoring Plumbing Cleanup +- Move the `Tracer` trait definition into `monitoring::api`; encapsulate global install/uninstall state in `monitoring::install`. +- Generate callback registration via a declarative table or macro to replace the duplicated functions in `monitoring/tracer.rs`. +- Ensure the public functions `install_tracer`, `uninstall_tracer`, and `flush_installed_tracer` remain accessible from `crate::monitoring`. +- Update or add tests validating callback dispatch and disable-on-error behaviour. +- Run `just test`. + +### 5. Runtime Tracer Modularisation +- Introduce collaborators for lifecycle management (trace file setup, teardown), event handling (py_start/line/return), filter cache lookup, and IO coordination. +- Refactor `RuntimeTracer` to compose these collaborators, keeping state injection explicit and eliminating unrelated helper functions from the main impl. +- Ensure failure injection hooks, telemetry counters, activation gating, and existing public methods (`begin`, `install_io_capture`, `flush`, `finish`) behave identically. +- Update `src/runtime/mod.rs` unit tests and add coverage for new components. +- Re-run `just test` plus targeted integration tests if available. + +### 6. Integration and Cleanup +- Harmonise module exports, update documentation comments referencing moved code, and ensure Python packaging metadata/build scripts still resolve module paths. +- Review for dead imports or obsolete helpers left behind after splits. +- Run the full test suite (`just test`), optionally `cargo fmt`/`cargo clippy` if part of CI requirements. +- Prepare follow-up documentation updates or status reports; confirm with stakeholders that milestones meet ADR intent. + +## Testing Strategy +- Incrementally adjust existing unit tests to target new modules. +- Add focused tests where previously impossible (e.g., loader-only tests without touching runtime). +- Maintain or enhance integration tests covering start/stop tracing flows. +- Execute `just test` after each milestone; add performance smoke benchmarks if regressions are suspected. + +## Risks & Mitigations +- **Regression risk:** Break tracing lifecycle when splitting modules. Mitigate with exhaustive unit tests and incremental commits. +- **Merge conflicts:** Large file churn may collide with parallel work. Communicate schedule early, stage PRs sequentially, and land high-churn files first. +- **Performance impact:** Additional abstraction layers could add overhead. Benchmark hot paths after Milestones 4 and 5; profile if slowdowns exceed 5 %. +- **Doc drift:** Architectural docs may become outdated. Schedule documentation updates during Milestone 6. + +## Rollout & Sign-off +- Track milestones via status files (e.g., `codetracer-architecture-refactor-implementation-plan.status.md`). +- Flip ADR 0011 to **Accepted** once Milestone 6 completes and maintainers confirm no behavioural regressions. +- Announce completion to stakeholders (runtime tracing users, DX tooling) and note any follow-up cleanups. diff --git a/design-docs/codetracer-architecture-refactor-implementation-plan.status.md b/design-docs/codetracer-architecture-refactor-implementation-plan.status.md new file mode 100644 index 0000000..646484e --- /dev/null +++ b/design-docs/codetracer-architecture-refactor-implementation-plan.status.md @@ -0,0 +1,122 @@ +# Codetracer Architecture Refactor – Status + +## Task Summary +- **Objective:** Execute ADR 0011 by modularising `codetracer-python-recorder`, starting with Milestone 1 (Trace Filter Decomposition) to restore single-responsibility boundaries and reduce coupling. + +## Relevant Design Docs +- `design-docs/adr/0011-codetracer-architecture-refactor.md` +- `design-docs/codetracer-architecture-refactor-implementation-plan.md` + +## Key Source Files (Milestone 1 Focus) +- `codetracer-python-recorder/src/trace_filter/config.rs` +- `codetracer-python-recorder/src/trace_filter/engine.rs` +- `codetracer-python-recorder/src/session/bootstrap.rs` +- `codetracer-python-recorder/src/runtime/mod.rs` +- Associated `trace_filter` unit tests under `codetracer-python-recorder/src/trace_filter/` + +## Progress Log +- ✅ Captured architectural intent in ADR 0011 and drafted the implementation plan with milestones and concept-to-file mapping. +- ✅ Logged this status tracker to maintain continuity across milestones. +- ✅ Milestone 1 Kickoff: catalogued existing `trace_filter` responsibilities and outlined target submodules (`model`, `loader`, `summary`, `engine` helpers). + - `trace_filter/config.rs` audit: + - **Model candidates:** `ExecDirective`, `ValueAction`, `IoStream`, `FilterMeta`, `IoConfig`, `ValuePattern`, `ScopeRule`, `FilterSource`, `FilterSummary`, `FilterSummaryEntry`, `TraceFilterConfig`. + - **Loader utilities:** `ConfigAggregator` and helpers (`ingest_*`, `finish`, `calculate_sha256`, `detect_project_root`, `parse_meta`, `resolve_defaults`, `parse_*`, `parse_rules`, `parse_value_patterns`), plus private `Raw*` serde structs. + - `trace_filter/engine.rs` audit: + - **Model/shared:** `ExecDecision`, `ValueKind`, `ValuePolicy`, `ScopeResolution`. + - **Engine core:** `TraceFilterEngine`, `CompiledScopeRule`, `CompiledValuePattern`, `ScopeContext`, compilation helpers (`compile_rules`, `compile_value_patterns`, `ScopeContext::derive`, `normalise_*`, `module_from_relative`, `py_attr_error`). + - **Tests:** rely on helper `filter_with_pkg_rule`; will need relocation once modules split. +- ✅ Milestone 1 skeleton: added placeholder modules `trace_filter::model`, `::loader`, and `::summary`; updated `trace_filter::mod` to expose them while retaining existing `config`/`engine` facades for compatibility. +- ✅ Step 1 complete: relocated shared model types (`ExecDirective`, `ValueAction`, `IoStream`, `IoConfig`, `ValuePattern`, `ScopeRule`, `FilterSource`, `FilterMeta`, `FilterSummary*`, `TraceFilterConfig`) into `trace_filter::model`, re-exported them from `config`, and removed duplicate impls. `just test` verified the crate after the move. +- ✅ Step 2 complete: extracted loader utilities and serde `Raw*` structures into `trace_filter::loader`, rewrote the config facade to use `ConfigAggregator`, and rebuilt selector normalisation via `Selector::parse`. `just test` (Rust + Python suites) confirmed parsing works post-move. +- ✅ Step 3 complete: moved summary construction into `trace_filter::summary`, updated `TraceFilterConfig::summary` to delegate to the new helper, and re-ran `just test` (all Rust/Python tests pass). +- ✅ Facade review: `trace_filter::config` now re-exports model types and delegates to the loader; no redundant helpers remain. Module exports verified via `just test`. + +- 🔄 Milestone 2 Kickoff: auditing `policy.rs` and `logging.rs` to classify responsibilities for modularisation. + - `policy.rs` audit: + - **Model candidates:** `OnRecorderError`, `IoCapturePolicy`, `RecorderPolicy`, `PolicyUpdate`, `PolicyPath`, `policy_snapshot`, POLICY cell helpers. + - **Environment parsing:** constants (`ENV_*`), `configure_policy_from_env`, `parse_bool`, `parse_capture_io`. + - **FFI bindings:** `configure_policy_py`, `py_configure_policy_from_env`, `py_policy_snapshot`, PyO3 imports/tests. + - `logging.rs` audit: + - **Logger core:** `RecorderLogger`, `FilterSpec`, init/apply helpers, destination management. + - **Metrics:** `RecorderMetrics` trait, `NoopMetrics`, `install_metrics`, `metrics_sink`, telemetry recorders. + - **Error trailers:** `emit_error_trailer`, trailer writer management. + - **Shared utilities:** `with_error_code[_opt]`, `set_active_trace_id`, `log_recorder_error`, `JSON_ERRORS_ENABLED`. +- ✅ Milestone 2 scaffolding: created placeholder modules `policy::{model, env, ffi}` and `logging::{logger, metrics, trailer}`; top-level `policy.rs`/`logging.rs` still host existing logic pending extraction. `just test` validates the skeletal split compiles. +- ✅ Milestone 2 Step 1: moved policy data structures and global helpers into `policy::model`, re-exported public APIs, updated tests, and reran Rust/Python suites (`cargo nextest`, `pytest`) successfully. +- ✅ Milestone 2 Step 2: migrated environment parsing/consts into `policy::env`, cleaned `policy.rs` to consume the facade, and refreshed unit tests. `uv run cargo nextest` + `uv run python -m pytest` both pass. +- ✅ Milestone 2 Step 3: relocated all PyO3 policy bindings into `policy::ffi`, updated the facade re-exports, and stretched unit coverage before re-running `just test`. + - `policy.rs` now only wires modules together while `policy::ffi` owns `configure_policy_py`, `py_configure_policy_from_env`, and `py_policy_snapshot` alongside focused tests (error translation, snapshot shape). + - `policy::ffi` imports model/env helpers via sibling modules and continues to use `crate::ffi::map_recorder_error`; `lib.rs` still registers these bindings via the facade exports so Python callers see no change. + - Simplified the PyO3 snapshot test to validate expected keys after verifying rust-side policy behaviour; broader value assertions remain covered by model/env tests. +- ✅ Milestone 2 Step 4: extracted logging responsibilities into `logging::{logger, metrics, trailer}`, leaving `logging.rs` as a thin facade that re-exports public APIs. + - `logger.rs` owns the log installation, filter parsing, policy application, and error-code scoping; it exposes helpers (`with_error_code`, `log_recorder_error`, `set_active_trace_id`) for the rest of the crate. + - `metrics.rs` encapsulates the `RecorderMetrics` trait, sink installation, and testing harness; `trailer.rs` manages JSON error toggles and payload emission via the logger's context snapshot. + - Updated facade tests (`structured_log_records`, `json_error_trailers_emit_payload`, metrics capture) to rely on the new modules; `just test` verifies Rust + Python suites after the split. +- ✅ Milestone 3 complete: `session/bootstrap` delegates to `filesystem`, `metadata`, and `filters` submodules, each with focused unit tests covering success and failure paths (e.g., unwritable directory, unsupported formats, missing filters). `TraceSessionBootstrap` now orchestrates these modules without additional helper functions, and `just test` (Rust + Python) confirms parity. +- 🔄 Milestone 4 Kickoff: surveying `monitoring/mod.rs` and `monitoring/tracer.rs` to stage the split into `monitoring::{api, install, callbacks}`. + - `api.rs` now hosts the `Tracer` trait and shared type aliases, leaving `tracer.rs` to consume it via the facade. + - `install.rs` and `callbacks.rs` currently re-export legacy plumbing while we prepare to migrate install/registration logic and PyO3 wrappers in subsequent steps. +- ✅ Milestone 4 Step 1: introduced a declarative `CALLBACK_SPECS` table and helper APIs in `monitoring::callbacks` to drive registration and teardown. + - `monitoring::callbacks` now exposes `register_enabled_callbacks`/`unregister_enabled_callbacks`, replacing the hand-written loops in `monitoring/tracer.rs`. + - Callback functions remain in `monitoring::tracer` for now but are exported as `pub(super)` so the next step can relocate them without changing call sites. + - Preserved the invariants from the kickoff audit (16 active events, shared error-handling helpers, tool ownership) and exercised them via the new table-driven helpers. +- ✅ Milestone 4 Step 2: migrated the PyO3 callback shims and error-handling helpers into `monitoring::callbacks`, centralising the shared global state. + - `Global`/`GLOBAL` now live alongside the callback metadata, and `handle_callback_error` channels disable-on-error flows through the shared helpers. + - Rewired `CALLBACK_SPECS` to wrap in-module functions and removed the duplicated definitions from `monitoring/tracer.rs`. + - `monitoring::tracer` shrank to installer plumbing ahead of the dedicated install split. +- ✅ Milestone 4 Step 3: lifted installation plumbing into `monitoring::install`, leaving `tracer.rs` as a compatibility facade. +- ✅ Milestone 4 Step 4: ran `just test` (Rust nextest + Python pytest) after the module split to ensure behaviour parity. + - All suites passed (1 perf test skipped), confirming the new callbacks/install layout preserves runtime semantics. + - No additional formatting or lint adjustments required beyond `cargo fmt`. + - `monitoring::install` now owns `install_tracer`, `uninstall_tracer`, `flush_installed_tracer`, and the internal `uninstall_locked` helper, all backed by `callbacks::GLOBAL`. + - `monitoring::callbacks` delegates disable-on-error teardown to `install::uninstall_locked`, while `monitoring::tracer` simply re-exports the install APIs. + - Updated module imports keep the public facade unchanged (still exported via `monitoring::install`), paving the way for runtime tracer refactors in Milestone 5. +- ✅ Milestone 4 Step 5: documentation pass to close out the milestone and queue the next phase. + - Summarised the refactor scope (status tracker + ADR 0011 update) and recorded the retrospective in `design-docs/codetracer-architecture-refactor-milestone-4-retrospective.md`. + - Repository remains test-clean; next work items roll into Milestone 5 prep. +- 🔄 Milestone 5 Kickoff: audited `runtime/mod.rs` to outline collaborator boundaries before extracting modules. + - **Lifecycle management:** `RuntimeTracer::new`, `finish`, `finalise_writer`, `cleanup_partial_outputs`, `notify_failure`, `require_trace_or_fail`, activation teardown, and metadata writers. + - **Event handling:** `Tracer` impl (`interest`, `on_py_start`, `on_line`, `on_py_return`) plus helpers (`ensure_function_id`, `mark_event`, `mark_failure`). + - **Filter cache:** `scope_resolution`, `should_trace_code`, `FilterStats`, ignore tracking, and filter summary appenders. + - **IO coordination:** `install_io_capture`, `flush_*`, `drain_io_chunks`, `record_io_chunk`, `build_io_metadata`, `teardown_io_capture`, and `io_flag_labels`. +- ✅ Milestone 5 Step 1: moved `RuntimeTracer` and companion helpers into `runtime::tracer::runtime_tracer`, re-exported the type via `runtime::tracer` and `runtime`, and kept module scaffolding for upcoming collaborators. `just test` (Rust nextest + Python pytest) confirms the relocation preserves behaviour. +- ✅ Milestone 5 Step 2: extracted IO coordination into `runtime::tracer::io::IoCoordinator`, delegating installation, flush/teardown, metadata enrichment, and snapshot tracking from `RuntimeTracer`. Updated callers to mark events on IO writes and re-ran `just test` to validate Rust and Python suites. +- ✅ Milestone 5 Step 3: introduced `runtime::tracer::filtering::FilterCoordinator` to own scope resolution, skip caching, telemetry stats, and metadata wiring. `RuntimeTracer` now delegates trace decisions and summary emission, while tests continue to validate skip behaviour and metadata shape with unchanged expectations. +- ✅ Milestone 5 Step 4: carved lifecycle orchestration into `runtime::tracer::lifecycle::LifecycleController`, covering activation gating, writer initialisation/finalisation, policy enforcement, failure cleanup, and trace id scoping. Added focused unit tests for the controller and re-ran `just test` (nextest + pytest) to verify no behavioural drift. +- ✅ Milestone 5 Step 5: shifted event handling into `runtime::tracer::events`, relocating the `Tracer` trait implementation alongside failure-injection helpers and telemetry wiring. `RuntimeTracer` now exposes a slim collaborator API (`mark_event`, `flush_io_before_step`, `ensure_function_id`), while tests import the trait explicitly. `just test` (nextest + pytest) confirms the callbacks behave identically after the split. +- ✅ Milestone 5 Step 6: harmonised the tracer module facade by tightening `IoCoordinator` visibility, pruning unused re-exports, documenting the `runtime::tracer` layout, and updating design docs that referenced the legacy `runtime_tracer.rs` path. `just test` (Rust nextest + Python pytest) verified the cleanup. +- 🔄 Milestone 6 Kickoff: audited crate exports to keep the new module tree internal by default (runtime tracer collaborators + monitoring submodules now `pub(crate)`), and confirmed packaging metadata/docs already reference the updated paths so no additional adjustments are required yet. Pending follow-up: sweep for dead imports and finalise documentation/CI updates before ADR 0011 sign-off. +- ✅ Milestone 6 Step 1: realigned the Python trace filter benchmark to account for the always-prepended `builtin-default` filter when validating metadata, restoring the smoke test with `just test` (nextest + pytest) coverage. +- ✅ Milestone 6 Step 2: finished the integration cleanup—tightened monitoring callback logging to use the new table metadata, confirmed no stale doc references or build script changes were required, and re-ran `just test` (nextest + pytest) as the final verification before ADR 0011 acceptance. + + +### Planned Extraction Order (Milestone 4) +1. **Callback metadata table:** Introduce a declarative structure in `monitoring::callbacks` that captures CPython event identifiers, binding names, and tracer entrypoints so registration/unregistration can iterate instead of hand-writing each branch. +2. **Callback relocation:** Move the `*_callback` PyO3 functions plus the `catch_callback` and `call_tracer_with_code` helpers into `monitoring::callbacks`, exposing a minimal API for registering callbacks against a tool id. +3. **Install plumbing:** Shift `install_tracer`, `flush_installed_tracer`, and `uninstall_tracer` into `monitoring::install`, ensuring tool acquisition, event mask negotiation, and disable-sentinel handling route through the new callback table. +4. **Tests and verification:** Update unit tests (including panic-to-pyerr coverage) to point at the new modules, add table-driven tests for registration completeness, and run `just test` to confirm the refactor preserves behaviour. + +### Planned Extraction Order (Milestone 5) +1. **Scaffold collaborators:** Introduce `runtime::tracer` with submodules for lifecycle, events, filtering, and IO; move `RuntimeTracer` into the new tree while keeping the public facade (`crate::runtime::RuntimeTracer`) stable. +2. **IO coordinator migration:** Extract IO capture installation/flush/record logic into `runtime::tracer::io::IoCoordinator`, delegating from `RuntimeTracer` and covering payload metadata helpers. +3. **Filter cache module:** Move scope resolution, ignore tracking, statistics, and metadata serialisation into `runtime::tracer::filtering`, exposing a collaborator that caches resolutions and records drops. +4. **Lifecycle controller:** Relocate writer setup/teardown, policy checks, failure handling, activation gating, and metadata finalisation into `runtime::tracer::lifecycle`. +5. **Event processor:** Shift `Tracer` trait implementation and per-event pipelines into `runtime::tracer::events`, wiring through the collaborators and updating unit/integration tests; run `just test` after the split. + +### Planned Extraction Order (Milestone 2) +1. **Policy model split:** Move data structures (`OnRecorderError`, `IoCapturePolicy`, `RecorderPolicy`, `PolicyUpdate`, `PolicyPath`) and policy cell helpers (`policy_cell`, `policy_snapshot`, `apply_policy_update`) into `policy::model`. Expose minimal APIs for environment/FFI modules. +2. **Policy environment parsing:** Relocate `configure_policy_from_env`, env variable constants, and helper parsers (`parse_bool`, `parse_capture_io`) into `policy::env`, depending on `policy::model` for mutations. +3. **Policy FFI layer:** Migrate PyO3 functions (`configure_policy_py`, `py_configure_policy_from_env`, `py_policy_snapshot`) into `policy::ffi`, keeping tests alongside; ensure `lib.rs` uses the new module exports. +4. **Logging module split:** Extract `RecorderLogger`, `FilterSpec`, `init_rust_logging_with_default`, `apply_policy`, and log helpers into `logging::logger`. Place metrics trait/sink logic into `logging::metrics`, error trailer functions into `logging::trailer`, leaving `logging.rs` as the facade orchestrating shared utilities (`with_error_code`, `set_active_trace_id`). +5. **Update tests & imports:** Adjust unit tests to target new modules, ensure re-exports keep existing public API stable, and run `just test` after each stage. + +### Planned Extraction Order (Milestone 1) +1. **Model types first:** Relocate shared enums/structs (`ExecDirective`, `ValueAction`, `IoStream`, `FilterMeta`, `IoConfig`, `ValuePattern`, `ScopeRule`, `FilterSource`, `FilterSummary*`, `TraceFilterConfig`) into `trace_filter::model`. Update `config.rs` to re-export or `use` the new module and adjust external call sites (`session/bootstrap.rs`, `runtime/mod.rs`, tests). +2. **Loader utilities next:** Port `ConfigAggregator`, parsing helpers (`ingest_*`, `calculate_sha256`, `detect_project_root`, `parse_*`, `parse_rules`, `parse_value_patterns`) and serde `Raw*` structs into `trace_filter::loader`. Provide a clean API (e.g., `Loader::finish() -> TraceFilterConfig`) consumed by the facade. +3. **Summary helpers:** Move filter summary construction into `trace_filter::summary`, ensuring metadata writers (`RuntimeTracer::append_filter_metadata`) switch to the new API. +4. **Facade cleanup:** Once pieces live in dedicated modules, shrink `config.rs` to a thin facade that orchestrates loader/model interactions and re-exports primary types. Keep backward-compatible function names for now. +5. **Tests:** After each move, update unit tests in `trace_filter` modules and dependent integration tests (`session/bootstrap.rs` tests, `runtime` tests). Targeted command: `just test` (covers Rust + Python suites). + +## Next Actions +1. Communicate completion to stakeholders and monitor for regression reports during adoption. +2. Archive supporting notes and update any external runbooks that referenced the pre-refactor layout. diff --git a/design-docs/codetracer-architecture-refactor-milestone-4-retrospective.md b/design-docs/codetracer-architecture-refactor-milestone-4-retrospective.md new file mode 100644 index 0000000..9b10c8d --- /dev/null +++ b/design-docs/codetracer-architecture-refactor-milestone-4-retrospective.md @@ -0,0 +1,26 @@ +# Codetracer Architecture Refactor – Milestone 4 Retrospective + +- **Milestone window:** 2025‑02‑17 → 2025‑03‑01 +- **Scope recap:** Detangle `monitoring` so `sys.monitoring` plumbing lives in cohesive modules (`api`, `callbacks`, `install`) and eliminate hand-rolled callback registration/teardown logic while preserving the existing public facade. + +## Outcomes +- Added a declarative `CALLBACK_SPECS` table plus helper APIs to drive callback registration/unregistration, replacing ~30 duplicate branches. +- Centralised tracer state and error handling in `monitoring::callbacks`, ensuring panic-to-PyErr conversion, policy-driven disable flows, and callback execution share the same instrumentation. +- Moved install/teardown logic into `monitoring::install`, leaving `monitoring::tracer` as a compatibility shim; consumers still import `install_tracer` et al. unchanged. +- `just test` (Rust `cargo nextest` + Python `pytest`) passes post-refactor, confirming behavioural parity; one existing perf test remained skipped as expected. + +## What Went Well +- Table-driven metadata drastically simplified maintenance—adding or removing CPython events is now a single-row change. +- Co-locating global state with callback helpers removed redundant locking/unwrap patterns spread across modules. +- Incremental updates to the status tracker kept context handy when the work paused between sessions. + +## Challenges & Mitigations +- Adapting PyO3 wrappers required careful lifetime handling; switching helper factories to accept `Bound<'py, PyModule>` avoided compile-time churn. +- Ensuring disable-on-error flows still reached teardown code meant delegating to `install::uninstall_locked`; unit paths relied on shared helpers to avoid divergence. +- Multiple modules touched by the split increased the risk of import regressions. Running `cargo fmt` and `just test` after each major change caught mistakes early. + +## Follow-Ups +1. Update developer docs (README/AGENTS) once Milestone 5 lands so the new monitoring structure is reflected in onboarding material. +2. Revisit milestone test coverage to see if table-driven registration merits additional unit tests (e.g., verifying `CALLBACK_SPECS` completeness via assertions). +3. Proceed to Milestone 5 (runtime tracer modularisation) using the newly isolated install/callback modules as building blocks. +4. Capture any stakeholder feedback and incorporate into ADR 0011 before final acceptance.*** End Patch diff --git a/design-docs/file-level-srp-refactor-plan.md b/design-docs/file-level-srp-refactor-plan.md index 476bfc6..255a5e6 100644 --- a/design-docs/file-level-srp-refactor-plan.md +++ b/design-docs/file-level-srp-refactor-plan.md @@ -7,7 +7,7 @@ ## Current State Observations - `src/lib.rs` is responsible for PyO3 module registration, lifecycle management for tracing sessions, global logging initialisation, and runtime format selection, which mixes unrelated concerns in one file. -- `src/runtime_tracer.rs` couples trace lifecycle control, activation toggling, and Python value encoding in a single module, making it difficult to unit test or substitute individual pieces. +- `src/runtime/tracer/runtime_tracer.rs` (previously the monolithic `runtime_tracer.rs`) couples trace lifecycle control, activation toggling, and Python value encoding in a single module, making it difficult to unit test or substitute individual pieces. - `src/tracer.rs` combines the `Tracer` trait definition, sys.monitoring shims, callback registration utilities, and thread-safe storage, meaning small changes can ripple through unrelated logic. - `codetracer_python_recorder/api.py` interleaves environment based auto-start, context-manager ergonomics, backend state management, and format constants, leaving no clearly isolated entry-point for CLI or library callers. diff --git a/design-docs/value-capture.md b/design-docs/value-capture.md index 4523806..b2de0a8 100644 --- a/design-docs/value-capture.md +++ b/design-docs/value-capture.md @@ -336,7 +336,7 @@ result = builtins_test([5, 3, 7]) # General Rules * This spec is for `/codetracer-python-recorder` project and NOT for `/codetracer-pure-python-recorder` -* Code and tests should be added to `/codetracer-python-recorder/src/runtime_tracer.rs` +* Code and tests should be added under `/codetracer-python-recorder/src/runtime/tracer/` (primarily `runtime_tracer.rs` and its collaborators) * Performance is important. Avoid using Python modules and functions and prefer PyO3 methods including the FFI API. * If you want to run Python do it like so `uv run python` This will set up the right venv. Similarly for running tests `uv run pytest`. * After every code change you need to run `just dev` to make sure that you are testing the new code. Otherwise some tests might run against the old code