From 6ae9093b94e75e4a857646082ca842cf7945d11d Mon Sep 17 00:00:00 2001 From: Nikolay Nikolov Date: Fri, 18 Jul 2025 23:29:43 +0300 Subject: [PATCH 01/28] feat(cbor_plus_zstd): added reference to zeekstd version 0.5.0 (the latest version) --- runtime_tracing/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/runtime_tracing/Cargo.toml b/runtime_tracing/Cargo.toml index 665bd2e..cd5ea35 100644 --- a/runtime_tracing/Cargo.toml +++ b/runtime_tracing/Cargo.toml @@ -20,6 +20,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_repr = "0.1" capnp = "0.21.1" +zeekstd = "0.5.0" [build-dependencies] capnpc = "0.21.0" From 331992e5987f5aa56b7cfe9f8d3db02271e08356 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolov Date: Fri, 18 Jul 2025 23:37:35 +0300 Subject: [PATCH 02/28] feat(cbor_plus_zstd): upgrade to Rust 2024 edition, because of zeekstd 0.5.0 --- runtime_tracing/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime_tracing/Cargo.toml b/runtime_tracing/Cargo.toml index cd5ea35..d74f3d2 100644 --- a/runtime_tracing/Cargo.toml +++ b/runtime_tracing/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "runtime_tracing" version = "0.13.0" -edition = "2021" +edition = "2024" authors = ["Metacraft Labs Ltd"] description = "A library for the schema and tracing helpers for the CodeTracer db trace format" readme = "README.md" From 23c232413b751d1294c11fc253b81d20c814df99 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolov Date: Tue, 22 Jul 2025 23:07:57 +0300 Subject: [PATCH 03/28] feat(cbor_plus_zstd): added reference to cbor4ii version 1.0.0 (the latest version) --- runtime_tracing/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/runtime_tracing/Cargo.toml b/runtime_tracing/Cargo.toml index d74f3d2..70320b4 100644 --- a/runtime_tracing/Cargo.toml +++ b/runtime_tracing/Cargo.toml @@ -21,6 +21,7 @@ serde_json = "1.0" serde_repr = "0.1" capnp = "0.21.1" zeekstd = "0.5.0" +cbor4ii = "1.0.0" [build-dependencies] capnpc = "0.21.0" From 07bc2ff79d569b59feeffe88aabdb61c28d3e11e Mon Sep 17 00:00:00 2001 From: Nikolay Nikolov Date: Tue, 22 Jul 2025 23:42:09 +0300 Subject: [PATCH 04/28] feat(cbor_plus_zstd): enable the serde1 feature of cbor4ii --- runtime_tracing/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime_tracing/Cargo.toml b/runtime_tracing/Cargo.toml index 70320b4..6de21ed 100644 --- a/runtime_tracing/Cargo.toml +++ b/runtime_tracing/Cargo.toml @@ -21,7 +21,7 @@ serde_json = "1.0" serde_repr = "0.1" capnp = "0.21.1" zeekstd = "0.5.0" -cbor4ii = "1.0.0" +cbor4ii = { version = "1.0.0", features = ["serde1"] } [build-dependencies] capnpc = "0.21.0" From 6bbc2c785c0dae080a8bf0882f1005e5e2efe497 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolov Date: Wed, 23 Jul 2025 01:33:33 +0300 Subject: [PATCH 05/28] feat(cbor_plus_zstd): started implementing a StreamingTraceWriter. Not finished, yet. --- runtime_tracing/src/cborzstdwriter.rs | 348 ++++++++++++++++++++++++++ runtime_tracing/src/lib.rs | 2 + 2 files changed, 350 insertions(+) create mode 100644 runtime_tracing/src/cborzstdwriter.rs diff --git a/runtime_tracing/src/cborzstdwriter.rs b/runtime_tracing/src/cborzstdwriter.rs new file mode 100644 index 0000000..2791e2a --- /dev/null +++ b/runtime_tracing/src/cborzstdwriter.rs @@ -0,0 +1,348 @@ +use std::{collections::HashMap, env, fs::{self, File}, io::Write, path::PathBuf}; +use zeekstd::Encoder; + +use crate::{tracer::TOP_LEVEL_FUNCTION_ID, AssignCellRecord, AssignCompoundItemRecord, AssignmentRecord, CallRecord, CellValueRecord, CompoundValueRecord, FullValueRecord, FunctionId, FunctionRecord, Line, PathId, RValue, RecordEvent, ReturnRecord, StepRecord, TraceLowLevelEvent, TraceMetadata, TraceWriter, TypeId, TypeKind, TypeRecord, TypeSpecificInfo, VariableCellRecord, VariableId, NONE_TYPE_ID}; + +pub struct StreamingTraceWriter<'a> { + // trace metadata: + workdir: PathBuf, + program: String, + args: Vec, + // trace events + //pub events: Vec, + // internal tracer state: + path_list: Vec, + function_list: Vec<(String, PathId, Line)>, + + paths: HashMap, + functions: HashMap, + variables: HashMap, + types: HashMap, + + //format: TraceEventsFileFormat, + trace_metadata_path: Option, + trace_events_path: Option, + trace_events_file_output: Option, + trace_events_file_zstd_encoder: Option>>, + trace_paths_path: Option, +} + +impl<'a> StreamingTraceWriter<'a> { + /// Create a new tracer instance for the given program and arguments. + pub fn new(program: &str, args: &[String]) -> Self { + StreamingTraceWriter { + workdir: env::current_dir().expect("can access the current dir"), + program: program.to_string(), + args: args.to_vec(), + //events: vec![], + + path_list: vec![], + function_list: vec![], + paths: HashMap::new(), + functions: HashMap::new(), + variables: HashMap::new(), + types: HashMap::new(), + + //format: TraceEventsFileFormat::Binary, + trace_metadata_path: None, + trace_events_path: None, + trace_events_file_output: None, + trace_events_file_zstd_encoder: None, + trace_paths_path: None, + } + } +} + +impl<'a> TraceWriter for StreamingTraceWriter<'a> { + fn begin_writing_trace_metadata(&mut self, path: &std::path::Path) -> Result<(), Box> { + self.trace_metadata_path = Some(path.to_path_buf()); + Ok(()) + } + + fn begin_writing_trace_events(&mut self, path: &std::path::Path) -> Result<(), Box> { + let pb = path.to_path_buf(); + self.trace_events_path = Some(pb.clone()); + self.trace_events_file_output = Some(std::fs::File::create(pb)?); + //self.trace_events_file_zstd_encoder = Some(Encoder::new(&self.trace_events_file_output)?); + + Ok(()) + } + + fn begin_writing_trace_paths(&mut self, path: &std::path::Path) -> Result<(), Box> { + self.trace_paths_path = Some(path.to_path_buf()); + Ok(()) + } + + fn start(&mut self, path: &std::path::Path, line: Line) { + let function_id = self.ensure_function_id("", path, line); + self.register_call(function_id, vec![]); + assert!(function_id == TOP_LEVEL_FUNCTION_ID); + + // probably we let the user choose, as different languages have + // different base types/names + // assert!(EXAMPLE_INT_TYPE_ID == self.load_type_id(TypeKind::Int, "Int")); + // assert!(EXAMPLE_FLOAT_TYPE_ID == self.load_type_id(TypeKind::Float, "Float")); + // assert!(EXAMPLE_BOOL_TYPE_ID == self.load_type_id(TypeKind::Bool, "Bool")); + // assert!(EXAMPLE_STRING_TYPE_ID == self.load_type_id(TypeKind::Bool, "String")); + assert!(NONE_TYPE_ID == self.ensure_type_id(TypeKind::None, "None")); + } + + fn ensure_path_id(&mut self, path: &std::path::Path) -> PathId { + if !self.paths.contains_key(path) { + self.paths.insert(path.to_path_buf(), PathId(self.paths.len())); + self.register_path(path); + } + *self.paths.get(path).unwrap() + } + + fn ensure_function_id(&mut self, function_name: &str, path: &std::path::Path, line: Line) -> FunctionId { + if !self.functions.contains_key(function_name) { + // same function names for different path line? TODO + self.functions.insert(function_name.to_string(), FunctionId(self.functions.len())); + self.register_function(function_name, path, line); + } + *self.functions.get(function_name).unwrap() + } + + fn ensure_type_id(&mut self, kind: crate::TypeKind, lang_type: &str) -> TypeId { + let typ = self.to_raw_type(kind, lang_type); + self.ensure_raw_type_id(typ) + } + + fn ensure_raw_type_id(&mut self, typ: crate::TypeRecord) -> TypeId { + if !self.types.contains_key(&typ.lang_type) { + self.types.insert(typ.lang_type.clone(), TypeId(self.types.len())); + self.register_raw_type(typ.clone()); + } + *self.types.get(&typ.lang_type).unwrap() + } + + fn ensure_variable_id(&mut self, variable_name: &str) -> VariableId { + if !self.variables.contains_key(variable_name) { + self.variables.insert(variable_name.to_string(), VariableId(self.variables.len())); + self.register_variable_name(variable_name); + } + *self.variables.get(variable_name).unwrap() + } + + fn register_path(&mut self, path: &std::path::Path) { + self.path_list.push(path.to_path_buf()); + self.add_event(TraceLowLevelEvent::Path(path.to_path_buf())); + } + + fn register_function(&mut self, name: &str, path: &std::path::Path, line: Line) { + let path_id = self.ensure_path_id(path); + self.function_list.push((name.to_string(), path_id, line)); + self.add_event(TraceLowLevelEvent::Function(FunctionRecord { + name: name.to_string(), + path_id, + line, + })); + } + + fn register_step(&mut self, path: &std::path::Path, line: Line) { + let path_id = self.ensure_path_id(path); + self.add_event(TraceLowLevelEvent::Step(StepRecord { path_id, line })); + } + + fn register_call(&mut self, function_id: FunctionId, args: Vec) { + // register a step for each call, the backend expects this for + // non-toplevel calls, so + // we ensure it directly from register_call + if function_id != TOP_LEVEL_FUNCTION_ID { + for arg in &args { + self.register_full_value(arg.variable_id, arg.value.clone()); + } + let function = &self.function_list[function_id.0]; + self.add_event(TraceLowLevelEvent::Step(StepRecord { + path_id: function.1, + line: function.2, + })); + } + // the actual call event: + self.add_event(TraceLowLevelEvent::Call(CallRecord { function_id, args })); + } + + fn arg(&mut self, name: &str, value: crate::ValueRecord) -> FullValueRecord { + let variable_id = self.ensure_variable_id(name); + FullValueRecord { variable_id, value } + } + + fn register_return(&mut self, return_value: crate::ValueRecord) { + self.add_event(TraceLowLevelEvent::Return(ReturnRecord { return_value })); + } + + fn register_special_event(&mut self, kind: crate::EventLogKind, content: &str) { + self.add_event(TraceLowLevelEvent::Event(RecordEvent { + kind, + metadata: "".to_string(), + content: content.to_string(), + })); + } + + fn to_raw_type(&self, kind: crate::TypeKind, lang_type: &str) -> crate::TypeRecord { + TypeRecord { + kind, + lang_type: lang_type.to_string(), + specific_info: TypeSpecificInfo::None, + } + } + + fn register_type(&mut self, kind: crate::TypeKind, lang_type: &str) { + let typ = self.to_raw_type(kind, lang_type); + self.add_event(TraceLowLevelEvent::Type(typ)); + } + + fn register_raw_type(&mut self, typ: crate::TypeRecord) { + self.add_event(TraceLowLevelEvent::Type(typ)); + } + + fn register_asm(&mut self, instructions: &[String]) { + self.add_event(TraceLowLevelEvent::Asm(instructions.to_vec())); + } + + fn register_variable_with_full_value(&mut self, name: &str, value: crate::ValueRecord) { + let variable_id = self.ensure_variable_id(name); + self.register_full_value(variable_id, value); + } + + fn register_variable_name(&mut self, variable_name: &str) { + self.add_event(TraceLowLevelEvent::VariableName(variable_name.to_string())); + } + + fn register_full_value(&mut self, variable_id: VariableId, value: crate::ValueRecord) { + self.add_event(TraceLowLevelEvent::Value(FullValueRecord { variable_id, value })); + } + + fn register_compound_value(&mut self, place: crate::Place, value: crate::ValueRecord) { + self.add_event(TraceLowLevelEvent::CompoundValue(CompoundValueRecord { place, value })); + } + + fn register_cell_value(&mut self, place: crate::Place, value: crate::ValueRecord) { + self.add_event(TraceLowLevelEvent::CellValue(CellValueRecord { place, value })); + } + + fn assign_compound_item(&mut self, place: crate::Place, index: usize, item_place: crate::Place) { + self.add_event(TraceLowLevelEvent::AssignCompoundItem(AssignCompoundItemRecord { + place, + index, + item_place, + })); + } + + fn assign_cell(&mut self, place: crate::Place, new_value: crate::ValueRecord) { + self.add_event(TraceLowLevelEvent::AssignCell(AssignCellRecord { place, new_value })); + } + + fn register_variable(&mut self, variable_name: &str, place: crate::Place) { + let variable_id = self.ensure_variable_id(variable_name); + self.add_event(TraceLowLevelEvent::VariableCell(VariableCellRecord { variable_id, place })); + } + + fn drop_variable(&mut self, variable_name: &str) { + let variable_id = self.ensure_variable_id(variable_name); + self.add_event(TraceLowLevelEvent::DropVariable(variable_id)); + } + + // history event helpers + fn assign(&mut self, variable_name: &str, rvalue: crate::RValue, pass_by: crate::PassBy) { + let variable_id = self.ensure_variable_id(variable_name); + self.add_event(TraceLowLevelEvent::Assignment(AssignmentRecord { + to: variable_id, + from: rvalue, + pass_by, + })); + } + + fn bind_variable(&mut self, variable_name: &str, place: crate::Place) { + let variable_id = self.ensure_variable_id(variable_name); + self.add_event(TraceLowLevelEvent::BindVariable(crate::BindVariableRecord { variable_id, place })); + } + + fn drop_variables(&mut self, variable_names: &[String]) { + let variable_ids: Vec = variable_names + .to_vec() + .iter() + .map(|variable_name| self.ensure_variable_id(variable_name)) + .collect(); + self.add_event(TraceLowLevelEvent::DropVariables(variable_ids)) + } + + fn simple_rvalue(&mut self, variable_name: &str) -> crate::RValue { + let variable_id = self.ensure_variable_id(variable_name); + RValue::Simple(variable_id) + } + + fn compound_rvalue(&mut self, variable_dependencies: &[String]) -> crate::RValue { + let variable_ids: Vec = variable_dependencies + .to_vec() + .iter() + .map(|variable_dependency| self.ensure_variable_id(variable_dependency)) + .collect(); + RValue::Compound(variable_ids) + } + + fn drop_last_step(&mut self) { + self.add_event(TraceLowLevelEvent::DropLastStep); + } + + fn add_event(&mut self, event: TraceLowLevelEvent) { + todo!() + } + + fn append_events(&mut self, events: &mut Vec) { + todo!() + } + + fn finish_writing_trace_metadata(&mut self) -> Result<(), Box> { + if let Some(path) = &self.trace_metadata_path { + let trace_metadata = TraceMetadata { + program: self.program.clone(), + args: self.args.clone(), + workdir: self.workdir.clone(), + }; + let json = serde_json::to_string(&trace_metadata)?; + fs::write(path, json)?; + Ok(()) + } else { + panic!("finish_writing_trace_metadata() called without previous call to begin_writing_trace_metadata()"); + } + } + + fn finish_writing_trace_events(&mut self) -> Result<(), Box> { + todo!() + } + + fn finish_writing_trace_paths(&mut self) -> Result<(), Box> { + if let Some(path) = &self.trace_paths_path { + let json = serde_json::to_string(&self.path_list)?; + fs::write(path, json)?; + Ok(()) + } else { + panic!("finish_writing_trace_paths() called without previous call to begin_writing_trace_paths()"); + } + } +} + +fn test_zstd() { + let output = std::fs::File::create("tralala.zstd").unwrap(); + let mut encoder = Encoder::new(output).unwrap(); + + let event: TraceLowLevelEvent = TraceLowLevelEvent::Path(PathBuf::new()); + + let buf: Vec = Vec::new(); + let q = cbor4ii::serde::to_vec(buf, &event).unwrap(); + encoder.write(&q).unwrap(); + + let f = encoder.finish().unwrap(); +} + +#[cfg(test)] +mod tests { + use crate::cborzstdwriter::test_zstd; + + #[test] + fn run_zstd_test() { + test_zstd(); + } +} diff --git a/runtime_tracing/src/lib.rs b/runtime_tracing/src/lib.rs index b9393de..9d62117 100644 --- a/runtime_tracing/src/lib.rs +++ b/runtime_tracing/src/lib.rs @@ -12,6 +12,8 @@ mod base64; mod capnptrace; mod tracer; mod types; +mod cborzstdwriter; + pub use crate::tracer::{ create_trace_reader, create_trace_writer, NonStreamingTraceWriter, TraceEventsFileFormat, TraceReader, TraceWriter, NONE_TYPE_ID, NONE_VALUE, }; From 84b053ba6b1cd45cd2d8a534e08b8796b2525152 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolov Date: Wed, 23 Jul 2025 01:41:35 +0300 Subject: [PATCH 06/28] feat(cbor_plus_zstd): introduced TraceEventsFileFormat::BinaryV0. Use that for the capnproto binary format, and Binary for the new (unfinished) CBOR + zstd encoding format. --- runtime_tracing/src/tracer.rs | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/runtime_tracing/src/tracer.rs b/runtime_tracing/src/tracer.rs index e384947..9ce383a 100644 --- a/runtime_tracing/src/tracer.rs +++ b/runtime_tracing/src/tracer.rs @@ -115,7 +115,8 @@ pub struct NonStreamingTraceWriter { #[derive(Debug, Clone, Copy)] pub enum TraceEventsFileFormat { Json, - Binary, + BinaryV0, + Binary } // we ensure in start they are registered with those id-s @@ -162,7 +163,9 @@ impl NonStreamingTraceWriter { let json = std::fs::read_to_string(path)?; self.events = serde_json::from_str(&json)?; } - TraceEventsFileFormat::Binary => { + TraceEventsFileFormat::Binary | + TraceEventsFileFormat::BinaryV0 => { + // TODO: autodetect the version of the format, from the header let file = fs::File::open(path)?; let mut buf_reader = BufReader::new(file); self.events = crate::capnptrace::read_trace(&mut buf_reader)?; @@ -433,10 +436,13 @@ impl TraceWriter for NonStreamingTraceWriter { let json = serde_json::to_string(&self.events)?; fs::write(path, json)?; } - TraceEventsFileFormat::Binary => { + TraceEventsFileFormat::BinaryV0 => { let mut file = fs::File::create(path)?; crate::capnptrace::write_trace(&self.events, &mut file)?; } + TraceEventsFileFormat::Binary => { + unreachable!() + } } Ok(()) } else { @@ -458,12 +464,21 @@ impl TraceWriter for NonStreamingTraceWriter { pub fn create_trace_reader(format: TraceEventsFileFormat) -> Box { match format { TraceEventsFileFormat::Json => Box::new(JsonTraceReader {}), + TraceEventsFileFormat::BinaryV0 | TraceEventsFileFormat::Binary => Box::new(BinaryTraceReader {}), } } pub fn create_trace_writer(program: &str, args: &[String], format: TraceEventsFileFormat) -> Box { - let mut result = Box::new(NonStreamingTraceWriter::new(program, args)); - result.set_format(format); - result + match format { + TraceEventsFileFormat::Json | + TraceEventsFileFormat::BinaryV0 => { + let mut result = Box::new(NonStreamingTraceWriter::new(program, args)); + result.set_format(format); + result + } + TraceEventsFileFormat::Binary => { + Box::new(crate::cborzstdwriter::StreamingTraceWriter::new(program, args)) + } + } } From c43e711e514ca78bba7ee390d6756e59a9007918 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolov Date: Wed, 23 Jul 2025 02:16:10 +0300 Subject: [PATCH 07/28] feat(cbor_plus_zstd): implemented the actual writing of the CBOR compressed file --- runtime_tracing/src/cborzstdwriter.rs | 34 ++++++++++++++++++++------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/runtime_tracing/src/cborzstdwriter.rs b/runtime_tracing/src/cborzstdwriter.rs index 2791e2a..d525abe 100644 --- a/runtime_tracing/src/cborzstdwriter.rs +++ b/runtime_tracing/src/cborzstdwriter.rs @@ -3,6 +3,13 @@ use zeekstd::Encoder; use crate::{tracer::TOP_LEVEL_FUNCTION_ID, AssignCellRecord, AssignCompoundItemRecord, AssignmentRecord, CallRecord, CellValueRecord, CompoundValueRecord, FullValueRecord, FunctionId, FunctionRecord, Line, PathId, RValue, RecordEvent, ReturnRecord, StepRecord, TraceLowLevelEvent, TraceMetadata, TraceWriter, TypeId, TypeKind, TypeRecord, TypeSpecificInfo, VariableCellRecord, VariableId, NONE_TYPE_ID}; +/// The next 3 bytes are reserved/version info. +/// The header is 8 bytes in size, ensuring 64-bit alignment for the rest of the file. +const HEADERV1: &[u8] = &[ + 0xC0, 0xDE, 0x72, 0xAC, 0xE2, // The first 5 bytes identify the file as a CodeTracer file (hex l33tsp33k - C0DE72ACE2 for "CodeTracer"). + 0x01, // Indicates version 1 of the file format + 0x00, 0x00]; // Reserved, must be zero in this version. + pub struct StreamingTraceWriter<'a> { // trace metadata: workdir: PathBuf, @@ -22,8 +29,7 @@ pub struct StreamingTraceWriter<'a> { //format: TraceEventsFileFormat, trace_metadata_path: Option, trace_events_path: Option, - trace_events_file_output: Option, - trace_events_file_zstd_encoder: Option>>, + trace_events_file_zstd_encoder: Option>, trace_paths_path: Option, } @@ -46,7 +52,6 @@ impl<'a> StreamingTraceWriter<'a> { //format: TraceEventsFileFormat::Binary, trace_metadata_path: None, trace_events_path: None, - trace_events_file_output: None, trace_events_file_zstd_encoder: None, trace_paths_path: None, } @@ -62,8 +67,9 @@ impl<'a> TraceWriter for StreamingTraceWriter<'a> { fn begin_writing_trace_events(&mut self, path: &std::path::Path) -> Result<(), Box> { let pb = path.to_path_buf(); self.trace_events_path = Some(pb.clone()); - self.trace_events_file_output = Some(std::fs::File::create(pb)?); - //self.trace_events_file_zstd_encoder = Some(Encoder::new(&self.trace_events_file_output)?); + let mut file_output = std::fs::File::create(pb)?; + file_output.write_all(HEADERV1)?; + self.trace_events_file_zstd_encoder = Some(Encoder::new(file_output)?); Ok(()) } @@ -287,11 +293,17 @@ impl<'a> TraceWriter for StreamingTraceWriter<'a> { } fn add_event(&mut self, event: TraceLowLevelEvent) { - todo!() + let buf: Vec = Vec::new(); + let q = cbor4ii::serde::to_vec(buf, &event).unwrap(); + if let Some(enc) = &mut self.trace_events_file_zstd_encoder { + enc.write(&q).unwrap(); + } } fn append_events(&mut self, events: &mut Vec) { - todo!() + for e in events { + self.add_event(e.clone()); + } } fn finish_writing_trace_metadata(&mut self) -> Result<(), Box> { @@ -310,7 +322,13 @@ impl<'a> TraceWriter for StreamingTraceWriter<'a> { } fn finish_writing_trace_events(&mut self) -> Result<(), Box> { - todo!() + if let Some(enc) = self.trace_events_file_zstd_encoder.take() { + enc.finish()?; + + Ok(()) + } else { + panic!("finish_writing_trace_events() called without previous call to begin_writing_trace_events()"); + } } fn finish_writing_trace_paths(&mut self) -> Result<(), Box> { From 4a74c09b9a80e3ae43f31dd8501dea2dc5f9035e Mon Sep 17 00:00:00 2001 From: Nikolay Nikolov Date: Wed, 23 Jul 2025 02:19:10 +0300 Subject: [PATCH 08/28] feat(cbor_plus_zstd): removed commented out code --- runtime_tracing/src/cborzstdwriter.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/runtime_tracing/src/cborzstdwriter.rs b/runtime_tracing/src/cborzstdwriter.rs index d525abe..19821e1 100644 --- a/runtime_tracing/src/cborzstdwriter.rs +++ b/runtime_tracing/src/cborzstdwriter.rs @@ -15,8 +15,6 @@ pub struct StreamingTraceWriter<'a> { workdir: PathBuf, program: String, args: Vec, - // trace events - //pub events: Vec, // internal tracer state: path_list: Vec, function_list: Vec<(String, PathId, Line)>, @@ -26,7 +24,6 @@ pub struct StreamingTraceWriter<'a> { variables: HashMap, types: HashMap, - //format: TraceEventsFileFormat, trace_metadata_path: Option, trace_events_path: Option, trace_events_file_zstd_encoder: Option>, @@ -40,7 +37,6 @@ impl<'a> StreamingTraceWriter<'a> { workdir: env::current_dir().expect("can access the current dir"), program: program.to_string(), args: args.to_vec(), - //events: vec![], path_list: vec![], function_list: vec![], @@ -49,7 +45,6 @@ impl<'a> StreamingTraceWriter<'a> { variables: HashMap::new(), types: HashMap::new(), - //format: TraceEventsFileFormat::Binary, trace_metadata_path: None, trace_events_path: None, trace_events_file_zstd_encoder: None, From e5f65a2311e795988193189cb79e5fe566a9a0c6 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolov Date: Wed, 23 Jul 2025 02:26:05 +0300 Subject: [PATCH 09/28] feat(cbor_plus_zstd): updated test to test both versions of the binary format --- runtime_tracing/tests/binary_format.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/runtime_tracing/tests/binary_format.rs b/runtime_tracing/tests/binary_format.rs index ae818f4..92ea700 100644 --- a/runtime_tracing/tests/binary_format.rs +++ b/runtime_tracing/tests/binary_format.rs @@ -2,8 +2,7 @@ use runtime_tracing::{create_trace_reader, create_trace_writer, TraceEventsFileF use std::fs; use std::path::Path; -#[test] -fn test_binary_roundtrip() { +fn test_binary_roundtrip(ver: TraceEventsFileFormat) { let json_path = Path::new("tests/data/trace.json"); let mut json_reader = create_trace_reader(TraceEventsFileFormat::Json); @@ -11,7 +10,7 @@ fn test_binary_roundtrip() { let bin_path = Path::new("tests/data/trace.bin"); - let mut bin_writer = create_trace_writer("", &[], TraceEventsFileFormat::Binary); + let mut bin_writer = create_trace_writer("", &[], ver); bin_writer.begin_writing_trace_events(bin_path).unwrap(); bin_writer.append_events(&mut original.clone()); bin_writer.finish_writing_trace_events().unwrap(); @@ -26,3 +25,13 @@ fn test_binary_roundtrip() { assert_eq!(orig_json, new_json); } + +#[test] +fn test_binary_roundtrip_v0() { + test_binary_roundtrip(TraceEventsFileFormat::BinaryV0); +} + +#[test] +fn test_binary_roundtrip_v1() { + test_binary_roundtrip(TraceEventsFileFormat::Binary); +} From 794b58cf75999ae81eb213079277efc1c44db359 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolov Date: Wed, 23 Jul 2025 02:27:27 +0300 Subject: [PATCH 10/28] feat(cbor_plus_zstd): removed the load_trace_events function from NonStreamingTraceWriter --- runtime_tracing/src/tracer.rs | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/runtime_tracing/src/tracer.rs b/runtime_tracing/src/tracer.rs index 9ce383a..68eb6c2 100644 --- a/runtime_tracing/src/tracer.rs +++ b/runtime_tracing/src/tracer.rs @@ -156,23 +156,6 @@ impl NonStreamingTraceWriter { pub fn set_format(&mut self, format: TraceEventsFileFormat) { self.format = format; } - - pub fn load_trace_events(&mut self, path: &Path, format: TraceEventsFileFormat) -> Result<(), Box> { - match format { - TraceEventsFileFormat::Json => { - let json = std::fs::read_to_string(path)?; - self.events = serde_json::from_str(&json)?; - } - TraceEventsFileFormat::Binary | - TraceEventsFileFormat::BinaryV0 => { - // TODO: autodetect the version of the format, from the header - let file = fs::File::open(path)?; - let mut buf_reader = BufReader::new(file); - self.events = crate::capnptrace::read_trace(&mut buf_reader)?; - } - } - Ok(()) - } } impl TraceWriter for NonStreamingTraceWriter { From 42667beec0105f7071caaf72e5e111ab3b348bcf Mon Sep 17 00:00:00 2001 From: Nikolay Nikolov Date: Wed, 23 Jul 2025 03:50:51 +0300 Subject: [PATCH 11/28] feat(cbor_plus_zstd): initial attempt at implementing reading of CBOR + zstd. Not working, yet. :( --- runtime_tracing/Cargo.toml | 3 +- runtime_tracing/src/capnptrace.rs | 2 +- runtime_tracing/src/cborzstdwriter.rs | 2 +- runtime_tracing/src/lib.rs | 1 + runtime_tracing/src/tracer.rs | 41 +++++++++++++++++++++++---- 5 files changed, 41 insertions(+), 8 deletions(-) diff --git a/runtime_tracing/Cargo.toml b/runtime_tracing/Cargo.toml index 6de21ed..1be984b 100644 --- a/runtime_tracing/Cargo.toml +++ b/runtime_tracing/Cargo.toml @@ -21,7 +21,8 @@ serde_json = "1.0" serde_repr = "0.1" capnp = "0.21.1" zeekstd = "0.5.0" -cbor4ii = { version = "1.0.0", features = ["serde1"] } +cbor4ii = { version = "1.0.0", features = ["serde1", "use_std"] } +fscommon = "0.1.1" [build-dependencies] capnpc = "0.21.0" diff --git a/runtime_tracing/src/capnptrace.rs b/runtime_tracing/src/capnptrace.rs index 7b3a4aa..5005f77 100644 --- a/runtime_tracing/src/capnptrace.rs +++ b/runtime_tracing/src/capnptrace.rs @@ -7,7 +7,7 @@ use std::str::FromStr; /// The next 3 bytes are reserved/version info. In the initial version, they are zero. Non-zero values might /// indicate incompatible future versions. /// The header is 8 bytes in size, ensuring 64-bit alignment for the rest of the file. -const HEADER: &[u8] = &[0xC0, 0xDE, 0x72, 0xAC, 0xE2, 0x00, 0x00, 0x00]; +pub const HEADER: &[u8] = &[0xC0, 0xDE, 0x72, 0xAC, 0xE2, 0x00, 0x00, 0x00]; impl From for trace::TypeKind { fn from(item: crate::TypeKind) -> Self { diff --git a/runtime_tracing/src/cborzstdwriter.rs b/runtime_tracing/src/cborzstdwriter.rs index 19821e1..78888db 100644 --- a/runtime_tracing/src/cborzstdwriter.rs +++ b/runtime_tracing/src/cborzstdwriter.rs @@ -5,7 +5,7 @@ use crate::{tracer::TOP_LEVEL_FUNCTION_ID, AssignCellRecord, AssignCompoundItemR /// The next 3 bytes are reserved/version info. /// The header is 8 bytes in size, ensuring 64-bit alignment for the rest of the file. -const HEADERV1: &[u8] = &[ +pub const HEADERV1: &[u8] = &[ 0xC0, 0xDE, 0x72, 0xAC, 0xE2, // The first 5 bytes identify the file as a CodeTracer file (hex l33tsp33k - C0DE72ACE2 for "CodeTracer"). 0x01, // Indicates version 1 of the file format 0x00, 0x00]; // Reserved, must be zero in this version. diff --git a/runtime_tracing/src/lib.rs b/runtime_tracing/src/lib.rs index 9d62117..0aa297c 100644 --- a/runtime_tracing/src/lib.rs +++ b/runtime_tracing/src/lib.rs @@ -12,6 +12,7 @@ mod base64; mod capnptrace; mod tracer; mod types; +mod cborzstdreader; mod cborzstdwriter; pub use crate::tracer::{ diff --git a/runtime_tracing/src/tracer.rs b/runtime_tracing/src/tracer.rs index 68eb6c2..4633113 100644 --- a/runtime_tracing/src/tracer.rs +++ b/runtime_tracing/src/tracer.rs @@ -3,10 +3,12 @@ use std::collections::HashMap; use std::env; use std::error::Error; -use std::fs; -use std::io::BufReader; +use std::fs::{self, File}; +use std::io::{BufReader, Read, Seek, SeekFrom}; use std::path::{Path, PathBuf}; +use crate::capnptrace::HEADER; +use crate::cborzstdwriter::HEADERV1; use crate::types::{ AssignCellRecord, AssignCompoundItemRecord, AssignmentRecord, CallRecord, CellValueRecord, CompoundValueRecord, EventLogKind, FullValueRecord, FunctionId, FunctionRecord, Line, PassBy, PathId, Place, RecordEvent, ReturnRecord, StepRecord, TraceLowLevelEvent, TraceMetadata, TypeId, @@ -29,11 +31,40 @@ impl TraceReader for JsonTraceReader { pub struct BinaryTraceReader {} +fn detect_bin_file_version(input: &mut File) -> Result, Box> { + input.seek(SeekFrom::Start(0))?; + let mut header_buf = [0; 8]; + input.read_exact(&mut header_buf)?; + input.seek(SeekFrom::Start(0))?; + + if header_buf == HEADER { + Ok(Some(TraceEventsFileFormat::BinaryV0)) + } else if header_buf == HEADERV1 { + Ok(Some(TraceEventsFileFormat::Binary)) + } else { + Ok(None) + } +} + impl TraceReader for BinaryTraceReader { fn load_trace_events(&mut self, path: &Path) -> Result, Box> { - let file = fs::File::open(path)?; - let mut buf_reader = BufReader::new(file); - Ok(crate::capnptrace::read_trace(&mut buf_reader)?) + let mut file = fs::File::open(path)?; + let ver = detect_bin_file_version(&mut file)?; + match ver { + Some(TraceEventsFileFormat::BinaryV0) => { + let mut buf_reader = BufReader::new(file); + Ok(crate::capnptrace::read_trace(&mut buf_reader)?) + } + Some(TraceEventsFileFormat::Binary) => { + Ok(crate::cborzstdreader::read_trace(&mut file)?) + } + Some(TraceEventsFileFormat::Json) => { + unreachable!() + } + None => { + panic!("Invalid file header (wrong file format or incompatible version)"); + } + } } } From 07f5796711aaf772a30e0ce51822c9ef7bff0b41 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolov Date: Wed, 23 Jul 2025 04:34:38 +0300 Subject: [PATCH 12/28] feat(cbor_plus_zstd): CBOR + zstd reading fixed. Also, forgot to add file in previous commit. --- runtime_tracing/src/cborzstdreader.rs | 38 +++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 runtime_tracing/src/cborzstdreader.rs diff --git a/runtime_tracing/src/cborzstdreader.rs b/runtime_tracing/src/cborzstdreader.rs new file mode 100644 index 0000000..38dd390 --- /dev/null +++ b/runtime_tracing/src/cborzstdreader.rs @@ -0,0 +1,38 @@ +use std::io::{self, BufRead, BufReader, Read, Seek, Write}; + +use fscommon::StreamSlice; +use zeekstd::{DecodeOptions, Decoder}; + +use crate::{cborzstdwriter::HEADERV1, TraceLowLevelEvent}; + +fn is_at_eof(reader: &mut R) -> io::Result { + let buffer = reader.fill_buf()?; + Ok(buffer.is_empty()) +} + +pub fn read_trace(input: &mut (impl Read + Write + Seek)) -> Result, Box> { + let end_pos = input.seek(io::SeekFrom::End(0))?; + input.seek(io::SeekFrom::Start(0))?; + + let mut header_buf = [0; 8]; + let mut buf_reader = BufReader::new(&mut *input); + buf_reader.read_exact(&mut header_buf)?; + if header_buf != HEADERV1 { + panic!("Invalid file header (wrong file format or incompatible version)"); + } + + input.seek(io::SeekFrom::Start(0))?; + let input2 = StreamSlice::new(&mut *input, 8, end_pos)?; + + let decoder = Decoder::new(input2)?; + let mut buf_reader = BufReader::new(decoder); + + let mut result: Vec = vec![]; + + while !is_at_eof(&mut buf_reader)? { + let obj = cbor4ii::serde::from_reader::(&mut buf_reader)?; + result.push(obj); + } + + Ok(result) +} From 8c8842c4bae0591aa83a53901fb8b4ef56f8e60e Mon Sep 17 00:00:00 2001 From: Nikolay Nikolov Date: Wed, 23 Jul 2025 04:40:16 +0300 Subject: [PATCH 13/28] feat(cbor_plus_zstd): fix tests --- runtime_tracing/tests/binary_format.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/runtime_tracing/tests/binary_format.rs b/runtime_tracing/tests/binary_format.rs index 92ea700..d4c02f8 100644 --- a/runtime_tracing/tests/binary_format.rs +++ b/runtime_tracing/tests/binary_format.rs @@ -2,13 +2,14 @@ use runtime_tracing::{create_trace_reader, create_trace_writer, TraceEventsFileF use std::fs; use std::path::Path; -fn test_binary_roundtrip(ver: TraceEventsFileFormat) { +fn test_binary_roundtrip(ver: TraceEventsFileFormat, binfile: &str) { let json_path = Path::new("tests/data/trace.json"); let mut json_reader = create_trace_reader(TraceEventsFileFormat::Json); let original = json_reader.load_trace_events(json_path).unwrap(); - let bin_path = Path::new("tests/data/trace.bin"); + let bin_path_str = format!("tests/data/{}", binfile); + let bin_path = Path::new(&bin_path_str); let mut bin_writer = create_trace_writer("", &[], ver); bin_writer.begin_writing_trace_events(bin_path).unwrap(); @@ -28,10 +29,10 @@ fn test_binary_roundtrip(ver: TraceEventsFileFormat) { #[test] fn test_binary_roundtrip_v0() { - test_binary_roundtrip(TraceEventsFileFormat::BinaryV0); + test_binary_roundtrip(TraceEventsFileFormat::BinaryV0, "trace.v0.bin"); } #[test] fn test_binary_roundtrip_v1() { - test_binary_roundtrip(TraceEventsFileFormat::Binary); + test_binary_roundtrip(TraceEventsFileFormat::Binary, "trace.v1.bin"); } From 96ec64b20c67438b6b6861d9a96f8fd7355a94c9 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolov Date: Wed, 23 Jul 2025 04:43:19 +0300 Subject: [PATCH 14/28] feat(cbor_plus_zstd): removed unused use --- runtime_tracing/src/cborzstdreader.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime_tracing/src/cborzstdreader.rs b/runtime_tracing/src/cborzstdreader.rs index 38dd390..fe7eedf 100644 --- a/runtime_tracing/src/cborzstdreader.rs +++ b/runtime_tracing/src/cborzstdreader.rs @@ -1,7 +1,7 @@ use std::io::{self, BufRead, BufReader, Read, Seek, Write}; use fscommon::StreamSlice; -use zeekstd::{DecodeOptions, Decoder}; +use zeekstd::Decoder; use crate::{cborzstdwriter::HEADERV1, TraceLowLevelEvent}; From b02cd9ae7253d44377d5b223eec80372949690ae Mon Sep 17 00:00:00 2001 From: Nikolay Nikolov Date: Wed, 23 Jul 2025 04:43:55 +0300 Subject: [PATCH 15/28] feat(cbor_plus_zstd): removed unused test code --- runtime_tracing/src/cborzstdwriter.rs | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/runtime_tracing/src/cborzstdwriter.rs b/runtime_tracing/src/cborzstdwriter.rs index 78888db..db89ece 100644 --- a/runtime_tracing/src/cborzstdwriter.rs +++ b/runtime_tracing/src/cborzstdwriter.rs @@ -336,26 +336,3 @@ impl<'a> TraceWriter for StreamingTraceWriter<'a> { } } } - -fn test_zstd() { - let output = std::fs::File::create("tralala.zstd").unwrap(); - let mut encoder = Encoder::new(output).unwrap(); - - let event: TraceLowLevelEvent = TraceLowLevelEvent::Path(PathBuf::new()); - - let buf: Vec = Vec::new(); - let q = cbor4ii::serde::to_vec(buf, &event).unwrap(); - encoder.write(&q).unwrap(); - - let f = encoder.finish().unwrap(); -} - -#[cfg(test)] -mod tests { - use crate::cborzstdwriter::test_zstd; - - #[test] - fn run_zstd_test() { - test_zstd(); - } -} From 5970c197eeb93f0488c1e679dd4d50d5b12e627f Mon Sep 17 00:00:00 2001 From: Nikolay Nikolov Date: Thu, 24 Jul 2025 12:50:53 +0300 Subject: [PATCH 16/28] feat(cbor_plus_zstd): refactored trace writer code, to avoid copy duplication --- runtime_tracing/src/abstracttracewriter.rs | 243 ++++++++++++ runtime_tracing/src/cborzstdwriter.rs | 296 ++------------ runtime_tracing/src/lib.rs | 1 + runtime_tracing/src/tracer.rs | 431 +++++++-------------- runtime_tracing/tests/binary_format.rs | 4 +- runtime_tracing_cli/src/main.rs | 4 +- 6 files changed, 434 insertions(+), 545 deletions(-) create mode 100644 runtime_tracing/src/abstracttracewriter.rs diff --git a/runtime_tracing/src/abstracttracewriter.rs b/runtime_tracing/src/abstracttracewriter.rs new file mode 100644 index 0000000..c2ad180 --- /dev/null +++ b/runtime_tracing/src/abstracttracewriter.rs @@ -0,0 +1,243 @@ +use std::{collections::HashMap, path::PathBuf}; + +use crate::{tracer::TOP_LEVEL_FUNCTION_ID, AssignCellRecord, AssignCompoundItemRecord, AssignmentRecord, CallRecord, CellValueRecord, CompoundValueRecord, FullValueRecord, FunctionId, FunctionRecord, Line, PathId, RValue, RecordEvent, ReturnRecord, StepRecord, TraceLowLevelEvent, TypeId, TypeKind, TypeRecord, TypeSpecificInfo, VariableCellRecord, VariableId, NONE_TYPE_ID}; + +pub struct AbstractTraceWriterData { + // trace metadata: + pub workdir: PathBuf, + pub program: String, + pub args: Vec, + // internal tracer state: + pub path_list: Vec, + pub function_list: Vec<(String, PathId, Line)>, + + pub paths: HashMap, + pub functions: HashMap, + pub variables: HashMap, + pub types: HashMap, +} + +pub trait AbstractTraceWriter { + fn get_data(&self) -> &AbstractTraceWriterData; + fn get_mut_data(&mut self) -> &mut AbstractTraceWriterData; + + fn add_event(&mut self, event: TraceLowLevelEvent); + fn append_events(&mut self, events: &mut Vec); + + fn start(&mut self, path: &std::path::Path, line: Line) { + let function_id = self.ensure_function_id("", path, line); + self.register_call(function_id, vec![]); + assert!(function_id == TOP_LEVEL_FUNCTION_ID); + + // probably we let the user choose, as different languages have + // different base types/names + // assert!(EXAMPLE_INT_TYPE_ID == self.load_type_id(TypeKind::Int, "Int")); + // assert!(EXAMPLE_FLOAT_TYPE_ID == self.load_type_id(TypeKind::Float, "Float")); + // assert!(EXAMPLE_BOOL_TYPE_ID == self.load_type_id(TypeKind::Bool, "Bool")); + // assert!(EXAMPLE_STRING_TYPE_ID == self.load_type_id(TypeKind::Bool, "String")); + assert!(NONE_TYPE_ID == self.ensure_type_id(TypeKind::None, "None")); + } + + fn ensure_path_id(&mut self, path: &std::path::Path) -> PathId { + if !self.get_data().paths.contains_key(path) { + let mut_data = self.get_mut_data(); + mut_data.paths.insert(path.to_path_buf(), PathId(mut_data.paths.len())); + self.register_path(path); + } + *self.get_data().paths.get(path).unwrap() + } + + fn ensure_function_id(&mut self, function_name: &str, path: &std::path::Path, line: Line) -> FunctionId { + if !self.get_data().functions.contains_key(function_name) { + // same function names for different path line? TODO + let mut_data = self.get_mut_data(); + mut_data.functions.insert(function_name.to_string(), FunctionId(mut_data.functions.len())); + self.register_function(function_name, path, line); + } + *self.get_data().functions.get(function_name).unwrap() + } + + fn ensure_type_id(&mut self, kind: crate::TypeKind, lang_type: &str) -> TypeId { + let typ = self.to_raw_type(kind, lang_type); + self.ensure_raw_type_id(typ) + } + + fn ensure_raw_type_id(&mut self, typ: crate::TypeRecord) -> TypeId { + if !self.get_data().types.contains_key(&typ.lang_type) { + let mut_data = self.get_mut_data(); + mut_data.types.insert(typ.lang_type.clone(), TypeId(mut_data.types.len())); + self.register_raw_type(typ.clone()); + } + *self.get_data().types.get(&typ.lang_type).unwrap() + } + + fn ensure_variable_id(&mut self, variable_name: &str) -> VariableId { + if !self.get_data().variables.contains_key(variable_name) { + let mut_data = self.get_mut_data(); + mut_data.variables.insert(variable_name.to_string(), VariableId(mut_data.variables.len())); + self.register_variable_name(variable_name); + } + *self.get_data().variables.get(variable_name).unwrap() + } + + fn register_path(&mut self, path: &std::path::Path) { + self.get_mut_data().path_list.push(path.to_path_buf()); + self.add_event(TraceLowLevelEvent::Path(path.to_path_buf())); + } + + fn register_function(&mut self, name: &str, path: &std::path::Path, line: Line) { + let path_id = self.ensure_path_id(path); + self.get_mut_data().function_list.push((name.to_string(), path_id, line)); + self.add_event(TraceLowLevelEvent::Function(FunctionRecord { + name: name.to_string(), + path_id, + line, + })); + } + + fn register_step(&mut self, path: &std::path::Path, line: Line) { + let path_id = self.ensure_path_id(path); + self.add_event(TraceLowLevelEvent::Step(StepRecord { path_id, line })); + } + + fn register_call(&mut self, function_id: FunctionId, args: Vec) { + // register a step for each call, the backend expects this for + // non-toplevel calls, so + // we ensure it directly from register_call + if function_id != TOP_LEVEL_FUNCTION_ID { + for arg in &args { + self.register_full_value(arg.variable_id, arg.value.clone()); + } + let function = &self.get_data().function_list[function_id.0]; + self.add_event(TraceLowLevelEvent::Step(StepRecord { + path_id: function.1, + line: function.2, + })); + } + // the actual call event: + self.add_event(TraceLowLevelEvent::Call(CallRecord { function_id, args })); + } + + fn arg(&mut self, name: &str, value: crate::ValueRecord) -> FullValueRecord { + let variable_id = self.ensure_variable_id(name); + FullValueRecord { variable_id, value } + } + + fn register_return(&mut self, return_value: crate::ValueRecord) { + self.add_event(TraceLowLevelEvent::Return(ReturnRecord { return_value })); + } + + fn register_special_event(&mut self, kind: crate::EventLogKind, content: &str) { + self.add_event(TraceLowLevelEvent::Event(RecordEvent { + kind, + metadata: "".to_string(), + content: content.to_string(), + })); + } + + fn to_raw_type(&self, kind: crate::TypeKind, lang_type: &str) -> crate::TypeRecord { + TypeRecord { + kind, + lang_type: lang_type.to_string(), + specific_info: TypeSpecificInfo::None, + } + } + + fn register_type(&mut self, kind: crate::TypeKind, lang_type: &str) { + let typ = self.to_raw_type(kind, lang_type); + self.add_event(TraceLowLevelEvent::Type(typ)); + } + + fn register_raw_type(&mut self, typ: crate::TypeRecord) { + self.add_event(TraceLowLevelEvent::Type(typ)); + } + + fn register_asm(&mut self, instructions: &[String]) { + self.add_event(TraceLowLevelEvent::Asm(instructions.to_vec())); + } + + fn register_variable_with_full_value(&mut self, name: &str, value: crate::ValueRecord) { + let variable_id = self.ensure_variable_id(name); + self.register_full_value(variable_id, value); + } + + fn register_variable_name(&mut self, variable_name: &str) { + self.add_event(TraceLowLevelEvent::VariableName(variable_name.to_string())); + } + + fn register_full_value(&mut self, variable_id: VariableId, value: crate::ValueRecord) { + self.add_event(TraceLowLevelEvent::Value(FullValueRecord { variable_id, value })); + } + + fn register_compound_value(&mut self, place: crate::Place, value: crate::ValueRecord) { + self.add_event(TraceLowLevelEvent::CompoundValue(CompoundValueRecord { place, value })); + } + + fn register_cell_value(&mut self, place: crate::Place, value: crate::ValueRecord) { + self.add_event(TraceLowLevelEvent::CellValue(CellValueRecord { place, value })); + } + + fn assign_compound_item(&mut self, place: crate::Place, index: usize, item_place: crate::Place) { + self.add_event(TraceLowLevelEvent::AssignCompoundItem(AssignCompoundItemRecord { + place, + index, + item_place, + })); + } + + fn assign_cell(&mut self, place: crate::Place, new_value: crate::ValueRecord) { + self.add_event(TraceLowLevelEvent::AssignCell(AssignCellRecord { place, new_value })); + } + + fn register_variable(&mut self, variable_name: &str, place: crate::Place) { + let variable_id = self.ensure_variable_id(variable_name); + self.add_event(TraceLowLevelEvent::VariableCell(VariableCellRecord { variable_id, place })); + } + + fn drop_variable(&mut self, variable_name: &str) { + let variable_id = self.ensure_variable_id(variable_name); + self.add_event(TraceLowLevelEvent::DropVariable(variable_id)); + } + + // history event helpers + fn assign(&mut self, variable_name: &str, rvalue: crate::RValue, pass_by: crate::PassBy) { + let variable_id = self.ensure_variable_id(variable_name); + self.add_event(TraceLowLevelEvent::Assignment(AssignmentRecord { + to: variable_id, + from: rvalue, + pass_by, + })); + } + + fn bind_variable(&mut self, variable_name: &str, place: crate::Place) { + let variable_id = self.ensure_variable_id(variable_name); + self.add_event(TraceLowLevelEvent::BindVariable(crate::BindVariableRecord { variable_id, place })); + } + + fn drop_variables(&mut self, variable_names: &[String]) { + let variable_ids: Vec = variable_names + .to_vec() + .iter() + .map(|variable_name| self.ensure_variable_id(variable_name)) + .collect(); + self.add_event(TraceLowLevelEvent::DropVariables(variable_ids)) + } + + fn simple_rvalue(&mut self, variable_name: &str) -> crate::RValue { + let variable_id = self.ensure_variable_id(variable_name); + RValue::Simple(variable_id) + } + + fn compound_rvalue(&mut self, variable_dependencies: &[String]) -> crate::RValue { + let variable_ids: Vec = variable_dependencies + .to_vec() + .iter() + .map(|variable_dependency| self.ensure_variable_id(variable_dependency)) + .collect(); + RValue::Compound(variable_ids) + } + + fn drop_last_step(&mut self) { + self.add_event(TraceLowLevelEvent::DropLastStep); + } +} diff --git a/runtime_tracing/src/cborzstdwriter.rs b/runtime_tracing/src/cborzstdwriter.rs index db89ece..4b65f81 100644 --- a/runtime_tracing/src/cborzstdwriter.rs +++ b/runtime_tracing/src/cborzstdwriter.rs @@ -1,7 +1,7 @@ use std::{collections::HashMap, env, fs::{self, File}, io::Write, path::PathBuf}; use zeekstd::Encoder; -use crate::{tracer::TOP_LEVEL_FUNCTION_ID, AssignCellRecord, AssignCompoundItemRecord, AssignmentRecord, CallRecord, CellValueRecord, CompoundValueRecord, FullValueRecord, FunctionId, FunctionRecord, Line, PathId, RValue, RecordEvent, ReturnRecord, StepRecord, TraceLowLevelEvent, TraceMetadata, TraceWriter, TypeId, TypeKind, TypeRecord, TypeSpecificInfo, VariableCellRecord, VariableId, NONE_TYPE_ID}; +use crate::{abstracttracewriter::{AbstractTraceWriter, AbstractTraceWriterData}, TraceLowLevelEvent, TraceMetadata, TraceWriter}; /// The next 3 bytes are reserved/version info. /// The header is 8 bytes in size, ensuring 64-bit alignment for the rest of the file. @@ -11,18 +11,7 @@ pub const HEADERV1: &[u8] = &[ 0x00, 0x00]; // Reserved, must be zero in this version. pub struct StreamingTraceWriter<'a> { - // trace metadata: - workdir: PathBuf, - program: String, - args: Vec, - // internal tracer state: - path_list: Vec, - function_list: Vec<(String, PathId, Line)>, - - paths: HashMap, - functions: HashMap, - variables: HashMap, - types: HashMap, + base: AbstractTraceWriterData, trace_metadata_path: Option, trace_events_path: Option, @@ -34,16 +23,18 @@ impl<'a> StreamingTraceWriter<'a> { /// Create a new tracer instance for the given program and arguments. pub fn new(program: &str, args: &[String]) -> Self { StreamingTraceWriter { - workdir: env::current_dir().expect("can access the current dir"), - program: program.to_string(), - args: args.to_vec(), - - path_list: vec![], - function_list: vec![], - paths: HashMap::new(), - functions: HashMap::new(), - variables: HashMap::new(), - types: HashMap::new(), + base: AbstractTraceWriterData { + workdir: env::current_dir().expect("can access the current dir"), + program: program.to_string(), + args: args.to_vec(), + + path_list: vec![], + function_list: vec![], + paths: HashMap::new(), + functions: HashMap::new(), + variables: HashMap::new(), + types: HashMap::new(), + }, trace_metadata_path: None, trace_events_path: None, @@ -53,6 +44,30 @@ impl<'a> StreamingTraceWriter<'a> { } } +impl<'a> AbstractTraceWriter for StreamingTraceWriter<'a> { + fn get_data(&self) -> &AbstractTraceWriterData { + &self.base + } + + fn get_mut_data(&mut self) -> &mut AbstractTraceWriterData { + &mut self.base + } + + fn add_event(&mut self, event: TraceLowLevelEvent) { + let buf: Vec = Vec::new(); + let q = cbor4ii::serde::to_vec(buf, &event).unwrap(); + if let Some(enc) = &mut self.trace_events_file_zstd_encoder { + enc.write(&q).unwrap(); + } + } + + fn append_events(&mut self, events: &mut Vec) { + for e in events { + AbstractTraceWriter::add_event(self, e.clone()); + } + } +} + impl<'a> TraceWriter for StreamingTraceWriter<'a> { fn begin_writing_trace_metadata(&mut self, path: &std::path::Path) -> Result<(), Box> { self.trace_metadata_path = Some(path.to_path_buf()); @@ -74,239 +89,12 @@ impl<'a> TraceWriter for StreamingTraceWriter<'a> { Ok(()) } - fn start(&mut self, path: &std::path::Path, line: Line) { - let function_id = self.ensure_function_id("", path, line); - self.register_call(function_id, vec![]); - assert!(function_id == TOP_LEVEL_FUNCTION_ID); - - // probably we let the user choose, as different languages have - // different base types/names - // assert!(EXAMPLE_INT_TYPE_ID == self.load_type_id(TypeKind::Int, "Int")); - // assert!(EXAMPLE_FLOAT_TYPE_ID == self.load_type_id(TypeKind::Float, "Float")); - // assert!(EXAMPLE_BOOL_TYPE_ID == self.load_type_id(TypeKind::Bool, "Bool")); - // assert!(EXAMPLE_STRING_TYPE_ID == self.load_type_id(TypeKind::Bool, "String")); - assert!(NONE_TYPE_ID == self.ensure_type_id(TypeKind::None, "None")); - } - - fn ensure_path_id(&mut self, path: &std::path::Path) -> PathId { - if !self.paths.contains_key(path) { - self.paths.insert(path.to_path_buf(), PathId(self.paths.len())); - self.register_path(path); - } - *self.paths.get(path).unwrap() - } - - fn ensure_function_id(&mut self, function_name: &str, path: &std::path::Path, line: Line) -> FunctionId { - if !self.functions.contains_key(function_name) { - // same function names for different path line? TODO - self.functions.insert(function_name.to_string(), FunctionId(self.functions.len())); - self.register_function(function_name, path, line); - } - *self.functions.get(function_name).unwrap() - } - - fn ensure_type_id(&mut self, kind: crate::TypeKind, lang_type: &str) -> TypeId { - let typ = self.to_raw_type(kind, lang_type); - self.ensure_raw_type_id(typ) - } - - fn ensure_raw_type_id(&mut self, typ: crate::TypeRecord) -> TypeId { - if !self.types.contains_key(&typ.lang_type) { - self.types.insert(typ.lang_type.clone(), TypeId(self.types.len())); - self.register_raw_type(typ.clone()); - } - *self.types.get(&typ.lang_type).unwrap() - } - - fn ensure_variable_id(&mut self, variable_name: &str) -> VariableId { - if !self.variables.contains_key(variable_name) { - self.variables.insert(variable_name.to_string(), VariableId(self.variables.len())); - self.register_variable_name(variable_name); - } - *self.variables.get(variable_name).unwrap() - } - - fn register_path(&mut self, path: &std::path::Path) { - self.path_list.push(path.to_path_buf()); - self.add_event(TraceLowLevelEvent::Path(path.to_path_buf())); - } - - fn register_function(&mut self, name: &str, path: &std::path::Path, line: Line) { - let path_id = self.ensure_path_id(path); - self.function_list.push((name.to_string(), path_id, line)); - self.add_event(TraceLowLevelEvent::Function(FunctionRecord { - name: name.to_string(), - path_id, - line, - })); - } - - fn register_step(&mut self, path: &std::path::Path, line: Line) { - let path_id = self.ensure_path_id(path); - self.add_event(TraceLowLevelEvent::Step(StepRecord { path_id, line })); - } - - fn register_call(&mut self, function_id: FunctionId, args: Vec) { - // register a step for each call, the backend expects this for - // non-toplevel calls, so - // we ensure it directly from register_call - if function_id != TOP_LEVEL_FUNCTION_ID { - for arg in &args { - self.register_full_value(arg.variable_id, arg.value.clone()); - } - let function = &self.function_list[function_id.0]; - self.add_event(TraceLowLevelEvent::Step(StepRecord { - path_id: function.1, - line: function.2, - })); - } - // the actual call event: - self.add_event(TraceLowLevelEvent::Call(CallRecord { function_id, args })); - } - - fn arg(&mut self, name: &str, value: crate::ValueRecord) -> FullValueRecord { - let variable_id = self.ensure_variable_id(name); - FullValueRecord { variable_id, value } - } - - fn register_return(&mut self, return_value: crate::ValueRecord) { - self.add_event(TraceLowLevelEvent::Return(ReturnRecord { return_value })); - } - - fn register_special_event(&mut self, kind: crate::EventLogKind, content: &str) { - self.add_event(TraceLowLevelEvent::Event(RecordEvent { - kind, - metadata: "".to_string(), - content: content.to_string(), - })); - } - - fn to_raw_type(&self, kind: crate::TypeKind, lang_type: &str) -> crate::TypeRecord { - TypeRecord { - kind, - lang_type: lang_type.to_string(), - specific_info: TypeSpecificInfo::None, - } - } - - fn register_type(&mut self, kind: crate::TypeKind, lang_type: &str) { - let typ = self.to_raw_type(kind, lang_type); - self.add_event(TraceLowLevelEvent::Type(typ)); - } - - fn register_raw_type(&mut self, typ: crate::TypeRecord) { - self.add_event(TraceLowLevelEvent::Type(typ)); - } - - fn register_asm(&mut self, instructions: &[String]) { - self.add_event(TraceLowLevelEvent::Asm(instructions.to_vec())); - } - - fn register_variable_with_full_value(&mut self, name: &str, value: crate::ValueRecord) { - let variable_id = self.ensure_variable_id(name); - self.register_full_value(variable_id, value); - } - - fn register_variable_name(&mut self, variable_name: &str) { - self.add_event(TraceLowLevelEvent::VariableName(variable_name.to_string())); - } - - fn register_full_value(&mut self, variable_id: VariableId, value: crate::ValueRecord) { - self.add_event(TraceLowLevelEvent::Value(FullValueRecord { variable_id, value })); - } - - fn register_compound_value(&mut self, place: crate::Place, value: crate::ValueRecord) { - self.add_event(TraceLowLevelEvent::CompoundValue(CompoundValueRecord { place, value })); - } - - fn register_cell_value(&mut self, place: crate::Place, value: crate::ValueRecord) { - self.add_event(TraceLowLevelEvent::CellValue(CellValueRecord { place, value })); - } - - fn assign_compound_item(&mut self, place: crate::Place, index: usize, item_place: crate::Place) { - self.add_event(TraceLowLevelEvent::AssignCompoundItem(AssignCompoundItemRecord { - place, - index, - item_place, - })); - } - - fn assign_cell(&mut self, place: crate::Place, new_value: crate::ValueRecord) { - self.add_event(TraceLowLevelEvent::AssignCell(AssignCellRecord { place, new_value })); - } - - fn register_variable(&mut self, variable_name: &str, place: crate::Place) { - let variable_id = self.ensure_variable_id(variable_name); - self.add_event(TraceLowLevelEvent::VariableCell(VariableCellRecord { variable_id, place })); - } - - fn drop_variable(&mut self, variable_name: &str) { - let variable_id = self.ensure_variable_id(variable_name); - self.add_event(TraceLowLevelEvent::DropVariable(variable_id)); - } - - // history event helpers - fn assign(&mut self, variable_name: &str, rvalue: crate::RValue, pass_by: crate::PassBy) { - let variable_id = self.ensure_variable_id(variable_name); - self.add_event(TraceLowLevelEvent::Assignment(AssignmentRecord { - to: variable_id, - from: rvalue, - pass_by, - })); - } - - fn bind_variable(&mut self, variable_name: &str, place: crate::Place) { - let variable_id = self.ensure_variable_id(variable_name); - self.add_event(TraceLowLevelEvent::BindVariable(crate::BindVariableRecord { variable_id, place })); - } - - fn drop_variables(&mut self, variable_names: &[String]) { - let variable_ids: Vec = variable_names - .to_vec() - .iter() - .map(|variable_name| self.ensure_variable_id(variable_name)) - .collect(); - self.add_event(TraceLowLevelEvent::DropVariables(variable_ids)) - } - - fn simple_rvalue(&mut self, variable_name: &str) -> crate::RValue { - let variable_id = self.ensure_variable_id(variable_name); - RValue::Simple(variable_id) - } - - fn compound_rvalue(&mut self, variable_dependencies: &[String]) -> crate::RValue { - let variable_ids: Vec = variable_dependencies - .to_vec() - .iter() - .map(|variable_dependency| self.ensure_variable_id(variable_dependency)) - .collect(); - RValue::Compound(variable_ids) - } - - fn drop_last_step(&mut self) { - self.add_event(TraceLowLevelEvent::DropLastStep); - } - - fn add_event(&mut self, event: TraceLowLevelEvent) { - let buf: Vec = Vec::new(); - let q = cbor4ii::serde::to_vec(buf, &event).unwrap(); - if let Some(enc) = &mut self.trace_events_file_zstd_encoder { - enc.write(&q).unwrap(); - } - } - - fn append_events(&mut self, events: &mut Vec) { - for e in events { - self.add_event(e.clone()); - } - } - fn finish_writing_trace_metadata(&mut self) -> Result<(), Box> { if let Some(path) = &self.trace_metadata_path { let trace_metadata = TraceMetadata { - program: self.program.clone(), - args: self.args.clone(), - workdir: self.workdir.clone(), + program: self.get_data().program.clone(), + args: self.get_data().args.clone(), + workdir: self.get_data().workdir.clone(), }; let json = serde_json::to_string(&trace_metadata)?; fs::write(path, json)?; @@ -328,7 +116,7 @@ impl<'a> TraceWriter for StreamingTraceWriter<'a> { fn finish_writing_trace_paths(&mut self) -> Result<(), Box> { if let Some(path) = &self.trace_paths_path { - let json = serde_json::to_string(&self.path_list)?; + let json = serde_json::to_string(&self.get_data().path_list)?; fs::write(path, json)?; Ok(()) } else { diff --git a/runtime_tracing/src/lib.rs b/runtime_tracing/src/lib.rs index 0aa297c..c7defbd 100644 --- a/runtime_tracing/src/lib.rs +++ b/runtime_tracing/src/lib.rs @@ -14,6 +14,7 @@ mod tracer; mod types; mod cborzstdreader; mod cborzstdwriter; +mod abstracttracewriter; pub use crate::tracer::{ create_trace_reader, create_trace_writer, NonStreamingTraceWriter, TraceEventsFileFormat, TraceReader, TraceWriter, NONE_TYPE_ID, NONE_VALUE, diff --git a/runtime_tracing/src/tracer.rs b/runtime_tracing/src/tracer.rs index 4633113..d174e5e 100644 --- a/runtime_tracing/src/tracer.rs +++ b/runtime_tracing/src/tracer.rs @@ -9,10 +9,11 @@ use std::path::{Path, PathBuf}; use crate::capnptrace::HEADER; use crate::cborzstdwriter::HEADERV1; +use crate::abstracttracewriter::{AbstractTraceWriter, AbstractTraceWriterData}; use crate::types::{ - AssignCellRecord, AssignCompoundItemRecord, AssignmentRecord, CallRecord, CellValueRecord, CompoundValueRecord, EventLogKind, FullValueRecord, - FunctionId, FunctionRecord, Line, PassBy, PathId, Place, RecordEvent, ReturnRecord, StepRecord, TraceLowLevelEvent, TraceMetadata, TypeId, - TypeKind, TypeRecord, TypeSpecificInfo, ValueRecord, VariableCellRecord, VariableId, + EventLogKind, FullValueRecord, + FunctionId, Line, PassBy, PathId, Place, TraceLowLevelEvent, TraceMetadata, TypeId, + TypeKind, TypeRecord, ValueRecord, VariableId, }; use crate::RValue; @@ -68,48 +69,116 @@ impl TraceReader for BinaryTraceReader { } } -pub trait TraceWriter { +pub trait TraceWriter: AbstractTraceWriter { fn begin_writing_trace_metadata(&mut self, path: &Path) -> Result<(), Box>; fn begin_writing_trace_events(&mut self, path: &Path) -> Result<(), Box>; fn begin_writing_trace_paths(&mut self, path: &Path) -> Result<(), Box>; - fn start(&mut self, path: &Path, line: Line); - fn ensure_path_id(&mut self, path: &Path) -> PathId; - fn ensure_function_id(&mut self, function_name: &str, path: &Path, line: Line) -> FunctionId; - fn ensure_type_id(&mut self, kind: TypeKind, lang_type: &str) -> TypeId; - fn ensure_raw_type_id(&mut self, typ: TypeRecord) -> TypeId; - fn ensure_variable_id(&mut self, variable_name: &str) -> VariableId; - fn register_path(&mut self, path: &Path); - fn register_function(&mut self, name: &str, path: &Path, line: Line); - fn register_step(&mut self, path: &Path, line: Line); - fn register_call(&mut self, function_id: FunctionId, args: Vec); - fn arg(&mut self, name: &str, value: ValueRecord) -> FullValueRecord; - fn register_return(&mut self, return_value: ValueRecord); + fn start(&mut self, path: &Path, line: Line) { + AbstractTraceWriter::start(self, path, line) + } + fn ensure_path_id(&mut self, path: &Path) -> PathId { + AbstractTraceWriter::ensure_path_id(self, path) + } + fn ensure_function_id(&mut self, function_name: &str, path: &Path, line: Line) -> FunctionId { + AbstractTraceWriter::ensure_function_id(self, function_name, path, line) + } + fn ensure_type_id(&mut self, kind: TypeKind, lang_type: &str) -> TypeId { + AbstractTraceWriter::ensure_type_id(self, kind, lang_type) + } + fn ensure_raw_type_id(&mut self, typ: TypeRecord) -> TypeId { + AbstractTraceWriter::ensure_raw_type_id(self, typ) + } + fn ensure_variable_id(&mut self, variable_name: &str) -> VariableId { + AbstractTraceWriter::ensure_variable_id(self, variable_name) + } + fn register_path(&mut self, path: &Path) { + AbstractTraceWriter::register_path(self, path) + } + fn register_function(&mut self, name: &str, path: &Path, line: Line) { + AbstractTraceWriter::register_function(self, name, path, line) + } + fn register_step(&mut self, path: &Path, line: Line) { + AbstractTraceWriter::register_step(self, path, line) + } + fn register_call(&mut self, function_id: FunctionId, args: Vec) { + AbstractTraceWriter::register_call(self, function_id, args) + } + fn arg(&mut self, name: &str, value: ValueRecord) -> FullValueRecord { + AbstractTraceWriter::arg(self, name, value) + } + fn register_return(&mut self, return_value: ValueRecord) { + AbstractTraceWriter::register_return(self, return_value) + } // TODO: add metadata arg - fn register_special_event(&mut self, kind: EventLogKind, content: &str); - fn to_raw_type(&self, kind: TypeKind, lang_type: &str) -> TypeRecord; - fn register_type(&mut self, kind: TypeKind, lang_type: &str); - fn register_raw_type(&mut self, typ: TypeRecord); - fn register_asm(&mut self, instructions: &[String]); - fn register_variable_with_full_value(&mut self, name: &str, value: ValueRecord); - fn register_variable_name(&mut self, variable_name: &str); - fn register_full_value(&mut self, variable_id: VariableId, value: ValueRecord); - fn register_compound_value(&mut self, place: Place, value: ValueRecord); - fn register_cell_value(&mut self, place: Place, value: ValueRecord); - fn assign_compound_item(&mut self, place: Place, index: usize, item_place: Place); - fn assign_cell(&mut self, place: Place, new_value: ValueRecord); - fn register_variable(&mut self, variable_name: &str, place: Place); - fn drop_variable(&mut self, variable_name: &str); + fn register_special_event(&mut self, kind: EventLogKind, content: &str) { + AbstractTraceWriter::register_special_event(self, kind, content) + } + fn to_raw_type(&self, kind: TypeKind, lang_type: &str) -> TypeRecord { + AbstractTraceWriter::to_raw_type(self, kind, lang_type) + } + fn register_type(&mut self, kind: TypeKind, lang_type: &str) { + AbstractTraceWriter::register_type(self, kind, lang_type) + } + fn register_raw_type(&mut self, typ: TypeRecord) { + AbstractTraceWriter::register_raw_type(self, typ) + } + fn register_asm(&mut self, instructions: &[String]) { + AbstractTraceWriter::register_asm(self, instructions) + } + fn register_variable_with_full_value(&mut self, name: &str, value: ValueRecord) { + AbstractTraceWriter::register_variable_with_full_value(self, name, value) + } + fn register_variable_name(&mut self, variable_name: &str) { + AbstractTraceWriter::register_variable_name(self, variable_name) + } + fn register_full_value(&mut self, variable_id: VariableId, value: ValueRecord) { + AbstractTraceWriter::register_full_value(self, variable_id, value) + } + fn register_compound_value(&mut self, place: Place, value: ValueRecord) { + AbstractTraceWriter::register_compound_value(self, place, value) + } + fn register_cell_value(&mut self, place: Place, value: ValueRecord) { + AbstractTraceWriter::register_cell_value(self, place, value) + } + fn assign_compound_item(&mut self, place: Place, index: usize, item_place: Place) { + AbstractTraceWriter::assign_compound_item(self, place, index, item_place) + } + fn assign_cell(&mut self, place: Place, new_value: ValueRecord) { + AbstractTraceWriter::assign_cell(self, place, new_value) + } + fn register_variable(&mut self, variable_name: &str, place: Place) { + AbstractTraceWriter::register_variable(self, variable_name, place) + } + fn drop_variable(&mut self, variable_name: &str) { + AbstractTraceWriter::drop_variable(self, variable_name) + } // history event helpers - fn assign(&mut self, variable_name: &str, rvalue: RValue, pass_by: PassBy); - fn bind_variable(&mut self, variable_name: &str, place: Place); - fn drop_variables(&mut self, variable_names: &[String]); - fn simple_rvalue(&mut self, variable_name: &str) -> RValue; - fn compound_rvalue(&mut self, variable_dependencies: &[String]) -> RValue; - fn drop_last_step(&mut self); + fn assign(&mut self, variable_name: &str, rvalue: RValue, pass_by: PassBy) { + AbstractTraceWriter::assign(self, variable_name, rvalue, pass_by) + } + fn bind_variable(&mut self, variable_name: &str, place: Place) { + AbstractTraceWriter::bind_variable(self, variable_name, place) + } + fn drop_variables(&mut self, variable_names: &[String]) { + AbstractTraceWriter::drop_variables(self, variable_names) + } + fn simple_rvalue(&mut self, variable_name: &str) -> RValue { + AbstractTraceWriter::simple_rvalue(self, variable_name) + } + fn compound_rvalue(&mut self, variable_dependencies: &[String]) -> RValue { + AbstractTraceWriter::compound_rvalue(self, variable_dependencies) + } + fn drop_last_step(&mut self) { + AbstractTraceWriter::drop_last_step(self) + } - fn add_event(&mut self, event: TraceLowLevelEvent); - fn append_events(&mut self, events: &mut Vec); + fn add_event(&mut self, event: TraceLowLevelEvent) { + AbstractTraceWriter::add_event(self, event) + } + fn append_events(&mut self, events: &mut Vec) { + AbstractTraceWriter::append_events(self, events) + } fn finish_writing_trace_metadata(&mut self) -> Result<(), Box>; fn finish_writing_trace_events(&mut self) -> Result<(), Box>; @@ -122,20 +191,10 @@ pub trait TraceWriter { /// disk when the `finish_writing_trace_*` methods are called. The in-memory event list /// is exposed publicly. pub struct NonStreamingTraceWriter { - // trace metadata: - workdir: PathBuf, - program: String, - args: Vec, + base: AbstractTraceWriterData, + // trace events pub events: Vec, - // internal tracer state: - path_list: Vec, - function_list: Vec<(String, PathId, Line)>, - - paths: HashMap, - functions: HashMap, - variables: HashMap, - types: HashMap, format: TraceEventsFileFormat, trace_metadata_path: Option, @@ -165,17 +224,20 @@ impl NonStreamingTraceWriter { /// Create a new tracer instance for the given program and arguments. pub fn new(program: &str, args: &[String]) -> Self { NonStreamingTraceWriter { - workdir: env::current_dir().expect("can access the current dir"), - program: program.to_string(), - args: args.to_vec(), - events: vec![], + base: AbstractTraceWriterData { + workdir: env::current_dir().expect("can access the current dir"), + program: program.to_string(), + args: args.to_vec(), + + path_list: vec![], + function_list: vec![], + paths: HashMap::new(), + functions: HashMap::new(), + variables: HashMap::new(), + types: HashMap::new(), + }, - path_list: vec![], - function_list: vec![], - paths: HashMap::new(), - functions: HashMap::new(), - variables: HashMap::new(), - types: HashMap::new(), + events: vec![], format: TraceEventsFileFormat::Binary, trace_metadata_path: None, @@ -189,6 +251,24 @@ impl NonStreamingTraceWriter { } } +impl AbstractTraceWriter for NonStreamingTraceWriter { + fn get_data(&self) -> &AbstractTraceWriterData { + &self.base + } + + fn get_mut_data(&mut self) -> &mut AbstractTraceWriterData { + &mut self.base + } + + fn add_event(&mut self, event: TraceLowLevelEvent) { + self.events.push(event) + } + + fn append_events(&mut self, events: &mut Vec) { + self.events.append(events) + } +} + impl TraceWriter for NonStreamingTraceWriter { fn begin_writing_trace_metadata(&mut self, path: &Path) -> Result<(), Box> { self.trace_metadata_path = Some(path.to_path_buf()); @@ -205,235 +285,12 @@ impl TraceWriter for NonStreamingTraceWriter { Ok(()) } - /// Begin tracing of a program starting at the given source location. - fn start(&mut self, path: &Path, line: Line) { - let function_id = self.ensure_function_id("", path, line); - self.register_call(function_id, vec![]); - assert!(function_id == TOP_LEVEL_FUNCTION_ID); - - // probably we let the user choose, as different languages have - // different base types/names - // assert!(EXAMPLE_INT_TYPE_ID == self.load_type_id(TypeKind::Int, "Int")); - // assert!(EXAMPLE_FLOAT_TYPE_ID == self.load_type_id(TypeKind::Float, "Float")); - // assert!(EXAMPLE_BOOL_TYPE_ID == self.load_type_id(TypeKind::Bool, "Bool")); - // assert!(EXAMPLE_STRING_TYPE_ID == self.load_type_id(TypeKind::Bool, "String")); - assert!(NONE_TYPE_ID == self.ensure_type_id(TypeKind::None, "None")); - } - - fn ensure_path_id(&mut self, path: &Path) -> PathId { - if !self.paths.contains_key(path) { - self.paths.insert(path.to_path_buf(), PathId(self.paths.len())); - self.register_path(path); - } - *self.paths.get(path).unwrap() - } - - fn ensure_function_id(&mut self, function_name: &str, path: &Path, line: Line) -> FunctionId { - if !self.functions.contains_key(function_name) { - // same function names for different path line? TODO - self.functions.insert(function_name.to_string(), FunctionId(self.functions.len())); - self.register_function(function_name, path, line); - } - *self.functions.get(function_name).unwrap() - } - - fn ensure_type_id(&mut self, kind: TypeKind, lang_type: &str) -> TypeId { - let typ = self.to_raw_type(kind, lang_type); - self.ensure_raw_type_id(typ) - } - - fn ensure_raw_type_id(&mut self, typ: TypeRecord) -> TypeId { - if !self.types.contains_key(&typ.lang_type) { - self.types.insert(typ.lang_type.clone(), TypeId(self.types.len())); - self.register_raw_type(typ.clone()); - } - *self.types.get(&typ.lang_type).unwrap() - } - - fn ensure_variable_id(&mut self, variable_name: &str) -> VariableId { - if !self.variables.contains_key(variable_name) { - self.variables.insert(variable_name.to_string(), VariableId(self.variables.len())); - self.register_variable_name(variable_name); - } - *self.variables.get(variable_name).unwrap() - } - - fn register_path(&mut self, path: &Path) { - self.path_list.push(path.to_path_buf()); - self.events.push(TraceLowLevelEvent::Path(path.to_path_buf())); - } - - fn register_function(&mut self, name: &str, path: &Path, line: Line) { - let path_id = self.ensure_path_id(path); - self.function_list.push((name.to_string(), path_id, line)); - self.events.push(TraceLowLevelEvent::Function(FunctionRecord { - name: name.to_string(), - path_id, - line, - })); - } - - fn register_step(&mut self, path: &Path, line: Line) { - let path_id = self.ensure_path_id(path); - self.events.push(TraceLowLevelEvent::Step(StepRecord { path_id, line })); - } - - fn register_call(&mut self, function_id: FunctionId, args: Vec) { - // register a step for each call, the backend expects this for - // non-toplevel calls, so - // we ensure it directly from register_call - if function_id != TOP_LEVEL_FUNCTION_ID { - for arg in &args { - self.register_full_value(arg.variable_id, arg.value.clone()); - } - let function = &self.function_list[function_id.0]; - self.events.push(TraceLowLevelEvent::Step(StepRecord { - path_id: function.1, - line: function.2, - })); - } - // the actual call event: - self.events.push(TraceLowLevelEvent::Call(CallRecord { function_id, args })); - } - - fn arg(&mut self, name: &str, value: ValueRecord) -> FullValueRecord { - let variable_id = self.ensure_variable_id(name); - FullValueRecord { variable_id, value } - } - - fn register_return(&mut self, return_value: ValueRecord) { - self.events.push(TraceLowLevelEvent::Return(ReturnRecord { return_value })); - } - - fn register_special_event(&mut self, kind: EventLogKind, content: &str) { - self.events.push(TraceLowLevelEvent::Event(RecordEvent { - kind, - metadata: "".to_string(), - content: content.to_string(), - })); - } - - fn to_raw_type(&self, kind: TypeKind, lang_type: &str) -> TypeRecord { - TypeRecord { - kind, - lang_type: lang_type.to_string(), - specific_info: TypeSpecificInfo::None, - } - } - - fn register_type(&mut self, kind: TypeKind, lang_type: &str) { - let typ = self.to_raw_type(kind, lang_type); - self.events.push(TraceLowLevelEvent::Type(typ)); - } - - fn register_raw_type(&mut self, typ: TypeRecord) { - self.events.push(TraceLowLevelEvent::Type(typ)); - } - - fn register_asm(&mut self, instructions: &[String]) { - self.events.push(TraceLowLevelEvent::Asm(instructions.to_vec())); - } - - fn register_variable_with_full_value(&mut self, name: &str, value: ValueRecord) { - let variable_id = self.ensure_variable_id(name); - self.register_full_value(variable_id, value); - } - - fn register_variable_name(&mut self, variable_name: &str) { - self.events.push(TraceLowLevelEvent::VariableName(variable_name.to_string())); - } - - fn register_full_value(&mut self, variable_id: VariableId, value: ValueRecord) { - self.events.push(TraceLowLevelEvent::Value(FullValueRecord { variable_id, value })); - } - - fn register_compound_value(&mut self, place: Place, value: ValueRecord) { - self.events.push(TraceLowLevelEvent::CompoundValue(CompoundValueRecord { place, value })); - } - - fn register_cell_value(&mut self, place: Place, value: ValueRecord) { - self.events.push(TraceLowLevelEvent::CellValue(CellValueRecord { place, value })); - } - - fn assign_compound_item(&mut self, place: Place, index: usize, item_place: Place) { - self.events.push(TraceLowLevelEvent::AssignCompoundItem(AssignCompoundItemRecord { - place, - index, - item_place, - })); - } - fn assign_cell(&mut self, place: Place, new_value: ValueRecord) { - self.events.push(TraceLowLevelEvent::AssignCell(AssignCellRecord { place, new_value })); - } - - fn register_variable(&mut self, variable_name: &str, place: Place) { - let variable_id = self.ensure_variable_id(variable_name); - self.events - .push(TraceLowLevelEvent::VariableCell(VariableCellRecord { variable_id, place })); - } - - fn drop_variable(&mut self, variable_name: &str) { - let variable_id = self.ensure_variable_id(variable_name); - self.events.push(TraceLowLevelEvent::DropVariable(variable_id)); - } - - // history event helpers - fn assign(&mut self, variable_name: &str, rvalue: RValue, pass_by: PassBy) { - let variable_id = self.ensure_variable_id(variable_name); - self.events.push(TraceLowLevelEvent::Assignment(AssignmentRecord { - to: variable_id, - from: rvalue, - pass_by, - })); - } - - fn bind_variable(&mut self, variable_name: &str, place: Place) { - let variable_id = self.ensure_variable_id(variable_name); - self.events - .push(TraceLowLevelEvent::BindVariable(crate::BindVariableRecord { variable_id, place })); - } - - fn drop_variables(&mut self, variable_names: &[String]) { - let variable_ids: Vec = variable_names - .to_vec() - .iter() - .map(|variable_name| self.ensure_variable_id(variable_name)) - .collect(); - self.events.push(TraceLowLevelEvent::DropVariables(variable_ids)) - } - - fn simple_rvalue(&mut self, variable_name: &str) -> RValue { - let variable_id = self.ensure_variable_id(variable_name); - RValue::Simple(variable_id) - } - - fn compound_rvalue(&mut self, variable_dependencies: &[String]) -> RValue { - let variable_ids: Vec = variable_dependencies - .to_vec() - .iter() - .map(|variable_dependency| self.ensure_variable_id(variable_dependency)) - .collect(); - RValue::Compound(variable_ids) - } - - fn drop_last_step(&mut self) { - self.events.push(TraceLowLevelEvent::DropLastStep); - } - - fn add_event(&mut self, event: TraceLowLevelEvent) { - self.events.push(event) - } - - fn append_events(&mut self, events: &mut Vec) { - self.events.append(events); - } - fn finish_writing_trace_metadata(&mut self) -> Result<(), Box> { if let Some(path) = &self.trace_metadata_path { let trace_metadata = TraceMetadata { - program: self.program.clone(), - args: self.args.clone(), - workdir: self.workdir.clone(), + program: self.get_data().program.clone(), + args: self.get_data().args.clone(), + workdir: self.get_data().workdir.clone(), }; let json = serde_json::to_string(&trace_metadata)?; fs::write(path, json)?; @@ -466,7 +323,7 @@ impl TraceWriter for NonStreamingTraceWriter { fn finish_writing_trace_paths(&mut self) -> Result<(), Box> { if let Some(path) = &self.trace_paths_path { - let json = serde_json::to_string(&self.path_list)?; + let json = serde_json::to_string(&self.get_data().path_list)?; fs::write(path, json)?; Ok(()) } else { diff --git a/runtime_tracing/tests/binary_format.rs b/runtime_tracing/tests/binary_format.rs index d4c02f8..a91812a 100644 --- a/runtime_tracing/tests/binary_format.rs +++ b/runtime_tracing/tests/binary_format.rs @@ -1,4 +1,4 @@ -use runtime_tracing::{create_trace_reader, create_trace_writer, TraceEventsFileFormat}; +use runtime_tracing::{create_trace_reader, create_trace_writer, TraceEventsFileFormat, TraceWriter}; use std::fs; use std::path::Path; @@ -13,7 +13,7 @@ fn test_binary_roundtrip(ver: TraceEventsFileFormat, binfile: &str) { let mut bin_writer = create_trace_writer("", &[], ver); bin_writer.begin_writing_trace_events(bin_path).unwrap(); - bin_writer.append_events(&mut original.clone()); + TraceWriter::append_events(bin_writer.as_mut(), &mut original.clone()); bin_writer.finish_writing_trace_events().unwrap(); let mut bin_reader = create_trace_reader(TraceEventsFileFormat::Binary); diff --git a/runtime_tracing_cli/src/main.rs b/runtime_tracing_cli/src/main.rs index 0877c1d..f3ceb81 100644 --- a/runtime_tracing_cli/src/main.rs +++ b/runtime_tracing_cli/src/main.rs @@ -2,7 +2,7 @@ use std::path::Path; use crate::fmt_trace_cmd::FmtTraceCommand; use clap::{Args, Parser, Subcommand}; -use runtime_tracing::{create_trace_reader, create_trace_writer, TraceEventsFileFormat}; +use runtime_tracing::{create_trace_reader, create_trace_writer, TraceEventsFileFormat, TraceWriter}; mod fmt_trace_cmd; #[derive(Debug, Clone, Args)] @@ -48,7 +48,7 @@ fn main() { let mut trace_writer = create_trace_writer("", &[], output_file_format); let mut trace_events = trace_reader.load_trace_events(Path::new(&convert_command.input_file)).unwrap(); trace_writer.begin_writing_trace_events(Path::new(&convert_command.output_file)).unwrap(); - trace_writer.append_events(&mut trace_events); + TraceWriter::append_events(trace_writer.as_mut(), &mut trace_events); trace_writer.finish_writing_trace_events().unwrap(); } RuntimeTracingCliCommand::FormatTrace(fmt_trace_cmd) => { From a12bd924f1fceb7302eacfe96e6996efe677f6a7 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolov Date: Thu, 24 Jul 2025 13:05:43 +0300 Subject: [PATCH 17/28] refactor(cbor_plus_zstd): abstracttracewriter.rs renamed abstract_trace_writer.rs --- .../src/{abstracttracewriter.rs => abstract_trace_writer.rs} | 0 runtime_tracing/src/cborzstdwriter.rs | 2 +- runtime_tracing/src/lib.rs | 2 +- runtime_tracing/src/tracer.rs | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename runtime_tracing/src/{abstracttracewriter.rs => abstract_trace_writer.rs} (100%) diff --git a/runtime_tracing/src/abstracttracewriter.rs b/runtime_tracing/src/abstract_trace_writer.rs similarity index 100% rename from runtime_tracing/src/abstracttracewriter.rs rename to runtime_tracing/src/abstract_trace_writer.rs diff --git a/runtime_tracing/src/cborzstdwriter.rs b/runtime_tracing/src/cborzstdwriter.rs index 4b65f81..79ace0b 100644 --- a/runtime_tracing/src/cborzstdwriter.rs +++ b/runtime_tracing/src/cborzstdwriter.rs @@ -1,7 +1,7 @@ use std::{collections::HashMap, env, fs::{self, File}, io::Write, path::PathBuf}; use zeekstd::Encoder; -use crate::{abstracttracewriter::{AbstractTraceWriter, AbstractTraceWriterData}, TraceLowLevelEvent, TraceMetadata, TraceWriter}; +use crate::{abstract_trace_writer::{AbstractTraceWriter, AbstractTraceWriterData}, TraceLowLevelEvent, TraceMetadata, TraceWriter}; /// The next 3 bytes are reserved/version info. /// The header is 8 bytes in size, ensuring 64-bit alignment for the rest of the file. diff --git a/runtime_tracing/src/lib.rs b/runtime_tracing/src/lib.rs index c7defbd..0504bac 100644 --- a/runtime_tracing/src/lib.rs +++ b/runtime_tracing/src/lib.rs @@ -14,7 +14,7 @@ mod tracer; mod types; mod cborzstdreader; mod cborzstdwriter; -mod abstracttracewriter; +mod abstract_trace_writer; pub use crate::tracer::{ create_trace_reader, create_trace_writer, NonStreamingTraceWriter, TraceEventsFileFormat, TraceReader, TraceWriter, NONE_TYPE_ID, NONE_VALUE, diff --git a/runtime_tracing/src/tracer.rs b/runtime_tracing/src/tracer.rs index d174e5e..9e5466f 100644 --- a/runtime_tracing/src/tracer.rs +++ b/runtime_tracing/src/tracer.rs @@ -9,7 +9,7 @@ use std::path::{Path, PathBuf}; use crate::capnptrace::HEADER; use crate::cborzstdwriter::HEADERV1; -use crate::abstracttracewriter::{AbstractTraceWriter, AbstractTraceWriterData}; +use crate::abstract_trace_writer::{AbstractTraceWriter, AbstractTraceWriterData}; use crate::types::{ EventLogKind, FullValueRecord, FunctionId, Line, PassBy, PathId, Place, TraceLowLevelEvent, TraceMetadata, TypeId, From e5960f731a5f44d1fc765f66c098470c644b62e1 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolov Date: Thu, 24 Jul 2025 13:07:32 +0300 Subject: [PATCH 18/28] refactor(cbor_plus_zstd): cborzstdreader.rs renamed cbor_zstd_reader.rs --- runtime_tracing/src/{cborzstdreader.rs => cbor_zstd_reader.rs} | 0 runtime_tracing/src/lib.rs | 2 +- runtime_tracing/src/tracer.rs | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename runtime_tracing/src/{cborzstdreader.rs => cbor_zstd_reader.rs} (100%) diff --git a/runtime_tracing/src/cborzstdreader.rs b/runtime_tracing/src/cbor_zstd_reader.rs similarity index 100% rename from runtime_tracing/src/cborzstdreader.rs rename to runtime_tracing/src/cbor_zstd_reader.rs diff --git a/runtime_tracing/src/lib.rs b/runtime_tracing/src/lib.rs index 0504bac..dcaf5f4 100644 --- a/runtime_tracing/src/lib.rs +++ b/runtime_tracing/src/lib.rs @@ -12,7 +12,7 @@ mod base64; mod capnptrace; mod tracer; mod types; -mod cborzstdreader; +mod cbor_zstd_reader; mod cborzstdwriter; mod abstract_trace_writer; diff --git a/runtime_tracing/src/tracer.rs b/runtime_tracing/src/tracer.rs index 9e5466f..25d9a06 100644 --- a/runtime_tracing/src/tracer.rs +++ b/runtime_tracing/src/tracer.rs @@ -57,7 +57,7 @@ impl TraceReader for BinaryTraceReader { Ok(crate::capnptrace::read_trace(&mut buf_reader)?) } Some(TraceEventsFileFormat::Binary) => { - Ok(crate::cborzstdreader::read_trace(&mut file)?) + Ok(crate::cbor_zstd_reader::read_trace(&mut file)?) } Some(TraceEventsFileFormat::Json) => { unreachable!() From 21c77ec1c7821f971322f3ab6566ec8c33bfa2e8 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolov Date: Thu, 24 Jul 2025 13:10:19 +0300 Subject: [PATCH 19/28] refactor(cbor_plus_zstd): cborzstdwriter.rs renamed cbor_zstd_writer.rs --- runtime_tracing/src/cbor_zstd_reader.rs | 2 +- .../src/{cborzstdwriter.rs => cbor_zstd_writer.rs} | 0 runtime_tracing/src/lib.rs | 2 +- runtime_tracing/src/tracer.rs | 4 ++-- 4 files changed, 4 insertions(+), 4 deletions(-) rename runtime_tracing/src/{cborzstdwriter.rs => cbor_zstd_writer.rs} (100%) diff --git a/runtime_tracing/src/cbor_zstd_reader.rs b/runtime_tracing/src/cbor_zstd_reader.rs index fe7eedf..0ce5442 100644 --- a/runtime_tracing/src/cbor_zstd_reader.rs +++ b/runtime_tracing/src/cbor_zstd_reader.rs @@ -3,7 +3,7 @@ use std::io::{self, BufRead, BufReader, Read, Seek, Write}; use fscommon::StreamSlice; use zeekstd::Decoder; -use crate::{cborzstdwriter::HEADERV1, TraceLowLevelEvent}; +use crate::{cbor_zstd_writer::HEADERV1, TraceLowLevelEvent}; fn is_at_eof(reader: &mut R) -> io::Result { let buffer = reader.fill_buf()?; diff --git a/runtime_tracing/src/cborzstdwriter.rs b/runtime_tracing/src/cbor_zstd_writer.rs similarity index 100% rename from runtime_tracing/src/cborzstdwriter.rs rename to runtime_tracing/src/cbor_zstd_writer.rs diff --git a/runtime_tracing/src/lib.rs b/runtime_tracing/src/lib.rs index dcaf5f4..2cdbf0d 100644 --- a/runtime_tracing/src/lib.rs +++ b/runtime_tracing/src/lib.rs @@ -13,7 +13,7 @@ mod capnptrace; mod tracer; mod types; mod cbor_zstd_reader; -mod cborzstdwriter; +mod cbor_zstd_writer; mod abstract_trace_writer; pub use crate::tracer::{ diff --git a/runtime_tracing/src/tracer.rs b/runtime_tracing/src/tracer.rs index 25d9a06..d283155 100644 --- a/runtime_tracing/src/tracer.rs +++ b/runtime_tracing/src/tracer.rs @@ -8,7 +8,7 @@ use std::io::{BufReader, Read, Seek, SeekFrom}; use std::path::{Path, PathBuf}; use crate::capnptrace::HEADER; -use crate::cborzstdwriter::HEADERV1; +use crate::cbor_zstd_writer::HEADERV1; use crate::abstract_trace_writer::{AbstractTraceWriter, AbstractTraceWriterData}; use crate::types::{ EventLogKind, FullValueRecord, @@ -349,7 +349,7 @@ pub fn create_trace_writer(program: &str, args: &[String], format: TraceEventsFi result } TraceEventsFileFormat::Binary => { - Box::new(crate::cborzstdwriter::StreamingTraceWriter::new(program, args)) + Box::new(crate::cbor_zstd_writer::StreamingTraceWriter::new(program, args)) } } } From b088823e2f38f3a29c2db20bb97d4fb8e3a2fded Mon Sep 17 00:00:00 2001 From: Nikolay Nikolov Date: Thu, 24 Jul 2025 13:22:25 +0300 Subject: [PATCH 20/28] refactor(cbor_plus_zstd): moved some code to trace_readers.rs --- runtime_tracing/src/lib.rs | 4 +- runtime_tracing/src/trace_readers.rs | 55 ++++++++++++++++++++++++++ runtime_tracing/src/tracer.rs | 58 +--------------------------- 3 files changed, 60 insertions(+), 57 deletions(-) create mode 100644 runtime_tracing/src/trace_readers.rs diff --git a/runtime_tracing/src/lib.rs b/runtime_tracing/src/lib.rs index 2cdbf0d..c3a9375 100644 --- a/runtime_tracing/src/lib.rs +++ b/runtime_tracing/src/lib.rs @@ -15,10 +15,12 @@ mod types; mod cbor_zstd_reader; mod cbor_zstd_writer; mod abstract_trace_writer; +mod trace_readers; pub use crate::tracer::{ - create_trace_reader, create_trace_writer, NonStreamingTraceWriter, TraceEventsFileFormat, TraceReader, TraceWriter, NONE_TYPE_ID, NONE_VALUE, + create_trace_reader, create_trace_writer, NonStreamingTraceWriter, TraceEventsFileFormat, TraceWriter, NONE_TYPE_ID, NONE_VALUE, }; +pub use crate::trace_readers::TraceReader; pub use crate::types::*; pub mod trace_capnp { diff --git a/runtime_tracing/src/trace_readers.rs b/runtime_tracing/src/trace_readers.rs new file mode 100644 index 0000000..1c1f9c1 --- /dev/null +++ b/runtime_tracing/src/trace_readers.rs @@ -0,0 +1,55 @@ +use std::{error::Error, fs::{self, File}, io::{BufReader, Read, Seek, SeekFrom}, path::Path}; + +use crate::{capnptrace::HEADER, cbor_zstd_writer::HEADERV1, TraceEventsFileFormat, TraceLowLevelEvent}; + +pub trait TraceReader { + fn load_trace_events(&mut self, path: &Path) -> Result, Box>; +} + +pub struct JsonTraceReader {} + +impl TraceReader for JsonTraceReader { + fn load_trace_events(&mut self, path: &Path) -> Result, Box> { + let json = std::fs::read_to_string(path)?; + Ok(serde_json::from_str(&json)?) + } +} + +pub struct BinaryTraceReader {} + +fn detect_bin_file_version(input: &mut File) -> Result, Box> { + input.seek(SeekFrom::Start(0))?; + let mut header_buf = [0; 8]; + input.read_exact(&mut header_buf)?; + input.seek(SeekFrom::Start(0))?; + + if header_buf == HEADER { + Ok(Some(TraceEventsFileFormat::BinaryV0)) + } else if header_buf == HEADERV1 { + Ok(Some(TraceEventsFileFormat::Binary)) + } else { + Ok(None) + } +} + +impl TraceReader for BinaryTraceReader { + fn load_trace_events(&mut self, path: &Path) -> Result, Box> { + let mut file = fs::File::open(path)?; + let ver = detect_bin_file_version(&mut file)?; + match ver { + Some(TraceEventsFileFormat::BinaryV0) => { + let mut buf_reader = BufReader::new(file); + Ok(crate::capnptrace::read_trace(&mut buf_reader)?) + } + Some(TraceEventsFileFormat::Binary) => { + Ok(crate::cbor_zstd_reader::read_trace(&mut file)?) + } + Some(TraceEventsFileFormat::Json) => { + unreachable!() + } + None => { + panic!("Invalid file header (wrong file format or incompatible version)"); + } + } + } +} diff --git a/runtime_tracing/src/tracer.rs b/runtime_tracing/src/tracer.rs index d283155..ee4ea6b 100644 --- a/runtime_tracing/src/tracer.rs +++ b/runtime_tracing/src/tracer.rs @@ -3,13 +3,11 @@ use std::collections::HashMap; use std::env; use std::error::Error; -use std::fs::{self, File}; -use std::io::{BufReader, Read, Seek, SeekFrom}; +use std::fs; use std::path::{Path, PathBuf}; -use crate::capnptrace::HEADER; -use crate::cbor_zstd_writer::HEADERV1; use crate::abstract_trace_writer::{AbstractTraceWriter, AbstractTraceWriterData}; +use crate::trace_readers::{BinaryTraceReader, JsonTraceReader, TraceReader}; use crate::types::{ EventLogKind, FullValueRecord, FunctionId, Line, PassBy, PathId, Place, TraceLowLevelEvent, TraceMetadata, TypeId, @@ -17,58 +15,6 @@ use crate::types::{ }; use crate::RValue; -pub trait TraceReader { - fn load_trace_events(&mut self, path: &Path) -> Result, Box>; -} - -pub struct JsonTraceReader {} - -impl TraceReader for JsonTraceReader { - fn load_trace_events(&mut self, path: &Path) -> Result, Box> { - let json = std::fs::read_to_string(path)?; - Ok(serde_json::from_str(&json)?) - } -} - -pub struct BinaryTraceReader {} - -fn detect_bin_file_version(input: &mut File) -> Result, Box> { - input.seek(SeekFrom::Start(0))?; - let mut header_buf = [0; 8]; - input.read_exact(&mut header_buf)?; - input.seek(SeekFrom::Start(0))?; - - if header_buf == HEADER { - Ok(Some(TraceEventsFileFormat::BinaryV0)) - } else if header_buf == HEADERV1 { - Ok(Some(TraceEventsFileFormat::Binary)) - } else { - Ok(None) - } -} - -impl TraceReader for BinaryTraceReader { - fn load_trace_events(&mut self, path: &Path) -> Result, Box> { - let mut file = fs::File::open(path)?; - let ver = detect_bin_file_version(&mut file)?; - match ver { - Some(TraceEventsFileFormat::BinaryV0) => { - let mut buf_reader = BufReader::new(file); - Ok(crate::capnptrace::read_trace(&mut buf_reader)?) - } - Some(TraceEventsFileFormat::Binary) => { - Ok(crate::cbor_zstd_reader::read_trace(&mut file)?) - } - Some(TraceEventsFileFormat::Json) => { - unreachable!() - } - None => { - panic!("Invalid file header (wrong file format or incompatible version)"); - } - } - } -} - pub trait TraceWriter: AbstractTraceWriter { fn begin_writing_trace_metadata(&mut self, path: &Path) -> Result<(), Box>; fn begin_writing_trace_events(&mut self, path: &Path) -> Result<(), Box>; From 747b17ff3491b8cc72ad962d8661730b548ad203 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolov Date: Thu, 24 Jul 2025 13:29:14 +0300 Subject: [PATCH 21/28] refactor(cbor_plus_zstd): moved the TraceWriter trait to a new file trace_writer.rs --- runtime_tracing/src/cbor_zstd_writer.rs | 2 +- runtime_tracing/src/lib.rs | 4 +- runtime_tracing/src/trace_writer.rs | 119 +++++++++++++++++++++++ runtime_tracing/src/tracer.rs | 122 +----------------------- 4 files changed, 126 insertions(+), 121 deletions(-) create mode 100644 runtime_tracing/src/trace_writer.rs diff --git a/runtime_tracing/src/cbor_zstd_writer.rs b/runtime_tracing/src/cbor_zstd_writer.rs index 79ace0b..dee96b9 100644 --- a/runtime_tracing/src/cbor_zstd_writer.rs +++ b/runtime_tracing/src/cbor_zstd_writer.rs @@ -1,7 +1,7 @@ use std::{collections::HashMap, env, fs::{self, File}, io::Write, path::PathBuf}; use zeekstd::Encoder; -use crate::{abstract_trace_writer::{AbstractTraceWriter, AbstractTraceWriterData}, TraceLowLevelEvent, TraceMetadata, TraceWriter}; +use crate::{abstract_trace_writer::{AbstractTraceWriter, AbstractTraceWriterData}, trace_writer::TraceWriter, TraceLowLevelEvent, TraceMetadata}; /// The next 3 bytes are reserved/version info. /// The header is 8 bytes in size, ensuring 64-bit alignment for the rest of the file. diff --git a/runtime_tracing/src/lib.rs b/runtime_tracing/src/lib.rs index c3a9375..68e51c9 100644 --- a/runtime_tracing/src/lib.rs +++ b/runtime_tracing/src/lib.rs @@ -16,11 +16,13 @@ mod cbor_zstd_reader; mod cbor_zstd_writer; mod abstract_trace_writer; mod trace_readers; +mod trace_writer; pub use crate::tracer::{ - create_trace_reader, create_trace_writer, NonStreamingTraceWriter, TraceEventsFileFormat, TraceWriter, NONE_TYPE_ID, NONE_VALUE, + create_trace_reader, create_trace_writer, NonStreamingTraceWriter, TraceEventsFileFormat, NONE_TYPE_ID, NONE_VALUE, }; pub use crate::trace_readers::TraceReader; +pub use crate::trace_writer::TraceWriter; pub use crate::types::*; pub mod trace_capnp { diff --git a/runtime_tracing/src/trace_writer.rs b/runtime_tracing/src/trace_writer.rs new file mode 100644 index 0000000..86767a7 --- /dev/null +++ b/runtime_tracing/src/trace_writer.rs @@ -0,0 +1,119 @@ +use std::{error::Error, path::Path}; + +use crate::{abstract_trace_writer::AbstractTraceWriter, EventLogKind, FullValueRecord, FunctionId, Line, PassBy, PathId, Place, RValue, TraceLowLevelEvent, TypeId, TypeKind, TypeRecord, ValueRecord, VariableId}; + +pub trait TraceWriter: AbstractTraceWriter { + fn begin_writing_trace_metadata(&mut self, path: &Path) -> Result<(), Box>; + fn begin_writing_trace_events(&mut self, path: &Path) -> Result<(), Box>; + fn begin_writing_trace_paths(&mut self, path: &Path) -> Result<(), Box>; + + fn start(&mut self, path: &Path, line: Line) { + AbstractTraceWriter::start(self, path, line) + } + fn ensure_path_id(&mut self, path: &Path) -> PathId { + AbstractTraceWriter::ensure_path_id(self, path) + } + fn ensure_function_id(&mut self, function_name: &str, path: &Path, line: Line) -> FunctionId { + AbstractTraceWriter::ensure_function_id(self, function_name, path, line) + } + fn ensure_type_id(&mut self, kind: TypeKind, lang_type: &str) -> TypeId { + AbstractTraceWriter::ensure_type_id(self, kind, lang_type) + } + fn ensure_raw_type_id(&mut self, typ: TypeRecord) -> TypeId { + AbstractTraceWriter::ensure_raw_type_id(self, typ) + } + fn ensure_variable_id(&mut self, variable_name: &str) -> VariableId { + AbstractTraceWriter::ensure_variable_id(self, variable_name) + } + fn register_path(&mut self, path: &Path) { + AbstractTraceWriter::register_path(self, path) + } + fn register_function(&mut self, name: &str, path: &Path, line: Line) { + AbstractTraceWriter::register_function(self, name, path, line) + } + fn register_step(&mut self, path: &Path, line: Line) { + AbstractTraceWriter::register_step(self, path, line) + } + fn register_call(&mut self, function_id: FunctionId, args: Vec) { + AbstractTraceWriter::register_call(self, function_id, args) + } + fn arg(&mut self, name: &str, value: ValueRecord) -> FullValueRecord { + AbstractTraceWriter::arg(self, name, value) + } + fn register_return(&mut self, return_value: ValueRecord) { + AbstractTraceWriter::register_return(self, return_value) + } + // TODO: add metadata arg + fn register_special_event(&mut self, kind: EventLogKind, content: &str) { + AbstractTraceWriter::register_special_event(self, kind, content) + } + fn to_raw_type(&self, kind: TypeKind, lang_type: &str) -> TypeRecord { + AbstractTraceWriter::to_raw_type(self, kind, lang_type) + } + fn register_type(&mut self, kind: TypeKind, lang_type: &str) { + AbstractTraceWriter::register_type(self, kind, lang_type) + } + fn register_raw_type(&mut self, typ: TypeRecord) { + AbstractTraceWriter::register_raw_type(self, typ) + } + fn register_asm(&mut self, instructions: &[String]) { + AbstractTraceWriter::register_asm(self, instructions) + } + fn register_variable_with_full_value(&mut self, name: &str, value: ValueRecord) { + AbstractTraceWriter::register_variable_with_full_value(self, name, value) + } + fn register_variable_name(&mut self, variable_name: &str) { + AbstractTraceWriter::register_variable_name(self, variable_name) + } + fn register_full_value(&mut self, variable_id: VariableId, value: ValueRecord) { + AbstractTraceWriter::register_full_value(self, variable_id, value) + } + fn register_compound_value(&mut self, place: Place, value: ValueRecord) { + AbstractTraceWriter::register_compound_value(self, place, value) + } + fn register_cell_value(&mut self, place: Place, value: ValueRecord) { + AbstractTraceWriter::register_cell_value(self, place, value) + } + fn assign_compound_item(&mut self, place: Place, index: usize, item_place: Place) { + AbstractTraceWriter::assign_compound_item(self, place, index, item_place) + } + fn assign_cell(&mut self, place: Place, new_value: ValueRecord) { + AbstractTraceWriter::assign_cell(self, place, new_value) + } + fn register_variable(&mut self, variable_name: &str, place: Place) { + AbstractTraceWriter::register_variable(self, variable_name, place) + } + fn drop_variable(&mut self, variable_name: &str) { + AbstractTraceWriter::drop_variable(self, variable_name) + } + // history event helpers + fn assign(&mut self, variable_name: &str, rvalue: RValue, pass_by: PassBy) { + AbstractTraceWriter::assign(self, variable_name, rvalue, pass_by) + } + fn bind_variable(&mut self, variable_name: &str, place: Place) { + AbstractTraceWriter::bind_variable(self, variable_name, place) + } + fn drop_variables(&mut self, variable_names: &[String]) { + AbstractTraceWriter::drop_variables(self, variable_names) + } + fn simple_rvalue(&mut self, variable_name: &str) -> RValue { + AbstractTraceWriter::simple_rvalue(self, variable_name) + } + fn compound_rvalue(&mut self, variable_dependencies: &[String]) -> RValue { + AbstractTraceWriter::compound_rvalue(self, variable_dependencies) + } + fn drop_last_step(&mut self) { + AbstractTraceWriter::drop_last_step(self) + } + + fn add_event(&mut self, event: TraceLowLevelEvent) { + AbstractTraceWriter::add_event(self, event) + } + fn append_events(&mut self, events: &mut Vec) { + AbstractTraceWriter::append_events(self, events) + } + + fn finish_writing_trace_metadata(&mut self) -> Result<(), Box>; + fn finish_writing_trace_events(&mut self) -> Result<(), Box>; + fn finish_writing_trace_paths(&mut self) -> Result<(), Box>; +} diff --git a/runtime_tracing/src/tracer.rs b/runtime_tracing/src/tracer.rs index ee4ea6b..6955923 100644 --- a/runtime_tracing/src/tracer.rs +++ b/runtime_tracing/src/tracer.rs @@ -9,127 +9,11 @@ use std::path::{Path, PathBuf}; use crate::abstract_trace_writer::{AbstractTraceWriter, AbstractTraceWriterData}; use crate::trace_readers::{BinaryTraceReader, JsonTraceReader, TraceReader}; use crate::types::{ - EventLogKind, FullValueRecord, - FunctionId, Line, PassBy, PathId, Place, TraceLowLevelEvent, TraceMetadata, TypeId, - TypeKind, TypeRecord, ValueRecord, VariableId, + FunctionId, TraceLowLevelEvent, TraceMetadata, TypeId, + ValueRecord, }; -use crate::RValue; +use crate::TraceWriter; -pub trait TraceWriter: AbstractTraceWriter { - fn begin_writing_trace_metadata(&mut self, path: &Path) -> Result<(), Box>; - fn begin_writing_trace_events(&mut self, path: &Path) -> Result<(), Box>; - fn begin_writing_trace_paths(&mut self, path: &Path) -> Result<(), Box>; - - fn start(&mut self, path: &Path, line: Line) { - AbstractTraceWriter::start(self, path, line) - } - fn ensure_path_id(&mut self, path: &Path) -> PathId { - AbstractTraceWriter::ensure_path_id(self, path) - } - fn ensure_function_id(&mut self, function_name: &str, path: &Path, line: Line) -> FunctionId { - AbstractTraceWriter::ensure_function_id(self, function_name, path, line) - } - fn ensure_type_id(&mut self, kind: TypeKind, lang_type: &str) -> TypeId { - AbstractTraceWriter::ensure_type_id(self, kind, lang_type) - } - fn ensure_raw_type_id(&mut self, typ: TypeRecord) -> TypeId { - AbstractTraceWriter::ensure_raw_type_id(self, typ) - } - fn ensure_variable_id(&mut self, variable_name: &str) -> VariableId { - AbstractTraceWriter::ensure_variable_id(self, variable_name) - } - fn register_path(&mut self, path: &Path) { - AbstractTraceWriter::register_path(self, path) - } - fn register_function(&mut self, name: &str, path: &Path, line: Line) { - AbstractTraceWriter::register_function(self, name, path, line) - } - fn register_step(&mut self, path: &Path, line: Line) { - AbstractTraceWriter::register_step(self, path, line) - } - fn register_call(&mut self, function_id: FunctionId, args: Vec) { - AbstractTraceWriter::register_call(self, function_id, args) - } - fn arg(&mut self, name: &str, value: ValueRecord) -> FullValueRecord { - AbstractTraceWriter::arg(self, name, value) - } - fn register_return(&mut self, return_value: ValueRecord) { - AbstractTraceWriter::register_return(self, return_value) - } - // TODO: add metadata arg - fn register_special_event(&mut self, kind: EventLogKind, content: &str) { - AbstractTraceWriter::register_special_event(self, kind, content) - } - fn to_raw_type(&self, kind: TypeKind, lang_type: &str) -> TypeRecord { - AbstractTraceWriter::to_raw_type(self, kind, lang_type) - } - fn register_type(&mut self, kind: TypeKind, lang_type: &str) { - AbstractTraceWriter::register_type(self, kind, lang_type) - } - fn register_raw_type(&mut self, typ: TypeRecord) { - AbstractTraceWriter::register_raw_type(self, typ) - } - fn register_asm(&mut self, instructions: &[String]) { - AbstractTraceWriter::register_asm(self, instructions) - } - fn register_variable_with_full_value(&mut self, name: &str, value: ValueRecord) { - AbstractTraceWriter::register_variable_with_full_value(self, name, value) - } - fn register_variable_name(&mut self, variable_name: &str) { - AbstractTraceWriter::register_variable_name(self, variable_name) - } - fn register_full_value(&mut self, variable_id: VariableId, value: ValueRecord) { - AbstractTraceWriter::register_full_value(self, variable_id, value) - } - fn register_compound_value(&mut self, place: Place, value: ValueRecord) { - AbstractTraceWriter::register_compound_value(self, place, value) - } - fn register_cell_value(&mut self, place: Place, value: ValueRecord) { - AbstractTraceWriter::register_cell_value(self, place, value) - } - fn assign_compound_item(&mut self, place: Place, index: usize, item_place: Place) { - AbstractTraceWriter::assign_compound_item(self, place, index, item_place) - } - fn assign_cell(&mut self, place: Place, new_value: ValueRecord) { - AbstractTraceWriter::assign_cell(self, place, new_value) - } - fn register_variable(&mut self, variable_name: &str, place: Place) { - AbstractTraceWriter::register_variable(self, variable_name, place) - } - fn drop_variable(&mut self, variable_name: &str) { - AbstractTraceWriter::drop_variable(self, variable_name) - } - // history event helpers - fn assign(&mut self, variable_name: &str, rvalue: RValue, pass_by: PassBy) { - AbstractTraceWriter::assign(self, variable_name, rvalue, pass_by) - } - fn bind_variable(&mut self, variable_name: &str, place: Place) { - AbstractTraceWriter::bind_variable(self, variable_name, place) - } - fn drop_variables(&mut self, variable_names: &[String]) { - AbstractTraceWriter::drop_variables(self, variable_names) - } - fn simple_rvalue(&mut self, variable_name: &str) -> RValue { - AbstractTraceWriter::simple_rvalue(self, variable_name) - } - fn compound_rvalue(&mut self, variable_dependencies: &[String]) -> RValue { - AbstractTraceWriter::compound_rvalue(self, variable_dependencies) - } - fn drop_last_step(&mut self) { - AbstractTraceWriter::drop_last_step(self) - } - - fn add_event(&mut self, event: TraceLowLevelEvent) { - AbstractTraceWriter::add_event(self, event) - } - fn append_events(&mut self, events: &mut Vec) { - AbstractTraceWriter::append_events(self, events) - } - - fn finish_writing_trace_metadata(&mut self) -> Result<(), Box>; - fn finish_writing_trace_events(&mut self) -> Result<(), Box>; - fn finish_writing_trace_paths(&mut self) -> Result<(), Box>; -} /// State machine used to record [`TraceLowLevelEvent`]s. /// From 5cc8b39a3ec760c5a27e12e3e4aba5d6e7f23cab Mon Sep 17 00:00:00 2001 From: Nikolay Nikolov Date: Thu, 24 Jul 2025 13:44:00 +0300 Subject: [PATCH 22/28] refactor(cbor_plus_zstd): moved the NonStreamingTraceWriter to a new file non_streaming_trace_writer.rs --- runtime_tracing/src/lib.rs | 4 +- .../src/non_streaming_trace_writer.rs | 132 +++++++++++++++++ runtime_tracing/src/tracer.rs | 139 +----------------- 3 files changed, 137 insertions(+), 138 deletions(-) create mode 100644 runtime_tracing/src/non_streaming_trace_writer.rs diff --git a/runtime_tracing/src/lib.rs b/runtime_tracing/src/lib.rs index 68e51c9..a3b14eb 100644 --- a/runtime_tracing/src/lib.rs +++ b/runtime_tracing/src/lib.rs @@ -17,12 +17,14 @@ mod cbor_zstd_writer; mod abstract_trace_writer; mod trace_readers; mod trace_writer; +mod non_streaming_trace_writer; pub use crate::tracer::{ - create_trace_reader, create_trace_writer, NonStreamingTraceWriter, TraceEventsFileFormat, NONE_TYPE_ID, NONE_VALUE, + create_trace_reader, create_trace_writer, TraceEventsFileFormat, NONE_TYPE_ID, NONE_VALUE, }; pub use crate::trace_readers::TraceReader; pub use crate::trace_writer::TraceWriter; +pub use crate::non_streaming_trace_writer::NonStreamingTraceWriter; pub use crate::types::*; pub mod trace_capnp { diff --git a/runtime_tracing/src/non_streaming_trace_writer.rs b/runtime_tracing/src/non_streaming_trace_writer.rs new file mode 100644 index 0000000..499a4b9 --- /dev/null +++ b/runtime_tracing/src/non_streaming_trace_writer.rs @@ -0,0 +1,132 @@ +use std::{collections::HashMap, env, error::Error, fs, path::{Path, PathBuf}}; + +use crate::{abstract_trace_writer::{AbstractTraceWriter, AbstractTraceWriterData}, TraceEventsFileFormat, TraceLowLevelEvent, TraceMetadata, TraceWriter}; + +/// State machine used to record [`TraceLowLevelEvent`]s. +/// +/// A `NonStreamingTraceWriter` instance accumulates events in memory and stores them on +/// disk when the `finish_writing_trace_*` methods are called. The in-memory event list +/// is exposed publicly. +pub struct NonStreamingTraceWriter { + base: AbstractTraceWriterData, + + // trace events + pub events: Vec, + + format: TraceEventsFileFormat, + trace_metadata_path: Option, + trace_events_path: Option, + trace_paths_path: Option, +} + +impl NonStreamingTraceWriter { + /// Create a new tracer instance for the given program and arguments. + pub fn new(program: &str, args: &[String]) -> Self { + NonStreamingTraceWriter { + base: AbstractTraceWriterData { + workdir: env::current_dir().expect("can access the current dir"), + program: program.to_string(), + args: args.to_vec(), + + path_list: vec![], + function_list: vec![], + paths: HashMap::new(), + functions: HashMap::new(), + variables: HashMap::new(), + types: HashMap::new(), + }, + + events: vec![], + + format: TraceEventsFileFormat::Binary, + trace_metadata_path: None, + trace_events_path: None, + trace_paths_path: None, + } + } + + pub fn set_format(&mut self, format: TraceEventsFileFormat) { + self.format = format; + } +} + +impl AbstractTraceWriter for NonStreamingTraceWriter { + fn get_data(&self) -> &AbstractTraceWriterData { + &self.base + } + + fn get_mut_data(&mut self) -> &mut AbstractTraceWriterData { + &mut self.base + } + + fn add_event(&mut self, event: TraceLowLevelEvent) { + self.events.push(event) + } + + fn append_events(&mut self, events: &mut Vec) { + self.events.append(events) + } +} + +impl TraceWriter for NonStreamingTraceWriter { + fn begin_writing_trace_metadata(&mut self, path: &Path) -> Result<(), Box> { + self.trace_metadata_path = Some(path.to_path_buf()); + Ok(()) + } + + fn begin_writing_trace_events(&mut self, path: &Path) -> Result<(), Box> { + self.trace_events_path = Some(path.to_path_buf()); + Ok(()) + } + + fn begin_writing_trace_paths(&mut self, path: &Path) -> Result<(), Box> { + self.trace_paths_path = Some(path.to_path_buf()); + Ok(()) + } + + fn finish_writing_trace_metadata(&mut self) -> Result<(), Box> { + if let Some(path) = &self.trace_metadata_path { + let trace_metadata = TraceMetadata { + program: self.get_data().program.clone(), + args: self.get_data().args.clone(), + workdir: self.get_data().workdir.clone(), + }; + let json = serde_json::to_string(&trace_metadata)?; + fs::write(path, json)?; + Ok(()) + } else { + panic!("finish_writing_trace_metadata() called without previous call to begin_writing_trace_metadata()"); + } + } + + fn finish_writing_trace_events(&mut self) -> Result<(), Box> { + if let Some(path) = &self.trace_events_path { + match self.format { + TraceEventsFileFormat::Json => { + let json = serde_json::to_string(&self.events)?; + fs::write(path, json)?; + } + TraceEventsFileFormat::BinaryV0 => { + let mut file = fs::File::create(path)?; + crate::capnptrace::write_trace(&self.events, &mut file)?; + } + TraceEventsFileFormat::Binary => { + unreachable!() + } + } + Ok(()) + } else { + panic!("finish_writing_trace_events() called without previous call to begin_writing_trace_events()"); + } + } + + fn finish_writing_trace_paths(&mut self) -> Result<(), Box> { + if let Some(path) = &self.trace_paths_path { + let json = serde_json::to_string(&self.get_data().path_list)?; + fs::write(path, json)?; + Ok(()) + } else { + panic!("finish_writing_trace_paths() called without previous call to begin_writing_trace_paths()"); + } + } +} diff --git a/runtime_tracing/src/tracer.rs b/runtime_tracing/src/tracer.rs index 6955923..40a6c10 100644 --- a/runtime_tracing/src/tracer.rs +++ b/runtime_tracing/src/tracer.rs @@ -1,37 +1,13 @@ //! Helper for generating trace events from a running program or interpreter. -use std::collections::HashMap; -use std::env; -use std::error::Error; -use std::fs; -use std::path::{Path, PathBuf}; - -use crate::abstract_trace_writer::{AbstractTraceWriter, AbstractTraceWriterData}; +use crate::non_streaming_trace_writer::NonStreamingTraceWriter; use crate::trace_readers::{BinaryTraceReader, JsonTraceReader, TraceReader}; use crate::types::{ - FunctionId, TraceLowLevelEvent, TraceMetadata, TypeId, + FunctionId, TypeId, ValueRecord, }; use crate::TraceWriter; - -/// State machine used to record [`TraceLowLevelEvent`]s. -/// -/// A `NonStreamingTraceWriter` instance accumulates events in memory and stores them on -/// disk when the `finish_writing_trace_*` methods are called. The in-memory event list -/// is exposed publicly. -pub struct NonStreamingTraceWriter { - base: AbstractTraceWriterData, - - // trace events - pub events: Vec, - - format: TraceEventsFileFormat, - trace_metadata_path: Option, - trace_events_path: Option, - trace_paths_path: Option, -} - #[derive(Debug, Clone, Copy)] pub enum TraceEventsFileFormat { Json, @@ -50,117 +26,6 @@ pub const NONE_VALUE: ValueRecord = ValueRecord::None { type_id: NONE_TYPE_ID }; pub const TOP_LEVEL_FUNCTION_ID: FunctionId = FunctionId(0); -impl NonStreamingTraceWriter { - /// Create a new tracer instance for the given program and arguments. - pub fn new(program: &str, args: &[String]) -> Self { - NonStreamingTraceWriter { - base: AbstractTraceWriterData { - workdir: env::current_dir().expect("can access the current dir"), - program: program.to_string(), - args: args.to_vec(), - - path_list: vec![], - function_list: vec![], - paths: HashMap::new(), - functions: HashMap::new(), - variables: HashMap::new(), - types: HashMap::new(), - }, - - events: vec![], - - format: TraceEventsFileFormat::Binary, - trace_metadata_path: None, - trace_events_path: None, - trace_paths_path: None, - } - } - - pub fn set_format(&mut self, format: TraceEventsFileFormat) { - self.format = format; - } -} - -impl AbstractTraceWriter for NonStreamingTraceWriter { - fn get_data(&self) -> &AbstractTraceWriterData { - &self.base - } - - fn get_mut_data(&mut self) -> &mut AbstractTraceWriterData { - &mut self.base - } - - fn add_event(&mut self, event: TraceLowLevelEvent) { - self.events.push(event) - } - - fn append_events(&mut self, events: &mut Vec) { - self.events.append(events) - } -} - -impl TraceWriter for NonStreamingTraceWriter { - fn begin_writing_trace_metadata(&mut self, path: &Path) -> Result<(), Box> { - self.trace_metadata_path = Some(path.to_path_buf()); - Ok(()) - } - - fn begin_writing_trace_events(&mut self, path: &Path) -> Result<(), Box> { - self.trace_events_path = Some(path.to_path_buf()); - Ok(()) - } - - fn begin_writing_trace_paths(&mut self, path: &Path) -> Result<(), Box> { - self.trace_paths_path = Some(path.to_path_buf()); - Ok(()) - } - - fn finish_writing_trace_metadata(&mut self) -> Result<(), Box> { - if let Some(path) = &self.trace_metadata_path { - let trace_metadata = TraceMetadata { - program: self.get_data().program.clone(), - args: self.get_data().args.clone(), - workdir: self.get_data().workdir.clone(), - }; - let json = serde_json::to_string(&trace_metadata)?; - fs::write(path, json)?; - Ok(()) - } else { - panic!("finish_writing_trace_metadata() called without previous call to begin_writing_trace_metadata()"); - } - } - - fn finish_writing_trace_events(&mut self) -> Result<(), Box> { - if let Some(path) = &self.trace_events_path { - match self.format { - TraceEventsFileFormat::Json => { - let json = serde_json::to_string(&self.events)?; - fs::write(path, json)?; - } - TraceEventsFileFormat::BinaryV0 => { - let mut file = fs::File::create(path)?; - crate::capnptrace::write_trace(&self.events, &mut file)?; - } - TraceEventsFileFormat::Binary => { - unreachable!() - } - } - Ok(()) - } else { - panic!("finish_writing_trace_events() called without previous call to begin_writing_trace_events()"); - } - } - - fn finish_writing_trace_paths(&mut self) -> Result<(), Box> { - if let Some(path) = &self.trace_paths_path { - let json = serde_json::to_string(&self.get_data().path_list)?; - fs::write(path, json)?; - Ok(()) - } else { - panic!("finish_writing_trace_paths() called without previous call to begin_writing_trace_paths()"); - } - } -} pub fn create_trace_reader(format: TraceEventsFileFormat) -> Box { match format { From de4cb0b7f5cef5313cfe67db00b35f8954a1f834 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolov Date: Thu, 24 Jul 2025 13:57:32 +0300 Subject: [PATCH 23/28] refactor(cbor_plus_zstd): factored out the repeated code for begin_writing_trace_metadata, begin_writing_trace_paths, finish_writing_trace_metadata and finish_writing_trace_paths into AbstractTraceWriterData --- runtime_tracing/src/abstract_trace_writer.rs | 42 ++++++++++++++++- runtime_tracing/src/cbor_zstd_writer.rs | 46 ++----------------- .../src/non_streaming_trace_writer.rs | 44 ++---------------- runtime_tracing/src/trace_writer.rs | 16 +++++-- 4 files changed, 61 insertions(+), 87 deletions(-) diff --git a/runtime_tracing/src/abstract_trace_writer.rs b/runtime_tracing/src/abstract_trace_writer.rs index c2ad180..cf3b2ec 100644 --- a/runtime_tracing/src/abstract_trace_writer.rs +++ b/runtime_tracing/src/abstract_trace_writer.rs @@ -1,6 +1,6 @@ -use std::{collections::HashMap, path::PathBuf}; +use std::{collections::HashMap, error::Error, fs, path::{Path, PathBuf}}; -use crate::{tracer::TOP_LEVEL_FUNCTION_ID, AssignCellRecord, AssignCompoundItemRecord, AssignmentRecord, CallRecord, CellValueRecord, CompoundValueRecord, FullValueRecord, FunctionId, FunctionRecord, Line, PathId, RValue, RecordEvent, ReturnRecord, StepRecord, TraceLowLevelEvent, TypeId, TypeKind, TypeRecord, TypeSpecificInfo, VariableCellRecord, VariableId, NONE_TYPE_ID}; +use crate::{tracer::TOP_LEVEL_FUNCTION_ID, AssignCellRecord, AssignCompoundItemRecord, AssignmentRecord, CallRecord, CellValueRecord, CompoundValueRecord, FullValueRecord, FunctionId, FunctionRecord, Line, PathId, RValue, RecordEvent, ReturnRecord, StepRecord, TraceLowLevelEvent, TraceMetadata, TypeId, TypeKind, TypeRecord, TypeSpecificInfo, VariableCellRecord, VariableId, NONE_TYPE_ID}; pub struct AbstractTraceWriterData { // trace metadata: @@ -15,6 +15,9 @@ pub struct AbstractTraceWriterData { pub functions: HashMap, pub variables: HashMap, pub types: HashMap, + + pub trace_metadata_path: Option, + pub trace_paths_path: Option, } pub trait AbstractTraceWriter { @@ -24,6 +27,16 @@ pub trait AbstractTraceWriter { fn add_event(&mut self, event: TraceLowLevelEvent); fn append_events(&mut self, events: &mut Vec); + fn begin_writing_trace_metadata(&mut self, path: &Path) -> Result<(), Box> { + self.get_mut_data().trace_metadata_path = Some(path.to_path_buf()); + Ok(()) + } + + fn begin_writing_trace_paths(&mut self, path: &Path) -> Result<(), Box> { + self.get_mut_data().trace_paths_path = Some(path.to_path_buf()); + Ok(()) + } + fn start(&mut self, path: &std::path::Path, line: Line) { let function_id = self.ensure_function_id("", path, line); self.register_call(function_id, vec![]); @@ -240,4 +253,29 @@ pub trait AbstractTraceWriter { fn drop_last_step(&mut self) { self.add_event(TraceLowLevelEvent::DropLastStep); } + + fn finish_writing_trace_metadata(&mut self) -> Result<(), Box> { + if let Some(path) = &self.get_data().trace_metadata_path { + let trace_metadata = TraceMetadata { + program: self.get_data().program.clone(), + args: self.get_data().args.clone(), + workdir: self.get_data().workdir.clone(), + }; + let json = serde_json::to_string(&trace_metadata)?; + fs::write(path, json)?; + Ok(()) + } else { + panic!("finish_writing_trace_metadata() called without previous call to begin_writing_trace_metadata()"); + } + } + + fn finish_writing_trace_paths(&mut self) -> Result<(), Box> { + if let Some(path) = &self.get_data().trace_paths_path { + let json = serde_json::to_string(&self.get_data().path_list)?; + fs::write(path, json)?; + Ok(()) + } else { + panic!("finish_writing_trace_paths() called without previous call to begin_writing_trace_paths()"); + } + } } diff --git a/runtime_tracing/src/cbor_zstd_writer.rs b/runtime_tracing/src/cbor_zstd_writer.rs index dee96b9..21278f9 100644 --- a/runtime_tracing/src/cbor_zstd_writer.rs +++ b/runtime_tracing/src/cbor_zstd_writer.rs @@ -1,7 +1,7 @@ -use std::{collections::HashMap, env, fs::{self, File}, io::Write, path::PathBuf}; +use std::{collections::HashMap, env, fs::File, io::Write, path::PathBuf}; use zeekstd::Encoder; -use crate::{abstract_trace_writer::{AbstractTraceWriter, AbstractTraceWriterData}, trace_writer::TraceWriter, TraceLowLevelEvent, TraceMetadata}; +use crate::{abstract_trace_writer::{AbstractTraceWriter, AbstractTraceWriterData}, trace_writer::TraceWriter, TraceLowLevelEvent}; /// The next 3 bytes are reserved/version info. /// The header is 8 bytes in size, ensuring 64-bit alignment for the rest of the file. @@ -13,10 +13,8 @@ pub const HEADERV1: &[u8] = &[ pub struct StreamingTraceWriter<'a> { base: AbstractTraceWriterData, - trace_metadata_path: Option, trace_events_path: Option, trace_events_file_zstd_encoder: Option>, - trace_paths_path: Option, } impl<'a> StreamingTraceWriter<'a> { @@ -34,12 +32,13 @@ impl<'a> StreamingTraceWriter<'a> { functions: HashMap::new(), variables: HashMap::new(), types: HashMap::new(), + + trace_metadata_path: None, + trace_paths_path: None, }, - trace_metadata_path: None, trace_events_path: None, trace_events_file_zstd_encoder: None, - trace_paths_path: None, } } } @@ -69,11 +68,6 @@ impl<'a> AbstractTraceWriter for StreamingTraceWriter<'a> { } impl<'a> TraceWriter for StreamingTraceWriter<'a> { - fn begin_writing_trace_metadata(&mut self, path: &std::path::Path) -> Result<(), Box> { - self.trace_metadata_path = Some(path.to_path_buf()); - Ok(()) - } - fn begin_writing_trace_events(&mut self, path: &std::path::Path) -> Result<(), Box> { let pb = path.to_path_buf(); self.trace_events_path = Some(pb.clone()); @@ -84,26 +78,6 @@ impl<'a> TraceWriter for StreamingTraceWriter<'a> { Ok(()) } - fn begin_writing_trace_paths(&mut self, path: &std::path::Path) -> Result<(), Box> { - self.trace_paths_path = Some(path.to_path_buf()); - Ok(()) - } - - fn finish_writing_trace_metadata(&mut self) -> Result<(), Box> { - if let Some(path) = &self.trace_metadata_path { - let trace_metadata = TraceMetadata { - program: self.get_data().program.clone(), - args: self.get_data().args.clone(), - workdir: self.get_data().workdir.clone(), - }; - let json = serde_json::to_string(&trace_metadata)?; - fs::write(path, json)?; - Ok(()) - } else { - panic!("finish_writing_trace_metadata() called without previous call to begin_writing_trace_metadata()"); - } - } - fn finish_writing_trace_events(&mut self) -> Result<(), Box> { if let Some(enc) = self.trace_events_file_zstd_encoder.take() { enc.finish()?; @@ -113,14 +87,4 @@ impl<'a> TraceWriter for StreamingTraceWriter<'a> { panic!("finish_writing_trace_events() called without previous call to begin_writing_trace_events()"); } } - - fn finish_writing_trace_paths(&mut self) -> Result<(), Box> { - if let Some(path) = &self.trace_paths_path { - let json = serde_json::to_string(&self.get_data().path_list)?; - fs::write(path, json)?; - Ok(()) - } else { - panic!("finish_writing_trace_paths() called without previous call to begin_writing_trace_paths()"); - } - } } diff --git a/runtime_tracing/src/non_streaming_trace_writer.rs b/runtime_tracing/src/non_streaming_trace_writer.rs index 499a4b9..083c417 100644 --- a/runtime_tracing/src/non_streaming_trace_writer.rs +++ b/runtime_tracing/src/non_streaming_trace_writer.rs @@ -1,6 +1,6 @@ use std::{collections::HashMap, env, error::Error, fs, path::{Path, PathBuf}}; -use crate::{abstract_trace_writer::{AbstractTraceWriter, AbstractTraceWriterData}, TraceEventsFileFormat, TraceLowLevelEvent, TraceMetadata, TraceWriter}; +use crate::{abstract_trace_writer::{AbstractTraceWriter, AbstractTraceWriterData}, TraceEventsFileFormat, TraceLowLevelEvent, TraceWriter}; /// State machine used to record [`TraceLowLevelEvent`]s. /// @@ -14,9 +14,7 @@ pub struct NonStreamingTraceWriter { pub events: Vec, format: TraceEventsFileFormat, - trace_metadata_path: Option, trace_events_path: Option, - trace_paths_path: Option, } impl NonStreamingTraceWriter { @@ -34,14 +32,15 @@ impl NonStreamingTraceWriter { functions: HashMap::new(), variables: HashMap::new(), types: HashMap::new(), + + trace_metadata_path: None, + trace_paths_path: None, }, events: vec![], format: TraceEventsFileFormat::Binary, - trace_metadata_path: None, trace_events_path: None, - trace_paths_path: None, } } @@ -69,36 +68,11 @@ impl AbstractTraceWriter for NonStreamingTraceWriter { } impl TraceWriter for NonStreamingTraceWriter { - fn begin_writing_trace_metadata(&mut self, path: &Path) -> Result<(), Box> { - self.trace_metadata_path = Some(path.to_path_buf()); - Ok(()) - } - fn begin_writing_trace_events(&mut self, path: &Path) -> Result<(), Box> { self.trace_events_path = Some(path.to_path_buf()); Ok(()) } - fn begin_writing_trace_paths(&mut self, path: &Path) -> Result<(), Box> { - self.trace_paths_path = Some(path.to_path_buf()); - Ok(()) - } - - fn finish_writing_trace_metadata(&mut self) -> Result<(), Box> { - if let Some(path) = &self.trace_metadata_path { - let trace_metadata = TraceMetadata { - program: self.get_data().program.clone(), - args: self.get_data().args.clone(), - workdir: self.get_data().workdir.clone(), - }; - let json = serde_json::to_string(&trace_metadata)?; - fs::write(path, json)?; - Ok(()) - } else { - panic!("finish_writing_trace_metadata() called without previous call to begin_writing_trace_metadata()"); - } - } - fn finish_writing_trace_events(&mut self) -> Result<(), Box> { if let Some(path) = &self.trace_events_path { match self.format { @@ -119,14 +93,4 @@ impl TraceWriter for NonStreamingTraceWriter { panic!("finish_writing_trace_events() called without previous call to begin_writing_trace_events()"); } } - - fn finish_writing_trace_paths(&mut self) -> Result<(), Box> { - if let Some(path) = &self.trace_paths_path { - let json = serde_json::to_string(&self.get_data().path_list)?; - fs::write(path, json)?; - Ok(()) - } else { - panic!("finish_writing_trace_paths() called without previous call to begin_writing_trace_paths()"); - } - } } diff --git a/runtime_tracing/src/trace_writer.rs b/runtime_tracing/src/trace_writer.rs index 86767a7..9823878 100644 --- a/runtime_tracing/src/trace_writer.rs +++ b/runtime_tracing/src/trace_writer.rs @@ -3,9 +3,13 @@ use std::{error::Error, path::Path}; use crate::{abstract_trace_writer::AbstractTraceWriter, EventLogKind, FullValueRecord, FunctionId, Line, PassBy, PathId, Place, RValue, TraceLowLevelEvent, TypeId, TypeKind, TypeRecord, ValueRecord, VariableId}; pub trait TraceWriter: AbstractTraceWriter { - fn begin_writing_trace_metadata(&mut self, path: &Path) -> Result<(), Box>; + fn begin_writing_trace_metadata(&mut self, path: &Path) -> Result<(), Box> { + AbstractTraceWriter::begin_writing_trace_metadata(self, path) + } fn begin_writing_trace_events(&mut self, path: &Path) -> Result<(), Box>; - fn begin_writing_trace_paths(&mut self, path: &Path) -> Result<(), Box>; + fn begin_writing_trace_paths(&mut self, path: &Path) -> Result<(), Box> { + AbstractTraceWriter::begin_writing_trace_paths(self, path) + } fn start(&mut self, path: &Path, line: Line) { AbstractTraceWriter::start(self, path, line) @@ -113,7 +117,11 @@ pub trait TraceWriter: AbstractTraceWriter { AbstractTraceWriter::append_events(self, events) } - fn finish_writing_trace_metadata(&mut self) -> Result<(), Box>; + fn finish_writing_trace_metadata(&mut self) -> Result<(), Box> { + AbstractTraceWriter::finish_writing_trace_metadata(self) + } fn finish_writing_trace_events(&mut self) -> Result<(), Box>; - fn finish_writing_trace_paths(&mut self) -> Result<(), Box>; + fn finish_writing_trace_paths(&mut self) -> Result<(), Box> { + AbstractTraceWriter::finish_writing_trace_paths(self) + } } From 295a012d9e6964598b38ae82a5bb5aab7abf1758 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolov Date: Thu, 24 Jul 2025 14:02:21 +0300 Subject: [PATCH 24/28] refactor(cbor_plus_zstd): factored out the repeated initialization code for AbstractTraceWriterData into a AbstractTraceWriterData::new() constructor-like function --- runtime_tracing/src/abstract_trace_writer.rs | 22 ++++++++++++++++++- runtime_tracing/src/cbor_zstd_writer.rs | 18 ++------------- .../src/non_streaming_trace_writer.rs | 18 ++------------- 3 files changed, 25 insertions(+), 33 deletions(-) diff --git a/runtime_tracing/src/abstract_trace_writer.rs b/runtime_tracing/src/abstract_trace_writer.rs index cf3b2ec..c7d12a1 100644 --- a/runtime_tracing/src/abstract_trace_writer.rs +++ b/runtime_tracing/src/abstract_trace_writer.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, error::Error, fs, path::{Path, PathBuf}}; +use std::{collections::HashMap, env, error::Error, fs, path::{Path, PathBuf}}; use crate::{tracer::TOP_LEVEL_FUNCTION_ID, AssignCellRecord, AssignCompoundItemRecord, AssignmentRecord, CallRecord, CellValueRecord, CompoundValueRecord, FullValueRecord, FunctionId, FunctionRecord, Line, PathId, RValue, RecordEvent, ReturnRecord, StepRecord, TraceLowLevelEvent, TraceMetadata, TypeId, TypeKind, TypeRecord, TypeSpecificInfo, VariableCellRecord, VariableId, NONE_TYPE_ID}; @@ -20,6 +20,26 @@ pub struct AbstractTraceWriterData { pub trace_paths_path: Option, } +impl AbstractTraceWriterData { + pub fn new(program: &str, args: &[String]) -> Self { + AbstractTraceWriterData { + workdir: env::current_dir().expect("can access the current dir"), + program: program.to_string(), + args: args.to_vec(), + + path_list: vec![], + function_list: vec![], + paths: HashMap::new(), + functions: HashMap::new(), + variables: HashMap::new(), + types: HashMap::new(), + + trace_metadata_path: None, + trace_paths_path: None, + } + } +} + pub trait AbstractTraceWriter { fn get_data(&self) -> &AbstractTraceWriterData; fn get_mut_data(&mut self) -> &mut AbstractTraceWriterData; diff --git a/runtime_tracing/src/cbor_zstd_writer.rs b/runtime_tracing/src/cbor_zstd_writer.rs index 21278f9..8b5bc02 100644 --- a/runtime_tracing/src/cbor_zstd_writer.rs +++ b/runtime_tracing/src/cbor_zstd_writer.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, env, fs::File, io::Write, path::PathBuf}; +use std::{fs::File, io::Write, path::PathBuf}; use zeekstd::Encoder; use crate::{abstract_trace_writer::{AbstractTraceWriter, AbstractTraceWriterData}, trace_writer::TraceWriter, TraceLowLevelEvent}; @@ -21,21 +21,7 @@ impl<'a> StreamingTraceWriter<'a> { /// Create a new tracer instance for the given program and arguments. pub fn new(program: &str, args: &[String]) -> Self { StreamingTraceWriter { - base: AbstractTraceWriterData { - workdir: env::current_dir().expect("can access the current dir"), - program: program.to_string(), - args: args.to_vec(), - - path_list: vec![], - function_list: vec![], - paths: HashMap::new(), - functions: HashMap::new(), - variables: HashMap::new(), - types: HashMap::new(), - - trace_metadata_path: None, - trace_paths_path: None, - }, + base: AbstractTraceWriterData::new(program, args), trace_events_path: None, trace_events_file_zstd_encoder: None, diff --git a/runtime_tracing/src/non_streaming_trace_writer.rs b/runtime_tracing/src/non_streaming_trace_writer.rs index 083c417..c710900 100644 --- a/runtime_tracing/src/non_streaming_trace_writer.rs +++ b/runtime_tracing/src/non_streaming_trace_writer.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, env, error::Error, fs, path::{Path, PathBuf}}; +use std::{error::Error, fs, path::{Path, PathBuf}}; use crate::{abstract_trace_writer::{AbstractTraceWriter, AbstractTraceWriterData}, TraceEventsFileFormat, TraceLowLevelEvent, TraceWriter}; @@ -21,21 +21,7 @@ impl NonStreamingTraceWriter { /// Create a new tracer instance for the given program and arguments. pub fn new(program: &str, args: &[String]) -> Self { NonStreamingTraceWriter { - base: AbstractTraceWriterData { - workdir: env::current_dir().expect("can access the current dir"), - program: program.to_string(), - args: args.to_vec(), - - path_list: vec![], - function_list: vec![], - paths: HashMap::new(), - functions: HashMap::new(), - variables: HashMap::new(), - types: HashMap::new(), - - trace_metadata_path: None, - trace_paths_path: None, - }, + base: AbstractTraceWriterData::new(program, args), events: vec![], From 2d52028e991a310b87c09e6fc3fea14d8a7a7b73 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolov Date: Thu, 24 Jul 2025 14:04:23 +0300 Subject: [PATCH 25/28] refactor(cbor_plus_zstd): StreamingTraceWriter renamed CborZstdTraceWriter --- runtime_tracing/src/cbor_zstd_writer.rs | 10 +++++----- runtime_tracing/src/tracer.rs | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/runtime_tracing/src/cbor_zstd_writer.rs b/runtime_tracing/src/cbor_zstd_writer.rs index 8b5bc02..6974e87 100644 --- a/runtime_tracing/src/cbor_zstd_writer.rs +++ b/runtime_tracing/src/cbor_zstd_writer.rs @@ -10,17 +10,17 @@ pub const HEADERV1: &[u8] = &[ 0x01, // Indicates version 1 of the file format 0x00, 0x00]; // Reserved, must be zero in this version. -pub struct StreamingTraceWriter<'a> { +pub struct CborZstdTraceWriter<'a> { base: AbstractTraceWriterData, trace_events_path: Option, trace_events_file_zstd_encoder: Option>, } -impl<'a> StreamingTraceWriter<'a> { +impl<'a> CborZstdTraceWriter<'a> { /// Create a new tracer instance for the given program and arguments. pub fn new(program: &str, args: &[String]) -> Self { - StreamingTraceWriter { + CborZstdTraceWriter { base: AbstractTraceWriterData::new(program, args), trace_events_path: None, @@ -29,7 +29,7 @@ impl<'a> StreamingTraceWriter<'a> { } } -impl<'a> AbstractTraceWriter for StreamingTraceWriter<'a> { +impl<'a> AbstractTraceWriter for CborZstdTraceWriter<'a> { fn get_data(&self) -> &AbstractTraceWriterData { &self.base } @@ -53,7 +53,7 @@ impl<'a> AbstractTraceWriter for StreamingTraceWriter<'a> { } } -impl<'a> TraceWriter for StreamingTraceWriter<'a> { +impl<'a> TraceWriter for CborZstdTraceWriter<'a> { fn begin_writing_trace_events(&mut self, path: &std::path::Path) -> Result<(), Box> { let pb = path.to_path_buf(); self.trace_events_path = Some(pb.clone()); diff --git a/runtime_tracing/src/tracer.rs b/runtime_tracing/src/tracer.rs index 40a6c10..13e9788 100644 --- a/runtime_tracing/src/tracer.rs +++ b/runtime_tracing/src/tracer.rs @@ -44,7 +44,7 @@ pub fn create_trace_writer(program: &str, args: &[String], format: TraceEventsFi result } TraceEventsFileFormat::Binary => { - Box::new(crate::cbor_zstd_writer::StreamingTraceWriter::new(program, args)) + Box::new(crate::cbor_zstd_writer::CborZstdTraceWriter::new(program, args)) } } } From 6084732bb17db1604bd6dfcf99b98802656e2acf Mon Sep 17 00:00:00 2001 From: Nikolay Nikolov Date: Thu, 24 Jul 2025 14:06:45 +0300 Subject: [PATCH 26/28] chore: cargo fmt --- runtime_tracing/src/abstract_trace_writer.rs | 16 ++++++++++++--- runtime_tracing/src/cbor_zstd_reader.rs | 4 ++-- runtime_tracing/src/cbor_zstd_writer.rs | 15 +++++++++----- runtime_tracing/src/lib.rs | 14 ++++++------- .../src/non_streaming_trace_writer.rs | 11 ++++++++-- runtime_tracing/src/trace_readers.rs | 13 +++++++----- runtime_tracing/src/trace_writer.rs | 5 ++++- runtime_tracing/src/tracer.rs | 20 ++++++------------- runtime_tracing/tests/binary_format.rs | 2 +- 9 files changed, 59 insertions(+), 41 deletions(-) diff --git a/runtime_tracing/src/abstract_trace_writer.rs b/runtime_tracing/src/abstract_trace_writer.rs index c7d12a1..4f350bf 100644 --- a/runtime_tracing/src/abstract_trace_writer.rs +++ b/runtime_tracing/src/abstract_trace_writer.rs @@ -1,6 +1,16 @@ -use std::{collections::HashMap, env, error::Error, fs, path::{Path, PathBuf}}; - -use crate::{tracer::TOP_LEVEL_FUNCTION_ID, AssignCellRecord, AssignCompoundItemRecord, AssignmentRecord, CallRecord, CellValueRecord, CompoundValueRecord, FullValueRecord, FunctionId, FunctionRecord, Line, PathId, RValue, RecordEvent, ReturnRecord, StepRecord, TraceLowLevelEvent, TraceMetadata, TypeId, TypeKind, TypeRecord, TypeSpecificInfo, VariableCellRecord, VariableId, NONE_TYPE_ID}; +use std::{ + collections::HashMap, + env, + error::Error, + fs, + path::{Path, PathBuf}, +}; + +use crate::{ + AssignCellRecord, AssignCompoundItemRecord, AssignmentRecord, CallRecord, CellValueRecord, CompoundValueRecord, FullValueRecord, FunctionId, + FunctionRecord, Line, NONE_TYPE_ID, PathId, RValue, RecordEvent, ReturnRecord, StepRecord, TraceLowLevelEvent, TraceMetadata, TypeId, TypeKind, + TypeRecord, TypeSpecificInfo, VariableCellRecord, VariableId, tracer::TOP_LEVEL_FUNCTION_ID, +}; pub struct AbstractTraceWriterData { // trace metadata: diff --git a/runtime_tracing/src/cbor_zstd_reader.rs b/runtime_tracing/src/cbor_zstd_reader.rs index 0ce5442..18f2d9e 100644 --- a/runtime_tracing/src/cbor_zstd_reader.rs +++ b/runtime_tracing/src/cbor_zstd_reader.rs @@ -3,14 +3,14 @@ use std::io::{self, BufRead, BufReader, Read, Seek, Write}; use fscommon::StreamSlice; use zeekstd::Decoder; -use crate::{cbor_zstd_writer::HEADERV1, TraceLowLevelEvent}; +use crate::{TraceLowLevelEvent, cbor_zstd_writer::HEADERV1}; fn is_at_eof(reader: &mut R) -> io::Result { let buffer = reader.fill_buf()?; Ok(buffer.is_empty()) } -pub fn read_trace(input: &mut (impl Read + Write + Seek)) -> Result, Box> { +pub fn read_trace(input: &mut (impl Read + Write + Seek)) -> Result, Box> { let end_pos = input.seek(io::SeekFrom::End(0))?; input.seek(io::SeekFrom::Start(0))?; diff --git a/runtime_tracing/src/cbor_zstd_writer.rs b/runtime_tracing/src/cbor_zstd_writer.rs index 6974e87..56bbdd0 100644 --- a/runtime_tracing/src/cbor_zstd_writer.rs +++ b/runtime_tracing/src/cbor_zstd_writer.rs @@ -1,14 +1,19 @@ use std::{fs::File, io::Write, path::PathBuf}; use zeekstd::Encoder; -use crate::{abstract_trace_writer::{AbstractTraceWriter, AbstractTraceWriterData}, trace_writer::TraceWriter, TraceLowLevelEvent}; +use crate::{ + TraceLowLevelEvent, + abstract_trace_writer::{AbstractTraceWriter, AbstractTraceWriterData}, + trace_writer::TraceWriter, +}; /// The next 3 bytes are reserved/version info. /// The header is 8 bytes in size, ensuring 64-bit alignment for the rest of the file. pub const HEADERV1: &[u8] = &[ - 0xC0, 0xDE, 0x72, 0xAC, 0xE2, // The first 5 bytes identify the file as a CodeTracer file (hex l33tsp33k - C0DE72ACE2 for "CodeTracer"). - 0x01, // Indicates version 1 of the file format - 0x00, 0x00]; // Reserved, must be zero in this version. + 0xC0, 0xDE, 0x72, 0xAC, 0xE2, // The first 5 bytes identify the file as a CodeTracer file (hex l33tsp33k - C0DE72ACE2 for "CodeTracer"). + 0x01, // Indicates version 1 of the file format + 0x00, 0x00, +]; // Reserved, must be zero in this version. pub struct CborZstdTraceWriter<'a> { base: AbstractTraceWriterData, @@ -33,7 +38,7 @@ impl<'a> AbstractTraceWriter for CborZstdTraceWriter<'a> { fn get_data(&self) -> &AbstractTraceWriterData { &self.base } - + fn get_mut_data(&mut self) -> &mut AbstractTraceWriterData { &mut self.base } diff --git a/runtime_tracing/src/lib.rs b/runtime_tracing/src/lib.rs index a3b14eb..3afd539 100644 --- a/runtime_tracing/src/lib.rs +++ b/runtime_tracing/src/lib.rs @@ -8,23 +8,21 @@ //! This crate provides the [`Tracer`] type for emitting trace events and a //! collection of serializable structures describing the trace format. //! The format is documented in `docs/` and the README. +mod abstract_trace_writer; mod base64; mod capnptrace; -mod tracer; -mod types; mod cbor_zstd_reader; mod cbor_zstd_writer; -mod abstract_trace_writer; +mod non_streaming_trace_writer; mod trace_readers; mod trace_writer; -mod non_streaming_trace_writer; +mod tracer; +mod types; -pub use crate::tracer::{ - create_trace_reader, create_trace_writer, TraceEventsFileFormat, NONE_TYPE_ID, NONE_VALUE, -}; +pub use crate::non_streaming_trace_writer::NonStreamingTraceWriter; pub use crate::trace_readers::TraceReader; pub use crate::trace_writer::TraceWriter; -pub use crate::non_streaming_trace_writer::NonStreamingTraceWriter; +pub use crate::tracer::{NONE_TYPE_ID, NONE_VALUE, TraceEventsFileFormat, create_trace_reader, create_trace_writer}; pub use crate::types::*; pub mod trace_capnp { diff --git a/runtime_tracing/src/non_streaming_trace_writer.rs b/runtime_tracing/src/non_streaming_trace_writer.rs index c710900..5a00c77 100644 --- a/runtime_tracing/src/non_streaming_trace_writer.rs +++ b/runtime_tracing/src/non_streaming_trace_writer.rs @@ -1,6 +1,13 @@ -use std::{error::Error, fs, path::{Path, PathBuf}}; +use std::{ + error::Error, + fs, + path::{Path, PathBuf}, +}; -use crate::{abstract_trace_writer::{AbstractTraceWriter, AbstractTraceWriterData}, TraceEventsFileFormat, TraceLowLevelEvent, TraceWriter}; +use crate::{ + TraceEventsFileFormat, TraceLowLevelEvent, TraceWriter, + abstract_trace_writer::{AbstractTraceWriter, AbstractTraceWriterData}, +}; /// State machine used to record [`TraceLowLevelEvent`]s. /// diff --git a/runtime_tracing/src/trace_readers.rs b/runtime_tracing/src/trace_readers.rs index 1c1f9c1..40d4821 100644 --- a/runtime_tracing/src/trace_readers.rs +++ b/runtime_tracing/src/trace_readers.rs @@ -1,6 +1,11 @@ -use std::{error::Error, fs::{self, File}, io::{BufReader, Read, Seek, SeekFrom}, path::Path}; +use std::{ + error::Error, + fs::{self, File}, + io::{BufReader, Read, Seek, SeekFrom}, + path::Path, +}; -use crate::{capnptrace::HEADER, cbor_zstd_writer::HEADERV1, TraceEventsFileFormat, TraceLowLevelEvent}; +use crate::{TraceEventsFileFormat, TraceLowLevelEvent, capnptrace::HEADER, cbor_zstd_writer::HEADERV1}; pub trait TraceReader { fn load_trace_events(&mut self, path: &Path) -> Result, Box>; @@ -41,9 +46,7 @@ impl TraceReader for BinaryTraceReader { let mut buf_reader = BufReader::new(file); Ok(crate::capnptrace::read_trace(&mut buf_reader)?) } - Some(TraceEventsFileFormat::Binary) => { - Ok(crate::cbor_zstd_reader::read_trace(&mut file)?) - } + Some(TraceEventsFileFormat::Binary) => Ok(crate::cbor_zstd_reader::read_trace(&mut file)?), Some(TraceEventsFileFormat::Json) => { unreachable!() } diff --git a/runtime_tracing/src/trace_writer.rs b/runtime_tracing/src/trace_writer.rs index 9823878..748f92b 100644 --- a/runtime_tracing/src/trace_writer.rs +++ b/runtime_tracing/src/trace_writer.rs @@ -1,6 +1,9 @@ use std::{error::Error, path::Path}; -use crate::{abstract_trace_writer::AbstractTraceWriter, EventLogKind, FullValueRecord, FunctionId, Line, PassBy, PathId, Place, RValue, TraceLowLevelEvent, TypeId, TypeKind, TypeRecord, ValueRecord, VariableId}; +use crate::{ + EventLogKind, FullValueRecord, FunctionId, Line, PassBy, PathId, Place, RValue, TraceLowLevelEvent, TypeId, TypeKind, TypeRecord, ValueRecord, + VariableId, abstract_trace_writer::AbstractTraceWriter, +}; pub trait TraceWriter: AbstractTraceWriter { fn begin_writing_trace_metadata(&mut self, path: &Path) -> Result<(), Box> { diff --git a/runtime_tracing/src/tracer.rs b/runtime_tracing/src/tracer.rs index 13e9788..49c4233 100644 --- a/runtime_tracing/src/tracer.rs +++ b/runtime_tracing/src/tracer.rs @@ -1,18 +1,15 @@ //! Helper for generating trace events from a running program or interpreter. +use crate::TraceWriter; use crate::non_streaming_trace_writer::NonStreamingTraceWriter; use crate::trace_readers::{BinaryTraceReader, JsonTraceReader, TraceReader}; -use crate::types::{ - FunctionId, TypeId, - ValueRecord, -}; -use crate::TraceWriter; +use crate::types::{FunctionId, TypeId, ValueRecord}; #[derive(Debug, Clone, Copy)] pub enum TraceEventsFileFormat { Json, BinaryV0, - Binary + Binary, } // we ensure in start they are registered with those id-s @@ -26,25 +23,20 @@ pub const NONE_VALUE: ValueRecord = ValueRecord::None { type_id: NONE_TYPE_ID }; pub const TOP_LEVEL_FUNCTION_ID: FunctionId = FunctionId(0); - pub fn create_trace_reader(format: TraceEventsFileFormat) -> Box { match format { TraceEventsFileFormat::Json => Box::new(JsonTraceReader {}), - TraceEventsFileFormat::BinaryV0 | - TraceEventsFileFormat::Binary => Box::new(BinaryTraceReader {}), + TraceEventsFileFormat::BinaryV0 | TraceEventsFileFormat::Binary => Box::new(BinaryTraceReader {}), } } pub fn create_trace_writer(program: &str, args: &[String], format: TraceEventsFileFormat) -> Box { match format { - TraceEventsFileFormat::Json | - TraceEventsFileFormat::BinaryV0 => { + TraceEventsFileFormat::Json | TraceEventsFileFormat::BinaryV0 => { let mut result = Box::new(NonStreamingTraceWriter::new(program, args)); result.set_format(format); result } - TraceEventsFileFormat::Binary => { - Box::new(crate::cbor_zstd_writer::CborZstdTraceWriter::new(program, args)) - } + TraceEventsFileFormat::Binary => Box::new(crate::cbor_zstd_writer::CborZstdTraceWriter::new(program, args)), } } diff --git a/runtime_tracing/tests/binary_format.rs b/runtime_tracing/tests/binary_format.rs index a91812a..e4804f8 100644 --- a/runtime_tracing/tests/binary_format.rs +++ b/runtime_tracing/tests/binary_format.rs @@ -1,4 +1,4 @@ -use runtime_tracing::{create_trace_reader, create_trace_writer, TraceEventsFileFormat, TraceWriter}; +use runtime_tracing::{TraceEventsFileFormat, TraceWriter, create_trace_reader, create_trace_writer}; use std::fs; use std::path::Path; From 23f65bbca397d3670ff5f34f99f64253227a555d Mon Sep 17 00:00:00 2001 From: Nikolay Nikolov Date: Thu, 24 Jul 2025 14:10:56 +0300 Subject: [PATCH 27/28] chore: use write_all() instead of write() in CborZstdTraceWriter::add_event --- runtime_tracing/src/cbor_zstd_writer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime_tracing/src/cbor_zstd_writer.rs b/runtime_tracing/src/cbor_zstd_writer.rs index 56bbdd0..db136a1 100644 --- a/runtime_tracing/src/cbor_zstd_writer.rs +++ b/runtime_tracing/src/cbor_zstd_writer.rs @@ -47,7 +47,7 @@ impl<'a> AbstractTraceWriter for CborZstdTraceWriter<'a> { let buf: Vec = Vec::new(); let q = cbor4ii::serde::to_vec(buf, &event).unwrap(); if let Some(enc) = &mut self.trace_events_file_zstd_encoder { - enc.write(&q).unwrap(); + enc.write_all(&q).unwrap(); } } From 923ca9bb4bea7398125685ce994c9f4d6c582f01 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolov Date: Thu, 24 Jul 2025 14:16:04 +0300 Subject: [PATCH 28/28] chore: magic Rust lifetime elision woo-woo Thanks you very much, Clippy, for the suggestion! The code looks so poetic now! --- runtime_tracing/src/cbor_zstd_writer.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime_tracing/src/cbor_zstd_writer.rs b/runtime_tracing/src/cbor_zstd_writer.rs index db136a1..0bd10d8 100644 --- a/runtime_tracing/src/cbor_zstd_writer.rs +++ b/runtime_tracing/src/cbor_zstd_writer.rs @@ -22,7 +22,7 @@ pub struct CborZstdTraceWriter<'a> { trace_events_file_zstd_encoder: Option>, } -impl<'a> CborZstdTraceWriter<'a> { +impl CborZstdTraceWriter<'_> { /// Create a new tracer instance for the given program and arguments. pub fn new(program: &str, args: &[String]) -> Self { CborZstdTraceWriter { @@ -34,7 +34,7 @@ impl<'a> CborZstdTraceWriter<'a> { } } -impl<'a> AbstractTraceWriter for CborZstdTraceWriter<'a> { +impl AbstractTraceWriter for CborZstdTraceWriter<'_> { fn get_data(&self) -> &AbstractTraceWriterData { &self.base } @@ -58,7 +58,7 @@ impl<'a> AbstractTraceWriter for CborZstdTraceWriter<'a> { } } -impl<'a> TraceWriter for CborZstdTraceWriter<'a> { +impl TraceWriter for CborZstdTraceWriter<'_> { fn begin_writing_trace_events(&mut self, path: &std::path::Path) -> Result<(), Box> { let pb = path.to_path_buf(); self.trace_events_path = Some(pb.clone());