diff --git a/runtime_tracing/Cargo.toml b/runtime_tracing/Cargo.toml index 4e2d48a..665bd2e 100644 --- a/runtime_tracing/Cargo.toml +++ b/runtime_tracing/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "runtime_tracing" -version = "0.12.2" +version = "0.13.0" edition = "2021" authors = ["Metacraft Labs Ltd"] description = "A library for the schema and tracing helpers for the CodeTracer db trace format" diff --git a/runtime_tracing/build.rs b/runtime_tracing/build.rs index e51548f..f139dff 100644 --- a/runtime_tracing/build.rs +++ b/runtime_tracing/build.rs @@ -1,6 +1,3 @@ fn main() { - ::capnpc::CompilerCommand::new() - .file("src/trace.capnp") - .run() - .expect("compiling schema") + ::capnpc::CompilerCommand::new().file("src/trace.capnp").run().expect("compiling schema") } diff --git a/runtime_tracing/src/base64.rs b/runtime_tracing/src/base64.rs index a38e8f9..c451918 100644 --- a/runtime_tracing/src/base64.rs +++ b/runtime_tracing/src/base64.rs @@ -9,5 +9,7 @@ pub fn serialize(v: &Vec, s: S) -> Result { pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { let base64 = String::deserialize(d)?; - base64::engine::general_purpose::STANDARD.decode(base64.as_bytes()).map_err(serde::de::Error::custom) + base64::engine::general_purpose::STANDARD + .decode(base64.as_bytes()) + .map_err(serde::de::Error::custom) } diff --git a/runtime_tracing/src/capnptrace.rs b/runtime_tracing/src/capnptrace.rs index 4d1a5af..7b3a4aa 100644 --- a/runtime_tracing/src/capnptrace.rs +++ b/runtime_tracing/src/capnptrace.rs @@ -1,7 +1,7 @@ -use std::str::FromStr; use crate::trace_capnp::trace; -use capnp::serialize_packed; use crate::{TraceLowLevelEvent, VariableId}; +use capnp::serialize_packed; +use std::str::FromStr; /// The first 5 bytes identify the file as a CodeTracer file (hex l33tsp33k - C0DE72ACE2 for "CodeTracer"). /// The next 3 bytes are reserved/version info. In the initial version, they are zero. Non-zero values might @@ -142,11 +142,7 @@ impl From for trace::PassBy { } } - -fn conv_valuerecord( - bldr: crate::trace_capnp::trace::value_record::Builder, - vr: &crate::ValueRecord, -) { +fn conv_valuerecord(bldr: crate::trace_capnp::trace::value_record::Builder, vr: &crate::ValueRecord) { match vr { crate::ValueRecord::Int { i, type_id } => { let mut qi = bldr.init_int(); @@ -172,15 +168,9 @@ fn conv_valuerecord( let mut q_typ_id = qs.init_type_id(); q_typ_id.set_i(type_id.0.try_into().unwrap()); } - crate::ValueRecord::Sequence { - elements, - is_slice, - type_id, - } => { + crate::ValueRecord::Sequence { elements, is_slice, type_id } => { let mut qseq = bldr.init_sequence(); - let mut elems = qseq - .reborrow() - .init_elements(elements.len().try_into().unwrap()); + let mut elems = qseq.reborrow().init_elements(elements.len().try_into().unwrap()); for i in 0..elements.len() { let ele = &elements[i]; let bele = elems.reborrow().get(i.try_into().unwrap()); @@ -192,9 +182,7 @@ fn conv_valuerecord( } crate::ValueRecord::Tuple { elements, type_id } => { let mut qtup = bldr.init_tuple(); - let mut elems = qtup - .reborrow() - .init_elements(elements.len().try_into().unwrap()); + let mut elems = qtup.reborrow().init_elements(elements.len().try_into().unwrap()); for i in 0..elements.len() { let ele = &elements[i]; let bele = elems.reborrow().get(i.try_into().unwrap()); @@ -203,14 +191,9 @@ fn conv_valuerecord( let mut q_typ_id = qtup.init_type_id(); q_typ_id.set_i(type_id.0.try_into().unwrap()); } - crate::ValueRecord::Struct { - field_values, - type_id, - } => { + crate::ValueRecord::Struct { field_values, type_id } => { let mut qstruc = bldr.init_struct(); - let mut elems = qstruc - .reborrow() - .init_field_values(field_values.len().try_into().unwrap()); + let mut elems = qstruc.reborrow().init_field_values(field_values.len().try_into().unwrap()); for i in 0..field_values.len() { let ele = &field_values[i]; let bele = elems.reborrow().get(i.try_into().unwrap()); @@ -311,9 +294,7 @@ pub fn write_trace(q: &[crate::TraceLowLevelEvent], output: &mut impl std::io::W typ_id.set_i(ftr.type_id.0.try_into().unwrap()); } } - crate::TypeSpecificInfo::Pointer { - dereference_type_id, - } => { + crate::TypeSpecificInfo::Pointer { dereference_type_id } => { let ptr = specific_info.init_pointer(); let mut deref_typ_id = ptr.init_dereference_type_id(); deref_typ_id.set_i(dereference_type_id.0.try_into().unwrap()); @@ -336,8 +317,7 @@ pub fn write_trace(q: &[crate::TraceLowLevelEvent], output: &mut impl std::io::W let mut call_record = event.init_call(); let mut function_id = call_record.reborrow().init_function_id(); function_id.set_i(callrecord.function_id.0.try_into().unwrap()); - let mut function_args = - call_record.init_args(callrecord.args.len().try_into().unwrap()); + let mut function_args = call_record.init_args(callrecord.args.len().try_into().unwrap()); for i in 0..callrecord.args.len() { let farg = &callrecord.args[i]; let mut arg = function_args.reborrow().get(i.try_into().unwrap()); @@ -401,14 +381,14 @@ pub fn write_trace(q: &[crate::TraceLowLevelEvent], output: &mut impl std::io::W crate::RValue::Simple(variable_id) => { let mut ret_from_simple = ret_from.init_simple(); ret_from_simple.set_i(variable_id.0.try_into().unwrap()); - }, + } crate::RValue::Compound(variable_ids) => { let mut ret_from_compound = ret_from.init_compound(variable_ids.len().try_into().unwrap()); for i in 0..variable_ids.len() { let mut r = ret_from_compound.reborrow().get(i.try_into().unwrap()); r.set_i(variable_ids[i].0.try_into().unwrap()); } - }, + } } } TraceLowLevelEvent::DropLastStep => { @@ -469,20 +449,15 @@ pub fn write_trace(q: &[crate::TraceLowLevelEvent], output: &mut impl std::io::W serialize_packed::write_message(output, &message) } -fn get_value_records( - r: capnp::struct_list::Reader, -) -> Result, capnp::Error> { - let mut res: Vec = - Vec::with_capacity(r.len().try_into().unwrap()); +fn get_value_records(r: capnp::struct_list::Reader) -> Result, capnp::Error> { + let mut res: Vec = Vec::with_capacity(r.len().try_into().unwrap()); for i in 0..r.len() { res.push(get_value_record(r.get(i))?); } Ok(res) } -fn get_value_record( - r: trace::value_record::Reader, -) -> Result { +fn get_value_record(r: trace::value_record::Reader) -> Result { match r.which() { Ok(trace::value_record::Which::Int(q)) => Ok(crate::ValueRecord::Int { i: q.get_i(), @@ -500,13 +475,11 @@ fn get_value_record( text: q.get_text()?.to_string()?, type_id: crate::TypeId(q.get_type_id()?.get_i().try_into().unwrap()), }), - Ok(trace::value_record::Which::Sequence(q)) => { - Ok(crate::ValueRecord::Sequence { - elements: get_value_records(q.get_elements()?)?, - is_slice: q.get_is_slice(), - type_id: crate::TypeId(q.get_type_id()?.get_i().try_into().unwrap()), - }) - } + Ok(trace::value_record::Which::Sequence(q)) => Ok(crate::ValueRecord::Sequence { + elements: get_value_records(q.get_elements()?)?, + is_slice: q.get_is_slice(), + type_id: crate::TypeId(q.get_type_id()?.get_i().try_into().unwrap()), + }), Ok(trace::value_record::Which::Tuple(q)) => Ok(crate::ValueRecord::Tuple { elements: get_value_records(q.get_elements()?)?, type_id: crate::TypeId(q.get_type_id()?.get_i().try_into().unwrap()), @@ -515,21 +488,17 @@ fn get_value_record( field_values: get_value_records(q.get_field_values()?)?, type_id: crate::TypeId(q.get_type_id()?.get_i().try_into().unwrap()), }), - Ok(trace::value_record::Which::Variant(q)) => { - Ok(crate::ValueRecord::Variant { - discriminator: q.get_discriminator()?.to_string()?, - contents: Box::new(get_value_record(q.get_contents()?)?), - type_id: crate::TypeId(q.get_type_id()?.get_i().try_into().unwrap()), - }) - } - Ok(trace::value_record::Which::Reference(q)) => { - Ok(crate::ValueRecord::Reference { - dereferenced: Box::new(get_value_record(q.get_dereferenced()?)?), - address: q.get_address(), - mutable: q.get_mutable(), - type_id: crate::TypeId(q.get_type_id()?.get_i().try_into().unwrap()), - }) - } + Ok(trace::value_record::Which::Variant(q)) => Ok(crate::ValueRecord::Variant { + discriminator: q.get_discriminator()?.to_string()?, + contents: Box::new(get_value_record(q.get_contents()?)?), + type_id: crate::TypeId(q.get_type_id()?.get_i().try_into().unwrap()), + }), + Ok(trace::value_record::Which::Reference(q)) => Ok(crate::ValueRecord::Reference { + dereferenced: Box::new(get_value_record(q.get_dereferenced()?)?), + address: q.get_address(), + mutable: q.get_mutable(), + type_id: crate::TypeId(q.get_type_id()?.get_i().try_into().unwrap()), + }), Ok(trace::value_record::Which::Raw(q)) => Ok(crate::ValueRecord::Raw { r: q.get_r()?.to_string()?, type_id: crate::TypeId(q.get_type_id()?.get_i().try_into().unwrap()), @@ -553,13 +522,9 @@ fn get_value_record( } } -fn get_full_value_record( - r: trace::full_value_record::Reader, -) -> Result { +fn get_full_value_record(r: trace::full_value_record::Reader) -> Result { Ok(crate::FullValueRecord { - variable_id: crate::VariableId( - r.get_variable_id()?.get_i().try_into().unwrap(), - ), + variable_id: crate::VariableId(r.get_variable_id()?.get_i().try_into().unwrap()), value: get_value_record(r.get_value()?)?, }) } @@ -570,83 +535,58 @@ pub fn read_trace(input: &mut impl std::io::BufRead) -> ::capnp::Result()?; - let mut res: Vec = - Vec::with_capacity(trace.get_events()?.len().try_into().unwrap()); + let mut res: Vec = Vec::with_capacity(trace.get_events()?.len().try_into().unwrap()); for event in trace.get_events()? { let q = match event.which() { Ok(trace::trace_low_level_event::Which::Step(step_record)) => { let step_record = step_record?; TraceLowLevelEvent::Step(crate::StepRecord { - path_id: crate::PathId( - step_record.get_path_id()?.get_i().try_into().unwrap(), - ), + path_id: crate::PathId(step_record.get_path_id()?.get_i().try_into().unwrap()), line: crate::Line(step_record.get_line()?.get_l()), }) } Ok(trace::trace_low_level_event::Which::Path(path_buf)) => { - TraceLowLevelEvent::Path( - std::path::PathBuf::from_str(path_buf?.get_p()?.to_str()?).unwrap(), - ) - } - Ok(trace::trace_low_level_event::Which::VariableName(variable_name)) => { - TraceLowLevelEvent::VariableName(variable_name?.to_string()?) - } - Ok(trace::trace_low_level_event::Which::Variable(variable)) => { - TraceLowLevelEvent::Variable(variable?.to_string()?) + TraceLowLevelEvent::Path(std::path::PathBuf::from_str(path_buf?.get_p()?.to_str()?).unwrap()) } + Ok(trace::trace_low_level_event::Which::VariableName(variable_name)) => TraceLowLevelEvent::VariableName(variable_name?.to_string()?), + Ok(trace::trace_low_level_event::Which::Variable(variable)) => TraceLowLevelEvent::Variable(variable?.to_string()?), Ok(trace::trace_low_level_event::Which::Type(type_record)) => { let type_record = type_record?; TraceLowLevelEvent::Type(crate::TypeRecord { kind: type_record.get_kind()?.into(), lang_type: type_record.get_lang_type()?.to_string()?, specific_info: match type_record.get_specific_info()?.which() { - Ok(trace::type_specific_info::Which::None(())) => { - crate::TypeSpecificInfo::None - } + Ok(trace::type_specific_info::Which::None(())) => crate::TypeSpecificInfo::None, Ok(trace::type_specific_info::Which::Struct(s)) => { let s_fields = s.get_fields()?; - let mut fields: Vec = - Vec::with_capacity(s_fields.len().try_into().unwrap()); + let mut fields: Vec = Vec::with_capacity(s_fields.len().try_into().unwrap()); for s_field in s_fields { fields.push(crate::FieldTypeRecord { name: s_field.get_name()?.to_string()?, - type_id: crate::TypeId( - s_field.get_type_id()?.get_i().try_into().unwrap(), - ), + type_id: crate::TypeId(s_field.get_type_id()?.get_i().try_into().unwrap()), }); } crate::TypeSpecificInfo::Struct { fields } } - Ok(trace::type_specific_info::Which::Pointer(p)) => { - crate::TypeSpecificInfo::Pointer { - dereference_type_id: crate::TypeId( - p.get_dereference_type_id()?.get_i().try_into().unwrap(), - ), - } - } + Ok(trace::type_specific_info::Which::Pointer(p)) => crate::TypeSpecificInfo::Pointer { + dereference_type_id: crate::TypeId(p.get_dereference_type_id()?.get_i().try_into().unwrap()), + }, Err(_) => { panic!() } }, }) } - Ok(trace::trace_low_level_event::Which::Value(fvr)) => { - TraceLowLevelEvent::Value(get_full_value_record(fvr?)?) - } + Ok(trace::trace_low_level_event::Which::Value(fvr)) => TraceLowLevelEvent::Value(get_full_value_record(fvr?)?), Ok(trace::trace_low_level_event::Which::Function(function_record)) => { let function_record = function_record?; TraceLowLevelEvent::Function(crate::FunctionRecord { - path_id: crate::PathId( - function_record.get_path_id()?.get_i().try_into().unwrap(), - ), + path_id: crate::PathId(function_record.get_path_id()?.get_i().try_into().unwrap()), line: crate::Line(function_record.get_line()?.get_l()), name: function_record.get_name()?.to_string()?, }) @@ -654,28 +594,21 @@ pub fn read_trace(input: &mut impl std::io::BufRead) -> ::capnp::Result { let call_record = call_record?; let sargs = call_record.get_args()?; - let mut args: Vec = - Vec::with_capacity(sargs.len().try_into().unwrap()); + let mut args: Vec = Vec::with_capacity(sargs.len().try_into().unwrap()); for sarg in sargs { args.push(crate::FullValueRecord { - variable_id: crate::VariableId( - sarg.get_variable_id()?.get_i().try_into().unwrap(), - ), + variable_id: crate::VariableId(sarg.get_variable_id()?.get_i().try_into().unwrap()), value: get_value_record(sarg.get_value()?)?, }); } TraceLowLevelEvent::Call(crate::CallRecord { - function_id: crate::FunctionId( - call_record.get_function_id()?.get_i().try_into().unwrap(), - ), + function_id: crate::FunctionId(call_record.get_function_id()?.get_i().try_into().unwrap()), args, }) } - Ok(trace::trace_low_level_event::Which::Return(return_record)) => { - TraceLowLevelEvent::Return(crate::ReturnRecord { - return_value: get_value_record(return_record?.get_return_value()?)?, - }) - } + Ok(trace::trace_low_level_event::Which::Return(return_record)) => TraceLowLevelEvent::Return(crate::ReturnRecord { + return_value: get_value_record(return_record?.get_return_value()?)?, + }), Ok(trace::trace_low_level_event::Which::Event(record_event)) => { let record_event = record_event?; TraceLowLevelEvent::Event(crate::RecordEvent { @@ -686,8 +619,7 @@ pub fn read_trace(input: &mut impl std::io::BufRead) -> ::capnp::Result { let asm_strings = asm_strings?; - let mut strs: Vec = - Vec::with_capacity(asm_strings.len().try_into().unwrap()); + let mut strs: Vec = Vec::with_capacity(asm_strings.len().try_into().unwrap()); for s in asm_strings { strs.push(s?.to_string()?); } @@ -696,40 +628,27 @@ pub fn read_trace(input: &mut impl std::io::BufRead) -> ::capnp::Result { let bind_variable_record = bind_variable_record?; TraceLowLevelEvent::BindVariable(crate::BindVariableRecord { - variable_id: crate::VariableId( - bind_variable_record - .get_variable_id()? - .get_i() - .try_into() - .unwrap(), - ), + variable_id: crate::VariableId(bind_variable_record.get_variable_id()?.get_i().try_into().unwrap()), place: crate::Place(bind_variable_record.get_place()?.get_p()), }) } Ok(trace::trace_low_level_event::Which::Assignment(assignment_record)) => { let assignment_record = assignment_record?; TraceLowLevelEvent::Assignment(crate::AssignmentRecord { - to: crate::VariableId( - assignment_record.get_to()?.get_i().try_into().unwrap(), - ), + to: crate::VariableId(assignment_record.get_to()?.get_i().try_into().unwrap()), pass_by: match assignment_record.get_pass_by()? { trace::PassBy::Value => crate::PassBy::Value, trace::PassBy::Reference => crate::PassBy::Reference, }, from: match assignment_record.get_from()?.which()? { trace::r_value::Which::Simple(variable_id) => { - crate::RValue::Simple(crate::VariableId( - variable_id?.get_i().try_into().unwrap(), - )) + crate::RValue::Simple(crate::VariableId(variable_id?.get_i().try_into().unwrap())) } trace::r_value::Which::Compound(variables) => { let variables = variables?; - let mut v: Vec = - Vec::with_capacity(variables.len().try_into().unwrap()); + let mut v: Vec = Vec::with_capacity(variables.len().try_into().unwrap()); for vv in variables { - v.push(crate::VariableId( - vv.get_i().try_into().unwrap(), - )); + v.push(crate::VariableId(vv.get_i().try_into().unwrap())); } crate::RValue::Compound(v) } @@ -738,8 +657,7 @@ pub fn read_trace(input: &mut impl std::io::BufRead) -> ::capnp::Result { let variables = variables?; - let mut v: Vec = - Vec::with_capacity(variables.len().try_into().unwrap()); + let mut v: Vec = Vec::with_capacity(variables.len().try_into().unwrap()); for vv in variables { v.push(crate::VariableId(vv.get_i().try_into().unwrap())) } @@ -759,21 +677,13 @@ pub fn read_trace(input: &mut impl std::io::BufRead) -> ::capnp::Result { + Ok(trace::trace_low_level_event::Which::AssignCompoundItem(assign_compound_item_record)) => { let assign_compound_item_record = assign_compound_item_record?; - TraceLowLevelEvent::AssignCompoundItem( - crate::AssignCompoundItemRecord { - place: crate::Place( - assign_compound_item_record.get_place()?.get_p(), - ), - index: assign_compound_item_record.get_index().try_into().unwrap(), - item_place: crate::Place( - assign_compound_item_record.get_item_place()?.get_p(), - ), - }, - ) + TraceLowLevelEvent::AssignCompoundItem(crate::AssignCompoundItemRecord { + place: crate::Place(assign_compound_item_record.get_place()?.get_p()), + index: assign_compound_item_record.get_index().try_into().unwrap(), + item_place: crate::Place(assign_compound_item_record.get_item_place()?.get_p()), + }) } Ok(trace::trace_low_level_event::Which::AssignCell(assign_cell_record)) => { let assign_cell_record = assign_cell_record?; @@ -785,24 +695,14 @@ pub fn read_trace(input: &mut impl std::io::BufRead) -> ::capnp::Result { let variable_cell_record = variable_cell_record?; TraceLowLevelEvent::VariableCell(crate::VariableCellRecord { - variable_id: crate::VariableId( - variable_cell_record - .get_variable_id()? - .get_i() - .try_into() - .unwrap(), - ), + variable_id: crate::VariableId(variable_cell_record.get_variable_id()?.get_i().try_into().unwrap()), place: crate::Place(variable_cell_record.get_place()?.get_p()), }) } Ok(trace::trace_low_level_event::Which::DropVariable(variable_id)) => { - TraceLowLevelEvent::DropVariable(crate::VariableId( - variable_id?.get_i().try_into().unwrap(), - )) - } - Ok(trace::trace_low_level_event::Which::DropLastStep(())) => { - TraceLowLevelEvent::DropLastStep + TraceLowLevelEvent::DropVariable(crate::VariableId(variable_id?.get_i().try_into().unwrap())) } + Ok(trace::trace_low_level_event::Which::DropLastStep(())) => TraceLowLevelEvent::DropLastStep, Err(_) => { panic!() } diff --git a/runtime_tracing/src/lib.rs b/runtime_tracing/src/lib.rs index c71252c..b9393de 100644 --- a/runtime_tracing/src/lib.rs +++ b/runtime_tracing/src/lib.rs @@ -8,11 +8,13 @@ //! This crate provides the [`Tracer`] type for emitting trace events and a //! collection of serializable structures describing the trace format. //! The format is documented in `docs/` and the README. -mod tracer; -mod types; mod base64; mod capnptrace; -pub use crate::tracer::{Tracer, TraceEventsFileFormat, NONE_TYPE_ID, NONE_VALUE}; +mod tracer; +mod types; +pub use crate::tracer::{ + create_trace_reader, create_trace_writer, NonStreamingTraceWriter, TraceEventsFileFormat, TraceReader, TraceWriter, NONE_TYPE_ID, NONE_VALUE, +}; pub use crate::types::*; pub mod trace_capnp { @@ -27,7 +29,7 @@ mod tests { #[test] fn test_simple_trace() { - let mut tracer = Tracer::new("path.small", &vec![]); + let mut tracer = NonStreamingTraceWriter::new("path.small", &vec![]); let path = Path::new("/test/path.small"); tracer.start(path, Line(1)); tracer.register_step(path, Line(1)); diff --git a/runtime_tracing/src/tracer.rs b/runtime_tracing/src/tracer.rs index ac2f80f..e384947 100644 --- a/runtime_tracing/src/tracer.rs +++ b/runtime_tracing/src/tracer.rs @@ -14,11 +14,83 @@ use crate::types::{ }; use crate::RValue; +pub trait TraceReader { + fn load_trace_events(&mut self, path: &Path) -> Result, Box>; +} + +pub struct JsonTraceReader {} + +impl TraceReader for JsonTraceReader { + fn load_trace_events(&mut self, path: &Path) -> Result, Box> { + let json = std::fs::read_to_string(path)?; + Ok(serde_json::from_str(&json)?) + } +} + +pub struct BinaryTraceReader {} + +impl TraceReader for BinaryTraceReader { + fn load_trace_events(&mut self, path: &Path) -> Result, Box> { + let file = fs::File::open(path)?; + let mut buf_reader = BufReader::new(file); + Ok(crate::capnptrace::read_trace(&mut buf_reader)?) + } +} + +pub trait TraceWriter { + fn begin_writing_trace_metadata(&mut self, path: &Path) -> Result<(), Box>; + fn begin_writing_trace_events(&mut self, path: &Path) -> Result<(), Box>; + fn begin_writing_trace_paths(&mut self, path: &Path) -> Result<(), Box>; + + fn start(&mut self, path: &Path, line: Line); + fn ensure_path_id(&mut self, path: &Path) -> PathId; + fn ensure_function_id(&mut self, function_name: &str, path: &Path, line: Line) -> FunctionId; + fn ensure_type_id(&mut self, kind: TypeKind, lang_type: &str) -> TypeId; + fn ensure_raw_type_id(&mut self, typ: TypeRecord) -> TypeId; + fn ensure_variable_id(&mut self, variable_name: &str) -> VariableId; + fn register_path(&mut self, path: &Path); + fn register_function(&mut self, name: &str, path: &Path, line: Line); + fn register_step(&mut self, path: &Path, line: Line); + fn register_call(&mut self, function_id: FunctionId, args: Vec); + fn arg(&mut self, name: &str, value: ValueRecord) -> FullValueRecord; + fn register_return(&mut self, return_value: ValueRecord); + // TODO: add metadata arg + fn register_special_event(&mut self, kind: EventLogKind, content: &str); + fn to_raw_type(&self, kind: TypeKind, lang_type: &str) -> TypeRecord; + fn register_type(&mut self, kind: TypeKind, lang_type: &str); + fn register_raw_type(&mut self, typ: TypeRecord); + fn register_asm(&mut self, instructions: &[String]); + fn register_variable_with_full_value(&mut self, name: &str, value: ValueRecord); + fn register_variable_name(&mut self, variable_name: &str); + fn register_full_value(&mut self, variable_id: VariableId, value: ValueRecord); + fn register_compound_value(&mut self, place: Place, value: ValueRecord); + fn register_cell_value(&mut self, place: Place, value: ValueRecord); + fn assign_compound_item(&mut self, place: Place, index: usize, item_place: Place); + fn assign_cell(&mut self, place: Place, new_value: ValueRecord); + fn register_variable(&mut self, variable_name: &str, place: Place); + fn drop_variable(&mut self, variable_name: &str); + // history event helpers + fn assign(&mut self, variable_name: &str, rvalue: RValue, pass_by: PassBy); + fn bind_variable(&mut self, variable_name: &str, place: Place); + fn drop_variables(&mut self, variable_names: &[String]); + fn simple_rvalue(&mut self, variable_name: &str) -> RValue; + fn compound_rvalue(&mut self, variable_dependencies: &[String]) -> RValue; + fn drop_last_step(&mut self); + + fn add_event(&mut self, event: TraceLowLevelEvent); + fn append_events(&mut self, events: &mut Vec); + + fn finish_writing_trace_metadata(&mut self) -> Result<(), Box>; + fn finish_writing_trace_events(&mut self) -> Result<(), Box>; + fn finish_writing_trace_paths(&mut self) -> Result<(), Box>; +} + /// State machine used to record [`TraceLowLevelEvent`]s. /// -/// A `Tracer` instance accumulates events and can store them on disk via the -/// `store_trace_*` methods. -pub struct Tracer { +/// A `NonStreamingTraceWriter` instance accumulates events in memory and stores them on +/// disk when the `finish_writing_trace_*` methods are called. The in-memory event list +/// is exposed publicly. +pub struct NonStreamingTraceWriter { // trace metadata: workdir: PathBuf, program: String, @@ -33,11 +105,17 @@ pub struct Tracer { functions: HashMap, variables: HashMap, types: HashMap, + + format: TraceEventsFileFormat, + trace_metadata_path: Option, + trace_events_path: Option, + trace_paths_path: Option, } +#[derive(Debug, Clone, Copy)] pub enum TraceEventsFileFormat { Json, - Binary + Binary, } // we ensure in start they are registered with those id-s @@ -51,10 +129,10 @@ pub const NONE_VALUE: ValueRecord = ValueRecord::None { type_id: NONE_TYPE_ID }; pub const TOP_LEVEL_FUNCTION_ID: FunctionId = FunctionId(0); -impl Tracer { +impl NonStreamingTraceWriter { /// Create a new tracer instance for the given program and arguments. pub fn new(program: &str, args: &[String]) -> Self { - Tracer { + NonStreamingTraceWriter { workdir: env::current_dir().expect("can access the current dir"), program: program.to_string(), args: args.to_vec(), @@ -66,11 +144,52 @@ impl Tracer { functions: HashMap::new(), variables: HashMap::new(), types: HashMap::new(), + + format: TraceEventsFileFormat::Binary, + trace_metadata_path: None, + trace_events_path: None, + trace_paths_path: None, + } + } + + pub fn set_format(&mut self, format: TraceEventsFileFormat) { + self.format = format; + } + + pub fn load_trace_events(&mut self, path: &Path, format: TraceEventsFileFormat) -> Result<(), Box> { + match format { + TraceEventsFileFormat::Json => { + let json = std::fs::read_to_string(path)?; + self.events = serde_json::from_str(&json)?; + } + TraceEventsFileFormat::Binary => { + let file = fs::File::open(path)?; + let mut buf_reader = BufReader::new(file); + self.events = crate::capnptrace::read_trace(&mut buf_reader)?; + } } + Ok(()) + } +} + +impl TraceWriter for NonStreamingTraceWriter { + fn begin_writing_trace_metadata(&mut self, path: &Path) -> Result<(), Box> { + self.trace_metadata_path = Some(path.to_path_buf()); + Ok(()) + } + + fn begin_writing_trace_events(&mut self, path: &Path) -> Result<(), Box> { + self.trace_events_path = Some(path.to_path_buf()); + Ok(()) + } + + fn begin_writing_trace_paths(&mut self, path: &Path) -> Result<(), Box> { + self.trace_paths_path = Some(path.to_path_buf()); + Ok(()) } /// Begin tracing of a program starting at the given source location. - pub fn start(&mut self, path: &Path, line: Line) { + fn start(&mut self, path: &Path, line: Line) { let function_id = self.ensure_function_id("", path, line); self.register_call(function_id, vec![]); assert!(function_id == TOP_LEVEL_FUNCTION_ID); @@ -84,7 +203,7 @@ impl Tracer { assert!(NONE_TYPE_ID == self.ensure_type_id(TypeKind::None, "None")); } - pub fn ensure_path_id(&mut self, path: &Path) -> PathId { + fn ensure_path_id(&mut self, path: &Path) -> PathId { if !self.paths.contains_key(path) { self.paths.insert(path.to_path_buf(), PathId(self.paths.len())); self.register_path(path); @@ -92,7 +211,7 @@ impl Tracer { *self.paths.get(path).unwrap() } - pub fn ensure_function_id(&mut self, function_name: &str, path: &Path, line: Line) -> FunctionId { + fn ensure_function_id(&mut self, function_name: &str, path: &Path, line: Line) -> FunctionId { if !self.functions.contains_key(function_name) { // same function names for different path line? TODO self.functions.insert(function_name.to_string(), FunctionId(self.functions.len())); @@ -101,12 +220,12 @@ impl Tracer { *self.functions.get(function_name).unwrap() } - pub fn ensure_type_id(&mut self, kind: TypeKind, lang_type: &str) -> TypeId { + fn ensure_type_id(&mut self, kind: TypeKind, lang_type: &str) -> TypeId { let typ = self.to_raw_type(kind, lang_type); self.ensure_raw_type_id(typ) } - pub fn ensure_raw_type_id(&mut self, typ: TypeRecord) -> TypeId { + fn ensure_raw_type_id(&mut self, typ: TypeRecord) -> TypeId { if !self.types.contains_key(&typ.lang_type) { self.types.insert(typ.lang_type.clone(), TypeId(self.types.len())); self.register_raw_type(typ.clone()); @@ -114,7 +233,7 @@ impl Tracer { *self.types.get(&typ.lang_type).unwrap() } - pub fn ensure_variable_id(&mut self, variable_name: &str) -> VariableId { + fn ensure_variable_id(&mut self, variable_name: &str) -> VariableId { if !self.variables.contains_key(variable_name) { self.variables.insert(variable_name.to_string(), VariableId(self.variables.len())); self.register_variable_name(variable_name); @@ -122,12 +241,12 @@ impl Tracer { *self.variables.get(variable_name).unwrap() } - pub fn register_path(&mut self, path: &Path) { + fn register_path(&mut self, path: &Path) { self.path_list.push(path.to_path_buf()); self.events.push(TraceLowLevelEvent::Path(path.to_path_buf())); } - pub fn register_function(&mut self, name: &str, path: &Path, line: Line) { + fn register_function(&mut self, name: &str, path: &Path, line: Line) { let path_id = self.ensure_path_id(path); self.function_list.push((name.to_string(), path_id, line)); self.events.push(TraceLowLevelEvent::Function(FunctionRecord { @@ -137,12 +256,12 @@ impl Tracer { })); } - pub fn register_step(&mut self, path: &Path, line: Line) { + fn register_step(&mut self, path: &Path, line: Line) { let path_id = self.ensure_path_id(path); self.events.push(TraceLowLevelEvent::Step(StepRecord { path_id, line })); } - pub fn register_call(&mut self, function_id: FunctionId, args: Vec) { + fn register_call(&mut self, function_id: FunctionId, args: Vec) { // register a step for each call, the backend expects this for // non-toplevel calls, so // we ensure it directly from register_call @@ -160,17 +279,16 @@ impl Tracer { self.events.push(TraceLowLevelEvent::Call(CallRecord { function_id, args })); } - pub fn arg(&mut self, name: &str, value: ValueRecord) -> FullValueRecord { + fn arg(&mut self, name: &str, value: ValueRecord) -> FullValueRecord { let variable_id = self.ensure_variable_id(name); FullValueRecord { variable_id, value } } - pub fn register_return(&mut self, return_value: ValueRecord) { + fn register_return(&mut self, return_value: ValueRecord) { self.events.push(TraceLowLevelEvent::Return(ReturnRecord { return_value })); } - // TODO: add metadata arg - pub fn register_special_event(&mut self, kind: EventLogKind, content: &str) { + fn register_special_event(&mut self, kind: EventLogKind, content: &str) { self.events.push(TraceLowLevelEvent::Event(RecordEvent { kind, metadata: "".to_string(), @@ -178,7 +296,7 @@ impl Tracer { })); } - pub fn to_raw_type(&self, kind: TypeKind, lang_type: &str) -> TypeRecord { + fn to_raw_type(&self, kind: TypeKind, lang_type: &str) -> TypeRecord { TypeRecord { kind, lang_type: lang_type.to_string(), @@ -186,64 +304,64 @@ impl Tracer { } } - pub fn register_type(&mut self, kind: TypeKind, lang_type: &str) { + fn register_type(&mut self, kind: TypeKind, lang_type: &str) { let typ = self.to_raw_type(kind, lang_type); self.events.push(TraceLowLevelEvent::Type(typ)); } - pub fn register_raw_type(&mut self, typ: TypeRecord) { + fn register_raw_type(&mut self, typ: TypeRecord) { self.events.push(TraceLowLevelEvent::Type(typ)); } - pub fn register_asm(&mut self, instructions: &[String]) { + fn register_asm(&mut self, instructions: &[String]) { self.events.push(TraceLowLevelEvent::Asm(instructions.to_vec())); } - pub fn register_variable_with_full_value(&mut self, name: &str, value: ValueRecord) { + fn register_variable_with_full_value(&mut self, name: &str, value: ValueRecord) { let variable_id = self.ensure_variable_id(name); self.register_full_value(variable_id, value); } - pub fn register_variable_name(&mut self, variable_name: &str) { + fn register_variable_name(&mut self, variable_name: &str) { self.events.push(TraceLowLevelEvent::VariableName(variable_name.to_string())); } - pub fn register_full_value(&mut self, variable_id: VariableId, value: ValueRecord) { + fn register_full_value(&mut self, variable_id: VariableId, value: ValueRecord) { self.events.push(TraceLowLevelEvent::Value(FullValueRecord { variable_id, value })); } - pub fn register_compound_value(&mut self, place: Place, value: ValueRecord) { + fn register_compound_value(&mut self, place: Place, value: ValueRecord) { self.events.push(TraceLowLevelEvent::CompoundValue(CompoundValueRecord { place, value })); } - pub fn register_cell_value(&mut self, place: Place, value: ValueRecord) { + fn register_cell_value(&mut self, place: Place, value: ValueRecord) { self.events.push(TraceLowLevelEvent::CellValue(CellValueRecord { place, value })); } - pub fn assign_compound_item(&mut self, place: Place, index: usize, item_place: Place) { + fn assign_compound_item(&mut self, place: Place, index: usize, item_place: Place) { self.events.push(TraceLowLevelEvent::AssignCompoundItem(AssignCompoundItemRecord { place, index, item_place, })); } - pub fn assign_cell(&mut self, place: Place, new_value: ValueRecord) { + fn assign_cell(&mut self, place: Place, new_value: ValueRecord) { self.events.push(TraceLowLevelEvent::AssignCell(AssignCellRecord { place, new_value })); } - pub fn register_variable(&mut self, variable_name: &str, place: Place) { + fn register_variable(&mut self, variable_name: &str, place: Place) { let variable_id = self.ensure_variable_id(variable_name); self.events .push(TraceLowLevelEvent::VariableCell(VariableCellRecord { variable_id, place })); } - pub fn drop_variable(&mut self, variable_name: &str) { + fn drop_variable(&mut self, variable_name: &str) { let variable_id = self.ensure_variable_id(variable_name); self.events.push(TraceLowLevelEvent::DropVariable(variable_id)); } // history event helpers - pub fn assign(&mut self, variable_name: &str, rvalue: RValue, pass_by: PassBy) { + fn assign(&mut self, variable_name: &str, rvalue: RValue, pass_by: PassBy) { let variable_id = self.ensure_variable_id(variable_name); self.events.push(TraceLowLevelEvent::Assignment(AssignmentRecord { to: variable_id, @@ -252,13 +370,13 @@ impl Tracer { })); } - pub fn bind_variable(&mut self, variable_name: &str, place: Place) { + fn bind_variable(&mut self, variable_name: &str, place: Place) { let variable_id = self.ensure_variable_id(variable_name); self.events .push(TraceLowLevelEvent::BindVariable(crate::BindVariableRecord { variable_id, place })); } - pub fn drop_variables(&mut self, variable_names: &[String]) { + fn drop_variables(&mut self, variable_names: &[String]) { let variable_ids: Vec = variable_names .to_vec() .iter() @@ -267,12 +385,12 @@ impl Tracer { self.events.push(TraceLowLevelEvent::DropVariables(variable_ids)) } - pub fn simple_rvalue(&mut self, variable_name: &str) -> RValue { + fn simple_rvalue(&mut self, variable_name: &str) -> RValue { let variable_id = self.ensure_variable_id(variable_name); RValue::Simple(variable_id) } - pub fn compound_rvalue(&mut self, variable_dependencies: &[String]) -> RValue { + fn compound_rvalue(&mut self, variable_dependencies: &[String]) -> RValue { let variable_ids: Vec = variable_dependencies .to_vec() .iter() @@ -281,53 +399,71 @@ impl Tracer { RValue::Compound(variable_ids) } - pub fn drop_last_step(&mut self) { + fn drop_last_step(&mut self) { self.events.push(TraceLowLevelEvent::DropLastStep); } - pub fn store_trace_metadata(&self, path: &Path) -> Result<(), Box> { - let trace_metadata = TraceMetadata { - program: self.program.clone(), - args: self.args.clone(), - workdir: self.workdir.clone(), - }; - let json = serde_json::to_string(&trace_metadata)?; - fs::write(path, json)?; - Ok(()) + fn add_event(&mut self, event: TraceLowLevelEvent) { + self.events.push(event) } - pub fn load_trace_events(&mut self, path: &Path, format: TraceEventsFileFormat) -> Result<(), Box> { - match format { - TraceEventsFileFormat::Json => { - let json = std::fs::read_to_string(path)?; - self.events = serde_json::from_str(&json)?; - } - TraceEventsFileFormat::Binary => { - let file = fs::File::open(path)?; - let mut buf_reader = BufReader::new(file); - self.events = crate::capnptrace::read_trace(&mut buf_reader)?; - } + fn append_events(&mut self, events: &mut Vec) { + self.events.append(events); + } + + fn finish_writing_trace_metadata(&mut self) -> Result<(), Box> { + if let Some(path) = &self.trace_metadata_path { + let trace_metadata = TraceMetadata { + program: self.program.clone(), + args: self.args.clone(), + workdir: self.workdir.clone(), + }; + let json = serde_json::to_string(&trace_metadata)?; + fs::write(path, json)?; + Ok(()) + } else { + panic!("finish_writing_trace_metadata() called without previous call to begin_writing_trace_metadata()"); } - Ok(()) } - pub fn store_trace_events(&self, path: &Path, format: TraceEventsFileFormat) -> Result<(), Box> { - match format { - TraceEventsFileFormat::Json => { - let json = serde_json::to_string(&self.events)?; - fs::write(path, json)?; - } - TraceEventsFileFormat::Binary => { - let mut file = fs::File::create(path)?; - crate::capnptrace::write_trace(&self.events, &mut file)?; + fn finish_writing_trace_events(&mut self) -> Result<(), Box> { + if let Some(path) = &self.trace_events_path { + match self.format { + TraceEventsFileFormat::Json => { + let json = serde_json::to_string(&self.events)?; + fs::write(path, json)?; + } + TraceEventsFileFormat::Binary => { + let mut file = fs::File::create(path)?; + crate::capnptrace::write_trace(&self.events, &mut file)?; + } } + Ok(()) + } else { + panic!("finish_writing_trace_events() called without previous call to begin_writing_trace_events()"); } - Ok(()) } - pub fn store_trace_paths(&self, path: &Path) -> Result<(), Box> { - let json = serde_json::to_string(&self.path_list)?; - fs::write(path, json)?; - Ok(()) + fn finish_writing_trace_paths(&mut self) -> Result<(), Box> { + if let Some(path) = &self.trace_paths_path { + let json = serde_json::to_string(&self.path_list)?; + fs::write(path, json)?; + Ok(()) + } else { + panic!("finish_writing_trace_paths() called without previous call to begin_writing_trace_paths()"); + } + } +} + +pub fn create_trace_reader(format: TraceEventsFileFormat) -> Box { + match format { + TraceEventsFileFormat::Json => Box::new(JsonTraceReader {}), + TraceEventsFileFormat::Binary => Box::new(BinaryTraceReader {}), } } + +pub fn create_trace_writer(program: &str, args: &[String], format: TraceEventsFileFormat) -> Box { + let mut result = Box::new(NonStreamingTraceWriter::new(program, args)); + result.set_format(format); + result +} diff --git a/runtime_tracing/src/types.rs b/runtime_tracing/src/types.rs index c7988f7..2b33788 100644 --- a/runtime_tracing/src/types.rs +++ b/runtime_tracing/src/types.rs @@ -6,10 +6,10 @@ use std::cmp::Ord; use std::ops; use std::path::PathBuf; +use crate::base64; use num_derive::FromPrimitive; use serde::{Deserialize, Serialize}; use serde_repr::*; -use crate::base64; // currently, we do assume that we record the whole program // so, we try to include minimal amount of data, @@ -474,5 +474,5 @@ pub enum EventLogKind { Error, // used for trace events TraceLogEvent, - EvmEvent + EvmEvent, } diff --git a/runtime_tracing/tests/binary_format.rs b/runtime_tracing/tests/binary_format.rs index 6be880e..ae818f4 100644 --- a/runtime_tracing/tests/binary_format.rs +++ b/runtime_tracing/tests/binary_format.rs @@ -1,32 +1,28 @@ -use runtime_tracing::{TraceEventsFileFormat, Tracer}; -use std::path::Path; +use runtime_tracing::{create_trace_reader, create_trace_writer, TraceEventsFileFormat}; use std::fs; +use std::path::Path; #[test] fn test_binary_roundtrip() { let json_path = Path::new("tests/data/trace.json"); - let mut tracer = Tracer::new("", &[]); - tracer - .load_trace_events(json_path, TraceEventsFileFormat::Json) - .unwrap(); - let original = tracer.events.clone(); + let mut json_reader = create_trace_reader(TraceEventsFileFormat::Json); + let original = json_reader.load_trace_events(json_path).unwrap(); let bin_path = Path::new("tests/data/trace.bin"); - tracer - .store_trace_events(bin_path, TraceEventsFileFormat::Binary) - .unwrap(); + let mut bin_writer = create_trace_writer("", &[], TraceEventsFileFormat::Binary); + bin_writer.begin_writing_trace_events(bin_path).unwrap(); + bin_writer.append_events(&mut original.clone()); + bin_writer.finish_writing_trace_events().unwrap(); - let mut tracer2 = Tracer::new("", &[]); - tracer2 - .load_trace_events(bin_path, TraceEventsFileFormat::Binary) - .unwrap(); + let mut bin_reader = create_trace_reader(TraceEventsFileFormat::Binary); + let tracer2_events = bin_reader.load_trace_events(bin_path).unwrap(); fs::remove_file(bin_path).unwrap(); let orig_json = serde_json::to_string(&original).unwrap(); - let new_json = serde_json::to_string(&tracer2.events).unwrap(); + let new_json = serde_json::to_string(&tracer2_events).unwrap(); assert_eq!(orig_json, new_json); } diff --git a/runtime_tracing_cli/src/main.rs b/runtime_tracing_cli/src/main.rs index ce12260..0877c1d 100644 --- a/runtime_tracing_cli/src/main.rs +++ b/runtime_tracing_cli/src/main.rs @@ -1,9 +1,8 @@ use std::path::Path; -use clap::{Args, Parser, Subcommand}; -use runtime_tracing::{TraceEventsFileFormat, Tracer}; - use crate::fmt_trace_cmd::FmtTraceCommand; +use clap::{Args, Parser, Subcommand}; +use runtime_tracing::{create_trace_reader, create_trace_writer, TraceEventsFileFormat}; mod fmt_trace_cmd; #[derive(Debug, Clone, Args)] @@ -45,10 +44,13 @@ fn main() { RuntimeTracingCliCommand::Convert(convert_command) => { let input_file_format = determine_file_format_from_name(&convert_command.input_file).unwrap(); let output_file_format = determine_file_format_from_name(&convert_command.output_file).unwrap(); - let mut trace = Tracer::new("", &[]); - trace.load_trace_events(Path::new(&convert_command.input_file), input_file_format).unwrap(); - trace.store_trace_events(Path::new(&convert_command.output_file), output_file_format).unwrap(); - }, + let mut trace_reader = create_trace_reader(input_file_format); + let mut trace_writer = create_trace_writer("", &[], output_file_format); + let mut trace_events = trace_reader.load_trace_events(Path::new(&convert_command.input_file)).unwrap(); + trace_writer.begin_writing_trace_events(Path::new(&convert_command.output_file)).unwrap(); + trace_writer.append_events(&mut trace_events); + trace_writer.finish_writing_trace_events().unwrap(); + } RuntimeTracingCliCommand::FormatTrace(fmt_trace_cmd) => { fmt_trace_cmd::run(fmt_trace_cmd); } diff --git a/trace_formatter/src/read_write_json.rs b/trace_formatter/src/read_write_json.rs index a801071..6c96843 100644 --- a/trace_formatter/src/read_write_json.rs +++ b/trace_formatter/src/read_write_json.rs @@ -4,8 +4,7 @@ use std::fs; pub fn serialize_file(src_filename: String) -> Value { let file_content = fs::read_to_string(src_filename).expect("Failed to read the file"); - serde_json::from_str(&file_content) - .expect("Failed to parse the json file that was given as a source") + serde_json::from_str(&file_content).expect("Failed to parse the json file that was given as a source") } pub fn save_to_file(dest_filename: String, json_string: String) { @@ -13,7 +12,5 @@ pub fn save_to_file(dest_filename: String, json_string: String) { if !json_string_copy.ends_with('\n') { json_string_copy.push('\n'); } - fs::write(&dest_filename, json_string_copy).unwrap_or_else(|_| { - panic!("Unable to write to destination file: {}", dest_filename.as_str()) - }); + fs::write(&dest_filename, json_string_copy).unwrap_or_else(|_| panic!("Unable to write to destination file: {}", dest_filename.as_str())); }