diff --git a/cranelift/codegen/meta/src/shared/instructions.rs b/cranelift/codegen/meta/src/shared/instructions.rs index 7480a4d44ea9..9604c6904bc0 100644 --- a/cranelift/codegen/meta/src/shared/instructions.rs +++ b/cranelift/codegen/meta/src/shared/instructions.rs @@ -3902,4 +3902,20 @@ pub(crate) fn define( Operand::new("a", &TxN.dynamic_to_vector()).with_doc("New fixed vector"), ]), ); + + ig.push( + Inst::new( + "sequence_point", + r#" + A compiler barrier that acts as an immovable marker from IR input to machine-code output. + + This "sequence point" can have debug tags attached to it, and these tags will be + noted in the output `MachBuffer`. + + It prevents motion of any other side-effects across this boundary. + "#, + &formats.nullary, + ) + .other_side_effects(), + ); } diff --git a/cranelift/codegen/src/inline.rs b/cranelift/codegen/src/inline.rs index 9a425086fdd2..1ca82203f3a2 100644 --- a/cranelift/codegen/src/inline.rs +++ b/cranelift/codegen/src/inline.rs @@ -20,7 +20,7 @@ //! Cranelift the body of the callee that is to be inlined. use crate::cursor::{Cursor as _, FuncCursor}; -use crate::ir::{self, ExceptionTableData, ExceptionTableItem, InstBuilder as _}; +use crate::ir::{self, DebugTag, ExceptionTableData, ExceptionTableItem, InstBuilder as _}; use crate::result::CodegenResult; use crate::trace; use crate::traversals::Dfs; @@ -366,6 +366,13 @@ fn inline_one( // callee. let mut last_inlined_block = inline_block_layout(func, call_block, callee, &entity_map); + // Get a copy of debug tags on the call instruction; these are + // prepended to debug tags on inlined instructions. Remove them + // from the call itself as it will be rewritten to a jump (which + // cannot have tags). + let call_debug_tags = func.debug_tags.get(call_inst).to_vec(); + func.debug_tags.set(call_inst, []); + // Translate each instruction from the callee into the caller, // appending them to their associated block in the caller. // @@ -403,6 +410,29 @@ fn inline_one( let inlined_inst = func.dfg.make_inst(inlined_inst_data); func.layout.append_inst(inlined_inst, inlined_block); + // Copy over debug tags, translating referenced entities + // as appropriate. + let debug_tags = callee.debug_tags.get(callee_inst); + // If there are tags on the inlined instruction, we always + // add tags, and we prepend any tags from the call + // instruction; but we don't add tags if only the callsite + // had them (this would otherwise mean that every single + // instruction in an inlined function body would get + // tags). + if !debug_tags.is_empty() { + let tags = call_debug_tags + .iter() + .cloned() + .chain(debug_tags.iter().map(|tag| match *tag { + DebugTag::User(value) => DebugTag::User(value), + DebugTag::StackSlot(slot) => { + DebugTag::StackSlot(entity_map.inlined_stack_slot(slot)) + } + })) + .collect::>(); + func.debug_tags.set(inlined_inst, tags); + } + let opcode = callee.dfg.insts[callee_inst].opcode(); if opcode.is_return() { // Instructions that return do not define any values, so we diff --git a/cranelift/codegen/src/inst_predicates.rs b/cranelift/codegen/src/inst_predicates.rs index cdaa5ebc2582..7b42e1728663 100644 --- a/cranelift/codegen/src/inst_predicates.rs +++ b/cranelift/codegen/src/inst_predicates.rs @@ -147,7 +147,8 @@ pub fn has_memory_fence_semantics(op: Opcode) -> bool { | Opcode::AtomicLoad | Opcode::AtomicStore | Opcode::Fence - | Opcode::Debugtrap => true, + | Opcode::Debugtrap + | Opcode::SequencePoint => true, Opcode::Call | Opcode::CallIndirect | Opcode::TryCall | Opcode::TryCallIndirect => true, op if op.can_trap() => true, _ => false, diff --git a/cranelift/codegen/src/ir/debug_tags.rs b/cranelift/codegen/src/ir/debug_tags.rs new file mode 100644 index 000000000000..39d137c2973b --- /dev/null +++ b/cranelift/codegen/src/ir/debug_tags.rs @@ -0,0 +1,141 @@ +//! Debug tag storage. +//! +//! Cranelift permits the embedder to place "debug tags" on +//! instructions in CLIF. These tags are sequences of items of various +//! kinds, with no other meaning imposed by Cranelift. They are passed +//! through to metadata provided alongside the compilation result. +//! +//! When Cranelift inlines a function, it will prepend any tags from +//! the call instruction at the inlining callsite to tags on all +//! inlined instructions. +//! +//! These tags can be used, for example, to identify stackslots that +//! store user state, or to denote positions in user source. In +//! general, the intent is to allow perfect reconstruction of original +//! (source-level) program state in an instrumentation-based +//! debug-info scheme, as long as the instruction(s) on which these +//! tags are attached are preserved. This will be the case for any +//! instructions with side-effects. +//! +//! A few answers to design questions that lead to this design: +//! +//! - Why not use the SourceLoc mechanism? Debug tags are richer than +//! that infrastructure because they preserve inlining location and +//! are interleaved properly with any other tags describing the +//! frame. +//! - Why not attach debug tags only to special sequence-point +//! instructions? This is driven by inlining: we should have the +//! semantic information about a callsite attached directly to the +//! call and observe it there, not have a magic "look backward to +//! find a sequence point" behavior in the inliner. +//! +//! In other words, the needs of preserving "virtual" frames across an +//! inlining transform drive this design. + +use crate::ir::{Inst, StackSlot}; +use alloc::collections::BTreeMap; +use alloc::vec::Vec; +use core::ops::Range; + +/// Debug tags for instructions. +#[derive(Clone, PartialEq, Hash, Default)] +#[cfg_attr( + feature = "enable-serde", + derive(serde_derive::Serialize, serde_derive::Deserialize) +)] +pub struct DebugTags { + /// Pool of tags, referred to by `insts` below. + tags: Vec, + + /// Per-instruction range for its list of tags in the tag pool (if + /// any). + /// + /// Note: we don't use `PackedOption` and `EntityList` here + /// because the values that we are storing are not entities. + insts: BTreeMap>, +} + +/// One debug tag. +#[derive(Clone, Debug, PartialEq, Hash)] +#[cfg_attr( + feature = "enable-serde", + derive(serde_derive::Serialize, serde_derive::Deserialize) +)] +pub enum DebugTag { + /// User-specified `u32` value, opaque to Cranelift. + User(u32), + + /// A stack slot reference. + StackSlot(StackSlot), +} + +impl DebugTags { + /// Set the tags on an instruction, overwriting existing tag list. + /// + /// Tags can only be set on call instructions (those for which + /// [`crate::Opcode::is_call()`] returns `true`) and on + /// `sequence_point` instructions. This property is checked by the + /// CLIF verifier. + pub fn set(&mut self, inst: Inst, tags: impl IntoIterator) { + let start = u32::try_from(self.tags.len()).unwrap(); + self.tags.extend(tags); + let end = u32::try_from(self.tags.len()).unwrap(); + if end > start { + self.insts.insert(inst, start..end); + } else { + self.insts.remove(&inst); + } + } + + /// Get the tags associated with an instruction. + pub fn get(&self, inst: Inst) -> &[DebugTag] { + if let Some(range) = self.insts.get(&inst) { + let start = usize::try_from(range.start).unwrap(); + let end = usize::try_from(range.end).unwrap(); + &self.tags[start..end] + } else { + &[] + } + } + + /// Does the given instruction have any tags? + pub fn has(&self, inst: Inst) -> bool { + // We rely on the invariant that an entry in the map is + // present only if the list range is non-empty. + self.insts.contains_key(&inst) + } + + /// Clone the tags from one instruction to another. + /// + /// This clone is cheap (references the same underlying storage) + /// because the tag lists are immutable. + pub fn clone_tags(&mut self, from: Inst, to: Inst) { + if let Some(range) = self.insts.get(&from).cloned() { + self.insts.insert(to, range); + } else { + self.insts.remove(&to); + } + } + + /// Are any debug tags present? + /// + /// This is used for adjusting margins when pretty-printing CLIF. + pub fn is_empty(&self) -> bool { + self.insts.is_empty() + } + + /// Clear all tags. + pub fn clear(&mut self) { + self.insts.clear(); + self.tags.clear(); + } +} + +impl core::fmt::Display for DebugTag { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + match self { + DebugTag::User(value) => write!(f, "{value}"), + DebugTag::StackSlot(slot) => write!(f, "{slot}"), + } + } +} diff --git a/cranelift/codegen/src/ir/function.rs b/cranelift/codegen/src/ir/function.rs index 02f2a31cc7ed..80926edcdf1c 100644 --- a/cranelift/codegen/src/ir/function.rs +++ b/cranelift/codegen/src/ir/function.rs @@ -5,6 +5,7 @@ use crate::HashMap; use crate::entity::{PrimaryMap, SecondaryMap}; +use crate::ir::DebugTags; use crate::ir::{ self, Block, DataFlowGraph, DynamicStackSlot, DynamicStackSlotData, DynamicStackSlots, DynamicType, ExtFuncData, FuncRef, GlobalValue, GlobalValueData, Inst, JumpTable, @@ -190,6 +191,22 @@ pub struct FunctionStencil { /// interpreted by Cranelift, only preserved. pub srclocs: SourceLocs, + /// Opaque debug-info tags on sequence-point and call + /// instructions. + /// + /// These tags are not interpreted by Cranelift, and are passed + /// through to compilation-result metadata. The only semantic + /// structure that Cranelift imposes is that when inlining, it + /// prepends the callsite call instruction's tags to the tags on + /// inlined instructions. + /// + /// In order to ensure clarity around guaranteed compiler + /// behavior, tags are only permitted on instructions whose + /// presence and sequence will remain the same in the compiled + /// output: namely, `sequence_point` instructions and ordinary + /// call instructions. + pub debug_tags: DebugTags, + /// An optional global value which represents an expression evaluating to /// the stack limit for this function. This `GlobalValue` will be /// interpreted in the prologue, if necessary, to insert a stack check to @@ -209,6 +226,7 @@ impl FunctionStencil { self.dfg.clear(); self.layout.clear(); self.srclocs.clear(); + self.debug_tags.clear(); self.stack_limit = None; } @@ -408,6 +426,7 @@ impl Function { layout: Layout::new(), srclocs: SecondaryMap::new(), stack_limit: None, + debug_tags: DebugTags::default(), }, params: FunctionParameters::new(), } diff --git a/cranelift/codegen/src/ir/mod.rs b/cranelift/codegen/src/ir/mod.rs index 4519d5d74ed3..eca17e7badda 100644 --- a/cranelift/codegen/src/ir/mod.rs +++ b/cranelift/codegen/src/ir/mod.rs @@ -4,6 +4,7 @@ mod atomic_rmw_op; mod builder; pub mod condcodes; pub mod constant; +mod debug_tags; pub mod dfg; pub mod dynamic_type; pub mod entities; @@ -36,6 +37,7 @@ pub use crate::ir::builder::{ InsertBuilder, InstBuilder, InstBuilderBase, InstInserterBase, ReplaceBuilder, }; pub use crate::ir::constant::{ConstantData, ConstantPool}; +pub use crate::ir::debug_tags::{DebugTag, DebugTags}; pub use crate::ir::dfg::{BlockData, DataFlowGraph, ValueDef}; pub use crate::ir::dynamic_type::{DynamicTypeData, DynamicTypes, dynamic_to_fixed}; pub use crate::ir::entities::{ @@ -64,7 +66,7 @@ pub use crate::ir::progpoint::ProgramPoint; pub use crate::ir::sourceloc::RelSourceLoc; pub use crate::ir::sourceloc::SourceLoc; pub use crate::ir::stackslot::{ - DynamicStackSlotData, DynamicStackSlots, StackSlotData, StackSlotKind, StackSlots, + DynamicStackSlotData, DynamicStackSlots, StackSlotData, StackSlotKey, StackSlotKind, StackSlots, }; pub use crate::ir::trapcode::TrapCode; pub use crate::ir::types::Type; diff --git a/cranelift/codegen/src/ir/stackslot.rs b/cranelift/codegen/src/ir/stackslot.rs index cfcb8b2c9c74..1ad2a5d7d871 100644 --- a/cranelift/codegen/src/ir/stackslot.rs +++ b/cranelift/codegen/src/ir/stackslot.rs @@ -69,6 +69,37 @@ pub struct StackSlotData { /// be aligned according to other considerations, such as minimum /// stack slot size or machine word size, as well. pub align_shift: u8, + + /// Opaque stackslot metadata handle, passed through to + /// compilation result metadata describing stackslot location. + /// + /// In the face of compiler transforms like inlining that may move + /// stackslots between functions, when an embedder wants to + /// externally observe stackslots, it needs a first-class way for + /// the identity of stackslots to be carried along with the IR + /// entities. This opaque `StackSlotKey` allows the embedder to do + /// so. + pub key: Option, +} + +/// An opaque key uniquely identifying a stack slot. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct StackSlotKey(u64); +impl StackSlotKey { + /// Construct a [`StackSlotKey`] from raw bits. + /// + /// An embedder can use any 64-bit value to describe a stack slot; + /// there are no restrictions, and the value does not mean + /// anything to Cranelift itself. + pub fn new(value: u64) -> StackSlotKey { + StackSlotKey(value) + } + + /// Get the raw bits from the [`StackSlotKey`]. + pub fn bits(&self) -> u64 { + self.0 + } } impl StackSlotData { @@ -78,23 +109,40 @@ impl StackSlotData { kind, size, align_shift, + key: None, + } + } + + /// Create a stack slot with the specified byte size and alignment + /// and the given user-defined key. + pub fn new_with_key( + kind: StackSlotKind, + size: StackSize, + align_shift: u8, + key: StackSlotKey, + ) -> Self { + Self { + kind, + size, + align_shift, + key: Some(key), } } } impl fmt::Display for StackSlotData { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - if self.align_shift != 0 { - write!( - f, - "{} {}, align = {}", - self.kind, - self.size, - 1u32 << self.align_shift - ) + let align_shift = if self.align_shift != 0 { + format!(", align = {}", 1u32 << self.align_shift) } else { - write!(f, "{} {}", self.kind, self.size) - } + "".into() + }; + let key = match self.key { + Some(value) => format!(", key = {}", value.bits()), + None => "".into(), + }; + + write!(f, "{} {}{align_shift}{key}", self.kind, self.size) } } diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle index 9251f4d8b1bf..2d1eb8f1b21b 100644 --- a/cranelift/codegen/src/isa/aarch64/inst.isle +++ b/cranelift/codegen/src/isa/aarch64/inst.isle @@ -997,6 +997,9 @@ (LabelAddress (dst WritableReg) (label MachLabel)) + ;; A pseudoinstruction that serves as a sequence point. + (SequencePoint) + ;; Emits an inline stack probe loop. ;; ;; Note that this is emitted post-regalloc so `start` and `end` can be @@ -5203,3 +5206,8 @@ (let ((dst WritableReg (temp_writable_reg $I64)) (_ Unit (emit (MInst.LabelAddress dst label)))) dst)) + +;; Helper for creating a `SequencePoint` instruction. +(decl a64_sequence_point () SideEffectNoResult) +(rule (a64_sequence_point) + (SideEffectNoResult.Inst (MInst.SequencePoint))) diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs index db910a650ad9..91beb84efb50 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs @@ -3547,6 +3547,10 @@ impl MachInstEmit for Inst { sink.use_label_at_offset(offset, label, LabelUse::Adr21); } + &Inst::SequencePoint { .. } => { + // Nothing. + } + &Inst::StackProbeLoop { start, end, step } => { assert!(emit_info.0.enable_probestack()); diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs index dbff1df1b625..1ba8ec50ff6d 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs @@ -919,6 +919,7 @@ fn aarch64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { Inst::LabelAddress { dst, .. } => { collector.reg_def(dst); } + Inst::SequencePoint { .. } => {} Inst::StackProbeLoop { start, end, .. } => { collector.reg_early_def(start); collector.reg_use(end); @@ -2893,6 +2894,9 @@ impl Inst { let dst = pretty_print_reg(dst.to_reg()); format!("label_address {dst}, {label:?}") } + &Inst::SequencePoint {} => { + format!("sequence_point") + } &Inst::StackProbeLoop { start, end, step } => { let start = pretty_print_reg(start.to_reg()); let end = pretty_print_reg(end); diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle index f304e5c620af..c32edb2b1fac 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.isle +++ b/cranelift/codegen/src/isa/aarch64/lower.isle @@ -3271,3 +3271,9 @@ (rule (lower (get_exception_handler_address (u64_from_imm64 idx) block)) (let ((succ_label MachLabel (block_exn_successor_label block idx))) (a64_label_address succ_label))) + +;; Rules for `sequence_point` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (sequence_point)) + (side_effect + (a64_sequence_point))) diff --git a/cranelift/codegen/src/isa/aarch64/mod.rs b/cranelift/codegen/src/isa/aarch64/mod.rs index 98f4d5c7d650..892911edd833 100644 --- a/cranelift/codegen/src/isa/aarch64/mod.rs +++ b/cranelift/codegen/src/isa/aarch64/mod.rs @@ -74,11 +74,8 @@ impl TargetIsa for AArch64Backend { let (vcode, regalloc_result) = self.compile_vcode(func, domtree, ctrl_plane)?; let emit_result = vcode.emit(®alloc_result, want_disasm, &self.flags, ctrl_plane); - let frame_size = emit_result.frame_size; let value_labels_ranges = emit_result.value_labels_ranges; let buffer = emit_result.buffer; - let sized_stackslot_offsets = emit_result.sized_stackslot_offsets; - let dynamic_stackslot_offsets = emit_result.dynamic_stackslot_offsets; if let Some(disasm) = emit_result.disasm.as_ref() { log::debug!("disassembly:\n{disasm}"); @@ -86,11 +83,8 @@ impl TargetIsa for AArch64Backend { Ok(CompiledCodeStencil { buffer, - frame_size, vcode: emit_result.disasm, value_labels_ranges, - sized_stackslot_offsets, - dynamic_stackslot_offsets, bb_starts: emit_result.bb_offsets, bb_edges: emit_result.bb_edges, }) diff --git a/cranelift/codegen/src/isa/pulley_shared/inst.isle b/cranelift/codegen/src/isa/pulley_shared/inst.isle index 4b008a8867a7..901753841cee 100644 --- a/cranelift/codegen/src/isa/pulley_shared/inst.isle +++ b/cranelift/codegen/src/isa/pulley_shared/inst.isle @@ -95,6 +95,9 @@ ;; A pseudoinstruction that loads the address of a label. (LabelAddress (dst WritableXReg) (label MachLabel)) + + ;; A pseudoinstruction that serves as a sequence point. + (SequencePoint) ) ) @@ -792,3 +795,8 @@ (let ((dst WritableReg (temp_writable_xreg)) (_ Unit (emit (MInst.LabelAddress dst label)))) dst)) + +;; Helper for creating a `SequencePoint` instruction. +(decl pulley_sequence_point () SideEffectNoResult) +(rule (pulley_sequence_point) + (SideEffectNoResult.Inst (MInst.SequencePoint))) diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs b/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs index 08d8b5eaf763..c8de06325999 100644 --- a/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs +++ b/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs @@ -596,6 +596,10 @@ fn pulley_emit

( let end = sink.cur_offset(); sink.use_label_at_offset(end - 4, *label, LabelUse::PcRel); } + + Inst::SequencePoint { .. } => { + // Nothing. + } } } diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs b/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs index bc2490f7bce2..606ec93ae3df 100644 --- a/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs +++ b/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs @@ -335,6 +335,8 @@ fn pulley_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { Inst::LabelAddress { dst, label: _ } => { collector.reg_def(dst); } + + Inst::SequencePoint { .. } => {} } } @@ -834,6 +836,10 @@ impl Inst { let dst = format_reg(dst.to_reg().to_reg()); format!("label_address {dst}, {label:?}") } + + Inst::SequencePoint {} => { + format!("sequence_point") + } } } } diff --git a/cranelift/codegen/src/isa/pulley_shared/lower.isle b/cranelift/codegen/src/isa/pulley_shared/lower.isle index 12770d844e0a..7cde5f2027e8 100644 --- a/cranelift/codegen/src/isa/pulley_shared/lower.isle +++ b/cranelift/codegen/src/isa/pulley_shared/lower.isle @@ -1836,3 +1836,9 @@ (rule (lower (get_exception_handler_address (u64_from_imm64 idx) block)) (let ((succ_label MachLabel (block_exn_successor_label block idx))) (pulley_label_address succ_label))) + +;; Rules for `sequence_point` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (sequence_point)) + (side_effect + (pulley_sequence_point))) diff --git a/cranelift/codegen/src/isa/pulley_shared/mod.rs b/cranelift/codegen/src/isa/pulley_shared/mod.rs index 0b2cf027f01e..c3f16efa2b33 100644 --- a/cranelift/codegen/src/isa/pulley_shared/mod.rs +++ b/cranelift/codegen/src/isa/pulley_shared/mod.rs @@ -180,11 +180,8 @@ where let want_disasm = want_disasm || (cfg!(feature = "trace-log") && log::log_enabled!(log::Level::Debug)); let emit_result = vcode.emit(®alloc_result, want_disasm, &self.flags, ctrl_plane); - let frame_size = emit_result.frame_size; let value_labels_ranges = emit_result.value_labels_ranges; let buffer = emit_result.buffer; - let sized_stackslot_offsets = emit_result.sized_stackslot_offsets; - let dynamic_stackslot_offsets = emit_result.dynamic_stackslot_offsets; if let Some(disasm) = emit_result.disasm.as_ref() { log::debug!("disassembly:\n{disasm}"); @@ -192,11 +189,8 @@ where Ok(CompiledCodeStencil { buffer, - frame_size, vcode: emit_result.disasm, value_labels_ranges, - sized_stackslot_offsets, - dynamic_stackslot_offsets, bb_starts: emit_result.bb_offsets, bb_edges: emit_result.bb_edges, }) diff --git a/cranelift/codegen/src/isa/riscv64/inst.isle b/cranelift/codegen/src/isa/riscv64/inst.isle index f118fcf7100f..62774b59fecd 100644 --- a/cranelift/codegen/src/isa/riscv64/inst.isle +++ b/cranelift/codegen/src/isa/riscv64/inst.isle @@ -360,6 +360,9 @@ (EmitIsland ;; The needed space before the next deadline. (needed_space u32)) + + ;; A pseudoinstruction that serves as a sequence point. + (SequencePoint) )) (type AtomicOP (enum @@ -3262,3 +3265,8 @@ (let ((dst WritableReg (temp_writable_reg $I64)) (_ Unit (emit (MInst.LabelAddress dst label)))) dst)) + +;; Helper for creating a `SequencePoint` instruction. +(decl rv64_sequence_point () SideEffectNoResult) +(rule (rv64_sequence_point) + (SideEffectNoResult.Inst (MInst.SequencePoint))) diff --git a/cranelift/codegen/src/isa/riscv64/inst/emit.rs b/cranelift/codegen/src/isa/riscv64/inst/emit.rs index e7a64b605f03..77bd801c4f3e 100644 --- a/cranelift/codegen/src/isa/riscv64/inst/emit.rs +++ b/cranelift/codegen/src/isa/riscv64/inst/emit.rs @@ -195,6 +195,7 @@ impl Inst { | Inst::Unwind { .. } | Inst::DummyUse { .. } | Inst::LabelAddress { .. } + | Inst::SequencePoint { .. } | Inst::Popcnt { .. } | Inst::Cltz { .. } | Inst::Brev8 { .. } @@ -2769,6 +2770,10 @@ impl Inst { sink.bind_label(jump_around_label, &mut state.ctrl_plane); } } + + Inst::SequencePoint { .. } => { + // Nothing. + } } } } diff --git a/cranelift/codegen/src/isa/riscv64/inst/mod.rs b/cranelift/codegen/src/isa/riscv64/inst/mod.rs index 5f4c037b29ff..4a3890b09319 100644 --- a/cranelift/codegen/src/isa/riscv64/inst/mod.rs +++ b/cranelift/codegen/src/isa/riscv64/inst/mod.rs @@ -699,6 +699,7 @@ fn riscv64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { Inst::LabelAddress { dst, .. } => { collector.reg_def(dst); } + Inst::SequencePoint { .. } => {} } } @@ -1680,6 +1681,10 @@ impl Inst { let dst = format_reg(dst.to_reg()); format!("label_address {dst}, {label:?}") } + + Inst::SequencePoint {} => { + format!("sequence_point") + } } } } diff --git a/cranelift/codegen/src/isa/riscv64/lower.isle b/cranelift/codegen/src/isa/riscv64/lower.isle index d2329133b747..920e5c211978 100644 --- a/cranelift/codegen/src/isa/riscv64/lower.isle +++ b/cranelift/codegen/src/isa/riscv64/lower.isle @@ -3138,3 +3138,9 @@ (rule (lower (get_exception_handler_address (u64_from_imm64 idx) block)) (let ((succ_label MachLabel (block_exn_successor_label block idx))) (rv64_label_address succ_label))) + +;; Rules for `sequence_point` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (sequence_point)) + (side_effect + (rv64_sequence_point))) diff --git a/cranelift/codegen/src/isa/riscv64/mod.rs b/cranelift/codegen/src/isa/riscv64/mod.rs index a196e53e89a4..f9595ace0c2c 100644 --- a/cranelift/codegen/src/isa/riscv64/mod.rs +++ b/cranelift/codegen/src/isa/riscv64/mod.rs @@ -75,11 +75,8 @@ impl TargetIsa for Riscv64Backend { let want_disasm = want_disasm || log::log_enabled!(log::Level::Debug); let emit_result = vcode.emit(®alloc_result, want_disasm, &self.flags, ctrl_plane); - let frame_size = emit_result.frame_size; let value_labels_ranges = emit_result.value_labels_ranges; let buffer = emit_result.buffer; - let sized_stackslot_offsets = emit_result.sized_stackslot_offsets; - let dynamic_stackslot_offsets = emit_result.dynamic_stackslot_offsets; if let Some(disasm) = emit_result.disasm.as_ref() { log::debug!("disassembly:\n{disasm}"); @@ -87,11 +84,8 @@ impl TargetIsa for Riscv64Backend { Ok(CompiledCodeStencil { buffer, - frame_size, vcode: emit_result.disasm, value_labels_ranges, - sized_stackslot_offsets, - dynamic_stackslot_offsets, bb_starts: emit_result.bb_offsets, bb_edges: emit_result.bb_edges, }) diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isle index 91d0227b3cc9..46576a42d42a 100644 --- a/cranelift/codegen/src/isa/s390x/inst.isle +++ b/cranelift/codegen/src/isa/s390x/inst.isle @@ -1019,6 +1019,9 @@ ;; A pseudoinstruction that loads the address of a label. (LabelAddress (dst WritableReg) (label MachLabel)) + + ;; A pseudoinstruction that serves as a sequence point. + (SequencePoint) )) ;; Primitive types used in instruction formats. @@ -4941,6 +4944,11 @@ (_ Unit (emit (MInst.LabelAddress dst label)))) dst)) +;; Helper for creating a `SequencePoint` instruction. +(decl s390x_sequence_point () SideEffectNoResult) +(rule (s390x_sequence_point) + (SideEffectNoResult.Inst (MInst.SequencePoint))) + ;; Implicit conversions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (convert WritableRegPair RegPair writable_regpair_to_regpair) diff --git a/cranelift/codegen/src/isa/s390x/inst/emit.rs b/cranelift/codegen/src/isa/s390x/inst/emit.rs index 98718d1ee117..d67bd5f56c02 100644 --- a/cranelift/codegen/src/isa/s390x/inst/emit.rs +++ b/cranelift/codegen/src/isa/s390x/inst/emit.rs @@ -3544,6 +3544,10 @@ impl Inst { }; inst.emit(sink, emit_info, state); } + + &Inst::SequencePoint { .. } => { + // Nothing. + } } state.clear_post_insn(); diff --git a/cranelift/codegen/src/isa/s390x/inst/mod.rs b/cranelift/codegen/src/isa/s390x/inst/mod.rs index 256de4150485..8c16117ec62e 100644 --- a/cranelift/codegen/src/isa/s390x/inst/mod.rs +++ b/cranelift/codegen/src/isa/s390x/inst/mod.rs @@ -284,6 +284,8 @@ impl Inst { Inst::DummyUse { .. } => InstructionSet::Base, Inst::LabelAddress { .. } => InstructionSet::Base, + + Inst::SequencePoint { .. } => InstructionSet::Base, } } @@ -1011,6 +1013,7 @@ fn s390x_get_operands(inst: &mut Inst, collector: &mut DenyReuseVisitor { collector.reg_def(dst); } + Inst::SequencePoint { .. } => {} } } @@ -3414,6 +3417,9 @@ impl Inst { let dst = pretty_print_reg(dst.to_reg()); format!("label_address {dst}, {label:?}") } + &Inst::SequencePoint {} => { + format!("sequence_point") + } } } } diff --git a/cranelift/codegen/src/isa/s390x/lower.isle b/cranelift/codegen/src/isa/s390x/lower.isle index b86358749c59..41d8507f6dc0 100644 --- a/cranelift/codegen/src/isa/s390x/lower.isle +++ b/cranelift/codegen/src/isa/s390x/lower.isle @@ -4073,3 +4073,9 @@ (rule (lower (get_exception_handler_address (u64_from_imm64 idx) block)) (let ((succ_label MachLabel (block_exn_successor_label block idx))) (s390x_label_address succ_label))) + +;; Rules for `sequence_point` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (sequence_point)) + (side_effect + (s390x_sequence_point))) diff --git a/cranelift/codegen/src/isa/s390x/mod.rs b/cranelift/codegen/src/isa/s390x/mod.rs index 0434c4fcba65..8768b4eebf84 100644 --- a/cranelift/codegen/src/isa/s390x/mod.rs +++ b/cranelift/codegen/src/isa/s390x/mod.rs @@ -74,11 +74,8 @@ impl TargetIsa for S390xBackend { let (vcode, regalloc_result) = self.compile_vcode(func, domtree, ctrl_plane)?; let emit_result = vcode.emit(®alloc_result, want_disasm, flags, ctrl_plane); - let frame_size = emit_result.frame_size; let value_labels_ranges = emit_result.value_labels_ranges; let buffer = emit_result.buffer; - let sized_stackslot_offsets = emit_result.sized_stackslot_offsets; - let dynamic_stackslot_offsets = emit_result.dynamic_stackslot_offsets; if let Some(disasm) = emit_result.disasm.as_ref() { log::debug!("disassembly:\n{disasm}"); @@ -86,11 +83,8 @@ impl TargetIsa for S390xBackend { Ok(CompiledCodeStencil { buffer, - frame_size, vcode: emit_result.disasm, value_labels_ranges, - sized_stackslot_offsets, - dynamic_stackslot_offsets, bb_starts: emit_result.bb_offsets, bb_edges: emit_result.bb_edges, }) diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index 454a08637c51..b285f2e0f2ee 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -320,6 +320,9 @@ (LabelAddress (dst WritableGpr) (label MachLabel)) + ;; A pseudoinstruction that serves as a sequence point. + (SequencePoint) + ;; An instruction assembled outside of cranelift-codegen. (External (inst AssemblerInst)))) @@ -4097,3 +4100,8 @@ (decl libcall_3 (LibCall Reg Reg Reg) Reg) (extern constructor libcall_3 libcall_3) + +;; Helper for creating a `SequencePoint` instruction. +(decl x64_sequence_point () SideEffectNoResult) +(rule (x64_sequence_point) + (SideEffectNoResult.Inst (MInst.SequencePoint))) diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index 3f1582ce8d99..a922cf73773a 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -1815,6 +1815,10 @@ pub(crate) fn emit( asm::inst::leaq_rm::new(*dst, Amode::rip_relative(*label)).emit(sink, info, state); } + Inst::SequencePoint { .. } => { + // Nothing. + } + Inst::External { inst } => { let frame = state.frame_layout(); emit_maybe_shrink( diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index f01dd8781bfe..c3ee37a958b0 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -106,7 +106,8 @@ impl Inst { | Inst::CoffTlsGetAddr { .. } | Inst::Unwind { .. } | Inst::DummyUse { .. } - | Inst::LabelAddress { .. } => true, + | Inst::LabelAddress { .. } + | Inst::SequencePoint => true, Inst::Atomic128RmwSeq { .. } | Inst::Atomic128XchgSeq { .. } => emit_info.cmpxchg16b(), @@ -828,6 +829,10 @@ impl PrettyPrint for Inst { format!("label_address {dst}, {label:?}") } + Inst::SequencePoint {} => { + format!("sequence_point") + } + Inst::External { inst } => { format!("{inst}") } @@ -1193,6 +1198,8 @@ fn x64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { collector.reg_def(dst); } + Inst::SequencePoint { .. } => {} + Inst::External { inst } => { inst.visit(&mut external::RegallocVisitor { collector }); } diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index a7a8104de9e4..f76c7ae7c738 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -5059,3 +5059,9 @@ (rule (lower (nop)) (invalid_reg)) + +;; Rules for `sequence_point` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (sequence_point)) + (side_effect + (x64_sequence_point))) diff --git a/cranelift/codegen/src/isa/x64/mod.rs b/cranelift/codegen/src/isa/x64/mod.rs index 80d154b434b5..920ad0ae95c1 100644 --- a/cranelift/codegen/src/isa/x64/mod.rs +++ b/cranelift/codegen/src/isa/x64/mod.rs @@ -83,11 +83,8 @@ impl TargetIsa for X64Backend { let (vcode, regalloc_result) = self.compile_vcode(func, domtree, ctrl_plane)?; let emit_result = vcode.emit(®alloc_result, want_disasm, &self.flags, ctrl_plane); - let frame_size = emit_result.frame_size; let value_labels_ranges = emit_result.value_labels_ranges; let buffer = emit_result.buffer; - let sized_stackslot_offsets = emit_result.sized_stackslot_offsets; - let dynamic_stackslot_offsets = emit_result.dynamic_stackslot_offsets; if let Some(disasm) = emit_result.disasm.as_ref() { crate::trace!("disassembly:\n{}", disasm); @@ -95,11 +92,8 @@ impl TargetIsa for X64Backend { Ok(CompiledCodeStencil { buffer, - frame_size, vcode: emit_result.disasm, value_labels_ranges, - sized_stackslot_offsets, - dynamic_stackslot_offsets, bb_starts: emit_result.bb_offsets, bb_edges: emit_result.bb_edges, }) diff --git a/cranelift/codegen/src/isa/x64/pcc.rs b/cranelift/codegen/src/isa/x64/pcc.rs index 75a0c8e3d26c..09b356d69958 100644 --- a/cranelift/codegen/src/isa/x64/pcc.rs +++ b/cranelift/codegen/src/isa/x64/pcc.rs @@ -211,6 +211,8 @@ pub(crate) fn check( Inst::LabelAddress { .. } => Err(PccError::UnimplementedInst), + Inst::SequencePoint { .. } => Ok(()), + Inst::External { .. } => Ok(()), // TODO: unsure what to do about this! } } diff --git a/cranelift/codegen/src/lib.rs b/cranelift/codegen/src/lib.rs index 64d37df76a64..425edb25e160 100644 --- a/cranelift/codegen/src/lib.rs +++ b/cranelift/codegen/src/lib.rs @@ -68,9 +68,10 @@ pub use crate::machinst::buffer::{ OpenPatchRegion, PatchRegion, }; pub use crate::machinst::{ - CallInfo, CompiledCode, Final, MachBuffer, MachBufferFinalized, MachInst, MachInstEmit, - MachInstEmitState, MachLabel, RealReg, Reg, RelocDistance, TextSectionBuilder, VCodeConstant, - VCodeConstantData, VCodeConstants, VCodeInst, Writable, + CallInfo, CompiledCode, Final, MachBuffer, MachBufferDebugTagList, MachBufferFinalized, + MachBufferFrameLayout, MachDebugTagPos, MachInst, MachInstEmit, MachInstEmitState, MachLabel, + RealReg, Reg, RelocDistance, TextSectionBuilder, VCodeConstant, VCodeConstantData, + VCodeConstants, VCodeInst, Writable, }; mod alias_analysis; diff --git a/cranelift/codegen/src/machinst/abi.rs b/cranelift/codegen/src/machinst/abi.rs index cc13a6d49db9..91510e5a46f4 100644 --- a/cranelift/codegen/src/machinst/abi.rs +++ b/cranelift/codegen/src/machinst/abi.rs @@ -100,8 +100,8 @@ use crate::CodegenError; use crate::entity::SecondaryMap; -use crate::ir::types::*; use crate::ir::{ArgumentExtension, ArgumentPurpose, ExceptionTag, Signature}; +use crate::ir::{StackSlotKey, types::*}; use crate::isa::TargetIsa; use crate::settings::ProbestackStrategy; use crate::{ir, isa}; @@ -1142,6 +1142,8 @@ pub struct Callee { dynamic_stackslots: PrimaryMap, /// Offsets to each sized stackslot. sized_stackslots: PrimaryMap, + /// Descriptors for sized stackslots. + sized_stackslot_keys: SecondaryMap>, /// Total stack size of all stackslots stackslots_size: u32, /// Stack size to be reserved for outgoing arguments. @@ -1227,6 +1229,7 @@ impl Callee { // Compute sized stackslot locations and total stackslot size. let mut end_offset: u32 = 0; let mut sized_stackslots = PrimaryMap::new(); + let mut sized_stackslot_keys = SecondaryMap::new(); for (stackslot, data) in f.sized_stack_slots.iter() { // We start our computation possibly unaligned where the previous @@ -1250,6 +1253,7 @@ impl Callee { debug_assert_eq!(stackslot.as_u32() as usize, sized_stackslots.len()); sized_stackslots.push(start_offset); + sized_stackslot_keys[stackslot] = data.key; } // Compute dynamic stackslot locations and total stackslot size. @@ -1304,6 +1308,7 @@ impl Callee { dynamic_stackslots, dynamic_type_sizes, sized_stackslots, + sized_stackslot_keys, stackslots_size, outgoing_args_size: 0, tail_args_size, @@ -2324,22 +2329,29 @@ impl Callee { .expect("frame layout not computed before prologue generation") } - /// Returns the full frame size for the given function, after prologue - /// emission has run. This comprises the spill slots and stack-storage - /// slots as well as storage for clobbered callee-save registers, but - /// not arguments arguments pushed at callsites within this function, - /// or other ephemeral pushes. - pub fn frame_size(&self) -> u32 { + /// Returns the offset from SP to FP for the given function, after + /// the prologue has set up the frame. This comprises the spill + /// slots and stack-storage slots as well as storage for clobbered + /// callee-save registers and outgoing arguments at callsites + /// (space for which is reserved during frame setup). + pub fn sp_to_fp_offset(&self) -> u32 { let frame_layout = self.frame_layout(); - frame_layout.clobber_size + frame_layout.fixed_frame_storage_size + frame_layout.clobber_size + + frame_layout.fixed_frame_storage_size + + frame_layout.outgoing_args_size } /// Returns offset from the slot base in the current frame to the caller's SP. pub fn slot_base_to_caller_sp_offset(&self) -> u32 { + // Note: this looks very similar to `frame_size()` above, but + // it differs in both endpoints: it measures from the bottom + // of stackslots, excluding outgoing args; and it includes the + // setup area (FP/LR) size and any extra tail-args space. let frame_layout = self.frame_layout(); frame_layout.clobber_size + frame_layout.fixed_frame_storage_size + frame_layout.setup_area_size + + (frame_layout.tail_args_size - frame_layout.incoming_args_size) } /// Returns the size of arguments expected on the stack. @@ -2390,6 +2402,28 @@ impl Callee { let from = StackAMode::Slot(sp_off); ::gen_load_stack(from, to_reg.map(Reg::from), ty) } + + /// Provide metadata to be emitted alongside machine code. + /// + /// This metadata describes the frame layout sufficiently to find + /// stack slots, so that runtimes and unwinders can observe state + /// set up by compiled code in stackslots allocated for that + /// purpose. + pub fn frame_slot_metadata(&self) -> MachBufferFrameLayout { + let frame_to_fp_offset = self.sp_to_fp_offset(); + let mut stackslots = SecondaryMap::with_capacity(self.sized_stackslots.len()); + let storage_area_base = self.frame_layout().outgoing_args_size; + for (slot, storage_area_offset) in &self.sized_stackslots { + stackslots[slot] = MachBufferStackSlot { + offset: storage_area_base.checked_add(*storage_area_offset).unwrap(), + key: self.sized_stackslot_keys[slot], + }; + } + MachBufferFrameLayout { + frame_to_fp_offset, + stackslots, + } + } } /// An input argument to a call instruction: the vreg that is used, diff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rs index b489b7c079d6..444ae2a25b1a 100644 --- a/cranelift/codegen/src/machinst/buffer.rs +++ b/cranelift/codegen/src/machinst/buffer.rs @@ -172,7 +172,7 @@ use crate::binemit::{Addend, CodeOffset, Reloc}; use crate::ir::function::FunctionParameters; -use crate::ir::{ExceptionTag, ExternalName, RelSourceLoc, SourceLoc, TrapCode}; +use crate::ir::{DebugTag, ExceptionTag, ExternalName, RelSourceLoc, SourceLoc, TrapCode}; use crate::isa::unwind::UnwindInst; use crate::machinst::{ BlockIndex, MachInstLabelUse, TextSectionBuilder, VCodeConstant, VCodeConstants, VCodeInst, @@ -182,7 +182,7 @@ use crate::{MachInstEmitState, ir}; use crate::{VCodeConstantData, timing}; use core::ops::Range; use cranelift_control::ControlPlane; -use cranelift_entity::{PrimaryMap, entity_impl}; +use cranelift_entity::{PrimaryMap, SecondaryMap, entity_impl}; use smallvec::SmallVec; use std::cmp::Ordering; use std::collections::BinaryHeap; @@ -254,6 +254,10 @@ pub struct MachBuffer { exception_handlers: SmallVec<[MachExceptionHandler; 16]>, /// Any source location mappings referring to this code. srclocs: SmallVec<[MachSrcLoc; 64]>, + /// Any debug tags referring to this code. + debug_tags: Vec, + /// Pool of debug tags referenced by `MachDebugTags` entries. + debug_tag_pool: Vec, /// Any user stack maps for this code. /// /// Each entry is an `(offset, span, stack_map)` triple. Entries are sorted @@ -322,6 +326,10 @@ pub struct MachBuffer { /// Indicates when a patchable region is currently open, to guard that it's /// not possible to nest patchable regions. open_patchable: bool, + /// Stack frame layout metadata. If provided for a MachBuffer + /// containing a function body, this allows interpretation of + /// runtime state given a view of an active stack frame. + frame_layout: Option, } impl MachBufferFinalized { @@ -338,9 +346,12 @@ impl MachBufferFinalized { .into_iter() .map(|srcloc| srcloc.apply_base_srcloc(base_srcloc)) .collect(), + debug_tags: self.debug_tags, + debug_tag_pool: self.debug_tag_pool, user_stack_maps: self.user_stack_maps, unwind_info: self.unwind_info, alignment: self.alignment, + frame_layout: self.frame_layout, } } } @@ -367,11 +378,19 @@ pub struct MachBufferFinalized { pub(crate) exception_handlers: SmallVec<[FinalizedMachExceptionHandler; 16]>, /// Any source location mappings referring to this code. pub(crate) srclocs: SmallVec<[T::MachSrcLocType; 64]>, + /// Any debug tags referring to this code. + pub(crate) debug_tags: Vec, + /// Pool of debug tags referenced by `MachDebugTags` entries. + pub(crate) debug_tag_pool: Vec, /// Any user stack maps for this code. /// /// Each entry is an `(offset, span, stack_map)` triple. Entries are sorted /// by code offset, and each stack map covers `span` bytes on the stack. pub(crate) user_stack_maps: SmallVec<[(CodeOffset, u32, ir::UserStackMap); 8]>, + /// Stack frame layout metadata. If provided for a MachBuffer + /// containing a function body, this allows interpretation of + /// runtime state given a view of an active stack frame. + pub(crate) frame_layout: Option, /// Any unwind info at a given location. pub unwind_info: SmallVec<[(CodeOffset, UnwindInst); 8]>, /// The required alignment of this buffer. @@ -447,6 +466,8 @@ impl MachBuffer { call_sites: SmallVec::new(), exception_handlers: SmallVec::new(), srclocs: SmallVec::new(), + debug_tags: vec![], + debug_tag_pool: vec![], user_stack_maps: SmallVec::new(), unwind_info: SmallVec::new(), cur_srcloc: None, @@ -464,6 +485,7 @@ impl MachBuffer { constants: Default::default(), used_constants: Default::default(), open_patchable: false, + frame_layout: None, } } @@ -838,6 +860,8 @@ impl MachBuffer { // (end of buffer) self.data.truncate(b.start as usize); self.pending_fixup_records.truncate(b.fixup); + + // Trim srclocs and debug tags now past the end of the buffer. while let Some(last_srcloc) = self.srclocs.last_mut() { if last_srcloc.end <= b.start { break; @@ -848,6 +872,13 @@ impl MachBuffer { } self.srclocs.pop(); } + while let Some(last_debug_tag) = self.debug_tags.last() { + if last_debug_tag.offset <= b.start { + break; + } + self.debug_tags.pop(); + } + // State: // [PRE CODE] // cur_off, Offset b.start, b.labels_at_this_branch: @@ -1535,9 +1566,12 @@ impl MachBuffer { call_sites: self.call_sites, exception_handlers: finalized_exception_handlers, srclocs, + debug_tags: self.debug_tags, + debug_tag_pool: self.debug_tag_pool, user_stack_maps: self.user_stack_maps, unwind_info: self.unwind_info, alignment, + frame_layout: self.frame_layout, } } @@ -1693,6 +1727,19 @@ impl MachBuffer { self.user_stack_maps.push((return_addr, span, stack_map)); } + /// Push a debug tag associated with the current buffer offset. + pub fn push_debug_tags(&mut self, pos: MachDebugTagPos, tags: &[DebugTag]) { + trace!("debug tags at offset {}: {tags:?}", self.cur_offset()); + let start = u32::try_from(self.debug_tag_pool.len()).unwrap(); + self.debug_tag_pool.extend(tags.iter().cloned()); + let end = u32::try_from(self.debug_tag_pool.len()).unwrap(); + self.debug_tags.push(MachDebugTags { + offset: self.cur_offset(), + pos, + range: start..end, + }); + } + /// Increase the alignment of the buffer to the given alignment if bigger /// than the current alignment. pub fn set_log2_min_function_alignment(&mut self, align_to: u8) { @@ -1701,6 +1748,12 @@ impl MachBuffer { .expect("log2_min_function_alignment too large"), ); } + + /// Set the frame layout metadata. + pub fn set_frame_layout(&mut self, frame_layout: MachBufferFrameLayout) { + debug_assert!(self.frame_layout.is_none()); + self.frame_layout = Some(frame_layout); + } } impl Extend for MachBuffer { @@ -1717,6 +1770,19 @@ impl MachBufferFinalized { &self.srclocs[..] } + /// Get all debug tags, sorted by associated offset. + pub fn debug_tags(&self) -> impl Iterator> { + self.debug_tags.iter().map(|tags| { + let start = usize::try_from(tags.range.start).unwrap(); + let end = usize::try_from(tags.range.end).unwrap(); + MachBufferDebugTagList { + offset: tags.offset, + pos: tags.pos, + tags: &self.debug_tag_pool[start..end], + } + }) + } + /// Get the total required size for the code. pub fn total_size(&self) -> CodeOffset { self.data.len() as CodeOffset @@ -1792,6 +1858,11 @@ impl MachBufferFinalized { } }) } + + /// Get the frame layout, if known. + pub fn frame_layout(&self) -> Option<&MachBufferFrameLayout> { + self.frame_layout.as_ref() + } } /// An item in the exception-handler list for a callsite, with label @@ -2117,6 +2188,118 @@ impl MachBranch { } } +/// Stack-frame layout information carried through to machine +/// code. This provides sufficient information to interpret an active +/// stack frame from a running function, if provided. +#[derive(Clone, Debug, PartialEq)] +#[cfg_attr( + feature = "enable-serde", + derive(serde_derive::Serialize, serde_derive::Deserialize) +)] +pub struct MachBufferFrameLayout { + /// Offset from bottom of frame to FP (near top of frame). This + /// allows reading the frame given only FP. + pub frame_to_fp_offset: u32, + /// Offset from bottom of frame for each StackSlot, + pub stackslots: SecondaryMap, +} + +/// Descriptor for a single stack slot in the compiled function. +#[derive(Clone, Debug, PartialEq, Default)] +#[cfg_attr( + feature = "enable-serde", + derive(serde_derive::Serialize, serde_derive::Deserialize) +)] +pub struct MachBufferStackSlot { + /// Offset from the bottom of the stack frame. + pub offset: u32, + + /// User-provided key to describe this stack slot. + pub key: Option, +} + +/// Debug tags: a sequence of references to a stack slot, or a +/// user-defined value, at a particular PC. +#[derive(Clone, Debug, PartialEq)] +#[cfg_attr( + feature = "enable-serde", + derive(serde_derive::Serialize, serde_derive::Deserialize) +)] +pub(crate) struct MachDebugTags { + /// Offset at which this tag applies. + pub offset: CodeOffset, + + /// Position on the attached instruction. This indicates whether + /// the tags attach to the prior instruction (i.e., as a return + /// point from a call) or the current instruction (i.e., as a PC + /// seen during a trap). + pub pos: MachDebugTagPos, + + /// The range in the tag pool. + pub range: Range, +} + +/// Debug tag position on an instruction. +/// +/// We need to distinguish position on an instruction, and not just +/// use offsets, because of the following case: +/// +/// ```plain +/// call ... +/// trapping_store ... +/// ``` +/// +/// If the stack is walked and interpreted with debug tags while +/// within the call, the PC seen will be the return point, i.e. the +/// address after the call. If the stack is walked and interpreted +/// with debug tags upon a trap of the following instruction, it will +/// be the PC of that instruction -- which is the same PC! Thus to +/// disambiguate which tags we want, we attach a "pre/post" flag to +/// every group of tags at an offset; and when we look up tags, we +/// look them up for an offset and "position" at that offset. +/// +/// Thus there are logically two positions at every offset -- so the +/// above will be emitted as +/// +/// ```plain +/// 0: call ... +/// 4, post: +/// 4, pre: +/// 4: trapping_store ... +/// ``` +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[cfg_attr( + feature = "enable-serde", + derive(serde_derive::Serialize, serde_derive::Deserialize) +)] +pub enum MachDebugTagPos { + /// Tags attached after the instruction that ends at this offset. + /// + /// This is used to attach tags to a call, because the PC we see + /// when walking the stack is the *return point*. + Post, + /// Tags attached before the instruction that starts at this offset. + /// + /// This is used to attach tags to every other kind of + /// instruction, because the PC we see when processing a trap of + /// that instruction is the PC of that instruction, not the + /// following one. + Pre, +} + +/// Iterator item for visiting debug tags. +pub struct MachBufferDebugTagList<'a> { + /// Offset at which this tag applies. + pub offset: CodeOffset, + + /// Position at this offset ("post", attaching to prior + /// instruction, or "pre", attaching to next instruction). + pub pos: MachDebugTagPos, + + /// The underlying tags. + pub tags: &'a [DebugTag], +} + /// Implementation of the `TextSectionBuilder` trait backed by `MachBuffer`. /// /// Note that `MachBuffer` was primarily written for intra-function references diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs index 90007f25f928..47eec0767ef5 100644 --- a/cranelift/codegen/src/machinst/lower.rs +++ b/cranelift/codegen/src/machinst/lower.rs @@ -873,6 +873,21 @@ impl<'func, I: VCodeInst> Lower<'func, I> { } } + // If the CLIF instruction had debug tags, copy them to + // the VCode. Place on all VCode instructions lowered from + // this CLIF instruction. + let debug_tags = self.f.debug_tags.get(inst); + if !debug_tags.is_empty() && self.vcode.vcode.num_insts() > 0 { + let end = self.vcode.vcode.num_insts(); + for i in start..end { + let backwards_index = BackwardsInsnIndex::new(i); + log::trace!( + "debug tags on {inst}; associating {debug_tags:?} with {backwards_index:?}" + ); + self.vcode.add_debug_tags(backwards_index, debug_tags); + } + } + // maybe insert random instruction if ctrl_plane.get_decision() { if ctrl_plane.get_decision() { diff --git a/cranelift/codegen/src/machinst/mod.rs b/cranelift/codegen/src/machinst/mod.rs index 38355b8cbdec..77033e1c15a4 100644 --- a/cranelift/codegen/src/machinst/mod.rs +++ b/cranelift/codegen/src/machinst/mod.rs @@ -389,16 +389,10 @@ pub trait MachInstEmitState: Default + Clone + Debug { pub struct CompiledCodeBase { /// Machine code. pub buffer: MachBufferFinalized, - /// Size of stack frame, in bytes. - pub frame_size: u32, /// Disassembly, if requested. pub vcode: Option, /// Debug info: value labels to registers/stackslots at code offsets. pub value_labels_ranges: ValueLabelsRanges, - /// Debug info: stackslots to stack pointer offsets. - pub sized_stackslot_offsets: PrimaryMap, - /// Debug info: stackslots to stack pointer offsets. - pub dynamic_stackslot_offsets: PrimaryMap, /// Basic-block layout info: block start offsets. /// /// This info is generated only if the `machine_code_cfg_info` @@ -418,11 +412,8 @@ impl CompiledCodeStencil { pub fn apply_params(self, params: &FunctionParameters) -> CompiledCode { CompiledCode { buffer: self.buffer.apply_base_srcloc(params.base_srcloc()), - frame_size: self.frame_size, vcode: self.vcode, value_labels_ranges: self.value_labels_ranges, - sized_stackslot_offsets: self.sized_stackslot_offsets, - dynamic_stackslot_offsets: self.dynamic_stackslot_offsets, bb_starts: self.bb_starts, bb_edges: self.bb_edges, } diff --git a/cranelift/codegen/src/machinst/vcode.rs b/cranelift/codegen/src/machinst/vcode.rs index cd6b8294ba53..9bb44e43d50f 100644 --- a/cranelift/codegen/src/machinst/vcode.rs +++ b/cranelift/codegen/src/machinst/vcode.rs @@ -34,6 +34,7 @@ use rustc_hash::FxHashMap; use core::cmp::Ordering; use core::fmt::{self, Write}; use core::mem::take; +use core::ops::Range; use cranelift_entity::{Keys, entity_impl}; use std::collections::HashMap; use std::collections::hash_map::Entry; @@ -103,6 +104,15 @@ pub struct VCode { /// user stack map. user_stack_maps: FxHashMap, + /// A map from backwards instruction index to the debug tags for + /// that instruction. Each entry indexes a range in the + /// `debug_tag_pool`. + debug_tags: FxHashMap>, + + /// Pooled storage for sequences of debug tags; indexed by entries + /// in `debug_tags`. + debug_tag_pool: Vec, + /// Operands: pre-regalloc references to virtual registers with /// constraints, in one flattened array. This allows the regalloc /// to efficiently access all operands without requiring expensive @@ -221,17 +231,8 @@ pub struct EmitResult { /// epilogue(s), and makes use of the regalloc results. pub disasm: Option, - /// Offsets of sized stackslots. - pub sized_stackslot_offsets: PrimaryMap, - - /// Offsets of dynamic stackslots. - pub dynamic_stackslot_offsets: PrimaryMap, - /// Value-labels information (debug metadata). pub value_labels_ranges: ValueLabelsRanges, - - /// Stack frame size. - pub frame_size: u32, } /// A builder for a VCode function body. @@ -617,6 +618,14 @@ impl VCodeBuilder { let old_entry = self.vcode.user_stack_maps.insert(inst, stack_map); debug_assert!(old_entry.is_none()); } + + /// Add debug tags for the associated instruction. + pub fn add_debug_tags(&mut self, inst: BackwardsInsnIndex, entries: &[ir::DebugTag]) { + let start = u32::try_from(self.vcode.debug_tag_pool.len()).unwrap(); + self.vcode.debug_tag_pool.extend(entries.iter().cloned()); + let end = u32::try_from(self.vcode.debug_tag_pool.len()).unwrap(); + self.vcode.debug_tags.insert(inst, start..end); + } } const NO_INST_OFFSET: CodeOffset = u32::MAX; @@ -637,6 +646,8 @@ impl VCode { vreg_types: vec![], insts: Vec::with_capacity(10 * n_blocks), user_stack_maps: FxHashMap::default(), + debug_tags: FxHashMap::default(), + debug_tag_pool: vec![], operands: Vec::with_capacity(30 * n_blocks), operand_ranges: Ranges::with_capacity(10 * n_blocks), clobbers: FxHashMap::default(), @@ -938,8 +949,11 @@ impl VCode { // function. let index = iix.to_backwards_insn_index(self.num_insts()); let user_stack_map = self.user_stack_maps.remove(&index); - let user_stack_map_disasm = - user_stack_map.as_ref().map(|m| format!(" ; {m:?}")); + let user_stack_map_disasm = if want_disasm { + user_stack_map.as_ref().map(|m| format!(" ; {m:?}")) + } else { + None + }; (user_stack_map, user_stack_map_disasm) }; @@ -950,11 +964,50 @@ impl VCode { None }; + // Place debug tags in the emission buffer + // either at the offset prior to the + // instruction or after the instruction, + // depending on whether this is a call. See + // the documentation on [`MachDebugTagPos`] + // for details on why. + let mut debug_tag_disasm = None; + let mut place_debug_tags = + |this: &VCode, pos: MachDebugTagPos, buffer: &mut MachBuffer| { + // As above, translate the forward instruction + // index to a backward index for the lookup. + let debug_tag_range = { + let index = iix.to_backwards_insn_index(this.num_insts()); + this.debug_tags.get(&index) + }; + if let Some(range) = debug_tag_range { + let start = usize::try_from(range.start).unwrap(); + let end = usize::try_from(range.end).unwrap(); + let tags = &this.debug_tag_pool[start..end]; + + if want_disasm { + debug_tag_disasm = + Some(format!(" ; ^-- debug @ {pos:?}: {tags:?}")); + } + buffer.push_debug_tags(pos, tags); + } + }; + let debug_tag_pos = + if self.insts[iix.index()].call_type() == CallType::Regular { + MachDebugTagPos::Post + } else { + MachDebugTagPos::Pre + }; + + if debug_tag_pos == MachDebugTagPos::Pre { + place_debug_tags(&self, debug_tag_pos, &mut buffer); + } + // If the instruction we are about to emit is // a return, place an epilogue at this point // (and don't emit the return; the actual // epilogue will contain it). if self.insts[iix.index()].is_term() == MachTerminator::Ret { + log::trace!("emitting epilogue"); for inst in self.abi.gen_epilogue() { do_emit(&inst, &mut disasm, &mut buffer, &mut state); } @@ -981,6 +1034,8 @@ impl VCode { ); debug_assert!(allocs.next().is_none()); + log::trace!("emitting: {:?}", self.insts[iix.index()]); + // Emit the instruction! do_emit( &self.insts[iix.index()], @@ -988,10 +1043,19 @@ impl VCode { &mut buffer, &mut state, ); + + if debug_tag_pos == MachDebugTagPos::Post { + place_debug_tags(&self, debug_tag_pos, &mut buffer); + } + if let Some(stack_map_disasm) = stack_map_disasm { disasm.push_str(&stack_map_disasm); disasm.push('\n'); } + if let Some(debug_tag_disasm) = debug_tag_disasm { + disasm.push_str(&debug_tag_disasm); + disasm.push('\n'); + } } } @@ -1113,17 +1177,16 @@ impl VCode { self.monotonize_inst_offsets(&mut inst_offsets[..], func_body_len); let value_labels_ranges = self.compute_value_labels_ranges(regalloc, &inst_offsets[..], func_body_len); - let frame_size = self.abi.frame_size(); + + // Store metadata about frame layout in the MachBuffer. + buffer.set_frame_layout(self.abi.frame_slot_metadata()); EmitResult { buffer: buffer.finish(&self.constants, ctrl_plane), bb_offsets, bb_edges, disasm: if want_disasm { Some(disasm) } else { None }, - sized_stackslot_offsets: self.abi.sized_stackslot_offsets().clone(), - dynamic_stackslot_offsets: self.abi.dynamic_stackslot_offsets().clone(), value_labels_ranges, - frame_size, } } diff --git a/cranelift/codegen/src/verifier/mod.rs b/cranelift/codegen/src/verifier/mod.rs index b91f1bcc0498..fbb04cd77153 100644 --- a/cranelift/codegen/src/verifier/mod.rs +++ b/cranelift/codegen/src/verifier/mod.rs @@ -2025,6 +2025,21 @@ impl<'a> Verifier<'a> { Ok(()) } + pub fn debug_tags(&self, inst: Inst, errors: &mut VerifierErrors) -> VerifierStepResult { + // Tags can only be present on calls and sequence points. + let op = self.func.dfg.insts[inst].opcode(); + let tags_allowed = op.is_call() || op == Opcode::SequencePoint; + let has_tags = self.func.debug_tags.has(inst); + if has_tags && !tags_allowed { + return errors.fatal(( + inst, + "debug tags present on non-call, non-sequence point instruction".to_string(), + )); + } + + Ok(()) + } + pub fn run(&self, errors: &mut VerifierErrors) -> VerifierStepResult { self.verify_global_values(errors)?; self.verify_memory_types(errors)?; @@ -2043,6 +2058,7 @@ impl<'a> Verifier<'a> { self.typecheck(inst, errors)?; self.immediate_constraints(inst, errors)?; self.iconst_bounds(inst, errors)?; + self.debug_tags(inst, errors)?; } self.encodable_as_bb(block, errors)?; diff --git a/cranelift/codegen/src/write.rs b/cranelift/codegen/src/write.rs index d2fbce11bb11..c0025aa6fca3 100644 --- a/cranelift/codegen/src/write.rs +++ b/cranelift/codegen/src/write.rs @@ -259,8 +259,12 @@ fn decorate_block( aliases: &SecondaryMap>, block: Block, ) -> fmt::Result { - // Indent all instructions if any srclocs are present. - let indent = if func.rel_srclocs().is_empty() { 4 } else { 36 }; + // Indent all instructions if any srclocs or debug tags are present. + let indent = if func.rel_srclocs().is_empty() && func.debug_tags.is_empty() { + 4 + } else { + 36 + }; func_w.write_block_header(w, func, block, indent)?; for a in func.dfg.block_params(block).iter().cloned() { @@ -336,7 +340,7 @@ fn write_instruction( func: &Function, aliases: &SecondaryMap>, inst: Inst, - indent: usize, + mut indent: usize, ) -> fmt::Result { // Prefix containing source location, encoding, and value locations. let mut s = String::with_capacity(16); @@ -347,6 +351,9 @@ fn write_instruction( write!(s, "{srcloc} ")?; } + // Write out any debug tags. + write_debug_tags(w, &func, inst, &mut indent)?; + // Write out prefix and indent the instruction. write!(w, "{s:indent$}")?; @@ -572,6 +579,26 @@ pub fn write_operands(w: &mut dyn Write, dfg: &DataFlowGraph, inst: Inst) -> fmt Ok(()) } +fn write_debug_tags( + w: &mut dyn Write, + func: &Function, + inst: Inst, + indent: &mut usize, +) -> fmt::Result { + let tags = func.debug_tags.get(inst); + if !tags.is_empty() { + let tags = tags + .iter() + .map(|tag| format!("{tag}")) + .collect::>() + .join(", "); + let s = format!("<{tags}> "); + write!(w, "{s}")?; + *indent = indent.saturating_sub(s.len()); + } + Ok(()) +} + fn write_user_stack_map_entries(w: &mut dyn Write, dfg: &DataFlowGraph, inst: Inst) -> fmt::Result { let entries = match dfg.user_stack_map_entries(inst) { None => return Ok(()), diff --git a/cranelift/filetests/filetests/inline/debug.clif b/cranelift/filetests/filetests/inline/debug.clif new file mode 100644 index 000000000000..a6820e3a33ad --- /dev/null +++ b/cranelift/filetests/filetests/inline/debug.clif @@ -0,0 +1,42 @@ +test inline precise-output +target x86_64 + +function %f0(i32, i32) -> i32 { + ss0 = explicit_slot 64, key = 0xfedc_ba98_7654_3210 +block0(v0: i32, v1: i32): + sequence_point + v2 = iadd v0, v1 + return v2 +} + +; (no functions inlined into %f0) + +function %f1() -> i32 { + fn0 = %f0(i32, i32) -> i32 + ss0 = explicit_slot 64, key = 0x1234_5678_9abc_def0 +block0(): + v0 = iconst.i32 10 + v1 = call fn0(v0, v0) + return v1 +} + +; function %f1() -> i32 fast { +; ss0 = explicit_slot 64, key = 1311768467463790320 +; ss1 = explicit_slot 64, key = 18364758544493064720 +; sig0 = (i32, i32) -> i32 fast +; fn0 = %f0 sig0 +; +; block0: +; v0 = iconst.i32 10 +; jump block1 +; +; block1: +; sequence_point +; v3 = iadd.i32 v0, v0 ; v0 = 10, v0 = 10 +; jump block2(v3) +; +; block2(v2: i32): +; v1 -> v2 +; return v1 +; } + diff --git a/cranelift/filetests/filetests/isa/x64/debug.clif b/cranelift/filetests/filetests/isa/x64/debug.clif new file mode 100644 index 000000000000..16fc15245e76 --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/debug.clif @@ -0,0 +1,108 @@ +test compile precise-output +target x86_64 + +function %f0(i32, i32) -> i32 { + ss0 = explicit_slot 64, key = 1234 + ;; not colocated -- shows interesting case where tags attach to + ;; multiple VCode insts. + fn0 = %f1(i32, i32) +block0(v0: i32, v1: i32): + call fn0(v0, v1) + sequence_point + v2 = iadd v0, v1 + return v2 +} + +; VCode: +; pushq %rbp +; movq %rsp, %rbp +; subq $0x50, %rsp +; movq %r12, 0x40(%rsp) +; movq %r14, 0x48(%rsp) +; block0: +; load_ext_name %f1+0, %rdx +; ; ^-- debug @ Pre: [StackSlot(ss0), User(0), User(0)] +; movq %rsi, %r12 +; movq %rdi, %r14 +; call *%rdx +; ; ^-- debug @ Post: [StackSlot(ss0), User(0), User(0)] +; sequence_point +; ; ^-- debug @ Pre: [StackSlot(ss0), User(1), User(0)] +; leal (%r14, %r12), %eax +; movq 0x40(%rsp), %r12 +; movq 0x48(%rsp), %r14 +; addq $0x50, %rsp +; movq %rbp, %rsp +; popq %rbp +; retq +; +; Disassembled: +; block0: ; offset 0x0 +; pushq %rbp +; movq %rsp, %rbp +; subq $0x50, %rsp +; movq %r12, 0x40(%rsp) +; movq %r14, 0x48(%rsp) +; block1: ; offset 0x12 +; movabsq $0, %rdx ; reloc_external Abs8 %f1 0 +; movq %rsi, %r12 +; movq %rdi, %r14 +; callq *%rdx +; leal (%r14, %r12), %eax +; movq 0x40(%rsp), %r12 +; movq 0x48(%rsp), %r14 +; addq $0x50, %rsp +; movq %rbp, %rsp +; popq %rbp +; retq + +function %f1(i32, i32) -> i32 { + ss0 = explicit_slot 64, key = 1234 + fn0 = colocated %f1(i32, i32) +block0(v0: i32, v1: i32): + call fn0(v0, v1) + sequence_point + v2 = iadd v0, v1 + return v2 +} + +; VCode: +; pushq %rbp +; movq %rsp, %rbp +; subq $0x50, %rsp +; movq %rbx, 0x40(%rsp) +; movq %r15, 0x48(%rsp) +; block0: +; movq %rdi, %rbx +; movq %rsi, %r15 +; call TestCase(%f1) +; ; ^-- debug @ Post: [StackSlot(ss0), User(0), User(0)] +; sequence_point +; ; ^-- debug @ Pre: [StackSlot(ss0), User(1), User(0)] +; leal (%rbx, %r15), %eax +; movq 0x40(%rsp), %rbx +; movq 0x48(%rsp), %r15 +; addq $0x50, %rsp +; movq %rbp, %rsp +; popq %rbp +; retq +; +; Disassembled: +; block0: ; offset 0x0 +; pushq %rbp +; movq %rsp, %rbp +; subq $0x50, %rsp +; movq %rbx, 0x40(%rsp) +; movq %r15, 0x48(%rsp) +; block1: ; offset 0x12 +; movq %rdi, %rbx +; movq %rsi, %r15 +; callq 0x1d ; reloc_external CallPCRel4 %f1 -4 +; leal (%rbx, %r15), %eax +; movq 0x40(%rsp), %rbx +; movq 0x48(%rsp), %r15 +; addq $0x50, %rsp +; movq %rbp, %rsp +; popq %rbp +; retq + diff --git a/cranelift/filetests/filetests/verifier/debug_tags.clif b/cranelift/filetests/filetests/verifier/debug_tags.clif new file mode 100644 index 000000000000..bb7fa77739f5 --- /dev/null +++ b/cranelift/filetests/filetests/verifier/debug_tags.clif @@ -0,0 +1,7 @@ +test verifier + +function %f(i32, i32) -> i32 { +block0(v0: i32, v1: i32): +<0, 1> v2 = iadd.i32 v0, v1 ; error: debug tags present on non-call, non-sequence point instruction + return v2 +} \ No newline at end of file diff --git a/cranelift/interpreter/src/step.rs b/cranelift/interpreter/src/step.rs index ac4bee433d96..8a7fad2323d9 100644 --- a/cranelift/interpreter/src/step.rs +++ b/cranelift/interpreter/src/step.rs @@ -1321,6 +1321,8 @@ where Opcode::TryCallIndirect => unimplemented!("TryCallIndirect"), Opcode::GetExceptionHandlerAddress => unimplemented!("GetExceptionHandlerAddress"), + + Opcode::SequencePoint => unimplemented!("SequencePoint"), }) } diff --git a/cranelift/reader/src/lexer.rs b/cranelift/reader/src/lexer.rs index 17b3f63c2240..ce8feb221523 100644 --- a/cranelift/reader/src/lexer.rs +++ b/cranelift/reader/src/lexer.rs @@ -19,6 +19,8 @@ pub enum Token<'a> { RBrace, // '}' LBracket, // '[' RBracket, // ']' + LAngle, // '<' + RAngle, // '>' Minus, // '-' Plus, // '+' Multiply, // '*' @@ -482,6 +484,8 @@ impl<'a> Lexer<'a> { Some('}') => Some(self.scan_char(Token::RBrace)), Some('[') => Some(self.scan_char(Token::LBracket)), Some(']') => Some(self.scan_char(Token::RBracket)), + Some('<') => Some(self.scan_char(Token::LAngle)), + Some('>') => Some(self.scan_char(Token::RAngle)), Some(',') => Some(self.scan_char(Token::Comma)), Some('.') => Some(self.scan_char(Token::Dot)), Some(':') => Some(self.scan_char(Token::Colon)), diff --git a/cranelift/reader/src/parser.rs b/cranelift/reader/src/parser.rs index 080476a44f67..56b751a8a162 100644 --- a/cranelift/reader/src/parser.rs +++ b/cranelift/reader/src/parser.rs @@ -15,8 +15,8 @@ use cranelift_codegen::ir::immediates::{ }; use cranelift_codegen::ir::instructions::{InstructionData, InstructionFormat, VariableArgs}; use cranelift_codegen::ir::pcc::{BaseExpr, Expr, Fact}; -use cranelift_codegen::ir::types::*; -use cranelift_codegen::ir::{self, UserExternalNameRef}; +use cranelift_codegen::ir::{self, StackSlotKey, UserExternalNameRef}; +use cranelift_codegen::ir::{DebugTag, types::*}; use cranelift_codegen::ir::{ AbiParam, ArgumentExtension, ArgumentPurpose, Block, BlockArg, Constant, ConstantData, @@ -956,6 +956,38 @@ impl<'a> Parser<'a> { } } + /// Parse an optional list of debug tags. + fn optional_debug_tags(&mut self) -> ParseResult> { + if self.optional(Token::LAngle) { + let mut tags = vec![]; + while !self.optional(Token::RAngle) { + match self.token() { + Some(Token::Integer(_)) => { + let value: u32 = self.match_uimm32("expected a u32 value")?.into(); + tags.push(DebugTag::User(value)); + } + Some(Token::StackSlot(slot)) => { + self.consume(); + tags.push(DebugTag::StackSlot(StackSlot::from_u32(slot))); + } + _ => { + return err!( + self.loc, + "expected integer user value or stack slot in debug tags" + ); + } + } + if !self.optional(Token::Comma) { + self.match_token(Token::RAngle, "expected `,` or `>`")?; + break; + } + } + Ok(tags) + } else { + Ok(vec![]) + } + } + /// Parse a list of literals (i.e. integers, floats, booleans); e.g. `0 1 2 3`, usually as /// part of something like `vconst.i32x4 [0 1 2 3]`. fn parse_literals_to_constant_data(&mut self, ty: Type) -> ParseResult { @@ -1510,7 +1542,7 @@ impl<'a> Parser<'a> { // | "spill_slot" // | "incoming_arg" // | "outgoing_arg" - // stack-slot-flag ::= "align" "=" Bytes + // stack-slot-flag ::= "align" "=" Bytes | "key" "=" uimm64 fn parse_stack_slot_decl(&mut self) -> ParseResult<(StackSlot, StackSlotData)> { let ss = self.match_ss("expected stack slot number: ss«n»")?; self.match_token(Token::Equal, "expected '=' in stack slot declaration")?; @@ -1527,29 +1559,42 @@ impl<'a> Parser<'a> { return err!(self.loc, "stack slot too large"); } - // Parse flags. - let align = if self.token() == Some(Token::Comma) { + let mut align = 1; + let mut key = None; + + while self.token() == Some(Token::Comma) { self.consume(); - self.match_token( - Token::Identifier("align"), - "expected a valid stack-slot flag (currently only `align`)", - )?; - self.match_token(Token::Equal, "expected `=` after flag")?; - let align: i64 = self - .match_imm64("expected alignment-size after `align` flag")? - .into(); - u32::try_from(align) - .map_err(|_| self.error("alignment must be a 32-bit unsigned integer"))? - } else { - 1 - }; + match self.token() { + Some(Token::Identifier("align")) => { + self.consume(); + self.match_token(Token::Equal, "expected `=` after flag")?; + let align64: i64 = self + .match_imm64("expected alignment-size after `align` flag")? + .into(); + align = u32::try_from(align64) + .map_err(|_| self.error("alignment must be a 32-bit unsigned integer"))?; + } + Some(Token::Identifier("key")) => { + self.consume(); + self.match_token(Token::Equal, "expected `=` after flag")?; + let value = self.match_uimm64("expected `u64` value for `key` flag")?; + key = Some(StackSlotKey::new(value.into())); + } + _ => { + return Err(self.error("invalid flag for stack slot")); + } + } + } if !align.is_power_of_two() { return err!(self.loc, "stack slot alignment is not a power of two"); } let align_shift = u8::try_from(align.ilog2()).unwrap(); // Always succeeds: range 0..=31. - let data = StackSlotData::new(kind, bytes as u32, align_shift); + let data = match key { + Some(key) => StackSlotData::new_with_key(kind, bytes as u32, align_shift, key), + None => StackSlotData::new(kind, bytes as u32, align_shift), + }; // Collect any trailing comments. self.token(); @@ -2103,11 +2148,14 @@ impl<'a> Parser<'a> { Some(Token::Value(_)) | Some(Token::Identifier(_)) | Some(Token::LBracket) - | Some(Token::SourceLoc(_)) => true, + | Some(Token::SourceLoc(_)) + | Some(Token::LAngle) => true, _ => false, } { let srcloc = self.optional_srcloc()?; + let debug_tags = self.optional_debug_tags()?; + // We need to parse instruction results here because they are shared // between the parsing of value aliases and the parsing of instructions. // @@ -2127,10 +2175,10 @@ impl<'a> Parser<'a> { } Some(Token::Equal) => { self.consume(); - self.parse_instruction(&results, srcloc, ctx, block)?; + self.parse_instruction(&results, srcloc, debug_tags, ctx, block)?; } _ if !results.is_empty() => return err!(self.loc, "expected -> or ="), - _ => self.parse_instruction(&results, srcloc, ctx, block)?, + _ => self.parse_instruction(&results, srcloc, debug_tags, ctx, block)?, } } @@ -2521,6 +2569,7 @@ impl<'a> Parser<'a> { &mut self, results: &[Value], srcloc: ir::SourceLoc, + debug_tags: Vec, ctx: &mut Context, block: Block, ) -> ParseResult<()> { @@ -2560,16 +2609,22 @@ impl<'a> Parser<'a> { // instruction ::= [inst-results "="] Opcode(opc) ["." Type] * ... let inst_data = self.parse_inst_operands(ctx, opcode, explicit_ctrl_type)?; - // We're done parsing the instruction data itself. - // - // We still need to check that the number of result values in the source - // matches the opcode or function call signature. We also need to create - // values with the right type for all the instruction results and parse - // and attach stack map entries, if present. let ctrl_typevar = self.infer_typevar(ctx, opcode, explicit_ctrl_type, &inst_data)?; let inst = ctx.function.dfg.make_inst(inst_data); - if opcode.is_call() && !opcode.is_return() && self.optional(Token::Comma) { - self.match_identifier("stack_map", "expected `stack_map = [...]`")?; + + // Attach stack map, if present. + if self.optional(Token::Comma) { + self.match_token( + Token::Identifier("stack_map"), + "expected `stack_map = [...]`", + )?; + if !opcode.is_call() || opcode.is_return() { + return err!( + self.loc, + "stack map can only be attached to a (non-tail) call" + ); + } + self.match_token(Token::Equal, "expected `= [...]`")?; self.match_token(Token::LBracket, "expected `[...]`")?; while !self.optional(Token::RBracket) { @@ -2594,6 +2649,13 @@ impl<'a> Parser<'a> { } } } + + // We're done parsing the instruction data itself. + // + // We still need to check that the number of result values in + // the source matches the opcode or function call + // signature. We also need to create values with the right + // type for all the instruction results. let num_results = ctx.function .dfg @@ -2606,6 +2668,9 @@ impl<'a> Parser<'a> { if !srcloc.is_default() { ctx.function.set_srcloc(inst, srcloc); } + if !debug_tags.is_empty() { + ctx.function.debug_tags.set(inst, debug_tags); + } if results.len() != num_results { return err!(