fix: accurate leaf detection (#11581)

* feat: add is_call() method to MachInst trait and VCode analysis

Add is_call() method to MachInst trait to enable accurate leaf function
detection during register allocation. Update VCode compute_clobbers() to
return (clobbers, is_leaf) tuple by analyzing actual call instructions
in machine code.

* feat: implement is_call() method across all architectures

Implement is_call() method for all architecture-specific MachInst
implementations:

- x64: Detects CallKnown, CallUnknown, ReturnCall variants, and TLS
calls (ElfTlsGetAddr, MachOTlsGetAddr)
- aarch64: Detects Call, CallInd, ReturnCall variants, and TLS calls
(ElfTlsGetAddr, MachOTlsGetAddr)
- riscv64: Detects Call, CallInd, ReturnCall variants, and ElfTlsGetAddr
- s390x: Detects CallKnown, CallUnknown, ReturnCall variants
- pulley: Detects Call, CallIndirect, ReturnCall variants

Co-authored-by: bjorn3 <[email protected]>

* feat: improve leaf function detection and pass is_leaf to FrameLayout

* test: add filetests for leaf detection

* test: update expected outputs for accurate leaf function detection

* test(riscv64): update filetests output

---------

Co-authored-by: bjorn3 <[email protected]>

Author: pnodet

cranelift/codegen/src/ir/function.rs

@@ -320,24 +320,6 @@ impl FunctionStencil {
         })
     }
 
-    /// Returns true if the function is function that doesn't call any other functions. This is not
-    /// to be confused with a "leaf function" in Windows terminology.
-    pub fn is_leaf(&self) -> bool {
-        // Conservative result: if there's at least one function signature referenced in this
-        // function, assume it is not a leaf.
-        let has_signatures = !self.dfg.signatures.is_empty();
-
-        // Under some TLS models, retrieving the address of a TLS variable requires calling a
-        // function. Conservatively assume that any function that references a tls global value
-        // is not a leaf.
-        let has_tls = self.global_values.values().any(|gv| match gv {
-            GlobalValueData::Symbol { tls, .. } => *tls,
-            _ => false,
-        });
-
-        !has_signatures && !has_tls
-    }
-
     /// Replace the `dst` instruction's data with the `src` instruction's data
     /// and then remove `src`.
     ///

cranelift/codegen/src/isa/aarch64/abi.rs

@@ -1167,6 +1167,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
             stackslots_size,
             outgoing_args_size,
             clobbered_callee_saves: regs,
+            is_leaf,
         }
     }
 

cranelift/codegen/src/isa/aarch64/inst/mod.rs

@@ -1000,6 +1000,18 @@ impl MachInst for Inst {
         }
     }
 
+    fn is_call(&self) -> bool {
+        match self {
+            Inst::Call { .. }
+            | Inst::CallInd { .. }
+            | Inst::ReturnCall { .. }
+            | Inst::ReturnCallInd { .. }
+            | Inst::ElfTlsGetAddr { .. }
+            | Inst::MachOTlsGetAddr { .. } => true,
+            _ => false,
+        }
+    }
+
     fn is_term(&self) -> MachTerminator {
         match self {
             &Inst::Rets { .. } => MachTerminator::Ret,

cranelift/codegen/src/isa/pulley_shared/abi.rs

@@ -537,6 +537,7 @@ where
             stackslots_size,
             outgoing_args_size,
             clobbered_callee_saves: regs,
+            is_leaf,
         }
     }
 

cranelift/codegen/src/isa/pulley_shared/inst/mod.rs

@@ -490,6 +490,17 @@ where
         todo!()
     }
 
+    fn is_call(&self) -> bool {
+        match &self.inst {
+            Inst::Call { .. }
+            | Inst::IndirectCall { .. }
+            | Inst::IndirectCallHost { .. }
+            | Inst::ReturnCall { .. }
+            | Inst::ReturnIndirectCall { .. } => true,
+            _ => false,
+        }
+    }
+
     fn gen_move(to_reg: Writable<Reg>, from_reg: Reg, ty: Type) -> Self {
         match ty {
             ir::types::I8 | ir::types::I16 | ir::types::I32 | ir::types::I64 => RawInst::Xmov {

cranelift/codegen/src/isa/riscv64/abi.rs

@@ -671,6 +671,7 @@ impl ABIMachineSpec for Riscv64MachineDeps {
             stackslots_size,
             outgoing_args_size,
             clobbered_callee_saves: regs,
+            is_leaf,
         }
     }
 

cranelift/codegen/src/isa/riscv64/inst/mod.rs

@@ -758,6 +758,17 @@ impl MachInst for Inst {
         }
     }
 
+    fn is_call(&self) -> bool {
+        match self {
+            Inst::Call { .. }
+            | Inst::CallInd { .. }
+            | Inst::ReturnCall { .. }
+            | Inst::ReturnCallInd { .. }
+            | Inst::ElfTlsGetAddr { .. } => true,
+            _ => false,
+        }
+    }
+
     fn is_term(&self) -> MachTerminator {
         match self {
             &Inst::Jal { .. } => MachTerminator::Branch,

cranelift/codegen/src/isa/s390x/abi.rs

@@ -896,7 +896,7 @@ impl ABIMachineSpec for S390xMachineDeps {
         flags: &settings::Flags,
         _sig: &Signature,
         regs: &[Writable<RealReg>],
-        _is_leaf: bool,
+        is_leaf: bool,
         incoming_args_size: u32,
         tail_args_size: u32,
         stackslots_size: u32,
@@ -975,6 +975,7 @@ impl ABIMachineSpec for S390xMachineDeps {
             stackslots_size,
             outgoing_args_size,
             clobbered_callee_saves: regs,
+            is_leaf,
         }
     }
 

cranelift/codegen/src/isa/s390x/inst/mod.rs

@@ -1122,6 +1122,13 @@ impl MachInst for Inst {
         }
     }
 
+    fn is_call(&self) -> bool {
+        match self {
+            Inst::Call { .. } | Inst::ReturnCall { .. } | Inst::ElfTlsGetOffset { .. } => true,
+            _ => false,
+        }
+    }
+
     fn gen_move(to_reg: Writable<Reg>, from_reg: Reg, ty: Type) -> Inst {
         assert!(ty.bits() <= 128);
         if ty.bits() <= 32 {

cranelift/codegen/src/isa/x64/abi.rs

@@ -900,7 +900,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
         flags: &settings::Flags,
         _sig: &Signature,
         regs: &[Writable<RealReg>],
-        _is_leaf: bool,
+        is_leaf: bool,
         incoming_args_size: u32,
         tail_args_size: u32,
         stackslots_size: u32,
@@ -947,6 +947,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
             stackslots_size,
             outgoing_args_size,
             clobbered_callee_saves: regs,
+            is_leaf,
         }
     }