feat: add granular tail call detection infrastructure to MachInst (#11599)

pnodet · web-flow · commit 3b85d8383b50 · 2025-09-03T22:47:50.000Z
* feat: add granular tail call detection infrastructure to machinst

Adds core infrastructure for distinguishing between regular calls and
tail calls at the instruction level.

* feat: implement call_type() method for all ISA backends

* refactor: pass around function_calls enum instead of boolean

* feat: add function_calls.update() logic
diff --git a/cranelift/codegen/src/isa/aarch64/abi.rs b/cranelift/codegen/src/isa/aarch64/abi.rs
@@ -1120,7 +1120,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
         flags: &settings::Flags,
         sig: &Signature,
         regs: &[Writable<RealReg>],
-        is_leaf: bool,
+        function_calls: FunctionCalls,
         incoming_args_size: u32,
         tail_args_size: u32,
         stackslots_size: u32,
@@ -1144,7 +1144,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
 
         // Compute linkage frame size.
         let setup_area_size = if flags.preserve_frame_pointers()
-            || !is_leaf
+            || function_calls != FunctionCalls::None
             // The function arguments that are passed on the stack are addressed
             // relative to the Frame Pointer.
             || incoming_args_size > 0
@@ -1167,7 +1167,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
             stackslots_size,
             outgoing_args_size,
             clobbered_callee_saves: regs,
-            is_leaf,
+            function_calls,
         }
     }
 
diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
@@ -1000,15 +1000,16 @@ impl MachInst for Inst {
         }
     }
 
-    fn is_call(&self) -> bool {
+    fn call_type(&self) -> CallType {
         match self {
             Inst::Call { .. }
             | Inst::CallInd { .. }
-            | Inst::ReturnCall { .. }
-            | Inst::ReturnCallInd { .. }
             | Inst::ElfTlsGetAddr { .. }
-            | Inst::MachOTlsGetAddr { .. } => true,
-            _ => false,
+            | Inst::MachOTlsGetAddr { .. } => CallType::Regular,
+
+            Inst::ReturnCall { .. } | Inst::ReturnCallInd { .. } => CallType::TailCall,
+
+            _ => CallType::None,
         }
     }
 
diff --git a/cranelift/codegen/src/isa/pulley_shared/abi.rs b/cranelift/codegen/src/isa/pulley_shared/abi.rs
@@ -495,7 +495,7 @@ where
         flags: &settings::Flags,
         _sig: &Signature,
         regs: &[Writable<RealReg>],
-        is_leaf: bool,
+        function_calls: FunctionCalls,
         incoming_args_size: u32,
         tail_args_size: u32,
         stackslots_size: u32,
@@ -515,7 +515,7 @@ where
 
         // Compute linkage frame size.
         let setup_area_size = if flags.preserve_frame_pointers()
-            || !is_leaf
+            || function_calls != FunctionCalls::None
             // The function arguments that are passed on the stack are addressed
             // relative to the Frame Pointer.
             || incoming_args_size > 0
@@ -537,7 +537,7 @@ where
             stackslots_size,
             outgoing_args_size,
             clobbered_callee_saves: regs,
-            is_leaf,
+            function_calls,
         }
     }
 
diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs b/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs
@@ -490,14 +490,15 @@ where
         todo!()
     }
 
-    fn is_call(&self) -> bool {
+    fn call_type(&self) -> CallType {
         match &self.inst {
-            Inst::Call { .. }
-            | Inst::IndirectCall { .. }
-            | Inst::IndirectCallHost { .. }
-            | Inst::ReturnCall { .. }
-            | Inst::ReturnIndirectCall { .. } => true,
-            _ => false,
+            Inst::Call { .. } | Inst::IndirectCall { .. } | Inst::IndirectCallHost { .. } => {
+                CallType::Regular
+            }
+
+            Inst::ReturnCall { .. } | Inst::ReturnIndirectCall { .. } => CallType::TailCall,
+
+            _ => CallType::None,
         }
     }
 
diff --git a/cranelift/codegen/src/isa/riscv64/abi.rs b/cranelift/codegen/src/isa/riscv64/abi.rs
@@ -628,7 +628,7 @@ impl ABIMachineSpec for Riscv64MachineDeps {
         flags: &settings::Flags,
         _sig: &Signature,
         regs: &[Writable<RealReg>],
-        is_leaf: bool,
+        function_calls: FunctionCalls,
         incoming_args_size: u32,
         tail_args_size: u32,
         stackslots_size: u32,
@@ -648,7 +648,7 @@ impl ABIMachineSpec for Riscv64MachineDeps {
 
         // Compute linkage frame size.
         let setup_area_size = if flags.preserve_frame_pointers()
-            || !is_leaf
+            || function_calls != FunctionCalls::None
             // The function arguments that are passed on the stack are addressed
             // relative to the Frame Pointer.
             || incoming_args_size > 0
@@ -671,7 +671,7 @@ impl ABIMachineSpec for Riscv64MachineDeps {
             stackslots_size,
             outgoing_args_size,
             clobbered_callee_saves: regs,
-            is_leaf,
+            function_calls,
         }
     }
 
diff --git a/cranelift/codegen/src/isa/riscv64/inst/mod.rs b/cranelift/codegen/src/isa/riscv64/inst/mod.rs
@@ -758,14 +758,15 @@ impl MachInst for Inst {
         }
     }
 
-    fn is_call(&self) -> bool {
+    fn call_type(&self) -> CallType {
         match self {
-            Inst::Call { .. }
-            | Inst::CallInd { .. }
-            | Inst::ReturnCall { .. }
-            | Inst::ReturnCallInd { .. }
-            | Inst::ElfTlsGetAddr { .. } => true,
-            _ => false,
+            Inst::Call { .. } | Inst::CallInd { .. } | Inst::ElfTlsGetAddr { .. } => {
+                CallType::Regular
+            }
+
+            Inst::ReturnCall { .. } | Inst::ReturnCallInd { .. } => CallType::TailCall,
+
+            _ => CallType::None,
         }
     }
 
diff --git a/cranelift/codegen/src/isa/s390x/abi.rs b/cranelift/codegen/src/isa/s390x/abi.rs
@@ -896,7 +896,7 @@ impl ABIMachineSpec for S390xMachineDeps {
         flags: &settings::Flags,
         _sig: &Signature,
         regs: &[Writable<RealReg>],
-        is_leaf: bool,
+        function_calls: FunctionCalls,
         incoming_args_size: u32,
         tail_args_size: u32,
         stackslots_size: u32,
@@ -975,7 +975,7 @@ impl ABIMachineSpec for S390xMachineDeps {
             stackslots_size,
             outgoing_args_size,
             clobbered_callee_saves: regs,
-            is_leaf,
+            function_calls,
         }
     }
 
diff --git a/cranelift/codegen/src/isa/s390x/inst/mod.rs b/cranelift/codegen/src/isa/s390x/inst/mod.rs
@@ -1122,10 +1122,13 @@ impl MachInst for Inst {
         }
     }
 
-    fn is_call(&self) -> bool {
+    fn call_type(&self) -> CallType {
         match self {
-            Inst::Call { .. } | Inst::ReturnCall { .. } | Inst::ElfTlsGetOffset { .. } => true,
-            _ => false,
+            Inst::Call { .. } | Inst::ElfTlsGetOffset { .. } => CallType::Regular,
+
+            Inst::ReturnCall { .. } => CallType::TailCall,
+
+            _ => CallType::None,
         }
     }
 
diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs
@@ -900,7 +900,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
         flags: &settings::Flags,
         _sig: &Signature,
         regs: &[Writable<RealReg>],
-        is_leaf: bool,
+        function_calls: FunctionCalls,
         incoming_args_size: u32,
         tail_args_size: u32,
         stackslots_size: u32,
@@ -947,7 +947,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
             stackslots_size,
             outgoing_args_size,
             clobbered_callee_saves: regs,
-            is_leaf,
+            function_calls,
         }
     }
 
diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs
@@ -1281,15 +1281,16 @@ impl MachInst for Inst {
         }
     }
 
-    fn is_call(&self) -> bool {
+    fn call_type(&self) -> CallType {
         match self {
             Inst::CallKnown { .. }
             | Inst::CallUnknown { .. }
-            | Inst::ReturnCallKnown { .. }
-            | Inst::ReturnCallUnknown { .. }
             | Inst::ElfTlsGetAddr { .. }
-            | Inst::MachOTlsGetAddr { .. } => true,
-            _ => false,
+            | Inst::MachOTlsGetAddr { .. } => CallType::Regular,
+
+            Inst::ReturnCallKnown { .. } | Inst::ReturnCallUnknown { .. } => CallType::TailCall,
+
+            _ => CallType::None,
         }
     }
 
diff --git a/cranelift/codegen/src/machinst/abi.rs b/cranelift/codegen/src/machinst/abi.rs
@@ -490,7 +490,7 @@ pub trait ABIMachineSpec {
         flags: &settings::Flags,
         sig: &Signature,
         regs: &[Writable<RealReg>],
-        is_leaf: bool,
+        function_calls: FunctionCalls,
         incoming_args_size: u32,
         tail_args_size: u32,
         stackslots_size: u32,
@@ -1084,8 +1084,8 @@ pub struct FrameLayout {
     /// restored by gen_clobber_save and gen_clobber_restore.
     pub clobbered_callee_saves: Vec<Writable<RealReg>>,
 
-    /// Whether this function is a leaf function (makes no calls).
-    pub is_leaf: bool,
+    /// The function's call pattern classification.
+    pub function_calls: FunctionCalls,
 }
 
 impl FrameLayout {
@@ -2186,7 +2186,7 @@ impl<M: ABIMachineSpec> Callee<M> {
         sigs: &SigSet,
         spillslots: usize,
         clobbered: Vec<Writable<RealReg>>,
-        is_leaf: bool,
+        function_calls: FunctionCalls,
     ) {
         let bytes = M::word_bytes();
         let total_stacksize = self.stackslots_size + bytes * spillslots as u32;
@@ -2197,7 +2197,7 @@ impl<M: ABIMachineSpec> Callee<M> {
             &self.flags,
             self.signature(),
             &clobbered,
-            is_leaf,
+            function_calls,
             self.stack_args_size(sigs),
             self.tail_args_size,
             self.stackslots_size,
@@ -2234,15 +2234,15 @@ impl<M: ABIMachineSpec> Callee<M> {
             + frame_layout.clobber_size
             + frame_layout.fixed_frame_storage_size
             + frame_layout.outgoing_args_size
-            + if frame_layout.is_leaf {
+            + if frame_layout.function_calls == FunctionCalls::None {
                 0
             } else {
                 frame_layout.setup_area_size
             };
 
         // Leaf functions with zero stack don't need a stack check if one's
         // specified, otherwise always insert the stack check.
-        if total_stacksize > 0 || !frame_layout.is_leaf {
+        if total_stacksize > 0 || frame_layout.function_calls != FunctionCalls::None {
             if let Some((reg, stack_limit_load)) = &self.stack_limit {
                 insts.extend(stack_limit_load.clone());
                 self.insert_stack_check(*reg, total_stacksize, &mut insts);
diff --git a/cranelift/codegen/src/machinst/mod.rs b/cranelift/codegen/src/machinst/mod.rs
@@ -109,9 +109,13 @@ pub trait MachInst: Clone + Debug {
     /// Is this an "args" pseudoinst?
     fn is_args(&self) -> bool;
 
-    /// Is this a call instruction (including indirect calls and tail calls)?
-    /// This is used to determine if a function is truly a leaf function.
-    fn is_call(&self) -> bool;
+    /// Classify the type of call instruction this is.
+    ///
+    /// This enables more granular function type analysis and optimization.
+    /// Returns `CallType::None` for non-call instructions, `CallType::Regular`
+    /// for normal calls that return to the caller, and `CallType::TailCall`
+    /// for tail calls that don't return to the caller.
+    fn call_type(&self) -> CallType;
 
     /// Should this instruction's clobber-list be included in the
     /// clobber-set?
@@ -269,6 +273,55 @@ pub trait MachInstLabelUse: Clone + Copy + Debug + Eq {
     fn from_reloc(reloc: Reloc, addend: Addend) -> Option<Self>;
 }
 
+/// Classification of call instruction types for granular analysis.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum CallType {
+    /// Not a call instruction.
+    None,
+    /// Regular call that returns to the caller.
+    Regular,
+    /// Tail call that doesn't return to the caller.
+    TailCall,
+}
+
+/// Function classification based on call patterns.
+///
+/// This enum classifies functions based on their calling behavior to enable
+/// targeted optimizations. Functions are categorized as:
+/// - `None`: No calls at all (can use simplified calling conventions)
+/// - `TailOnly`: Only tail calls (may skip frame setup in some cases)
+/// - `Regular`: Has regular calls (requires full calling convention support)
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub enum FunctionCalls {
+    /// Function makes no calls at all.
+    #[default]
+    None,
+    /// Function only makes tail calls (no regular calls).
+    TailOnly,
+    /// Function makes at least one regular call (may also have tail calls).
+    Regular,
+}
+
+impl FunctionCalls {
+    /// Update the function classification based on a new call instruction.
+    ///
+    /// This method implements the merge logic for accumulating call patterns:
+    /// - Any regular call makes the function Regular
+    /// - Tail calls upgrade None to TailOnly
+    /// - Regular always stays Regular
+    pub fn update(&mut self, call_type: CallType) {
+        *self = match (*self, call_type) {
+            // No call instruction - state unchanged
+            (current, CallType::None) => current,
+            // Regular call always results in Regular classification
+            (_, CallType::Regular) => FunctionCalls::Regular,
+            // Tail call: None becomes TailOnly, others unchanged
+            (FunctionCalls::None, CallType::TailCall) => FunctionCalls::TailOnly,
+            (current, CallType::TailCall) => current,
+        };
+    }
+}
+
 /// Describes a block terminator (not call) in the VCode.
 ///
 /// Actual targets are not included: the single-source-of-truth for
diff --git a/cranelift/codegen/src/machinst/vcode.rs b/cranelift/codegen/src/machinst/vcode.rs

Original file line number	Diff line number	Diff line change
`@@ -1000,15 +1000,16 @@ impl MachInst for Inst {`
`1000`	`1000`	`}`
`1001`	`1001`	`}`
`1002`	`1002`
`1003`		`- fn is_call(&self) -> bool {`
	`1003`	`+ fn call_type(&self) -> CallType {`
`1004`	`1004`	`match self {`
`1005`	`1005`	`Inst::Call { .. }`
`1006`	`1006`	`\| Inst::CallInd { .. }`
`1007`		`- \| Inst::ReturnCall { .. }`
`1008`		`- \| Inst::ReturnCallInd { .. }`
`1009`	`1007`	`\| Inst::ElfTlsGetAddr { .. }`
`1010`		`- \| Inst::MachOTlsGetAddr { .. } => true,`
`1011`		`- _ => false,`
	`1008`	`+ \| Inst::MachOTlsGetAddr { .. } => CallType::Regular,`
	`1009`	`+`
	`1010`	`+ Inst::ReturnCall { .. } \| Inst::ReturnCallInd { .. } => CallType::TailCall,`
	`1011`	`+`
	`1012`	`+ _ => CallType::None,`
`1012`	`1013`	`}`
`1013`	`1014`	`}`
`1014`	`1015`
Original file line number	Diff line number	Diff line change
`@@ -758,14 +758,15 @@ impl MachInst for Inst {`
`758`	`758`	`}`
`759`	`759`	`}`
`760`	`760`
`761`		`- fn is_call(&self) -> bool {`
	`761`	`+ fn call_type(&self) -> CallType {`
`762`	`762`	`match self {`
`763`		`- Inst::Call { .. }`
`764`		`- \| Inst::CallInd { .. }`
`765`		`- \| Inst::ReturnCall { .. }`
`766`		`- \| Inst::ReturnCallInd { .. }`
`767`		`- \| Inst::ElfTlsGetAddr { .. } => true,`
`768`		`- _ => false,`
	`763`	`+ Inst::Call { .. } \| Inst::CallInd { .. } \| Inst::ElfTlsGetAddr { .. } => {`
	`764`	`+ CallType::Regular`
	`765`	`+ }`
	`766`	`+`
	`767`	`+ Inst::ReturnCall { .. } \| Inst::ReturnCallInd { .. } => CallType::TailCall,`
	`768`	`+`
	`769`	`+ _ => CallType::None,`
`769`	`770`	`}`
`770`	`771`	`}`
`771`	`772`
Original file line number	Diff line number	Diff line change
`@@ -1122,10 +1122,13 @@ impl MachInst for Inst {`
`1122`	`1122`	`}`
`1123`	`1123`	`}`
`1124`	`1124`
`1125`		`- fn is_call(&self) -> bool {`
	`1125`	`+ fn call_type(&self) -> CallType {`
`1126`	`1126`	`match self {`
`1127`		`- Inst::Call { .. } \| Inst::ReturnCall { .. } \| Inst::ElfTlsGetOffset { .. } => true,`
`1128`		`- _ => false,`
	`1127`	`+ Inst::Call { .. } \| Inst::ElfTlsGetOffset { .. } => CallType::Regular,`
	`1128`	`+`
	`1129`	`+ Inst::ReturnCall { .. } => CallType::TailCall,`
	`1130`	`+`
	`1131`	`+ _ => CallType::None,`
`1129`	`1132`	`}`
`1130`	`1133`	`}`
`1131`	`1134`
Original file line number	Diff line number	Diff line change
`@@ -1281,15 +1281,16 @@ impl MachInst for Inst {`
`1281`	`1281`	`}`
`1282`	`1282`	`}`
`1283`	`1283`
`1284`		`- fn is_call(&self) -> bool {`
	`1284`	`+ fn call_type(&self) -> CallType {`
`1285`	`1285`	`match self {`
`1286`	`1286`	`Inst::CallKnown { .. }`
`1287`	`1287`	`\| Inst::CallUnknown { .. }`
`1288`		`- \| Inst::ReturnCallKnown { .. }`
`1289`		`- \| Inst::ReturnCallUnknown { .. }`
`1290`	`1288`	`\| Inst::ElfTlsGetAddr { .. }`
`1291`		`- \| Inst::MachOTlsGetAddr { .. } => true,`
`1292`		`- _ => false,`
	`1289`	`+ \| Inst::MachOTlsGetAddr { .. } => CallType::Regular,`
	`1290`	`+`
	`1291`	`+ Inst::ReturnCallKnown { .. } \| Inst::ReturnCallUnknown { .. } => CallType::TailCall,`
	`1292`	`+`
	`1293`	`+ _ => CallType::None,`
`1293`	`1294`	`}`
`1294`	`1295`	`}`
`1295`	`1296`