x64: Add conditional jumps to the new assembler (#11196)

alexcrichton · web-flow · commit 22f51bca37bf · 2025-07-09T18:32:09.000Z
While this doesn't remove any pseudo insts from ISLE (as was thought
might be the case with `WinchJmpIf`) this does clean up some emission
code to use some helpers and make it more clear what's happening.
diff --git a/cranelift/assembler-x64/meta/src/instructions/jmp.rs b/cranelift/assembler-x64/meta/src/instructions/jmp.rs
@@ -1,9 +1,66 @@
 use crate::dsl::{Customization::*, Feature::*, Inst, Location::*};
-use crate::dsl::{fmt, inst, r, rex};
+use crate::dsl::{fmt, inst, r, rex, sxq};
 
 #[rustfmt::skip] // Keeps instructions on a single line.
 pub fn list() -> Vec<Inst> {
     vec![
         inst("jmpq", fmt("M", [r(rm64)]), rex([0xFF]).digit(4), _64b).custom(Display),
+
+        inst("jmp", fmt("D8", [r(sxq(imm8))]), rex([0xEB]).ib(), _64b | compat).custom(Display),
+        inst("jmp", fmt("D32", [r(sxq(imm32))]), rex([0xE9]).id(), _64b | compat).custom(Display),
+
+        // Note that the Intel manual lists many mnemonics for this family of
+        // instructions which are duplicates of other mnemonics. The order here
+        // matches the order in the manual and comments are left when variants
+        // are omitted due to the instructions being duplicates of another.
+        inst("ja", fmt("D8", [r(sxq(imm8))]), rex([0x77]).ib(), _64b | compat).custom(Display),
+        inst("ja", fmt("D32", [r(sxq(imm32))]), rex([0x0F, 0x87]).id(), _64b | compat).custom(Display),
+        inst("jae", fmt("D8", [r(sxq(imm8))]), rex([0x73]).ib(), _64b | compat).custom(Display),
+        inst("jae", fmt("D32", [r(sxq(imm32))]), rex([0x0F, 0x83]).id(), _64b | compat).custom(Display),
+        inst("jb", fmt("D8", [r(sxq(imm8))]), rex([0x72]).ib(), _64b | compat).custom(Display),
+        inst("jb", fmt("D32", [r(sxq(imm32))]), rex([0x0F, 0x82]).id(), _64b | compat).custom(Display),
+        inst("jbe", fmt("D8", [r(sxq(imm8))]), rex([0x76]).ib(), _64b | compat).custom(Display),
+        inst("jbe", fmt("D32", [r(sxq(imm32))]), rex([0x0F, 0x86]).id(), _64b | compat).custom(Display),
+        // jc == jb
+        // TODO: jcx
+        // TODO: jecx
+        // TODO: jrcx
+        inst("je", fmt("D8", [r(sxq(imm8))]), rex([0x74]).ib(), _64b | compat).custom(Display),
+        inst("je", fmt("D32", [r(sxq(imm32))]), rex([0x0F, 0x84]).id(), _64b | compat).custom(Display),
+        inst("jg", fmt("D8", [r(sxq(imm8))]), rex([0x7F]).ib(), _64b | compat).custom(Display),
+        inst("jg", fmt("D32", [r(sxq(imm32))]), rex([0x0F, 0x8F]).id(), _64b | compat).custom(Display),
+        inst("jge", fmt("D8", [r(sxq(imm8))]), rex([0x7D]).ib(), _64b | compat).custom(Display),
+        inst("jge", fmt("D32", [r(sxq(imm32))]), rex([0x0F, 0x8D]).id(), _64b | compat).custom(Display),
+        inst("jl", fmt("D8", [r(sxq(imm8))]), rex([0x7C]).ib(), _64b | compat).custom(Display),
+        inst("jl", fmt("D32", [r(sxq(imm32))]), rex([0x0F, 0x8C]).id(), _64b | compat).custom(Display),
+        inst("jle", fmt("D8", [r(sxq(imm8))]), rex([0x7E]).ib(), _64b | compat).custom(Display),
+        inst("jle", fmt("D32", [r(sxq(imm32))]), rex([0x0F, 0x8E]).id(), _64b | compat).custom(Display),
+        // jna == jbe
+        // jnae == jb
+        // jnb == jae
+        // jnbe == ja
+        // jnc == jae
+        inst("jne", fmt("D8", [r(sxq(imm8))]), rex([0x75]).ib(), _64b | compat).custom(Display),
+        inst("jne", fmt("D32", [r(sxq(imm32))]), rex([0x0F, 0x85]).id(), _64b | compat).custom(Display),
+        // jng == jle
+        // jnge == jl
+        // jnl == jge
+        // jnle == jg
+        inst("jno", fmt("D8", [r(sxq(imm8))]), rex([0x71]).ib(), _64b | compat).custom(Display),
+        inst("jno", fmt("D32", [r(sxq(imm32))]), rex([0x0F, 0x81]).id(), _64b | compat).custom(Display),
+        inst("jnp", fmt("D8", [r(sxq(imm8))]), rex([0x7B]).ib(), _64b | compat).custom(Display),
+        inst("jnp", fmt("D32", [r(sxq(imm32))]), rex([0x0F, 0x8B]).id(), _64b | compat).custom(Display),
+        inst("jns", fmt("D8", [r(sxq(imm8))]), rex([0x79]).ib(), _64b | compat).custom(Display),
+        inst("jns", fmt("D32", [r(sxq(imm32))]), rex([0x0F, 0x89]).id(), _64b | compat).custom(Display),
+        // jnz == jne
+        inst("jo", fmt("D8", [r(sxq(imm8))]), rex([0x70]).ib(), _64b | compat).custom(Display),
+        inst("jo", fmt("D32", [r(sxq(imm32))]), rex([0x0F, 0x80]).id(), _64b | compat).custom(Display),
+        inst("jp", fmt("D8", [r(sxq(imm8))]), rex([0x7A]).ib(), _64b | compat).custom(Display),
+        inst("jp", fmt("D32", [r(sxq(imm32))]), rex([0x0F, 0x8A]).id(), _64b | compat).custom(Display),
+        // jpe == jp
+        // jpo == jnp
+        inst("js", fmt("D8", [r(sxq(imm8))]), rex([0x78]).ib(), _64b | compat).custom(Display),
+        inst("js", fmt("D32", [r(sxq(imm32))]), rex([0x0F, 0x88]).id(), _64b | compat).custom(Display),
+        // jz == je
     ]
 }
diff --git a/cranelift/assembler-x64/src/custom.rs b/cranelift/assembler-x64/src/custom.rs
@@ -195,12 +195,7 @@ pub mod display {
 
     pub fn callq_d(f: &mut fmt::Formatter, inst: &inst::callq_d) -> fmt::Result {
         let inst::callq_d { imm32 } = inst;
-        let displacement = i64::from(imm32.value()) + 5;
-        if displacement >= 0 && displacement < 10 {
-            write!(f, "callq {displacement:}")
-        } else {
-            write!(f, "callq {displacement:#x}")
-        }
+        display_displacement(f, "callq", i64::from(imm32.value()) + 5)
     }
 
     pub fn callq_m<R: Registers>(f: &mut fmt::Formatter, inst: &inst::callq_m<R>) -> fmt::Result {
@@ -585,6 +580,61 @@ pub mod display {
         let rm64 = rm64.to_string(Size::Quadword);
         write!(f, "jmpq *{rm64}")
     }
+
+    pub fn jmp_d8(f: &mut fmt::Formatter<'_>, jmp: &inst::jmp_d8) -> fmt::Result {
+        let inst::jmp_d8 { imm8 } = jmp;
+        display_displacement(f, "jmp", i64::from(imm8.value()) + 2)
+    }
+
+    pub fn jmp_d32(f: &mut fmt::Formatter<'_>, jmp: &inst::jmp_d32) -> fmt::Result {
+        let inst::jmp_d32 { imm32 } = jmp;
+        display_displacement(f, "jmp", i64::from(imm32.value()) + 5)
+    }
+
+    macro_rules! jcc {
+        ($($mnemonic:tt = $j8:ident / $j32:ident;)*) => ($(
+            pub fn $j8(f: &mut fmt::Formatter<'_>, jmp: &inst::$j8) -> fmt::Result {
+                let inst::$j8 { imm8 } = jmp;
+                display_displacement(f, $mnemonic, i64::from(imm8.value()) + 2)
+            }
+
+            pub fn $j32(f: &mut fmt::Formatter<'_>, jmp: &inst::$j32) -> fmt::Result {
+                let inst::$j32 { imm32 } = jmp;
+                display_displacement(f, $mnemonic, i64::from(imm32.value()) + 6)
+            }
+        )*)
+    }
+
+    jcc! {
+        "ja" = ja_d8 / ja_d32;
+        "jae" = jae_d8 / jae_d32;
+        "jb" = jb_d8 / jb_d32;
+        "jbe" = jbe_d8 / jbe_d32;
+        "je" = je_d8 / je_d32;
+        "jg" = jg_d8 / jg_d32;
+        "jge" = jge_d8 / jge_d32;
+        "jl" = jl_d8 / jl_d32;
+        "jle" = jle_d8 / jle_d32;
+        "jne" = jne_d8 / jne_d32;
+        "jno" = jno_d8 / jno_d32;
+        "jnp" = jnp_d8 / jnp_d32;
+        "jns" = jns_d8 / jns_d32;
+        "jo" = jo_d8 / jo_d32;
+        "jp" = jp_d8 / jp_d32;
+        "js" = js_d8 / js_d32;
+    }
+
+    fn display_displacement(
+        f: &mut fmt::Formatter<'_>,
+        mnemonic: &str,
+        displacement: i64,
+    ) -> fmt::Result {
+        if displacement >= 0 && displacement < 10 {
+            write!(f, "{mnemonic} {displacement}")
+        } else {
+            write!(f, "{mnemonic} {displacement:#x}")
+        }
+    }
 }
 
 pub mod visit {
diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs
@@ -35,9 +35,73 @@ fn one_way_jmp(sink: &mut MachBuffer<Inst>, cc: CC, label: MachLabel) {
     let cond_start = sink.cur_offset();
     let cond_disp_off = cond_start + 2;
     sink.use_label_at_offset(cond_disp_off, label, LabelUse::JmpRel32);
-    sink.put1(0x0F);
-    sink.put1(0x80 + cc.get_enc());
-    sink.put4(0x0);
+    emit_jcc_no_offset(sink, cc);
+    debug_assert_eq!(sink.cur_offset(), cond_disp_off + 4);
+}
+
+/// Like `one_way_jmp` above emitting a conditional jump, but also using
+/// `MachBuffer::add_cond_branch`.
+fn cond_jmp(sink: &mut MachBuffer<Inst>, cc: CC, label: MachLabel) {
+    let cond_start = sink.cur_offset();
+    let cond_disp_off = cond_start + 2;
+    let cond_end = cond_start + 6;
+
+    sink.use_label_at_offset(cond_disp_off, label, LabelUse::JmpRel32);
+    // FIXME: ideally this `inverted` calculation would go through the external
+    // assembler, but for now it's left done manually.
+    let inverted: [u8; 6] = [0x0F, 0x80 + (cc.invert().get_enc()), 0x00, 0x00, 0x00, 0x00];
+    sink.add_cond_branch(cond_start, cond_end, label, &inverted[..]);
+
+    emit_jcc_no_offset(sink, cc);
+
+    debug_assert_eq!(sink.cur_offset(), cond_disp_off + 4);
+    debug_assert_eq!(sink.cur_offset(), cond_end);
+}
+
+fn emit_jcc_no_offset(sink: &mut MachBuffer<Inst>, cc: CC) {
+    // Note that the disassembler matches Capstone which doesn't match the `CC`
+    // enum directly as Intel has multiple mnemonics use the same encoding.
+    let inst: AsmInst = match cc {
+        CC::Z => asm::inst::je_d32::new(0).into(),   // jz == je
+        CC::NZ => asm::inst::jne_d32::new(0).into(), // jnz == jne
+        CC::B => asm::inst::jb_d32::new(0).into(),
+        CC::NB => asm::inst::jae_d32::new(0).into(), // jnb == jae
+        CC::BE => asm::inst::jbe_d32::new(0).into(),
+        CC::NBE => asm::inst::ja_d32::new(0).into(), // jnbe == ja
+        CC::L => asm::inst::jl_d32::new(0).into(),
+        CC::LE => asm::inst::jle_d32::new(0).into(),
+        CC::NL => asm::inst::jge_d32::new(0).into(), // jnl == jge
+        CC::NLE => asm::inst::jg_d32::new(0).into(), // jnle == jg
+        CC::O => asm::inst::jo_d32::new(0).into(),
+        CC::NO => asm::inst::jno_d32::new(0).into(),
+        CC::P => asm::inst::jp_d32::new(0).into(),
+        CC::NP => asm::inst::jnp_d32::new(0).into(),
+        CC::S => asm::inst::js_d32::new(0).into(),
+        CC::NS => asm::inst::jns_d32::new(0).into(),
+    };
+    inst.encode(&mut external::AsmCodeSink {
+        sink,
+        incoming_arg_offset: 0,
+        slot_offset: 0,
+    });
+}
+
+/// Emits an unconditional branch.
+fn uncond_jmp(sink: &mut MachBuffer<Inst>, label: MachLabel) {
+    let uncond_start = sink.cur_offset();
+    let uncond_disp_off = uncond_start + 1;
+    let uncond_end = uncond_start + 5;
+
+    sink.use_label_at_offset(uncond_disp_off, label, LabelUse::JmpRel32);
+    sink.add_uncond_branch(uncond_start, uncond_end, label);
+
+    asm::inst::jmp_d32::new(0).encode(&mut external::AsmCodeSink {
+        sink,
+        incoming_arg_offset: 0,
+        slot_offset: 0,
+    });
+    debug_assert_eq!(sink.cur_offset(), uncond_disp_off + 4);
+    debug_assert_eq!(sink.cur_offset(), uncond_end);
 }
 
 /// Emits a relocation, attaching the current source location as well.
@@ -427,11 +491,11 @@ pub(crate) fn emit(
             // Note: this is not `Inst::Jmp { .. }.emit(..)` because we have
             // different metadata in this case: we don't have a label for the
             // target, but rather a function relocation.
-            sink.put1(0xE9);
+            asm::inst::jmp_d32::new(0).emit(sink, info, state);
+            let offset = sink.cur_offset();
             // The addend adjusts for the difference between the end of the instruction and the
             // beginning of the immediate field.
-            emit_reloc(sink, Reloc::X86CallPCRel4, &call_info.dest, -4);
-            sink.put4(0);
+            sink.add_reloc_at_offset(offset - 4, Reloc::X86CallPCRel4, &call_info.dest, -4);
             sink.add_call_site(&[]);
         }
 
@@ -595,62 +659,17 @@ pub(crate) fn emit(
             sink.bind_label(resume, state.ctrl_plane_mut());
         }
 
-        Inst::JmpKnown { dst } => {
-            let br_start = sink.cur_offset();
-            let br_disp_off = br_start + 1;
-            let br_end = br_start + 5;
+        Inst::JmpKnown { dst } => uncond_jmp(sink, *dst),
 
-            sink.use_label_at_offset(br_disp_off, *dst, LabelUse::JmpRel32);
-            sink.add_uncond_branch(br_start, br_end, *dst);
-
-            sink.put1(0xE9);
-            // Placeholder for the label value.
-            sink.put4(0x0);
-        }
-
-        Inst::WinchJmpIf { cc, taken } => {
-            let cond_start = sink.cur_offset();
-            let cond_disp_off = cond_start + 2;
-
-            sink.use_label_at_offset(cond_disp_off, *taken, LabelUse::JmpRel32);
-            // Since this is not a terminator, don't enroll in the branch inversion mechanism.
-
-            sink.put1(0x0F);
-            sink.put1(0x80 + cc.get_enc());
-            // Placeholder for the label value.
-            sink.put4(0x0);
-        }
+        Inst::WinchJmpIf { cc, taken } => one_way_jmp(sink, *cc, *taken),
 
         Inst::JmpCond {
             cc,
             taken,
             not_taken,
         } => {
-            // If taken.
-            let cond_start = sink.cur_offset();
-            let cond_disp_off = cond_start + 2;
-            let cond_end = cond_start + 6;
-
-            sink.use_label_at_offset(cond_disp_off, *taken, LabelUse::JmpRel32);
-            let inverted: [u8; 6] = [0x0F, 0x80 + (cc.invert().get_enc()), 0x00, 0x00, 0x00, 0x00];
-            sink.add_cond_branch(cond_start, cond_end, *taken, &inverted[..]);
-
-            sink.put1(0x0F);
-            sink.put1(0x80 + cc.get_enc());
-            // Placeholder for the label value.
-            sink.put4(0x0);
-
-            // If not taken.
-            let uncond_start = sink.cur_offset();
-            let uncond_disp_off = uncond_start + 1;
-            let uncond_end = uncond_start + 5;
-
-            sink.use_label_at_offset(uncond_disp_off, *not_taken, LabelUse::JmpRel32);
-            sink.add_uncond_branch(uncond_start, uncond_end, *not_taken);
-
-            sink.put1(0xE9);
-            // Placeholder for the label value.
-            sink.put4(0x0);
+            cond_jmp(sink, *cc, *taken);
+            uncond_jmp(sink, *not_taken);
         }
 
         Inst::JmpCondOr {
@@ -671,56 +690,9 @@ pub(crate) fn emit(
             // not_taken and that one block is the fallthrough block,
             // all three branches can disappear.
 
-            // jcc1 taken
-            let cond_1_start = sink.cur_offset();
-            let cond_1_disp_off = cond_1_start + 2;
-            let cond_1_end = cond_1_start + 6;
-
-            sink.use_label_at_offset(cond_1_disp_off, *taken, LabelUse::JmpRel32);
-            let inverted: [u8; 6] = [
-                0x0F,
-                0x80 + (cc1.invert().get_enc()),
-                0x00,
-                0x00,
-                0x00,
-                0x00,
-            ];
-            sink.add_cond_branch(cond_1_start, cond_1_end, *taken, &inverted[..]);
-
-            sink.put1(0x0F);
-            sink.put1(0x80 + cc1.get_enc());
-            sink.put4(0x0);
-
-            // jcc2 taken
-            let cond_2_start = sink.cur_offset();
-            let cond_2_disp_off = cond_2_start + 2;
-            let cond_2_end = cond_2_start + 6;
-
-            sink.use_label_at_offset(cond_2_disp_off, *taken, LabelUse::JmpRel32);
-            let inverted: [u8; 6] = [
-                0x0F,
-                0x80 + (cc2.invert().get_enc()),
-                0x00,
-                0x00,
-                0x00,
-                0x00,
-            ];
-            sink.add_cond_branch(cond_2_start, cond_2_end, *taken, &inverted[..]);
-
-            sink.put1(0x0F);
-            sink.put1(0x80 + cc2.get_enc());
-            sink.put4(0x0);
-
-            // jmp not_taken
-            let uncond_start = sink.cur_offset();
-            let uncond_disp_off = uncond_start + 1;
-            let uncond_end = uncond_start + 5;
-
-            sink.use_label_at_offset(uncond_disp_off, *not_taken, LabelUse::JmpRel32);
-            sink.add_uncond_branch(uncond_start, uncond_end, *not_taken);
-
-            sink.put1(0xE9);
-            sink.put4(0x0);
+            cond_jmp(sink, *cc1, *taken);
+            cond_jmp(sink, *cc2, *taken);
+            uncond_jmp(sink, *not_taken);
         }
 
         &Inst::JmpTableSeq {
