x64: Migrate "UnaryRmR" instructions to the new assembler (#10810)

* x64: Migrate "UnaryRmR" instructions to the new assembler

This includes `bsr`, `bsf`, `tzcnt`, `popcnt`, and `lzcnt`.

* Fix emit tests

Author: alexcrichton

cranelift/assembler-x64/meta/src/dsl/features.rs

@@ -66,6 +66,9 @@ pub enum Feature {
     sse2,
     ssse3,
     sse41,
+    bmi1,
+    lzcnt,
+    popcnt,
 }
 
 /// List all CPU features.
@@ -82,18 +85,14 @@ pub const ALL_FEATURES: &[Feature] = &[
     Feature::sse2,
     Feature::ssse3,
     Feature::sse41,
+    Feature::bmi1,
+    Feature::lzcnt,
+    Feature::popcnt,
 ];
 
 impl fmt::Display for Feature {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match self {
-            Feature::_64b => write!(f, "_64b"),
-            Feature::compat => write!(f, "compat"),
-            Feature::sse => write!(f, "sse"),
-            Feature::sse2 => write!(f, "sse2"),
-            Feature::ssse3 => write!(f, "ssse3"),
-            Feature::sse41 => write!(f, "sse41"),
-        }
+        fmt::Debug::fmt(self, f)
     }
 }
 

cranelift/assembler-x64/meta/src/instructions.rs

@@ -2,6 +2,7 @@
 
 mod add;
 mod and;
+mod bitmanip;
 mod cvt;
 mod mul;
 mod neg;
@@ -17,6 +18,7 @@ pub fn list() -> Vec<Inst> {
     let mut all = vec![];
     all.extend(add::list());
     all.extend(and::list());
+    all.extend(bitmanip::list());
     all.extend(cvt::list());
     all.extend(mul::list());
     all.extend(neg::list());

cranelift/assembler-x64/meta/src/instructions/bitmanip.rs

@@ -0,0 +1,27 @@
+use crate::dsl::{Feature::*, Inst, Location::*};
+use crate::dsl::{fmt, inst, r, rex, w};
+
+#[rustfmt::skip] // Keeps instructions on a single line.
+pub fn list() -> Vec<Inst> {
+    vec![
+        inst("bsfw", fmt("RM", [w(r16), r(rm16)]), rex([0x66, 0x0F, 0xBC]).r(), _64b | compat),
+        inst("bsfl", fmt("RM", [w(r32), r(rm32)]), rex([0x0F, 0xBC]).r(), _64b | compat),
+        inst("bsfq", fmt("RM", [w(r64), r(rm64)]), rex([0x0F, 0xBC]).r().w(), _64b),
+
+        inst("bsrw", fmt("RM", [w(r16), r(rm16)]), rex([0x66, 0x0F, 0xBD]).r(), _64b | compat),
+        inst("bsrl", fmt("RM", [w(r32), r(rm32)]), rex([0x0F, 0xBD]).r(), _64b | compat),
+        inst("bsrq", fmt("RM", [w(r64), r(rm64)]), rex([0x0F, 0xBD]).r().w(), _64b),
+
+        inst("tzcntw", fmt("A", [w(r16), r(rm16)]), rex([0x66, 0xF3, 0x0F, 0xBC]).r(), _64b | compat | bmi1),
+        inst("tzcntl", fmt("A", [w(r32), r(rm32)]), rex([0xF3, 0x0F, 0xBC]).r(), _64b | compat | bmi1),
+        inst("tzcntq", fmt("A", [w(r64), r(rm64)]), rex([0xF3, 0x0F, 0xBC]).r().w(), _64b | bmi1),
+
+        inst("lzcntw", fmt("RM", [w(r16), r(rm16)]), rex([0x66, 0xF3, 0x0F, 0xBD]).r(), _64b | compat | lzcnt),
+        inst("lzcntl", fmt("RM", [w(r32), r(rm32)]), rex([0xF3, 0x0F, 0xBD]).r(), _64b | compat | lzcnt),
+        inst("lzcntq", fmt("RM", [w(r64), r(rm64)]), rex([0xF3, 0x0F, 0xBD]).r().w(), _64b | lzcnt),
+
+        inst("popcntw", fmt("RM", [w(r16), r(rm16)]), rex([0x66, 0xF3, 0x0F, 0xB8]).r(), _64b | compat | popcnt),
+        inst("popcntl", fmt("RM", [w(r32), r(rm32)]), rex([0xF3, 0x0F, 0xB8]).r(), _64b | compat | popcnt),
+        inst("popcntq", fmt("RM", [w(r64), r(rm64)]), rex([0xF3, 0x0F, 0xB8]).r().w(), _64b | popcnt),
+    ]
+}

cranelift/assembler-x64/src/gpr.rs

@@ -46,6 +46,12 @@ impl<R: AsReg> AsMut<R> for Gpr<R> {
     }
 }
 
+impl<R: AsReg> From<R> for Gpr<R> {
+    fn from(reg: R) -> Gpr<R> {
+        Gpr(reg)
+    }
+}
+
 /// A single x64 register encoding can access a different number of bits.
 #[derive(Copy, Clone, Debug)]
 pub enum Size {
@@ -96,6 +102,12 @@ impl<R: AsReg> AsMut<R> for NonRspGpr<R> {
     }
 }
 
+impl<R: AsReg> From<R> for NonRspGpr<R> {
+    fn from(reg: R) -> NonRspGpr<R> {
+        NonRspGpr(reg)
+    }
+}
+
 /// Encode x64 registers.
 pub mod enc {
     use super::Size;

cranelift/assembler-x64/src/mem.rs

@@ -300,6 +300,18 @@ impl<R: AsReg, M: AsReg> GprMem<R, M> {
     }
 }
 
+impl<R: AsReg, M: AsReg> From<R> for GprMem<R, M> {
+    fn from(reg: R) -> GprMem<R, M> {
+        GprMem::Gpr(reg)
+    }
+}
+
+impl<R: AsReg, M: AsReg> From<Amode<M>> for GprMem<R, M> {
+    fn from(amode: Amode<M>) -> GprMem<R, M> {
+        GprMem::Mem(amode)
+    }
+}
+
 /// An XMM register or memory operand.
 #[derive(Clone, Debug)]
 #[cfg_attr(any(test, feature = "fuzz"), derive(arbitrary::Arbitrary))]
@@ -349,6 +361,18 @@ impl<R: AsReg, M: AsReg> XmmMem<R, M> {
     }
 }
 
+impl<R: AsReg, M: AsReg> From<R> for XmmMem<R, M> {
+    fn from(reg: R) -> XmmMem<R, M> {
+        XmmMem::Xmm(reg)
+    }
+}
+
+impl<R: AsReg, M: AsReg> From<Amode<M>> for XmmMem<R, M> {
+    fn from(amode: Amode<M>) -> XmmMem<R, M> {
+        XmmMem::Mem(amode)
+    }
+}
+
 /// Emit the ModRM/SIB/displacement sequence for a memory operand.
 fn emit_modrm_sib_disp<R: AsReg>(
     sink: &mut impl CodeSink,

cranelift/assembler-x64/src/xmm.rs

@@ -47,6 +47,12 @@ impl<R: AsReg> AsMut<R> for Xmm<R> {
     }
 }
 
+impl<R: AsReg> From<R> for Xmm<R> {
+    fn from(reg: R) -> Xmm<R> {
+        Xmm(reg)
+    }
+}
+
 /// Encode xmm registers.
 pub mod enc {
     pub const XMM0: u8 = 0;

cranelift/codegen/src/isa/x64/inst.isle

@@ -22,13 +22,6 @@
                   (src2 GprMem)
                   (dst WritableGpr))
 
-       ;; Instructions on general-purpose registers that only read src and
-       ;; defines dst (dst is not modified). `bsr`, etc.
-       (UnaryRmR (size OperandSize) ;; 2, 4, or 8
-                 (op UnaryRmROpcode)
-                 (src GprMem)
-                 (dst WritableGpr))
-
        ;; Same as `UnaryRmR` but with the VEX prefix for BMI1 instructions.
        (UnaryRmRVex (size OperandSize)
                     (op UnaryRmRVexOpcode)
@@ -2143,13 +2136,6 @@
             (_ Unit (emit (MInst.XmmRmRVex3 op src1 src2 src3 dst))))
         dst))
 
-;; Helper for creating `MInst.UnaryRmR` instructions.
-(decl unary_rm_r (UnaryRmROpcode Gpr OperandSize) Gpr)
-(rule (unary_rm_r op src size)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (_ Unit (emit (MInst.UnaryRmR size op src dst))))
-        dst))
-
 ;; Helper for creating `MInst.UnaryRmRVex` instructions.
 (decl unary_rm_r_vex (UnaryRmRVexOpcode GprMem OperandSize) Gpr)
 (rule (unary_rm_r_vex op src size)
@@ -4502,22 +4488,22 @@
       (SideEffectNoResult.Inst (MInst.Hlt)))
 
 ;; Helper for creating `lzcnt` instructions.
-(decl x64_lzcnt (Type Gpr) Gpr)
-(rule (x64_lzcnt ty src)
-      (unary_rm_r (UnaryRmROpcode.Lzcnt) src (operand_size_of_type_32_64 ty)))
+(decl x64_lzcnt (Type GprMem) Gpr)
+(rule (x64_lzcnt $I16 src) (x64_lzcntw_rm src))
+(rule (x64_lzcnt $I32 src) (x64_lzcntl_rm src))
+(rule (x64_lzcnt $I64 src) (x64_lzcntq_rm src))
 
 ;; Helper for creating `tzcnt` instructions.
-(decl x64_tzcnt (Type Gpr) Gpr)
-(rule (x64_tzcnt ty src)
-      (unary_rm_r (UnaryRmROpcode.Tzcnt) src (operand_size_of_type_32_64 ty)))
+(decl x64_tzcnt (Type GprMem) Gpr)
+(rule (x64_tzcnt $I16 src) (x64_tzcntw_a src))
+(rule (x64_tzcnt $I32 src) (x64_tzcntl_a src))
+(rule (x64_tzcnt $I64 src) (x64_tzcntq_a src))
 
 ;; Helper for creating `bsr` instructions.
-(decl x64_bsr (Type Gpr) ProducesFlags)
-(rule (x64_bsr ty src)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (size OperandSize (operand_size_of_type_32_64 ty))
-            (inst MInst (MInst.UnaryRmR size (UnaryRmROpcode.Bsr) src dst)))
-        (ProducesFlags.ProducesFlagsReturnsReg inst dst)))
+(decl x64_bsr (Type GprMem) ProducesFlags)
+(rule (x64_bsr $I16 src) (asm_produce_flags (x64_bsrw_rm_raw src)))
+(rule (x64_bsr $I32 src) (asm_produce_flags (x64_bsrl_rm_raw src)))
+(rule (x64_bsr $I64 src) (asm_produce_flags (x64_bsrq_rm_raw src)))
 
 ;; Helper for creating `bsr + cmov` instruction pairs that produce the
 ;; result of the `bsr`, or `alt` if the input was zero.
@@ -4532,12 +4518,10 @@
         (with_flags_reg (produces_flags_ignore bsr) cmove)))
 
 ;; Helper for creating `bsf` instructions.
-(decl x64_bsf (Type Gpr) ProducesFlags)
-(rule (x64_bsf ty src)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (size OperandSize (operand_size_of_type_32_64 ty))
-            (inst MInst (MInst.UnaryRmR size (UnaryRmROpcode.Bsf) src dst)))
-        (ProducesFlags.ProducesFlagsReturnsReg inst dst)))
+(decl x64_bsf (Type GprMem) ProducesFlags)
+(rule (x64_bsf $I16 src) (asm_produce_flags (x64_bsfw_rm_raw src)))
+(rule (x64_bsf $I32 src) (asm_produce_flags (x64_bsfl_rm_raw src)))
+(rule (x64_bsf $I64 src) (asm_produce_flags (x64_bsfq_rm_raw src)))
 
 ;; Helper for creating `bsf + cmov` instruction pairs that produce the
 ;; result of the `bsf`, or `alt` if the input was zero.
@@ -4590,9 +4574,10 @@
                           imm))
 
 ;; Helper for creating `popcnt` instructions.
-(decl x64_popcnt (Type Gpr) Gpr)
-(rule (x64_popcnt ty src)
-      (unary_rm_r (UnaryRmROpcode.Popcnt) src (operand_size_of_type_32_64 ty)))
+(decl x64_popcnt (Type GprMem) Gpr)
+(rule (x64_popcnt $I16 src) (x64_popcntw_rm src))
+(rule (x64_popcnt $I32 src) (x64_popcntl_rm src))
+(rule (x64_popcnt $I64 src) (x64_popcntq_rm src))
 
 ;; Helper for creating `minss` instructions.
 (decl x64_minss (Xmm XmmMem) Xmm)

cranelift/codegen/src/isa/x64/inst/args.rs

@@ -824,50 +824,6 @@ impl fmt::Display for AluRmROpcode {
     }
 }
 
-#[derive(Clone, PartialEq)]
-/// Unary operations requiring register or memory and register operands.
-pub enum UnaryRmROpcode {
-    /// Bit-scan reverse.
-    Bsr,
-    /// Bit-scan forward.
-    Bsf,
-    /// Counts leading zeroes (Leading Zero CouNT).
-    Lzcnt,
-    /// Counts trailing zeroes (Trailing Zero CouNT).
-    Tzcnt,
-    /// Counts the number of ones (POPulation CouNT).
-    Popcnt,
-}
-
-impl UnaryRmROpcode {
-    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
-        match self {
-            UnaryRmROpcode::Bsr | UnaryRmROpcode::Bsf => smallvec![],
-            UnaryRmROpcode::Lzcnt => smallvec![InstructionSet::Lzcnt],
-            UnaryRmROpcode::Tzcnt => smallvec![InstructionSet::BMI1],
-            UnaryRmROpcode::Popcnt => smallvec![InstructionSet::Popcnt],
-        }
-    }
-}
-
-impl fmt::Debug for UnaryRmROpcode {
-    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
-        match self {
-            UnaryRmROpcode::Bsr => write!(fmt, "bsr"),
-            UnaryRmROpcode::Bsf => write!(fmt, "bsf"),
-            UnaryRmROpcode::Lzcnt => write!(fmt, "lzcnt"),
-            UnaryRmROpcode::Tzcnt => write!(fmt, "tzcnt"),
-            UnaryRmROpcode::Popcnt => write!(fmt, "popcnt"),
-        }
-    }
-}
-
-impl fmt::Display for UnaryRmROpcode {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        fmt::Debug::fmt(self, f)
-    }
-}
-
 pub use crate::isa::x64::lower::isle::generated_code::UnaryRmRVexOpcode;
 
 impl UnaryRmRVexOpcode {

cranelift/codegen/src/isa/x64/inst/emit.rs

@@ -188,41 +188,6 @@ pub(crate) fn emit(
                 .encode(sink);
         }
 
-        Inst::UnaryRmR { size, op, src, dst } => {
-            let dst = dst.to_reg().to_reg();
-            let rex_flags = RexFlags::from(*size);
-            use UnaryRmROpcode::*;
-            let prefix = match size {
-                OperandSize::Size16 => match op {
-                    Bsr | Bsf => LegacyPrefixes::_66,
-                    Lzcnt | Tzcnt | Popcnt => LegacyPrefixes::_66F3,
-                },
-                OperandSize::Size32 | OperandSize::Size64 => match op {
-                    Bsr | Bsf => LegacyPrefixes::None,
-                    Lzcnt | Tzcnt | Popcnt => LegacyPrefixes::_F3,
-                },
-                _ => unreachable!(),
-            };
-
-            let (opcode, num_opcodes) = match op {
-                Bsr => (0x0fbd, 2),
-                Bsf => (0x0fbc, 2),
-                Lzcnt => (0x0fbd, 2),
-                Tzcnt => (0x0fbc, 2),
-                Popcnt => (0x0fb8, 2),
-            };
-
-            match src.clone().into() {
-                RegMem::Reg { reg: src } => {
-                    emit_std_reg_reg(sink, prefix, opcode, num_opcodes, dst, src, rex_flags);
-                }
-                RegMem::Mem { addr: src } => {
-                    let amode = src.finalize(state.frame_layout(), sink).clone();
-                    emit_std_reg_mem(sink, prefix, opcode, num_opcodes, dst, &amode, rex_flags, 0);
-                }
-            }
-        }
-
         Inst::UnaryRmRVex { size, op, src, dst } => {
             let dst = dst.to_reg().to_reg();
             let src = match src.clone().to_reg_mem().clone() {

cranelift/codegen/src/isa/x64/inst/emit_tests.rs

@@ -1081,30 +1081,6 @@ fn test_x64_emit() {
     // guidelines commented just prior to `fn x64_emit`.
     //
 
-    // ========================================================
-    // UnaryRmR
-
-    insns.push((
-        Inst::unary_rm_r(
-            OperandSize::Size32,
-            UnaryRmROpcode::Bsr,
-            RegMem::reg(rsi),
-            w_rdi,
-        ),
-        "0FBDFE",
-        "bsrl    %esi, %edi",
-    ));
-    insns.push((
-        Inst::unary_rm_r(
-            OperandSize::Size64,
-            UnaryRmROpcode::Bsr,
-            RegMem::reg(r15),
-            w_rax,
-        ),
-        "490FBDC7",
-        "bsrq    %r15, %rax",
-    ));
-
     // ========================================================
     // Div
     insns.push((