x64: convert all multiplication-related instructions (#10782)

* asm: add SSE4.1 feature

* asm: refactor format constructors with expanding structs

* asm: add implicit operands

Implicit operands are used by an instruction but not present in its
disassembled output. Instructions like `mul`, e.g., will write to the
`%rax` and `%rdx` registers, but this is all invisible in disassembly.
Implicit operands are always fixed (i.e., the register is known), but
not all fixed operands are implicit (i.e., some fixed registers _are_
disassembled).

* asm: add `mul*` instructions, vector and scalar

* asm: generate ISLE for multi-return constructors--`AssemblerOutputs::RetValueRegs`

Certain `mul*` instructions write to multiple registers. For register
allocation, Cranelift needs to know about all of these registers. This
change uses the pre-existing pattern of returning a `ValueRegs` type to
indicate this. This change is limited to what is needed now: the only
multi-return needed now uses two fixed registers.

* x64: convert SSE multiplications

* x64: convert scalar multiplications

* x64: remove old `Inst::Mul*` variants

* winch: convert multiplications

This does not include any special "small immediate resizing" rules for
Winch, so the Winch disassembly tests gain a few bytes (e.g., some
immediates that _could_ fit in 8 bits are emitted as the full 32 bits).

* refactor: move `i32`-related extractors to `prelude.isle`

* review: print the implicit registers as well

* fix: the instruction size has changed (again)

Author: abrown

cranelift/assembler-x64/meta/src/dsl.rs

@@ -13,8 +13,8 @@ pub use encoding::{
 };
 pub use encoding::{rex, vex};
 pub use features::{ALL_FEATURES, Feature, Features};
-pub use format::{Extension, Format, Location, Mutability, Operand, OperandKind};
-pub use format::{align, fmt, r, rw, sxl, sxq, sxw, w};
+pub use format::{Extension, Format, Location, Mutability, Operand, OperandKind, RegClass};
+pub use format::{align, fmt, implicit, r, rw, sxl, sxq, sxw, w};
 
 /// Abbreviated constructor for an x64 instruction.
 pub fn inst(

cranelift/assembler-x64/meta/src/dsl/features.rs

@@ -65,6 +65,7 @@ pub enum Feature {
     sse,
     sse2,
     ssse3,
+    sse41,
 }
 
 /// List all CPU features.
@@ -80,6 +81,7 @@ pub const ALL_FEATURES: &[Feature] = &[
     Feature::sse,
     Feature::sse2,
     Feature::ssse3,
+    Feature::sse41,
 ];
 
 impl fmt::Display for Feature {
@@ -90,6 +92,7 @@ impl fmt::Display for Feature {
             Feature::sse => write!(f, "sse"),
             Feature::sse2 => write!(f, "sse2"),
             Feature::ssse3 => write!(f, "ssse3"),
+            Feature::sse41 => write!(f, "sse41"),
         }
     }
 }

cranelift/assembler-x64/meta/src/dsl/format.rs

@@ -34,13 +34,12 @@ pub fn fmt(
 /// This function panics if the location is an immediate (i.e., an immediate
 /// cannot be written to).
 #[must_use]
-pub fn rw(location: Location) -> Operand {
-    assert!(!matches!(location.kind(), OperandKind::Imm(_)));
+pub fn rw(op: impl Into<Operand>) -> Operand {
+    let op = op.into();
+    assert!(!matches!(op.location.kind(), OperandKind::Imm(_)));
     Operand {
-        location,
         mutability: Mutability::ReadWrite,
-        extension: Extension::default(),
-        align: false,
+        ..op
     }
 }
 
@@ -54,23 +53,30 @@ pub fn r(op: impl Into<Operand>) -> Operand {
 
 /// An abbreviated constructor for a "write" operand.
 #[must_use]
-pub fn w(location: Location) -> Operand {
+pub fn w(op: impl Into<Operand>) -> Operand {
+    let op = op.into();
     Operand {
-        location,
         mutability: Mutability::Write,
-        extension: Extension::None,
-        align: false,
+        ..op
     }
 }
 
 /// An abbreviated constructor for a memory operand that requires alignment.
 pub fn align(location: Location) -> Operand {
     assert!(location.uses_memory());
     Operand {
-        location,
-        mutability: Mutability::Read,
-        extension: Extension::None,
         align: true,
+        ..Operand::from(location)
+    }
+}
+
+/// An abbreviated constructor for an operand that is used by the instruction
+/// but not visible in its disassembly.
+pub fn implicit(location: Location) -> Operand {
+    assert!(matches!(location.kind(), OperandKind::FixedReg(_)));
+    Operand {
+        implicit: true,
+        ..Operand::from(location)
     }
 }
 
@@ -84,10 +90,8 @@ pub fn align(location: Location) -> Operand {
 pub fn sxq(location: Location) -> Operand {
     assert!(location.bits() <= 64);
     Operand {
-        location,
-        mutability: Mutability::Read,
         extension: Extension::SignExtendQuad,
-        align: false,
+        ..Operand::from(location)
     }
 }
 
@@ -101,10 +105,8 @@ pub fn sxq(location: Location) -> Operand {
 pub fn sxl(location: Location) -> Operand {
     assert!(location.bits() <= 32);
     Operand {
-        location,
-        mutability: Mutability::Read,
         extension: Extension::SignExtendLong,
-        align: false,
+        ..Operand::from(location)
     }
 }
 
@@ -118,10 +120,8 @@ pub fn sxl(location: Location) -> Operand {
 pub fn sxw(location: Location) -> Operand {
     assert!(location.bits() <= 16);
     Operand {
-        location,
-        mutability: Mutability::Read,
         extension: Extension::SignExtendWord,
-        align: false,
+        ..Operand::from(location)
     }
 }
 
@@ -204,6 +204,9 @@ pub struct Operand {
     /// address used in the operand must align to the size of the operand (e.g.,
     /// `m128` must be 16-byte aligned).
     pub align: bool,
+    /// Some register operands are implicit: that is, they do not appear in the
+    /// disassembled output even though they are used in the instruction.
+    pub implicit: bool,
 }
 
 impl core::fmt::Display for Operand {
@@ -213,6 +216,7 @@ impl core::fmt::Display for Operand {
             mutability,
             extension,
             align,
+            implicit,
         } = self;
         write!(f, "{location}")?;
         let mut flags = vec![];
@@ -225,6 +229,9 @@ impl core::fmt::Display for Operand {
         if *align != false {
             flags.push("align".to_owned());
         }
+        if *implicit {
+            flags.push("implicit".to_owned());
+        }
         if !flags.is_empty() {
             write!(f, "[{}]", flags.join(","))?;
         }
@@ -237,11 +244,13 @@ impl From<Location> for Operand {
         let mutability = Mutability::default();
         let extension = Extension::default();
         let align = false;
+        let implicit = false;
         Self {
             location,
             mutability,
             extension,
             align,
+            implicit,
         }
     }
 }
@@ -270,6 +279,9 @@ pub enum Location {
     ax,
     eax,
     rax,
+    dx,
+    edx,
+    rdx,
     cl,
 
     // Immediate values.
@@ -307,9 +319,9 @@ impl Location {
         use Location::*;
         match self {
             al | cl | imm8 | r8 | rm8 | m8 => 8,
-            ax | imm16 | r16 | rm16 | m16 => 16,
-            eax | imm32 | r32 | rm32 | m32 | xmm_m32 => 32,
-            rax | r64 | rm64 | m64 | xmm_m64 => 64,
+            ax | dx | imm16 | r16 | rm16 | m16 => 16,
+            eax | edx | imm32 | r32 | rm32 | m32 | xmm_m32 => 32,
+            rax | rdx | r64 | rm64 | m64 | xmm_m64 => 64,
             xmm | xmm_m128 => 128,
         }
     }
@@ -325,7 +337,8 @@ impl Location {
     pub fn uses_memory(&self) -> bool {
         use Location::*;
         match self {
-            al | cl | ax | eax | rax | imm8 | imm16 | imm32 | r8 | r16 | r32 | r64 | xmm => false,
+            al | ax | eax | rax | cl | dx | edx | rdx | imm8 | imm16 | imm32 | r8 | r16 | r32
+            | r64 | xmm => false,
             rm8 | rm16 | rm32 | rm64 | xmm_m32 | xmm_m64 | xmm_m128 | m8 | m16 | m32 | m64 => true,
         }
     }
@@ -337,8 +350,8 @@ impl Location {
         use Location::*;
         match self {
             imm8 | imm16 | imm32 => false,
-            al | ax | eax | rax | cl | r8 | r16 | r32 | r64 | rm8 | rm16 | rm32 | rm64 | xmm
-            | xmm_m32 | xmm_m64 | xmm_m128 | m8 | m16 | m32 | m64 => true,
+            al | ax | eax | rax | cl | dx | edx | rdx | r8 | r16 | r32 | r64 | rm8 | rm16
+            | rm32 | rm64 | xmm | xmm_m32 | xmm_m64 | xmm_m128 | m8 | m16 | m32 | m64 => true,
         }
     }
 
@@ -347,7 +360,7 @@ impl Location {
     pub fn kind(&self) -> OperandKind {
         use Location::*;
         match self {
-            al | ax | eax | rax | cl => OperandKind::FixedReg(*self),
+            al | ax | eax | rax | cl | dx | edx | rdx => OperandKind::FixedReg(*self),
             imm8 | imm16 | imm32 => OperandKind::Imm(*self),
             r8 | r16 | r32 | r64 | xmm => OperandKind::Reg(*self),
             rm8 | rm16 | rm32 | rm64 | xmm_m32 | xmm_m64 | xmm_m128 => OperandKind::RegMem(*self),
@@ -364,9 +377,8 @@ impl Location {
         use Location::*;
         match self {
             imm8 | imm16 | imm32 | m8 | m16 | m32 | m64 => None,
-            al | ax | eax | rax | cl | r8 | r16 | r32 | r64 | rm8 | rm16 | rm32 | rm64 => {
-                Some(RegClass::Gpr)
-            }
+            al | ax | eax | rax | cl | dx | edx | rdx | r8 | r16 | r32 | r64 | rm8 | rm16
+            | rm32 | rm64 => Some(RegClass::Gpr),
             xmm | xmm_m32 | xmm_m64 | xmm_m128 => Some(RegClass::Xmm),
         }
     }
@@ -385,6 +397,9 @@ impl core::fmt::Display for Location {
             eax => write!(f, "eax"),
             rax => write!(f, "rax"),
             cl => write!(f, "cl"),
+            dx => write!(f, "dx"),
+            edx => write!(f, "edx"),
+            rdx => write!(f, "rdx"),
 
             r8 => write!(f, "r8"),
             r16 => write!(f, "r16"),

cranelift/assembler-x64/meta/src/generate/format.rs

@@ -14,17 +14,33 @@ impl dsl::Format {
     /// once Cranelift has switched to using this assembler predominantly
     /// (TODO).
     #[must_use]
-    pub fn generate_att_style_operands(&self) -> String {
+    pub(crate) fn generate_att_style_operands(&self) -> String {
         let ordered_ops: Vec<_> = self
             .operands
             .iter()
+            .filter(|o| !o.implicit)
             .rev()
             .map(|o| format!("{{{}}}", o.location))
             .collect();
         ordered_ops.join(", ")
     }
 
-    pub fn generate_rex_encoding(&self, f: &mut Formatter, rex: &dsl::Rex) {
+    #[must_use]
+    pub(crate) fn generate_implicit_operands(&self) -> String {
+        let ops: Vec<_> = self
+            .operands
+            .iter()
+            .filter(|o| o.implicit)
+            .map(|o| format!("{{{}}}", o.location))
+            .collect();
+        if ops.is_empty() {
+            String::new()
+        } else {
+            format!(" ;; implicit: {}", ops.join(", "))
+        }
+    }
+
+    pub(crate) fn generate_rex_encoding(&self, f: &mut Formatter, rex: &dsl::Rex) {
         self.generate_prefixes(f, rex);
         self.generate_rex_prefix(f, rex);
         self.generate_opcodes(f, rex);
@@ -91,12 +107,17 @@ impl dsl::Format {
                 fmtln!(f, "let dst = self.{dst}.enc();");
                 fmtln!(f, "let rex = RexPrefix::two_op(digit, dst, {bits});");
             }
+            [FixedReg(_), RegMem(mem)] | [FixedReg(_), FixedReg(_), RegMem(mem)] => {
+                let digit = rex.digit.unwrap();
+                fmtln!(f, "let digit = 0x{digit:x};");
+                fmtln!(f, "let rex = self.{mem}.as_rex_prefix(digit, {bits});");
+            }
             [Mem(dst), Imm(_)] | [RegMem(dst), Imm(_)] | [RegMem(dst)] => {
                 let digit = rex.digit.unwrap();
                 fmtln!(f, "let digit = 0x{digit:x};");
                 fmtln!(f, "let rex = self.{dst}.as_rex_prefix(digit, {bits});");
             }
-            [Reg(dst), RegMem(src)] => {
+            [Reg(dst), RegMem(src)] | [Reg(dst), RegMem(src), Imm(_)] => {
                 fmtln!(f, "let dst = self.{dst}.enc();");
                 fmtln!(f, "let rex = self.{src}.as_rex_prefix(dst, {bits});");
             }
@@ -135,12 +156,17 @@ impl dsl::Format {
                 fmtln!(f, "let digit = 0x{digit:x};");
                 fmtln!(f, "self.{reg}.encode_modrm(buf, digit);");
             }
-            [Mem(mem), Imm(_)] | [RegMem(mem), Imm(_)] | [RegMem(mem)] => {
+            [Mem(mem), Imm(_)]
+            | [RegMem(mem), Imm(_)]
+            | [RegMem(mem)]
+            | [FixedReg(_), RegMem(mem)]
+            | [FixedReg(_), FixedReg(_), RegMem(mem)] => {
                 let digit = rex.digit.unwrap();
                 fmtln!(f, "let digit = 0x{digit:x};");
                 fmtln!(f, "self.{mem}.encode_rex_suffixes(buf, off, digit, 0);");
             }
             [Reg(reg), RegMem(mem)]
+            | [Reg(reg), RegMem(mem), Imm(_)]
             | [Mem(mem), Reg(reg)]
             | [RegMem(mem), Reg(reg)]
             | [RegMem(mem), Reg(reg), Imm(_)]

cranelift/assembler-x64/meta/src/generate/inst.rs

@@ -194,7 +194,7 @@ impl dsl::Inst {
                 f.add_block(
                     "fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result",
                     |f| {
-                        for op in &self.format.operands {
+                        for op in self.format.operands.iter() {
                             let location = op.location;
                             let to_string = location.generate_to_string(op.extension);
                             fmtln!(f, "let {location} = {to_string};");
@@ -207,7 +207,8 @@ impl dsl::Inst {
                             &self.mnemonic
                         };
                         let ordered_ops = self.format.generate_att_style_operands();
-                        fmtln!(f, "write!(f, \"{inst_name} {ordered_ops}\")");
+                        let implicit_ops = self.format.generate_implicit_operands();
+                        fmtln!(f, "write!(f, \"{inst_name} {ordered_ops}{implicit_ops}\")");
                     },
                 );
             },

cranelift/assembler-x64/meta/src/generate/operand.rs

@@ -14,10 +14,11 @@ impl dsl::Operand {
                     format!("Imm{bits}")
                 }
             }
-            al | ax | eax | rax | cl => {
+            al | ax | eax | rax | cl | dx | edx | rdx => {
                 let enc = match self.location {
                     al | ax | eax | rax => "{ gpr::enc::RAX }",
                     cl => "{ gpr::enc::RCX }",
+                    dx | edx | rdx => "{ gpr::enc::RDX }",
                     _ => unreachable!(),
                 };
                 format!("Fixed<R::{mut_}Gpr, {enc}>")
@@ -44,6 +45,9 @@ impl dsl::Location {
             eax => "\"%eax\"".into(),
             rax => "\"%rax\"".into(),
             cl => "\"%cl\"".into(),
+            dx => "\"%dx\"".into(),
+            edx => "\"%edx\"".into(),
+            rdx => "\"%rdx\"".into(),
             imm8 | imm16 | imm32 => {
                 if extension.is_sign_extended() {
                     let variant = extension.generate_variant();
@@ -67,7 +71,7 @@ impl dsl::Location {
     fn generate_size(&self) -> Option<&str> {
         use dsl::Location::*;
         match self {
-            al | ax | eax | rax | cl | imm8 | imm16 | imm32 => None,
+            al | ax | eax | rax | cl | dx | edx | rdx | imm8 | imm16 | imm32 => None,
             r8 | rm8 => Some("Size::Byte"),
             r16 | rm16 => Some("Size::Word"),
             r32 | rm32 => Some("Size::Doubleword"),

cranelift/assembler-x64/meta/src/instructions.rs

@@ -3,6 +3,7 @@
 mod add;
 mod and;
 mod cvt;
+mod mul;
 mod neg;
 mod or;
 mod shift;
@@ -17,6 +18,7 @@ pub fn list() -> Vec<Inst> {
     all.extend(add::list());
     all.extend(and::list());
     all.extend(cvt::list());
+    all.extend(mul::list());
     all.extend(neg::list());
     all.extend(or::list());
     all.extend(shift::list());