cranelift(x64): Swap operands to save AVX instruction encoding size (bytecodealliance#7093)

For `XmmRmiRVex`-format instructions, when the opcode is commutative, the first
operand is one of xmm{0..7}, and the second operand is one of xmm{8..15}, then
we can swap the operands to save a byte on instruction encoding.

Author: fitzgen

cranelift/codegen/src/isa/x64/inst/args.rs

@@ -669,6 +669,12 @@ impl From<RegMem> for RegMemImm {
     }
 }
 
+impl From<Reg> for RegMemImm {
+    fn from(reg: Reg) -> Self {
+        RegMemImm::Reg { reg }
+    }
+}
+
 impl PrettyPrint for RegMemImm {
     fn pretty_print(&self, size: u8, allocs: &mut AllocationConsumer<'_>) -> String {
         match self {
@@ -1761,6 +1767,47 @@ impl AvxOpcode {
             }
         }
     }
+
+    /// Is the opcode known to be commutative?
+    ///
+    /// Note that this method is not exhaustive, and there may be commutative
+    /// opcodes that we don't recognize as commutative.
+    pub(crate) fn is_commutative(&self) -> bool {
+        match *self {
+            AvxOpcode::Vpaddb
+            | AvxOpcode::Vpaddw
+            | AvxOpcode::Vpaddd
+            | AvxOpcode::Vpaddq
+            | AvxOpcode::Vpaddsb
+            | AvxOpcode::Vpaddsw
+            | AvxOpcode::Vpaddusb
+            | AvxOpcode::Vpaddusw
+            | AvxOpcode::Vpand
+            | AvxOpcode::Vandps
+            | AvxOpcode::Vandpd
+            | AvxOpcode::Vpor
+            | AvxOpcode::Vorps
+            | AvxOpcode::Vorpd
+            | AvxOpcode::Vpxor
+            | AvxOpcode::Vxorps
+            | AvxOpcode::Vxorpd
+            | AvxOpcode::Vpmuldq
+            | AvxOpcode::Vpmuludq
+            | AvxOpcode::Vaddps
+            | AvxOpcode::Vaddpd
+            | AvxOpcode::Vmulps
+            | AvxOpcode::Vmulpd
+            | AvxOpcode::Vpcmpeqb
+            | AvxOpcode::Vpcmpeqw
+            | AvxOpcode::Vpcmpeqd
+            | AvxOpcode::Vpcmpeqq
+            | AvxOpcode::Vaddss
+            | AvxOpcode::Vaddsd
+            | AvxOpcode::Vmulss
+            | AvxOpcode::Vmulsd => true,
+            _ => false,
+        }
+    }
 }
 
 impl fmt::Display for AvxOpcode {

cranelift/codegen/src/isa/x64/inst/emit.rs

@@ -2285,6 +2285,20 @@ pub(crate) fn emit(
             let src1 = allocs.next(src1.to_reg());
             let src2 = src2.clone().to_reg_mem_imm().with_allocs(allocs);
 
+            // When the opcode is commutative, src1 is xmm{0..7}, and src2 is
+            // xmm{8..15}, then we can swap the operands to save one byte on the
+            // instruction's encoding.
+            let (src1, src2) = match (src1, src2) {
+                (src1, RegMemImm::Reg { reg: src2 })
+                    if op.is_commutative()
+                        && src1.to_real_reg().unwrap().hw_enc() < 8
+                        && src2.to_real_reg().unwrap().hw_enc() >= 8 =>
+                {
+                    (src2, RegMemImm::Reg { reg: src1 })
+                }
+                (src1, src2) => (src1, src2),
+            };
+
             let src2 = match src2 {
                 // For opcodes where one of the operands is an immediate the
                 // encoding is a bit different, notably the usage of
@@ -2319,6 +2333,7 @@ pub(crate) fn emit(
                 }
                 RegMemImm::Mem { addr } => RegisterOrAmode::Amode(addr.finalize(state, sink)),
             };
+
             let (prefix, map, opcode) = match op {
                 AvxOpcode::Vminps => (LP::None, OM::_0F, 0x5D),
                 AvxOpcode::Vminpd => (LP::_66, OM::_0F, 0x5D),

cranelift/codegen/src/isa/x64/inst/emit_tests.rs

@@ -13,7 +13,7 @@
 //!   -- isa::x64::inst::emit_tests::test_x64_emit
 
 use super::*;
-use crate::ir::UserExternalNameRef;
+use crate::ir::{MemFlags, UserExternalNameRef};
 use crate::isa::x64;
 use alloc::boxed::Box;
 use alloc::vec::Vec;
@@ -5159,6 +5159,77 @@ fn test_x64_emit() {
         "roundpd $0, %xmm15, %xmm15",
     ));
 
+    // ========================================================
+    // XmmRmiRVex
+
+    // Standard instruction w/ XmmMemImm::Reg operand.
+    insns.push((
+        Inst::XmmRmiRVex {
+            op: AvxOpcode::Vpmaxub,
+            dst: Writable::from_reg(Xmm::new(xmm13).unwrap()),
+            src1: Xmm::new(xmm1).unwrap(),
+            src2: XmmMemImm::new(xmm12.into()).unwrap(),
+        },
+        "C44171DEEC",
+        "vpmaxub %xmm1, %xmm12, %xmm13",
+    ));
+
+    // Standard instruction w/ XmmMemImm::Mem operand.
+    insns.push((
+        Inst::XmmRmiRVex {
+            op: AvxOpcode::Vpmaxub,
+            dst: Writable::from_reg(Xmm::new(xmm13).unwrap()),
+            src1: Xmm::new(xmm1).unwrap(),
+            src2: XmmMemImm::new(RegMemImm::Mem {
+                addr: Amode::ImmReg {
+                    simm32: 10,
+                    base: rax,
+                    flags: MemFlags::trusted(),
+                }
+                .into(),
+            })
+            .unwrap(),
+        },
+        "C571DE680A",
+        "vpmaxub %xmm1, 10(%rax), %xmm13",
+    ));
+
+    // When there's an immediate.
+    insns.push((
+        Inst::XmmRmiRVex {
+            op: AvxOpcode::Vpsrlw,
+            dst: Writable::from_reg(Xmm::new(xmm13).unwrap()),
+            src1: Xmm::new(xmm1).unwrap(),
+            src2: XmmMemImm::new(RegMemImm::Imm { simm32: 36 }).unwrap(),
+        },
+        "C59171D124",
+        "vpsrlw  %xmm1, $36, %xmm13",
+    ));
+
+    // Certain commutative ops get their operands swapped to avoid relying on an
+    // extra prefix byte, when possible. Note that these two instructions encode
+    // to the same bytes, and are 4-byte encodings rather than 5-byte encodings.
+    insns.push((
+        Inst::XmmRmiRVex {
+            op: AvxOpcode::Vmulsd,
+            dst: Writable::from_reg(Xmm::new(xmm13).unwrap()),
+            src1: Xmm::new(xmm1).unwrap(),
+            src2: XmmMemImm::new(xmm12.into()).unwrap(),
+        },
+        "C51B59E9",
+        "vmulsd  %xmm1, %xmm12, %xmm13",
+    ));
+    insns.push((
+        Inst::XmmRmiRVex {
+            op: AvxOpcode::Vmulsd,
+            dst: Writable::from_reg(Xmm::new(xmm13).unwrap()),
+            src1: Xmm::new(xmm12).unwrap(),
+            src2: XmmMemImm::new(xmm1.into()).unwrap(),
+        },
+        "C51B59E9",
+        "vmulsd  %xmm12, %xmm1, %xmm13",
+    ));
+
     // ========================================================
     // XmmRmRImmVex
     insns.push((