add SSE2 ABI for x86, and use it for float and SIMD passing

Author: RalfJung

compiler/rustc_target/src/callconv/mod.rs

@@ -9,7 +9,7 @@ use crate::abi::{
     self, AddressSpace, Align, BackendRepr, HasDataLayout, Pointer, Size, TyAbiInterface,
     TyAndLayout,
 };
-use crate::spec::{HasTargetSpec, HasWasmCAbiOpt, HasX86AbiOpt, WasmCAbi};
+use crate::spec::{HasTargetSpec, HasWasmCAbiOpt, HasX86AbiOpt, RustcAbi, WasmCAbi};
 
 mod aarch64;
 mod amdgpu;
@@ -748,6 +748,22 @@ impl<'a, Ty> FnAbi<'a, Ty> {
             _ => {}
         };
 
+        // Decides whether we can pass the given SIMD argument via `PassMode::Direct`.
+        // May only return `true` if the target will always pass those arguments the same way,
+        // no matter what the user does with `-Ctarget-feature`! In other words, whatever
+        // target features are required to pass a SIMD value in registers must be listed in
+        // the `abi_required_features` for the current target and ABI.
+        let can_pass_simd_directly = |arg: &ArgAbi<'_, Ty>| match &*spec.arch {
+            // On x86, if we have SSE2 (which we have by default for x86_64), we can always pass up
+            // to 128-bit-sized vectors.
+            "x86" if spec.rustc_abi == Some(RustcAbi::X86Sse2) => arg.layout.size.bits() <= 128,
+            "x86_64" if spec.rustc_abi != Some(RustcAbi::X86Softfloat) => {
+                arg.layout.size.bits() <= 128
+            }
+            // So far, we haven't implemented this logic for any other target.
+            _ => false,
+        };
+
         for (arg_idx, arg) in self
             .args
             .iter_mut()
@@ -847,7 +863,10 @@ impl<'a, Ty> FnAbi<'a, Ty> {
                     //
                     // Note that the intrinsic ABI is exempt here as those are not
                     // real functions anyway, and the backend expects very specific types.
-                    if abi != ExternAbi::RustIntrinsic && spec.simd_types_indirect {
+                    if abi != ExternAbi::RustIntrinsic
+                        && spec.simd_types_indirect
+                        && !can_pass_simd_directly(arg)
+                    {
                         arg.make_indirect();
                     }
                 }

compiler/rustc_target/src/callconv/x86.rs

@@ -2,8 +2,8 @@ use crate::abi::call::{ArgAttribute, FnAbi, PassMode, Reg, RegKind};
 use crate::abi::{
     AddressSpace, Align, BackendRepr, Float, HasDataLayout, Pointer, TyAbiInterface, TyAndLayout,
 };
-use crate::spec::HasTargetSpec;
 use crate::spec::abi::Abi as SpecAbi;
+use crate::spec::{HasTargetSpec, RustcAbi};
 
 #[derive(PartialEq)]
 pub(crate) enum Flavor {
@@ -234,7 +234,13 @@ where
             _ => false, // anyway not passed via registers on x86
         };
         if has_float {
-            if fn_abi.ret.layout.size <= Pointer(AddressSpace::DATA).size(cx) {
+            if cx.target_spec().rustc_abi == Some(RustcAbi::X86Sse2)
+                && fn_abi.ret.layout.backend_repr.is_scalar()
+                && fn_abi.ret.layout.size.bits() <= 128
+            {
+                // This is a single scalar that fits into an SSE register.
+                fn_abi.ret.cast_to(Reg { kind: RegKind::Vector, size: fn_abi.ret.layout.size });
+            } else if fn_abi.ret.layout.size <= Pointer(AddressSpace::DATA).size(cx) {
                 // Same size or smaller than pointer, return in an integer register.
                 fn_abi.ret.cast_to(Reg { kind: RegKind::Integer, size: fn_abi.ret.layout.size });
             } else {

compiler/rustc_target/src/spec/base/apple/mod.rs

@@ -2,8 +2,8 @@ use std::borrow::Cow;
 use std::env;
 
 use crate::spec::{
-    Cc, DebuginfoKind, FloatAbi, FramePointer, LinkerFlavor, Lld, SplitDebuginfo, StackProbeType,
-    StaticCow, TargetOptions, cvs,
+    Cc, DebuginfoKind, FloatAbi, FramePointer, LinkerFlavor, Lld, RustcAbi, SplitDebuginfo,
+    StackProbeType, StaticCow, TargetOptions, cvs,
 };
 
 #[cfg(test)]
@@ -103,7 +103,7 @@ pub(crate) fn base(
     arch: Arch,
     abi: TargetAbi,
 ) -> (TargetOptions, StaticCow<str>, StaticCow<str>) {
-    let opts = TargetOptions {
+    let mut opts = TargetOptions {
         abi: abi.target_abi().into(),
         llvm_floatabi: Some(FloatAbi::Hard),
         os: os.into(),
@@ -154,6 +154,10 @@ pub(crate) fn base(
 
         ..Default::default()
     };
+    if matches!(arch, Arch::I386 | Arch::I686) {
+        // All Apple x86-32 targets have SSE2.
+        opts.rustc_abi = Some(RustcAbi::X86Sse2);
+    }
     (opts, unversioned_llvm_target(os, arch, abi), arch.target_arch())
 }
 

compiler/rustc_target/src/spec/mod.rs

@@ -1117,6 +1117,8 @@ impl ToJson for FloatAbi {
 /// The Rustc-specific variant of the ABI used for this target.
 #[derive(Clone, Copy, PartialEq, Hash, Debug)]
 pub enum RustcAbi {
+    /// On x86-32 only: make use of SSE and SSE2 for ABI purposes.
+    X86Sse2,
     /// On x86-32/64 only: do not use any FPU or SIMD registers for the ABI/
     X86Softfloat,
 }
@@ -1126,6 +1128,7 @@ impl FromStr for RustcAbi {
 
     fn from_str(s: &str) -> Result<RustcAbi, ()> {
         Ok(match s {
+            "x86-sse2" => RustcAbi::X86Sse2,
             "x86-softfloat" => RustcAbi::X86Softfloat,
             _ => return Err(()),
         })
@@ -1135,6 +1138,7 @@ impl FromStr for RustcAbi {
 impl ToJson for RustcAbi {
     fn to_json(&self) -> Json {
         match *self {
+            RustcAbi::X86Sse2 => "x86-sse2",
             RustcAbi::X86Softfloat => "x86-softfloat",
         }
         .to_json()
@@ -3269,6 +3273,11 @@ impl Target {
         // Check consistency of Rust ABI declaration.
         if let Some(rust_abi) = self.rustc_abi {
             match rust_abi {
+                RustcAbi::X86Sse2 => check_matches!(
+                    &*self.arch,
+                    "x86",
+                    "`x86-sse2` ABI is only valid for x86-32 targets"
+                ),
                 RustcAbi::X86Softfloat => check_matches!(
                     &*self.arch,
                     "x86" | "x86_64",

compiler/rustc_target/src/spec/targets/i586_pc_nto_qnx700.rs

@@ -1,5 +1,5 @@
 use crate::spec::base::nto_qnx;
-use crate::spec::{StackProbeType, Target, TargetOptions, base};
+use crate::spec::{RustcAbi, StackProbeType, Target, TargetOptions, base};
 
 pub(crate) fn target() -> Target {
     let mut meta = nto_qnx::meta();
@@ -14,6 +14,7 @@ pub(crate) fn target() -> Target {
             .into(),
         arch: "x86".into(),
         options: TargetOptions {
+            rustc_abi: Some(RustcAbi::X86Sse2),
             cpu: "pentium4".into(),
             max_atomic_width: Some(64),
             pre_link_args: nto_qnx::pre_link_args(

compiler/rustc_target/src/spec/targets/i686_linux_android.rs

@@ -1,4 +1,4 @@
-use crate::spec::{SanitizerSet, StackProbeType, Target, TargetOptions, base};
+use crate::spec::{RustcAbi, SanitizerSet, StackProbeType, Target, TargetOptions, base};
 
 // See https://developer.android.com/ndk/guides/abis.html#x86
 // for target ABI requirements.
@@ -8,6 +8,7 @@ pub(crate) fn target() -> Target {
 
     base.max_atomic_width = Some(64);
 
+    base.rustc_abi = Some(RustcAbi::X86Sse2);
     // https://developer.android.com/ndk/guides/abis.html#x86
     base.cpu = "pentiumpro".into();
     base.features = "+mmx,+sse,+sse2,+sse3,+ssse3".into();

compiler/rustc_target/src/spec/targets/i686_pc_windows_gnu.rs

@@ -1,7 +1,8 @@
-use crate::spec::{Cc, FramePointer, LinkerFlavor, Lld, Target, base};
+use crate::spec::{Cc, FramePointer, LinkerFlavor, Lld, RustcAbi, Target, base};
 
 pub(crate) fn target() -> Target {
     let mut base = base::windows_gnu::opts();
+    base.rustc_abi = Some(RustcAbi::X86Sse2);
     base.cpu = "pentium4".into();
     base.max_atomic_width = Some(64);
     base.frame_pointer = FramePointer::Always; // Required for backtraces

compiler/rustc_target/src/spec/targets/i686_pc_windows_gnullvm.rs

@@ -1,7 +1,8 @@
-use crate::spec::{Cc, FramePointer, LinkerFlavor, Lld, Target, base};
+use crate::spec::{Cc, FramePointer, LinkerFlavor, Lld, RustcAbi, Target, base};
 
 pub(crate) fn target() -> Target {
     let mut base = base::windows_gnullvm::opts();
+    base.rustc_abi = Some(RustcAbi::X86Sse2);
     base.cpu = "pentium4".into();
     base.max_atomic_width = Some(64);
     base.frame_pointer = FramePointer::Always; // Required for backtraces

compiler/rustc_target/src/spec/targets/i686_pc_windows_msvc.rs

@@ -1,7 +1,8 @@
-use crate::spec::{LinkerFlavor, Lld, SanitizerSet, Target, base};
+use crate::spec::{LinkerFlavor, Lld, RustcAbi, SanitizerSet, Target, base};
 
 pub(crate) fn target() -> Target {
     let mut base = base::windows_msvc::opts();
+    base.rustc_abi = Some(RustcAbi::X86Sse2);
     base.cpu = "pentium4".into();
     base.max_atomic_width = Some(64);
     base.supported_sanitizers = SanitizerSet::ADDRESS;

compiler/rustc_target/src/spec/targets/i686_unknown_freebsd.rs

@@ -1,7 +1,8 @@
-use crate::spec::{Cc, LinkerFlavor, Lld, StackProbeType, Target, base};
+use crate::spec::{Cc, LinkerFlavor, Lld, RustcAbi, StackProbeType, Target, base};
 
 pub(crate) fn target() -> Target {
     let mut base = base::freebsd::opts();
+    base.rustc_abi = Some(RustcAbi::X86Sse2);
     base.cpu = "pentium4".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gnu(Cc::Yes, Lld::No), &["-m32", "-Wl,-znotext"]);