[WIP] Bunch of hacks to make vectors smaller than 128bit work with the new Rust ABI

Author: bjorn3

src/abi/pass_mode.rs

@@ -24,7 +24,9 @@ fn reg_to_abi_param(reg: Reg) -> AbiParam {
         (RegKind::Integer, 9..=16) => types::I128,
         (RegKind::Float, 4) => types::F32,
         (RegKind::Float, 8) => types::F64,
-        (RegKind::Vector, size) => types::I8.by(u32::try_from(size).unwrap()).unwrap(),
+        // FIXME(bytecodealliance/wasmtime#10254): Increasing vector size to 128bit to workaround
+        // load and store for smaller vectors not being supported.
+        (RegKind::Vector, size) => types::I8.by(u32::try_from(size.max(128)).unwrap()).unwrap(),
         _ => unreachable!("{:?}", reg),
     };
     AbiParam::new(clif_ty)
@@ -222,7 +224,36 @@ pub(super) fn adjust_arg_for_abi<'tcx>(
     assert_assignable(fx, arg.layout().ty, arg_abi.layout.ty, 16);
     match arg_abi.mode {
         PassMode::Ignore => smallvec![],
-        PassMode::Direct(_) => smallvec![arg.load_scalar(fx)],
+        PassMode::Direct(_) => match arg_abi.layout.backend_repr {
+            BackendRepr::Scalar(_) => smallvec![arg.load_scalar(fx)],
+            BackendRepr::Vector { .. } => {
+                let vector_ty = crate::intrinsics::clif_vector_type(fx.tcx, arg_abi.layout);
+                if u64::from(vector_ty.bytes()) > arg_abi.layout.size.bytes() {
+                    // FIXME(bytecodealliance/wasmtime#10254): In reg_to_abi_param we increased the
+                    // vector size to 128bit to workaround load and store for smaller vectors not
+                    // being supported. As such we need to truncate the vector.
+                    // FIXME(bytecodealliance/wasmtime#6104) do something more efficient for
+                    // transmutes between vectors and integers.
+                    let tmp_ptr = fx.create_stack_slot(vector_ty.bytes(), vector_ty.bytes());
+                    let tmp_ptr_addr = tmp_ptr.get_addr(fx);
+                    let val_ptr = arg.force_stack(fx).0.get_addr(fx);
+                    fx.bcx.emit_small_memory_copy(
+                        fx.target_config,
+                        tmp_ptr_addr,
+                        val_ptr,
+                        arg_abi.layout.size.bytes(),
+                        vector_ty.bytes().try_into().unwrap(),
+                        arg_abi.layout.align.abi.bytes().try_into().unwrap(),
+                        true,
+                        MemFlags::trusted(),
+                    );
+                    smallvec![tmp_ptr.load(fx, vector_ty, MemFlags::trusted())]
+                } else {
+                    smallvec![arg.load_scalar(fx)]
+                }
+            }
+            _ => unreachable!("{:?}", arg_abi.layout.backend_repr),
+        },
         PassMode::Pair(_, _) => {
             let (a, b) = arg.load_scalar_pair(fx);
             smallvec![a, b]

src/abi/returning.rs

@@ -111,9 +111,45 @@ pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, '_>) {
             unreachable!("unsized return value")
         }
         PassMode::Direct(_) => {
-            let place = fx.get_local_place(RETURN_PLACE);
-            let ret_val = place.to_cvalue(fx).load_scalar(fx);
-            fx.bcx.ins().return_(&[ret_val]);
+            match fx.fn_abi.ret.layout.backend_repr {
+                BackendRepr::Scalar(_) => {
+                    let place = fx.get_local_place(RETURN_PLACE);
+                    let ret_val = place.to_cvalue(fx).load_scalar(fx);
+                    fx.bcx.ins().return_(&[ret_val]);
+                }
+                BackendRepr::Vector { .. } => {
+                    let vector_ty =
+                        crate::intrinsics::clif_vector_type(fx.tcx, fx.fn_abi.ret.layout);
+                    if u64::from(vector_ty.bytes()) > fx.fn_abi.ret.layout.size.bytes() {
+                        // FIXME(bytecodealliance/wasmtime#10254): In reg_to_abi_param we increased the
+                        // vector size to 128bit to workaround load and store for smaller vectors not
+                        // being supported. As such we need to truncate the vector.
+                        // FIXME(bytecodealliance/wasmtime#6104) do something more efficient for
+                        // transmutes between vectors and integers.
+                        let tmp_ptr = fx.create_stack_slot(vector_ty.bytes(), vector_ty.bytes());
+                        let tmp_ptr_addr = tmp_ptr.get_addr(fx);
+                        let place = fx.get_local_place(RETURN_PLACE);
+                        let ret_ptr = place.to_cvalue(fx).force_stack(fx).0.get_addr(fx);
+                        fx.bcx.emit_small_memory_copy(
+                            fx.target_config,
+                            tmp_ptr_addr,
+                            ret_ptr,
+                            vector_ty.bytes().try_into().unwrap(),
+                            vector_ty.bytes().try_into().unwrap(),
+                            fx.fn_abi.ret.layout.align.abi.bytes().try_into().unwrap(),
+                            true,
+                            MemFlags::trusted(),
+                        );
+                        let ret_val = tmp_ptr.load(fx, vector_ty, MemFlags::trusted());
+                        fx.bcx.ins().return_(&[ret_val]);
+                    } else {
+                        let place = fx.get_local_place(RETURN_PLACE);
+                        let ret_val = place.to_cvalue(fx).load_scalar(fx);
+                        fx.bcx.ins().return_(&[ret_val]);
+                    }
+                }
+                _ => unreachable!("{:?}", fx.fn_abi.ret.layout.backend_repr),
+            }
         }
         PassMode::Pair(_, _) => {
             let place = fx.get_local_place(RETURN_PLACE);

src/intrinsics/mod.rs

@@ -57,7 +57,10 @@ pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx
         _ => unreachable!(),
     };
 
-    scalar_to_clif_type(tcx, element).by(u32::try_from(count).unwrap()).unwrap()
+    // FIXME(bytecodealliance/wasmtime#10254): Increasing vector size to 128bit to workaround
+    // load and store for smaller vectors not being supported.
+    let scalar = scalar_to_clif_type(tcx, element);
+    scalar.by(u32::try_from(count.max((128 / scalar.bits()).try_into().unwrap())).unwrap()).unwrap()
 }
 
 fn simd_for_each_lane<'tcx>(

src/value_and_place.rs

@@ -173,9 +173,13 @@ impl<'tcx> CValue<'tcx> {
             CValueInner::ByRef(ptr, None) => {
                 let clif_ty = match layout.backend_repr {
                     BackendRepr::Scalar(scalar) => scalar_to_clif_type(fx.tcx, scalar),
-                    BackendRepr::Vector { element, count } => scalar_to_clif_type(fx.tcx, element)
-                        .by(u32::try_from(count).unwrap())
-                        .unwrap(),
+                    BackendRepr::Vector { element, count } => {
+                        let ty = scalar_to_clif_type(fx.tcx, element)
+                            .by(u32::try_from(count).unwrap())
+                            .unwrap();
+                        assert!(ty.bits() >= 128);
+                        ty
+                    }
                     _ => unreachable!("{:?}", layout.ty),
                 };
                 let mut flags = MemFlags::new();