Add support for SVE types and registers in inline assembly

compiler/rustc_abi/src/layout.rs

@@ -231,7 +231,8 @@ pub trait LayoutCalculator {
                         hide_niches(a);
                         hide_niches(b);
                     }
-                    Abi::Vector { element, count: _ } => hide_niches(element),
+                    Abi::Vector { element, .. } => hide_niches(element),
+                    Abi::ScalableVector { element, .. } => hide_niches(element),
                     Abi::Aggregate { sized: _ } => {}
                 }
                 st.largest_niche = None;

compiler/rustc_abi/src/lib.rs

@@ -41,9 +41,11 @@ bitflags! {
         // If true, the type's layout can be randomized using
         // the seed stored in `ReprOptions.layout_seed`
         const RANDOMIZE_LAYOUT   = 1 << 4;
+        const IS_SCALABLE = 1 << 5;
         // Any of these flags being set prevent field reordering optimisation.
         const IS_UNOPTIMISABLE   = ReprFlags::IS_C.bits
                                  | ReprFlags::IS_SIMD.bits
+                                 | ReprFlags::IS_SCALABLE.bits
                                  | ReprFlags::IS_LINEAR.bits;
     }
 }
@@ -76,6 +78,7 @@ pub struct ReprOptions {
     pub align: Option<Align>,
     pub pack: Option<Align>,
     pub flags: ReprFlags,
+    pub scalable: Option<u32>,
     /// The seed to be used for randomizing a type's layout
     ///
     /// Note: This could technically be a `u128` which would
@@ -92,6 +95,11 @@ impl ReprOptions {
         self.flags.contains(ReprFlags::IS_SIMD)
     }
 
+    #[inline]
+    pub fn scalable(&self) -> bool {
+        self.flags.contains(ReprFlags::IS_SCALABLE)
+    }
+
     #[inline]
     pub fn c(&self) -> bool {
         self.flags.contains(ReprFlags::IS_C)
@@ -1243,6 +1251,10 @@ pub enum Abi {
     Uninhabited,
     Scalar(Scalar),
     ScalarPair(Scalar, Scalar),
+    ScalableVector {
+        element: Scalar,
+        elt: u64,
+    },
     Vector {
         element: Scalar,
         count: u64,
@@ -1260,6 +1272,7 @@ impl Abi {
         match *self {
             Abi::Uninhabited | Abi::Scalar(_) | Abi::ScalarPair(..) | Abi::Vector { .. } => false,
             Abi::Aggregate { sized } => !sized,
+            Abi::ScalableVector { .. } => true,
         }
     }
 
@@ -1306,7 +1319,7 @@ impl Abi {
             Abi::Vector { element, count } => {
                 cx.data_layout().vector_align(element.size(cx) * count)
             }
-            Abi::Uninhabited | Abi::Aggregate { .. } => return None,
+            Abi::Uninhabited | Abi::Aggregate { .. } | Abi::ScalableVector { .. } => return None,
         })
     }
 
@@ -1327,7 +1340,7 @@ impl Abi {
                 // to make the size a multiple of align (e.g. for vectors of size 3).
                 (element.size(cx) * count).align_to(self.inherent_align(cx)?.abi)
             }
-            Abi::Uninhabited | Abi::Aggregate { .. } => return None,
+            Abi::Uninhabited | Abi::Aggregate { .. } | Abi::ScalableVector { .. } => return None,
         })
     }
 
@@ -1337,6 +1350,9 @@ impl Abi {
             Abi::Scalar(s) => Abi::Scalar(s.to_union()),
             Abi::ScalarPair(s1, s2) => Abi::ScalarPair(s1.to_union(), s2.to_union()),
             Abi::Vector { element, count } => Abi::Vector { element: element.to_union(), count },
+            Abi::ScalableVector { element, elt } => {
+                Abi::ScalableVector { element: element.to_union(), elt }
+            }
             Abi::Uninhabited | Abi::Aggregate { .. } => Abi::Aggregate { sized: true },
         }
     }
@@ -1620,6 +1636,11 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutS<FieldIdx, VariantIdx> {
         self.is_sized() && self.size.bytes() == 0 && self.align.abi.bytes() == 1
     }
 
+    /// Returns true if the size of the type is only known at runtime.
+    pub fn is_runtime_sized(&self) -> bool {
+        matches!(self.abi, Abi::ScalableVector { .. })
+    }
+
     /// Returns `true` if the type is a ZST and not unsized.
     ///
     /// Note that this does *not* imply that the type is irrelevant for layout! It can still have
@@ -1629,6 +1650,7 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutS<FieldIdx, VariantIdx> {
             Abi::Scalar(_) | Abi::ScalarPair(..) | Abi::Vector { .. } => false,
             Abi::Uninhabited => self.size.bytes() == 0,
             Abi::Aggregate { sized } => sized && self.size.bytes() == 0,
+            Abi::ScalableVector { .. } => false,
         }
     }
 

compiler/rustc_ast_passes/src/feature_gate.rs

@@ -261,6 +261,15 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
                                 "SIMD types are experimental and possibly buggy"
                             );
                         }
+
+                        if item.has_name(sym::scalable) {
+                            gate!(
+                                &self,
+                                repr_scalable,
+                                attr.span,
+                                "Scalable SIMD types are experimental and possibly buggy"
+                            );
+                        }
                     }
                 }
             }

compiler/rustc_attr/messages.ftl

@@ -88,6 +88,9 @@ attr_rustc_allowed_unstable_pairing =
 attr_rustc_promotable_pairing =
     `rustc_promotable` attribute must be paired with either a `rustc_const_unstable` or a `rustc_const_stable` attribute
 
+attr_scalable_missing_n =
+    invalid `scalable(num)` attribute: `scalable` needs an argument
+    .suggestion = supply an argument here
 attr_soft_no_args =
     `soft` should not have any arguments
 

compiler/rustc_attr/src/builtin.rs

@@ -909,6 +909,7 @@ pub enum ReprAttr {
     ReprSimd,
     ReprTransparent,
     ReprAlign(u32),
+    ReprScalable(u32),
 }
 
 #[derive(Eq, PartialEq, Debug, Copy, Clone)]
@@ -964,6 +965,13 @@ pub fn parse_repr_attr(sess: &Session, attr: &Attribute) -> Vec<ReprAttr> {
                         recognised = true;
                         None
                     }
+                    sym::scalable => {
+                        sess.emit_err(session_diagnostics::ScalableAttrMissingN {
+                            span: item.span(),
+                        });
+                        recognised = true;
+                        None
+                    }
                     name => int_type_of_word(name).map(ReprInt),
                 };
 
@@ -985,6 +993,12 @@ pub fn parse_repr_attr(sess: &Session, attr: &Attribute) -> Vec<ReprAttr> {
                         Ok(literal) => acc.push(ReprPacked(literal)),
                         Err(message) => literal_error = Some(message),
                     };
+                } else if name == sym::scalable {
+                    recognised = true;
+                    match parse_scalable(&value.kind) {
+                        Ok(literal) => acc.push(ReprScalable(literal)),
+                        Err(message) => literal_error = Some(message),
+                    };
                 } else if matches!(name, sym::C | sym::simd | sym::transparent)
                     || int_type_of_word(name).is_some()
                 {
@@ -1004,7 +1018,10 @@ pub fn parse_repr_attr(sess: &Session, attr: &Attribute) -> Vec<ReprAttr> {
             } else if let Some(meta_item) = item.meta_item() {
                 match &meta_item.kind {
                     MetaItemKind::NameValue(value) => {
-                        if meta_item.has_name(sym::align) || meta_item.has_name(sym::packed) {
+                        if meta_item.has_name(sym::align)
+                            || meta_item.has_name(sym::packed)
+                            || meta_item.has_name(sym::scalable)
+                        {
                             let name = meta_item.name_or_empty().to_ident_string();
                             recognised = true;
                             sess.emit_err(session_diagnostics::IncorrectReprFormatGeneric {
@@ -1199,3 +1216,11 @@ pub fn parse_confusables(attr: &Attribute) -> Option<Vec<Symbol>> {
 
     return Some(candidates);
 }
+
+pub fn parse_scalable(node: &ast::LitKind) -> Result<u32, &'static str> {
+    if let ast::LitKind::Int(literal, ast::LitIntType::Unsuffixed) = node {
+        (*literal).try_into().map_err(|_| "integer too large")
+    } else {
+        Err("not an unsuffixed integer")
+    }
+}

compiler/rustc_attr/src/session_diagnostics.rs

@@ -390,3 +390,11 @@ pub(crate) struct UnknownVersionLiteral {
     #[primary_span]
     pub span: Span,
 }
+
+#[derive(Diagnostic)]
+#[diag(attr_scalable_missing_n, code = "E0796")]
+pub(crate) struct ScalableAttrMissingN {
+    #[primary_span]
+    #[suggestion(applicability = "has-placeholders", code = "scalable(...)")]
+    pub span: Span,
+}

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -515,7 +515,15 @@ impl<'a, 'b, 'tcx> TypeVerifier<'a, 'b, 'tcx> {
             place_ty = self.sanitize_projection(place_ty, elem, place, location, context);
         }
 
-        if let PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy) = context {
+        // The Copy trait isn't implemented for scalable SIMD types.
+        // These types live somewhere between `Sized` and `Unsize`.
+        // The bounds on `Copy` disallow the trait from being
+        // implemented for them. As a result of this no bounds from
+        // `Copy` apply for the type, therefore, skipping this check
+        // should be perfectly legal.
+        if let PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy) = context
+            && !place_ty.ty.is_scalable_simd()
+        {
             let tcx = self.tcx();
             let trait_ref =
                 ty::TraitRef::from_lang_item(tcx, LangItem::Copy, self.last_span, [place_ty.ty]);
@@ -1763,11 +1771,13 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             // expressions evaluate through `as_temp` or `into` a return
             // slot or local, so to find all unsized rvalues it is enough
             // to check all temps, return slots and locals.
-            if self.reported_errors.replace((ty, span)).is_none() {
-                // While this is located in `nll::typeck` this error is not
-                // an NLL error, it's a required check to prevent creation
-                // of unsized rvalues in a call expression.
-                self.tcx().sess.emit_err(MoveUnsized { ty, span });
+            if !ty.is_scalable_simd() {
+                if self.reported_errors.replace((ty, span)).is_none() {
+                    // While this is located in `nll::typeck` this error is not
+                    // an NLL error, it's a required check to prevent creation
+                    // of unsized rvalues in a call expression.
+                    self.tcx().sess.emit_err(MoveUnsized { ty, span });
+                }
             }
         }
     }

compiler/rustc_codegen_gcc/src/abi.rs

@@ -94,6 +94,7 @@ impl GccType for Reg {
                 }
             },
             RegKind::Vector => unimplemented!(), //cx.type_vector(cx.type_i8(), self.size.bytes()),
+            RegKind::ScalableVector => unimplemented!(),
         }
     }
 }

compiler/rustc_codegen_gcc/src/asm.rs

@@ -571,6 +571,13 @@ fn reg_to_gcc(reg: InlineAsmRegOrRegClass) -> ConstraintOrRegister {
             InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => {
                 unreachable!("clobber-only")
             }
+            InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::zreg)
+            | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::zreg_low16)
+            | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::zreg_low8)
+            | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg_low8)
+            | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::ffr_reg) => {
+                unimplemented()
+            },
             InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => "r",
             InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg)
             | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16)
@@ -647,9 +654,14 @@ fn reg_to_gcc(reg: InlineAsmRegOrRegClass) -> ConstraintOrRegister {
 fn dummy_output_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, reg: InlineAsmRegClass) -> Type<'gcc> {
     match reg {
         InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(),
-        InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => unimplemented!(),
         InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg)
-        | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
+        | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16)
+        | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::zreg)
+        | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::zreg_low16)
+        | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::zreg_low8)
+        | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg)
+        | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg_low8)
+        | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::ffr_reg) => {
             unimplemented!()
         }
         InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg)=> cx.type_i32(),
@@ -791,6 +803,13 @@ fn modifier_to_gcc(arch: InlineAsmArch, reg: InlineAsmRegClass, modifier: Option
         InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => {
             unreachable!("clobber-only")
         }
+        InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::zreg)
+        | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::zreg_low16)
+        | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::zreg_low8)
+        | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg_low8)
+        | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::ffr_reg) => {
+            unsupported();
+        }
         InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => None,
         InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg)
         | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => None,

compiler/rustc_codegen_gcc/src/intrinsic/mod.rs

@@ -274,7 +274,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                     let layout = self.layout_of(tp_ty).layout;
                     let _use_integer_compare = match layout.abi() {
                         Scalar(_) | ScalarPair(_, _) => true,
-                        Uninhabited | Vector { .. } => false,
+                        Uninhabited | Vector { .. } | ScalableVector { .. } => false,
                         Aggregate { .. } => {
                             // For rusty ABIs, small aggregates are actually passed
                             // as `RegKind::Integer` (see `FnAbi::adjust_for_abi`),

compiler/rustc_codegen_gcc/src/type_of.rs

@@ -80,6 +80,7 @@ fn uncached_gcc_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, layout: TyAndLayout
                 false,
             );
         }
+        Abi::ScalableVector { .. } => todo!(),
         Abi::Uninhabited | Abi::Aggregate { .. } => {}
     }
 
@@ -158,15 +159,15 @@ pub trait LayoutGccExt<'tcx> {
 impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
     fn is_gcc_immediate(&self) -> bool {
         match self.abi {
-            Abi::Scalar(_) | Abi::Vector { .. } => true,
+            Abi::Scalar(_) | Abi::Vector { .. } | Abi::ScalableVector { .. } => true,
             Abi::ScalarPair(..) | Abi::Uninhabited | Abi::Aggregate { .. } => false,
         }
     }
 
     fn is_gcc_scalar_pair(&self) -> bool {
         match self.abi {
             Abi::ScalarPair(..) => true,
-            Abi::Uninhabited | Abi::Scalar(_) | Abi::Vector { .. } | Abi::Aggregate { .. } => false,
+            Abi::Uninhabited | Abi::Scalar(_) | Abi::Vector { .. } | Abi::ScalableVector { .. } | Abi::Aggregate { .. } => false,
         }
     }
 

compiler/rustc_codegen_llvm/src/abi.rs

@@ -124,6 +124,7 @@ impl LlvmType for Reg {
                 _ => bug!("unsupported float: {:?}", self),
             },
             RegKind::Vector => cx.type_vector(cx.type_i8(), self.size.bytes()),
+            RegKind::ScalableVector => cx.type_scalable_vector(cx.type_i8(), 16),
         }
     }
 }