diff --git a/Cargo.lock b/Cargo.lock index e67898756fad5..973214a802e6e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4805,7 +4805,6 @@ name = "rustdoc-gui-test" version = "0.1.0" dependencies = [ "build_helper", - "camino", "compiletest", "getopts", "walkdir", diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs index 6dc6d1026f621..e1231312a2afd 100644 --- a/compiler/rustc_ast/src/token.rs +++ b/compiler/rustc_ast/src/token.rs @@ -881,11 +881,11 @@ impl Token { } pub fn is_qpath_start(&self) -> bool { - self == &Lt || self == &Shl + matches!(self.kind, Lt | Shl) } pub fn is_path_start(&self) -> bool { - self == &PathSep + self.kind == PathSep || self.is_qpath_start() || matches!(self.is_metavar_seq(), Some(MetaVarKind::Path)) || self.is_path_segment_keyword() diff --git a/compiler/rustc_attr_parsing/src/attributes/codegen_attrs.rs b/compiler/rustc_attr_parsing/src/attributes/codegen_attrs.rs index 7978bf28214cf..f09b02251e4d0 100644 --- a/compiler/rustc_attr_parsing/src/attributes/codegen_attrs.rs +++ b/compiler/rustc_attr_parsing/src/attributes/codegen_attrs.rs @@ -6,6 +6,7 @@ use crate::session_diagnostics::{ NakedFunctionIncompatibleAttribute, NullOnExport, NullOnObjcClass, NullOnObjcSelector, ObjcClassExpectedStringLiteral, ObjcSelectorExpectedStringLiteral, }; +use crate::target_checking::Policy::AllowSilent; pub(crate) struct OptimizeParser; @@ -362,6 +363,8 @@ impl NoArgsAttributeParser for NoMangleParser { Allow(Target::Static), Allow(Target::Method(MethodKind::Inherent)), Allow(Target::Method(MethodKind::TraitImpl)), + AllowSilent(Target::Const), // Handled in the `InvalidNoMangleItems` pass + Error(Target::Closure), ]); const CREATE: fn(Span) -> AttributeKind = AttributeKind::NoMangle; } diff --git a/compiler/rustc_attr_parsing/src/target_checking.rs b/compiler/rustc_attr_parsing/src/target_checking.rs index d28f43943ba84..fabd364d3d7f8 100644 --- a/compiler/rustc_attr_parsing/src/target_checking.rs +++ b/compiler/rustc_attr_parsing/src/target_checking.rs @@ -31,7 +31,9 @@ impl AllowedTargets { pub(crate) fn is_allowed(&self, target: Target) -> AllowedResult { match self { AllowedTargets::AllowList(list) => { - if list.contains(&Policy::Allow(target)) { + if list.contains(&Policy::Allow(target)) + || list.contains(&Policy::AllowSilent(target)) + { AllowedResult::Allowed } else if list.contains(&Policy::Warn(target)) { AllowedResult::Warn @@ -40,7 +42,9 @@ impl AllowedTargets { } } AllowedTargets::AllowListWarnRest(list) => { - if list.contains(&Policy::Allow(target)) { + if list.contains(&Policy::Allow(target)) + || list.contains(&Policy::AllowSilent(target)) + { AllowedResult::Allowed } else if list.contains(&Policy::Error(target)) { AllowedResult::Error @@ -61,6 +65,7 @@ impl AllowedTargets { .iter() .filter_map(|target| match target { Policy::Allow(target) => Some(*target), + Policy::AllowSilent(_) => None, // Not listed in possible targets Policy::Warn(_) => None, Policy::Error(_) => None, }) @@ -68,10 +73,18 @@ impl AllowedTargets { } } +/// This policy determines what diagnostics should be emitted based on the `Target` of the attribute. #[derive(Debug, Eq, PartialEq)] pub(crate) enum Policy { + /// A target that is allowed. Allow(Target), + /// A target that is allowed and not listed in the possible targets. + /// This is useful if the target is checked elsewhere. + AllowSilent(Target), + /// Emits a FCW on this target. + /// This is useful if the target was previously allowed but should not be. Warn(Target), + /// Emits an error on this target. Error(Target), } diff --git a/compiler/rustc_builtin_macros/src/asm.rs b/compiler/rustc_builtin_macros/src/asm.rs index ae62b5ea2a097..971b68d14757a 100644 --- a/compiler/rustc_builtin_macros/src/asm.rs +++ b/compiler/rustc_builtin_macros/src/asm.rs @@ -368,7 +368,7 @@ fn expand_preparsed_asm( if args.options.contains(ast::InlineAsmOptions::RAW) { template.push(ast::InlineAsmTemplatePiece::String(template_str.to_string().into())); let template_num_lines = 1 + template_str.matches('\n').count(); - line_spans.extend(std::iter::repeat(template_sp).take(template_num_lines)); + line_spans.extend(std::iter::repeat_n(template_sp, template_num_lines)); continue; } @@ -523,7 +523,7 @@ fn expand_preparsed_asm( if parser.line_spans.is_empty() { let template_num_lines = 1 + template_str.matches('\n').count(); - line_spans.extend(std::iter::repeat(template_sp).take(template_num_lines)); + line_spans.extend(std::iter::repeat_n(template_sp, template_num_lines)); } else { line_spans.extend( parser diff --git a/compiler/rustc_builtin_macros/src/format.rs b/compiler/rustc_builtin_macros/src/format.rs index bffc0407e8112..a0ee7ac19899b 100644 --- a/compiler/rustc_builtin_macros/src/format.rs +++ b/compiler/rustc_builtin_macros/src/format.rs @@ -69,35 +69,26 @@ struct MacroInput { /// Ok((fmtstr, parsed arguments)) /// ``` fn parse_args<'a>(ecx: &ExtCtxt<'a>, sp: Span, tts: TokenStream) -> PResult<'a, MacroInput> { - let mut args = FormatArguments::new(); - let mut p = ecx.new_parser_from_tts(tts); - if p.token == token::Eof { - return Err(ecx.dcx().create_err(errors::FormatRequiresString { span: sp })); - } - - let first_token = &p.token; - - let fmtstr = if let token::Literal(lit) = first_token.kind - && matches!(lit.kind, token::Str | token::StrRaw(_)) - { + // parse the format string + let fmtstr = match p.token.kind { + token::Eof => return Err(ecx.dcx().create_err(errors::FormatRequiresString { span: sp })), // This allows us to properly handle cases when the first comma // after the format string is mistakenly replaced with any operator, // which cause the expression parser to eat too much tokens. - p.parse_literal_maybe_minus()? - } else { + token::Literal(token::Lit { kind: token::Str | token::StrRaw(_), .. }) => { + p.parse_literal_maybe_minus()? + } // Otherwise, we fall back to the expression parser. - p.parse_expr()? + _ => p.parse_expr()?, }; - // Only allow implicit captures to be used when the argument is a direct literal - // instead of a macro expanding to one. - let is_direct_literal = matches!(fmtstr.kind, ExprKind::Lit(_)); - + // parse comma FormatArgument pairs + let mut args = FormatArguments::new(); let mut first = true; - while p.token != token::Eof { + // parse a comma, or else report an error if !p.eat(exp!(Comma)) { if first { p.clear_expected_token_types(); @@ -120,9 +111,11 @@ fn parse_args<'a>(ecx: &ExtCtxt<'a>, sp: Span, tts: TokenStream) -> PResult<'a, } } first = false; + // accept a trailing comma if p.token == token::Eof { break; - } // accept trailing commas + } + // parse a FormatArgument match p.token.ident() { Some((ident, _)) if p.look_ahead(1, |t| *t == token::Eq) => { p.bump(); @@ -156,6 +149,10 @@ fn parse_args<'a>(ecx: &ExtCtxt<'a>, sp: Span, tts: TokenStream) -> PResult<'a, } } } + + // Only allow implicit captures for direct literals + let is_direct_literal = matches!(fmtstr.kind, ExprKind::Lit(_)); + Ok(MacroInput { fmtstr, args, is_direct_literal }) } diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs index a56466750e75c..faca92957e1a6 100644 --- a/compiler/rustc_codegen_cranelift/src/constant.rs +++ b/compiler/rustc_codegen_cranelift/src/constant.rs @@ -318,7 +318,7 @@ fn data_id_for_static( let mut data = DataDescription::new(); data.set_align(align); let data_gv = module.declare_data_in_data(data_id, &mut data); - data.define(std::iter::repeat(0).take(pointer_ty(tcx).bytes() as usize).collect()); + data.define(std::iter::repeat_n(0, pointer_ty(tcx).bytes() as usize).collect()); data.write_data_addr(0, data_gv, 0); match module.define_data(ref_data_id, &data) { // Every time the static is referenced there will be another definition of this global, diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index e14f0098dd9b4..9844c9444b3d0 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -240,11 +240,13 @@ impl<'ll> CodegenCx<'ll, '_> { let gv = self.define_global(&name, self.val_ty(cv)).unwrap_or_else(|| { bug!("symbol `{}` is already defined", name); }); - llvm::set_linkage(gv, llvm::Linkage::PrivateLinkage); gv } - _ => self.define_private_global(self.val_ty(cv)), + _ => self.define_global("", self.val_ty(cv)).unwrap_or_else(|| { + bug!("anonymous global symbol is already defined"); + }), }; + llvm::set_linkage(gv, llvm::Linkage::PrivateLinkage); llvm::set_initializer(gv, cv); set_global_alignment(self, gv, align); llvm::set_unnamed_address(gv, llvm::UnnamedAddr::Global); diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index 48b2d09afb674..8f69f176138cf 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -230,13 +230,6 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { } } - /// Declare a private global - /// - /// Use this function when you intend to define a global without a name. - pub(crate) fn define_private_global(&self, ty: &'ll Type) -> &'ll Value { - unsafe { llvm::LLVMRustInsertPrivateGlobal(self.llmod(), ty) } - } - /// Gets declared value by name. pub(crate) fn get_declared_value(&self, name: &str) -> Option<&'ll Value> { debug!("get_declared_value(name={:?})", name); diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 7e62630f3f98f..2661b0c4d15b4 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1955,7 +1955,6 @@ unsafe extern "C" { NameLen: size_t, T: &'a Type, ) -> &'a Value; - pub(crate) fn LLVMRustInsertPrivateGlobal<'a>(M: &'a Module, T: &'a Type) -> &'a Value; pub(crate) fn LLVMRustGetNamedValue( M: &Module, Name: *const c_char, diff --git a/compiler/rustc_codegen_ssa/messages.ftl b/compiler/rustc_codegen_ssa/messages.ftl index 97ff04f66daec..e321b0773ec39 100644 --- a/compiler/rustc_codegen_ssa/messages.ftl +++ b/compiler/rustc_codegen_ssa/messages.ftl @@ -223,8 +223,6 @@ codegen_ssa_multiple_main_functions = entry symbol `main` declared multiple time codegen_ssa_no_field = no field `{$name}` -codegen_ssa_no_mangle_nameless = `#[no_mangle]` cannot be used on {$definition} as it has no name - codegen_ssa_no_module_named = no module named `{$user_path}` (mangled: {$cgu_name}). available modules: {$cgu_names} diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs index 2c7643e46ceaa..e8c8729f597b9 100644 --- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs +++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs @@ -19,7 +19,6 @@ use rustc_span::{Ident, Span, sym}; use rustc_target::spec::SanitizerSet; use crate::errors; -use crate::errors::NoMangleNameless; use crate::target_features::{ check_target_feature_trait_unsafe, check_tied_features, from_target_feature_attr, }; @@ -182,14 +181,10 @@ fn process_builtin_attrs( if tcx.opt_item_name(did.to_def_id()).is_some() { codegen_fn_attrs.flags |= CodegenFnAttrFlags::NO_MANGLE; } else { - tcx.dcx().emit_err(NoMangleNameless { - span: *attr_span, - definition: format!( - "{} {}", - tcx.def_descr_article(did.to_def_id()), - tcx.def_descr(did.to_def_id()) - ), - }); + tcx.dcx().span_delayed_bug( + *attr_span, + "no_mangle should be on a named function", + ); } } AttributeKind::Optimize(optimize, _) => codegen_fn_attrs.optimize = *optimize, diff --git a/compiler/rustc_codegen_ssa/src/errors.rs b/compiler/rustc_codegen_ssa/src/errors.rs index d5c30c5c7a6b0..2dd7c6fa7c080 100644 --- a/compiler/rustc_codegen_ssa/src/errors.rs +++ b/compiler/rustc_codegen_ssa/src/errors.rs @@ -1284,14 +1284,6 @@ impl Diagnostic<'_, G> for TargetFeatureDisableOrEnable<'_ } } -#[derive(Diagnostic)] -#[diag(codegen_ssa_no_mangle_nameless)] -pub(crate) struct NoMangleNameless { - #[primary_span] - pub span: Span, - pub definition: String, -} - #[derive(Diagnostic)] #[diag(codegen_ssa_feature_not_valid)] pub(crate) struct FeatureNotValid<'a> { diff --git a/compiler/rustc_const_eval/src/const_eval/error.rs b/compiler/rustc_const_eval/src/const_eval/error.rs index e00fb2c1eaf98..69a8f163ca93c 100644 --- a/compiler/rustc_const_eval/src/const_eval/error.rs +++ b/compiler/rustc_const_eval/src/const_eval/error.rs @@ -110,7 +110,7 @@ pub fn get_span_and_frames<'tcx>( if frame.times < 3 { let times = frame.times; frame.times = 0; - frames.extend(std::iter::repeat(frame).take(times as usize)); + frames.extend(std::iter::repeat_n(frame, times as usize)); } else { frames.push(frame); } diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs index 630e99ad4c74e..fc7f1166af99a 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs @@ -2,6 +2,8 @@ //! looking at their MIR. Intrinsics/functions supported here are shared by CTFE //! and miri. +mod simd; + use std::assert_matches::assert_matches; use rustc_abi::{FieldIdx, HasDataLayout, Size}; @@ -9,8 +11,8 @@ use rustc_apfloat::ieee::{Double, Half, Quad, Single}; use rustc_middle::mir::interpret::{CTFE_ALLOC_SALT, read_target_uint, write_target_uint}; use rustc_middle::mir::{self, BinOp, ConstValue, NonDivergingIntrinsic}; use rustc_middle::ty::layout::TyAndLayout; -use rustc_middle::ty::{Ty, TyCtxt}; -use rustc_middle::{bug, ty}; +use rustc_middle::ty::{FloatTy, Ty, TyCtxt}; +use rustc_middle::{bug, span_bug, ty}; use rustc_span::{Symbol, sym}; use tracing::trace; @@ -121,6 +123,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { ) -> InterpResult<'tcx, bool> { let instance_args = instance.args; let intrinsic_name = self.tcx.item_name(instance.def_id()); + + if intrinsic_name.as_str().starts_with("simd_") { + return self.eval_simd_intrinsic(intrinsic_name, instance_args, args, dest, ret); + } + let tcx = self.tcx.tcx; match intrinsic_name { @@ -454,37 +461,6 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { self.exact_div(&val, &size, dest)?; } - sym::simd_insert => { - let index = u64::from(self.read_scalar(&args[1])?.to_u32()?); - let elem = &args[2]; - let (input, input_len) = self.project_to_simd(&args[0])?; - let (dest, dest_len) = self.project_to_simd(dest)?; - assert_eq!(input_len, dest_len, "Return vector length must match input length"); - // Bounds are not checked by typeck so we have to do it ourselves. - if index >= input_len { - throw_ub_format!( - "`simd_insert` index {index} is out-of-bounds of vector with length {input_len}" - ); - } - - for i in 0..dest_len { - let place = self.project_index(&dest, i)?; - let value = - if i == index { elem.clone() } else { self.project_index(&input, i)? }; - self.copy_op(&value, &place)?; - } - } - sym::simd_extract => { - let index = u64::from(self.read_scalar(&args[1])?.to_u32()?); - let (input, input_len) = self.project_to_simd(&args[0])?; - // Bounds are not checked by typeck so we have to do it ourselves. - if index >= input_len { - throw_ub_format!( - "`simd_extract` index {index} is out-of-bounds of vector with length {input_len}" - ); - } - self.copy_op(&self.project_index(&input, index)?, dest)?; - } sym::black_box => { // These just return their argument self.copy_op(&args[0], dest)?; @@ -1081,4 +1057,66 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { self.write_scalar(res, dest)?; interp_ok(()) } + + /// Converts `src` from floating point to integer type `dest_ty` + /// after rounding with mode `round`. + /// Returns `None` if `f` is NaN or out of range. + pub fn float_to_int_checked( + &self, + src: &ImmTy<'tcx, M::Provenance>, + cast_to: TyAndLayout<'tcx>, + round: rustc_apfloat::Round, + ) -> InterpResult<'tcx, Option>> { + fn float_to_int_inner<'tcx, F: rustc_apfloat::Float, M: Machine<'tcx>>( + ecx: &InterpCx<'tcx, M>, + src: F, + cast_to: TyAndLayout<'tcx>, + round: rustc_apfloat::Round, + ) -> (Scalar, rustc_apfloat::Status) { + let int_size = cast_to.layout.size; + match cast_to.ty.kind() { + // Unsigned + ty::Uint(_) => { + let res = src.to_u128_r(int_size.bits_usize(), round, &mut false); + (Scalar::from_uint(res.value, int_size), res.status) + } + // Signed + ty::Int(_) => { + let res = src.to_i128_r(int_size.bits_usize(), round, &mut false); + (Scalar::from_int(res.value, int_size), res.status) + } + // Nothing else + _ => span_bug!( + ecx.cur_span(), + "attempted float-to-int conversion with non-int output type {}", + cast_to.ty, + ), + } + } + + let ty::Float(fty) = src.layout.ty.kind() else { + bug!("float_to_int_checked: non-float input type {}", src.layout.ty) + }; + + let (val, status) = match fty { + FloatTy::F16 => float_to_int_inner(self, src.to_scalar().to_f16()?, cast_to, round), + FloatTy::F32 => float_to_int_inner(self, src.to_scalar().to_f32()?, cast_to, round), + FloatTy::F64 => float_to_int_inner(self, src.to_scalar().to_f64()?, cast_to, round), + FloatTy::F128 => float_to_int_inner(self, src.to_scalar().to_f128()?, cast_to, round), + }; + + if status.intersects( + rustc_apfloat::Status::INVALID_OP + | rustc_apfloat::Status::OVERFLOW + | rustc_apfloat::Status::UNDERFLOW, + ) { + // Floating point value is NaN (flagged with INVALID_OP) or outside the range + // of values of the integer type (flagged with OVERFLOW or UNDERFLOW). + interp_ok(None) + } else { + // Floating point value can be represented by the integer type after rounding. + // The INEXACT flag is ignored on purpose to allow rounding. + interp_ok(Some(ImmTy::from_scalar(val, cast_to))) + } + } } diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics/simd.rs b/compiler/rustc_const_eval/src/interpret/intrinsics/simd.rs new file mode 100644 index 0000000000000..0dba66ae93721 --- /dev/null +++ b/compiler/rustc_const_eval/src/interpret/intrinsics/simd.rs @@ -0,0 +1,782 @@ +use either::Either; +use rustc_abi::Endian; +use rustc_apfloat::{Float, Round}; +use rustc_middle::mir::interpret::{InterpErrorKind, UndefinedBehaviorInfo}; +use rustc_middle::ty::FloatTy; +use rustc_middle::{bug, err_ub_format, mir, span_bug, throw_unsup_format, ty}; +use rustc_span::{Symbol, sym}; +use tracing::trace; + +use super::{ + ImmTy, InterpCx, InterpResult, Machine, OpTy, PlaceTy, Provenance, Scalar, Size, interp_ok, + throw_ub_format, +}; +use crate::interpret::Writeable; + +#[derive(Copy, Clone)] +pub(crate) enum MinMax { + Min, + Max, +} + +impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { + /// Returns `true` if emulation happened. + /// Here we implement the intrinsics that are common to all CTFE instances; individual machines can add their own + /// intrinsic handling. + pub fn eval_simd_intrinsic( + &mut self, + intrinsic_name: Symbol, + generic_args: ty::GenericArgsRef<'tcx>, + args: &[OpTy<'tcx, M::Provenance>], + dest: &PlaceTy<'tcx, M::Provenance>, + ret: Option, + ) -> InterpResult<'tcx, bool> { + let dest = dest.force_mplace(self)?; + + match intrinsic_name { + sym::simd_insert => { + let index = u64::from(self.read_scalar(&args[1])?.to_u32()?); + let elem = &args[2]; + let (input, input_len) = self.project_to_simd(&args[0])?; + let (dest, dest_len) = self.project_to_simd(&dest)?; + assert_eq!(input_len, dest_len, "Return vector length must match input length"); + // Bounds are not checked by typeck so we have to do it ourselves. + if index >= input_len { + throw_ub_format!( + "`simd_insert` index {index} is out-of-bounds of vector with length {input_len}" + ); + } + + for i in 0..dest_len { + let place = self.project_index(&dest, i)?; + let value = + if i == index { elem.clone() } else { self.project_index(&input, i)? }; + self.copy_op(&value, &place)?; + } + } + sym::simd_extract => { + let index = u64::from(self.read_scalar(&args[1])?.to_u32()?); + let (input, input_len) = self.project_to_simd(&args[0])?; + // Bounds are not checked by typeck so we have to do it ourselves. + if index >= input_len { + throw_ub_format!( + "`simd_extract` index {index} is out-of-bounds of vector with length {input_len}" + ); + } + self.copy_op(&self.project_index(&input, index)?, &dest)?; + } + sym::simd_neg + | sym::simd_fabs + | sym::simd_ceil + | sym::simd_floor + | sym::simd_round + | sym::simd_round_ties_even + | sym::simd_trunc + | sym::simd_ctlz + | sym::simd_ctpop + | sym::simd_cttz + | sym::simd_bswap + | sym::simd_bitreverse => { + let (op, op_len) = self.project_to_simd(&args[0])?; + let (dest, dest_len) = self.project_to_simd(&dest)?; + + assert_eq!(dest_len, op_len); + + #[derive(Copy, Clone)] + enum Op { + MirOp(mir::UnOp), + Abs, + Round(rustc_apfloat::Round), + Numeric(Symbol), + } + let which = match intrinsic_name { + sym::simd_neg => Op::MirOp(mir::UnOp::Neg), + sym::simd_fabs => Op::Abs, + sym::simd_ceil => Op::Round(rustc_apfloat::Round::TowardPositive), + sym::simd_floor => Op::Round(rustc_apfloat::Round::TowardNegative), + sym::simd_round => Op::Round(rustc_apfloat::Round::NearestTiesToAway), + sym::simd_round_ties_even => Op::Round(rustc_apfloat::Round::NearestTiesToEven), + sym::simd_trunc => Op::Round(rustc_apfloat::Round::TowardZero), + sym::simd_ctlz => Op::Numeric(sym::ctlz), + sym::simd_ctpop => Op::Numeric(sym::ctpop), + sym::simd_cttz => Op::Numeric(sym::cttz), + sym::simd_bswap => Op::Numeric(sym::bswap), + sym::simd_bitreverse => Op::Numeric(sym::bitreverse), + _ => unreachable!(), + }; + + for i in 0..dest_len { + let op = self.read_immediate(&self.project_index(&op, i)?)?; + let dest = self.project_index(&dest, i)?; + let val = match which { + Op::MirOp(mir_op) => { + // this already does NaN adjustments + self.unary_op(mir_op, &op)?.to_scalar() + } + Op::Abs => { + // Works for f32 and f64. + let ty::Float(float_ty) = op.layout.ty.kind() else { + span_bug!( + self.cur_span(), + "{} operand is not a float", + intrinsic_name + ) + }; + let op = op.to_scalar(); + // "Bitwise" operation, no NaN adjustments + match float_ty { + FloatTy::F16 => unimplemented!("f16_f128"), + FloatTy::F32 => Scalar::from_f32(op.to_f32()?.abs()), + FloatTy::F64 => Scalar::from_f64(op.to_f64()?.abs()), + FloatTy::F128 => unimplemented!("f16_f128"), + } + } + Op::Round(rounding) => { + let ty::Float(float_ty) = op.layout.ty.kind() else { + span_bug!( + self.cur_span(), + "{} operand is not a float", + intrinsic_name + ) + }; + match float_ty { + FloatTy::F16 => unimplemented!("f16_f128"), + FloatTy::F32 => { + let f = op.to_scalar().to_f32()?; + let res = f.round_to_integral(rounding).value; + let res = self.adjust_nan(res, &[f]); + Scalar::from_f32(res) + } + FloatTy::F64 => { + let f = op.to_scalar().to_f64()?; + let res = f.round_to_integral(rounding).value; + let res = self.adjust_nan(res, &[f]); + Scalar::from_f64(res) + } + FloatTy::F128 => unimplemented!("f16_f128"), + } + } + Op::Numeric(name) => { + self.numeric_intrinsic(name, op.to_scalar(), op.layout, op.layout)? + } + }; + self.write_scalar(val, &dest)?; + } + } + sym::simd_add + | sym::simd_sub + | sym::simd_mul + | sym::simd_div + | sym::simd_rem + | sym::simd_shl + | sym::simd_shr + | sym::simd_and + | sym::simd_or + | sym::simd_xor + | sym::simd_eq + | sym::simd_ne + | sym::simd_lt + | sym::simd_le + | sym::simd_gt + | sym::simd_ge + | sym::simd_fmax + | sym::simd_fmin + | sym::simd_saturating_add + | sym::simd_saturating_sub + | sym::simd_arith_offset => { + use mir::BinOp; + + let (left, left_len) = self.project_to_simd(&args[0])?; + let (right, right_len) = self.project_to_simd(&args[1])?; + let (dest, dest_len) = self.project_to_simd(&dest)?; + + assert_eq!(dest_len, left_len); + assert_eq!(dest_len, right_len); + + enum Op { + MirOp(BinOp), + SaturatingOp(BinOp), + FMinMax(MinMax), + WrappingOffset, + } + let which = match intrinsic_name { + sym::simd_add => Op::MirOp(BinOp::Add), + sym::simd_sub => Op::MirOp(BinOp::Sub), + sym::simd_mul => Op::MirOp(BinOp::Mul), + sym::simd_div => Op::MirOp(BinOp::Div), + sym::simd_rem => Op::MirOp(BinOp::Rem), + sym::simd_shl => Op::MirOp(BinOp::ShlUnchecked), + sym::simd_shr => Op::MirOp(BinOp::ShrUnchecked), + sym::simd_and => Op::MirOp(BinOp::BitAnd), + sym::simd_or => Op::MirOp(BinOp::BitOr), + sym::simd_xor => Op::MirOp(BinOp::BitXor), + sym::simd_eq => Op::MirOp(BinOp::Eq), + sym::simd_ne => Op::MirOp(BinOp::Ne), + sym::simd_lt => Op::MirOp(BinOp::Lt), + sym::simd_le => Op::MirOp(BinOp::Le), + sym::simd_gt => Op::MirOp(BinOp::Gt), + sym::simd_ge => Op::MirOp(BinOp::Ge), + sym::simd_fmax => Op::FMinMax(MinMax::Max), + sym::simd_fmin => Op::FMinMax(MinMax::Min), + sym::simd_saturating_add => Op::SaturatingOp(BinOp::Add), + sym::simd_saturating_sub => Op::SaturatingOp(BinOp::Sub), + sym::simd_arith_offset => Op::WrappingOffset, + _ => unreachable!(), + }; + + for i in 0..dest_len { + let left = self.read_immediate(&self.project_index(&left, i)?)?; + let right = self.read_immediate(&self.project_index(&right, i)?)?; + let dest = self.project_index(&dest, i)?; + let val = match which { + Op::MirOp(mir_op) => { + // this does NaN adjustments. + let val = self.binary_op(mir_op, &left, &right).map_err_kind(|kind| { + match kind { + InterpErrorKind::UndefinedBehavior(UndefinedBehaviorInfo::ShiftOverflow { shift_amount, .. }) => { + // this resets the interpreter backtrace, but it's not worth avoiding that. + let shift_amount = match shift_amount { + Either::Left(v) => v.to_string(), + Either::Right(v) => v.to_string(), + }; + err_ub_format!("overflowing shift by {shift_amount} in `{intrinsic_name}` in lane {i}") + } + kind => kind + } + })?; + if matches!( + mir_op, + BinOp::Eq + | BinOp::Ne + | BinOp::Lt + | BinOp::Le + | BinOp::Gt + | BinOp::Ge + ) { + // Special handling for boolean-returning operations + assert_eq!(val.layout.ty, self.tcx.types.bool); + let val = val.to_scalar().to_bool().unwrap(); + bool_to_simd_element(val, dest.layout.size) + } else { + assert_ne!(val.layout.ty, self.tcx.types.bool); + assert_eq!(val.layout.ty, dest.layout.ty); + val.to_scalar() + } + } + Op::SaturatingOp(mir_op) => self.saturating_arith(mir_op, &left, &right)?, + Op::WrappingOffset => { + let ptr = left.to_scalar().to_pointer(self)?; + let offset_count = right.to_scalar().to_target_isize(self)?; + let pointee_ty = left.layout.ty.builtin_deref(true).unwrap(); + + let pointee_size = + i64::try_from(self.layout_of(pointee_ty)?.size.bytes()).unwrap(); + let offset_bytes = offset_count.wrapping_mul(pointee_size); + let offset_ptr = ptr.wrapping_signed_offset(offset_bytes, self); + Scalar::from_maybe_pointer(offset_ptr, self) + } + Op::FMinMax(op) => self.fminmax_op(op, &left, &right)?, + }; + self.write_scalar(val, &dest)?; + } + } + sym::simd_reduce_and + | sym::simd_reduce_or + | sym::simd_reduce_xor + | sym::simd_reduce_any + | sym::simd_reduce_all + | sym::simd_reduce_max + | sym::simd_reduce_min => { + use mir::BinOp; + + let (op, op_len) = self.project_to_simd(&args[0])?; + + let imm_from_bool = |b| { + ImmTy::from_scalar( + Scalar::from_bool(b), + self.layout_of(self.tcx.types.bool).unwrap(), + ) + }; + + enum Op { + MirOp(BinOp), + MirOpBool(BinOp), + MinMax(MinMax), + } + let which = match intrinsic_name { + sym::simd_reduce_and => Op::MirOp(BinOp::BitAnd), + sym::simd_reduce_or => Op::MirOp(BinOp::BitOr), + sym::simd_reduce_xor => Op::MirOp(BinOp::BitXor), + sym::simd_reduce_any => Op::MirOpBool(BinOp::BitOr), + sym::simd_reduce_all => Op::MirOpBool(BinOp::BitAnd), + sym::simd_reduce_max => Op::MinMax(MinMax::Max), + sym::simd_reduce_min => Op::MinMax(MinMax::Min), + _ => unreachable!(), + }; + + // Initialize with first lane, then proceed with the rest. + let mut res = self.read_immediate(&self.project_index(&op, 0)?)?; + if matches!(which, Op::MirOpBool(_)) { + // Convert to `bool` scalar. + res = imm_from_bool(simd_element_to_bool(res)?); + } + for i in 1..op_len { + let op = self.read_immediate(&self.project_index(&op, i)?)?; + res = match which { + Op::MirOp(mir_op) => self.binary_op(mir_op, &res, &op)?, + Op::MirOpBool(mir_op) => { + let op = imm_from_bool(simd_element_to_bool(op)?); + self.binary_op(mir_op, &res, &op)? + } + Op::MinMax(mmop) => { + if matches!(res.layout.ty.kind(), ty::Float(_)) { + ImmTy::from_scalar(self.fminmax_op(mmop, &res, &op)?, res.layout) + } else { + // Just boring integers, so NaNs to worry about + let mirop = match mmop { + MinMax::Min => BinOp::Le, + MinMax::Max => BinOp::Ge, + }; + if self.binary_op(mirop, &res, &op)?.to_scalar().to_bool()? { + res + } else { + op + } + } + } + }; + } + self.write_immediate(*res, &dest)?; + } + sym::simd_reduce_add_ordered | sym::simd_reduce_mul_ordered => { + use mir::BinOp; + + let (op, op_len) = self.project_to_simd(&args[0])?; + let init = self.read_immediate(&args[1])?; + + let mir_op = match intrinsic_name { + sym::simd_reduce_add_ordered => BinOp::Add, + sym::simd_reduce_mul_ordered => BinOp::Mul, + _ => unreachable!(), + }; + + let mut res = init; + for i in 0..op_len { + let op = self.read_immediate(&self.project_index(&op, i)?)?; + res = self.binary_op(mir_op, &res, &op)?; + } + self.write_immediate(*res, &dest)?; + } + sym::simd_select => { + let (mask, mask_len) = self.project_to_simd(&args[0])?; + let (yes, yes_len) = self.project_to_simd(&args[1])?; + let (no, no_len) = self.project_to_simd(&args[2])?; + let (dest, dest_len) = self.project_to_simd(&dest)?; + + assert_eq!(dest_len, mask_len); + assert_eq!(dest_len, yes_len); + assert_eq!(dest_len, no_len); + + for i in 0..dest_len { + let mask = self.read_immediate(&self.project_index(&mask, i)?)?; + let yes = self.read_immediate(&self.project_index(&yes, i)?)?; + let no = self.read_immediate(&self.project_index(&no, i)?)?; + let dest = self.project_index(&dest, i)?; + + let val = if simd_element_to_bool(mask)? { yes } else { no }; + self.write_immediate(*val, &dest)?; + } + } + // Variant of `select` that takes a bitmask rather than a "vector of bool". + sym::simd_select_bitmask => { + let mask = &args[0]; + let (yes, yes_len) = self.project_to_simd(&args[1])?; + let (no, no_len) = self.project_to_simd(&args[2])?; + let (dest, dest_len) = self.project_to_simd(&dest)?; + let bitmask_len = dest_len.next_multiple_of(8); + if bitmask_len > 64 { + throw_unsup_format!( + "simd_select_bitmask: vectors larger than 64 elements are currently not supported" + ); + } + + assert_eq!(dest_len, yes_len); + assert_eq!(dest_len, no_len); + + // Read the mask, either as an integer or as an array. + let mask: u64 = match mask.layout.ty.kind() { + ty::Uint(_) => { + // Any larger integer type is fine. + assert!(mask.layout.size.bits() >= bitmask_len); + self.read_scalar(mask)?.to_bits(mask.layout.size)?.try_into().unwrap() + } + ty::Array(elem, _len) if elem == &self.tcx.types.u8 => { + // The array must have exactly the right size. + assert_eq!(mask.layout.size.bits(), bitmask_len); + // Read the raw bytes. + let mask = mask.assert_mem_place(); // arrays cannot be immediate + let mask_bytes = + self.read_bytes_ptr_strip_provenance(mask.ptr(), mask.layout.size)?; + // Turn them into a `u64` in the right way. + let mask_size = mask.layout.size.bytes_usize(); + let mut mask_arr = [0u8; 8]; + match self.tcx.data_layout.endian { + Endian::Little => { + // Fill the first N bytes. + mask_arr[..mask_size].copy_from_slice(mask_bytes); + u64::from_le_bytes(mask_arr) + } + Endian::Big => { + // Fill the last N bytes. + let i = mask_arr.len().strict_sub(mask_size); + mask_arr[i..].copy_from_slice(mask_bytes); + u64::from_be_bytes(mask_arr) + } + } + } + _ => bug!("simd_select_bitmask: invalid mask type {}", mask.layout.ty), + }; + + let dest_len = u32::try_from(dest_len).unwrap(); + for i in 0..dest_len { + let bit_i = simd_bitmask_index(i, dest_len, self.tcx.data_layout.endian); + let mask = mask & 1u64.strict_shl(bit_i); + let yes = self.read_immediate(&self.project_index(&yes, i.into())?)?; + let no = self.read_immediate(&self.project_index(&no, i.into())?)?; + let dest = self.project_index(&dest, i.into())?; + + let val = if mask != 0 { yes } else { no }; + self.write_immediate(*val, &dest)?; + } + // The remaining bits of the mask are ignored. + } + // Converts a "vector of bool" into a bitmask. + sym::simd_bitmask => { + let (op, op_len) = self.project_to_simd(&args[0])?; + let bitmask_len = op_len.next_multiple_of(8); + if bitmask_len > 64 { + throw_unsup_format!( + "simd_bitmask: vectors larger than 64 elements are currently not supported" + ); + } + + let op_len = u32::try_from(op_len).unwrap(); + let mut res = 0u64; + for i in 0..op_len { + let op = self.read_immediate(&self.project_index(&op, i.into())?)?; + if simd_element_to_bool(op)? { + let bit_i = simd_bitmask_index(i, op_len, self.tcx.data_layout.endian); + res |= 1u64.strict_shl(bit_i); + } + } + // Write the result, depending on the `dest` type. + // Returns either an unsigned integer or array of `u8`. + match dest.layout.ty.kind() { + ty::Uint(_) => { + // Any larger integer type is fine, it will be zero-extended. + assert!(dest.layout.size.bits() >= bitmask_len); + self.write_scalar(Scalar::from_uint(res, dest.layout.size), &dest)?; + } + ty::Array(elem, _len) if elem == &self.tcx.types.u8 => { + // The array must have exactly the right size. + assert_eq!(dest.layout.size.bits(), bitmask_len); + // We have to write the result byte-for-byte. + let res_size = dest.layout.size.bytes_usize(); + let res_bytes; + let res_bytes_slice = match self.tcx.data_layout.endian { + Endian::Little => { + res_bytes = res.to_le_bytes(); + &res_bytes[..res_size] // take the first N bytes + } + Endian::Big => { + res_bytes = res.to_be_bytes(); + &res_bytes[res_bytes.len().strict_sub(res_size)..] // take the last N bytes + } + }; + self.write_bytes_ptr(dest.ptr(), res_bytes_slice.iter().cloned())?; + } + _ => bug!("simd_bitmask: invalid return type {}", dest.layout.ty), + } + } + sym::simd_cast + | sym::simd_as + | sym::simd_cast_ptr + | sym::simd_with_exposed_provenance => { + let (op, op_len) = self.project_to_simd(&args[0])?; + let (dest, dest_len) = self.project_to_simd(&dest)?; + + assert_eq!(dest_len, op_len); + + let unsafe_cast = intrinsic_name == sym::simd_cast; + let safe_cast = intrinsic_name == sym::simd_as; + let ptr_cast = intrinsic_name == sym::simd_cast_ptr; + let from_exposed_cast = intrinsic_name == sym::simd_with_exposed_provenance; + + for i in 0..dest_len { + let op = self.read_immediate(&self.project_index(&op, i)?)?; + let dest = self.project_index(&dest, i)?; + + let val = match (op.layout.ty.kind(), dest.layout.ty.kind()) { + // Int-to-(int|float): always safe + (ty::Int(_) | ty::Uint(_), ty::Int(_) | ty::Uint(_) | ty::Float(_)) + if safe_cast || unsafe_cast => + self.int_to_int_or_float(&op, dest.layout)?, + // Float-to-float: always safe + (ty::Float(_), ty::Float(_)) if safe_cast || unsafe_cast => + self.float_to_float_or_int(&op, dest.layout)?, + // Float-to-int in safe mode + (ty::Float(_), ty::Int(_) | ty::Uint(_)) if safe_cast => + self.float_to_float_or_int(&op, dest.layout)?, + // Float-to-int in unchecked mode + (ty::Float(_), ty::Int(_) | ty::Uint(_)) if unsafe_cast => { + self.float_to_int_checked(&op, dest.layout, Round::TowardZero)? + .ok_or_else(|| { + err_ub_format!( + "`simd_cast` intrinsic called on {op} which cannot be represented in target type `{:?}`", + dest.layout.ty + ) + })? + } + // Ptr-to-ptr cast + (ty::RawPtr(..), ty::RawPtr(..)) if ptr_cast => + self.ptr_to_ptr(&op, dest.layout)?, + // Int->Ptr casts + (ty::Int(_) | ty::Uint(_), ty::RawPtr(..)) if from_exposed_cast => + self.pointer_with_exposed_provenance_cast(&op, dest.layout)?, + // Error otherwise + _ => + throw_unsup_format!( + "Unsupported SIMD cast from element type {from_ty} to {to_ty}", + from_ty = op.layout.ty, + to_ty = dest.layout.ty, + ), + }; + self.write_immediate(*val, &dest)?; + } + } + sym::simd_shuffle_const_generic => { + let (left, left_len) = self.project_to_simd(&args[0])?; + let (right, right_len) = self.project_to_simd(&args[1])?; + let (dest, dest_len) = self.project_to_simd(&dest)?; + + let index = generic_args[2].expect_const().to_value().valtree.unwrap_branch(); + let index_len = index.len(); + + assert_eq!(left_len, right_len); + assert_eq!(u64::try_from(index_len).unwrap(), dest_len); + + for i in 0..dest_len { + let src_index: u64 = + index[usize::try_from(i).unwrap()].unwrap_leaf().to_u32().into(); + let dest = self.project_index(&dest, i)?; + + let val = if src_index < left_len { + self.read_immediate(&self.project_index(&left, src_index)?)? + } else if src_index < left_len.strict_add(right_len) { + let right_idx = src_index.strict_sub(left_len); + self.read_immediate(&self.project_index(&right, right_idx)?)? + } else { + throw_ub_format!( + "`simd_shuffle_const_generic` index {src_index} is out-of-bounds for 2 vectors with length {dest_len}" + ); + }; + self.write_immediate(*val, &dest)?; + } + } + sym::simd_shuffle => { + let (left, left_len) = self.project_to_simd(&args[0])?; + let (right, right_len) = self.project_to_simd(&args[1])?; + let (index, index_len) = self.project_to_simd(&args[2])?; + let (dest, dest_len) = self.project_to_simd(&dest)?; + + assert_eq!(left_len, right_len); + assert_eq!(index_len, dest_len); + + for i in 0..dest_len { + let src_index: u64 = self + .read_immediate(&self.project_index(&index, i)?)? + .to_scalar() + .to_u32()? + .into(); + let dest = self.project_index(&dest, i)?; + + let val = if src_index < left_len { + self.read_immediate(&self.project_index(&left, src_index)?)? + } else if src_index < left_len.strict_add(right_len) { + let right_idx = src_index.strict_sub(left_len); + self.read_immediate(&self.project_index(&right, right_idx)?)? + } else { + throw_ub_format!( + "`simd_shuffle` index {src_index} is out-of-bounds for 2 vectors with length {dest_len}" + ); + }; + self.write_immediate(*val, &dest)?; + } + } + sym::simd_gather => { + let (passthru, passthru_len) = self.project_to_simd(&args[0])?; + let (ptrs, ptrs_len) = self.project_to_simd(&args[1])?; + let (mask, mask_len) = self.project_to_simd(&args[2])?; + let (dest, dest_len) = self.project_to_simd(&dest)?; + + assert_eq!(dest_len, passthru_len); + assert_eq!(dest_len, ptrs_len); + assert_eq!(dest_len, mask_len); + + for i in 0..dest_len { + let passthru = self.read_immediate(&self.project_index(&passthru, i)?)?; + let ptr = self.read_immediate(&self.project_index(&ptrs, i)?)?; + let mask = self.read_immediate(&self.project_index(&mask, i)?)?; + let dest = self.project_index(&dest, i)?; + + let val = if simd_element_to_bool(mask)? { + let place = self.deref_pointer(&ptr)?; + self.read_immediate(&place)? + } else { + passthru + }; + self.write_immediate(*val, &dest)?; + } + } + sym::simd_scatter => { + let (value, value_len) = self.project_to_simd(&args[0])?; + let (ptrs, ptrs_len) = self.project_to_simd(&args[1])?; + let (mask, mask_len) = self.project_to_simd(&args[2])?; + + assert_eq!(ptrs_len, value_len); + assert_eq!(ptrs_len, mask_len); + + for i in 0..ptrs_len { + let value = self.read_immediate(&self.project_index(&value, i)?)?; + let ptr = self.read_immediate(&self.project_index(&ptrs, i)?)?; + let mask = self.read_immediate(&self.project_index(&mask, i)?)?; + + if simd_element_to_bool(mask)? { + let place = self.deref_pointer(&ptr)?; + self.write_immediate(*value, &place)?; + } + } + } + sym::simd_masked_load => { + let (mask, mask_len) = self.project_to_simd(&args[0])?; + let ptr = self.read_pointer(&args[1])?; + let (default, default_len) = self.project_to_simd(&args[2])?; + let (dest, dest_len) = self.project_to_simd(&dest)?; + + assert_eq!(dest_len, mask_len); + assert_eq!(dest_len, default_len); + + for i in 0..dest_len { + let mask = self.read_immediate(&self.project_index(&mask, i)?)?; + let default = self.read_immediate(&self.project_index(&default, i)?)?; + let dest = self.project_index(&dest, i)?; + + let val = if simd_element_to_bool(mask)? { + // Size * u64 is implemented as always checked + let ptr = ptr.wrapping_offset(dest.layout.size * i, self); + let place = self.ptr_to_mplace(ptr, dest.layout); + self.read_immediate(&place)? + } else { + default + }; + self.write_immediate(*val, &dest)?; + } + } + sym::simd_masked_store => { + let (mask, mask_len) = self.project_to_simd(&args[0])?; + let ptr = self.read_pointer(&args[1])?; + let (vals, vals_len) = self.project_to_simd(&args[2])?; + + assert_eq!(mask_len, vals_len); + + for i in 0..vals_len { + let mask = self.read_immediate(&self.project_index(&mask, i)?)?; + let val = self.read_immediate(&self.project_index(&vals, i)?)?; + + if simd_element_to_bool(mask)? { + // Size * u64 is implemented as always checked + let ptr = ptr.wrapping_offset(val.layout.size * i, self); + let place = self.ptr_to_mplace(ptr, val.layout); + self.write_immediate(*val, &place)? + }; + } + } + + // Unsupported intrinsic: skip the return_to_block below. + _ => return interp_ok(false), + } + + trace!("{:?}", self.dump_place(&dest.clone().into())); + self.return_to_block(ret)?; + interp_ok(true) + } + + fn fminmax_op( + &self, + op: MinMax, + left: &ImmTy<'tcx, Prov>, + right: &ImmTy<'tcx, Prov>, + ) -> InterpResult<'tcx, Scalar> { + assert_eq!(left.layout.ty, right.layout.ty); + let ty::Float(float_ty) = left.layout.ty.kind() else { + bug!("fmax operand is not a float") + }; + let left = left.to_scalar(); + let right = right.to_scalar(); + interp_ok(match float_ty { + FloatTy::F16 => unimplemented!("f16_f128"), + FloatTy::F32 => { + let left = left.to_f32()?; + let right = right.to_f32()?; + let res = match op { + MinMax::Min => left.min(right), + MinMax::Max => left.max(right), + }; + let res = self.adjust_nan(res, &[left, right]); + Scalar::from_f32(res) + } + FloatTy::F64 => { + let left = left.to_f64()?; + let right = right.to_f64()?; + let res = match op { + MinMax::Min => left.min(right), + MinMax::Max => left.max(right), + }; + let res = self.adjust_nan(res, &[left, right]); + Scalar::from_f64(res) + } + FloatTy::F128 => unimplemented!("f16_f128"), + }) + } +} + +fn simd_bitmask_index(idx: u32, vec_len: u32, endianness: Endian) -> u32 { + assert!(idx < vec_len); + match endianness { + Endian::Little => idx, + #[expect(clippy::arithmetic_side_effects)] // idx < vec_len + Endian::Big => vec_len - 1 - idx, // reverse order of bits + } +} + +fn bool_to_simd_element(b: bool, size: Size) -> Scalar { + // SIMD uses all-1 as pattern for "true". In two's complement, + // -1 has all its bits set to one and `from_int` will truncate or + // sign-extend it to `size` as required. + let val = if b { -1 } else { 0 }; + Scalar::from_int(val, size) +} + +fn simd_element_to_bool(elem: ImmTy<'_, Prov>) -> InterpResult<'_, bool> { + assert!( + matches!(elem.layout.ty.kind(), ty::Int(_) | ty::Uint(_)), + "SIMD mask element type must be an integer, but this is `{}`", + elem.layout.ty + ); + let val = elem.to_scalar().to_int(elem.layout.size)?; + interp_ok(match val { + 0 => false, + -1 => true, + _ => throw_ub_format!("each element of a SIMD mask must be all-0-bits or all-1-bits"), + }) +} diff --git a/compiler/rustc_const_eval/src/util/check_validity_requirement.rs b/compiler/rustc_const_eval/src/util/check_validity_requirement.rs index 1dea7e4252d7f..34bd8423e8e15 100644 --- a/compiler/rustc_const_eval/src/util/check_validity_requirement.rs +++ b/compiler/rustc_const_eval/src/util/check_validity_requirement.rs @@ -59,7 +59,7 @@ fn check_validity_requirement_strict<'tcx>( if kind == ValidityRequirement::Zero { cx.write_bytes_ptr( allocated.ptr(), - std::iter::repeat(0_u8).take(ty.layout.size().bytes_usize()), + std::iter::repeat_n(0_u8, ty.layout.size().bytes_usize()), ) .expect("failed to write bytes for zero valid check"); } diff --git a/compiler/rustc_hir_analysis/src/collect.rs b/compiler/rustc_hir_analysis/src/collect.rs index f78eb2bae8ada..b6a662f425602 100644 --- a/compiler/rustc_hir_analysis/src/collect.rs +++ b/compiler/rustc_hir_analysis/src/collect.rs @@ -590,7 +590,7 @@ fn get_new_lifetime_name<'tcx>( let a_to_z_repeat_n = |n| { (b'a'..=b'z').map(move |c| { let mut s = '\''.to_string(); - s.extend(std::iter::repeat(char::from(c)).take(n)); + s.extend(std::iter::repeat_n(char::from(c), n)); s }) }; diff --git a/compiler/rustc_hir_analysis/src/errors/wrong_number_of_generic_args.rs b/compiler/rustc_hir_analysis/src/errors/wrong_number_of_generic_args.rs index 8a9f9130feacb..30324da3c6516 100644 --- a/compiler/rustc_hir_analysis/src/errors/wrong_number_of_generic_args.rs +++ b/compiler/rustc_hir_analysis/src/errors/wrong_number_of_generic_args.rs @@ -339,8 +339,7 @@ impl<'a, 'tcx> WrongNumberOfGenericArgs<'a, 'tcx> { hir::GenericArg::Lifetime(lt) => Some(lt), _ => None, }) { - return std::iter::repeat(lt.to_string()) - .take(num_params_to_take) + return std::iter::repeat_n(lt.to_string(), num_params_to_take) .collect::>() .join(", "); } @@ -362,8 +361,7 @@ impl<'a, 'tcx> WrongNumberOfGenericArgs<'a, 'tcx> { matches!(fn_decl.output, hir::FnRetTy::Return(ty) if ty.hir_id == ty_id); if in_arg || (in_ret && fn_decl.lifetime_elision_allowed) { - return std::iter::repeat("'_".to_owned()) - .take(num_params_to_take) + return std::iter::repeat_n("'_".to_owned(), num_params_to_take) .collect::>() .join(", "); } @@ -388,10 +386,12 @@ impl<'a, 'tcx> WrongNumberOfGenericArgs<'a, 'tcx> { }) | hir::Node::AnonConst(..) = node { - return std::iter::repeat("'static".to_owned()) - .take(num_params_to_take.saturating_sub(ret.len())) - .collect::>() - .join(", "); + return std::iter::repeat_n( + "'static".to_owned(), + num_params_to_take.saturating_sub(ret.len()), + ) + .collect::>() + .join(", "); } let params = if let Some(generics) = node.generics() { diff --git a/compiler/rustc_hir_typeck/src/expr.rs b/compiler/rustc_hir_typeck/src/expr.rs index e663dd25e0ea9..1aaf02646c794 100644 --- a/compiler/rustc_hir_typeck/src/expr.rs +++ b/compiler/rustc_hir_typeck/src/expr.rs @@ -2602,7 +2602,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { let suggestion = |name, args| { format!( "::{name}({})", - std::iter::repeat("_").take(args).collect::>().join(", ") + std::iter::repeat_n("_", args).collect::>().join(", ") ) }; match &items[..] { diff --git a/compiler/rustc_lint/src/if_let_rescope.rs b/compiler/rustc_lint/src/if_let_rescope.rs index ff67ed1bc5532..2521c2b4eb6ac 100644 --- a/compiler/rustc_lint/src/if_let_rescope.rs +++ b/compiler/rustc_lint/src/if_let_rescope.rs @@ -1,4 +1,4 @@ -use std::iter::repeat; +use std::iter::repeat_n; use std::ops::ControlFlow; use hir::intravisit::{self, Visitor}; @@ -351,7 +351,7 @@ impl Subdiagnostic for IfLetRescopeRewrite { .then_some(" _ => {}".chars()) .into_iter() .flatten() - .chain(repeat('}').take(closing_brackets.count)) + .chain(repeat_n('}', closing_brackets.count)) .collect(), )); let msg = diag.eagerly_translate(crate::fluent_generated::lint_suggestion); diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 38d677cfa3403..5c5b7eea219b9 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -205,12 +205,6 @@ extern "C" LLVMValueRef LLVMRustGetOrInsertGlobal(LLVMModuleRef M, return wrap(GV); } -extern "C" LLVMValueRef LLVMRustInsertPrivateGlobal(LLVMModuleRef M, - LLVMTypeRef Ty) { - return wrap(new GlobalVariable(*unwrap(M), unwrap(Ty), false, - GlobalValue::PrivateLinkage, nullptr)); -} - // Must match the layout of `rustc_codegen_llvm::llvm::ffi::AttributeKind`. enum class LLVMRustAttributeKind { AlwaysInline = 0, diff --git a/compiler/rustc_middle/src/values.rs b/compiler/rustc_middle/src/values.rs index 4d70a70873267..bc73d36216ef4 100644 --- a/compiler/rustc_middle/src/values.rs +++ b/compiler/rustc_middle/src/values.rs @@ -63,7 +63,7 @@ impl<'tcx> Value> for ty::Binder<'_, ty::FnSig<'_>> { }; let fn_sig = ty::Binder::dummy(tcx.mk_fn_sig( - std::iter::repeat(err).take(arity), + std::iter::repeat_n(err, arity), err, false, rustc_hir::Safety::Safe, diff --git a/compiler/rustc_mir_build/src/check_tail_calls.rs b/compiler/rustc_mir_build/src/check_tail_calls.rs index d40c77d145f47..9115c17f37525 100644 --- a/compiler/rustc_mir_build/src/check_tail_calls.rs +++ b/compiler/rustc_mir_build/src/check_tail_calls.rs @@ -283,7 +283,7 @@ impl<'tcx> TailCallCkVisitor<'_, 'tcx> { fn report_calling_closure(&mut self, fun: &Expr<'_>, tupled_args: Ty<'_>, expr: &Expr<'_>) { let underscored_args = match tupled_args.kind() { ty::Tuple(tys) if tys.is_empty() => "".to_owned(), - ty::Tuple(tys) => std::iter::repeat("_, ").take(tys.len() - 1).chain(["_"]).collect(), + ty::Tuple(tys) => std::iter::repeat_n("_, ", tys.len() - 1).chain(["_"]).collect(), _ => "_".to_owned(), }; diff --git a/compiler/rustc_mir_transform/src/ctfe_limit.rs b/compiler/rustc_mir_transform/src/ctfe_limit.rs index ac46336b83473..e2f518bb4ee81 100644 --- a/compiler/rustc_mir_transform/src/ctfe_limit.rs +++ b/compiler/rustc_mir_transform/src/ctfe_limit.rs @@ -28,12 +28,12 @@ impl<'tcx> crate::MirPass<'tcx> for CtfeLimit { } }) .collect(); + + let basic_blocks = body.basic_blocks.as_mut_preserves_cfg(); for index in indices { - insert_counter( - body.basic_blocks_mut() - .get_mut(index) - .expect("basic_blocks index {index} should exist"), - ); + let bbdata = &mut basic_blocks[index]; + let source_info = bbdata.terminator().source_info; + bbdata.statements.push(Statement::new(source_info, StatementKind::ConstEvalCounter)); } } @@ -53,10 +53,3 @@ fn has_back_edge( // Check if any of the dominators of the node are also the node's successor. node_data.terminator().successors().any(|succ| doms.dominates(succ, node)) } - -fn insert_counter(basic_block_data: &mut BasicBlockData<'_>) { - basic_block_data.statements.push(Statement::new( - basic_block_data.terminator().source_info, - StatementKind::ConstEvalCounter, - )); -} diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs index 80879c88d1ba8..cba6243fa1098 100644 --- a/compiler/rustc_passes/src/check_attr.rs +++ b/compiler/rustc_passes/src/check_attr.rs @@ -2019,10 +2019,13 @@ impl<'tcx> CheckAttrVisitor<'tcx> { } let expected_sig = tcx.mk_fn_sig( - std::iter::repeat(token_stream).take(match kind { - ProcMacroKind::Attribute => 2, - ProcMacroKind::Derive | ProcMacroKind::FunctionLike => 1, - }), + std::iter::repeat_n( + token_stream, + match kind { + ProcMacroKind::Attribute => 2, + ProcMacroKind::Derive | ProcMacroKind::FunctionLike => 1, + }, + ), token_stream, false, Safety::Safe, diff --git a/compiler/rustc_query_system/src/dep_graph/serialized.rs b/compiler/rustc_query_system/src/dep_graph/serialized.rs index 43cac015c32af..0012bf79a1f5d 100644 --- a/compiler/rustc_query_system/src/dep_graph/serialized.rs +++ b/compiler/rustc_query_system/src/dep_graph/serialized.rs @@ -566,7 +566,7 @@ impl EncoderState { edge_count: 0, node_count: 0, encoder: MemEncoder::new(), - kind_stats: iter::repeat(0).take(D::DEP_KIND_MAX as usize + 1).collect(), + kind_stats: iter::repeat_n(0, D::DEP_KIND_MAX as usize + 1).collect(), }) }), marker: PhantomData, @@ -735,7 +735,7 @@ impl EncoderState { let mut encoder = self.file.lock().take().unwrap(); - let mut kind_stats: Vec = iter::repeat(0).take(D::DEP_KIND_MAX as usize + 1).collect(); + let mut kind_stats: Vec = iter::repeat_n(0, D::DEP_KIND_MAX as usize + 1).collect(); let mut node_max = 0; let mut node_count = 0; diff --git a/compiler/rustc_resolve/src/late/diagnostics.rs b/compiler/rustc_resolve/src/late/diagnostics.rs index 8c2ddda7f9834..1589e3f4092c4 100644 --- a/compiler/rustc_resolve/src/late/diagnostics.rs +++ b/compiler/rustc_resolve/src/late/diagnostics.rs @@ -2214,10 +2214,7 @@ impl<'ast, 'ra, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> { .collect::>(); items.sort_by_key(|(order, _, _)| *order); let suggestion = |name, args| { - format!( - "::{name}({})", - std::iter::repeat("_").take(args).collect::>().join(", ") - ) + format!("::{name}({})", std::iter::repeat_n("_", args).collect::>().join(", ")) }; match &items[..] { [] => {} @@ -3485,17 +3482,14 @@ impl<'ast, 'ra, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> { (lt.span.shrink_to_hi(), format!("{existing_name} ")) } MissingLifetimeKind::Comma => { - let sugg: String = std::iter::repeat([existing_name.as_str(), ", "]) - .take(lt.count) + let sugg: String = std::iter::repeat_n([existing_name.as_str(), ", "], lt.count) .flatten() .collect(); (lt.span.shrink_to_hi(), sugg) } MissingLifetimeKind::Brackets => { let sugg: String = std::iter::once("<") - .chain( - std::iter::repeat(existing_name.as_str()).take(lt.count).intersperse(", "), - ) + .chain(std::iter::repeat_n(existing_name.as_str(), lt.count).intersperse(", ")) .chain([">"]) .collect(); (lt.span.shrink_to_hi(), sugg) diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 1318f28ff89f3..71b0f408b5d88 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -981,10 +981,12 @@ symbols! { external_doc, f, f16, + f16_consts_mod, f16_epsilon, f16_nan, f16c_target_feature, f32, + f32_consts_mod, f32_epsilon, f32_legacy_const_digits, f32_legacy_const_epsilon, @@ -1002,6 +1004,7 @@ symbols! { f32_legacy_const_radix, f32_nan, f64, + f64_consts_mod, f64_epsilon, f64_legacy_const_digits, f64_legacy_const_epsilon, @@ -1019,6 +1022,7 @@ symbols! { f64_legacy_const_radix, f64_nan, f128, + f128_consts_mod, f128_epsilon, f128_nan, fabsf16, diff --git a/compiler/rustc_trait_selection/src/error_reporting/infer/nice_region_error/static_impl_trait.rs b/compiler/rustc_trait_selection/src/error_reporting/infer/nice_region_error/static_impl_trait.rs index 5b8315841765b..3ee6e6b739c03 100644 --- a/compiler/rustc_trait_selection/src/error_reporting/infer/nice_region_error/static_impl_trait.rs +++ b/compiler/rustc_trait_selection/src/error_reporting/infer/nice_region_error/static_impl_trait.rs @@ -305,7 +305,7 @@ fn make_elided_region_spans_suggs<'a>( consecutive_brackets += 1; } else if let Some(bracket_span) = bracket_span.take() { let sugg = std::iter::once("<") - .chain(std::iter::repeat(name).take(consecutive_brackets).intersperse(", ")) + .chain(std::iter::repeat_n(name, consecutive_brackets).intersperse(", ")) .chain([">"]) .collect(); spans_suggs.push((bracket_span.shrink_to_hi(), sugg)); diff --git a/compiler/rustc_transmute/src/layout/tree.rs b/compiler/rustc_transmute/src/layout/tree.rs index 7f626e8c4e886..d9ebdadf3b79c 100644 --- a/compiler/rustc_transmute/src/layout/tree.rs +++ b/compiler/rustc_transmute/src/layout/tree.rs @@ -326,8 +326,7 @@ pub(crate) mod rustc { let inner_layout = layout_of(cx, *inner_ty)?; assert_eq!(*stride, inner_layout.size); let elt = Tree::from_ty(*inner_ty, cx)?; - Ok(std::iter::repeat(elt) - .take(*count as usize) + Ok(std::iter::repeat_n(elt, *count as usize) .fold(Tree::unit(), |tree, elt| tree.then(elt))) } diff --git a/library/core/src/iter/sources/repeat.rs b/library/core/src/iter/sources/repeat.rs index 4bcd5b16aea6a..7fcd6f8963a48 100644 --- a/library/core/src/iter/sources/repeat.rs +++ b/library/core/src/iter/sources/repeat.rs @@ -8,6 +8,9 @@ use crate::num::NonZero; /// Infinite iterators like `repeat()` are often used with adapters like /// [`Iterator::take()`], in order to make them finite. /// +/// If you know the number of repetitions in advance, consider using [`repeat_n()`] +/// instead, as it is more efficient and conveys the intent more clearly. +/// /// Use [`str::repeat()`] instead of this function if you just want to repeat /// a char/string `n` times. /// @@ -15,6 +18,7 @@ use crate::num::NonZero; /// or if you do not want to keep the repeated element in memory, you can /// instead use the [`repeat_with()`] function. /// +/// [`repeat_n()`]: crate::iter::repeat_n /// [`repeat_with()`]: crate::iter::repeat_with /// [`str::repeat()`]: ../../std/primitive.str.html#method.repeat /// diff --git a/library/core/src/num/f128.rs b/library/core/src/num/f128.rs index 4fe4735e304c9..e7101537b298f 100644 --- a/library/core/src/num/f128.rs +++ b/library/core/src/num/f128.rs @@ -18,6 +18,7 @@ use crate::{intrinsics, mem}; /// Basic mathematical constants. #[unstable(feature = "f128", issue = "116909")] +#[rustc_diagnostic_item = "f128_consts_mod"] pub mod consts { // FIXME: replace with mathematical constants from cmath. diff --git a/library/core/src/num/f16.rs b/library/core/src/num/f16.rs index 0bea6bc8801d8..aa8342a22ad58 100644 --- a/library/core/src/num/f16.rs +++ b/library/core/src/num/f16.rs @@ -20,6 +20,7 @@ use crate::{intrinsics, mem}; /// Basic mathematical constants. #[unstable(feature = "f16", issue = "116909")] +#[rustc_diagnostic_item = "f16_consts_mod"] pub mod consts { // FIXME: replace with mathematical constants from cmath. diff --git a/library/core/src/num/f32.rs b/library/core/src/num/f32.rs index e380cc698f574..3070e1dedbe43 100644 --- a/library/core/src/num/f32.rs +++ b/library/core/src/num/f32.rs @@ -277,6 +277,7 @@ pub const NEG_INFINITY: f32 = f32::NEG_INFINITY; /// Basic mathematical constants. #[stable(feature = "rust1", since = "1.0.0")] +#[rustc_diagnostic_item = "f32_consts_mod"] pub mod consts { // FIXME: replace with mathematical constants from cmath. diff --git a/library/core/src/num/f64.rs b/library/core/src/num/f64.rs index ff7449fd996ce..dc8ccc551b2da 100644 --- a/library/core/src/num/f64.rs +++ b/library/core/src/num/f64.rs @@ -277,6 +277,7 @@ pub const NEG_INFINITY: f64 = f64::NEG_INFINITY; /// Basic mathematical constants. #[stable(feature = "rust1", since = "1.0.0")] +#[rustc_diagnostic_item = "f64_consts_mod"] pub mod consts { // FIXME: replace with mathematical constants from cmath. diff --git a/library/core/src/panicking.rs b/library/core/src/panicking.rs index b5150837e6a94..448f4ffc3dae0 100644 --- a/library/core/src/panicking.rs +++ b/library/core/src/panicking.rs @@ -36,7 +36,8 @@ use crate::panic::{Location, PanicInfo}; compile_error!( "panic_immediate_abort is now a real panic strategy! \ Enable it with `panic = \"immediate-abort\"` in Cargo.toml, \ - or with the compiler flags `-Zunstable-options -Cpanic=immediate-abort`" + or with the compiler flags `-Zunstable-options -Cpanic=immediate-abort`. \ + In both cases, you still need to build core, e.g. with `-Zbuild-std`" ); // First we define the two main entry points that all panics go through. diff --git a/library/core/src/slice/specialize.rs b/library/core/src/slice/specialize.rs index 80eb590587f99..17436395fee69 100644 --- a/library/core/src/slice/specialize.rs +++ b/library/core/src/slice/specialize.rs @@ -15,9 +15,54 @@ impl SpecFill for [T] { } impl SpecFill for [T] { - fn spec_fill(&mut self, value: T) { + default fn spec_fill(&mut self, value: T) { for item in self.iter_mut() { *item = value; } } } + +impl SpecFill for [u8] { + fn spec_fill(&mut self, value: u8) { + // SAFETY: The pointer is derived from a reference, so it's writable. + unsafe { + crate::intrinsics::write_bytes(self.as_mut_ptr(), value, self.len()); + } + } +} + +impl SpecFill for [i8] { + fn spec_fill(&mut self, value: i8) { + // SAFETY: The pointer is derived from a reference, so it's writable. + unsafe { + crate::intrinsics::write_bytes(self.as_mut_ptr(), value.cast_unsigned(), self.len()); + } + } +} + +macro spec_fill_int { + ($($type:ty)*) => {$( + impl SpecFill<$type> for [$type] { + #[inline] + fn spec_fill(&mut self, value: $type) { + // We always take this fastpath in Miri for long slices as the manual `for` + // loop can be prohibitively slow. + if (cfg!(miri) && self.len() > 32) || crate::intrinsics::is_val_statically_known(value) { + let bytes = value.to_ne_bytes(); + if value == <$type>::from_ne_bytes([bytes[0]; size_of::<$type>()]) { + // SAFETY: The pointer is derived from a reference, so it's writable. + unsafe { + crate::intrinsics::write_bytes(self.as_mut_ptr(), bytes[0], self.len()); + } + return; + } + } + for item in self.iter_mut() { + *item = value; + } + } + } + )*} +} + +spec_fill_int! { u16 i16 u32 i32 u64 i64 u128 i128 usize isize } diff --git a/library/std/src/lib.rs b/library/std/src/lib.rs index da41c1216c4d5..46eb120d65de3 100644 --- a/library/std/src/lib.rs +++ b/library/std/src/lib.rs @@ -63,10 +63,10 @@ //! type, but not the all-important methods. //! //! So for example there is a [page for the primitive type -//! `i32`](primitive::i32) that lists all the methods that can be called on -//! 32-bit integers (very useful), and there is a [page for the module -//! `std::i32`] that documents the constant values [`MIN`] and [`MAX`] (rarely -//! useful). +//! `char`](primitive::char) that lists all the methods that can be called on +//! characters (very useful), and there is a [page for the module +//! `std::char`] that documents iterator and error types created by these methods +//! (rarely useful). //! //! Note the documentation for the primitives [`str`] and [`[T]`][prim@slice] (also //! called 'slice'). Many method calls on [`String`] and [`Vec`] are actually diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index c0ddd031821a6..87d54c2b5ad30 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -529,7 +529,7 @@ pub fn std_cargo(builder: &Builder<'_>, target: TargetSelection, cargo: &mut Car // Query rustc for the deployment target, and the associated env var. // The env var is one of the standard `*_DEPLOYMENT_TARGET` vars, i.e. // `MACOSX_DEPLOYMENT_TARGET`, `IPHONEOS_DEPLOYMENT_TARGET`, etc. - let mut cmd = command(builder.rustc(cargo.compiler())); + let mut cmd = builder.rustc_cmd(cargo.compiler()); cmd.arg("--target").arg(target.rustc_target_arg()); cmd.arg("--print=deployment-target"); let output = cmd.run_capture_stdout(builder).stdout(); diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index b79d2cb413db7..411d42962644d 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -625,7 +625,7 @@ fn generate_target_spec_json_schema(builder: &Builder<'_>, sysroot: &Path) { // We do this by using the stage 1 compiler, which is always compiled for the host, // even in a cross build. let stage1_host = builder.compiler(1, builder.host_target); - let mut rustc = command(builder.rustc(stage1_host)).fail_fast(); + let mut rustc = builder.rustc_cmd(stage1_host).fail_fast(); rustc .env("RUSTC_BOOTSTRAP", "1") .args(["--print=target-spec-json-schema", "-Zunstable-options"]); diff --git a/src/bootstrap/src/core/build_steps/synthetic_targets.rs b/src/bootstrap/src/core/build_steps/synthetic_targets.rs index 21733c5d9e3f7..2cc691832b5f0 100644 --- a/src/bootstrap/src/core/build_steps/synthetic_targets.rs +++ b/src/bootstrap/src/core/build_steps/synthetic_targets.rs @@ -10,7 +10,6 @@ use crate::Compiler; use crate::core::builder::{Builder, ShouldRun, Step}; use crate::core::config::TargetSelection; -use crate::utils::exec::command; #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub(crate) struct MirOptPanicAbortSyntheticTarget { @@ -55,7 +54,7 @@ fn create_synthetic_target( return TargetSelection::create_synthetic(&name, path.to_str().unwrap()); } - let mut cmd = command(builder.rustc(compiler)); + let mut cmd = builder.rustc_cmd(compiler); cmd.arg("--target").arg(base.rustc_target_arg()); cmd.args(["-Zunstable-options", "--print", "target-spec-json"]); diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 00aea8feab7de..154b209b20252 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -1974,9 +1974,6 @@ HELP: You can add it into `bootstrap.toml` in `rust.codegen-backends = [{name:?} } else if mode == "rustdoc-js" { panic!("need nodejs to run rustdoc-js suite"); } - if let Some(ref npm) = builder.config.npm { - cmd.arg("--npm").arg(npm); - } if builder.config.rust_optimize_tests { cmd.arg("--optimize-tests"); } diff --git a/src/bootstrap/src/core/builder/cargo.rs b/src/bootstrap/src/core/builder/cargo.rs index a404aec512091..c2029f97391d4 100644 --- a/src/bootstrap/src/core/builder/cargo.rs +++ b/src/bootstrap/src/core/builder/cargo.rs @@ -728,10 +728,8 @@ impl Builder<'_> { // Build proc macros both for the host and the target unless proc-macros are not // supported by the target. if target != compiler.host && cmd_kind != Kind::Check { - let mut rustc_cmd = command(self.rustc(compiler)); - self.add_rustc_lib_path(compiler, &mut rustc_cmd); - - let error = rustc_cmd + let error = self + .rustc_cmd(compiler) .arg("--target") .arg(target.rustc_target_arg()) .arg("--print=file-names") diff --git a/src/bootstrap/src/core/builder/mod.rs b/src/bootstrap/src/core/builder/mod.rs index fc06db8f80b9d..1ddfbd51df523 100644 --- a/src/bootstrap/src/core/builder/mod.rs +++ b/src/bootstrap/src/core/builder/mod.rs @@ -1605,6 +1605,14 @@ Alternatively, you can set `build.local-rebuild=true` and use a stage0 compiler } } + /// Gets a command to run the compiler specified, including the dynamic library + /// path in case the executable has not been build with `rpath` enabled. + pub fn rustc_cmd(&self, compiler: Compiler) -> BootstrapCommand { + let mut cmd = command(self.rustc(compiler)); + self.add_rustc_lib_path(compiler, &mut cmd); + cmd + } + /// Gets the paths to all of the compiler's codegen backends. fn codegen_backends(&self, compiler: Compiler) -> impl Iterator { fs::read_dir(self.sysroot_codegen_backends(compiler)) diff --git a/src/tools/compiletest/src/bin/main.rs b/src/tools/compiletest/src/bin/main.rs index 8fac6ccdc582b..3e608d57e81a7 100644 --- a/src/tools/compiletest/src/bin/main.rs +++ b/src/tools/compiletest/src/bin/main.rs @@ -1,23 +1,3 @@ -use std::env; -use std::io::IsTerminal; -use std::sync::Arc; - -use compiletest::{early_config_check, parse_config, run_tests}; - fn main() { - tracing_subscriber::fmt::init(); - - // colored checks stdout by default, but for some reason only stderr is a terminal. - // compiletest *does* print many things to stdout, but it doesn't really matter. - if std::io::stderr().is_terminal() - && matches!(std::env::var("NO_COLOR").as_deref(), Err(_) | Ok("0")) - { - colored::control::set_override(true); - } - - let config = Arc::new(parse_config(env::args().collect())); - - early_config_check(&config); - - run_tests(config); + compiletest::cli::main(); } diff --git a/src/tools/compiletest/src/cli.rs b/src/tools/compiletest/src/cli.rs new file mode 100644 index 0000000000000..763c46550b411 --- /dev/null +++ b/src/tools/compiletest/src/cli.rs @@ -0,0 +1,26 @@ +//! Isolates the APIs used by `bin/main.rs`, to help minimize the surface area +//! of public exports from the compiletest library crate. + +use std::env; +use std::io::IsTerminal; +use std::sync::Arc; + +use crate::{early_config_check, parse_config, run_tests}; + +pub fn main() { + tracing_subscriber::fmt::init(); + + // colored checks stdout by default, but for some reason only stderr is a terminal. + // compiletest *does* print many things to stdout, but it doesn't really matter. + if std::io::stderr().is_terminal() + && matches!(std::env::var("NO_COLOR").as_deref(), Err(_) | Ok("0")) + { + colored::control::set_override(true); + } + + let config = Arc::new(parse_config(env::args().collect())); + + early_config_check(&config); + + run_tests(config); +} diff --git a/src/tools/compiletest/src/common.rs b/src/tools/compiletest/src/common.rs index 65db816ad1a87..58330ed20dc88 100644 --- a/src/tools/compiletest/src/common.rs +++ b/src/tools/compiletest/src/common.rs @@ -631,8 +631,6 @@ pub struct Config { /// Path to a NodeJS executable. Used for JS doctests, emscripten and WASM tests. pub nodejs: Option, - /// Path to a npm executable. Used for rustdoc GUI tests. - pub npm: Option, /// Whether to rerun tests even if the inputs are unchanged. pub force_rerun: bool, @@ -686,110 +684,6 @@ pub struct Config { } impl Config { - /// Incomplete config intended for `src/tools/rustdoc-gui-test` **only** as - /// `src/tools/rustdoc-gui-test` wants to reuse `compiletest`'s directive -> test property - /// handling for `//@ {compile,run}-flags`, do not use for any other purpose. - /// - /// FIXME(#143827): this setup feels very hacky. It so happens that `tests/rustdoc-gui/` - /// **only** uses `//@ {compile,run}-flags` for now and not any directives that actually rely on - /// info that is assumed available in a fully populated [`Config`]. - pub fn incomplete_for_rustdoc_gui_test() -> Config { - // FIXME(#143827): spelling this out intentionally, because this is questionable. - // - // For instance, `//@ ignore-stage1` will not work at all. - Config { - mode: TestMode::Rustdoc, - // E.g. this has no sensible default tbh. - suite: TestSuite::Ui, - - // Dummy values. - edition: Default::default(), - bless: Default::default(), - fail_fast: Default::default(), - compile_lib_path: Utf8PathBuf::default(), - run_lib_path: Utf8PathBuf::default(), - rustc_path: Utf8PathBuf::default(), - cargo_path: Default::default(), - stage0_rustc_path: Default::default(), - query_rustc_path: Default::default(), - rustdoc_path: Default::default(), - coverage_dump_path: Default::default(), - python: Default::default(), - jsondocck_path: Default::default(), - jsondoclint_path: Default::default(), - llvm_filecheck: Default::default(), - llvm_bin_dir: Default::default(), - run_clang_based_tests_with: Default::default(), - src_root: Utf8PathBuf::default(), - src_test_suite_root: Utf8PathBuf::default(), - build_root: Utf8PathBuf::default(), - build_test_suite_root: Utf8PathBuf::default(), - sysroot_base: Utf8PathBuf::default(), - stage: Default::default(), - stage_id: String::default(), - debugger: Default::default(), - run_ignored: Default::default(), - with_rustc_debug_assertions: Default::default(), - with_std_debug_assertions: Default::default(), - filters: Default::default(), - skip: Default::default(), - filter_exact: Default::default(), - force_pass_mode: Default::default(), - run: Default::default(), - runner: Default::default(), - host_rustcflags: Default::default(), - target_rustcflags: Default::default(), - rust_randomized_layout: Default::default(), - optimize_tests: Default::default(), - target: Default::default(), - host: Default::default(), - cdb: Default::default(), - cdb_version: Default::default(), - gdb: Default::default(), - gdb_version: Default::default(), - lldb_version: Default::default(), - llvm_version: Default::default(), - system_llvm: Default::default(), - android_cross_path: Default::default(), - adb_path: Default::default(), - adb_test_dir: Default::default(), - adb_device_status: Default::default(), - lldb_python_dir: Default::default(), - verbose: Default::default(), - color: Default::default(), - remote_test_client: Default::default(), - compare_mode: Default::default(), - rustfix_coverage: Default::default(), - has_html_tidy: Default::default(), - has_enzyme: Default::default(), - channel: Default::default(), - git_hash: Default::default(), - cc: Default::default(), - cxx: Default::default(), - cflags: Default::default(), - cxxflags: Default::default(), - ar: Default::default(), - target_linker: Default::default(), - host_linker: Default::default(), - llvm_components: Default::default(), - nodejs: Default::default(), - npm: Default::default(), - force_rerun: Default::default(), - only_modified: Default::default(), - target_cfgs: Default::default(), - builtin_cfg_names: Default::default(), - supported_crate_types: Default::default(), - nocapture: Default::default(), - nightly_branch: Default::default(), - git_merge_commit_email: Default::default(), - profiler_runtime: Default::default(), - diff_command: Default::default(), - minicore_path: Default::default(), - default_codegen_backend: CodegenBackend::Llvm, - override_codegen_backend: None, - } - } - /// FIXME: this run scheme is... confusing. pub fn run_enabled(&self) -> bool { self.run.unwrap_or_else(|| { @@ -825,7 +719,8 @@ impl Config { self.target_cfg().abi == abi } - pub fn matches_family(&self, family: &str) -> bool { + #[cfg_attr(not(test), expect(dead_code, reason = "only used by tests for `ignore-{family}`"))] + pub(crate) fn matches_family(&self, family: &str) -> bool { self.target_cfg().families.iter().any(|f| f == family) } diff --git a/src/tools/compiletest/src/directives.rs b/src/tools/compiletest/src/directives.rs index 3490ba8e64d91..34a9d77ce0dfa 100644 --- a/src/tools/compiletest/src/directives.rs +++ b/src/tools/compiletest/src/directives.rs @@ -198,8 +198,6 @@ pub struct TestProps { pub filecheck_flags: Vec, /// Don't automatically insert any `--check-cfg` args pub no_auto_check_cfg: bool, - /// Run tests which require enzyme being build - pub has_enzyme: bool, /// Build and use `minicore` as `core` stub for `no_core` tests in cross-compilation scenarios /// that don't otherwise want/need `-Z build-std`. pub add_core_stubs: bool, @@ -314,7 +312,6 @@ impl TestProps { llvm_cov_flags: vec![], filecheck_flags: vec![], no_auto_check_cfg: false, - has_enzyme: false, add_core_stubs: false, core_stubs_compile_flags: vec![], dont_require_annotations: Default::default(), diff --git a/src/tools/compiletest/src/lib.rs b/src/tools/compiletest/src/lib.rs index ac1a8226112a9..48ae38b61b93d 100644 --- a/src/tools/compiletest/src/lib.rs +++ b/src/tools/compiletest/src/lib.rs @@ -3,20 +3,22 @@ #[cfg(test)] mod tests; -pub mod common; +pub mod cli; +mod common; mod debuggers; -pub mod diagnostics; -pub mod directives; -pub mod edition; -pub mod errors; +mod diagnostics; +mod directives; +mod edition; +mod errors; mod executor; mod json; mod output_capture; mod panic_hook; mod raise_fd_limit; mod read2; -pub mod runtest; -pub mod util; +mod runtest; +pub mod rustdoc_gui_test; +mod util; use core::panic; use std::collections::HashSet; @@ -48,7 +50,7 @@ use crate::executor::{CollectedTest, ColorConfig}; /// The config mostly reflects command-line arguments, but there might also be /// some code here that inspects environment variables or even runs executables /// (e.g. when discovering debugger versions). -pub fn parse_config(args: Vec) -> Config { +fn parse_config(args: Vec) -> Config { let mut opts = Options::new(); opts.reqopt("", "compile-lib-path", "path to host shared libraries", "PATH") .reqopt("", "run-lib-path", "path to target shared libraries", "PATH") @@ -462,7 +464,6 @@ pub fn parse_config(args: Vec) -> Config { host_linker: matches.opt_str("host-linker"), llvm_components: matches.opt_str("llvm-components").unwrap(), nodejs: matches.opt_str("nodejs"), - npm: matches.opt_str("npm"), force_rerun: matches.opt_present("force-rerun"), @@ -486,14 +487,7 @@ pub fn parse_config(args: Vec) -> Config { } } -pub fn opt_str(maybestr: &Option) -> &str { - match *maybestr { - None => "(none)", - Some(ref s) => s, - } -} - -pub fn opt_str2(maybestr: Option) -> String { +fn opt_str2(maybestr: Option) -> String { match maybestr { None => "(none)".to_owned(), Some(s) => s, @@ -501,7 +495,7 @@ pub fn opt_str2(maybestr: Option) -> String { } /// Called by `main` after the config has been parsed. -pub fn run_tests(config: Arc) { +fn run_tests(config: Arc) { debug!(?config, "run_tests"); panic_hook::install_panic_hook(); @@ -639,7 +633,7 @@ impl TestCollector { /// FIXME(Zalathar): Now that we no longer rely on libtest, try to overhaul /// test discovery to take into account the filters/tests specified on the /// command-line, instead of having to enumerate everything. -pub(crate) fn collect_and_make_tests(config: Arc) -> Vec { +fn collect_and_make_tests(config: Arc) -> Vec { debug!("making tests from {}", config.src_test_suite_root); let common_inputs_stamp = common_inputs_stamp(&config); let modified_tests = @@ -851,7 +845,7 @@ fn collect_tests_from_dir( } /// Returns true if `file_name` looks like a proper test file name. -pub fn is_test(file_name: &str) -> bool { +fn is_test(file_name: &str) -> bool { if !file_name.ends_with(".rs") { return false; } @@ -1131,7 +1125,7 @@ fn check_for_overlapping_test_paths(found_path_stems: &HashSet) { } } -pub fn early_config_check(config: &Config) { +fn early_config_check(config: &Config) { if !config.has_html_tidy && config.mode == TestMode::Rustdoc { warning!("`tidy` (html-tidy.org) is not installed; diffs will not be generated"); } diff --git a/src/tools/compiletest/src/rustdoc_gui_test.rs b/src/tools/compiletest/src/rustdoc_gui_test.rs new file mode 100644 index 0000000000000..4bd3531f4b433 --- /dev/null +++ b/src/tools/compiletest/src/rustdoc_gui_test.rs @@ -0,0 +1,142 @@ +//! This module isolates the compiletest APIs used by the rustdoc-gui-test tool. +//! +//! Thanks to this isolation layer, changes to compiletest directive parsing +//! might require changes to the items in this module, but shouldn't require +//! changes to rustdoc-gui-test itself. +//! +//! The current relationship between compiletest and rustdoc-gui-test is +//! awkward. Ideally, rustdoc-gui-test should either split off its own +//! directive parser and become fully independent, or be incorporated into +//! compiletest as another test mode. +//! +//! See for more context. + +use std::path::Path; + +use camino::{Utf8Path, Utf8PathBuf}; + +use crate::common::{CodegenBackend, Config, TestMode, TestSuite}; +use crate::directives::TestProps; + +/// Subset of compiletest directive values that are actually used by +/// rustdoc-gui-test. +#[derive(Debug)] +pub struct RustdocGuiTestProps { + pub compile_flags: Vec, + pub run_flags: Vec, +} + +impl RustdocGuiTestProps { + pub fn from_file(test_file_path: &Path) -> Self { + let test_file_path = Utf8Path::from_path(test_file_path).unwrap(); + let config = incomplete_config_for_rustdoc_gui_test(); + + let props = TestProps::from_file(test_file_path, None, &config); + + let TestProps { compile_flags, run_flags, .. } = props; + Self { compile_flags, run_flags } + } +} + +/// Incomplete config intended for `src/tools/rustdoc-gui-test` **only** as +/// `src/tools/rustdoc-gui-test` wants to reuse `compiletest`'s directive -> test property +/// handling for `//@ {compile,run}-flags`, do not use for any other purpose. +/// +/// FIXME(#143827): this setup feels very hacky. It so happens that `tests/rustdoc-gui/` +/// **only** uses `//@ {compile,run}-flags` for now and not any directives that actually rely on +/// info that is assumed available in a fully populated [`Config`]. +fn incomplete_config_for_rustdoc_gui_test() -> Config { + // FIXME(#143827): spelling this out intentionally, because this is questionable. + // + // For instance, `//@ ignore-stage1` will not work at all. + Config { + mode: TestMode::Rustdoc, + // E.g. this has no sensible default tbh. + suite: TestSuite::Ui, + + // Dummy values. + edition: Default::default(), + bless: Default::default(), + fail_fast: Default::default(), + compile_lib_path: Utf8PathBuf::default(), + run_lib_path: Utf8PathBuf::default(), + rustc_path: Utf8PathBuf::default(), + cargo_path: Default::default(), + stage0_rustc_path: Default::default(), + query_rustc_path: Default::default(), + rustdoc_path: Default::default(), + coverage_dump_path: Default::default(), + python: Default::default(), + jsondocck_path: Default::default(), + jsondoclint_path: Default::default(), + llvm_filecheck: Default::default(), + llvm_bin_dir: Default::default(), + run_clang_based_tests_with: Default::default(), + src_root: Utf8PathBuf::default(), + src_test_suite_root: Utf8PathBuf::default(), + build_root: Utf8PathBuf::default(), + build_test_suite_root: Utf8PathBuf::default(), + sysroot_base: Utf8PathBuf::default(), + stage: Default::default(), + stage_id: String::default(), + debugger: Default::default(), + run_ignored: Default::default(), + with_rustc_debug_assertions: Default::default(), + with_std_debug_assertions: Default::default(), + filters: Default::default(), + skip: Default::default(), + filter_exact: Default::default(), + force_pass_mode: Default::default(), + run: Default::default(), + runner: Default::default(), + host_rustcflags: Default::default(), + target_rustcflags: Default::default(), + rust_randomized_layout: Default::default(), + optimize_tests: Default::default(), + target: Default::default(), + host: Default::default(), + cdb: Default::default(), + cdb_version: Default::default(), + gdb: Default::default(), + gdb_version: Default::default(), + lldb_version: Default::default(), + llvm_version: Default::default(), + system_llvm: Default::default(), + android_cross_path: Default::default(), + adb_path: Default::default(), + adb_test_dir: Default::default(), + adb_device_status: Default::default(), + lldb_python_dir: Default::default(), + verbose: Default::default(), + color: Default::default(), + remote_test_client: Default::default(), + compare_mode: Default::default(), + rustfix_coverage: Default::default(), + has_html_tidy: Default::default(), + has_enzyme: Default::default(), + channel: Default::default(), + git_hash: Default::default(), + cc: Default::default(), + cxx: Default::default(), + cflags: Default::default(), + cxxflags: Default::default(), + ar: Default::default(), + target_linker: Default::default(), + host_linker: Default::default(), + llvm_components: Default::default(), + nodejs: Default::default(), + force_rerun: Default::default(), + only_modified: Default::default(), + target_cfgs: Default::default(), + builtin_cfg_names: Default::default(), + supported_crate_types: Default::default(), + nocapture: Default::default(), + nightly_branch: Default::default(), + git_merge_commit_email: Default::default(), + profiler_runtime: Default::default(), + diff_command: Default::default(), + minicore_path: Default::default(), + default_codegen_backend: CodegenBackend::Llvm, + override_codegen_backend: None, + } +} diff --git a/src/tools/compiletest/src/util.rs b/src/tools/compiletest/src/util.rs index 558e9a58697e7..a189c1d1c2dfc 100644 --- a/src/tools/compiletest/src/util.rs +++ b/src/tools/compiletest/src/util.rs @@ -102,7 +102,9 @@ macro_rules! string_enum { } impl $name { + #[allow(dead_code)] $vis const VARIANTS: &'static [Self] = &[$(Self::$variant,)*]; + #[allow(dead_code)] $vis const STR_VARIANTS: &'static [&'static str] = &[$(Self::$variant.to_str(),)*]; $vis const fn to_str(&self) -> &'static str { diff --git a/src/tools/miri/src/helpers.rs b/src/tools/miri/src/helpers.rs index e0c077e99319a..d6646f9586aa6 100644 --- a/src/tools/miri/src/helpers.rs +++ b/src/tools/miri/src/helpers.rs @@ -5,7 +5,6 @@ use std::{cmp, iter}; use rand::RngCore; use rustc_abi::{Align, ExternAbi, FieldIdx, FieldsShape, Size, Variants}; use rustc_apfloat::Float; -use rustc_apfloat::ieee::{Double, Half, Quad, Single}; use rustc_hir::Safety; use rustc_hir::def::{DefKind, Namespace}; use rustc_hir::def_id::{CRATE_DEF_INDEX, CrateNum, DefId, LOCAL_CRATE}; @@ -14,7 +13,7 @@ use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::middle::dependency_format::Linkage; use rustc_middle::middle::exported_symbols::ExportedSymbol; use rustc_middle::ty::layout::{LayoutOf, MaybeResult, TyAndLayout}; -use rustc_middle::ty::{self, FloatTy, IntTy, Ty, TyCtxt, UintTy}; +use rustc_middle::ty::{self, IntTy, Ty, TyCtxt, UintTy}; use rustc_session::config::CrateType; use rustc_span::{Span, Symbol}; use rustc_symbol_mangling::mangle_internal_symbol; @@ -961,75 +960,6 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { this.alloc_mark_immutable(provenance.get_alloc_id().unwrap()).unwrap(); } - /// Converts `src` from floating point to integer type `dest_ty` - /// after rounding with mode `round`. - /// Returns `None` if `f` is NaN or out of range. - fn float_to_int_checked( - &self, - src: &ImmTy<'tcx>, - cast_to: TyAndLayout<'tcx>, - round: rustc_apfloat::Round, - ) -> InterpResult<'tcx, Option>> { - let this = self.eval_context_ref(); - - fn float_to_int_inner<'tcx, F: rustc_apfloat::Float>( - ecx: &MiriInterpCx<'tcx>, - src: F, - cast_to: TyAndLayout<'tcx>, - round: rustc_apfloat::Round, - ) -> (Scalar, rustc_apfloat::Status) { - let int_size = cast_to.layout.size; - match cast_to.ty.kind() { - // Unsigned - ty::Uint(_) => { - let res = src.to_u128_r(int_size.bits_usize(), round, &mut false); - (Scalar::from_uint(res.value, int_size), res.status) - } - // Signed - ty::Int(_) => { - let res = src.to_i128_r(int_size.bits_usize(), round, &mut false); - (Scalar::from_int(res.value, int_size), res.status) - } - // Nothing else - _ => - span_bug!( - ecx.cur_span(), - "attempted float-to-int conversion with non-int output type {}", - cast_to.ty, - ), - } - } - - let ty::Float(fty) = src.layout.ty.kind() else { - bug!("float_to_int_checked: non-float input type {}", src.layout.ty) - }; - - let (val, status) = match fty { - FloatTy::F16 => - float_to_int_inner::(this, src.to_scalar().to_f16()?, cast_to, round), - FloatTy::F32 => - float_to_int_inner::(this, src.to_scalar().to_f32()?, cast_to, round), - FloatTy::F64 => - float_to_int_inner::(this, src.to_scalar().to_f64()?, cast_to, round), - FloatTy::F128 => - float_to_int_inner::(this, src.to_scalar().to_f128()?, cast_to, round), - }; - - if status.intersects( - rustc_apfloat::Status::INVALID_OP - | rustc_apfloat::Status::OVERFLOW - | rustc_apfloat::Status::UNDERFLOW, - ) { - // Floating point value is NaN (flagged with INVALID_OP) or outside the range - // of values of the integer type (flagged with OVERFLOW or UNDERFLOW). - interp_ok(None) - } else { - // Floating point value can be represented by the integer type after rounding. - // The INEXACT flag is ignored on purpose to allow rounding. - interp_ok(Some(ImmTy::from_scalar(val, cast_to))) - } - } - /// Returns an integer type that is twice wide as `ty` fn get_twice_wide_int_ty(&self, ty: Ty<'tcx>) -> Ty<'tcx> { let this = self.eval_context_ref(); @@ -1194,20 +1124,6 @@ pub(crate) fn bool_to_simd_element(b: bool, size: Size) -> Scalar { Scalar::from_int(val, size) } -pub(crate) fn simd_element_to_bool(elem: ImmTy<'_>) -> InterpResult<'_, bool> { - assert!( - matches!(elem.layout.ty.kind(), ty::Int(_) | ty::Uint(_)), - "SIMD mask element type must be an integer, but this is `{}`", - elem.layout.ty - ); - let val = elem.to_scalar().to_int(elem.layout.size)?; - interp_ok(match val { - 0 => false, - -1 => true, - _ => throw_ub_format!("each element of a SIMD mask must be all-0-bits or all-1-bits"), - }) -} - /// Check whether an operation that writes to a target buffer was successful. /// Accordingly select return value. /// Local helper function to be used in Windows shims. diff --git a/src/tools/miri/src/intrinsics/mod.rs b/src/tools/miri/src/intrinsics/mod.rs index a80b939d84ea9..f09fc6c187896 100644 --- a/src/tools/miri/src/intrinsics/mod.rs +++ b/src/tools/miri/src/intrinsics/mod.rs @@ -118,7 +118,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { return this.emulate_atomic_intrinsic(name, generic_args, args, dest); } if let Some(name) = intrinsic_name.strip_prefix("simd_") { - return this.emulate_simd_intrinsic(name, generic_args, args, dest); + return this.emulate_simd_intrinsic(name, args, dest); } match intrinsic_name { diff --git a/src/tools/miri/src/intrinsics/simd.rs b/src/tools/miri/src/intrinsics/simd.rs index b26516c0ff0e8..5f75657e0a220 100644 --- a/src/tools/miri/src/intrinsics/simd.rs +++ b/src/tools/miri/src/intrinsics/simd.rs @@ -1,21 +1,12 @@ -use either::Either; use rand::Rng; -use rustc_abi::{Endian, HasDataLayout}; -use rustc_apfloat::{Float, Round}; +use rustc_apfloat::Float; use rustc_middle::ty::FloatTy; -use rustc_middle::{mir, ty}; -use rustc_span::{Symbol, sym}; +use rustc_middle::ty; use super::check_intrinsic_arg_count; -use crate::helpers::{ToHost, ToSoft, bool_to_simd_element, simd_element_to_bool}; +use crate::helpers::{ToHost, ToSoft}; use crate::*; -#[derive(Copy, Clone)] -pub(crate) enum MinMax { - Min, - Max, -} - impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {} pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { /// Calls the simd intrinsic `intrinsic`; the `simd_` prefix has already been removed. @@ -23,20 +14,12 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { fn emulate_simd_intrinsic( &mut self, intrinsic_name: &str, - generic_args: ty::GenericArgsRef<'tcx>, args: &[OpTy<'tcx>], dest: &MPlaceTy<'tcx>, ) -> InterpResult<'tcx, EmulateItemResult> { let this = self.eval_context_mut(); match intrinsic_name { #[rustfmt::skip] - | "neg" - | "fabs" - | "ceil" - | "floor" - | "round" - | "round_ties_even" - | "trunc" | "fsqrt" | "fsin" | "fcos" @@ -45,11 +28,6 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { | "flog" | "flog2" | "flog10" - | "ctlz" - | "ctpop" - | "cttz" - | "bswap" - | "bitreverse" => { let [op] = check_intrinsic_arg_count(args)?; let (op, op_len) = this.project_to_simd(op)?; @@ -57,235 +35,51 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { assert_eq!(dest_len, op_len); - #[derive(Copy, Clone)] - enum Op<'a> { - MirOp(mir::UnOp), - Abs, - Round(rustc_apfloat::Round), - Numeric(Symbol), - HostOp(&'a str), - } - let which = match intrinsic_name { - "neg" => Op::MirOp(mir::UnOp::Neg), - "fabs" => Op::Abs, - "ceil" => Op::Round(rustc_apfloat::Round::TowardPositive), - "floor" => Op::Round(rustc_apfloat::Round::TowardNegative), - "round" => Op::Round(rustc_apfloat::Round::NearestTiesToAway), - "round_ties_even" => Op::Round(rustc_apfloat::Round::NearestTiesToEven), - "trunc" => Op::Round(rustc_apfloat::Round::TowardZero), - "ctlz" => Op::Numeric(sym::ctlz), - "ctpop" => Op::Numeric(sym::ctpop), - "cttz" => Op::Numeric(sym::cttz), - "bswap" => Op::Numeric(sym::bswap), - "bitreverse" => Op::Numeric(sym::bitreverse), - _ => Op::HostOp(intrinsic_name), - }; - for i in 0..dest_len { let op = this.read_immediate(&this.project_index(&op, i)?)?; let dest = this.project_index(&dest, i)?; - let val = match which { - Op::MirOp(mir_op) => { - // This already does NaN adjustments - this.unary_op(mir_op, &op)?.to_scalar() - } - Op::Abs => { - // Works for f32 and f64. - let ty::Float(float_ty) = op.layout.ty.kind() else { - span_bug!(this.cur_span(), "{} operand is not a float", intrinsic_name) - }; - let op = op.to_scalar(); - // "Bitwise" operation, no NaN adjustments - match float_ty { - FloatTy::F16 => unimplemented!("f16_f128"), - FloatTy::F32 => Scalar::from_f32(op.to_f32()?.abs()), - FloatTy::F64 => Scalar::from_f64(op.to_f64()?.abs()), - FloatTy::F128 => unimplemented!("f16_f128"), - } - } - Op::HostOp(host_op) => { - let ty::Float(float_ty) = op.layout.ty.kind() else { - span_bug!(this.cur_span(), "{} operand is not a float", intrinsic_name) + let ty::Float(float_ty) = op.layout.ty.kind() else { + span_bug!(this.cur_span(), "{} operand is not a float", intrinsic_name) + }; + // Using host floats except for sqrt (but it's fine, these operations do not + // have guaranteed precision). + let val = match float_ty { + FloatTy::F16 => unimplemented!("f16_f128"), + FloatTy::F32 => { + let f = op.to_scalar().to_f32()?; + let res = match intrinsic_name { + "fsqrt" => math::sqrt(f), + "fsin" => f.to_host().sin().to_soft(), + "fcos" => f.to_host().cos().to_soft(), + "fexp" => f.to_host().exp().to_soft(), + "fexp2" => f.to_host().exp2().to_soft(), + "flog" => f.to_host().ln().to_soft(), + "flog2" => f.to_host().log2().to_soft(), + "flog10" => f.to_host().log10().to_soft(), + _ => bug!(), }; - // Using host floats except for sqrt (but it's fine, these operations do not - // have guaranteed precision). - match float_ty { - FloatTy::F16 => unimplemented!("f16_f128"), - FloatTy::F32 => { - let f = op.to_scalar().to_f32()?; - let res = match host_op { - "fsqrt" => math::sqrt(f), - "fsin" => f.to_host().sin().to_soft(), - "fcos" => f.to_host().cos().to_soft(), - "fexp" => f.to_host().exp().to_soft(), - "fexp2" => f.to_host().exp2().to_soft(), - "flog" => f.to_host().ln().to_soft(), - "flog2" => f.to_host().log2().to_soft(), - "flog10" => f.to_host().log10().to_soft(), - _ => bug!(), - }; - let res = this.adjust_nan(res, &[f]); - Scalar::from(res) - } - FloatTy::F64 => { - let f = op.to_scalar().to_f64()?; - let res = match host_op { - "fsqrt" => math::sqrt(f), - "fsin" => f.to_host().sin().to_soft(), - "fcos" => f.to_host().cos().to_soft(), - "fexp" => f.to_host().exp().to_soft(), - "fexp2" => f.to_host().exp2().to_soft(), - "flog" => f.to_host().ln().to_soft(), - "flog2" => f.to_host().log2().to_soft(), - "flog10" => f.to_host().log10().to_soft(), - _ => bug!(), - }; - let res = this.adjust_nan(res, &[f]); - Scalar::from(res) - } - FloatTy::F128 => unimplemented!("f16_f128"), - } + let res = this.adjust_nan(res, &[f]); + Scalar::from(res) } - Op::Round(rounding) => { - let ty::Float(float_ty) = op.layout.ty.kind() else { - span_bug!(this.cur_span(), "{} operand is not a float", intrinsic_name) + FloatTy::F64 => { + let f = op.to_scalar().to_f64()?; + let res = match intrinsic_name { + "fsqrt" => math::sqrt(f), + "fsin" => f.to_host().sin().to_soft(), + "fcos" => f.to_host().cos().to_soft(), + "fexp" => f.to_host().exp().to_soft(), + "fexp2" => f.to_host().exp2().to_soft(), + "flog" => f.to_host().ln().to_soft(), + "flog2" => f.to_host().log2().to_soft(), + "flog10" => f.to_host().log10().to_soft(), + _ => bug!(), }; - match float_ty { - FloatTy::F16 => unimplemented!("f16_f128"), - FloatTy::F32 => { - let f = op.to_scalar().to_f32()?; - let res = f.round_to_integral(rounding).value; - let res = this.adjust_nan(res, &[f]); - Scalar::from_f32(res) - } - FloatTy::F64 => { - let f = op.to_scalar().to_f64()?; - let res = f.round_to_integral(rounding).value; - let res = this.adjust_nan(res, &[f]); - Scalar::from_f64(res) - } - FloatTy::F128 => unimplemented!("f16_f128"), - } - } - Op::Numeric(name) => { - this.numeric_intrinsic(name, op.to_scalar(), op.layout, op.layout)? - } - }; - this.write_scalar(val, &dest)?; - } - } - #[rustfmt::skip] - | "add" - | "sub" - | "mul" - | "div" - | "rem" - | "shl" - | "shr" - | "and" - | "or" - | "xor" - | "eq" - | "ne" - | "lt" - | "le" - | "gt" - | "ge" - | "fmax" - | "fmin" - | "saturating_add" - | "saturating_sub" - | "arith_offset" - => { - use mir::BinOp; - - let [left, right] = check_intrinsic_arg_count(args)?; - let (left, left_len) = this.project_to_simd(left)?; - let (right, right_len) = this.project_to_simd(right)?; - let (dest, dest_len) = this.project_to_simd(dest)?; - - assert_eq!(dest_len, left_len); - assert_eq!(dest_len, right_len); - - enum Op { - MirOp(BinOp), - SaturatingOp(BinOp), - FMinMax(MinMax), - WrappingOffset, - } - let which = match intrinsic_name { - "add" => Op::MirOp(BinOp::Add), - "sub" => Op::MirOp(BinOp::Sub), - "mul" => Op::MirOp(BinOp::Mul), - "div" => Op::MirOp(BinOp::Div), - "rem" => Op::MirOp(BinOp::Rem), - "shl" => Op::MirOp(BinOp::ShlUnchecked), - "shr" => Op::MirOp(BinOp::ShrUnchecked), - "and" => Op::MirOp(BinOp::BitAnd), - "or" => Op::MirOp(BinOp::BitOr), - "xor" => Op::MirOp(BinOp::BitXor), - "eq" => Op::MirOp(BinOp::Eq), - "ne" => Op::MirOp(BinOp::Ne), - "lt" => Op::MirOp(BinOp::Lt), - "le" => Op::MirOp(BinOp::Le), - "gt" => Op::MirOp(BinOp::Gt), - "ge" => Op::MirOp(BinOp::Ge), - "fmax" => Op::FMinMax(MinMax::Max), - "fmin" => Op::FMinMax(MinMax::Min), - "saturating_add" => Op::SaturatingOp(BinOp::Add), - "saturating_sub" => Op::SaturatingOp(BinOp::Sub), - "arith_offset" => Op::WrappingOffset, - _ => unreachable!(), - }; - - for i in 0..dest_len { - let left = this.read_immediate(&this.project_index(&left, i)?)?; - let right = this.read_immediate(&this.project_index(&right, i)?)?; - let dest = this.project_index(&dest, i)?; - let val = match which { - Op::MirOp(mir_op) => { - // This does NaN adjustments. - let val = this.binary_op(mir_op, &left, &right).map_err_kind(|kind| { - match kind { - InterpErrorKind::UndefinedBehavior(UndefinedBehaviorInfo::ShiftOverflow { shift_amount, .. }) => { - // This resets the interpreter backtrace, but it's not worth avoiding that. - let shift_amount = match shift_amount { - Either::Left(v) => v.to_string(), - Either::Right(v) => v.to_string(), - }; - err_ub_format!("overflowing shift by {shift_amount} in `simd_{intrinsic_name}` in lane {i}") - } - kind => kind - } - })?; - if matches!(mir_op, BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge) { - // Special handling for boolean-returning operations - assert_eq!(val.layout.ty, this.tcx.types.bool); - let val = val.to_scalar().to_bool().unwrap(); - bool_to_simd_element(val, dest.layout.size) - } else { - assert_ne!(val.layout.ty, this.tcx.types.bool); - assert_eq!(val.layout.ty, dest.layout.ty); - val.to_scalar() - } - } - Op::SaturatingOp(mir_op) => { - this.saturating_arith(mir_op, &left, &right)? - } - Op::WrappingOffset => { - let ptr = left.to_scalar().to_pointer(this)?; - let offset_count = right.to_scalar().to_target_isize(this)?; - let pointee_ty = left.layout.ty.builtin_deref(true).unwrap(); - - let pointee_size = i64::try_from(this.layout_of(pointee_ty)?.size.bytes()).unwrap(); - let offset_bytes = offset_count.wrapping_mul(pointee_size); - let offset_ptr = ptr.wrapping_signed_offset(offset_bytes, this); - Scalar::from_maybe_pointer(offset_ptr, this) - } - Op::FMinMax(op) => { - this.fminmax_op(op, &left, &right)? + let res = this.adjust_nan(res, &[f]); + Scalar::from(res) } + FloatTy::F128 => unimplemented!("f16_f128"), }; + this.write_scalar(val, &dest)?; } } @@ -345,279 +139,25 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { this.write_scalar(val, &dest)?; } } - #[rustfmt::skip] - | "reduce_and" - | "reduce_or" - | "reduce_xor" - | "reduce_any" - | "reduce_all" - | "reduce_max" - | "reduce_min" => { - use mir::BinOp; - - let [op] = check_intrinsic_arg_count(args)?; - let (op, op_len) = this.project_to_simd(op)?; - - let imm_from_bool = - |b| ImmTy::from_scalar(Scalar::from_bool(b), this.machine.layouts.bool); - - enum Op { - MirOp(BinOp), - MirOpBool(BinOp), - MinMax(MinMax), - } - let which = match intrinsic_name { - "reduce_and" => Op::MirOp(BinOp::BitAnd), - "reduce_or" => Op::MirOp(BinOp::BitOr), - "reduce_xor" => Op::MirOp(BinOp::BitXor), - "reduce_any" => Op::MirOpBool(BinOp::BitOr), - "reduce_all" => Op::MirOpBool(BinOp::BitAnd), - "reduce_max" => Op::MinMax(MinMax::Max), - "reduce_min" => Op::MinMax(MinMax::Min), - _ => unreachable!(), - }; - - // Initialize with first lane, then proceed with the rest. - let mut res = this.read_immediate(&this.project_index(&op, 0)?)?; - if matches!(which, Op::MirOpBool(_)) { - // Convert to `bool` scalar. - res = imm_from_bool(simd_element_to_bool(res)?); - } - for i in 1..op_len { - let op = this.read_immediate(&this.project_index(&op, i)?)?; - res = match which { - Op::MirOp(mir_op) => { - this.binary_op(mir_op, &res, &op)? - } - Op::MirOpBool(mir_op) => { - let op = imm_from_bool(simd_element_to_bool(op)?); - this.binary_op(mir_op, &res, &op)? - } - Op::MinMax(mmop) => { - if matches!(res.layout.ty.kind(), ty::Float(_)) { - ImmTy::from_scalar(this.fminmax_op(mmop, &res, &op)?, res.layout) - } else { - // Just boring integers, so NaNs to worry about - let mirop = match mmop { - MinMax::Min => BinOp::Le, - MinMax::Max => BinOp::Ge, - }; - if this.binary_op(mirop, &res, &op)?.to_scalar().to_bool()? { - res - } else { - op - } - } - } - }; - } - this.write_immediate(*res, dest)?; - } - #[rustfmt::skip] - | "reduce_add_ordered" - | "reduce_mul_ordered" => { - use mir::BinOp; - - let [op, init] = check_intrinsic_arg_count(args)?; - let (op, op_len) = this.project_to_simd(op)?; - let init = this.read_immediate(init)?; - - let mir_op = match intrinsic_name { - "reduce_add_ordered" => BinOp::Add, - "reduce_mul_ordered" => BinOp::Mul, - _ => unreachable!(), - }; - - let mut res = init; - for i in 0..op_len { - let op = this.read_immediate(&this.project_index(&op, i)?)?; - res = this.binary_op(mir_op, &res, &op)?; - } - this.write_immediate(*res, dest)?; - } - "select" => { - let [mask, yes, no] = check_intrinsic_arg_count(args)?; - let (mask, mask_len) = this.project_to_simd(mask)?; - let (yes, yes_len) = this.project_to_simd(yes)?; - let (no, no_len) = this.project_to_simd(no)?; - let (dest, dest_len) = this.project_to_simd(dest)?; - - assert_eq!(dest_len, mask_len); - assert_eq!(dest_len, yes_len); - assert_eq!(dest_len, no_len); - - for i in 0..dest_len { - let mask = this.read_immediate(&this.project_index(&mask, i)?)?; - let yes = this.read_immediate(&this.project_index(&yes, i)?)?; - let no = this.read_immediate(&this.project_index(&no, i)?)?; - let dest = this.project_index(&dest, i)?; - - let val = if simd_element_to_bool(mask)? { yes } else { no }; - this.write_immediate(*val, &dest)?; - } - } - // Variant of `select` that takes a bitmask rather than a "vector of bool". - "select_bitmask" => { - let [mask, yes, no] = check_intrinsic_arg_count(args)?; - let (yes, yes_len) = this.project_to_simd(yes)?; - let (no, no_len) = this.project_to_simd(no)?; - let (dest, dest_len) = this.project_to_simd(dest)?; - let bitmask_len = dest_len.next_multiple_of(8); - if bitmask_len > 64 { - throw_unsup_format!( - "simd_select_bitmask: vectors larger than 64 elements are currently not supported" - ); - } - - assert_eq!(dest_len, yes_len); - assert_eq!(dest_len, no_len); - - // Read the mask, either as an integer or as an array. - let mask: u64 = match mask.layout.ty.kind() { - ty::Uint(_) => { - // Any larger integer type is fine. - assert!(mask.layout.size.bits() >= bitmask_len); - this.read_scalar(mask)?.to_bits(mask.layout.size)?.try_into().unwrap() - } - ty::Array(elem, _len) if elem == &this.tcx.types.u8 => { - // The array must have exactly the right size. - assert_eq!(mask.layout.size.bits(), bitmask_len); - // Read the raw bytes. - let mask = mask.assert_mem_place(); // arrays cannot be immediate - let mask_bytes = - this.read_bytes_ptr_strip_provenance(mask.ptr(), mask.layout.size)?; - // Turn them into a `u64` in the right way. - let mask_size = mask.layout.size.bytes_usize(); - let mut mask_arr = [0u8; 8]; - match this.data_layout().endian { - Endian::Little => { - // Fill the first N bytes. - mask_arr[..mask_size].copy_from_slice(mask_bytes); - u64::from_le_bytes(mask_arr) - } - Endian::Big => { - // Fill the last N bytes. - let i = mask_arr.len().strict_sub(mask_size); - mask_arr[i..].copy_from_slice(mask_bytes); - u64::from_be_bytes(mask_arr) - } - } - } - _ => bug!("simd_select_bitmask: invalid mask type {}", mask.layout.ty), - }; - - let dest_len = u32::try_from(dest_len).unwrap(); - for i in 0..dest_len { - let bit_i = simd_bitmask_index(i, dest_len, this.data_layout().endian); - let mask = mask & 1u64.strict_shl(bit_i); - let yes = this.read_immediate(&this.project_index(&yes, i.into())?)?; - let no = this.read_immediate(&this.project_index(&no, i.into())?)?; - let dest = this.project_index(&dest, i.into())?; - - let val = if mask != 0 { yes } else { no }; - this.write_immediate(*val, &dest)?; - } - // The remaining bits of the mask are ignored. - } - // Converts a "vector of bool" into a bitmask. - "bitmask" => { - let [op] = check_intrinsic_arg_count(args)?; - let (op, op_len) = this.project_to_simd(op)?; - let bitmask_len = op_len.next_multiple_of(8); - if bitmask_len > 64 { - throw_unsup_format!( - "simd_bitmask: vectors larger than 64 elements are currently not supported" - ); - } - - let op_len = u32::try_from(op_len).unwrap(); - let mut res = 0u64; - for i in 0..op_len { - let op = this.read_immediate(&this.project_index(&op, i.into())?)?; - if simd_element_to_bool(op)? { - let bit_i = simd_bitmask_index(i, op_len, this.data_layout().endian); - res |= 1u64.strict_shl(bit_i); - } - } - // Write the result, depending on the `dest` type. - // Returns either an unsigned integer or array of `u8`. - match dest.layout.ty.kind() { - ty::Uint(_) => { - // Any larger integer type is fine, it will be zero-extended. - assert!(dest.layout.size.bits() >= bitmask_len); - this.write_int(res, dest)?; - } - ty::Array(elem, _len) if elem == &this.tcx.types.u8 => { - // The array must have exactly the right size. - assert_eq!(dest.layout.size.bits(), bitmask_len); - // We have to write the result byte-for-byte. - let res_size = dest.layout.size.bytes_usize(); - let res_bytes; - let res_bytes_slice = match this.data_layout().endian { - Endian::Little => { - res_bytes = res.to_le_bytes(); - &res_bytes[..res_size] // take the first N bytes - } - Endian::Big => { - res_bytes = res.to_be_bytes(); - &res_bytes[res_bytes.len().strict_sub(res_size)..] // take the last N bytes - } - }; - this.write_bytes_ptr(dest.ptr(), res_bytes_slice.iter().cloned())?; - } - _ => bug!("simd_bitmask: invalid return type {}", dest.layout.ty), - } - } - "cast" | "as" | "cast_ptr" | "expose_provenance" | "with_exposed_provenance" => { + "expose_provenance" => { let [op] = check_intrinsic_arg_count(args)?; let (op, op_len) = this.project_to_simd(op)?; let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(dest_len, op_len); - let unsafe_cast = intrinsic_name == "cast"; - let safe_cast = intrinsic_name == "as"; - let ptr_cast = intrinsic_name == "cast_ptr"; - let expose_cast = intrinsic_name == "expose_provenance"; - let from_exposed_cast = intrinsic_name == "with_exposed_provenance"; - for i in 0..dest_len { let op = this.read_immediate(&this.project_index(&op, i)?)?; let dest = this.project_index(&dest, i)?; let val = match (op.layout.ty.kind(), dest.layout.ty.kind()) { - // Int-to-(int|float): always safe - (ty::Int(_) | ty::Uint(_), ty::Int(_) | ty::Uint(_) | ty::Float(_)) - if safe_cast || unsafe_cast => - this.int_to_int_or_float(&op, dest.layout)?, - // Float-to-float: always safe - (ty::Float(_), ty::Float(_)) if safe_cast || unsafe_cast => - this.float_to_float_or_int(&op, dest.layout)?, - // Float-to-int in safe mode - (ty::Float(_), ty::Int(_) | ty::Uint(_)) if safe_cast => - this.float_to_float_or_int(&op, dest.layout)?, - // Float-to-int in unchecked mode - (ty::Float(_), ty::Int(_) | ty::Uint(_)) if unsafe_cast => { - this.float_to_int_checked(&op, dest.layout, Round::TowardZero)? - .ok_or_else(|| { - err_ub_format!( - "`simd_cast` intrinsic called on {op} which cannot be represented in target type `{:?}`", - dest.layout.ty - ) - })? - } - // Ptr-to-ptr cast - (ty::RawPtr(..), ty::RawPtr(..)) if ptr_cast => - this.ptr_to_ptr(&op, dest.layout)?, // Ptr/Int casts - (ty::RawPtr(..), ty::Int(_) | ty::Uint(_)) if expose_cast => + (ty::RawPtr(..), ty::Int(_) | ty::Uint(_)) => this.pointer_expose_provenance_cast(&op, dest.layout)?, - (ty::Int(_) | ty::Uint(_), ty::RawPtr(..)) if from_exposed_cast => - this.pointer_with_exposed_provenance_cast(&op, dest.layout)?, // Error otherwise _ => throw_unsup_format!( - "Unsupported SIMD cast from element type {from_ty} to {to_ty}", + "Unsupported `simd_expose_provenance` from element type {from_ty} to {to_ty}", from_ty = op.layout.ty, to_ty = dest.layout.ty, ), @@ -625,210 +165,9 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { this.write_immediate(*val, &dest)?; } } - "shuffle_const_generic" => { - let [left, right] = check_intrinsic_arg_count(args)?; - let (left, left_len) = this.project_to_simd(left)?; - let (right, right_len) = this.project_to_simd(right)?; - let (dest, dest_len) = this.project_to_simd(dest)?; - - let index = generic_args[2].expect_const().to_value().valtree.unwrap_branch(); - let index_len = index.len(); - - assert_eq!(left_len, right_len); - assert_eq!(u64::try_from(index_len).unwrap(), dest_len); - - for i in 0..dest_len { - let src_index: u64 = - index[usize::try_from(i).unwrap()].unwrap_leaf().to_u32().into(); - let dest = this.project_index(&dest, i)?; - - let val = if src_index < left_len { - this.read_immediate(&this.project_index(&left, src_index)?)? - } else if src_index < left_len.strict_add(right_len) { - let right_idx = src_index.strict_sub(left_len); - this.read_immediate(&this.project_index(&right, right_idx)?)? - } else { - throw_ub_format!( - "`simd_shuffle_const_generic` index {src_index} is out-of-bounds for 2 vectors with length {dest_len}" - ); - }; - this.write_immediate(*val, &dest)?; - } - } - "shuffle" => { - let [left, right, index] = check_intrinsic_arg_count(args)?; - let (left, left_len) = this.project_to_simd(left)?; - let (right, right_len) = this.project_to_simd(right)?; - let (index, index_len) = this.project_to_simd(index)?; - let (dest, dest_len) = this.project_to_simd(dest)?; - - assert_eq!(left_len, right_len); - assert_eq!(index_len, dest_len); - - for i in 0..dest_len { - let src_index: u64 = this - .read_immediate(&this.project_index(&index, i)?)? - .to_scalar() - .to_u32()? - .into(); - let dest = this.project_index(&dest, i)?; - - let val = if src_index < left_len { - this.read_immediate(&this.project_index(&left, src_index)?)? - } else if src_index < left_len.strict_add(right_len) { - let right_idx = src_index.strict_sub(left_len); - this.read_immediate(&this.project_index(&right, right_idx)?)? - } else { - throw_ub_format!( - "`simd_shuffle` index {src_index} is out-of-bounds for 2 vectors with length {dest_len}" - ); - }; - this.write_immediate(*val, &dest)?; - } - } - "gather" => { - let [passthru, ptrs, mask] = check_intrinsic_arg_count(args)?; - let (passthru, passthru_len) = this.project_to_simd(passthru)?; - let (ptrs, ptrs_len) = this.project_to_simd(ptrs)?; - let (mask, mask_len) = this.project_to_simd(mask)?; - let (dest, dest_len) = this.project_to_simd(dest)?; - - assert_eq!(dest_len, passthru_len); - assert_eq!(dest_len, ptrs_len); - assert_eq!(dest_len, mask_len); - - for i in 0..dest_len { - let passthru = this.read_immediate(&this.project_index(&passthru, i)?)?; - let ptr = this.read_immediate(&this.project_index(&ptrs, i)?)?; - let mask = this.read_immediate(&this.project_index(&mask, i)?)?; - let dest = this.project_index(&dest, i)?; - - let val = if simd_element_to_bool(mask)? { - let place = this.deref_pointer(&ptr)?; - this.read_immediate(&place)? - } else { - passthru - }; - this.write_immediate(*val, &dest)?; - } - } - "scatter" => { - let [value, ptrs, mask] = check_intrinsic_arg_count(args)?; - let (value, value_len) = this.project_to_simd(value)?; - let (ptrs, ptrs_len) = this.project_to_simd(ptrs)?; - let (mask, mask_len) = this.project_to_simd(mask)?; - - assert_eq!(ptrs_len, value_len); - assert_eq!(ptrs_len, mask_len); - - for i in 0..ptrs_len { - let value = this.read_immediate(&this.project_index(&value, i)?)?; - let ptr = this.read_immediate(&this.project_index(&ptrs, i)?)?; - let mask = this.read_immediate(&this.project_index(&mask, i)?)?; - - if simd_element_to_bool(mask)? { - let place = this.deref_pointer(&ptr)?; - this.write_immediate(*value, &place)?; - } - } - } - "masked_load" => { - let [mask, ptr, default] = check_intrinsic_arg_count(args)?; - let (mask, mask_len) = this.project_to_simd(mask)?; - let ptr = this.read_pointer(ptr)?; - let (default, default_len) = this.project_to_simd(default)?; - let (dest, dest_len) = this.project_to_simd(dest)?; - - assert_eq!(dest_len, mask_len); - assert_eq!(dest_len, default_len); - - for i in 0..dest_len { - let mask = this.read_immediate(&this.project_index(&mask, i)?)?; - let default = this.read_immediate(&this.project_index(&default, i)?)?; - let dest = this.project_index(&dest, i)?; - - let val = if simd_element_to_bool(mask)? { - // Size * u64 is implemented as always checked - let ptr = ptr.wrapping_offset(dest.layout.size * i, this); - let place = this.ptr_to_mplace(ptr, dest.layout); - this.read_immediate(&place)? - } else { - default - }; - this.write_immediate(*val, &dest)?; - } - } - "masked_store" => { - let [mask, ptr, vals] = check_intrinsic_arg_count(args)?; - let (mask, mask_len) = this.project_to_simd(mask)?; - let ptr = this.read_pointer(ptr)?; - let (vals, vals_len) = this.project_to_simd(vals)?; - - assert_eq!(mask_len, vals_len); - - for i in 0..vals_len { - let mask = this.read_immediate(&this.project_index(&mask, i)?)?; - let val = this.read_immediate(&this.project_index(&vals, i)?)?; - - if simd_element_to_bool(mask)? { - // Size * u64 is implemented as always checked - let ptr = ptr.wrapping_offset(val.layout.size * i, this); - let place = this.ptr_to_mplace(ptr, val.layout); - this.write_immediate(*val, &place)? - }; - } - } _ => return interp_ok(EmulateItemResult::NotSupported), } interp_ok(EmulateItemResult::NeedsReturn) } - - fn fminmax_op( - &self, - op: MinMax, - left: &ImmTy<'tcx>, - right: &ImmTy<'tcx>, - ) -> InterpResult<'tcx, Scalar> { - let this = self.eval_context_ref(); - assert_eq!(left.layout.ty, right.layout.ty); - let ty::Float(float_ty) = left.layout.ty.kind() else { - bug!("fmax operand is not a float") - }; - let left = left.to_scalar(); - let right = right.to_scalar(); - interp_ok(match float_ty { - FloatTy::F16 => unimplemented!("f16_f128"), - FloatTy::F32 => { - let left = left.to_f32()?; - let right = right.to_f32()?; - let res = match op { - MinMax::Min => left.min(right), - MinMax::Max => left.max(right), - }; - let res = this.adjust_nan(res, &[left, right]); - Scalar::from_f32(res) - } - FloatTy::F64 => { - let left = left.to_f64()?; - let right = right.to_f64()?; - let res = match op { - MinMax::Min => left.min(right), - MinMax::Max => left.max(right), - }; - let res = this.adjust_nan(res, &[left, right]); - Scalar::from_f64(res) - } - FloatTy::F128 => unimplemented!("f16_f128"), - }) - } -} - -fn simd_bitmask_index(idx: u32, vec_len: u32, endianness: Endian) -> u32 { - assert!(idx < vec_len); - match endianness { - Endian::Little => idx, - #[expect(clippy::arithmetic_side_effects)] // idx < vec_len - Endian::Big => vec_len - 1 - idx, // reverse order of bits - } } diff --git a/src/tools/rustdoc-gui-test/Cargo.toml b/src/tools/rustdoc-gui-test/Cargo.toml index 8d958ac94f30e..f7384a98f8565 100644 --- a/src/tools/rustdoc-gui-test/Cargo.toml +++ b/src/tools/rustdoc-gui-test/Cargo.toml @@ -5,7 +5,6 @@ edition = "2021" [dependencies] build_helper = { path = "../../build_helper" } -camino = "1" compiletest = { path = "../compiletest" } getopts = "0.2" walkdir = "2" diff --git a/src/tools/rustdoc-gui-test/src/main.rs b/src/tools/rustdoc-gui-test/src/main.rs index 5062c2597f15e..cd10882fc2269 100644 --- a/src/tools/rustdoc-gui-test/src/main.rs +++ b/src/tools/rustdoc-gui-test/src/main.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use build_helper::npm; use build_helper::util::try_run; -use compiletest::directives::TestProps; +use compiletest::rustdoc_gui_test::RustdocGuiTestProps; use config::Config; mod config; @@ -43,13 +43,7 @@ fn main() -> Result<(), ()> { .current_dir(path); if let Some(librs) = find_librs(entry.path()) { - let compiletest_c = compiletest::common::Config::incomplete_for_rustdoc_gui_test(); - - let test_props = TestProps::from_file( - &camino::Utf8PathBuf::try_from(librs).unwrap(), - None, - &compiletest_c, - ); + let test_props = RustdocGuiTestProps::from_file(&librs); if !test_props.compile_flags.is_empty() { cargo.env("RUSTDOCFLAGS", test_props.compile_flags.join(" ")); diff --git a/tests/codegen-llvm/lib-optimizations/slice_fill.rs b/tests/codegen-llvm/lib-optimizations/slice_fill.rs new file mode 100644 index 0000000000000..2d924ebf726d8 --- /dev/null +++ b/tests/codegen-llvm/lib-optimizations/slice_fill.rs @@ -0,0 +1,28 @@ +//@ compile-flags: -Copt-level=3 +#![crate_type = "lib"] + +use std::mem::MaybeUninit; + +// CHECK-LABEL: @slice_fill_pass_undef +#[no_mangle] +pub fn slice_fill_pass_undef(s: &mut [MaybeUninit], v: MaybeUninit) { + // CHECK: tail call void @llvm.memset.{{.*}}(ptr nonnull align 1 %s.0, i8 %v, {{.*}} %s.1, i1 false) + // CHECK: ret + s.fill(v); +} + +// CHECK-LABEL: @slice_fill_uninit +#[no_mangle] +pub fn slice_fill_uninit(s: &mut [MaybeUninit]) { + // CHECK-NOT: call + // CHECK: ret void + s.fill(MaybeUninit::uninit()); +} + +// CHECK-LABEL: @slice_wide_memset +#[no_mangle] +pub fn slice_wide_memset(s: &mut [u16]) { + // CHECK: tail call void @llvm.memset.{{.*}}(ptr nonnull align 2 %s.0, i8 -1 + // CHECK: ret + s.fill(0xFFFF); +} diff --git a/tests/ui/attributes/no-mangle-closure.rs b/tests/ui/attributes/no-mangle-closure.rs index c76baa27f38a0..05dcbf8e5bed7 100644 --- a/tests/ui/attributes/no-mangle-closure.rs +++ b/tests/ui/attributes/no-mangle-closure.rs @@ -7,5 +7,5 @@ pub struct S([usize; 8]); pub fn outer_function(x: S, y: S) -> usize { (#[no_mangle] || y.0[0])() - //~^ ERROR `#[no_mangle]` cannot be used on a closure as it has no name + //~^ ERROR `#[no_mangle]` attribute cannot be used on closures } diff --git a/tests/ui/attributes/no-mangle-closure.stderr b/tests/ui/attributes/no-mangle-closure.stderr index c183783493bdb..f81e65d926752 100644 --- a/tests/ui/attributes/no-mangle-closure.stderr +++ b/tests/ui/attributes/no-mangle-closure.stderr @@ -1,8 +1,10 @@ -error: `#[no_mangle]` cannot be used on a closure as it has no name +error: `#[no_mangle]` attribute cannot be used on closures --> $DIR/no-mangle-closure.rs:9:6 | LL | (#[no_mangle] || y.0[0])() | ^^^^^^^^^^^^ + | + = help: `#[no_mangle]` can be applied to functions, methods, and statics error: aborting due to 1 previous error diff --git a/tests/ui/issues/issue-45562.fixed b/tests/ui/issues/issue-45562.fixed index 8dcdd3a541ce4..529b5bd744e03 100644 --- a/tests/ui/issues/issue-45562.fixed +++ b/tests/ui/issues/issue-45562.fixed @@ -1,5 +1,7 @@ //@ run-rustfix +#![deny(unused_attributes)] + #[no_mangle] pub static RAH: usize = 5; //~^ ERROR const items should never be `#[no_mangle]` diff --git a/tests/ui/issues/issue-45562.rs b/tests/ui/issues/issue-45562.rs index 08f6c8046dce4..7c30a967c7848 100644 --- a/tests/ui/issues/issue-45562.rs +++ b/tests/ui/issues/issue-45562.rs @@ -1,5 +1,7 @@ //@ run-rustfix +#![deny(unused_attributes)] + #[no_mangle] pub const RAH: usize = 5; //~^ ERROR const items should never be `#[no_mangle]` diff --git a/tests/ui/issues/issue-45562.stderr b/tests/ui/issues/issue-45562.stderr index 6fae86f9f31ca..55d35f76a0198 100644 --- a/tests/ui/issues/issue-45562.stderr +++ b/tests/ui/issues/issue-45562.stderr @@ -1,5 +1,5 @@ error: const items should never be `#[no_mangle]` - --> $DIR/issue-45562.rs:3:14 + --> $DIR/issue-45562.rs:5:14 | LL | #[no_mangle] pub const RAH: usize = 5; | ---------^^^^^^^^^^^^^^^^