Skip to content

Port the remaining SIMD intrinsics to const-eval #147520

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
+181 −216
Open

Port the remaining SIMD intrinsics to const-eval #147520

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
effe469
Implement `simd_fma` and `simd_relaxed_fma` in const-eval
sayantn Oct 9, 2025
c1878f3
Refactor implementation of float minmax intrinsics
sayantn Oct 9, 2025
69a2513
Add f16-f128 support for SIMD intrinsics
sayantn Oct 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
215 changes: 117 additions & 98 deletions compiler/rustc_const_eval/src/interpret/intrinsics.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,27 @@ use super::{
};
use crate::fluent_generated as fluent;

#[derive(Copy, Clone, Debug, PartialEq, Eq)]
enum MulAddType {
/// Used with `fma` and `simd_fma`, always uses fused-multiply-add
Fused,
/// Used with `fmuladd` and `simd_relaxed_fma`, nondeterministically determines whether to use
/// fma or simple multiply-add
Nondeterministic,
}

#[derive(Copy, Clone)]
pub(crate) enum MinMax {
/// The IEEE `Minimum` operation - see `f32::minimum` etc
Minimum,
/// The IEEE `MinNum` operation - see `f32::min` etc
Minnum,
/// The IEEE `Maximum` operation - see `f32::maximum` etc
Maximum,
/// The IEEE `MaxNum` operation - see `f32::max` etc
Maxnum,
Comment on lines +39 to +46
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
/// The IEEE `Minimum` operation - see `f32::minimum` etc
Minimum,
/// The IEEE `MinNum` operation - see `f32::min` etc
Minnum,
/// The IEEE `Maximum` operation - see `f32::maximum` etc
Maximum,
/// The IEEE `MaxNum` operation - see `f32::max` etc
Maxnum,
/// The IEEE `Minimum` operation - see `f32::minimum` etc.
/// In particular, `-0.0` is considered smaller than `+0.0`.
Minimum,
/// The IEEE `MinNum` operation - see `f32::min` etc.
/// In particular, if the inputs are `-0.0` and `+0.0`, the result is non-deterministic.
Minnum,
/// The IEEE `Maximum` operation - see `f32::maximum` etc.
/// In particular, `-0.0` is considered smaller than `+0.0`.
Maximum,
/// The IEEE `MaxNum` operation - see `f32::max` etc.
/// In particular, if the inputs are `-0.0` and `+0.0`, the result is non-deterministic.
Maxnum,

}

/// Directly returns an `Allocation` containing an absolute path representation of the given type.
pub(crate) fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> (AllocId, u64) {
let path = crate::util::type_name(tcx, ty);
Expand Down Expand Up @@ -486,25 +507,33 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
self.write_scalar(Scalar::from_target_usize(align.bytes(), self), dest)?;
}

sym::minnumf16 => self.float_min_intrinsic::<Half>(args, dest)?,
sym::minnumf32 => self.float_min_intrinsic::<Single>(args, dest)?,
sym::minnumf64 => self.float_min_intrinsic::<Double>(args, dest)?,
sym::minnumf128 => self.float_min_intrinsic::<Quad>(args, dest)?,
sym::minnumf16 => self.float_minmax_intrinsic::<Half>(args, MinMax::Minnum, dest)?,
sym::minnumf32 => self.float_minmax_intrinsic::<Single>(args, MinMax::Minnum, dest)?,
sym::minnumf64 => self.float_minmax_intrinsic::<Double>(args, MinMax::Minnum, dest)?,
sym::minnumf128 => self.float_minmax_intrinsic::<Quad>(args, MinMax::Minnum, dest)?,

sym::minimumf16 => self.float_minimum_intrinsic::<Half>(args, dest)?,
sym::minimumf32 => self.float_minimum_intrinsic::<Single>(args, dest)?,
sym::minimumf64 => self.float_minimum_intrinsic::<Double>(args, dest)?,
sym::minimumf128 => self.float_minimum_intrinsic::<Quad>(args, dest)?,
sym::minimumf16 => self.float_minmax_intrinsic::<Half>(args, MinMax::Minimum, dest)?,
sym::minimumf32 => {
self.float_minmax_intrinsic::<Single>(args, MinMax::Minimum, dest)?
}
sym::minimumf64 => {
self.float_minmax_intrinsic::<Double>(args, MinMax::Minimum, dest)?
}
sym::minimumf128 => self.float_minmax_intrinsic::<Quad>(args, MinMax::Minimum, dest)?,

sym::maxnumf16 => self.float_max_intrinsic::<Half>(args, dest)?,
sym::maxnumf32 => self.float_max_intrinsic::<Single>(args, dest)?,
sym::maxnumf64 => self.float_max_intrinsic::<Double>(args, dest)?,
sym::maxnumf128 => self.float_max_intrinsic::<Quad>(args, dest)?,
sym::maxnumf16 => self.float_minmax_intrinsic::<Half>(args, MinMax::Maxnum, dest)?,
sym::maxnumf32 => self.float_minmax_intrinsic::<Single>(args, MinMax::Maxnum, dest)?,
sym::maxnumf64 => self.float_minmax_intrinsic::<Double>(args, MinMax::Maxnum, dest)?,
sym::maxnumf128 => self.float_minmax_intrinsic::<Quad>(args, MinMax::Maxnum, dest)?,

sym::maximumf16 => self.float_maximum_intrinsic::<Half>(args, dest)?,
sym::maximumf32 => self.float_maximum_intrinsic::<Single>(args, dest)?,
sym::maximumf64 => self.float_maximum_intrinsic::<Double>(args, dest)?,
sym::maximumf128 => self.float_maximum_intrinsic::<Quad>(args, dest)?,
sym::maximumf16 => self.float_minmax_intrinsic::<Half>(args, MinMax::Maximum, dest)?,
sym::maximumf32 => {
self.float_minmax_intrinsic::<Single>(args, MinMax::Maximum, dest)?
}
sym::maximumf64 => {
self.float_minmax_intrinsic::<Double>(args, MinMax::Maximum, dest)?
}
sym::maximumf128 => self.float_minmax_intrinsic::<Quad>(args, MinMax::Maximum, dest)?,

sym::copysignf16 => self.float_copysign_intrinsic::<Half>(args, dest)?,
sym::copysignf32 => self.float_copysign_intrinsic::<Single>(args, dest)?,
Expand Down Expand Up @@ -612,14 +641,22 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
dest,
rustc_apfloat::Round::NearestTiesToEven,
)?,
sym::fmaf16 => self.fma_intrinsic::<Half>(args, dest)?,
sym::fmaf32 => self.fma_intrinsic::<Single>(args, dest)?,
sym::fmaf64 => self.fma_intrinsic::<Double>(args, dest)?,
sym::fmaf128 => self.fma_intrinsic::<Quad>(args, dest)?,
sym::fmuladdf16 => self.float_muladd_intrinsic::<Half>(args, dest)?,
sym::fmuladdf32 => self.float_muladd_intrinsic::<Single>(args, dest)?,
sym::fmuladdf64 => self.float_muladd_intrinsic::<Double>(args, dest)?,
sym::fmuladdf128 => self.float_muladd_intrinsic::<Quad>(args, dest)?,
sym::fmaf16 => self.float_muladd_intrinsic::<Half>(args, dest, MulAddType::Fused)?,
sym::fmaf32 => self.float_muladd_intrinsic::<Single>(args, dest, MulAddType::Fused)?,
sym::fmaf64 => self.float_muladd_intrinsic::<Double>(args, dest, MulAddType::Fused)?,
sym::fmaf128 => self.float_muladd_intrinsic::<Quad>(args, dest, MulAddType::Fused)?,
sym::fmuladdf16 => {
self.float_muladd_intrinsic::<Half>(args, dest, MulAddType::Nondeterministic)?
}
sym::fmuladdf32 => {
self.float_muladd_intrinsic::<Single>(args, dest, MulAddType::Nondeterministic)?
}
sym::fmuladdf64 => {
self.float_muladd_intrinsic::<Double>(args, dest, MulAddType::Nondeterministic)?
}
sym::fmuladdf128 => {
self.float_muladd_intrinsic::<Quad>(args, dest, MulAddType::Nondeterministic)?
}

// Unsupported intrinsic: skip the return_to_block below.
_ => return interp_ok(false),
Expand Down Expand Up @@ -901,76 +938,45 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
interp_ok(Scalar::from_bool(lhs_bytes == rhs_bytes))
}

fn float_min_intrinsic<F>(
&mut self,
args: &[OpTy<'tcx, M::Provenance>],
dest: &PlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, ()>
where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
{
let a: F = self.read_scalar(&args[0])?.to_float()?;
let b: F = self.read_scalar(&args[1])?.to_float()?;
let res = if a == b {
// They are definitely not NaN (those are never equal), but they could be `+0` and `-0`.
// Let the machine decide which one to return.
M::equal_float_min_max(self, a, b)
} else {
self.adjust_nan(a.min(b), &[a, b])
};
self.write_scalar(res, dest)?;
interp_ok(())
}

fn float_max_intrinsic<F>(
&mut self,
args: &[OpTy<'tcx, M::Provenance>],
dest: &PlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, ()>
fn float_minmax<F>(
&self,
a: Scalar<M::Provenance>,
b: Scalar<M::Provenance>,
op: MinMax,
) -> InterpResult<'tcx, F>
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please make the function return a Scalar -- it's an odd mismatch to have the argument be Scalar but the result be F.

You can add an F: Into<Scalar> bound to make this possible.

where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F>,
{
let a: F = self.read_scalar(&args[0])?.to_float()?;
let b: F = self.read_scalar(&args[1])?.to_float()?;
let res = if a == b {
let a: F = a.to_float()?;
let b: F = b.to_float()?;
let res = if matches!(op, MinMax::Minnum | MinMax::Maxnum) && a == b {
// They are definitely not NaN (those are never equal), but they could be `+0` and `-0`.
// Let the machine decide which one to return.
M::equal_float_min_max(self, a, b)
} else {
self.adjust_nan(a.max(b), &[a, b])
let result = match op {
MinMax::Minimum => a.minimum(b),
MinMax::Minnum => a.min(b),
MinMax::Maximum => a.maximum(b),
MinMax::Maxnum => a.max(b),
};
self.adjust_nan(result, &[a, b])
};
self.write_scalar(res, dest)?;
interp_ok(())
}

fn float_minimum_intrinsic<F>(
&mut self,
args: &[OpTy<'tcx, M::Provenance>],
dest: &PlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, ()>
where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
{
let a: F = self.read_scalar(&args[0])?.to_float()?;
let b: F = self.read_scalar(&args[1])?.to_float()?;
let res = a.minimum(b);
let res = self.adjust_nan(res, &[a, b]);
self.write_scalar(res, dest)?;
interp_ok(())
interp_ok(res)
}

fn float_maximum_intrinsic<F>(
fn float_minmax_intrinsic<F>(
&mut self,
args: &[OpTy<'tcx, M::Provenance>],
op: MinMax,
dest: &PlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, ()>
where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
{
let a: F = self.read_scalar(&args[0])?.to_float()?;
let b: F = self.read_scalar(&args[1])?.to_float()?;
let res = a.maximum(b);
let res = self.adjust_nan(res, &[a, b]);
let res =
self.float_minmax::<F>(self.read_scalar(&args[0])?, self.read_scalar(&args[1])?, op)?;
self.write_scalar(res, dest)?;
interp_ok(())
}
Expand Down Expand Up @@ -1004,6 +1010,20 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
interp_ok(())
}

fn float_round<F>(
&mut self,
x: Scalar<M::Provenance>,
mode: rustc_apfloat::Round,
) -> InterpResult<'tcx, F>
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please make the function return a Scalar as well.

where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F>,
{
let x: F = x.to_float()?;
let res = x.round_to_integral(mode).value;
let res = self.adjust_nan(res, &[x]);
interp_ok(res)
}

fn float_round_intrinsic<F>(
&mut self,
args: &[OpTy<'tcx, M::Provenance>],
Expand All @@ -1013,47 +1033,46 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
{
let x: F = self.read_scalar(&args[0])?.to_float()?;
let res = x.round_to_integral(mode).value;
let res = self.adjust_nan(res, &[x]);
let res = self.float_round::<F>(self.read_scalar(&args[0])?, mode)?;
self.write_scalar(res, dest)?;
interp_ok(())
}

fn fma_intrinsic<F>(
&mut self,
args: &[OpTy<'tcx, M::Provenance>],
dest: &PlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, ()>
fn float_muladd<F>(
&self,
a: Scalar<M::Provenance>,
b: Scalar<M::Provenance>,
c: Scalar<M::Provenance>,
typ: MulAddType,
) -> InterpResult<'tcx, F>
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please make the function return a Scalar as well.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

works, I just though this looked nice because we can use Scalar::from_fX in the callsite instead of using turbofish argument (I find them somewhat ugly)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah the turbofish aren't great here... but the asymmetric signature is also ugly and so I'd rather have the turbofish (we already have that in the intrinsic handling code so it's also more consistent).

where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
{
let a: F = self.read_scalar(&args[0])?.to_float()?;
let b: F = self.read_scalar(&args[1])?.to_float()?;
let c: F = self.read_scalar(&args[2])?.to_float()?;
let a: F = a.to_float()?;
let b: F = b.to_float()?;
let c: F = c.to_float()?;

let fuse = typ == MulAddType::Fused || M::float_fuse_mul_add(self);

let res = a.mul_add(b, c).value;
let res = if fuse { a.mul_add(b, c).value } else { ((a * b).value + c).value };
let res = self.adjust_nan(res, &[a, b, c]);
self.write_scalar(res, dest)?;
interp_ok(())
interp_ok(res)
}

fn float_muladd_intrinsic<F>(
&mut self,
args: &[OpTy<'tcx, M::Provenance>],
dest: &PlaceTy<'tcx, M::Provenance>,
typ: MulAddType,
) -> InterpResult<'tcx, ()>
where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
{
let a: F = self.read_scalar(&args[0])?.to_float()?;
let b: F = self.read_scalar(&args[1])?.to_float()?;
let c: F = self.read_scalar(&args[2])?.to_float()?;

let fuse = M::float_fuse_mul_add(self);
let a = self.read_scalar(&args[0])?;
let b = self.read_scalar(&args[1])?;
let c = self.read_scalar(&args[2])?;

let res = if fuse { a.mul_add(b, c).value } else { ((a * b).value + c).value };
let res = self.adjust_nan(res, &[a, b, c]);
let res = self.float_muladd::<F>(a, b, c, typ)?;
self.write_scalar(res, dest)?;
interp_ok(())
}
Expand Down
Loading
Loading