From 2b2ad090f63bba9a261d572238af2706b32b8aa9 Mon Sep 17 00:00:00 2001
From: Daniel McNab <36049421+DJMcNab@users.noreply.github.com>
Date: Tue, 14 Oct 2025 12:42:54 +0100
Subject: [PATCH 01/19] Save the version before running the x86 generator
---
Cargo.lock | 15 +-
Cargo.toml | 2 +
fearless_simd_core/Cargo.toml | 17 +
fearless_simd_core/gen/Cargo.toml | 6 +
fearless_simd_core/gen/src/data.rs | 2 +
fearless_simd_core/gen/src/data/x86.rs | 370 ++++++++++++++++++++
fearless_simd_core/gen/src/main.rs | 166 +++++++++
fearless_simd_core/gen/templates/aarch64.rs | 0
fearless_simd_core/gen/templates/x86.rs | 90 +++++
fearless_simd_core/src/lib.rs | 282 +++++++++++++++
fearless_simd_core/src/trampoline.rs | 231 ++++++++++++
fearless_simd_core/src/x86/mod.rs | 15 +
fearless_simd_core/src/x86/v1/fxsr.rs | 80 +++++
fearless_simd_core/src/x86/v1/mod.rs | 38 ++
fearless_simd_core/src/x86/v1/sse.rs | 90 +++++
15 files changed, 1402 insertions(+), 2 deletions(-)
create mode 100644 fearless_simd_core/Cargo.toml
create mode 100644 fearless_simd_core/gen/Cargo.toml
create mode 100644 fearless_simd_core/gen/src/data.rs
create mode 100644 fearless_simd_core/gen/src/data/x86.rs
create mode 100644 fearless_simd_core/gen/src/main.rs
create mode 100644 fearless_simd_core/gen/templates/aarch64.rs
create mode 100644 fearless_simd_core/gen/templates/x86.rs
create mode 100644 fearless_simd_core/src/lib.rs
create mode 100644 fearless_simd_core/src/trampoline.rs
create mode 100644 fearless_simd_core/src/x86/mod.rs
create mode 100644 fearless_simd_core/src/x86/v1/fxsr.rs
create mode 100644 fearless_simd_core/src/x86/v1/mod.rs
create mode 100644 fearless_simd_core/src/x86/v1/sse.rs
diff --git a/Cargo.lock b/Cargo.lock
index 161950a6..672a0913 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -60,9 +60,9 @@ checksum = "793db76d6187cd04dff33004d8e6c9cc4e05cd330500379d2394209271b4aeee"
[[package]]
name = "bytemuck"
-version = "1.23.1"
+version = "1.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c76a5792e44e4abe34d3abf15636779261d45a7450612059293d1d2cfc63422"
+checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4"
[[package]]
name = "cc"
@@ -133,6 +133,17 @@ dependencies = [
"libm",
]
+[[package]]
+name = "fearless_simd_core"
+version = "0.1.0"
+dependencies = [
+ "bytemuck",
+]
+
+[[package]]
+name = "fearless_simd_core_gen"
+version = "0.1.0"
+
[[package]]
name = "fearless_simd_dev_macros"
version = "0.1.0"
diff --git a/Cargo.toml b/Cargo.toml
index 81395978..e84d0a2e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -2,6 +2,8 @@
resolver = "2"
members = [
"fearless_simd",
+ "fearless_simd_core",
+ "fearless_simd_core/gen",
"fearless_simd_dev_macros",
"fearless_simd_gen",
"fearless_simd_tests",
diff --git a/fearless_simd_core/Cargo.toml b/fearless_simd_core/Cargo.toml
new file mode 100644
index 00000000..e16a9823
--- /dev/null
+++ b/fearless_simd_core/Cargo.toml
@@ -0,0 +1,17 @@
+[package]
+name = "fearless_simd_core"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+
+[dev-dependencies]
+bytemuck = { version = "1.24.0", features = ["must_cast"] }
+
+[lints]
+workspace = true
+
+[features]
+default = ["std"]
+std = []
diff --git a/fearless_simd_core/gen/Cargo.toml b/fearless_simd_core/gen/Cargo.toml
new file mode 100644
index 00000000..5617f7be
--- /dev/null
+++ b/fearless_simd_core/gen/Cargo.toml
@@ -0,0 +1,6 @@
+[package]
+name = "fearless_simd_core_gen"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
diff --git a/fearless_simd_core/gen/src/data.rs b/fearless_simd_core/gen/src/data.rs
new file mode 100644
index 00000000..87b9ec46
--- /dev/null
+++ b/fearless_simd_core/gen/src/data.rs
@@ -0,0 +1,2 @@
+mod x86;
+pub(crate) use x86::{X86_FEATURES, X86_TEMPLATE};
diff --git a/fearless_simd_core/gen/src/data/x86.rs b/fearless_simd_core/gen/src/data/x86.rs
new file mode 100644
index 00000000..39e59990
--- /dev/null
+++ b/fearless_simd_core/gen/src/data/x86.rs
@@ -0,0 +1,370 @@
+use crate::Feature;
+
+macro_rules! f {
+ ($(#[doc = $doc_addition: literal])*
+ struct ::$module: ident:: $struct_name: ident($display_name: literal): $feature_name: literal + [$($implicitly_enabled: literal),*]
+ fn $example_function_name: ident
+ $($additional_impls: tt)*
+ ) => {
+ Feature {
+ struct_name: stringify!($struct_name),
+ feature_name: $feature_name,
+ directly_implicitly_enabled: &[$($implicitly_enabled),*],
+ extra_docs: concat!($($doc_addition, "\n",)*),
+ example_function_name: stringify!($example_function_name),
+ feature_docs_name: $display_name,
+ additional_impls: stringify!($($additional_impls)*),
+ module: stringify!($module)
+ }
+ }
+}
+
+pub(crate) const X86_TEMPLATE: &str = include_str!("../../templates/x86.rs");
+
+// Data taken from: https://doc.rust-lang.org/reference/attributes/codegen.html#r-attributes.codegen.target_feature.x86
+// (specifically, at https://github.com/rust-lang/reference/blob/1d930e1d5a27e114b4d22a50b0b6cd3771b92e31/src/attributes/codegen.md#x86-or-x86_64)
+// TODO: Do we need to add their license attribution to our license?
+// TODO: Check set against https://doc.rust-lang.org/stable/std/macro.is_x86_feature_detected.html
+// In particular, we're missing lahfsahf
+pub(crate) const X86_FEATURES: &[Feature] = &[
+ f!(
+ /// [ADX] --- Multi-Precision Add-Carry Instruction Extensions
+ /// [ADX]: https://en.wikipedia.org/wiki/Intel_ADX
+ struct ::adx::Adx("ADX"): "adx" + []
+ fn uses_adx
+ ),
+ f!(
+ /// [AES] --- Advanced Encryption Standard
+ /// [AES]: https://en.wikipedia.org/wiki/AES_instruction_set
+ struct ::crypto::Aes("AES"): "aes" + ["sse2"]
+ fn uses_aes
+ ),
+ f!(
+ /// [AVX] --- Advanced Vector Extensions
+ /// [AVX]: https://en.wikipedia.org/wiki/Advanced_Vector_Extensions
+ struct ::avx::Avx("AVX"): "avx" + ["sse4.2"]
+ fn uses_avx
+ ),
+ f!(
+ /// [AVX2] --- Advanced Vector Extensions 2
+ /// [AVX2]: https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#AVX2
+ struct ::avx::Avx2("AVX2"): "avx2" + ["avx"]
+ fn uses_avx2
+ ),
+ f!(
+ /// [AVX512-BF16] --- Advanced Vector Extensions 512-bit - Bfloat16 Extensions
+ /// [AVX512-BF16]: https://en.wikipedia.org/wiki/AVX-512#BF16
+ struct ::avx512::Avx512bf16("AVX512-BF16"): "avx512bf16" + ["avx512bw"]
+ fn uses_avx512bf16
+ ),
+ f!(
+ /// [AVX512-BITALG] --- Advanced Vector Extensions 512-bit - Bit Algorithms
+ /// [AVX512-BITALG]: https://en.wikipedia.org/wiki/AVX-512#VPOPCNTDQ_and_BITALG
+ struct ::avx512::Avx512bitalg("AVX512-BITALG"): "avx512bitalg" + ["avx512bw"]
+ fn uses_avx512bitalg
+ ),
+ f!(
+ /// [AVX512-BW] --- Advanced Vector Extensions 512-bit - Byte and Word Instructions
+ /// [AVX512-BW]: https://en.wikipedia.org/wiki/AVX-512#BW,_DQ_and_VBMI
+ struct ::avx512::Avx512bw("AVX512-BW"): "avx512bw" + ["avx512f"]
+ fn uses_avx512bw
+ ),
+ f!(
+ /// [AVX512-CD] --- Advanced Vector Extensions 512-bit - Conflict Detection Instructions
+ /// [AVX512-CD]: https://en.wikipedia.org/wiki/AVX-512#Conflict_detection
+ struct ::avx512::Avx512cd("AVX512-CD"): "avx512cd" + ["avx512f"]
+ fn uses_avx512cd
+ ),
+ f!(
+ /// [AVX512-DQ] --- Advanced Vector Extensions 512-bit - Doubleword and Quadword Instructions
+ /// [AVX512-DQ]: https://en.wikipedia.org/wiki/AVX-512#BW,_DQ_and_VBMI
+ struct ::avx512::Avx512dq("AVX512-DQ"): "avx512dq" + ["avx512f"]
+ fn uses_avx512dq
+ ),
+ f!(
+ /// [AVX512-F] --- Advanced Vector Extensions 512-bit - Foundation
+ /// [AVX512-F]: https://en.wikipedia.org/wiki/AVX-512
+ struct ::avx512::Avx512f("AVX512-F"): "avx512f" + ["avx2", "fma", "f16c"]
+ fn uses_avx512f
+ ),
+ f!(
+ /// [AVX512-FP16] --- Advanced Vector Extensions 512-bit - Float16 Extensions
+ /// [AVX512-FP16]: https://en.wikipedia.org/wiki/AVX-512#FP16
+ struct ::avx512::Avx512fp16("AVX512-FP16"): "avx512fp16" + ["avx512bw"]
+ fn uses_avx512fp16
+ ),
+ f!(
+ /// [AVX512-IFMA] --- Advanced Vector Extensions 512-bit - Integer Fused Multiply Add
+ /// [AVX512-IFMA]: https://en.wikipedia.org/wiki/AVX-512#IFMA
+ struct ::avx512::Avx512ifma("AVX512-IFMA"): "avx512ifma" + ["avx512f"]
+ fn uses_avx512ifma
+ ),
+ f!(
+ /// [AVX512-VBMI] --- Advanced Vector Extensions 512-bit - Vector Byte Manipulation Instructions
+ /// [AVX512-VBMI]: https://en.wikipedia.org/wiki/AVX-512#BW,_DQ_and_VBMI
+ struct ::avx512::Avx512vbmi("AVX512-VBMI"): "avx512vbmi" + ["avx512bw"]
+ fn uses_avx512vbmi
+ ),
+ f!(
+ /// [AVX512-VBMI2] --- Advanced Vector Extensions 512-bit - Vector Byte Manipulation Instructions 2
+ /// [AVX512-VBMI2]: https://en.wikipedia.org/wiki/AVX-512#VBMI2
+ struct ::avx512::Avx512vbmi2("AVX512-VBMI2"): "avx512vbmi2" + ["avx512bw"]
+ fn uses_avx512vbmi2
+ ),
+ f!(
+ /// [AVX512-VL] --- Advanced Vector Extensions 512-bit - Vector Length Extensions
+ /// [AVX512-VL]: https://en.wikipedia.org/wiki/AVX-512
+ struct ::avx512::Avx512vl("AVX512-VL"): "avx512vl" + ["avx512f"]
+ fn uses_avx512vl
+ ),
+ f!(
+ /// [AVX512-VNNI] --- Advanced Vector Extensions 512-bit - Vector Neural Network Instructions
+ /// [AVX512-VNNI]: https://en.wikipedia.org/wiki/AVX-512#VNNI
+ struct ::avx512::Avx512vnni("AVX512-VNNI"): "avx512vnni" + ["avx512f"]
+ fn uses_avx512vnni
+ ),
+ f!(
+ /// [AVX512-VP2INTERSECT] --- Advanced Vector Extensions 512-bit - Vector Pair Intersection to a Pair of Mask Registers
+ /// [AVX512-VP2INTERSECT]: https://en.wikipedia.org/wiki/AVX-512#VP2INTERSECT
+ struct ::avx512::Avx512vp2intersect("AVX512-VP2INTERSECT"): "avx512vp2intersect" + ["avx512f"]
+ fn uses_avx512vp2intersect
+ ),
+ f!(
+ /// [AVX512-VPOPCNTDQ] --- Advanced Vector Extensions 512-bit - Vector Population Count Instruction
+ /// [AVX512-VPOPCNTDQ]:https://en.wikipedia.org/wiki/AVX-512#VPOPCNTDQ_and_BITALG
+ struct ::avx512::Avx512vpopcntdq("AVX512-VPOPCNTDQ"): "avx512vpopcntdq" + ["avx512f"]
+ fn uses_avx512vpopcntdq
+ ),
+ f!(
+ /// [AVX-IFMA] --- Advanced Vector Extensions - Integer Fused Multiply Add
+ /// [AVX-IFMA]: https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#AVX-VNNI,_AVX-IFMA
+ struct ::avx::Avxifma("AVX-IFMA"): "avxifma" + ["avx2"]
+ fn uses_avxifma
+ ),
+ f!(
+ /// [AVX-NE-CONVERT] --- Advanced Vector Extensions - No-Exception Floating-Point conversion Instructions
+ /// [AVX-NE-CONVERT]: https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#AVX-VNNI,_AVX-IFMA
+ struct ::avx::Avxneconvert("AVX-NE-CONVERT"): "avxneconvert" + ["avx2"]
+ fn uses_avxneconvert
+ ),
+ f!(
+ /// [AVX-VNNI] --- Advanced Vector Extensions - Vector Neural Network Instructions
+ /// [AVX-VNNI]: https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#AVX-VNNI,_AVX-IFMA
+ struct ::avx::Avxvnni("AVX-VNNI"): "avxvnni" + ["avx2"]
+ fn uses_avxvnni
+ ),
+ f!(
+ /// [AVX-VNNI-INT16] --- Advanced Vector Extensions - Vector Neural Network Instructions with 16-bit Integers
+ /// [AVX-VNNI-INT16]: https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#AVX-VNNI,_AVX-IFMA
+ struct ::avx::Avxvnniint16("AVX-VNNI-INT16"): "avxvnniint16" + ["avx2"]
+ fn uses_avxvnniint16
+ ),
+ f!(
+ /// [AVX-VNNI-INT8] --- Advanced Vector Extensions - Vector Neural Network Instructions with 8-bit Integers
+ /// [AVX-VNNI-INT8]: https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#AVX-VNNI,_AVX-IFMA
+ struct ::avx::Avxvnniint8("AVX-VNNI-INT8"): "avxvnniint8" + ["avx2"]
+ fn uses_avxvnniint8
+ ),
+ f!(
+ /// [BMI1] --- Bit Manipulation Instruction Sets
+ /// [BMI1]: https://en.wikipedia.org/wiki/Bit_Manipulation_Instruction_Sets
+ struct ::v3::Bmi1(" 1"): "bmi1" + []
+ fn uses_bmi1
+ ),
+ f!(
+ /// [BMI2] --- Bit Manipulation Instruction Sets 2
+ /// [BMI2]: https://en.wikipedia.org/wiki/Bit_Manipulation_Instruction_Sets#BMI2
+ struct ::v3::Bmi2("BMI2"): "bmi2" + []
+ fn uses_bmi2
+ ),
+ f!(
+ /// ["cmpxchg16b"] --- Compares and exchange 16 bytes (128 bits) of data atomically
+ /// ["cmpxchg16b"]: https://www.felixcloutier.com/x86/cmpxchg8b:cmpxchg16b
+ struct ::v2::Cmpxchg16b("`cmpxchg16b`"): "cmpxchg16b" + []
+ fn uses_cmpxchg16b
+ ),
+ f!(
+ /// [F16C] --- 16-bit floating point conversion instructions
+ /// [F16C]: https://en.wikipedia.org/wiki/F16C
+ struct ::v3::F16c("F16C"): "f16c" + ["avx"]
+ fn uses_f16c
+ ),
+ f!(
+ /// [FMA3] --- Three-operand fused multiply-add
+ /// [FMA3]: https://en.wikipedia.org/wiki/FMA_instruction_set
+ struct ::v3::Fma("FMA3"): "fma" + ["avx"]
+ fn uses_fma
+ ),
+ f!(
+ /// ["fxsave"] and ["fxrstor"] --- Save and restore x87 FPU, MMX Technology, and SSE State
+ /// ["fxsave"]: https://www.felixcloutier.com/x86/fxsave,
+ struct ::sse::Fxsr("`fxsave + fxrstor`"): "fxsr" + []
+ fn uses_fxsr
+ ),
+ f!(
+ /// [GFNI] --- Galois Field New Instructions
+ /// [GFNI]: https://en.wikipedia.org/wiki/AVX-512#GFNI
+ struct ::crypto::Gfni("GFNI"): "gfni" + ["sse2"]
+ fn uses_gfni
+ ),
+ f!(
+ /// [KEYLOCKER] --- Intel Key Locker Instructions
+ /// [KEYLOCKER]: https://en.wikipedia.org/wiki/List_of_x86_cryptographic_instructions#Intel_Key_Locker_instructions
+ struct ::crypto::Keylocker("KEYLOCKER"): "kl" + []
+ fn uses_keylocker
+ ),
+ f!(
+ /// ["lzcnt"] --- Leading zeros count
+ /// ["lzcnt"]: https://www.felixcloutier.com/x86/lzcnt
+ struct ::v3::Lzcnt("`lzcnt`"): "lzcnt" + []
+ fn uses_lzcnt
+ ),
+ f!(
+ /// ["movbe"] --- Move data after swapping bytes
+ /// ["movbe"]: https://www.felixcloutier.com/x86/movbe
+ struct ::v3::Movbe("`movbe`"): "movbe" + []
+ fn uses_movbe
+ ),
+ f!(
+ /// ["pclmulqdq"] --- Packed carry-less multiplication quadword
+ /// ["pclmulqdq"]: https://www.felixcloutier.com/x86/pclmulqdq
+ struct ::crypto::Pclmulqdq("`pclmulqdq`"): "pclmulqdq" + ["sse2"]
+ fn uses_pclmulqdq
+ ),
+ f!(
+ /// ["popcnt"] --- Count of bits set to 1
+ /// ["popcnt"]: https://www.felixcloutier.com/x86/popcnt
+ struct ::v2::Popcnt("`popcnt`"): "popcnt" + []
+ fn uses_popcnt
+ ),
+ f!(
+ /// ["rdrand"] --- Read random number
+ /// ["rdrand"]: https://en.wikipedia.org/wiki/RdRand
+ struct ::crypto::Rdrand("`rdrand`"): "rdrand" + []
+ fn uses_rdrand
+ ),
+ f!(
+ /// ["rdseed"] --- Read random seed
+ /// ["rdseed"]: https://en.wikipedia.org/wiki/RdRand
+ struct ::crypto::Rdseed("`rdseed"): "rdseed" + []
+ fn uses_rdseed
+ ),
+ f!(
+ /// [SHA] --- Secure Hash Algorithm
+ /// [SHA]: https://en.wikipedia.org/wiki/Intel_SHA_extensions
+ struct ::crypto::Sha("SHA"): "sha" + ["sse2"]
+ fn uses_sha
+ ),
+ f!(
+ /// [SHA512] --- Secure Hash Algorithm with 512-bit digest
+ /// [SHA512]: https://en.wikipedia.org/wiki/Intel_SHA_extensions
+ struct ::crypto::Sha512("SHA512"): "sha512" + ["avx2"]
+ fn uses_sha512
+ ),
+ f!(
+ /// [SM3] --- ShangMi 3 Hash Algorithm
+ /// [SM3]: https://en.wikipedia.org/wiki/List_of_x86_cryptographic_instructions#Intel_SHA_and_SM3_instructions
+ struct ::crypto::Sm3("SM3"): "sm3" + ["avx"]
+ fn uses_sm3
+ ),
+ f!(
+ /// [SM4] --- ShangMi 4 Cipher Algorithm
+ /// [SM4]: https://en.wikipedia.org/wiki/List_of_x86_cryptographic_instructions#Intel_SHA_and_SM3_instructions
+ struct ::crypto::Sm4("SM4"): "sm4" + ["avx2"]
+ fn uses_sm4
+ ),
+ f!(
+ /// [SSE] --- Streaming SIMD Extensions
+ /// [SSE]: https://en.wikipedia.org/wiki/Streaming_SIMD_Extensions
+ struct ::sse::Sse("SSE"): "sse" + []
+ fn uses_sse
+ ),
+ f!(
+ /// [SSE2] --- Streaming SIMD Extensions 2
+ /// [SSE2]: https://en.wikipedia.org/wiki/SSE2
+ struct ::sse::Sse2("SSE2"): "sse2" + ["sse"]
+ fn uses_sse2
+ ),
+ f!(
+ /// [SSE3] --- Streaming SIMD Extensions 3
+ /// [SSE3]: https://en.wikipedia.org/wiki/SSE3
+ struct ::sse::Sse3("SSE3"): "sse3" + ["sse2"]
+ fn uses_sse3
+ ),
+ f!(
+ /// [SSE4.1] --- Streaming SIMD Extensions 4.1
+ /// [SSE4.1]: https://en.wikipedia.org/wiki/SSE4#SSE4.1
+ struct ::sse::Sse4_1("SSE4.1"): "sse4.1" + ["ssse3"]
+ fn uses_sse4
+ ),
+ f!(
+ /// [SSE4.2] --- StreamingSIMDExtensions 4.2
+ /// [SSE4.2]: https://en.wikipedia.org/wiki/SSE4#SSE4.2
+ struct ::sse::Sse4_2("SSE4.2"): "sse4.2" + ["sse4.1"]
+ fn uses_sse4
+ ),
+ // // TODO: This only exists from 1.91 and above (current beta)
+ // f!(
+ // /// [SSE4a] --- StreamingSIMDExtensions 4a
+ // /// [SSE4a]: https://en.wikipedia.org/wiki/SSE4#SSE4a
+ // struct Sse4a("SSE4a"): "sse4a" + ["sse3"]
+ // fn uses_sse4a
+ // ),
+ f!(
+ /// [SSSE3] --- Supplemental StreamingSIMDExtensions 3
+ /// [SSSE3]: https://en.wikipedia.org/wiki/SSSE3
+ struct ::sse::SupplementalSse3("SSSE3"): "ssse3" + ["sse3"]
+ fn uses_ssse3
+ ),
+ f!(
+ /// [TBM] --- Trailing Bit Manipulation
+ /// [TBM]: https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set#TBM_(Trailing_Bit_Manipulation)
+ struct ::discontinued::Tbm("TBM"): "tbm" + []
+ fn uses_tbm
+ ),
+ f!(
+ /// [VAES] --- Vector AES Instructions
+ /// [VAES]: https://en.wikipedia.org/wiki/AVX-512#VAES
+ struct ::crypto::Vaes("VAES"): "vaes" + ["avx2", "aes"]
+ fn uses_vaes
+ ),
+ f!(
+ /// [VPCLMULQDQ] --- Vector Carry-less multiplication of Quadwords
+ /// [VPCLMULQDQ]: https://en.wikipedia.org/wiki/AVX-512#VPCLMULQDQ
+ struct ::crypto::Vpclmulqdq("VPCLMULQDQ"): "vpclmulqdq" + ["avx", "pclmulqdq"]
+ fn uses_vpclmulqdq
+ ),
+ f!(
+ /// [KEYLOCKER_WIDE] --- Intel Wide Keylocker Instructions
+ /// [KEYLOCKER_WIDE]: https://en.wikipedia.org/wiki/List_of_x86_cryptographic_instructions#Intel_Key_Locker_instructions
+ struct ::crypto::WideKeylocker("KEYLOCKER_WIDE"): "widekl" + ["kl"]
+ fn uses_wide_keylocker
+ ),
+ f!(
+ /// [`xsave`] --- Save processor extended states
+ /// ["xsave"]: https://www.felixcloutier.com/x86/xsave
+ struct ::xsave::Xsave("`xsave`"): "xsave" + []
+ fn uses_xsave
+ ),
+ f!(
+ /// ["xsavec"] --- Save processor extended states with compaction
+ /// ["xsavec"]: https://www.felixcloutier.com/x86/xsavec
+ struct ::xsave::Xsavec("`xsavec`"): "xsavec" + []
+ fn uses_xsavec
+ ),
+ f!(
+ /// ["xsaveopt"] --- Save processor extended states optimized
+ /// ["xsaveopt"]: https://www.felixcloutier.com/x86/xsaveopt
+ struct ::xsave::Xsaveopt("`xsaveopt`"): "xsaveopt" + []
+ fn uses_xsaveopt
+ ),
+ f!(
+ /// ["xsaves"] --- Save processor extended states supervisor
+ /// ["xsaves"]: https://www.felixcloutier.com/x86/xsaves
+ struct ::xsave::Xsaves("`xsaves`"): "xsaves" + []
+ fn uses_xsaves
+ ),
+];
+
+#[test]
+fn all_features_included() {}
diff --git a/fearless_simd_core/gen/src/main.rs b/fearless_simd_core/gen/src/main.rs
new file mode 100644
index 00000000..2961bd33
--- /dev/null
+++ b/fearless_simd_core/gen/src/main.rs
@@ -0,0 +1,166 @@
+mod data;
+
+use std::fmt::{Write, format};
+use std::fs;
+use std::{
+ cell::RefCell,
+ collections::HashMap,
+ fs::create_dir_all,
+ io,
+ path::{Path, PathBuf},
+};
+
+fn main() {
+ let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+ let src_dir = manifest_dir.ancestors().nth(1).unwrap().join("src");
+ generate_for_arch(&src_dir, "x86", data::X86_TEMPLATE, data::X86_FEATURES).unwrap();
+}
+
+fn generate_for_arch(
+ root_dir: &Path,
+ arch_module_name: &str,
+ template: &str,
+ features: &'static [Feature],
+) -> io::Result<()> {
+ let arch_dir = root_dir.join(arch_module_name);
+ let features = normalize_features(features);
+ for feature in &features {
+ let mut new_docs = String::new();
+ for line in feature.feature.extra_docs.lines() {
+ writeln!(&mut new_docs, "///{line}").unwrap();
+ }
+ let enabled_feature_docs = format!("`{}`", feature.children.join("`, `"));
+ let enabled_feature_str_list = format!(r#""{}""#, feature.children.join(r#"", ""#));
+ let mut from_impls = String::new();
+ for child in &feature.children {
+ let from_feature = features
+ .iter()
+ .find(|it| it.feature.feature_name == *child)
+ .unwrap();
+ let type_path = format!(
+ "crate::{arch_module_name}::{}::{}",
+ from_feature.feature.module, from_feature.feature.struct_name
+ );
+ write!(
+ from_impls,
+ r#"\n\
+ impl From for {type_path} {{
+ fn from(value: Self) -> {type_path} {{
+ trampoline!([Self = value] => "{{FEATURE_ID}}", fn() -> {type_path} {{ {{type_path}}::new() }})
+ }}
+ }}\n
+ "#
+ ).unwrap();
+ }
+ let mut result = format!(
+ "// This file is automatically generated by `fearless_simd_core_gen`.\n\
+ // Its template can be found in `fearless_simd_core/gen/templates`.\n\n\
+ {template}"
+ );
+ // We replace the from impls first, as they use template variables from the rest of this.
+ result = result.replace("/*{FROM_IMPLS}*/", &from_impls);
+ result = result.replace("{FEATURE_DOCS_NAME}", feature.feature.feature_docs_name);
+ result = result.replace("/// {NEW_DOCS}", &new_docs);
+ result = result.replace("{FEATURE_ID}", feature.feature.feature_name);
+ result = result.replace("{ENABLED_FEATURES_DOCS_LIST}", &enabled_feature_docs);
+ result = result.replace(
+ "{EXAMPLE_FUNCTION_NAME}",
+ feature.feature.example_function_name,
+ );
+ result = result.replace("FEATURE_STRUCT_NAME", feature.feature.struct_name);
+ result = result.replace("{ENABLED_FEATURES_STR_LIST}", &enabled_feature_str_list);
+ let module_dir = arch_dir.join(feature.feature.module);
+ create_dir_all(&module_dir)?;
+ let mut file = module_dir.join(feature.feature.feature_name);
+ file.set_extension("rs");
+ fs::write(file, result)?;
+ }
+ Ok(())
+}
+
+#[derive(Debug)]
+struct Feature {
+ /// The name of the struct to be generated.
+ struct_name: &'static str,
+ /// The Rust name for the feature, e.g. `"sse"`.
+ feature_name: &'static str,
+ /// The array of features which are implicitly enabled by this feature.
+ /// Note that this array does not include transitive enabled features.
+ directly_implicitly_enabled: &'static [&'static str],
+ /// Any additional docs which we want to add to the module.
+ extra_docs: &'static str,
+ /// The name of the function used in the examples.
+ /// Ideally, we'd make this optional, but that starts making the templating look more complicated.
+ example_function_name: &'static str,
+ /// The "display name" for the feature, used inside the docs.
+ feature_docs_name: &'static str,
+ /// Extra code added at the end.
+ /// Used for implicitly enabled features.
+ additional_impls: &'static str,
+ /// The module (if any) this feature will belong to.
+ ///
+ /// (Note that imports into the module are checked to exist, but not automatically inserted).
+ module: &'static str,
+}
+
+/// Implementation detail intermediate struct of `normalize_features`.
+struct MaybeNormalizedFeature {
+ /// The actual feature.
+ feature: &'static Feature,
+ /// The fully deduplicated, sorted list of target features enabled by this feature, including with all
+ /// implicitly enabled features resolved.
+ ///
+ /// Note that this *excludes* the parent target feature.
+ // We use a RefCell here as we know there cannot be loops.
+ children: RefCell>>,
+}
+
+#[derive(Debug)]
+struct NormalizedFeature {
+ feature: &'static Feature,
+ children: Vec<&'static str>,
+}
+
+fn normalize_features(features: &'static [Feature]) -> Vec {
+ let mut state = HashMap::new();
+ for feature in features {
+ state.insert(
+ feature.feature_name,
+ MaybeNormalizedFeature {
+ feature,
+ children: RefCell::new(None),
+ },
+ );
+ }
+ fn handle_item(state: &HashMap<&str, MaybeNormalizedFeature>, item: &MaybeNormalizedFeature) {
+ // We borrow for the entire lifetime to avoid infinite loops.
+ let mut borrowed_children = item.children.borrow_mut();
+ if borrowed_children.is_some() {
+ return;
+ }
+ let mut new_children = Vec::new();
+ for child in item.feature.directly_implicitly_enabled {
+ new_children.push(*child);
+ let child = state
+ .get(child)
+ .expect("Every implicitly enabled feature should exist.");
+ handle_item(state, child);
+ new_children.extend_from_slice(child.children.borrow().as_ref().unwrap());
+ }
+ new_children.sort();
+ new_children.dedup();
+ *borrowed_children = Some(new_children);
+ }
+ for feature in state.values() {
+ handle_item(&state, feature);
+ }
+ let mut output = Vec::new();
+ for (_, feature) in state {
+ output.push(NormalizedFeature {
+ feature: feature.feature,
+ children: feature.children.into_inner().unwrap(),
+ });
+ }
+ output.sort_by_key(|it| it.feature.feature_name);
+ output
+}
diff --git a/fearless_simd_core/gen/templates/aarch64.rs b/fearless_simd_core/gen/templates/aarch64.rs
new file mode 100644
index 00000000..e69de29b
diff --git a/fearless_simd_core/gen/templates/x86.rs b/fearless_simd_core/gen/templates/x86.rs
new file mode 100644
index 00000000..a37207a3
--- /dev/null
+++ b/fearless_simd_core/gen/templates/x86.rs
@@ -0,0 +1,90 @@
+//! The {FEATURE_DOCS_NAME} target feature.
+
+use crate::{TargetFeatureToken, trampoline};
+
+use core::fmt::Debug;
+
+/// {NEW_DOCS}
+///
+/// A token indicating that the current CPU has the `{FEATURE_ID}` target feature.
+///
+/// This feature also enables {ENABLED_FEATURES_DOCS_LIST};
+/// the tokens for these features can be created using [`From`] implementations.
+///
+/// # Example
+///
+/// This can be used to [`trampoline!`] into:
+///
+/// ```rust
+/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+/// #[target_feature(enable = "{FEATURE_ID}")]
+/// fn {EXAMPLE_FUNCTION_NAME}() {
+/// // ...
+/// }
+/// ```
+#[derive(Copy, Clone, Hash, PartialEq, Eq)]
+pub struct FEATURE_STRUCT_NAME {
+ // We don't use non_exhaustive because we don't want this struct to be constructible.
+ // in different modules in this crate.
+ _private: (),
+}
+
+impl Debug for FEATURE_STRUCT_NAME {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ write!(f, r#""{FEATURE_ID}" enabled."#)
+ }
+}
+
+unsafe impl TargetFeatureToken for Sse {
+ const FEATURES: &[&str] = &[{ ENABLED_FEATURES_STR_LIST }];
+
+ #[inline(always)]
+ fn vectorize(self, f: impl FnOnce() -> R) -> R {
+ trampoline!([Self = self] => "{FEATURE_ID}", <(R)> fn<(R)>(f: impl FnOnce() -> R = f) -> R { f() })
+ }
+}
+
+impl Sse {
+ #[cfg(feature = "std")]
+ /// Create a new token if the `"{FEATURE_ID}"` target feature is detected as enabled.
+ ///
+ /// This does not do any caching internally, although note that the standard
+ /// library does internally cache the features it detects.
+ // TODO: Consider a manual override feature/env var?
+ pub fn try_new() -> Option {
+ // Feature flag required to make docs compile.
+ // TODO: Extract into a (private) crate::x86::is_x86_feature_detected?
+ #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+ if std::arch::is_x86_feature_detected!("{FEATURE_ID}") {
+ // Safety: The required CPU feature was detected.
+ unsafe { Some(Self::new()) }
+ } else {
+ None
+ }
+ }
+
+ #[target_feature(enable = "{FEATURE_ID}")]
+ /// Create a new token for the "{FEATURE_ID}" target feature is enabled.
+ ///
+ /// This method is useful to get a new token if you have an external proof that
+ /// {FEATURE_ID} is available. This could happen if you are in a target feature
+ /// function called by an external library user.
+ ///
+ /// # Safety
+ ///
+ /// No conditions other than those inherited from the target feature attribute,
+ /// i.e. that the "{FEATURE_DOCS_NAME}" target feature is available.
+ ///
+ /// [implicitly enables]: https://doc.rust-lang.org/beta/reference/attributes/codegen.html?highlight=implicitly%20enabled#r-attributes.codegen.target_feature.safety-restrictions
+ pub fn new() -> Self {
+ Self { _private: () }
+ }
+}
+/*{FROM_IMPLS}*/
+
+const _: () = {
+ assert!(
+ core::mem::size_of::() == 0,
+ "Target feature tokens should be zero sized."
+ );
+};
diff --git a/fearless_simd_core/src/lib.rs b/fearless_simd_core/src/lib.rs
new file mode 100644
index 00000000..e212b240
--- /dev/null
+++ b/fearless_simd_core/src/lib.rs
@@ -0,0 +1,282 @@
+//! Tooling for Rust's target features.
+
+// LINEBENDER LINT SET - lib.rs - v4
+// See https://linebender.org/wiki/canonical-lints/
+// These lints shouldn't apply to examples or tests.
+#![cfg_attr(not(test), warn(unused_crate_dependencies))]
+// These lints shouldn't apply to examples.
+#![warn(clippy::print_stdout, clippy::print_stderr)]
+// Targeting e.g. 32-bit means structs containing usize can give false positives for 64-bit.
+#![cfg_attr(target_pointer_width = "64", warn(clippy::trivially_copy_pass_by_ref))]
+// END LINEBENDER LINT SET
+#![cfg_attr(docsrs, feature(doc_cfg))]
+#![no_std]
+
+// TODO: Do we want both an `x86` and `x86_64` module?
+#[cfg(any(target_arch = "x86", target_arch = "x86_64", doc))]
+pub mod x86;
+
+pub mod trampoline;
+
+#[cfg(feature = "std")]
+extern crate std;
+
+/// Token that a set of target feature is available.
+///
+/// Note that this trait is only meaningful when there are values of this type.
+/// That is, to enable the target features in `FEATURES`, you *must* have a value
+/// of this type.
+///
+/// Values which implement this trait are used in the second argument to [`trampoline!`],
+/// which is a safe abstraction over enabling target features.
+///
+/// # Safety
+///
+/// To construct a value of a type implementing this trait, you must have proven that each
+/// target feature in `FEATURES` is available.
+pub unsafe trait TargetFeatureToken: Copy {
+ /// The set of target features which are enabled for this run, if
+ /// you have a value of this type.
+ const FEATURES: &[&str];
+
+ /// Enable the target features in `FEATURES` for a single run of `f`, and run it.
+ ///
+ /// `f` must be marked `#[inline(always)]` for this to work.
+ ///
+ /// Note that this does *not* enable the target features on the Rust side (e.g. for calling).
+ /// To do so, you should instead use [`trampoline!`] directly - this is a convenience wrapper around `trampoline`
+ /// for cases where the dispatch of simd values is handled elsewhere.
+ fn vectorize(self, f: impl FnOnce() -> R) -> R;
+}
+
+/// Run an operation in a context with specific target features enabled, validated with [`TargetFeatureToken`] values.
+///
+/// This is effectively a stable implementation of the "Struct Target Features" Rust feature,
+/// which at the time of writing is neither in stable or nightly Rust.
+/// This macro can be used to make SIMD dispatch safe in addition to make explicit SIMD, both safely.
+///
+/// # Reference
+///
+/// These reference examples presume that you have (values in brackets are the "variables"):
+///
+/// - An expression (`token`) of a type (`Token`) which is `TargetFeatureToken` for some target features (`"f1,f2,f3"`);
+/// - A function (signature `fn uses_simd(val: [f32; 4]) -> [f32; 4]`) which is safe but enables a subset of those target features (`"f1,f2"`);
+/// - Local values of types corresponding to the argument types (`a` of type `[f32; 4]`)
+///
+/// ```rust,ignore
+/// trampoline!(Token = token => "f1,f2", uses_simd(a: [f32; 4]) -> [f32; 4])
+/// ```
+///
+/// Multiple tokens are also supported by providing them in a sequence in square brackets:
+///
+/// ```rust,ignore
+/// trampoline!([Token = token, Sse = my_sse] => "f1,f2,sse", uses_simd(a: [f32; 4]) -> [f32; 4])
+/// ```
+///
+/// A more advanced syntax is available if you need to use generics.
+/// That syntax is explained in comments around the macro's definition, which can be seen above.
+/// For reference, the implementation used to implement [`vectorize`](TargetFeatureToken::vectorize) for `"sse"` is:
+///
+/// ```rust,ignore
+/// trampoline!([Self = self] => "sse", <(R)> fn<(R)>(f: impl FnOnce() -> R = f) -> R { f() })
+/// ```
+///
+/// There is also support for where clauses after the return type.
+///
+/// # Motivation
+///
+/// In Fearless SIMD, this macro has two primary use cases:
+///
+/// 1) To dispatch to a specialised SIMD implementation of a function using target specific
+/// instructions which will be more efficient than generic version written using the portable subset.
+/// 2) To implement the portable subset of SIMD operations.
+///
+/// To expand on use case 1, when using Fearless SIMD you will often be writing functions which are
+/// instantiated for multiple different SIMD levels (using generics).
+/// However, for certain SIMD levels, there may be specific instructions which solve your problem more
+/// efficiently than using the generic implementations (as an example, consider SHA256 hashing, which has
+/// built-in instructions on several architectures).
+/// However, in such generic implementations, the Rust type system doesn't know which target features are enabled,
+/// so it would ordinarily require writing code to:
+///
+/// - detect whether a specific target feature is supported.
+/// - unsafely, enter a context where the target feature is enabled in a way which makes the type system aware of this.
+///
+/// This macro provides a way to do the second safely once you have completed the first.
+///
+/// # Example
+///
+/// This expands upon the example in the reference, written out completely.
+///
+/// ```rust,ignore
+/// // Just once, acquire a token.
+/// let token = Token::try_new();
+/// // Later, dispatch based on whether that token is available, potentially multiple times:
+///
+/// /// Perform some computation using SIMD.
+/// #[target_feature(enable = "f1,f2")]
+/// fn uses_simd(val: [f32; 4]) -> [f32; 4] {
+/// // ...
+/// }
+///
+/// let a = [1., 2., 3., 4.];
+/// let Some(token) = token else { return scalar_fallback(a) };
+///
+/// trampoline!(Token = token => "f1,f2", uses_simd(a: [f32; 4]) -> [f32; 4])
+/// ```
+///
+/// Note that a function only operating on 128 bytes is probably too small for checking
+/// whether a token exists just for it is worthwhile.
+/// However, if you have amorphised the cost of that check between many function calls,
+/// the `trampoline!` macro itself compiles down to a function call.
+/// (This would be the case when this macro is being used to implement the portable subset of SIMD operations)
+///
+// TODO: We could write an example for each of ARM, x86, and conditionally compile it in?
+/// Note that our examples are all ignored as there is no target feature which is available on every platform,
+/// but we need these docs to compile for users on any platform.
+///
+/// # Soundness
+///
+/// This macro is designed to be sound, i.e. no input to this macro can lead to undefined behaviour
+/// without using the `unsafe` keyword.
+///
+/// The operation provided will only ever be immediately called once on the same thread as the macro caller,
+/// so safety justifications within the operation can rely on the context of the call site of this macro.
+/// The shorthand format does not allow calling unsafe functions.
+#[macro_export]
+macro_rules! trampoline {
+ // [Sse = sse] for "sse", <(u32)> fn<(T: Int)>(a: [T; 4]) -> T where (...) {...}
+ (
+ // The token types, with an expression to get a value of that token kind.
+ [$($token_type: path = $token: expr),+$(,)?]
+ // The target feature to enable. Must be a string literal.
+ => $to_enable: literal,
+ // The generic arguments to instantiate the call to the generated function with.
+ // Note the inner brackets, needed because we can't write a parser for this in macros.
+ $(<($($generic_instantiation: tt)+)>)?
+ // The generic parameters to give the inner generated function.
+ // Brackets needed as above.
+ fn$(<($($generic_args: tt)*)>)?
+ // The arguments to the function, with provided explicit values, plus return type and where clause.
+ ($($arg_name: ident: $arg_type: ty = $arg_value: expr),*$(,)?) $(-> $ret: ty)?
+ // The where clause of the generated function.
+ // Note the inner brackets after `where`, needed as above.
+ $(where ($($where: tt)*))?
+ // The operation to run inside the context with the target feature enabled.
+ $op: block
+ ) => {{
+ #[target_feature(enable = $to_enable)]
+ #[inline]
+ // TODO: Do we want any other attributes here?
+ // Soundness: We wrap the $op in a wrapping block, to ensure that any inner attributes don't apply to the function.
+ // This ensures that the user can't add `#![target_feature(enable = "xxx")]` to their block.
+ // Soundness: Either of generic_args and `$where` could be used to exit the function item early, so aren't
+ // inside an unsafe block.
+ fn trampoline_impl$(<$($generic_args)*>)?($($arg_name: $arg_type),*) $(-> $ret)? $(where $($where)*)? { $op }
+
+ $(
+ // We validate that we actually have a token of each claimed type.
+ let _: $token_type = $token;
+ )+
+ const {
+ // And that the claimed types justify enabling the enabled target features.
+ $crate::trampoline::is_feature_subset($to_enable, [$(<$token_type as $crate::TargetFeatureToken>::FEATURES),+])
+ // TODO: Better failure message here (i.e. at least concatting the set of requested features)
+ .unwrap();
+ }
+
+ $(
+ // Soundness: We use `arg_value` outside of the macro body to ensure it doesn't
+ // accidentally gain an unsafe capability.
+ #[allow(clippy::redundant_locals, reason="Required for consistency/safety.")]
+ let $arg_name = $arg_value;
+ )*
+ // Safety: We have validated that the target features enabled in `trampoline_impl` are enabled,
+ // because we have values of token types which implement $crate::TargetFeatureToken
+ // Soundness: `$generic_args` could be used to exit the path expression early. As `<>` are
+ // not treated as "real" brackets by macros, this isn't practical to detect and avoid statically.
+ // To try and ensure that this can't turn into unsoundess, the
+ // `trampoline_impl::<$generic_instantiation>` is evaluated outside of an unsafe block.
+ // In theory, if a user could make the value of `func` be an `unsafe` fn pointer or
+ // item type, this would still be unsound.
+ // However, we haven't found a way for this to compile given the trailing `>`,
+ // so aren't aware of any actual unsoundess. But note that this hasn't been rigorously proven,
+ // and new Rust features could open this up wider.
+ let func = trampoline_impl$(::<$($generic_instantiation)*>)?;
+ unsafe { func($($arg_name),*) }
+ }};
+ // Sse = sse => "sse", sse_do_x(a: [f32; 4], b: [f32; 4]) -> [f32; 4]
+ ($token_type: path = $token: expr => $to_enable: literal, $function: ident($($arg_name: ident: $arg_type: ty),*$(,)?) $(-> $ret: ty)?) => {
+ $crate::trampoline!(
+ [$token_type = $token]
+ => $to_enable,
+ $function($($arg_name: $arg_type),*) $(-> $ret)?
+ )
+ };
+ // [Sse = sse] => "sse", sse_do_x(a: [f32; 4], b: [f32; 4]) -> [f32; 4]
+ ([$($token_type: path = $token: expr),+$(,)?] => $to_enable: literal, $function: ident($($arg_name: ident: $arg_type: ty),*$(,)?) $(-> $ret: ty)?) => {
+ $crate::trampoline!(
+ [$($token_type = $token),+]
+ => $to_enable,
+ fn($($arg_name: $arg_type = $arg_name),*) $(-> $ret)? { $function($($arg_name),*) }
+ )
+ };
+}
+
+#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+#[cfg(test)]
+mod example_expansion {
+ use core::arch::x86_64::{__m128, _mm_mul_ps};
+
+ use crate::x86::{self, v1::Sse};
+
+ #[target_feature(enable = "sse")]
+ fn sse_mul_f32s(a: [f32; 4], b: [f32; 4]) -> [f32; 4] {
+ let a: __m128 = bytemuck::must_cast(a);
+ let b: __m128 = bytemuck::must_cast(b);
+ bytemuck::must_cast(_mm_mul_ps(a, b))
+ }
+
+ #[test]
+ // This is a test so that it is runnable
+ fn example_output() {
+ let Some(sse) = x86::v1::Sse::try_new() else {
+ panic!("Example code")
+ };
+ let a = [10_f32, 20_f32, 30_f32, 40_f32];
+ let b = [4_f32, 5_f32, 6_f32, 7_f32];
+
+ // Both of these example expansions, the former using the shorthand form:
+ let res =
+ trampoline!(Sse = sse => "sse", sse_mul_f32s(a: [f32; 4], b: [f32; 4]) -> [f32; 4]);
+ assert_eq!(res, [40_f32, 100_f32, 180_f32, 280_f32]);
+ let res = trampoline!([Sse = sse] => "sse", fn(a: [f32; 4] = a, b: [f32; 4] = b) -> [f32; 4] { sse_mul_f32s(a, b)});
+ assert_eq!(res, [40_f32, 100_f32, 180_f32, 280_f32]);
+ // will expand to:
+ #[expect(unused_braces, reason = "Required for macro soundness.")]
+ // Start expansion:
+ let res = {
+ #[target_feature(enable = "sse")]
+ #[inline]
+ fn trampoline_impl(a: [f32; 4], b: [f32; 4]) -> [f32; 4] {
+ { sse_mul_f32s(a, b) }
+ }
+ let _: Sse = sse;
+ const {
+ crate::trampoline::is_feature_subset(
+ "sse",
+ [::FEATURES],
+ )
+ .unwrap();
+ }
+ #[allow(clippy::redundant_locals, reason = "Required for consistency/safety.")]
+ let a = a;
+ #[allow(clippy::redundant_locals, reason = "Required for consistency/safety.")]
+ let b = b;
+ let func = trampoline_impl;
+ unsafe { func(a, b) }
+ };
+ // End expansion
+ assert_eq!(res, [40_f32, 100_f32, 180_f32, 280_f32]);
+ }
+}
diff --git a/fearless_simd_core/src/trampoline.rs b/fearless_simd_core/src/trampoline.rs
new file mode 100644
index 00000000..14c73f62
--- /dev/null
+++ b/fearless_simd_core/src/trampoline.rs
@@ -0,0 +1,231 @@
+//! Support for the safety checks in [`trampoline!`](crate::trampoline!).
+//!
+//! Methods to compute whether a each feature in a target feature string (e.g. "sse2,fma")
+//! is supported by a set of target features.
+//!
+//! The [`trampoline`](crate::trampoline!) macro takes both a target feature string,
+//! and one (or more) [`TargetFeatureToken`](crate::TargetFeatureToken).
+//! It uses the functions in this module to validate that the target feature string is
+//! supported by the provided tokens.
+//!
+//! Because evaluating whether this is safe needs to happen at compile time (for both performance
+//! and predictability), the methods in this file are written as `const` functions.
+//! This leads to a bit of weirdness, including treating strings as `&[u8]` internally, as that
+//! actually allows slicing (i.e. reading individual bytes). As far as I know, that isn't
+//! currently possibly in const contexts for strings.
+//! Note that the code is still written to be UTF-8 compatible, although we believe that
+//! all currently supported target features are ASCII anyway.
+
+/// The result of `is_feature_subset`.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[must_use]
+pub enum SubsetResult {
+ /// The required features are a subset of the permitted features.
+ Yes,
+ /// The required features are not all available.
+ No {
+ /// The feature which was found to be missing (there may be several such features).
+ failing: &'static str,
+ },
+}
+
+impl SubsetResult {
+ /// A utility method to panic if the target features aren't supported.
+ // TODO: How much more context would we be able to give if we inlined this?
+ pub const fn unwrap(self) {
+ match self {
+ Self::Yes => (),
+ // This is const, so we can't actually format out the failing value :(
+ Self::No { .. } => panic!("Tokens provided are missing a necessary target feature."),
+ }
+ }
+}
+
+/// Determine whether the features in the target feature string `required` are a subset of the features in `permitted`.
+/// See the module level docs [self].
+///
+/// We require static lifetimes as this is primarily internal to the macro.
+pub const fn is_feature_subset(
+ required: &'static str,
+ permitted: [&[&'static str]; N],
+) -> SubsetResult {
+ let mut required_bytes = required.as_bytes();
+ let mut finished = false;
+ 'input_feature: while !finished {
+ let mut comma_idx = 0;
+ // Find the first comma in required_bytes, or the end of the string.
+ while comma_idx < required_bytes.len() && required_bytes[comma_idx] != b',' {
+ comma_idx += 1;
+ }
+ // `comma_idx` is now the index of the comma, e.g. if the string was "sse,", idx would be 3
+ // This is the feature we need to validate exists in permitted.
+ let (to_find, remaining_required) = &required_bytes.split_at(comma_idx);
+ if let [comma, rest @ ..] = remaining_required {
+ if *comma != b',' {
+ panic!("Internal failure of expected behaviour.");
+ } else {
+ required_bytes = rest;
+ }
+ } else {
+ // Exit out of the loop after this iteration.
+ // Note that for input of `""`` and "sse,", we still need to search
+ // for the input target feature `` (i.e. the empty string), to match Rust's behaviour here.
+ finished = true;
+ }
+
+ let mut local_permitted = permitted.as_slice();
+ while let [to_test, rest @ ..] = local_permitted {
+ local_permitted = rest;
+ if str_array_contains(to_test, to_find) {
+ continue 'input_feature;
+ }
+ }
+ // We tried all of the items, and `to_find` wasn't one of them.
+ // Therefore, at least one of the features in the requested features wasn't supported
+ return SubsetResult::No {
+ failing: match core::str::from_utf8(to_find) {
+ Ok(x) => x,
+ Err(_) => panic!(
+ "We either found a comma or the end of the string, so before then should have been valid UTF-8."
+ ),
+ },
+ };
+ }
+ // We found all of the required features.
+ SubsetResult::Yes
+}
+
+const fn str_array_contains(mut haystack: &[&str], needle: &[u8]) -> bool {
+ while let [to_test, rest @ ..] = haystack {
+ haystack = rest;
+ if byte_arrays_eq(to_test.as_bytes(), needle) {
+ return true;
+ }
+ }
+ false
+}
+
+const fn byte_arrays_eq(lhs: &[u8], rhs: &[u8]) -> bool {
+ if lhs.len() != rhs.len() {
+ return false;
+ }
+ let mut idx = 0;
+ while idx < lhs.len() {
+ if lhs[idx] != rhs[idx] {
+ return false;
+ }
+ idx += 1;
+ }
+ true
+}
+
+#[cfg(test)]
+mod tests {
+ use super::{SubsetResult, is_feature_subset};
+
+ /// Test if each feature in the feature string `required` is an element in `permitted`.
+ ///
+ /// Should be equivalent to [`is_feature_subset`], but not written to be const compatible.
+ fn is_feature_subset_simple(
+ required: &'static str,
+ permitted: [&[&'static str]; N],
+ ) -> SubsetResult {
+ 'feature: for feature in required.split(',') {
+ for permitted_group in &permitted {
+ for permitted_feature in *permitted_group {
+ if feature == *permitted_feature {
+ continue 'feature;
+ }
+ }
+ }
+ // We tried all permitted feature, and this item wasn't present.
+ return SubsetResult::No { failing: feature };
+ }
+ SubsetResult::Yes
+ }
+
+ /// Expect `is_feature_subset` to succeed.
+ #[track_caller]
+ fn expect_success(required: &'static str, permitted: [&[&'static str]; N]) {
+ let res1 = is_feature_subset(required, permitted);
+ assert_eq!(res1, SubsetResult::Yes, "Const version failed.");
+ // Sanity check against the "trivially correct" version.
+ let res2 = is_feature_subset_simple(required, permitted);
+ assert_eq!(res2, SubsetResult::Yes, "Simpler version failed.");
+ }
+
+ /// Expect `is_feature_subset` to fail (with only a single possible failure).
+ #[track_caller]
+ fn expect_failure(
+ required: &'static str,
+ permitted: [&[&'static str]; N],
+ failing: &'static str,
+ ) {
+ let res1 = is_feature_subset(required, permitted);
+ assert_eq!(res1, SubsetResult::No { failing }, "Const version failed.");
+ // Sanity check against the "trivially correct" version.
+ let res2 = is_feature_subset_simple(required, permitted);
+ assert_eq!(
+ res2,
+ SubsetResult::No { failing },
+ "Simpler version failed."
+ );
+ }
+
+ /// Expect `is_feature_subset` to fail, possibly with multiple potential missing features.
+ #[track_caller]
+ fn expect_any_failure(required: &'static str, permitted: [&[&'static str]; N]) {
+ let res1 = is_feature_subset(required, permitted);
+ assert!(
+ matches!(res1, SubsetResult::No { .. }),
+ "Const version failed."
+ );
+ // Sanity check against the "trivially correct" version.
+ let res2 = is_feature_subset_simple(required, permitted);
+ assert!(
+ matches!(res2, SubsetResult::No { .. }),
+ "Simpler version failed."
+ );
+ }
+
+ #[test]
+ fn simple_cases() {
+ expect_success("a,b,c", [&["a", "b", "c"]]);
+ expect_failure("a,b,c", [&["a", "b"]], "c");
+ expect_success("c,a,b", [&["a", "b", "c"]]);
+ expect_failure("c,a,b", [&["a", "b"]], "c");
+ expect_success("a,b", [&["a", "b", "c"]]);
+ expect_failure("a,b", [&["a", "c"]], "b");
+
+ // Check it correctly catches more than single item failures
+ expect_success("a1,a2,a3", [&["a1", "a2", "a3"]]);
+ expect_failure("a1,a2,a3", [&["a1", "a2"]], "a3");
+ expect_success("a3,a1,a2", [&["a1", "a2", "a3"]]);
+ expect_failure("a3,a1,a2", [&["a1", "a2"]], "a3");
+ expect_success("a1,a2", [&["a1", "a2", "a3"]]);
+ expect_failure("a1,a2", [&["a1", "a3"]], "a2");
+
+ // Check it doesn't have false positives with prefixes
+ expect_failure("a1,a2,a3", [&["a1", "a2", "a"]], "a3");
+ expect_any_failure("a3,a1,a2", [&["a"]]);
+ expect_success("a1,a2", [&["a1", "a2", "a3"]]);
+ expect_failure("a1,a2", [&["a1", "a3"]], "a2");
+
+ expect_failure("a1b,a2b", [&["a1b", "a3b"]], "a2b");
+ expect_failure("a1b,a2b", [&["a1b", "a3b"]], "a2b");
+ expect_failure("a1b,a2b", [&["a1b", "a3b"]], "a2b");
+ expect_failure("a1b,a2b", [&["a1b", "a3b"]], "a2b");
+ }
+
+ #[test]
+ fn empty_feature() {
+ expect_failure("a,b,", [&["a", "b"]], "");
+ expect_failure("", [&["a", "b"]], "");
+ }
+
+ #[test]
+ fn non_ascii_features() {
+ expect_success("café", [&["café"]]);
+ expect_failure("café", [&["cafe"]], "café");
+ }
+}
diff --git a/fearless_simd_core/src/x86/mod.rs b/fearless_simd_core/src/x86/mod.rs
new file mode 100644
index 00000000..10caa27a
--- /dev/null
+++ b/fearless_simd_core/src/x86/mod.rs
@@ -0,0 +1,15 @@
+//! Target feature tokens for the x86 and x86-64 CPU families.
+//!
+//! The general compuotation CPU features associated with each [microarchitecture level] can
+//! be found in their corresponding modules:
+//!
+//! - [`v1`] for x86-64-v1.
+//! - [`v2`] for x86-64-v2.
+//! - [`v3`] for x86-64-v3.
+//! - [`v4`] for x86-64-v4.
+//!
+//! Tokens for target features which not associated with these levels can be found in this module.
+//!
+//! [microarchitecture level]: https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels
+
+pub mod v1;
diff --git a/fearless_simd_core/src/x86/v1/fxsr.rs b/fearless_simd_core/src/x86/v1/fxsr.rs
new file mode 100644
index 00000000..a4c99085
--- /dev/null
+++ b/fearless_simd_core/src/x86/v1/fxsr.rs
@@ -0,0 +1,80 @@
+//! The FXSR target feature.
+
+use core::fmt::Debug;
+
+use crate::{TargetFeatureToken, trampoline};
+
+/// A token indicating that the current CPU has the FXSR target feature.
+///
+/// The Rust target feature name for this feature is `fxsr`.
+/// For example, this can be used to [`trampoline!`] into:
+///
+/// ```rust
+/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+/// #[target_feature(enable = "fxsr")]
+/// fn uses_fxsr() {
+/// // ...
+/// }
+/// ```
+#[derive(Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Fxsr {
+ // We don't use non_exhaustive because we don't want this struct to be constructible.
+ // in different modules in this crate.
+ _private: (),
+}
+
+impl Debug for Fxsr {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ write!(f, r#""fxsr" enabled."#)
+ }
+}
+
+unsafe impl TargetFeatureToken for Fxsr {
+ const FEATURES: &[&str] = &["fxsr"];
+
+ #[inline(always)]
+ fn vectorize(self, f: impl FnOnce() -> R) -> R {
+ trampoline!([Self = self] => "fxsr", <(R)> fn<(R)>(f: impl FnOnce() -> R = f) -> R { f() })
+ }
+}
+
+impl Fxsr {
+ #[cfg(feature = "std")]
+ /// Create a new token if the "fxsr" target feature is detected as enabled.
+ ///
+ /// This does not do any caching internally, although note that the standard
+ /// library does internally cache the features it detects.
+ // TODO: Consider a manual override feature/env var?
+ pub fn try_new() -> Option {
+ // Feature flag required to make docs compile.
+ // TODO: Extract into a (private) crate::x86::is_x86_feature_detected?
+ #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+ if std::arch::is_x86_feature_detected!("fxsr") {
+ // Safety: The required CPU feature was detected.
+ unsafe { Some(Self::new()) }
+ } else {
+ None
+ }
+ }
+
+ #[target_feature(enable = "fxsr")]
+ /// Create a new token for the "fxsr" target feature is enabled.
+ ///
+ /// This method is useful to get a new token if you have an
+ /// external proof that FXSR is available.
+ ///
+ /// # Safety
+ ///
+ /// No conditions other than those inherited from the target feature attribute,
+ /// i.e. that the "fxsr" target feature is available.
+ pub fn new() -> Self {
+ Self { _private: () }
+ }
+}
+
+const _: () = {
+ assert!(
+ core::mem::size_of::() == 0,
+ "Target feature tokens should be zero sized."
+ );
+};
diff --git a/fearless_simd_core/src/x86/v1/mod.rs b/fearless_simd_core/src/x86/v1/mod.rs
new file mode 100644
index 00000000..0e360b50
--- /dev/null
+++ b/fearless_simd_core/src/x86/v1/mod.rs
@@ -0,0 +1,38 @@
+//! Target features enabled in the `x86-64-v1` [microarchitecture level](https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels) on x86 and x86-64.
+//!
+//! This is the baseline for x86-64 support.
+
+mod sse;
+pub use sse::Sse;
+
+mod fxsr;
+pub use fxsr::Fxsr;
+
+/// A token that the current CPU is on the x86-64-v1 microarchitecture level.
+// TODO: (This is currently incomplete)
+pub struct V1 {
+ pub sse: Sse,
+ pub fxsr: Fxsr,
+}
+
+impl V1 {
+ /// Create a new token if the current CPU is at the x86-64-v1 microarchitecture level or better.
+ ///
+ /// This does not do any caching internally, although note that the standard
+ /// library does internally cache the features it detects.
+ #[cfg(feature = "std")]
+ pub fn try_new() -> Option {
+ // TODO: Caching
+ Some(Self {
+ fxsr: Fxsr::try_new()?,
+ sse: Sse::try_new()?,
+ })
+ }
+}
+
+const _: () = {
+ assert!(
+ core::mem::size_of::() == 0,
+ "Target feature tokens should be zero sized."
+ );
+};
diff --git a/fearless_simd_core/src/x86/v1/sse.rs b/fearless_simd_core/src/x86/v1/sse.rs
new file mode 100644
index 00000000..67f2160c
--- /dev/null
+++ b/fearless_simd_core/src/x86/v1/sse.rs
@@ -0,0 +1,90 @@
+//! The SSE target feature.
+
+use crate::{TargetFeatureToken, trampoline};
+
+use core::fmt::Debug;
+
+/// A token indicating that the current CPU has the SSE target feature.
+///
+/// The Rust target feature name for this feature is `sse`.
+///
+/// See for more information about these instructions.
+/// This feature also implictily enables
+///
+/// # Example
+///
+/// This can be used to [`trampoline!`] into:
+///
+/// ```rust
+/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+/// #[target_feature(enable = "sse")]
+/// fn uses_sse() {
+/// // ...
+/// }
+/// ```
+///
+#[derive(Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Sse {
+ // We don't use non_exhaustive because we don't want this struct to be constructible.
+ // in different modules in this crate.
+ _private: (),
+}
+
+impl Debug for Sse {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ write!(f, r#""sse" enabled."#)
+ }
+}
+
+unsafe impl TargetFeatureToken for Sse {
+ const FEATURES: &[&str] = &["sse"];
+
+ #[inline(always)]
+ fn vectorize(self, f: impl FnOnce() -> R) -> R {
+ trampoline!([Self = self] => "sse", <(R)> fn<(R)>(f: impl FnOnce() -> R = f) -> R { f() })
+ }
+}
+
+impl Sse {
+ #[cfg(feature = "std")]
+ /// Create a new token if the `"sse"` target feature is detected as enabled.
+ ///
+ /// This does not do any caching internally, although note that the standard
+ /// library does internally cache the features it detects.
+ // TODO: Consider a manual override feature/env var?
+ pub fn try_new() -> Option {
+ // Feature flag required to make docs compile.
+ // TODO: Extract into a (private) crate::x86::is_x86_feature_detected?
+ #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+ if std::arch::is_x86_feature_detected!("sse") {
+ // Safety: The required CPU feature was detected.
+ unsafe { Some(Self::new()) }
+ } else {
+ None
+ }
+ }
+
+ #[target_feature(enable = "sse")]
+ /// Create a new token for the "sse" target feature is enabled.
+ ///
+ /// This method is useful to get a new token if you have an external proof that
+ /// SSE is available. This could happen if you have a token for a target feature
+ /// which [implicitly enables] `sse`.
+ ///
+ /// # Safety
+ ///
+ /// No conditions other than those inherited from the target feature attribute,
+ /// i.e. that the "sse" target feature is available.
+ ///
+ /// [implicitly enables]: https://doc.rust-lang.org/beta/reference/attributes/codegen.html#r-attributes.codegen.target_feature.x86
+ pub fn new() -> Self {
+ Self { _private: () }
+ }
+}
+
+const _: () = {
+ assert!(
+ core::mem::size_of::() == 0,
+ "Target feature tokens should be zero sized."
+ );
+};
From 78d23d8f79ea48e047018e77568f1da06ab03443 Mon Sep 17 00:00:00 2001
From: Daniel McNab <36049421+DJMcNab@users.noreply.github.com>
Date: Tue, 14 Oct 2025 14:43:24 +0100
Subject: [PATCH 02/19] Remove old v1 items
---
fearless_simd_core/src/x86/v1/fxsr.rs | 80 ------------------------
fearless_simd_core/src/x86/v1/sse.rs | 90 ---------------------------
2 files changed, 170 deletions(-)
delete mode 100644 fearless_simd_core/src/x86/v1/fxsr.rs
delete mode 100644 fearless_simd_core/src/x86/v1/sse.rs
diff --git a/fearless_simd_core/src/x86/v1/fxsr.rs b/fearless_simd_core/src/x86/v1/fxsr.rs
deleted file mode 100644
index a4c99085..00000000
--- a/fearless_simd_core/src/x86/v1/fxsr.rs
+++ /dev/null
@@ -1,80 +0,0 @@
-//! The FXSR target feature.
-
-use core::fmt::Debug;
-
-use crate::{TargetFeatureToken, trampoline};
-
-/// A token indicating that the current CPU has the FXSR target feature.
-///
-/// The Rust target feature name for this feature is `fxsr`.
-/// For example, this can be used to [`trampoline!`] into:
-///
-/// ```rust
-/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
-/// #[target_feature(enable = "fxsr")]
-/// fn uses_fxsr() {
-/// // ...
-/// }
-/// ```
-#[derive(Copy, Clone, Hash, PartialEq, Eq)]
-pub struct Fxsr {
- // We don't use non_exhaustive because we don't want this struct to be constructible.
- // in different modules in this crate.
- _private: (),
-}
-
-impl Debug for Fxsr {
- fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
- write!(f, r#""fxsr" enabled."#)
- }
-}
-
-unsafe impl TargetFeatureToken for Fxsr {
- const FEATURES: &[&str] = &["fxsr"];
-
- #[inline(always)]
- fn vectorize(self, f: impl FnOnce() -> R) -> R {
- trampoline!([Self = self] => "fxsr", <(R)> fn<(R)>(f: impl FnOnce() -> R = f) -> R { f() })
- }
-}
-
-impl Fxsr {
- #[cfg(feature = "std")]
- /// Create a new token if the "fxsr" target feature is detected as enabled.
- ///
- /// This does not do any caching internally, although note that the standard
- /// library does internally cache the features it detects.
- // TODO: Consider a manual override feature/env var?
- pub fn try_new() -> Option {
- // Feature flag required to make docs compile.
- // TODO: Extract into a (private) crate::x86::is_x86_feature_detected?
- #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
- if std::arch::is_x86_feature_detected!("fxsr") {
- // Safety: The required CPU feature was detected.
- unsafe { Some(Self::new()) }
- } else {
- None
- }
- }
-
- #[target_feature(enable = "fxsr")]
- /// Create a new token for the "fxsr" target feature is enabled.
- ///
- /// This method is useful to get a new token if you have an
- /// external proof that FXSR is available.
- ///
- /// # Safety
- ///
- /// No conditions other than those inherited from the target feature attribute,
- /// i.e. that the "fxsr" target feature is available.
- pub fn new() -> Self {
- Self { _private: () }
- }
-}
-
-const _: () = {
- assert!(
- core::mem::size_of::() == 0,
- "Target feature tokens should be zero sized."
- );
-};
diff --git a/fearless_simd_core/src/x86/v1/sse.rs b/fearless_simd_core/src/x86/v1/sse.rs
deleted file mode 100644
index 67f2160c..00000000
--- a/fearless_simd_core/src/x86/v1/sse.rs
+++ /dev/null
@@ -1,90 +0,0 @@
-//! The SSE target feature.
-
-use crate::{TargetFeatureToken, trampoline};
-
-use core::fmt::Debug;
-
-/// A token indicating that the current CPU has the SSE target feature.
-///
-/// The Rust target feature name for this feature is `sse`.
-///
-/// See for more information about these instructions.
-/// This feature also implictily enables
-///
-/// # Example
-///
-/// This can be used to [`trampoline!`] into:
-///
-/// ```rust
-/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
-/// #[target_feature(enable = "sse")]
-/// fn uses_sse() {
-/// // ...
-/// }
-/// ```
-///
-#[derive(Copy, Clone, Hash, PartialEq, Eq)]
-pub struct Sse {
- // We don't use non_exhaustive because we don't want this struct to be constructible.
- // in different modules in this crate.
- _private: (),
-}
-
-impl Debug for Sse {
- fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
- write!(f, r#""sse" enabled."#)
- }
-}
-
-unsafe impl TargetFeatureToken for Sse {
- const FEATURES: &[&str] = &["sse"];
-
- #[inline(always)]
- fn vectorize(self, f: impl FnOnce() -> R) -> R {
- trampoline!([Self = self] => "sse", <(R)> fn<(R)>(f: impl FnOnce() -> R = f) -> R { f() })
- }
-}
-
-impl Sse {
- #[cfg(feature = "std")]
- /// Create a new token if the `"sse"` target feature is detected as enabled.
- ///
- /// This does not do any caching internally, although note that the standard
- /// library does internally cache the features it detects.
- // TODO: Consider a manual override feature/env var?
- pub fn try_new() -> Option {
- // Feature flag required to make docs compile.
- // TODO: Extract into a (private) crate::x86::is_x86_feature_detected?
- #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
- if std::arch::is_x86_feature_detected!("sse") {
- // Safety: The required CPU feature was detected.
- unsafe { Some(Self::new()) }
- } else {
- None
- }
- }
-
- #[target_feature(enable = "sse")]
- /// Create a new token for the "sse" target feature is enabled.
- ///
- /// This method is useful to get a new token if you have an external proof that
- /// SSE is available. This could happen if you have a token for a target feature
- /// which [implicitly enables] `sse`.
- ///
- /// # Safety
- ///
- /// No conditions other than those inherited from the target feature attribute,
- /// i.e. that the "sse" target feature is available.
- ///
- /// [implicitly enables]: https://doc.rust-lang.org/beta/reference/attributes/codegen.html#r-attributes.codegen.target_feature.x86
- pub fn new() -> Self {
- Self { _private: () }
- }
-}
-
-const _: () = {
- assert!(
- core::mem::size_of::() == 0,
- "Target feature tokens should be zero sized."
- );
-};
From 9cfa3345c085a88997f73a115428b1d723461e35 Mon Sep 17 00:00:00 2001
From: Daniel McNab <36049421+DJMcNab@users.noreply.github.com>
Date: Tue, 14 Oct 2025 14:43:37 +0100
Subject: [PATCH 03/19] Improve the generator
---
fearless_simd_core/gen/src/data/x86.rs | 170 ++++++++++++++++--------
fearless_simd_core/gen/src/main.rs | 38 ++++--
fearless_simd_core/gen/templates/x86.rs | 14 +-
3 files changed, 144 insertions(+), 78 deletions(-)
diff --git a/fearless_simd_core/gen/src/data/x86.rs b/fearless_simd_core/gen/src/data/x86.rs
index 39e59990..346a9f79 100644
--- a/fearless_simd_core/gen/src/data/x86.rs
+++ b/fearless_simd_core/gen/src/data/x86.rs
@@ -2,7 +2,7 @@ use crate::Feature;
macro_rules! f {
($(#[doc = $doc_addition: literal])*
- struct ::$module: ident:: $struct_name: ident($display_name: literal): $feature_name: literal + [$($implicitly_enabled: literal),*]
+ struct $module: ident:: $struct_name: ident($display_name: literal): $feature_name: literal + [$($implicitly_enabled: literal),*]
fn $example_function_name: ident
$($additional_impls: tt)*
) => {
@@ -29,339 +29,397 @@ pub(crate) const X86_TEMPLATE: &str = include_str!("../../templates/x86.rs");
pub(crate) const X86_FEATURES: &[Feature] = &[
f!(
/// [ADX] --- Multi-Precision Add-Carry Instruction Extensions
+ ///
/// [ADX]: https://en.wikipedia.org/wiki/Intel_ADX
- struct ::adx::Adx("ADX"): "adx" + []
+ struct adx::Adx("ADX"): "adx" + []
fn uses_adx
),
f!(
/// [AES] --- Advanced Encryption Standard
+ ///
/// [AES]: https://en.wikipedia.org/wiki/AES_instruction_set
- struct ::crypto::Aes("AES"): "aes" + ["sse2"]
+ struct crypto::Aes("AES"): "aes" + ["sse2"]
fn uses_aes
),
f!(
/// [AVX] --- Advanced Vector Extensions
+ ///
/// [AVX]: https://en.wikipedia.org/wiki/Advanced_Vector_Extensions
- struct ::avx::Avx("AVX"): "avx" + ["sse4.2"]
+ struct avx::Avx("AVX"): "avx" + ["sse4.2"]
fn uses_avx
),
f!(
/// [AVX2] --- Advanced Vector Extensions 2
+ ///
/// [AVX2]: https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#AVX2
- struct ::avx::Avx2("AVX2"): "avx2" + ["avx"]
+ struct avx::Avx2("AVX2"): "avx2" + ["avx"]
fn uses_avx2
),
f!(
/// [AVX512-BF16] --- Advanced Vector Extensions 512-bit - Bfloat16 Extensions
+ ///
/// [AVX512-BF16]: https://en.wikipedia.org/wiki/AVX-512#BF16
- struct ::avx512::Avx512bf16("AVX512-BF16"): "avx512bf16" + ["avx512bw"]
+ struct avx512::Avx512bf16("AVX512-BF16"): "avx512bf16" + ["avx512bw"]
fn uses_avx512bf16
),
f!(
/// [AVX512-BITALG] --- Advanced Vector Extensions 512-bit - Bit Algorithms
+ ///
+ ///
/// [AVX512-BITALG]: https://en.wikipedia.org/wiki/AVX-512#VPOPCNTDQ_and_BITALG
- struct ::avx512::Avx512bitalg("AVX512-BITALG"): "avx512bitalg" + ["avx512bw"]
+ struct avx512::Avx512bitalg("AVX512-BITALG"): "avx512bitalg" + ["avx512bw"]
fn uses_avx512bitalg
),
f!(
/// [AVX512-BW] --- Advanced Vector Extensions 512-bit - Byte and Word Instructions
+ ///
/// [AVX512-BW]: https://en.wikipedia.org/wiki/AVX-512#BW,_DQ_and_VBMI
- struct ::avx512::Avx512bw("AVX512-BW"): "avx512bw" + ["avx512f"]
+ struct avx512::Avx512bw("AVX512-BW"): "avx512bw" + ["avx512f"]
fn uses_avx512bw
),
f!(
/// [AVX512-CD] --- Advanced Vector Extensions 512-bit - Conflict Detection Instructions
+ ///
/// [AVX512-CD]: https://en.wikipedia.org/wiki/AVX-512#Conflict_detection
- struct ::avx512::Avx512cd("AVX512-CD"): "avx512cd" + ["avx512f"]
+ struct avx512::Avx512cd("AVX512-CD"): "avx512cd" + ["avx512f"]
fn uses_avx512cd
),
f!(
/// [AVX512-DQ] --- Advanced Vector Extensions 512-bit - Doubleword and Quadword Instructions
+ ///
/// [AVX512-DQ]: https://en.wikipedia.org/wiki/AVX-512#BW,_DQ_and_VBMI
- struct ::avx512::Avx512dq("AVX512-DQ"): "avx512dq" + ["avx512f"]
+ struct avx512::Avx512dq("AVX512-DQ"): "avx512dq" + ["avx512f"]
fn uses_avx512dq
),
f!(
/// [AVX512-F] --- Advanced Vector Extensions 512-bit - Foundation
+ ///
/// [AVX512-F]: https://en.wikipedia.org/wiki/AVX-512
- struct ::avx512::Avx512f("AVX512-F"): "avx512f" + ["avx2", "fma", "f16c"]
+ struct avx512::Avx512f("AVX512-F"): "avx512f" + ["avx2", "fma", "f16c"]
fn uses_avx512f
),
f!(
/// [AVX512-FP16] --- Advanced Vector Extensions 512-bit - Float16 Extensions
+ ///
/// [AVX512-FP16]: https://en.wikipedia.org/wiki/AVX-512#FP16
- struct ::avx512::Avx512fp16("AVX512-FP16"): "avx512fp16" + ["avx512bw"]
+ struct avx512::Avx512fp16("AVX512-FP16"): "avx512fp16" + ["avx512bw"]
fn uses_avx512fp16
),
f!(
/// [AVX512-IFMA] --- Advanced Vector Extensions 512-bit - Integer Fused Multiply Add
+ ///
/// [AVX512-IFMA]: https://en.wikipedia.org/wiki/AVX-512#IFMA
- struct ::avx512::Avx512ifma("AVX512-IFMA"): "avx512ifma" + ["avx512f"]
+ struct avx512::Avx512ifma("AVX512-IFMA"): "avx512ifma" + ["avx512f"]
fn uses_avx512ifma
),
f!(
/// [AVX512-VBMI] --- Advanced Vector Extensions 512-bit - Vector Byte Manipulation Instructions
+ ///
/// [AVX512-VBMI]: https://en.wikipedia.org/wiki/AVX-512#BW,_DQ_and_VBMI
- struct ::avx512::Avx512vbmi("AVX512-VBMI"): "avx512vbmi" + ["avx512bw"]
+ struct avx512::Avx512vbmi("AVX512-VBMI"): "avx512vbmi" + ["avx512bw"]
fn uses_avx512vbmi
),
f!(
/// [AVX512-VBMI2] --- Advanced Vector Extensions 512-bit - Vector Byte Manipulation Instructions 2
+ ///
/// [AVX512-VBMI2]: https://en.wikipedia.org/wiki/AVX-512#VBMI2
- struct ::avx512::Avx512vbmi2("AVX512-VBMI2"): "avx512vbmi2" + ["avx512bw"]
+ struct avx512::Avx512vbmi2("AVX512-VBMI2"): "avx512vbmi2" + ["avx512bw"]
fn uses_avx512vbmi2
),
f!(
/// [AVX512-VL] --- Advanced Vector Extensions 512-bit - Vector Length Extensions
+ ///
/// [AVX512-VL]: https://en.wikipedia.org/wiki/AVX-512
- struct ::avx512::Avx512vl("AVX512-VL"): "avx512vl" + ["avx512f"]
+ struct avx512::Avx512vl("AVX512-VL"): "avx512vl" + ["avx512f"]
fn uses_avx512vl
),
f!(
/// [AVX512-VNNI] --- Advanced Vector Extensions 512-bit - Vector Neural Network Instructions
+ ///
/// [AVX512-VNNI]: https://en.wikipedia.org/wiki/AVX-512#VNNI
- struct ::avx512::Avx512vnni("AVX512-VNNI"): "avx512vnni" + ["avx512f"]
+ struct avx512::Avx512vnni("AVX512-VNNI"): "avx512vnni" + ["avx512f"]
fn uses_avx512vnni
),
f!(
/// [AVX512-VP2INTERSECT] --- Advanced Vector Extensions 512-bit - Vector Pair Intersection to a Pair of Mask Registers
+ ///
/// [AVX512-VP2INTERSECT]: https://en.wikipedia.org/wiki/AVX-512#VP2INTERSECT
- struct ::avx512::Avx512vp2intersect("AVX512-VP2INTERSECT"): "avx512vp2intersect" + ["avx512f"]
+ struct avx512::Avx512vp2intersect("AVX512-VP2INTERSECT"): "avx512vp2intersect" + ["avx512f"]
fn uses_avx512vp2intersect
),
f!(
/// [AVX512-VPOPCNTDQ] --- Advanced Vector Extensions 512-bit - Vector Population Count Instruction
+ ///
/// [AVX512-VPOPCNTDQ]:https://en.wikipedia.org/wiki/AVX-512#VPOPCNTDQ_and_BITALG
- struct ::avx512::Avx512vpopcntdq("AVX512-VPOPCNTDQ"): "avx512vpopcntdq" + ["avx512f"]
+ struct avx512::Avx512vpopcntdq("AVX512-VPOPCNTDQ"): "avx512vpopcntdq" + ["avx512f"]
fn uses_avx512vpopcntdq
),
f!(
/// [AVX-IFMA] --- Advanced Vector Extensions - Integer Fused Multiply Add
+ ///
/// [AVX-IFMA]: https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#AVX-VNNI,_AVX-IFMA
- struct ::avx::Avxifma("AVX-IFMA"): "avxifma" + ["avx2"]
+ struct avx::Avxifma("AVX-IFMA"): "avxifma" + ["avx2"]
fn uses_avxifma
),
f!(
/// [AVX-NE-CONVERT] --- Advanced Vector Extensions - No-Exception Floating-Point conversion Instructions
+ ///
/// [AVX-NE-CONVERT]: https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#AVX-VNNI,_AVX-IFMA
- struct ::avx::Avxneconvert("AVX-NE-CONVERT"): "avxneconvert" + ["avx2"]
+ struct avx::Avxneconvert("AVX-NE-CONVERT"): "avxneconvert" + ["avx2"]
fn uses_avxneconvert
),
f!(
/// [AVX-VNNI] --- Advanced Vector Extensions - Vector Neural Network Instructions
+ ///
/// [AVX-VNNI]: https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#AVX-VNNI,_AVX-IFMA
- struct ::avx::Avxvnni("AVX-VNNI"): "avxvnni" + ["avx2"]
+ struct avx::Avxvnni("AVX-VNNI"): "avxvnni" + ["avx2"]
fn uses_avxvnni
),
f!(
/// [AVX-VNNI-INT16] --- Advanced Vector Extensions - Vector Neural Network Instructions with 16-bit Integers
+ ///
/// [AVX-VNNI-INT16]: https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#AVX-VNNI,_AVX-IFMA
- struct ::avx::Avxvnniint16("AVX-VNNI-INT16"): "avxvnniint16" + ["avx2"]
+ struct avx::Avxvnniint16("AVX-VNNI-INT16"): "avxvnniint16" + ["avx2"]
fn uses_avxvnniint16
),
f!(
/// [AVX-VNNI-INT8] --- Advanced Vector Extensions - Vector Neural Network Instructions with 8-bit Integers
+ ///
/// [AVX-VNNI-INT8]: https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#AVX-VNNI,_AVX-IFMA
- struct ::avx::Avxvnniint8("AVX-VNNI-INT8"): "avxvnniint8" + ["avx2"]
+ struct avx::Avxvnniint8("AVX-VNNI-INT8"): "avxvnniint8" + ["avx2"]
fn uses_avxvnniint8
),
f!(
/// [BMI1] --- Bit Manipulation Instruction Sets
+ ///
/// [BMI1]: https://en.wikipedia.org/wiki/Bit_Manipulation_Instruction_Sets
- struct ::v3::Bmi1(" 1"): "bmi1" + []
+ struct v3::Bmi1(" 1"): "bmi1" + []
fn uses_bmi1
),
f!(
/// [BMI2] --- Bit Manipulation Instruction Sets 2
+ ///
/// [BMI2]: https://en.wikipedia.org/wiki/Bit_Manipulation_Instruction_Sets#BMI2
- struct ::v3::Bmi2("BMI2"): "bmi2" + []
+ struct v3::Bmi2("BMI2"): "bmi2" + []
fn uses_bmi2
),
f!(
/// ["cmpxchg16b"] --- Compares and exchange 16 bytes (128 bits) of data atomically
+ ///
/// ["cmpxchg16b"]: https://www.felixcloutier.com/x86/cmpxchg8b:cmpxchg16b
- struct ::v2::Cmpxchg16b("`cmpxchg16b`"): "cmpxchg16b" + []
+ struct v2::Cmpxchg16b("`cmpxchg16b`"): "cmpxchg16b" + []
fn uses_cmpxchg16b
),
f!(
/// [F16C] --- 16-bit floating point conversion instructions
+ ///
/// [F16C]: https://en.wikipedia.org/wiki/F16C
- struct ::v3::F16c("F16C"): "f16c" + ["avx"]
+ struct v3::F16c("F16C"): "f16c" + ["avx"]
fn uses_f16c
),
f!(
/// [FMA3] --- Three-operand fused multiply-add
+ ///
/// [FMA3]: https://en.wikipedia.org/wiki/FMA_instruction_set
- struct ::v3::Fma("FMA3"): "fma" + ["avx"]
+ struct v3::Fma("FMA3"): "fma" + ["avx"]
fn uses_fma
),
f!(
/// ["fxsave"] and ["fxrstor"] --- Save and restore x87 FPU, MMX Technology, and SSE State
+ ///
/// ["fxsave"]: https://www.felixcloutier.com/x86/fxsave,
- struct ::sse::Fxsr("`fxsave + fxrstor`"): "fxsr" + []
+ /// ["fxrstor"]: https://www.felixcloutier.com/x86/fxrstor,
+ struct sse::Fxsr("`fxsave + fxrstor`"): "fxsr" + []
fn uses_fxsr
),
f!(
/// [GFNI] --- Galois Field New Instructions
+ ///
/// [GFNI]: https://en.wikipedia.org/wiki/AVX-512#GFNI
- struct ::crypto::Gfni("GFNI"): "gfni" + ["sse2"]
+ struct crypto::Gfni("GFNI"): "gfni" + ["sse2"]
fn uses_gfni
),
f!(
/// [KEYLOCKER] --- Intel Key Locker Instructions
+ ///
/// [KEYLOCKER]: https://en.wikipedia.org/wiki/List_of_x86_cryptographic_instructions#Intel_Key_Locker_instructions
- struct ::crypto::Keylocker("KEYLOCKER"): "kl" + []
+ struct crypto::Keylocker("KEYLOCKER"): "kl" + []
fn uses_keylocker
),
f!(
/// ["lzcnt"] --- Leading zeros count
+ ///
/// ["lzcnt"]: https://www.felixcloutier.com/x86/lzcnt
- struct ::v3::Lzcnt("`lzcnt`"): "lzcnt" + []
+ struct v3::Lzcnt("`lzcnt`"): "lzcnt" + []
fn uses_lzcnt
),
f!(
/// ["movbe"] --- Move data after swapping bytes
+ ///
/// ["movbe"]: https://www.felixcloutier.com/x86/movbe
- struct ::v3::Movbe("`movbe`"): "movbe" + []
+ struct v3::Movbe("`movbe`"): "movbe" + []
fn uses_movbe
),
f!(
/// ["pclmulqdq"] --- Packed carry-less multiplication quadword
+ ///
/// ["pclmulqdq"]: https://www.felixcloutier.com/x86/pclmulqdq
- struct ::crypto::Pclmulqdq("`pclmulqdq`"): "pclmulqdq" + ["sse2"]
+ struct crypto::Pclmulqdq("`pclmulqdq`"): "pclmulqdq" + ["sse2"]
fn uses_pclmulqdq
),
f!(
/// ["popcnt"] --- Count of bits set to 1
+ ///
/// ["popcnt"]: https://www.felixcloutier.com/x86/popcnt
- struct ::v2::Popcnt("`popcnt`"): "popcnt" + []
+ struct v2::Popcnt("`popcnt`"): "popcnt" + []
fn uses_popcnt
),
f!(
/// ["rdrand"] --- Read random number
+ ///
/// ["rdrand"]: https://en.wikipedia.org/wiki/RdRand
- struct ::crypto::Rdrand("`rdrand`"): "rdrand" + []
+ struct crypto::Rdrand("`rdrand`"): "rdrand" + []
fn uses_rdrand
),
f!(
/// ["rdseed"] --- Read random seed
+ ///
/// ["rdseed"]: https://en.wikipedia.org/wiki/RdRand
- struct ::crypto::Rdseed("`rdseed"): "rdseed" + []
+ struct crypto::Rdseed("`rdseed"): "rdseed" + []
fn uses_rdseed
),
f!(
/// [SHA] --- Secure Hash Algorithm
+ ///
/// [SHA]: https://en.wikipedia.org/wiki/Intel_SHA_extensions
- struct ::crypto::Sha("SHA"): "sha" + ["sse2"]
+ struct crypto::Sha("SHA"): "sha" + ["sse2"]
fn uses_sha
),
f!(
/// [SHA512] --- Secure Hash Algorithm with 512-bit digest
+ ///
/// [SHA512]: https://en.wikipedia.org/wiki/Intel_SHA_extensions
- struct ::crypto::Sha512("SHA512"): "sha512" + ["avx2"]
+ struct crypto::Sha512("SHA512"): "sha512" + ["avx2"]
fn uses_sha512
),
f!(
/// [SM3] --- ShangMi 3 Hash Algorithm
+ ///
/// [SM3]: https://en.wikipedia.org/wiki/List_of_x86_cryptographic_instructions#Intel_SHA_and_SM3_instructions
- struct ::crypto::Sm3("SM3"): "sm3" + ["avx"]
+ struct crypto::Sm3("SM3"): "sm3" + ["avx"]
fn uses_sm3
),
f!(
/// [SM4] --- ShangMi 4 Cipher Algorithm
+ ///
/// [SM4]: https://en.wikipedia.org/wiki/List_of_x86_cryptographic_instructions#Intel_SHA_and_SM3_instructions
- struct ::crypto::Sm4("SM4"): "sm4" + ["avx2"]
+ struct crypto::Sm4("SM4"): "sm4" + ["avx2"]
fn uses_sm4
),
f!(
/// [SSE] --- Streaming SIMD Extensions
+ ///
/// [SSE]: https://en.wikipedia.org/wiki/Streaming_SIMD_Extensions
- struct ::sse::Sse("SSE"): "sse" + []
+ struct sse::Sse("SSE"): "sse" + []
fn uses_sse
),
f!(
/// [SSE2] --- Streaming SIMD Extensions 2
+ ///
/// [SSE2]: https://en.wikipedia.org/wiki/SSE2
- struct ::sse::Sse2("SSE2"): "sse2" + ["sse"]
+ struct sse::Sse2("SSE2"): "sse2" + ["sse"]
fn uses_sse2
),
f!(
/// [SSE3] --- Streaming SIMD Extensions 3
+ ///
/// [SSE3]: https://en.wikipedia.org/wiki/SSE3
- struct ::sse::Sse3("SSE3"): "sse3" + ["sse2"]
+ struct sse::Sse3("SSE3"): "sse3" + ["sse2"]
fn uses_sse3
),
f!(
/// [SSE4.1] --- Streaming SIMD Extensions 4.1
+ ///
/// [SSE4.1]: https://en.wikipedia.org/wiki/SSE4#SSE4.1
- struct ::sse::Sse4_1("SSE4.1"): "sse4.1" + ["ssse3"]
+ struct sse::Sse4_1("SSE4.1"): "sse4.1" + ["ssse3"]
fn uses_sse4
),
f!(
/// [SSE4.2] --- StreamingSIMDExtensions 4.2
+ ///
/// [SSE4.2]: https://en.wikipedia.org/wiki/SSE4#SSE4.2
- struct ::sse::Sse4_2("SSE4.2"): "sse4.2" + ["sse4.1"]
+ struct sse::Sse4_2("SSE4.2"): "sse4.2" + ["sse4.1"]
fn uses_sse4
),
// // TODO: This only exists from 1.91 and above (current beta)
// f!(
// /// [SSE4a] --- StreamingSIMDExtensions 4a
+
// /// [SSE4a]: https://en.wikipedia.org/wiki/SSE4#SSE4a
// struct Sse4a("SSE4a"): "sse4a" + ["sse3"]
// fn uses_sse4a
// ),
f!(
/// [SSSE3] --- Supplemental StreamingSIMDExtensions 3
+ ///
/// [SSSE3]: https://en.wikipedia.org/wiki/SSSE3
- struct ::sse::SupplementalSse3("SSSE3"): "ssse3" + ["sse3"]
+ struct sse::SupplementalSse3("SSSE3"): "ssse3" + ["sse3"]
fn uses_ssse3
),
f!(
/// [TBM] --- Trailing Bit Manipulation
+ ///
/// [TBM]: https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set#TBM_(Trailing_Bit_Manipulation)
- struct ::discontinued::Tbm("TBM"): "tbm" + []
+ struct discontinued::Tbm("TBM"): "tbm" + []
fn uses_tbm
),
f!(
/// [VAES] --- Vector AES Instructions
+ ///
/// [VAES]: https://en.wikipedia.org/wiki/AVX-512#VAES
- struct ::crypto::Vaes("VAES"): "vaes" + ["avx2", "aes"]
+ struct crypto::Vaes("VAES"): "vaes" + ["avx2", "aes"]
fn uses_vaes
),
f!(
/// [VPCLMULQDQ] --- Vector Carry-less multiplication of Quadwords
+ ///
/// [VPCLMULQDQ]: https://en.wikipedia.org/wiki/AVX-512#VPCLMULQDQ
- struct ::crypto::Vpclmulqdq("VPCLMULQDQ"): "vpclmulqdq" + ["avx", "pclmulqdq"]
+ struct crypto::Vpclmulqdq("VPCLMULQDQ"): "vpclmulqdq" + ["avx", "pclmulqdq"]
fn uses_vpclmulqdq
),
f!(
/// [KEYLOCKER_WIDE] --- Intel Wide Keylocker Instructions
+ ///
/// [KEYLOCKER_WIDE]: https://en.wikipedia.org/wiki/List_of_x86_cryptographic_instructions#Intel_Key_Locker_instructions
- struct ::crypto::WideKeylocker("KEYLOCKER_WIDE"): "widekl" + ["kl"]
+ struct crypto::WideKeylocker("WIDE KEYLOCKER"): "widekl" + ["kl"]
fn uses_wide_keylocker
),
f!(
/// [`xsave`] --- Save processor extended states
+ ///
/// ["xsave"]: https://www.felixcloutier.com/x86/xsave
- struct ::xsave::Xsave("`xsave`"): "xsave" + []
+ struct xsave::Xsave("`xsave`"): "xsave" + []
fn uses_xsave
),
f!(
/// ["xsavec"] --- Save processor extended states with compaction
+ ///
/// ["xsavec"]: https://www.felixcloutier.com/x86/xsavec
- struct ::xsave::Xsavec("`xsavec`"): "xsavec" + []
+ struct xsave::Xsavec("`xsavec`"): "xsavec" + []
fn uses_xsavec
),
f!(
/// ["xsaveopt"] --- Save processor extended states optimized
+ ///
/// ["xsaveopt"]: https://www.felixcloutier.com/x86/xsaveopt
- struct ::xsave::Xsaveopt("`xsaveopt`"): "xsaveopt" + []
+ struct xsave::Xsaveopt("`xsaveopt`"): "xsaveopt" + []
fn uses_xsaveopt
),
f!(
/// ["xsaves"] --- Save processor extended states supervisor
+ ///
/// ["xsaves"]: https://www.felixcloutier.com/x86/xsaves
- struct ::xsave::Xsaves("`xsaves`"): "xsaves" + []
+ struct xsave::Xsaves("`xsaves`"): "xsaves" + []
fn uses_xsaves
),
];
diff --git a/fearless_simd_core/gen/src/main.rs b/fearless_simd_core/gen/src/main.rs
index 2961bd33..1c0c8d43 100644
--- a/fearless_simd_core/gen/src/main.rs
+++ b/fearless_simd_core/gen/src/main.rs
@@ -1,6 +1,6 @@
mod data;
-use std::fmt::{Write, format};
+use std::fmt::Write;
use std::fs;
use std::{
cell::RefCell,
@@ -29,8 +29,16 @@ fn generate_for_arch(
for line in feature.feature.extra_docs.lines() {
writeln!(&mut new_docs, "///{line}").unwrap();
}
- let enabled_feature_docs = format!("`{}`", feature.children.join("`, `"));
- let enabled_feature_str_list = format!(r#""{}""#, feature.children.join(r#"", ""#));
+ let enabled_feature_str_list = format!(
+ r#""{}", {}"#,
+ feature.feature.feature_name,
+ feature
+ .children
+ .iter()
+ .map(|it| format!(r#""{it}""#))
+ .collect::>()
+ .join(", ")
+ );
let mut from_impls = String::new();
for child in &feature.children {
let from_feature = features
@@ -43,13 +51,13 @@ fn generate_for_arch(
);
write!(
from_impls,
- r#"\n\
- impl From for {type_path} {{
- fn from(value: Self) -> {type_path} {{
- trampoline!([Self = value] => "{{FEATURE_ID}}", fn() -> {type_path} {{ {{type_path}}::new() }})
- }}
- }}\n
- "#
+ "\n\
+impl From for {type_path} {{
+ fn from(value: FEATURE_STRUCT_NAME) -> Self {{
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([FEATURE_STRUCT_NAME = value] => \"{{FEATURE_ID}}\", fn() -> {type_path} {{ {type_path}::new() }})
+ }}
+}}\n"
).unwrap();
}
let mut result = format!(
@@ -60,18 +68,20 @@ fn generate_for_arch(
// We replace the from impls first, as they use template variables from the rest of this.
result = result.replace("/*{FROM_IMPLS}*/", &from_impls);
result = result.replace("{FEATURE_DOCS_NAME}", feature.feature.feature_docs_name);
- result = result.replace("/// {NEW_DOCS}", &new_docs);
+ result = result.replace("/// {NEW_DOCS}\n", &new_docs);
result = result.replace("{FEATURE_ID}", feature.feature.feature_name);
- result = result.replace("{ENABLED_FEATURES_DOCS_LIST}", &enabled_feature_docs);
result = result.replace(
"{EXAMPLE_FUNCTION_NAME}",
feature.feature.example_function_name,
);
result = result.replace("FEATURE_STRUCT_NAME", feature.feature.struct_name);
- result = result.replace("{ENABLED_FEATURES_STR_LIST}", &enabled_feature_str_list);
+ result = result.replace(
+ r#""{ENABLED_FEATURES_STR_LIST}""#,
+ &enabled_feature_str_list,
+ );
let module_dir = arch_dir.join(feature.feature.module);
create_dir_all(&module_dir)?;
- let mut file = module_dir.join(feature.feature.feature_name);
+ let mut file = module_dir.join(feature.feature.feature_name.replace(".", "_"));
file.set_extension("rs");
fs::write(file, result)?;
}
diff --git a/fearless_simd_core/gen/templates/x86.rs b/fearless_simd_core/gen/templates/x86.rs
index a37207a3..48a294d6 100644
--- a/fearless_simd_core/gen/templates/x86.rs
+++ b/fearless_simd_core/gen/templates/x86.rs
@@ -8,12 +8,9 @@ use core::fmt::Debug;
///
/// A token indicating that the current CPU has the `{FEATURE_ID}` target feature.
///
-/// This feature also enables {ENABLED_FEATURES_DOCS_LIST};
-/// the tokens for these features can be created using [`From`] implementations.
-///
/// # Example
///
-/// This can be used to [`trampoline!`] into:
+/// This can be used to [`trampoline!`] into functions like:
///
/// ```rust
/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
@@ -35,16 +32,17 @@ impl Debug for FEATURE_STRUCT_NAME {
}
}
-unsafe impl TargetFeatureToken for Sse {
- const FEATURES: &[&str] = &[{ ENABLED_FEATURES_STR_LIST }];
+unsafe impl TargetFeatureToken for FEATURE_STRUCT_NAME {
+ const FEATURES: &[&str] = &["{ENABLED_FEATURES_STR_LIST}"];
#[inline(always)]
fn vectorize(self, f: impl FnOnce() -> R) -> R {
- trampoline!([Self = self] => "{FEATURE_ID}", <(R)> fn<(R)>(f: impl FnOnce() -> R = f) -> R { f() })
+ // Because we want this constant to be eagerly evaluated.
+ trampoline!([FEATURE_STRUCT_NAME = self] => "{FEATURE_ID}", <(R)> fn<(R)>(f: impl FnOnce() -> R = f) -> R { f() })
}
}
-impl Sse {
+impl FEATURE_STRUCT_NAME {
#[cfg(feature = "std")]
/// Create a new token if the `"{FEATURE_ID}"` target feature is detected as enabled.
///
From 594a525b65eb2444f5fc09041a67a1c92675e8cd Mon Sep 17 00:00:00 2001
From: Daniel McNab <36049421+DJMcNab@users.noreply.github.com>
Date: Tue, 14 Oct 2025 14:50:50 +0100
Subject: [PATCH 04/19] Minor fixups in the generator
Also removes unused additional impl support
---
fearless_simd_core/gen/src/data/x86.rs | 4 +---
fearless_simd_core/gen/src/main.rs | 3 ---
fearless_simd_core/gen/templates/x86.rs | 6 +++---
3 files changed, 4 insertions(+), 9 deletions(-)
diff --git a/fearless_simd_core/gen/src/data/x86.rs b/fearless_simd_core/gen/src/data/x86.rs
index 346a9f79..52c13c4d 100644
--- a/fearless_simd_core/gen/src/data/x86.rs
+++ b/fearless_simd_core/gen/src/data/x86.rs
@@ -4,7 +4,6 @@ macro_rules! f {
($(#[doc = $doc_addition: literal])*
struct $module: ident:: $struct_name: ident($display_name: literal): $feature_name: literal + [$($implicitly_enabled: literal),*]
fn $example_function_name: ident
- $($additional_impls: tt)*
) => {
Feature {
struct_name: stringify!($struct_name),
@@ -13,7 +12,6 @@ macro_rules! f {
extra_docs: concat!($($doc_addition, "\n",)*),
example_function_name: stringify!($example_function_name),
feature_docs_name: $display_name,
- additional_impls: stringify!($($additional_impls)*),
module: stringify!($module)
}
}
@@ -285,7 +283,7 @@ pub(crate) const X86_FEATURES: &[Feature] = &[
/// ["rdseed"] --- Read random seed
///
/// ["rdseed"]: https://en.wikipedia.org/wiki/RdRand
- struct crypto::Rdseed("`rdseed"): "rdseed" + []
+ struct crypto::Rdseed("`rdseed`"): "rdseed" + []
fn uses_rdseed
),
f!(
diff --git a/fearless_simd_core/gen/src/main.rs b/fearless_simd_core/gen/src/main.rs
index 1c0c8d43..aec7057f 100644
--- a/fearless_simd_core/gen/src/main.rs
+++ b/fearless_simd_core/gen/src/main.rs
@@ -104,9 +104,6 @@ struct Feature {
example_function_name: &'static str,
/// The "display name" for the feature, used inside the docs.
feature_docs_name: &'static str,
- /// Extra code added at the end.
- /// Used for implicitly enabled features.
- additional_impls: &'static str,
/// The module (if any) this feature will belong to.
///
/// (Note that imports into the module are checked to exist, but not automatically inserted).
diff --git a/fearless_simd_core/gen/templates/x86.rs b/fearless_simd_core/gen/templates/x86.rs
index 48a294d6..7742b8e9 100644
--- a/fearless_simd_core/gen/templates/x86.rs
+++ b/fearless_simd_core/gen/templates/x86.rs
@@ -62,16 +62,16 @@ impl FEATURE_STRUCT_NAME {
}
#[target_feature(enable = "{FEATURE_ID}")]
- /// Create a new token for the "{FEATURE_ID}" target feature is enabled.
+ /// Create a new token for the "{FEATURE_ID}" target feature.
///
/// This method is useful to get a new token if you have an external proof that
- /// {FEATURE_ID} is available. This could happen if you are in a target feature
+ /// {FEATURE_DOCS_NAME} is available. This could happen if you are in a target feature
/// function called by an external library user.
///
/// # Safety
///
/// No conditions other than those inherited from the target feature attribute,
- /// i.e. that the "{FEATURE_DOCS_NAME}" target feature is available.
+ /// i.e. that the "{FEATURE_ID}" target feature is available.
///
/// [implicitly enables]: https://doc.rust-lang.org/beta/reference/attributes/codegen.html?highlight=implicitly%20enabled#r-attributes.codegen.target_feature.safety-restrictions
pub fn new() -> Self {
From 7b595b44f51bcb713bed15984c0be0421f28c9d7 Mon Sep 17 00:00:00 2001
From: Daniel McNab <36049421+DJMcNab@users.noreply.github.com>
Date: Tue, 14 Oct 2025 16:51:21 +0100
Subject: [PATCH 05/19] Add the generated x86 code
---
fearless_simd_core/src/x86/adx/adx.rs | 92 +++++++++
fearless_simd_core/src/x86/adx/mod.rs | 8 +
fearless_simd_core/src/x86/avx/avx.rs | 134 ++++++++++++
fearless_simd_core/src/x86/avx/avx2.rs | 143 +++++++++++++
fearless_simd_core/src/x86/avx/avxifma.rs | 150 ++++++++++++++
.../src/x86/avx/avxneconvert.rs | 158 +++++++++++++++
fearless_simd_core/src/x86/avx/avxvnni.rs | 150 ++++++++++++++
.../src/x86/avx/avxvnniint16.rs | 158 +++++++++++++++
fearless_simd_core/src/x86/avx/avxvnniint8.rs | 158 +++++++++++++++
fearless_simd_core/src/x86/avx/mod.rs | 24 +++
.../src/x86/avx512/avx512bf16.rs | 190 +++++++++++++++++
.../src/x86/avx512/avx512bitalg.rs | 191 ++++++++++++++++++
fearless_simd_core/src/x86/avx512/avx512bw.rs | 172 ++++++++++++++++
fearless_simd_core/src/x86/avx512/avx512cd.rs | 172 ++++++++++++++++
fearless_simd_core/src/x86/avx512/avx512dq.rs | 172 ++++++++++++++++
fearless_simd_core/src/x86/avx512/avx512f.rs | 164 +++++++++++++++
.../src/x86/avx512/avx512fp16.rs | 190 +++++++++++++++++
.../src/x86/avx512/avx512ifma.rs | 182 +++++++++++++++++
.../src/x86/avx512/avx512vbmi.rs | 190 +++++++++++++++++
.../src/x86/avx512/avx512vbmi2.rs | 190 +++++++++++++++++
fearless_simd_core/src/x86/avx512/avx512vl.rs | 172 ++++++++++++++++
.../src/x86/avx512/avx512vnni.rs | 182 +++++++++++++++++
.../src/x86/avx512/avx512vp2intersect.rs | 182 +++++++++++++++++
.../src/x86/avx512/avx512vpopcntdq.rs | 182 +++++++++++++++++
fearless_simd_core/src/x86/avx512/mod.rs | 41 ++++
fearless_simd_core/src/x86/crypto/aes.rs | 106 ++++++++++
fearless_simd_core/src/x86/crypto/gfni.rs | 106 ++++++++++
fearless_simd_core/src/x86/crypto/kl.rs | 92 +++++++++
fearless_simd_core/src/x86/crypto/mod.rs | 38 ++++
.../src/x86/crypto/pclmulqdq.rs | 106 ++++++++++
fearless_simd_core/src/x86/crypto/rdrand.rs | 92 +++++++++
fearless_simd_core/src/x86/crypto/rdseed.rs | 92 +++++++++
fearless_simd_core/src/x86/crypto/sha.rs | 106 ++++++++++
fearless_simd_core/src/x86/crypto/sha512.rs | 150 ++++++++++++++
fearless_simd_core/src/x86/crypto/sm3.rs | 143 +++++++++++++
fearless_simd_core/src/x86/crypto/sm4.rs | 150 ++++++++++++++
fearless_simd_core/src/x86/crypto/vaes.rs | 157 ++++++++++++++
.../src/x86/crypto/vpclmulqdq.rs | 158 +++++++++++++++
fearless_simd_core/src/x86/crypto/widekl.rs | 99 +++++++++
.../src/x86/discontinued/mod.rs | 3 +
.../src/x86/discontinued/tbm.rs | 93 +++++++++
fearless_simd_core/src/x86/mod.rs | 8 +
fearless_simd_core/src/x86/sse/fxsr.rs | 93 +++++++++
fearless_simd_core/src/x86/sse/mod.rs | 24 +++
fearless_simd_core/src/x86/sse/sse.rs | 92 +++++++++
fearless_simd_core/src/x86/sse/sse2.rs | 99 +++++++++
fearless_simd_core/src/x86/sse/sse3.rs | 106 ++++++++++
fearless_simd_core/src/x86/sse/sse4_1.rs | 120 +++++++++++
fearless_simd_core/src/x86/sse/sse4_2.rs | 127 ++++++++++++
fearless_simd_core/src/x86/sse/ssse3.rs | 113 +++++++++++
fearless_simd_core/src/x86/v1/mod.rs | 9 +-
fearless_simd_core/src/x86/v2/cmpxchg16b.rs | 92 +++++++++
fearless_simd_core/src/x86/v2/mod.rs | 5 +
fearless_simd_core/src/x86/v2/popcnt.rs | 92 +++++++++
fearless_simd_core/src/x86/v3/bmi1.rs | 92 +++++++++
fearless_simd_core/src/x86/v3/bmi2.rs | 92 +++++++++
fearless_simd_core/src/x86/v3/f16c.rs | 143 +++++++++++++
fearless_simd_core/src/x86/v3/fma.rs | 143 +++++++++++++
fearless_simd_core/src/x86/v3/lzcnt.rs | 92 +++++++++
fearless_simd_core/src/x86/v3/mod.rs | 17 ++
fearless_simd_core/src/x86/v3/movbe.rs | 92 +++++++++
fearless_simd_core/src/x86/xsave/xsave.rs | 93 +++++++++
fearless_simd_core/src/x86/xsave/xsavec.rs | 93 +++++++++
fearless_simd_core/src/x86/xsave/xsaveopt.rs | 93 +++++++++
fearless_simd_core/src/x86/xsave/xsaves.rs | 93 +++++++++
65 files changed, 7455 insertions(+), 6 deletions(-)
create mode 100644 fearless_simd_core/src/x86/adx/adx.rs
create mode 100644 fearless_simd_core/src/x86/adx/mod.rs
create mode 100644 fearless_simd_core/src/x86/avx/avx.rs
create mode 100644 fearless_simd_core/src/x86/avx/avx2.rs
create mode 100644 fearless_simd_core/src/x86/avx/avxifma.rs
create mode 100644 fearless_simd_core/src/x86/avx/avxneconvert.rs
create mode 100644 fearless_simd_core/src/x86/avx/avxvnni.rs
create mode 100644 fearless_simd_core/src/x86/avx/avxvnniint16.rs
create mode 100644 fearless_simd_core/src/x86/avx/avxvnniint8.rs
create mode 100644 fearless_simd_core/src/x86/avx/mod.rs
create mode 100644 fearless_simd_core/src/x86/avx512/avx512bf16.rs
create mode 100644 fearless_simd_core/src/x86/avx512/avx512bitalg.rs
create mode 100644 fearless_simd_core/src/x86/avx512/avx512bw.rs
create mode 100644 fearless_simd_core/src/x86/avx512/avx512cd.rs
create mode 100644 fearless_simd_core/src/x86/avx512/avx512dq.rs
create mode 100644 fearless_simd_core/src/x86/avx512/avx512f.rs
create mode 100644 fearless_simd_core/src/x86/avx512/avx512fp16.rs
create mode 100644 fearless_simd_core/src/x86/avx512/avx512ifma.rs
create mode 100644 fearless_simd_core/src/x86/avx512/avx512vbmi.rs
create mode 100644 fearless_simd_core/src/x86/avx512/avx512vbmi2.rs
create mode 100644 fearless_simd_core/src/x86/avx512/avx512vl.rs
create mode 100644 fearless_simd_core/src/x86/avx512/avx512vnni.rs
create mode 100644 fearless_simd_core/src/x86/avx512/avx512vp2intersect.rs
create mode 100644 fearless_simd_core/src/x86/avx512/avx512vpopcntdq.rs
create mode 100644 fearless_simd_core/src/x86/avx512/mod.rs
create mode 100644 fearless_simd_core/src/x86/crypto/aes.rs
create mode 100644 fearless_simd_core/src/x86/crypto/gfni.rs
create mode 100644 fearless_simd_core/src/x86/crypto/kl.rs
create mode 100644 fearless_simd_core/src/x86/crypto/mod.rs
create mode 100644 fearless_simd_core/src/x86/crypto/pclmulqdq.rs
create mode 100644 fearless_simd_core/src/x86/crypto/rdrand.rs
create mode 100644 fearless_simd_core/src/x86/crypto/rdseed.rs
create mode 100644 fearless_simd_core/src/x86/crypto/sha.rs
create mode 100644 fearless_simd_core/src/x86/crypto/sha512.rs
create mode 100644 fearless_simd_core/src/x86/crypto/sm3.rs
create mode 100644 fearless_simd_core/src/x86/crypto/sm4.rs
create mode 100644 fearless_simd_core/src/x86/crypto/vaes.rs
create mode 100644 fearless_simd_core/src/x86/crypto/vpclmulqdq.rs
create mode 100644 fearless_simd_core/src/x86/crypto/widekl.rs
create mode 100644 fearless_simd_core/src/x86/discontinued/mod.rs
create mode 100644 fearless_simd_core/src/x86/discontinued/tbm.rs
create mode 100644 fearless_simd_core/src/x86/sse/fxsr.rs
create mode 100644 fearless_simd_core/src/x86/sse/mod.rs
create mode 100644 fearless_simd_core/src/x86/sse/sse.rs
create mode 100644 fearless_simd_core/src/x86/sse/sse2.rs
create mode 100644 fearless_simd_core/src/x86/sse/sse3.rs
create mode 100644 fearless_simd_core/src/x86/sse/sse4_1.rs
create mode 100644 fearless_simd_core/src/x86/sse/sse4_2.rs
create mode 100644 fearless_simd_core/src/x86/sse/ssse3.rs
create mode 100644 fearless_simd_core/src/x86/v2/cmpxchg16b.rs
create mode 100644 fearless_simd_core/src/x86/v2/mod.rs
create mode 100644 fearless_simd_core/src/x86/v2/popcnt.rs
create mode 100644 fearless_simd_core/src/x86/v3/bmi1.rs
create mode 100644 fearless_simd_core/src/x86/v3/bmi2.rs
create mode 100644 fearless_simd_core/src/x86/v3/f16c.rs
create mode 100644 fearless_simd_core/src/x86/v3/fma.rs
create mode 100644 fearless_simd_core/src/x86/v3/lzcnt.rs
create mode 100644 fearless_simd_core/src/x86/v3/mod.rs
create mode 100644 fearless_simd_core/src/x86/v3/movbe.rs
create mode 100644 fearless_simd_core/src/x86/xsave/xsave.rs
create mode 100644 fearless_simd_core/src/x86/xsave/xsavec.rs
create mode 100644 fearless_simd_core/src/x86/xsave/xsaveopt.rs
create mode 100644 fearless_simd_core/src/x86/xsave/xsaves.rs
diff --git a/fearless_simd_core/src/x86/adx/adx.rs b/fearless_simd_core/src/x86/adx/adx.rs
new file mode 100644
index 00000000..b10b6379
--- /dev/null
+++ b/fearless_simd_core/src/x86/adx/adx.rs
@@ -0,0 +1,92 @@
+// This file is automatically generated by `fearless_simd_core_gen`.
+// Its template can be found in `fearless_simd_core/gen/templates`.
+
+//! The ADX target feature.
+
+use crate::{TargetFeatureToken, trampoline};
+
+use core::fmt::Debug;
+
+/// [ADX] --- Multi-Precision Add-Carry Instruction Extensions
+///
+/// [ADX]: https://en.wikipedia.org/wiki/Intel_ADX
+///
+/// A token indicating that the current CPU has the `adx` target feature.
+///
+/// # Example
+///
+/// This can be used to [`trampoline!`] into functions like:
+///
+/// ```rust
+/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+/// #[target_feature(enable = "adx")]
+/// fn uses_adx() {
+/// // ...
+/// }
+/// ```
+#[derive(Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Adx {
+ // We don't use non_exhaustive because we don't want this struct to be constructible.
+ // in different modules in this crate.
+ _private: (),
+}
+
+impl Debug for Adx {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ write!(f, r#""adx" enabled."#)
+ }
+}
+
+unsafe impl TargetFeatureToken for Adx {
+ const FEATURES: &[&str] = &["adx"];
+
+ #[inline(always)]
+ fn vectorize(self, f: impl FnOnce() -> R) -> R {
+ // Because we want this constant to be eagerly evaluated.
+ trampoline!([Adx = self] => "adx", <(R)> fn<(R)>(f: impl FnOnce() -> R = f) -> R { f() })
+ }
+}
+
+impl Adx {
+ #[cfg(feature = "std")]
+ /// Create a new token if the `"adx"` target feature is detected as enabled.
+ ///
+ /// This does not do any caching internally, although note that the standard
+ /// library does internally cache the features it detects.
+ // TODO: Consider a manual override feature/env var?
+ pub fn try_new() -> Option {
+ // Feature flag required to make docs compile.
+ // TODO: Extract into a (private) crate::x86::is_x86_feature_detected?
+ #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+ if std::arch::is_x86_feature_detected!("adx") {
+ // Safety: The required CPU feature was detected.
+ unsafe { Some(Self::new()) }
+ } else {
+ None
+ }
+ }
+
+ #[target_feature(enable = "adx")]
+ /// Create a new token for the "adx" target feature.
+ ///
+ /// This method is useful to get a new token if you have an external proof that
+ /// ADX is available. This could happen if you are in a target feature
+ /// function called by an external library user.
+ ///
+ /// # Safety
+ ///
+ /// No conditions other than those inherited from the target feature attribute,
+ /// i.e. that the "adx" target feature is available.
+ ///
+ /// [implicitly enables]: https://doc.rust-lang.org/beta/reference/attributes/codegen.html?highlight=implicitly%20enabled#r-attributes.codegen.target_feature.safety-restrictions
+ pub fn new() -> Self {
+ Self { _private: () }
+ }
+}
+
+const _: () = {
+ assert!(
+ core::mem::size_of::() == 0,
+ "Target feature tokens should be zero sized."
+ );
+};
diff --git a/fearless_simd_core/src/x86/adx/mod.rs b/fearless_simd_core/src/x86/adx/mod.rs
new file mode 100644
index 00000000..3c74dc60
--- /dev/null
+++ b/fearless_simd_core/src/x86/adx/mod.rs
@@ -0,0 +1,8 @@
+//! The "adx" target feature.
+
+#[expect(
+ clippy::module_inception,
+ reason = "The inner module is automatically generated."
+)]
+mod adx;
+pub use adx::Adx;
diff --git a/fearless_simd_core/src/x86/avx/avx.rs b/fearless_simd_core/src/x86/avx/avx.rs
new file mode 100644
index 00000000..baa56e36
--- /dev/null
+++ b/fearless_simd_core/src/x86/avx/avx.rs
@@ -0,0 +1,134 @@
+// This file is automatically generated by `fearless_simd_core_gen`.
+// Its template can be found in `fearless_simd_core/gen/templates`.
+
+//! The AVX target feature.
+
+use crate::{TargetFeatureToken, trampoline};
+
+use core::fmt::Debug;
+
+/// [AVX] --- Advanced Vector Extensions
+///
+/// [AVX]: https://en.wikipedia.org/wiki/Advanced_Vector_Extensions
+///
+/// A token indicating that the current CPU has the `avx` target feature.
+///
+/// # Example
+///
+/// This can be used to [`trampoline!`] into functions like:
+///
+/// ```rust
+/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+/// #[target_feature(enable = "avx")]
+/// fn uses_avx() {
+/// // ...
+/// }
+/// ```
+#[derive(Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Avx {
+ // We don't use non_exhaustive because we don't want this struct to be constructible.
+ // in different modules in this crate.
+ _private: (),
+}
+
+impl Debug for Avx {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ write!(f, r#""avx" enabled."#)
+ }
+}
+
+unsafe impl TargetFeatureToken for Avx {
+ const FEATURES: &[&str] = &["avx", "sse", "sse2", "sse3", "sse4.1", "sse4.2", "ssse3"];
+
+ #[inline(always)]
+ fn vectorize(self, f: impl FnOnce() -> R) -> R {
+ // Because we want this constant to be eagerly evaluated.
+ trampoline!([Avx = self] => "avx", <(R)> fn<(R)>(f: impl FnOnce() -> R = f) -> R { f() })
+ }
+}
+
+impl Avx {
+ #[cfg(feature = "std")]
+ /// Create a new token if the `"avx"` target feature is detected as enabled.
+ ///
+ /// This does not do any caching internally, although note that the standard
+ /// library does internally cache the features it detects.
+ // TODO: Consider a manual override feature/env var?
+ pub fn try_new() -> Option {
+ // Feature flag required to make docs compile.
+ // TODO: Extract into a (private) crate::x86::is_x86_feature_detected?
+ #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+ if std::arch::is_x86_feature_detected!("avx") {
+ // Safety: The required CPU feature was detected.
+ unsafe { Some(Self::new()) }
+ } else {
+ None
+ }
+ }
+
+ #[target_feature(enable = "avx")]
+ /// Create a new token for the "avx" target feature.
+ ///
+ /// This method is useful to get a new token if you have an external proof that
+ /// AVX is available. This could happen if you are in a target feature
+ /// function called by an external library user.
+ ///
+ /// # Safety
+ ///
+ /// No conditions other than those inherited from the target feature attribute,
+ /// i.e. that the "avx" target feature is available.
+ ///
+ /// [implicitly enables]: https://doc.rust-lang.org/beta/reference/attributes/codegen.html?highlight=implicitly%20enabled#r-attributes.codegen.target_feature.safety-restrictions
+ pub fn new() -> Self {
+ Self { _private: () }
+ }
+}
+
+impl From for crate::x86::sse::Sse {
+ fn from(value: Avx) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx = value] => "avx", fn() -> crate::x86::sse::Sse { crate::x86::sse::Sse::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse2 {
+ fn from(value: Avx) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx = value] => "avx", fn() -> crate::x86::sse::Sse2 { crate::x86::sse::Sse2::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse3 {
+ fn from(value: Avx) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx = value] => "avx", fn() -> crate::x86::sse::Sse3 { crate::x86::sse::Sse3::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_1 {
+ fn from(value: Avx) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx = value] => "avx", fn() -> crate::x86::sse::Sse4_1 { crate::x86::sse::Sse4_1::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_2 {
+ fn from(value: Avx) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx = value] => "avx", fn() -> crate::x86::sse::Sse4_2 { crate::x86::sse::Sse4_2::new() })
+ }
+}
+
+impl From for crate::x86::sse::SupplementalSse3 {
+ fn from(value: Avx) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx = value] => "avx", fn() -> crate::x86::sse::SupplementalSse3 { crate::x86::sse::SupplementalSse3::new() })
+ }
+}
+
+const _: () = {
+ assert!(
+ core::mem::size_of::() == 0,
+ "Target feature tokens should be zero sized."
+ );
+};
diff --git a/fearless_simd_core/src/x86/avx/avx2.rs b/fearless_simd_core/src/x86/avx/avx2.rs
new file mode 100644
index 00000000..b6f252bf
--- /dev/null
+++ b/fearless_simd_core/src/x86/avx/avx2.rs
@@ -0,0 +1,143 @@
+// This file is automatically generated by `fearless_simd_core_gen`.
+// Its template can be found in `fearless_simd_core/gen/templates`.
+
+//! The AVX2 target feature.
+
+use crate::{TargetFeatureToken, trampoline};
+
+use core::fmt::Debug;
+
+/// [AVX2] --- Advanced Vector Extensions 2
+///
+/// [AVX2]: https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#AVX2
+///
+/// A token indicating that the current CPU has the `avx2` target feature.
+///
+/// # Example
+///
+/// This can be used to [`trampoline!`] into functions like:
+///
+/// ```rust
+/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+/// #[target_feature(enable = "avx2")]
+/// fn uses_avx2() {
+/// // ...
+/// }
+/// ```
+#[derive(Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Avx2 {
+ // We don't use non_exhaustive because we don't want this struct to be constructible.
+ // in different modules in this crate.
+ _private: (),
+}
+
+impl Debug for Avx2 {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ write!(f, r#""avx2" enabled."#)
+ }
+}
+
+unsafe impl TargetFeatureToken for Avx2 {
+ const FEATURES: &[&str] = &[
+ "avx2", "avx", "sse", "sse2", "sse3", "sse4.1", "sse4.2", "ssse3",
+ ];
+
+ #[inline(always)]
+ fn vectorize(self, f: impl FnOnce() -> R) -> R {
+ // Because we want this constant to be eagerly evaluated.
+ trampoline!([Avx2 = self] => "avx2", <(R)> fn<(R)>(f: impl FnOnce() -> R = f) -> R { f() })
+ }
+}
+
+impl Avx2 {
+ #[cfg(feature = "std")]
+ /// Create a new token if the `"avx2"` target feature is detected as enabled.
+ ///
+ /// This does not do any caching internally, although note that the standard
+ /// library does internally cache the features it detects.
+ // TODO: Consider a manual override feature/env var?
+ pub fn try_new() -> Option {
+ // Feature flag required to make docs compile.
+ // TODO: Extract into a (private) crate::x86::is_x86_feature_detected?
+ #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+ if std::arch::is_x86_feature_detected!("avx2") {
+ // Safety: The required CPU feature was detected.
+ unsafe { Some(Self::new()) }
+ } else {
+ None
+ }
+ }
+
+ #[target_feature(enable = "avx2")]
+ /// Create a new token for the "avx2" target feature.
+ ///
+ /// This method is useful to get a new token if you have an external proof that
+ /// AVX2 is available. This could happen if you are in a target feature
+ /// function called by an external library user.
+ ///
+ /// # Safety
+ ///
+ /// No conditions other than those inherited from the target feature attribute,
+ /// i.e. that the "avx2" target feature is available.
+ ///
+ /// [implicitly enables]: https://doc.rust-lang.org/beta/reference/attributes/codegen.html?highlight=implicitly%20enabled#r-attributes.codegen.target_feature.safety-restrictions
+ pub fn new() -> Self {
+ Self { _private: () }
+ }
+}
+
+impl From for crate::x86::avx::Avx {
+ fn from(value: Avx2) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx2 = value] => "avx2", fn() -> crate::x86::avx::Avx { crate::x86::avx::Avx::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse {
+ fn from(value: Avx2) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx2 = value] => "avx2", fn() -> crate::x86::sse::Sse { crate::x86::sse::Sse::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse2 {
+ fn from(value: Avx2) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx2 = value] => "avx2", fn() -> crate::x86::sse::Sse2 { crate::x86::sse::Sse2::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse3 {
+ fn from(value: Avx2) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx2 = value] => "avx2", fn() -> crate::x86::sse::Sse3 { crate::x86::sse::Sse3::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_1 {
+ fn from(value: Avx2) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx2 = value] => "avx2", fn() -> crate::x86::sse::Sse4_1 { crate::x86::sse::Sse4_1::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_2 {
+ fn from(value: Avx2) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx2 = value] => "avx2", fn() -> crate::x86::sse::Sse4_2 { crate::x86::sse::Sse4_2::new() })
+ }
+}
+
+impl From for crate::x86::sse::SupplementalSse3 {
+ fn from(value: Avx2) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx2 = value] => "avx2", fn() -> crate::x86::sse::SupplementalSse3 { crate::x86::sse::SupplementalSse3::new() })
+ }
+}
+
+const _: () = {
+ assert!(
+ core::mem::size_of::() == 0,
+ "Target feature tokens should be zero sized."
+ );
+};
diff --git a/fearless_simd_core/src/x86/avx/avxifma.rs b/fearless_simd_core/src/x86/avx/avxifma.rs
new file mode 100644
index 00000000..12fda758
--- /dev/null
+++ b/fearless_simd_core/src/x86/avx/avxifma.rs
@@ -0,0 +1,150 @@
+// This file is automatically generated by `fearless_simd_core_gen`.
+// Its template can be found in `fearless_simd_core/gen/templates`.
+
+//! The AVX-IFMA target feature.
+
+use crate::{TargetFeatureToken, trampoline};
+
+use core::fmt::Debug;
+
+/// [AVX-IFMA] --- Advanced Vector Extensions - Integer Fused Multiply Add
+///
+/// [AVX-IFMA]: https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#AVX-VNNI,_AVX-IFMA
+///
+/// A token indicating that the current CPU has the `avxifma` target feature.
+///
+/// # Example
+///
+/// This can be used to [`trampoline!`] into functions like:
+///
+/// ```rust
+/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+/// #[target_feature(enable = "avxifma")]
+/// fn uses_avxifma() {
+/// // ...
+/// }
+/// ```
+#[derive(Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Avxifma {
+ // We don't use non_exhaustive because we don't want this struct to be constructible.
+ // in different modules in this crate.
+ _private: (),
+}
+
+impl Debug for Avxifma {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ write!(f, r#""avxifma" enabled."#)
+ }
+}
+
+unsafe impl TargetFeatureToken for Avxifma {
+ const FEATURES: &[&str] = &[
+ "avxifma", "avx", "avx2", "sse", "sse2", "sse3", "sse4.1", "sse4.2", "ssse3",
+ ];
+
+ #[inline(always)]
+ fn vectorize(self, f: impl FnOnce() -> R) -> R {
+ // Because we want this constant to be eagerly evaluated.
+ trampoline!([Avxifma = self] => "avxifma", <(R)> fn<(R)>(f: impl FnOnce() -> R = f) -> R { f() })
+ }
+}
+
+impl Avxifma {
+ #[cfg(feature = "std")]
+ /// Create a new token if the `"avxifma"` target feature is detected as enabled.
+ ///
+ /// This does not do any caching internally, although note that the standard
+ /// library does internally cache the features it detects.
+ // TODO: Consider a manual override feature/env var?
+ pub fn try_new() -> Option {
+ // Feature flag required to make docs compile.
+ // TODO: Extract into a (private) crate::x86::is_x86_feature_detected?
+ #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+ if std::arch::is_x86_feature_detected!("avxifma") {
+ // Safety: The required CPU feature was detected.
+ unsafe { Some(Self::new()) }
+ } else {
+ None
+ }
+ }
+
+ #[target_feature(enable = "avxifma")]
+ /// Create a new token for the "avxifma" target feature.
+ ///
+ /// This method is useful to get a new token if you have an external proof that
+ /// AVX-IFMA is available. This could happen if you are in a target feature
+ /// function called by an external library user.
+ ///
+ /// # Safety
+ ///
+ /// No conditions other than those inherited from the target feature attribute,
+ /// i.e. that the "avxifma" target feature is available.
+ ///
+ /// [implicitly enables]: https://doc.rust-lang.org/beta/reference/attributes/codegen.html?highlight=implicitly%20enabled#r-attributes.codegen.target_feature.safety-restrictions
+ pub fn new() -> Self {
+ Self { _private: () }
+ }
+}
+
+impl From for crate::x86::avx::Avx {
+ fn from(value: Avxifma) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxifma = value] => "avxifma", fn() -> crate::x86::avx::Avx { crate::x86::avx::Avx::new() })
+ }
+}
+
+impl From for crate::x86::avx::Avx2 {
+ fn from(value: Avxifma) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxifma = value] => "avxifma", fn() -> crate::x86::avx::Avx2 { crate::x86::avx::Avx2::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse {
+ fn from(value: Avxifma) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxifma = value] => "avxifma", fn() -> crate::x86::sse::Sse { crate::x86::sse::Sse::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse2 {
+ fn from(value: Avxifma) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxifma = value] => "avxifma", fn() -> crate::x86::sse::Sse2 { crate::x86::sse::Sse2::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse3 {
+ fn from(value: Avxifma) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxifma = value] => "avxifma", fn() -> crate::x86::sse::Sse3 { crate::x86::sse::Sse3::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_1 {
+ fn from(value: Avxifma) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxifma = value] => "avxifma", fn() -> crate::x86::sse::Sse4_1 { crate::x86::sse::Sse4_1::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_2 {
+ fn from(value: Avxifma) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxifma = value] => "avxifma", fn() -> crate::x86::sse::Sse4_2 { crate::x86::sse::Sse4_2::new() })
+ }
+}
+
+impl From for crate::x86::sse::SupplementalSse3 {
+ fn from(value: Avxifma) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxifma = value] => "avxifma", fn() -> crate::x86::sse::SupplementalSse3 { crate::x86::sse::SupplementalSse3::new() })
+ }
+}
+
+const _: () = {
+ assert!(
+ core::mem::size_of::() == 0,
+ "Target feature tokens should be zero sized."
+ );
+};
diff --git a/fearless_simd_core/src/x86/avx/avxneconvert.rs b/fearless_simd_core/src/x86/avx/avxneconvert.rs
new file mode 100644
index 00000000..a2adef3c
--- /dev/null
+++ b/fearless_simd_core/src/x86/avx/avxneconvert.rs
@@ -0,0 +1,158 @@
+// This file is automatically generated by `fearless_simd_core_gen`.
+// Its template can be found in `fearless_simd_core/gen/templates`.
+
+//! The AVX-NE-CONVERT target feature.
+
+use crate::{TargetFeatureToken, trampoline};
+
+use core::fmt::Debug;
+
+/// [AVX-NE-CONVERT] --- Advanced Vector Extensions - No-Exception Floating-Point conversion Instructions
+///
+/// [AVX-NE-CONVERT]: https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#AVX-VNNI,_AVX-IFMA
+///
+/// A token indicating that the current CPU has the `avxneconvert` target feature.
+///
+/// # Example
+///
+/// This can be used to [`trampoline!`] into functions like:
+///
+/// ```rust
+/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+/// #[target_feature(enable = "avxneconvert")]
+/// fn uses_avxneconvert() {
+/// // ...
+/// }
+/// ```
+#[derive(Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Avxneconvert {
+ // We don't use non_exhaustive because we don't want this struct to be constructible.
+ // in different modules in this crate.
+ _private: (),
+}
+
+impl Debug for Avxneconvert {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ write!(f, r#""avxneconvert" enabled."#)
+ }
+}
+
+unsafe impl TargetFeatureToken for Avxneconvert {
+ const FEATURES: &[&str] = &[
+ "avxneconvert",
+ "avx",
+ "avx2",
+ "sse",
+ "sse2",
+ "sse3",
+ "sse4.1",
+ "sse4.2",
+ "ssse3",
+ ];
+
+ #[inline(always)]
+ fn vectorize(self, f: impl FnOnce() -> R) -> R {
+ // Because we want this constant to be eagerly evaluated.
+ trampoline!([Avxneconvert = self] => "avxneconvert", <(R)> fn<(R)>(f: impl FnOnce() -> R = f) -> R { f() })
+ }
+}
+
+impl Avxneconvert {
+ #[cfg(feature = "std")]
+ /// Create a new token if the `"avxneconvert"` target feature is detected as enabled.
+ ///
+ /// This does not do any caching internally, although note that the standard
+ /// library does internally cache the features it detects.
+ // TODO: Consider a manual override feature/env var?
+ pub fn try_new() -> Option {
+ // Feature flag required to make docs compile.
+ // TODO: Extract into a (private) crate::x86::is_x86_feature_detected?
+ #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+ if std::arch::is_x86_feature_detected!("avxneconvert") {
+ // Safety: The required CPU feature was detected.
+ unsafe { Some(Self::new()) }
+ } else {
+ None
+ }
+ }
+
+ #[target_feature(enable = "avxneconvert")]
+ /// Create a new token for the "avxneconvert" target feature.
+ ///
+ /// This method is useful to get a new token if you have an external proof that
+ /// AVX-NE-CONVERT is available. This could happen if you are in a target feature
+ /// function called by an external library user.
+ ///
+ /// # Safety
+ ///
+ /// No conditions other than those inherited from the target feature attribute,
+ /// i.e. that the "avxneconvert" target feature is available.
+ ///
+ /// [implicitly enables]: https://doc.rust-lang.org/beta/reference/attributes/codegen.html?highlight=implicitly%20enabled#r-attributes.codegen.target_feature.safety-restrictions
+ pub fn new() -> Self {
+ Self { _private: () }
+ }
+}
+
+impl From for crate::x86::avx::Avx {
+ fn from(value: Avxneconvert) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxneconvert = value] => "avxneconvert", fn() -> crate::x86::avx::Avx { crate::x86::avx::Avx::new() })
+ }
+}
+
+impl From for crate::x86::avx::Avx2 {
+ fn from(value: Avxneconvert) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxneconvert = value] => "avxneconvert", fn() -> crate::x86::avx::Avx2 { crate::x86::avx::Avx2::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse {
+ fn from(value: Avxneconvert) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxneconvert = value] => "avxneconvert", fn() -> crate::x86::sse::Sse { crate::x86::sse::Sse::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse2 {
+ fn from(value: Avxneconvert) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxneconvert = value] => "avxneconvert", fn() -> crate::x86::sse::Sse2 { crate::x86::sse::Sse2::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse3 {
+ fn from(value: Avxneconvert) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxneconvert = value] => "avxneconvert", fn() -> crate::x86::sse::Sse3 { crate::x86::sse::Sse3::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_1 {
+ fn from(value: Avxneconvert) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxneconvert = value] => "avxneconvert", fn() -> crate::x86::sse::Sse4_1 { crate::x86::sse::Sse4_1::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_2 {
+ fn from(value: Avxneconvert) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxneconvert = value] => "avxneconvert", fn() -> crate::x86::sse::Sse4_2 { crate::x86::sse::Sse4_2::new() })
+ }
+}
+
+impl From for crate::x86::sse::SupplementalSse3 {
+ fn from(value: Avxneconvert) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxneconvert = value] => "avxneconvert", fn() -> crate::x86::sse::SupplementalSse3 { crate::x86::sse::SupplementalSse3::new() })
+ }
+}
+
+const _: () = {
+ assert!(
+ core::mem::size_of::() == 0,
+ "Target feature tokens should be zero sized."
+ );
+};
diff --git a/fearless_simd_core/src/x86/avx/avxvnni.rs b/fearless_simd_core/src/x86/avx/avxvnni.rs
new file mode 100644
index 00000000..48148a0c
--- /dev/null
+++ b/fearless_simd_core/src/x86/avx/avxvnni.rs
@@ -0,0 +1,150 @@
+// This file is automatically generated by `fearless_simd_core_gen`.
+// Its template can be found in `fearless_simd_core/gen/templates`.
+
+//! The AVX-VNNI target feature.
+
+use crate::{TargetFeatureToken, trampoline};
+
+use core::fmt::Debug;
+
+/// [AVX-VNNI] --- Advanced Vector Extensions - Vector Neural Network Instructions
+///
+/// [AVX-VNNI]: https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#AVX-VNNI,_AVX-IFMA
+///
+/// A token indicating that the current CPU has the `avxvnni` target feature.
+///
+/// # Example
+///
+/// This can be used to [`trampoline!`] into functions like:
+///
+/// ```rust
+/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+/// #[target_feature(enable = "avxvnni")]
+/// fn uses_avxvnni() {
+/// // ...
+/// }
+/// ```
+#[derive(Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Avxvnni {
+ // We don't use non_exhaustive because we don't want this struct to be constructible.
+ // in different modules in this crate.
+ _private: (),
+}
+
+impl Debug for Avxvnni {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ write!(f, r#""avxvnni" enabled."#)
+ }
+}
+
+unsafe impl TargetFeatureToken for Avxvnni {
+ const FEATURES: &[&str] = &[
+ "avxvnni", "avx", "avx2", "sse", "sse2", "sse3", "sse4.1", "sse4.2", "ssse3",
+ ];
+
+ #[inline(always)]
+ fn vectorize(self, f: impl FnOnce() -> R) -> R {
+ // Because we want this constant to be eagerly evaluated.
+ trampoline!([Avxvnni = self] => "avxvnni", <(R)> fn<(R)>(f: impl FnOnce() -> R = f) -> R { f() })
+ }
+}
+
+impl Avxvnni {
+ #[cfg(feature = "std")]
+ /// Create a new token if the `"avxvnni"` target feature is detected as enabled.
+ ///
+ /// This does not do any caching internally, although note that the standard
+ /// library does internally cache the features it detects.
+ // TODO: Consider a manual override feature/env var?
+ pub fn try_new() -> Option {
+ // Feature flag required to make docs compile.
+ // TODO: Extract into a (private) crate::x86::is_x86_feature_detected?
+ #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+ if std::arch::is_x86_feature_detected!("avxvnni") {
+ // Safety: The required CPU feature was detected.
+ unsafe { Some(Self::new()) }
+ } else {
+ None
+ }
+ }
+
+ #[target_feature(enable = "avxvnni")]
+ /// Create a new token for the "avxvnni" target feature.
+ ///
+ /// This method is useful to get a new token if you have an external proof that
+ /// AVX-VNNI is available. This could happen if you are in a target feature
+ /// function called by an external library user.
+ ///
+ /// # Safety
+ ///
+ /// No conditions other than those inherited from the target feature attribute,
+ /// i.e. that the "avxvnni" target feature is available.
+ ///
+ /// [implicitly enables]: https://doc.rust-lang.org/beta/reference/attributes/codegen.html?highlight=implicitly%20enabled#r-attributes.codegen.target_feature.safety-restrictions
+ pub fn new() -> Self {
+ Self { _private: () }
+ }
+}
+
+impl From for crate::x86::avx::Avx {
+ fn from(value: Avxvnni) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxvnni = value] => "avxvnni", fn() -> crate::x86::avx::Avx { crate::x86::avx::Avx::new() })
+ }
+}
+
+impl From for crate::x86::avx::Avx2 {
+ fn from(value: Avxvnni) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxvnni = value] => "avxvnni", fn() -> crate::x86::avx::Avx2 { crate::x86::avx::Avx2::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse {
+ fn from(value: Avxvnni) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxvnni = value] => "avxvnni", fn() -> crate::x86::sse::Sse { crate::x86::sse::Sse::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse2 {
+ fn from(value: Avxvnni) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxvnni = value] => "avxvnni", fn() -> crate::x86::sse::Sse2 { crate::x86::sse::Sse2::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse3 {
+ fn from(value: Avxvnni) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxvnni = value] => "avxvnni", fn() -> crate::x86::sse::Sse3 { crate::x86::sse::Sse3::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_1 {
+ fn from(value: Avxvnni) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxvnni = value] => "avxvnni", fn() -> crate::x86::sse::Sse4_1 { crate::x86::sse::Sse4_1::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_2 {
+ fn from(value: Avxvnni) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxvnni = value] => "avxvnni", fn() -> crate::x86::sse::Sse4_2 { crate::x86::sse::Sse4_2::new() })
+ }
+}
+
+impl From for crate::x86::sse::SupplementalSse3 {
+ fn from(value: Avxvnni) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxvnni = value] => "avxvnni", fn() -> crate::x86::sse::SupplementalSse3 { crate::x86::sse::SupplementalSse3::new() })
+ }
+}
+
+const _: () = {
+ assert!(
+ core::mem::size_of::() == 0,
+ "Target feature tokens should be zero sized."
+ );
+};
diff --git a/fearless_simd_core/src/x86/avx/avxvnniint16.rs b/fearless_simd_core/src/x86/avx/avxvnniint16.rs
new file mode 100644
index 00000000..36b16a41
--- /dev/null
+++ b/fearless_simd_core/src/x86/avx/avxvnniint16.rs
@@ -0,0 +1,158 @@
+// This file is automatically generated by `fearless_simd_core_gen`.
+// Its template can be found in `fearless_simd_core/gen/templates`.
+
+//! The AVX-VNNI-INT16 target feature.
+
+use crate::{TargetFeatureToken, trampoline};
+
+use core::fmt::Debug;
+
+/// [AVX-VNNI-INT16] --- Advanced Vector Extensions - Vector Neural Network Instructions with 16-bit Integers
+///
+/// [AVX-VNNI-INT16]: https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#AVX-VNNI,_AVX-IFMA
+///
+/// A token indicating that the current CPU has the `avxvnniint16` target feature.
+///
+/// # Example
+///
+/// This can be used to [`trampoline!`] into functions like:
+///
+/// ```rust
+/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+/// #[target_feature(enable = "avxvnniint16")]
+/// fn uses_avxvnniint16() {
+/// // ...
+/// }
+/// ```
+#[derive(Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Avxvnniint16 {
+ // We don't use non_exhaustive because we don't want this struct to be constructible.
+ // in different modules in this crate.
+ _private: (),
+}
+
+impl Debug for Avxvnniint16 {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ write!(f, r#""avxvnniint16" enabled."#)
+ }
+}
+
+unsafe impl TargetFeatureToken for Avxvnniint16 {
+ const FEATURES: &[&str] = &[
+ "avxvnniint16",
+ "avx",
+ "avx2",
+ "sse",
+ "sse2",
+ "sse3",
+ "sse4.1",
+ "sse4.2",
+ "ssse3",
+ ];
+
+ #[inline(always)]
+ fn vectorize(self, f: impl FnOnce() -> R) -> R {
+ // Because we want this constant to be eagerly evaluated.
+ trampoline!([Avxvnniint16 = self] => "avxvnniint16", <(R)> fn<(R)>(f: impl FnOnce() -> R = f) -> R { f() })
+ }
+}
+
+impl Avxvnniint16 {
+ #[cfg(feature = "std")]
+ /// Create a new token if the `"avxvnniint16"` target feature is detected as enabled.
+ ///
+ /// This does not do any caching internally, although note that the standard
+ /// library does internally cache the features it detects.
+ // TODO: Consider a manual override feature/env var?
+ pub fn try_new() -> Option {
+ // Feature flag required to make docs compile.
+ // TODO: Extract into a (private) crate::x86::is_x86_feature_detected?
+ #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+ if std::arch::is_x86_feature_detected!("avxvnniint16") {
+ // Safety: The required CPU feature was detected.
+ unsafe { Some(Self::new()) }
+ } else {
+ None
+ }
+ }
+
+ #[target_feature(enable = "avxvnniint16")]
+ /// Create a new token for the "avxvnniint16" target feature.
+ ///
+ /// This method is useful to get a new token if you have an external proof that
+ /// AVX-VNNI-INT16 is available. This could happen if you are in a target feature
+ /// function called by an external library user.
+ ///
+ /// # Safety
+ ///
+ /// No conditions other than those inherited from the target feature attribute,
+ /// i.e. that the "avxvnniint16" target feature is available.
+ ///
+ /// [implicitly enables]: https://doc.rust-lang.org/beta/reference/attributes/codegen.html?highlight=implicitly%20enabled#r-attributes.codegen.target_feature.safety-restrictions
+ pub fn new() -> Self {
+ Self { _private: () }
+ }
+}
+
+impl From for crate::x86::avx::Avx {
+ fn from(value: Avxvnniint16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxvnniint16 = value] => "avxvnniint16", fn() -> crate::x86::avx::Avx { crate::x86::avx::Avx::new() })
+ }
+}
+
+impl From for crate::x86::avx::Avx2 {
+ fn from(value: Avxvnniint16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxvnniint16 = value] => "avxvnniint16", fn() -> crate::x86::avx::Avx2 { crate::x86::avx::Avx2::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse {
+ fn from(value: Avxvnniint16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxvnniint16 = value] => "avxvnniint16", fn() -> crate::x86::sse::Sse { crate::x86::sse::Sse::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse2 {
+ fn from(value: Avxvnniint16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxvnniint16 = value] => "avxvnniint16", fn() -> crate::x86::sse::Sse2 { crate::x86::sse::Sse2::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse3 {
+ fn from(value: Avxvnniint16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxvnniint16 = value] => "avxvnniint16", fn() -> crate::x86::sse::Sse3 { crate::x86::sse::Sse3::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_1 {
+ fn from(value: Avxvnniint16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxvnniint16 = value] => "avxvnniint16", fn() -> crate::x86::sse::Sse4_1 { crate::x86::sse::Sse4_1::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_2 {
+ fn from(value: Avxvnniint16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxvnniint16 = value] => "avxvnniint16", fn() -> crate::x86::sse::Sse4_2 { crate::x86::sse::Sse4_2::new() })
+ }
+}
+
+impl From for crate::x86::sse::SupplementalSse3 {
+ fn from(value: Avxvnniint16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxvnniint16 = value] => "avxvnniint16", fn() -> crate::x86::sse::SupplementalSse3 { crate::x86::sse::SupplementalSse3::new() })
+ }
+}
+
+const _: () = {
+ assert!(
+ core::mem::size_of::() == 0,
+ "Target feature tokens should be zero sized."
+ );
+};
diff --git a/fearless_simd_core/src/x86/avx/avxvnniint8.rs b/fearless_simd_core/src/x86/avx/avxvnniint8.rs
new file mode 100644
index 00000000..2a0eaf4a
--- /dev/null
+++ b/fearless_simd_core/src/x86/avx/avxvnniint8.rs
@@ -0,0 +1,158 @@
+// This file is automatically generated by `fearless_simd_core_gen`.
+// Its template can be found in `fearless_simd_core/gen/templates`.
+
+//! The AVX-VNNI-INT8 target feature.
+
+use crate::{TargetFeatureToken, trampoline};
+
+use core::fmt::Debug;
+
+/// [AVX-VNNI-INT8] --- Advanced Vector Extensions - Vector Neural Network Instructions with 8-bit Integers
+///
+/// [AVX-VNNI-INT8]: https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#AVX-VNNI,_AVX-IFMA
+///
+/// A token indicating that the current CPU has the `avxvnniint8` target feature.
+///
+/// # Example
+///
+/// This can be used to [`trampoline!`] into functions like:
+///
+/// ```rust
+/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+/// #[target_feature(enable = "avxvnniint8")]
+/// fn uses_avxvnniint8() {
+/// // ...
+/// }
+/// ```
+#[derive(Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Avxvnniint8 {
+ // We don't use non_exhaustive because we don't want this struct to be constructible.
+ // in different modules in this crate.
+ _private: (),
+}
+
+impl Debug for Avxvnniint8 {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ write!(f, r#""avxvnniint8" enabled."#)
+ }
+}
+
+unsafe impl TargetFeatureToken for Avxvnniint8 {
+ const FEATURES: &[&str] = &[
+ "avxvnniint8",
+ "avx",
+ "avx2",
+ "sse",
+ "sse2",
+ "sse3",
+ "sse4.1",
+ "sse4.2",
+ "ssse3",
+ ];
+
+ #[inline(always)]
+ fn vectorize(self, f: impl FnOnce() -> R) -> R {
+ // Because we want this constant to be eagerly evaluated.
+ trampoline!([Avxvnniint8 = self] => "avxvnniint8", <(R)> fn<(R)>(f: impl FnOnce() -> R = f) -> R { f() })
+ }
+}
+
+impl Avxvnniint8 {
+ #[cfg(feature = "std")]
+ /// Create a new token if the `"avxvnniint8"` target feature is detected as enabled.
+ ///
+ /// This does not do any caching internally, although note that the standard
+ /// library does internally cache the features it detects.
+ // TODO: Consider a manual override feature/env var?
+ pub fn try_new() -> Option {
+ // Feature flag required to make docs compile.
+ // TODO: Extract into a (private) crate::x86::is_x86_feature_detected?
+ #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+ if std::arch::is_x86_feature_detected!("avxvnniint8") {
+ // Safety: The required CPU feature was detected.
+ unsafe { Some(Self::new()) }
+ } else {
+ None
+ }
+ }
+
+ #[target_feature(enable = "avxvnniint8")]
+ /// Create a new token for the "avxvnniint8" target feature.
+ ///
+ /// This method is useful to get a new token if you have an external proof that
+ /// AVX-VNNI-INT8 is available. This could happen if you are in a target feature
+ /// function called by an external library user.
+ ///
+ /// # Safety
+ ///
+ /// No conditions other than those inherited from the target feature attribute,
+ /// i.e. that the "avxvnniint8" target feature is available.
+ ///
+ /// [implicitly enables]: https://doc.rust-lang.org/beta/reference/attributes/codegen.html?highlight=implicitly%20enabled#r-attributes.codegen.target_feature.safety-restrictions
+ pub fn new() -> Self {
+ Self { _private: () }
+ }
+}
+
+impl From for crate::x86::avx::Avx {
+ fn from(value: Avxvnniint8) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxvnniint8 = value] => "avxvnniint8", fn() -> crate::x86::avx::Avx { crate::x86::avx::Avx::new() })
+ }
+}
+
+impl From for crate::x86::avx::Avx2 {
+ fn from(value: Avxvnniint8) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxvnniint8 = value] => "avxvnniint8", fn() -> crate::x86::avx::Avx2 { crate::x86::avx::Avx2::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse {
+ fn from(value: Avxvnniint8) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxvnniint8 = value] => "avxvnniint8", fn() -> crate::x86::sse::Sse { crate::x86::sse::Sse::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse2 {
+ fn from(value: Avxvnniint8) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxvnniint8 = value] => "avxvnniint8", fn() -> crate::x86::sse::Sse2 { crate::x86::sse::Sse2::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse3 {
+ fn from(value: Avxvnniint8) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxvnniint8 = value] => "avxvnniint8", fn() -> crate::x86::sse::Sse3 { crate::x86::sse::Sse3::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_1 {
+ fn from(value: Avxvnniint8) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxvnniint8 = value] => "avxvnniint8", fn() -> crate::x86::sse::Sse4_1 { crate::x86::sse::Sse4_1::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_2 {
+ fn from(value: Avxvnniint8) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxvnniint8 = value] => "avxvnniint8", fn() -> crate::x86::sse::Sse4_2 { crate::x86::sse::Sse4_2::new() })
+ }
+}
+
+impl From for crate::x86::sse::SupplementalSse3 {
+ fn from(value: Avxvnniint8) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avxvnniint8 = value] => "avxvnniint8", fn() -> crate::x86::sse::SupplementalSse3 { crate::x86::sse::SupplementalSse3::new() })
+ }
+}
+
+const _: () = {
+ assert!(
+ core::mem::size_of::() == 0,
+ "Target feature tokens should be zero sized."
+ );
+};
diff --git a/fearless_simd_core/src/x86/avx/mod.rs b/fearless_simd_core/src/x86/avx/mod.rs
new file mode 100644
index 00000000..f047d055
--- /dev/null
+++ b/fearless_simd_core/src/x86/avx/mod.rs
@@ -0,0 +1,24 @@
+#[expect(
+ clippy::module_inception,
+ reason = "The inner module is automatically generated."
+)]
+mod avx;
+pub use avx::Avx;
+
+mod avx2;
+pub use avx2::Avx2;
+
+mod avxifma;
+pub use avxifma::Avxifma;
+
+mod avxneconvert;
+pub use avxneconvert::Avxneconvert;
+
+mod avxvnni;
+pub use avxvnni::Avxvnni;
+
+mod avxvnniint8;
+pub use avxvnniint8::Avxvnniint8;
+
+mod avxvnniint16;
+pub use avxvnniint16::Avxvnniint16;
diff --git a/fearless_simd_core/src/x86/avx512/avx512bf16.rs b/fearless_simd_core/src/x86/avx512/avx512bf16.rs
new file mode 100644
index 00000000..52396972
--- /dev/null
+++ b/fearless_simd_core/src/x86/avx512/avx512bf16.rs
@@ -0,0 +1,190 @@
+// This file is automatically generated by `fearless_simd_core_gen`.
+// Its template can be found in `fearless_simd_core/gen/templates`.
+
+//! The AVX512-BF16 target feature.
+
+use crate::{TargetFeatureToken, trampoline};
+
+use core::fmt::Debug;
+
+/// [AVX512-BF16] --- Advanced Vector Extensions 512-bit - Bfloat16 Extensions
+///
+/// [AVX512-BF16]: https://en.wikipedia.org/wiki/AVX-512#BF16
+///
+/// A token indicating that the current CPU has the `avx512bf16` target feature.
+///
+/// # Example
+///
+/// This can be used to [`trampoline!`] into functions like:
+///
+/// ```rust
+/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+/// #[target_feature(enable = "avx512bf16")]
+/// fn uses_avx512bf16() {
+/// // ...
+/// }
+/// ```
+#[derive(Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Avx512bf16 {
+ // We don't use non_exhaustive because we don't want this struct to be constructible.
+ // in different modules in this crate.
+ _private: (),
+}
+
+impl Debug for Avx512bf16 {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ write!(f, r#""avx512bf16" enabled."#)
+ }
+}
+
+unsafe impl TargetFeatureToken for Avx512bf16 {
+ const FEATURES: &[&str] = &[
+ "avx512bf16",
+ "avx",
+ "avx2",
+ "avx512bw",
+ "avx512f",
+ "f16c",
+ "fma",
+ "sse",
+ "sse2",
+ "sse3",
+ "sse4.1",
+ "sse4.2",
+ "ssse3",
+ ];
+
+ #[inline(always)]
+ fn vectorize(self, f: impl FnOnce() -> R) -> R {
+ // Because we want this constant to be eagerly evaluated.
+ trampoline!([Avx512bf16 = self] => "avx512bf16", <(R)> fn<(R)>(f: impl FnOnce() -> R = f) -> R { f() })
+ }
+}
+
+impl Avx512bf16 {
+ #[cfg(feature = "std")]
+ /// Create a new token if the `"avx512bf16"` target feature is detected as enabled.
+ ///
+ /// This does not do any caching internally, although note that the standard
+ /// library does internally cache the features it detects.
+ // TODO: Consider a manual override feature/env var?
+ pub fn try_new() -> Option {
+ // Feature flag required to make docs compile.
+ // TODO: Extract into a (private) crate::x86::is_x86_feature_detected?
+ #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+ if std::arch::is_x86_feature_detected!("avx512bf16") {
+ // Safety: The required CPU feature was detected.
+ unsafe { Some(Self::new()) }
+ } else {
+ None
+ }
+ }
+
+ #[target_feature(enable = "avx512bf16")]
+ /// Create a new token for the "avx512bf16" target feature.
+ ///
+ /// This method is useful to get a new token if you have an external proof that
+ /// AVX512-BF16 is available. This could happen if you are in a target feature
+ /// function called by an external library user.
+ ///
+ /// # Safety
+ ///
+ /// No conditions other than those inherited from the target feature attribute,
+ /// i.e. that the "avx512bf16" target feature is available.
+ ///
+ /// [implicitly enables]: https://doc.rust-lang.org/beta/reference/attributes/codegen.html?highlight=implicitly%20enabled#r-attributes.codegen.target_feature.safety-restrictions
+ pub fn new() -> Self {
+ Self { _private: () }
+ }
+}
+
+impl From for crate::x86::avx::Avx {
+ fn from(value: Avx512bf16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bf16 = value] => "avx512bf16", fn() -> crate::x86::avx::Avx { crate::x86::avx::Avx::new() })
+ }
+}
+
+impl From for crate::x86::avx::Avx2 {
+ fn from(value: Avx512bf16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bf16 = value] => "avx512bf16", fn() -> crate::x86::avx::Avx2 { crate::x86::avx::Avx2::new() })
+ }
+}
+
+impl From for crate::x86::avx512::Avx512bw {
+ fn from(value: Avx512bf16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bf16 = value] => "avx512bf16", fn() -> crate::x86::avx512::Avx512bw { crate::x86::avx512::Avx512bw::new() })
+ }
+}
+
+impl From for crate::x86::avx512::Avx512f {
+ fn from(value: Avx512bf16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bf16 = value] => "avx512bf16", fn() -> crate::x86::avx512::Avx512f { crate::x86::avx512::Avx512f::new() })
+ }
+}
+
+impl From for crate::x86::v3::F16c {
+ fn from(value: Avx512bf16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bf16 = value] => "avx512bf16", fn() -> crate::x86::v3::F16c { crate::x86::v3::F16c::new() })
+ }
+}
+
+impl From for crate::x86::v3::Fma {
+ fn from(value: Avx512bf16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bf16 = value] => "avx512bf16", fn() -> crate::x86::v3::Fma { crate::x86::v3::Fma::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse {
+ fn from(value: Avx512bf16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bf16 = value] => "avx512bf16", fn() -> crate::x86::sse::Sse { crate::x86::sse::Sse::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse2 {
+ fn from(value: Avx512bf16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bf16 = value] => "avx512bf16", fn() -> crate::x86::sse::Sse2 { crate::x86::sse::Sse2::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse3 {
+ fn from(value: Avx512bf16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bf16 = value] => "avx512bf16", fn() -> crate::x86::sse::Sse3 { crate::x86::sse::Sse3::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_1 {
+ fn from(value: Avx512bf16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bf16 = value] => "avx512bf16", fn() -> crate::x86::sse::Sse4_1 { crate::x86::sse::Sse4_1::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_2 {
+ fn from(value: Avx512bf16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bf16 = value] => "avx512bf16", fn() -> crate::x86::sse::Sse4_2 { crate::x86::sse::Sse4_2::new() })
+ }
+}
+
+impl From for crate::x86::sse::SupplementalSse3 {
+ fn from(value: Avx512bf16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bf16 = value] => "avx512bf16", fn() -> crate::x86::sse::SupplementalSse3 { crate::x86::sse::SupplementalSse3::new() })
+ }
+}
+
+const _: () = {
+ assert!(
+ core::mem::size_of::() == 0,
+ "Target feature tokens should be zero sized."
+ );
+};
diff --git a/fearless_simd_core/src/x86/avx512/avx512bitalg.rs b/fearless_simd_core/src/x86/avx512/avx512bitalg.rs
new file mode 100644
index 00000000..66bb543a
--- /dev/null
+++ b/fearless_simd_core/src/x86/avx512/avx512bitalg.rs
@@ -0,0 +1,191 @@
+// This file is automatically generated by `fearless_simd_core_gen`.
+// Its template can be found in `fearless_simd_core/gen/templates`.
+
+//! The AVX512-BITALG target feature.
+
+use crate::{TargetFeatureToken, trampoline};
+
+use core::fmt::Debug;
+
+/// [AVX512-BITALG] --- Advanced Vector Extensions 512-bit - Bit Algorithms
+///
+///
+/// [AVX512-BITALG]: https://en.wikipedia.org/wiki/AVX-512#VPOPCNTDQ_and_BITALG
+///
+/// A token indicating that the current CPU has the `avx512bitalg` target feature.
+///
+/// # Example
+///
+/// This can be used to [`trampoline!`] into functions like:
+///
+/// ```rust
+/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+/// #[target_feature(enable = "avx512bitalg")]
+/// fn uses_avx512bitalg() {
+/// // ...
+/// }
+/// ```
+#[derive(Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Avx512bitalg {
+ // We don't use non_exhaustive because we don't want this struct to be constructible.
+ // in different modules in this crate.
+ _private: (),
+}
+
+impl Debug for Avx512bitalg {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ write!(f, r#""avx512bitalg" enabled."#)
+ }
+}
+
+unsafe impl TargetFeatureToken for Avx512bitalg {
+ const FEATURES: &[&str] = &[
+ "avx512bitalg",
+ "avx",
+ "avx2",
+ "avx512bw",
+ "avx512f",
+ "f16c",
+ "fma",
+ "sse",
+ "sse2",
+ "sse3",
+ "sse4.1",
+ "sse4.2",
+ "ssse3",
+ ];
+
+ #[inline(always)]
+ fn vectorize(self, f: impl FnOnce() -> R) -> R {
+ // Because we want this constant to be eagerly evaluated.
+ trampoline!([Avx512bitalg = self] => "avx512bitalg", <(R)> fn<(R)>(f: impl FnOnce() -> R = f) -> R { f() })
+ }
+}
+
+impl Avx512bitalg {
+ #[cfg(feature = "std")]
+ /// Create a new token if the `"avx512bitalg"` target feature is detected as enabled.
+ ///
+ /// This does not do any caching internally, although note that the standard
+ /// library does internally cache the features it detects.
+ // TODO: Consider a manual override feature/env var?
+ pub fn try_new() -> Option {
+ // Feature flag required to make docs compile.
+ // TODO: Extract into a (private) crate::x86::is_x86_feature_detected?
+ #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+ if std::arch::is_x86_feature_detected!("avx512bitalg") {
+ // Safety: The required CPU feature was detected.
+ unsafe { Some(Self::new()) }
+ } else {
+ None
+ }
+ }
+
+ #[target_feature(enable = "avx512bitalg")]
+ /// Create a new token for the "avx512bitalg" target feature.
+ ///
+ /// This method is useful to get a new token if you have an external proof that
+ /// AVX512-BITALG is available. This could happen if you are in a target feature
+ /// function called by an external library user.
+ ///
+ /// # Safety
+ ///
+ /// No conditions other than those inherited from the target feature attribute,
+ /// i.e. that the "avx512bitalg" target feature is available.
+ ///
+ /// [implicitly enables]: https://doc.rust-lang.org/beta/reference/attributes/codegen.html?highlight=implicitly%20enabled#r-attributes.codegen.target_feature.safety-restrictions
+ pub fn new() -> Self {
+ Self { _private: () }
+ }
+}
+
+impl From for crate::x86::avx::Avx {
+ fn from(value: Avx512bitalg) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bitalg = value] => "avx512bitalg", fn() -> crate::x86::avx::Avx { crate::x86::avx::Avx::new() })
+ }
+}
+
+impl From for crate::x86::avx::Avx2 {
+ fn from(value: Avx512bitalg) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bitalg = value] => "avx512bitalg", fn() -> crate::x86::avx::Avx2 { crate::x86::avx::Avx2::new() })
+ }
+}
+
+impl From for crate::x86::avx512::Avx512bw {
+ fn from(value: Avx512bitalg) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bitalg = value] => "avx512bitalg", fn() -> crate::x86::avx512::Avx512bw { crate::x86::avx512::Avx512bw::new() })
+ }
+}
+
+impl From for crate::x86::avx512::Avx512f {
+ fn from(value: Avx512bitalg) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bitalg = value] => "avx512bitalg", fn() -> crate::x86::avx512::Avx512f { crate::x86::avx512::Avx512f::new() })
+ }
+}
+
+impl From for crate::x86::v3::F16c {
+ fn from(value: Avx512bitalg) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bitalg = value] => "avx512bitalg", fn() -> crate::x86::v3::F16c { crate::x86::v3::F16c::new() })
+ }
+}
+
+impl From for crate::x86::v3::Fma {
+ fn from(value: Avx512bitalg) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bitalg = value] => "avx512bitalg", fn() -> crate::x86::v3::Fma { crate::x86::v3::Fma::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse {
+ fn from(value: Avx512bitalg) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bitalg = value] => "avx512bitalg", fn() -> crate::x86::sse::Sse { crate::x86::sse::Sse::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse2 {
+ fn from(value: Avx512bitalg) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bitalg = value] => "avx512bitalg", fn() -> crate::x86::sse::Sse2 { crate::x86::sse::Sse2::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse3 {
+ fn from(value: Avx512bitalg) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bitalg = value] => "avx512bitalg", fn() -> crate::x86::sse::Sse3 { crate::x86::sse::Sse3::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_1 {
+ fn from(value: Avx512bitalg) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bitalg = value] => "avx512bitalg", fn() -> crate::x86::sse::Sse4_1 { crate::x86::sse::Sse4_1::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_2 {
+ fn from(value: Avx512bitalg) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bitalg = value] => "avx512bitalg", fn() -> crate::x86::sse::Sse4_2 { crate::x86::sse::Sse4_2::new() })
+ }
+}
+
+impl From for crate::x86::sse::SupplementalSse3 {
+ fn from(value: Avx512bitalg) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bitalg = value] => "avx512bitalg", fn() -> crate::x86::sse::SupplementalSse3 { crate::x86::sse::SupplementalSse3::new() })
+ }
+}
+
+const _: () = {
+ assert!(
+ core::mem::size_of::() == 0,
+ "Target feature tokens should be zero sized."
+ );
+};
diff --git a/fearless_simd_core/src/x86/avx512/avx512bw.rs b/fearless_simd_core/src/x86/avx512/avx512bw.rs
new file mode 100644
index 00000000..7213b3da
--- /dev/null
+++ b/fearless_simd_core/src/x86/avx512/avx512bw.rs
@@ -0,0 +1,172 @@
+// This file is automatically generated by `fearless_simd_core_gen`.
+// Its template can be found in `fearless_simd_core/gen/templates`.
+
+//! The AVX512-BW target feature.
+
+use crate::{TargetFeatureToken, trampoline};
+
+use core::fmt::Debug;
+
+/// [AVX512-BW] --- Advanced Vector Extensions 512-bit - Byte and Word Instructions
+///
+/// [AVX512-BW]: https://en.wikipedia.org/wiki/AVX-512#BW,_DQ_and_VBMI
+///
+/// A token indicating that the current CPU has the `avx512bw` target feature.
+///
+/// # Example
+///
+/// This can be used to [`trampoline!`] into functions like:
+///
+/// ```rust
+/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+/// #[target_feature(enable = "avx512bw")]
+/// fn uses_avx512bw() {
+/// // ...
+/// }
+/// ```
+#[derive(Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Avx512bw {
+ // We don't use non_exhaustive because we don't want this struct to be constructible.
+ // in different modules in this crate.
+ _private: (),
+}
+
+impl Debug for Avx512bw {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ write!(f, r#""avx512bw" enabled."#)
+ }
+}
+
+unsafe impl TargetFeatureToken for Avx512bw {
+ const FEATURES: &[&str] = &[
+ "avx512bw", "avx", "avx2", "avx512f", "f16c", "fma", "sse", "sse2", "sse3", "sse4.1",
+ "sse4.2", "ssse3",
+ ];
+
+ #[inline(always)]
+ fn vectorize(self, f: impl FnOnce() -> R) -> R {
+ // Because we want this constant to be eagerly evaluated.
+ trampoline!([Avx512bw = self] => "avx512bw", <(R)> fn<(R)>(f: impl FnOnce() -> R = f) -> R { f() })
+ }
+}
+
+impl Avx512bw {
+ #[cfg(feature = "std")]
+ /// Create a new token if the `"avx512bw"` target feature is detected as enabled.
+ ///
+ /// This does not do any caching internally, although note that the standard
+ /// library does internally cache the features it detects.
+ // TODO: Consider a manual override feature/env var?
+ pub fn try_new() -> Option {
+ // Feature flag required to make docs compile.
+ // TODO: Extract into a (private) crate::x86::is_x86_feature_detected?
+ #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+ if std::arch::is_x86_feature_detected!("avx512bw") {
+ // Safety: The required CPU feature was detected.
+ unsafe { Some(Self::new()) }
+ } else {
+ None
+ }
+ }
+
+ #[target_feature(enable = "avx512bw")]
+ /// Create a new token for the "avx512bw" target feature.
+ ///
+ /// This method is useful to get a new token if you have an external proof that
+ /// AVX512-BW is available. This could happen if you are in a target feature
+ /// function called by an external library user.
+ ///
+ /// # Safety
+ ///
+ /// No conditions other than those inherited from the target feature attribute,
+ /// i.e. that the "avx512bw" target feature is available.
+ ///
+ /// [implicitly enables]: https://doc.rust-lang.org/beta/reference/attributes/codegen.html?highlight=implicitly%20enabled#r-attributes.codegen.target_feature.safety-restrictions
+ pub fn new() -> Self {
+ Self { _private: () }
+ }
+}
+
+impl From for crate::x86::avx::Avx {
+ fn from(value: Avx512bw) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bw = value] => "avx512bw", fn() -> crate::x86::avx::Avx { crate::x86::avx::Avx::new() })
+ }
+}
+
+impl From for crate::x86::avx::Avx2 {
+ fn from(value: Avx512bw) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bw = value] => "avx512bw", fn() -> crate::x86::avx::Avx2 { crate::x86::avx::Avx2::new() })
+ }
+}
+
+impl From for crate::x86::avx512::Avx512f {
+ fn from(value: Avx512bw) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bw = value] => "avx512bw", fn() -> crate::x86::avx512::Avx512f { crate::x86::avx512::Avx512f::new() })
+ }
+}
+
+impl From for crate::x86::v3::F16c {
+ fn from(value: Avx512bw) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bw = value] => "avx512bw", fn() -> crate::x86::v3::F16c { crate::x86::v3::F16c::new() })
+ }
+}
+
+impl From for crate::x86::v3::Fma {
+ fn from(value: Avx512bw) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bw = value] => "avx512bw", fn() -> crate::x86::v3::Fma { crate::x86::v3::Fma::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse {
+ fn from(value: Avx512bw) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bw = value] => "avx512bw", fn() -> crate::x86::sse::Sse { crate::x86::sse::Sse::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse2 {
+ fn from(value: Avx512bw) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bw = value] => "avx512bw", fn() -> crate::x86::sse::Sse2 { crate::x86::sse::Sse2::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse3 {
+ fn from(value: Avx512bw) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bw = value] => "avx512bw", fn() -> crate::x86::sse::Sse3 { crate::x86::sse::Sse3::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_1 {
+ fn from(value: Avx512bw) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bw = value] => "avx512bw", fn() -> crate::x86::sse::Sse4_1 { crate::x86::sse::Sse4_1::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_2 {
+ fn from(value: Avx512bw) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bw = value] => "avx512bw", fn() -> crate::x86::sse::Sse4_2 { crate::x86::sse::Sse4_2::new() })
+ }
+}
+
+impl From for crate::x86::sse::SupplementalSse3 {
+ fn from(value: Avx512bw) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512bw = value] => "avx512bw", fn() -> crate::x86::sse::SupplementalSse3 { crate::x86::sse::SupplementalSse3::new() })
+ }
+}
+
+const _: () = {
+ assert!(
+ core::mem::size_of::() == 0,
+ "Target feature tokens should be zero sized."
+ );
+};
diff --git a/fearless_simd_core/src/x86/avx512/avx512cd.rs b/fearless_simd_core/src/x86/avx512/avx512cd.rs
new file mode 100644
index 00000000..b3b9c8c3
--- /dev/null
+++ b/fearless_simd_core/src/x86/avx512/avx512cd.rs
@@ -0,0 +1,172 @@
+// This file is automatically generated by `fearless_simd_core_gen`.
+// Its template can be found in `fearless_simd_core/gen/templates`.
+
+//! The AVX512-CD target feature.
+
+use crate::{TargetFeatureToken, trampoline};
+
+use core::fmt::Debug;
+
+/// [AVX512-CD] --- Advanced Vector Extensions 512-bit - Conflict Detection Instructions
+///
+/// [AVX512-CD]: https://en.wikipedia.org/wiki/AVX-512#Conflict_detection
+///
+/// A token indicating that the current CPU has the `avx512cd` target feature.
+///
+/// # Example
+///
+/// This can be used to [`trampoline!`] into functions like:
+///
+/// ```rust
+/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+/// #[target_feature(enable = "avx512cd")]
+/// fn uses_avx512cd() {
+/// // ...
+/// }
+/// ```
+#[derive(Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Avx512cd {
+ // We don't use non_exhaustive because we don't want this struct to be constructible.
+ // in different modules in this crate.
+ _private: (),
+}
+
+impl Debug for Avx512cd {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ write!(f, r#""avx512cd" enabled."#)
+ }
+}
+
+unsafe impl TargetFeatureToken for Avx512cd {
+ const FEATURES: &[&str] = &[
+ "avx512cd", "avx", "avx2", "avx512f", "f16c", "fma", "sse", "sse2", "sse3", "sse4.1",
+ "sse4.2", "ssse3",
+ ];
+
+ #[inline(always)]
+ fn vectorize(self, f: impl FnOnce() -> R) -> R {
+ // Because we want this constant to be eagerly evaluated.
+ trampoline!([Avx512cd = self] => "avx512cd", <(R)> fn<(R)>(f: impl FnOnce() -> R = f) -> R { f() })
+ }
+}
+
+impl Avx512cd {
+ #[cfg(feature = "std")]
+ /// Create a new token if the `"avx512cd"` target feature is detected as enabled.
+ ///
+ /// This does not do any caching internally, although note that the standard
+ /// library does internally cache the features it detects.
+ // TODO: Consider a manual override feature/env var?
+ pub fn try_new() -> Option {
+ // Feature flag required to make docs compile.
+ // TODO: Extract into a (private) crate::x86::is_x86_feature_detected?
+ #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+ if std::arch::is_x86_feature_detected!("avx512cd") {
+ // Safety: The required CPU feature was detected.
+ unsafe { Some(Self::new()) }
+ } else {
+ None
+ }
+ }
+
+ #[target_feature(enable = "avx512cd")]
+ /// Create a new token for the "avx512cd" target feature.
+ ///
+ /// This method is useful to get a new token if you have an external proof that
+ /// AVX512-CD is available. This could happen if you are in a target feature
+ /// function called by an external library user.
+ ///
+ /// # Safety
+ ///
+ /// No conditions other than those inherited from the target feature attribute,
+ /// i.e. that the "avx512cd" target feature is available.
+ ///
+ /// [implicitly enables]: https://doc.rust-lang.org/beta/reference/attributes/codegen.html?highlight=implicitly%20enabled#r-attributes.codegen.target_feature.safety-restrictions
+ pub fn new() -> Self {
+ Self { _private: () }
+ }
+}
+
+impl From for crate::x86::avx::Avx {
+ fn from(value: Avx512cd) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512cd = value] => "avx512cd", fn() -> crate::x86::avx::Avx { crate::x86::avx::Avx::new() })
+ }
+}
+
+impl From for crate::x86::avx::Avx2 {
+ fn from(value: Avx512cd) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512cd = value] => "avx512cd", fn() -> crate::x86::avx::Avx2 { crate::x86::avx::Avx2::new() })
+ }
+}
+
+impl From for crate::x86::avx512::Avx512f {
+ fn from(value: Avx512cd) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512cd = value] => "avx512cd", fn() -> crate::x86::avx512::Avx512f { crate::x86::avx512::Avx512f::new() })
+ }
+}
+
+impl From for crate::x86::v3::F16c {
+ fn from(value: Avx512cd) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512cd = value] => "avx512cd", fn() -> crate::x86::v3::F16c { crate::x86::v3::F16c::new() })
+ }
+}
+
+impl From for crate::x86::v3::Fma {
+ fn from(value: Avx512cd) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512cd = value] => "avx512cd", fn() -> crate::x86::v3::Fma { crate::x86::v3::Fma::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse {
+ fn from(value: Avx512cd) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512cd = value] => "avx512cd", fn() -> crate::x86::sse::Sse { crate::x86::sse::Sse::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse2 {
+ fn from(value: Avx512cd) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512cd = value] => "avx512cd", fn() -> crate::x86::sse::Sse2 { crate::x86::sse::Sse2::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse3 {
+ fn from(value: Avx512cd) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512cd = value] => "avx512cd", fn() -> crate::x86::sse::Sse3 { crate::x86::sse::Sse3::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_1 {
+ fn from(value: Avx512cd) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512cd = value] => "avx512cd", fn() -> crate::x86::sse::Sse4_1 { crate::x86::sse::Sse4_1::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_2 {
+ fn from(value: Avx512cd) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512cd = value] => "avx512cd", fn() -> crate::x86::sse::Sse4_2 { crate::x86::sse::Sse4_2::new() })
+ }
+}
+
+impl From for crate::x86::sse::SupplementalSse3 {
+ fn from(value: Avx512cd) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512cd = value] => "avx512cd", fn() -> crate::x86::sse::SupplementalSse3 { crate::x86::sse::SupplementalSse3::new() })
+ }
+}
+
+const _: () = {
+ assert!(
+ core::mem::size_of::() == 0,
+ "Target feature tokens should be zero sized."
+ );
+};
diff --git a/fearless_simd_core/src/x86/avx512/avx512dq.rs b/fearless_simd_core/src/x86/avx512/avx512dq.rs
new file mode 100644
index 00000000..fb6d3670
--- /dev/null
+++ b/fearless_simd_core/src/x86/avx512/avx512dq.rs
@@ -0,0 +1,172 @@
+// This file is automatically generated by `fearless_simd_core_gen`.
+// Its template can be found in `fearless_simd_core/gen/templates`.
+
+//! The AVX512-DQ target feature.
+
+use crate::{TargetFeatureToken, trampoline};
+
+use core::fmt::Debug;
+
+/// [AVX512-DQ] --- Advanced Vector Extensions 512-bit - Doubleword and Quadword Instructions
+///
+/// [AVX512-DQ]: https://en.wikipedia.org/wiki/AVX-512#BW,_DQ_and_VBMI
+///
+/// A token indicating that the current CPU has the `avx512dq` target feature.
+///
+/// # Example
+///
+/// This can be used to [`trampoline!`] into functions like:
+///
+/// ```rust
+/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+/// #[target_feature(enable = "avx512dq")]
+/// fn uses_avx512dq() {
+/// // ...
+/// }
+/// ```
+#[derive(Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Avx512dq {
+ // We don't use non_exhaustive because we don't want this struct to be constructible.
+ // in different modules in this crate.
+ _private: (),
+}
+
+impl Debug for Avx512dq {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ write!(f, r#""avx512dq" enabled."#)
+ }
+}
+
+unsafe impl TargetFeatureToken for Avx512dq {
+ const FEATURES: &[&str] = &[
+ "avx512dq", "avx", "avx2", "avx512f", "f16c", "fma", "sse", "sse2", "sse3", "sse4.1",
+ "sse4.2", "ssse3",
+ ];
+
+ #[inline(always)]
+ fn vectorize(self, f: impl FnOnce() -> R) -> R {
+ // Because we want this constant to be eagerly evaluated.
+ trampoline!([Avx512dq = self] => "avx512dq", <(R)> fn<(R)>(f: impl FnOnce() -> R = f) -> R { f() })
+ }
+}
+
+impl Avx512dq {
+ #[cfg(feature = "std")]
+ /// Create a new token if the `"avx512dq"` target feature is detected as enabled.
+ ///
+ /// This does not do any caching internally, although note that the standard
+ /// library does internally cache the features it detects.
+ // TODO: Consider a manual override feature/env var?
+ pub fn try_new() -> Option {
+ // Feature flag required to make docs compile.
+ // TODO: Extract into a (private) crate::x86::is_x86_feature_detected?
+ #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+ if std::arch::is_x86_feature_detected!("avx512dq") {
+ // Safety: The required CPU feature was detected.
+ unsafe { Some(Self::new()) }
+ } else {
+ None
+ }
+ }
+
+ #[target_feature(enable = "avx512dq")]
+ /// Create a new token for the "avx512dq" target feature.
+ ///
+ /// This method is useful to get a new token if you have an external proof that
+ /// AVX512-DQ is available. This could happen if you are in a target feature
+ /// function called by an external library user.
+ ///
+ /// # Safety
+ ///
+ /// No conditions other than those inherited from the target feature attribute,
+ /// i.e. that the "avx512dq" target feature is available.
+ ///
+ /// [implicitly enables]: https://doc.rust-lang.org/beta/reference/attributes/codegen.html?highlight=implicitly%20enabled#r-attributes.codegen.target_feature.safety-restrictions
+ pub fn new() -> Self {
+ Self { _private: () }
+ }
+}
+
+impl From for crate::x86::avx::Avx {
+ fn from(value: Avx512dq) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512dq = value] => "avx512dq", fn() -> crate::x86::avx::Avx { crate::x86::avx::Avx::new() })
+ }
+}
+
+impl From for crate::x86::avx::Avx2 {
+ fn from(value: Avx512dq) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512dq = value] => "avx512dq", fn() -> crate::x86::avx::Avx2 { crate::x86::avx::Avx2::new() })
+ }
+}
+
+impl From for crate::x86::avx512::Avx512f {
+ fn from(value: Avx512dq) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512dq = value] => "avx512dq", fn() -> crate::x86::avx512::Avx512f { crate::x86::avx512::Avx512f::new() })
+ }
+}
+
+impl From for crate::x86::v3::F16c {
+ fn from(value: Avx512dq) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512dq = value] => "avx512dq", fn() -> crate::x86::v3::F16c { crate::x86::v3::F16c::new() })
+ }
+}
+
+impl From for crate::x86::v3::Fma {
+ fn from(value: Avx512dq) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512dq = value] => "avx512dq", fn() -> crate::x86::v3::Fma { crate::x86::v3::Fma::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse {
+ fn from(value: Avx512dq) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512dq = value] => "avx512dq", fn() -> crate::x86::sse::Sse { crate::x86::sse::Sse::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse2 {
+ fn from(value: Avx512dq) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512dq = value] => "avx512dq", fn() -> crate::x86::sse::Sse2 { crate::x86::sse::Sse2::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse3 {
+ fn from(value: Avx512dq) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512dq = value] => "avx512dq", fn() -> crate::x86::sse::Sse3 { crate::x86::sse::Sse3::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_1 {
+ fn from(value: Avx512dq) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512dq = value] => "avx512dq", fn() -> crate::x86::sse::Sse4_1 { crate::x86::sse::Sse4_1::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_2 {
+ fn from(value: Avx512dq) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512dq = value] => "avx512dq", fn() -> crate::x86::sse::Sse4_2 { crate::x86::sse::Sse4_2::new() })
+ }
+}
+
+impl From for crate::x86::sse::SupplementalSse3 {
+ fn from(value: Avx512dq) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512dq = value] => "avx512dq", fn() -> crate::x86::sse::SupplementalSse3 { crate::x86::sse::SupplementalSse3::new() })
+ }
+}
+
+const _: () = {
+ assert!(
+ core::mem::size_of::() == 0,
+ "Target feature tokens should be zero sized."
+ );
+};
diff --git a/fearless_simd_core/src/x86/avx512/avx512f.rs b/fearless_simd_core/src/x86/avx512/avx512f.rs
new file mode 100644
index 00000000..fa6adb77
--- /dev/null
+++ b/fearless_simd_core/src/x86/avx512/avx512f.rs
@@ -0,0 +1,164 @@
+// This file is automatically generated by `fearless_simd_core_gen`.
+// Its template can be found in `fearless_simd_core/gen/templates`.
+
+//! The AVX512-F target feature.
+
+use crate::{TargetFeatureToken, trampoline};
+
+use core::fmt::Debug;
+
+/// [AVX512-F] --- Advanced Vector Extensions 512-bit - Foundation
+///
+/// [AVX512-F]: https://en.wikipedia.org/wiki/AVX-512
+///
+/// A token indicating that the current CPU has the `avx512f` target feature.
+///
+/// # Example
+///
+/// This can be used to [`trampoline!`] into functions like:
+///
+/// ```rust
+/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+/// #[target_feature(enable = "avx512f")]
+/// fn uses_avx512f() {
+/// // ...
+/// }
+/// ```
+#[derive(Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Avx512f {
+ // We don't use non_exhaustive because we don't want this struct to be constructible.
+ // in different modules in this crate.
+ _private: (),
+}
+
+impl Debug for Avx512f {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ write!(f, r#""avx512f" enabled."#)
+ }
+}
+
+unsafe impl TargetFeatureToken for Avx512f {
+ const FEATURES: &[&str] = &[
+ "avx512f", "avx", "avx2", "f16c", "fma", "sse", "sse2", "sse3", "sse4.1", "sse4.2", "ssse3",
+ ];
+
+ #[inline(always)]
+ fn vectorize(self, f: impl FnOnce() -> R) -> R {
+ // Because we want this constant to be eagerly evaluated.
+ trampoline!([Avx512f = self] => "avx512f", <(R)> fn<(R)>(f: impl FnOnce() -> R = f) -> R { f() })
+ }
+}
+
+impl Avx512f {
+ #[cfg(feature = "std")]
+ /// Create a new token if the `"avx512f"` target feature is detected as enabled.
+ ///
+ /// This does not do any caching internally, although note that the standard
+ /// library does internally cache the features it detects.
+ // TODO: Consider a manual override feature/env var?
+ pub fn try_new() -> Option {
+ // Feature flag required to make docs compile.
+ // TODO: Extract into a (private) crate::x86::is_x86_feature_detected?
+ #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+ if std::arch::is_x86_feature_detected!("avx512f") {
+ // Safety: The required CPU feature was detected.
+ unsafe { Some(Self::new()) }
+ } else {
+ None
+ }
+ }
+
+ #[target_feature(enable = "avx512f")]
+ /// Create a new token for the "avx512f" target feature.
+ ///
+ /// This method is useful to get a new token if you have an external proof that
+ /// AVX512-F is available. This could happen if you are in a target feature
+ /// function called by an external library user.
+ ///
+ /// # Safety
+ ///
+ /// No conditions other than those inherited from the target feature attribute,
+ /// i.e. that the "avx512f" target feature is available.
+ ///
+ /// [implicitly enables]: https://doc.rust-lang.org/beta/reference/attributes/codegen.html?highlight=implicitly%20enabled#r-attributes.codegen.target_feature.safety-restrictions
+ pub fn new() -> Self {
+ Self { _private: () }
+ }
+}
+
+impl From for crate::x86::avx::Avx {
+ fn from(value: Avx512f) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512f = value] => "avx512f", fn() -> crate::x86::avx::Avx { crate::x86::avx::Avx::new() })
+ }
+}
+
+impl From for crate::x86::avx::Avx2 {
+ fn from(value: Avx512f) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512f = value] => "avx512f", fn() -> crate::x86::avx::Avx2 { crate::x86::avx::Avx2::new() })
+ }
+}
+
+impl From for crate::x86::v3::F16c {
+ fn from(value: Avx512f) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512f = value] => "avx512f", fn() -> crate::x86::v3::F16c { crate::x86::v3::F16c::new() })
+ }
+}
+
+impl From for crate::x86::v3::Fma {
+ fn from(value: Avx512f) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512f = value] => "avx512f", fn() -> crate::x86::v3::Fma { crate::x86::v3::Fma::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse {
+ fn from(value: Avx512f) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512f = value] => "avx512f", fn() -> crate::x86::sse::Sse { crate::x86::sse::Sse::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse2 {
+ fn from(value: Avx512f) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512f = value] => "avx512f", fn() -> crate::x86::sse::Sse2 { crate::x86::sse::Sse2::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse3 {
+ fn from(value: Avx512f) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512f = value] => "avx512f", fn() -> crate::x86::sse::Sse3 { crate::x86::sse::Sse3::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_1 {
+ fn from(value: Avx512f) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512f = value] => "avx512f", fn() -> crate::x86::sse::Sse4_1 { crate::x86::sse::Sse4_1::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_2 {
+ fn from(value: Avx512f) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512f = value] => "avx512f", fn() -> crate::x86::sse::Sse4_2 { crate::x86::sse::Sse4_2::new() })
+ }
+}
+
+impl From for crate::x86::sse::SupplementalSse3 {
+ fn from(value: Avx512f) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512f = value] => "avx512f", fn() -> crate::x86::sse::SupplementalSse3 { crate::x86::sse::SupplementalSse3::new() })
+ }
+}
+
+const _: () = {
+ assert!(
+ core::mem::size_of::() == 0,
+ "Target feature tokens should be zero sized."
+ );
+};
diff --git a/fearless_simd_core/src/x86/avx512/avx512fp16.rs b/fearless_simd_core/src/x86/avx512/avx512fp16.rs
new file mode 100644
index 00000000..f3ed6089
--- /dev/null
+++ b/fearless_simd_core/src/x86/avx512/avx512fp16.rs
@@ -0,0 +1,190 @@
+// This file is automatically generated by `fearless_simd_core_gen`.
+// Its template can be found in `fearless_simd_core/gen/templates`.
+
+//! The AVX512-FP16 target feature.
+
+use crate::{TargetFeatureToken, trampoline};
+
+use core::fmt::Debug;
+
+/// [AVX512-FP16] --- Advanced Vector Extensions 512-bit - Float16 Extensions
+///
+/// [AVX512-FP16]: https://en.wikipedia.org/wiki/AVX-512#FP16
+///
+/// A token indicating that the current CPU has the `avx512fp16` target feature.
+///
+/// # Example
+///
+/// This can be used to [`trampoline!`] into functions like:
+///
+/// ```rust
+/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+/// #[target_feature(enable = "avx512fp16")]
+/// fn uses_avx512fp16() {
+/// // ...
+/// }
+/// ```
+#[derive(Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Avx512fp16 {
+ // We don't use non_exhaustive because we don't want this struct to be constructible.
+ // in different modules in this crate.
+ _private: (),
+}
+
+impl Debug for Avx512fp16 {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ write!(f, r#""avx512fp16" enabled."#)
+ }
+}
+
+unsafe impl TargetFeatureToken for Avx512fp16 {
+ const FEATURES: &[&str] = &[
+ "avx512fp16",
+ "avx",
+ "avx2",
+ "avx512bw",
+ "avx512f",
+ "f16c",
+ "fma",
+ "sse",
+ "sse2",
+ "sse3",
+ "sse4.1",
+ "sse4.2",
+ "ssse3",
+ ];
+
+ #[inline(always)]
+ fn vectorize(self, f: impl FnOnce() -> R) -> R {
+ // Because we want this constant to be eagerly evaluated.
+ trampoline!([Avx512fp16 = self] => "avx512fp16", <(R)> fn<(R)>(f: impl FnOnce() -> R = f) -> R { f() })
+ }
+}
+
+impl Avx512fp16 {
+ #[cfg(feature = "std")]
+ /// Create a new token if the `"avx512fp16"` target feature is detected as enabled.
+ ///
+ /// This does not do any caching internally, although note that the standard
+ /// library does internally cache the features it detects.
+ // TODO: Consider a manual override feature/env var?
+ pub fn try_new() -> Option {
+ // Feature flag required to make docs compile.
+ // TODO: Extract into a (private) crate::x86::is_x86_feature_detected?
+ #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+ if std::arch::is_x86_feature_detected!("avx512fp16") {
+ // Safety: The required CPU feature was detected.
+ unsafe { Some(Self::new()) }
+ } else {
+ None
+ }
+ }
+
+ #[target_feature(enable = "avx512fp16")]
+ /// Create a new token for the "avx512fp16" target feature.
+ ///
+ /// This method is useful to get a new token if you have an external proof that
+ /// AVX512-FP16 is available. This could happen if you are in a target feature
+ /// function called by an external library user.
+ ///
+ /// # Safety
+ ///
+ /// No conditions other than those inherited from the target feature attribute,
+ /// i.e. that the "avx512fp16" target feature is available.
+ ///
+ /// [implicitly enables]: https://doc.rust-lang.org/beta/reference/attributes/codegen.html?highlight=implicitly%20enabled#r-attributes.codegen.target_feature.safety-restrictions
+ pub fn new() -> Self {
+ Self { _private: () }
+ }
+}
+
+impl From for crate::x86::avx::Avx {
+ fn from(value: Avx512fp16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512fp16 = value] => "avx512fp16", fn() -> crate::x86::avx::Avx { crate::x86::avx::Avx::new() })
+ }
+}
+
+impl From for crate::x86::avx::Avx2 {
+ fn from(value: Avx512fp16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512fp16 = value] => "avx512fp16", fn() -> crate::x86::avx::Avx2 { crate::x86::avx::Avx2::new() })
+ }
+}
+
+impl From for crate::x86::avx512::Avx512bw {
+ fn from(value: Avx512fp16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512fp16 = value] => "avx512fp16", fn() -> crate::x86::avx512::Avx512bw { crate::x86::avx512::Avx512bw::new() })
+ }
+}
+
+impl From for crate::x86::avx512::Avx512f {
+ fn from(value: Avx512fp16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512fp16 = value] => "avx512fp16", fn() -> crate::x86::avx512::Avx512f { crate::x86::avx512::Avx512f::new() })
+ }
+}
+
+impl From for crate::x86::v3::F16c {
+ fn from(value: Avx512fp16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512fp16 = value] => "avx512fp16", fn() -> crate::x86::v3::F16c { crate::x86::v3::F16c::new() })
+ }
+}
+
+impl From for crate::x86::v3::Fma {
+ fn from(value: Avx512fp16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512fp16 = value] => "avx512fp16", fn() -> crate::x86::v3::Fma { crate::x86::v3::Fma::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse {
+ fn from(value: Avx512fp16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512fp16 = value] => "avx512fp16", fn() -> crate::x86::sse::Sse { crate::x86::sse::Sse::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse2 {
+ fn from(value: Avx512fp16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512fp16 = value] => "avx512fp16", fn() -> crate::x86::sse::Sse2 { crate::x86::sse::Sse2::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse3 {
+ fn from(value: Avx512fp16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512fp16 = value] => "avx512fp16", fn() -> crate::x86::sse::Sse3 { crate::x86::sse::Sse3::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_1 {
+ fn from(value: Avx512fp16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512fp16 = value] => "avx512fp16", fn() -> crate::x86::sse::Sse4_1 { crate::x86::sse::Sse4_1::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_2 {
+ fn from(value: Avx512fp16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512fp16 = value] => "avx512fp16", fn() -> crate::x86::sse::Sse4_2 { crate::x86::sse::Sse4_2::new() })
+ }
+}
+
+impl From for crate::x86::sse::SupplementalSse3 {
+ fn from(value: Avx512fp16) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512fp16 = value] => "avx512fp16", fn() -> crate::x86::sse::SupplementalSse3 { crate::x86::sse::SupplementalSse3::new() })
+ }
+}
+
+const _: () = {
+ assert!(
+ core::mem::size_of::() == 0,
+ "Target feature tokens should be zero sized."
+ );
+};
diff --git a/fearless_simd_core/src/x86/avx512/avx512ifma.rs b/fearless_simd_core/src/x86/avx512/avx512ifma.rs
new file mode 100644
index 00000000..b7ab646c
--- /dev/null
+++ b/fearless_simd_core/src/x86/avx512/avx512ifma.rs
@@ -0,0 +1,182 @@
+// This file is automatically generated by `fearless_simd_core_gen`.
+// Its template can be found in `fearless_simd_core/gen/templates`.
+
+//! The AVX512-IFMA target feature.
+
+use crate::{TargetFeatureToken, trampoline};
+
+use core::fmt::Debug;
+
+/// [AVX512-IFMA] --- Advanced Vector Extensions 512-bit - Integer Fused Multiply Add
+///
+/// [AVX512-IFMA]: https://en.wikipedia.org/wiki/AVX-512#IFMA
+///
+/// A token indicating that the current CPU has the `avx512ifma` target feature.
+///
+/// # Example
+///
+/// This can be used to [`trampoline!`] into functions like:
+///
+/// ```rust
+/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+/// #[target_feature(enable = "avx512ifma")]
+/// fn uses_avx512ifma() {
+/// // ...
+/// }
+/// ```
+#[derive(Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Avx512ifma {
+ // We don't use non_exhaustive because we don't want this struct to be constructible.
+ // in different modules in this crate.
+ _private: (),
+}
+
+impl Debug for Avx512ifma {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ write!(f, r#""avx512ifma" enabled."#)
+ }
+}
+
+unsafe impl TargetFeatureToken for Avx512ifma {
+ const FEATURES: &[&str] = &[
+ "avx512ifma",
+ "avx",
+ "avx2",
+ "avx512f",
+ "f16c",
+ "fma",
+ "sse",
+ "sse2",
+ "sse3",
+ "sse4.1",
+ "sse4.2",
+ "ssse3",
+ ];
+
+ #[inline(always)]
+ fn vectorize(self, f: impl FnOnce() -> R) -> R {
+ // Because we want this constant to be eagerly evaluated.
+ trampoline!([Avx512ifma = self] => "avx512ifma", <(R)> fn<(R)>(f: impl FnOnce() -> R = f) -> R { f() })
+ }
+}
+
+impl Avx512ifma {
+ #[cfg(feature = "std")]
+ /// Create a new token if the `"avx512ifma"` target feature is detected as enabled.
+ ///
+ /// This does not do any caching internally, although note that the standard
+ /// library does internally cache the features it detects.
+ // TODO: Consider a manual override feature/env var?
+ pub fn try_new() -> Option {
+ // Feature flag required to make docs compile.
+ // TODO: Extract into a (private) crate::x86::is_x86_feature_detected?
+ #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+ if std::arch::is_x86_feature_detected!("avx512ifma") {
+ // Safety: The required CPU feature was detected.
+ unsafe { Some(Self::new()) }
+ } else {
+ None
+ }
+ }
+
+ #[target_feature(enable = "avx512ifma")]
+ /// Create a new token for the "avx512ifma" target feature.
+ ///
+ /// This method is useful to get a new token if you have an external proof that
+ /// AVX512-IFMA is available. This could happen if you are in a target feature
+ /// function called by an external library user.
+ ///
+ /// # Safety
+ ///
+ /// No conditions other than those inherited from the target feature attribute,
+ /// i.e. that the "avx512ifma" target feature is available.
+ ///
+ /// [implicitly enables]: https://doc.rust-lang.org/beta/reference/attributes/codegen.html?highlight=implicitly%20enabled#r-attributes.codegen.target_feature.safety-restrictions
+ pub fn new() -> Self {
+ Self { _private: () }
+ }
+}
+
+impl From for crate::x86::avx::Avx {
+ fn from(value: Avx512ifma) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512ifma = value] => "avx512ifma", fn() -> crate::x86::avx::Avx { crate::x86::avx::Avx::new() })
+ }
+}
+
+impl From for crate::x86::avx::Avx2 {
+ fn from(value: Avx512ifma) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512ifma = value] => "avx512ifma", fn() -> crate::x86::avx::Avx2 { crate::x86::avx::Avx2::new() })
+ }
+}
+
+impl From for crate::x86::avx512::Avx512f {
+ fn from(value: Avx512ifma) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512ifma = value] => "avx512ifma", fn() -> crate::x86::avx512::Avx512f { crate::x86::avx512::Avx512f::new() })
+ }
+}
+
+impl From for crate::x86::v3::F16c {
+ fn from(value: Avx512ifma) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512ifma = value] => "avx512ifma", fn() -> crate::x86::v3::F16c { crate::x86::v3::F16c::new() })
+ }
+}
+
+impl From for crate::x86::v3::Fma {
+ fn from(value: Avx512ifma) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512ifma = value] => "avx512ifma", fn() -> crate::x86::v3::Fma { crate::x86::v3::Fma::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse {
+ fn from(value: Avx512ifma) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512ifma = value] => "avx512ifma", fn() -> crate::x86::sse::Sse { crate::x86::sse::Sse::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse2 {
+ fn from(value: Avx512ifma) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512ifma = value] => "avx512ifma", fn() -> crate::x86::sse::Sse2 { crate::x86::sse::Sse2::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse3 {
+ fn from(value: Avx512ifma) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512ifma = value] => "avx512ifma", fn() -> crate::x86::sse::Sse3 { crate::x86::sse::Sse3::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_1 {
+ fn from(value: Avx512ifma) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512ifma = value] => "avx512ifma", fn() -> crate::x86::sse::Sse4_1 { crate::x86::sse::Sse4_1::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_2 {
+ fn from(value: Avx512ifma) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512ifma = value] => "avx512ifma", fn() -> crate::x86::sse::Sse4_2 { crate::x86::sse::Sse4_2::new() })
+ }
+}
+
+impl From for crate::x86::sse::SupplementalSse3 {
+ fn from(value: Avx512ifma) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512ifma = value] => "avx512ifma", fn() -> crate::x86::sse::SupplementalSse3 { crate::x86::sse::SupplementalSse3::new() })
+ }
+}
+
+const _: () = {
+ assert!(
+ core::mem::size_of::() == 0,
+ "Target feature tokens should be zero sized."
+ );
+};
diff --git a/fearless_simd_core/src/x86/avx512/avx512vbmi.rs b/fearless_simd_core/src/x86/avx512/avx512vbmi.rs
new file mode 100644
index 00000000..9d0ad4da
--- /dev/null
+++ b/fearless_simd_core/src/x86/avx512/avx512vbmi.rs
@@ -0,0 +1,190 @@
+// This file is automatically generated by `fearless_simd_core_gen`.
+// Its template can be found in `fearless_simd_core/gen/templates`.
+
+//! The AVX512-VBMI target feature.
+
+use crate::{TargetFeatureToken, trampoline};
+
+use core::fmt::Debug;
+
+/// [AVX512-VBMI] --- Advanced Vector Extensions 512-bit - Vector Byte Manipulation Instructions
+///
+/// [AVX512-VBMI]: https://en.wikipedia.org/wiki/AVX-512#BW,_DQ_and_VBMI
+///
+/// A token indicating that the current CPU has the `avx512vbmi` target feature.
+///
+/// # Example
+///
+/// This can be used to [`trampoline!`] into functions like:
+///
+/// ```rust
+/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+/// #[target_feature(enable = "avx512vbmi")]
+/// fn uses_avx512vbmi() {
+/// // ...
+/// }
+/// ```
+#[derive(Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Avx512vbmi {
+ // We don't use non_exhaustive because we don't want this struct to be constructible.
+ // in different modules in this crate.
+ _private: (),
+}
+
+impl Debug for Avx512vbmi {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ write!(f, r#""avx512vbmi" enabled."#)
+ }
+}
+
+unsafe impl TargetFeatureToken for Avx512vbmi {
+ const FEATURES: &[&str] = &[
+ "avx512vbmi",
+ "avx",
+ "avx2",
+ "avx512bw",
+ "avx512f",
+ "f16c",
+ "fma",
+ "sse",
+ "sse2",
+ "sse3",
+ "sse4.1",
+ "sse4.2",
+ "ssse3",
+ ];
+
+ #[inline(always)]
+ fn vectorize(self, f: impl FnOnce() -> R) -> R {
+ // Because we want this constant to be eagerly evaluated.
+ trampoline!([Avx512vbmi = self] => "avx512vbmi", <(R)> fn<(R)>(f: impl FnOnce() -> R = f) -> R { f() })
+ }
+}
+
+impl Avx512vbmi {
+ #[cfg(feature = "std")]
+ /// Create a new token if the `"avx512vbmi"` target feature is detected as enabled.
+ ///
+ /// This does not do any caching internally, although note that the standard
+ /// library does internally cache the features it detects.
+ // TODO: Consider a manual override feature/env var?
+ pub fn try_new() -> Option {
+ // Feature flag required to make docs compile.
+ // TODO: Extract into a (private) crate::x86::is_x86_feature_detected?
+ #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+ if std::arch::is_x86_feature_detected!("avx512vbmi") {
+ // Safety: The required CPU feature was detected.
+ unsafe { Some(Self::new()) }
+ } else {
+ None
+ }
+ }
+
+ #[target_feature(enable = "avx512vbmi")]
+ /// Create a new token for the "avx512vbmi" target feature.
+ ///
+ /// This method is useful to get a new token if you have an external proof that
+ /// AVX512-VBMI is available. This could happen if you are in a target feature
+ /// function called by an external library user.
+ ///
+ /// # Safety
+ ///
+ /// No conditions other than those inherited from the target feature attribute,
+ /// i.e. that the "avx512vbmi" target feature is available.
+ ///
+ /// [implicitly enables]: https://doc.rust-lang.org/beta/reference/attributes/codegen.html?highlight=implicitly%20enabled#r-attributes.codegen.target_feature.safety-restrictions
+ pub fn new() -> Self {
+ Self { _private: () }
+ }
+}
+
+impl From for crate::x86::avx::Avx {
+ fn from(value: Avx512vbmi) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512vbmi = value] => "avx512vbmi", fn() -> crate::x86::avx::Avx { crate::x86::avx::Avx::new() })
+ }
+}
+
+impl From for crate::x86::avx::Avx2 {
+ fn from(value: Avx512vbmi) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512vbmi = value] => "avx512vbmi", fn() -> crate::x86::avx::Avx2 { crate::x86::avx::Avx2::new() })
+ }
+}
+
+impl From for crate::x86::avx512::Avx512bw {
+ fn from(value: Avx512vbmi) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512vbmi = value] => "avx512vbmi", fn() -> crate::x86::avx512::Avx512bw { crate::x86::avx512::Avx512bw::new() })
+ }
+}
+
+impl From for crate::x86::avx512::Avx512f {
+ fn from(value: Avx512vbmi) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512vbmi = value] => "avx512vbmi", fn() -> crate::x86::avx512::Avx512f { crate::x86::avx512::Avx512f::new() })
+ }
+}
+
+impl From for crate::x86::v3::F16c {
+ fn from(value: Avx512vbmi) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512vbmi = value] => "avx512vbmi", fn() -> crate::x86::v3::F16c { crate::x86::v3::F16c::new() })
+ }
+}
+
+impl From for crate::x86::v3::Fma {
+ fn from(value: Avx512vbmi) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512vbmi = value] => "avx512vbmi", fn() -> crate::x86::v3::Fma { crate::x86::v3::Fma::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse {
+ fn from(value: Avx512vbmi) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512vbmi = value] => "avx512vbmi", fn() -> crate::x86::sse::Sse { crate::x86::sse::Sse::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse2 {
+ fn from(value: Avx512vbmi) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512vbmi = value] => "avx512vbmi", fn() -> crate::x86::sse::Sse2 { crate::x86::sse::Sse2::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse3 {
+ fn from(value: Avx512vbmi) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512vbmi = value] => "avx512vbmi", fn() -> crate::x86::sse::Sse3 { crate::x86::sse::Sse3::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_1 {
+ fn from(value: Avx512vbmi) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512vbmi = value] => "avx512vbmi", fn() -> crate::x86::sse::Sse4_1 { crate::x86::sse::Sse4_1::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_2 {
+ fn from(value: Avx512vbmi) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512vbmi = value] => "avx512vbmi", fn() -> crate::x86::sse::Sse4_2 { crate::x86::sse::Sse4_2::new() })
+ }
+}
+
+impl From for crate::x86::sse::SupplementalSse3 {
+ fn from(value: Avx512vbmi) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512vbmi = value] => "avx512vbmi", fn() -> crate::x86::sse::SupplementalSse3 { crate::x86::sse::SupplementalSse3::new() })
+ }
+}
+
+const _: () = {
+ assert!(
+ core::mem::size_of::() == 0,
+ "Target feature tokens should be zero sized."
+ );
+};
diff --git a/fearless_simd_core/src/x86/avx512/avx512vbmi2.rs b/fearless_simd_core/src/x86/avx512/avx512vbmi2.rs
new file mode 100644
index 00000000..cfff6b25
--- /dev/null
+++ b/fearless_simd_core/src/x86/avx512/avx512vbmi2.rs
@@ -0,0 +1,190 @@
+// This file is automatically generated by `fearless_simd_core_gen`.
+// Its template can be found in `fearless_simd_core/gen/templates`.
+
+//! The AVX512-VBMI2 target feature.
+
+use crate::{TargetFeatureToken, trampoline};
+
+use core::fmt::Debug;
+
+/// [AVX512-VBMI2] --- Advanced Vector Extensions 512-bit - Vector Byte Manipulation Instructions 2
+///
+/// [AVX512-VBMI2]: https://en.wikipedia.org/wiki/AVX-512#VBMI2
+///
+/// A token indicating that the current CPU has the `avx512vbmi2` target feature.
+///
+/// # Example
+///
+/// This can be used to [`trampoline!`] into functions like:
+///
+/// ```rust
+/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+/// #[target_feature(enable = "avx512vbmi2")]
+/// fn uses_avx512vbmi2() {
+/// // ...
+/// }
+/// ```
+#[derive(Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Avx512vbmi2 {
+ // We don't use non_exhaustive because we don't want this struct to be constructible.
+ // in different modules in this crate.
+ _private: (),
+}
+
+impl Debug for Avx512vbmi2 {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ write!(f, r#""avx512vbmi2" enabled."#)
+ }
+}
+
+unsafe impl TargetFeatureToken for Avx512vbmi2 {
+ const FEATURES: &[&str] = &[
+ "avx512vbmi2",
+ "avx",
+ "avx2",
+ "avx512bw",
+ "avx512f",
+ "f16c",
+ "fma",
+ "sse",
+ "sse2",
+ "sse3",
+ "sse4.1",
+ "sse4.2",
+ "ssse3",
+ ];
+
+ #[inline(always)]
+ fn vectorize(self, f: impl FnOnce() -> R) -> R {
+ // Because we want this constant to be eagerly evaluated.
+ trampoline!([Avx512vbmi2 = self] => "avx512vbmi2", <(R)> fn<(R)>(f: impl FnOnce() -> R = f) -> R { f() })
+ }
+}
+
+impl Avx512vbmi2 {
+ #[cfg(feature = "std")]
+ /// Create a new token if the `"avx512vbmi2"` target feature is detected as enabled.
+ ///
+ /// This does not do any caching internally, although note that the standard
+ /// library does internally cache the features it detects.
+ // TODO: Consider a manual override feature/env var?
+ pub fn try_new() -> Option {
+ // Feature flag required to make docs compile.
+ // TODO: Extract into a (private) crate::x86::is_x86_feature_detected?
+ #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+ if std::arch::is_x86_feature_detected!("avx512vbmi2") {
+ // Safety: The required CPU feature was detected.
+ unsafe { Some(Self::new()) }
+ } else {
+ None
+ }
+ }
+
+ #[target_feature(enable = "avx512vbmi2")]
+ /// Create a new token for the "avx512vbmi2" target feature.
+ ///
+ /// This method is useful to get a new token if you have an external proof that
+ /// AVX512-VBMI2 is available. This could happen if you are in a target feature
+ /// function called by an external library user.
+ ///
+ /// # Safety
+ ///
+ /// No conditions other than those inherited from the target feature attribute,
+ /// i.e. that the "avx512vbmi2" target feature is available.
+ ///
+ /// [implicitly enables]: https://doc.rust-lang.org/beta/reference/attributes/codegen.html?highlight=implicitly%20enabled#r-attributes.codegen.target_feature.safety-restrictions
+ pub fn new() -> Self {
+ Self { _private: () }
+ }
+}
+
+impl From for crate::x86::avx::Avx {
+ fn from(value: Avx512vbmi2) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512vbmi2 = value] => "avx512vbmi2", fn() -> crate::x86::avx::Avx { crate::x86::avx::Avx::new() })
+ }
+}
+
+impl From for crate::x86::avx::Avx2 {
+ fn from(value: Avx512vbmi2) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512vbmi2 = value] => "avx512vbmi2", fn() -> crate::x86::avx::Avx2 { crate::x86::avx::Avx2::new() })
+ }
+}
+
+impl From for crate::x86::avx512::Avx512bw {
+ fn from(value: Avx512vbmi2) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512vbmi2 = value] => "avx512vbmi2", fn() -> crate::x86::avx512::Avx512bw { crate::x86::avx512::Avx512bw::new() })
+ }
+}
+
+impl From for crate::x86::avx512::Avx512f {
+ fn from(value: Avx512vbmi2) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512vbmi2 = value] => "avx512vbmi2", fn() -> crate::x86::avx512::Avx512f { crate::x86::avx512::Avx512f::new() })
+ }
+}
+
+impl From for crate::x86::v3::F16c {
+ fn from(value: Avx512vbmi2) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512vbmi2 = value] => "avx512vbmi2", fn() -> crate::x86::v3::F16c { crate::x86::v3::F16c::new() })
+ }
+}
+
+impl From for crate::x86::v3::Fma {
+ fn from(value: Avx512vbmi2) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512vbmi2 = value] => "avx512vbmi2", fn() -> crate::x86::v3::Fma { crate::x86::v3::Fma::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse {
+ fn from(value: Avx512vbmi2) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512vbmi2 = value] => "avx512vbmi2", fn() -> crate::x86::sse::Sse { crate::x86::sse::Sse::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse2 {
+ fn from(value: Avx512vbmi2) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512vbmi2 = value] => "avx512vbmi2", fn() -> crate::x86::sse::Sse2 { crate::x86::sse::Sse2::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse3 {
+ fn from(value: Avx512vbmi2) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512vbmi2 = value] => "avx512vbmi2", fn() -> crate::x86::sse::Sse3 { crate::x86::sse::Sse3::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_1 {
+ fn from(value: Avx512vbmi2) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512vbmi2 = value] => "avx512vbmi2", fn() -> crate::x86::sse::Sse4_1 { crate::x86::sse::Sse4_1::new() })
+ }
+}
+
+impl From for crate::x86::sse::Sse4_2 {
+ fn from(value: Avx512vbmi2) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512vbmi2 = value] => "avx512vbmi2", fn() -> crate::x86::sse::Sse4_2 { crate::x86::sse::Sse4_2::new() })
+ }
+}
+
+impl From for crate::x86::sse::SupplementalSse3 {
+ fn from(value: Avx512vbmi2) -> Self {
+ // This also serves as a correctness check of the implicitly enabled features.
+ trampoline!([Avx512vbmi2 = value] => "avx512vbmi2", fn() -> crate::x86::sse::SupplementalSse3 { crate::x86::sse::SupplementalSse3::new() })
+ }
+}
+
+const _: () = {
+ assert!(
+ core::mem::size_of::