Skip to content

Commit 22dd9e3

Browse files
silvanshadesilvanshade
committed
Migrate to intrinsics for VAES
1 parent 5a4b683 commit 22dd9e3

File tree

16 files changed

+1768
-159
lines changed

16 files changed

+1768
-159
lines changed

‎aes/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ hazmat = [] # Expose cryptographically hazardous APIs
3131

3232
[lints.rust.unexpected_cfgs]
3333
level = "warn"
34-
check-cfg = ["cfg(aes_compact)", "cfg(aes_force_soft)"]
34+
check-cfg = ["cfg(aes_compact)", "cfg(aes_force_soft)", "cfg(aes_avx256)", "cfg(aes_avx512)"]
3535

3636
[package.metadata.docs.rs]
3737
all-features = true

‎aes/src/armv8.rs‎

Lines changed: 7 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,13 @@ use cipher::{
2525
};
2626
use core::fmt;
2727

28+
pub(crate) mod features {
29+
cpufeatures::new!(features_aes, "aes");
30+
pub(crate) mod aes {
31+
pub use super::features_aes::*;
32+
}
33+
}
34+
2835
impl_backends!(
2936
enc_name = Aes128BackEnc,
3037
dec_name = Aes128BackDec,
@@ -86,18 +93,6 @@ macro_rules! define_aes_impl {
8693
decrypt: $name_back_dec,
8794
}
8895

89-
impl $name {
90-
#[inline(always)]
91-
pub(crate) fn get_enc_backend(&self) -> &$name_back_enc {
92-
&self.encrypt
93-
}
94-
95-
#[inline(always)]
96-
pub(crate) fn get_dec_backend(&self) -> &$name_back_dec {
97-
&self.decrypt
98-
}
99-
}
100-
10196
impl KeySizeUser for $name {
10297
type KeySize = $key_size;
10398
}
@@ -182,13 +177,6 @@ macro_rules! define_aes_impl {
182177
backend: $name_back_enc,
183178
}
184179

185-
impl $name_enc {
186-
#[inline(always)]
187-
pub(crate) fn get_enc_backend(&self) -> &$name_back_enc {
188-
&self.backend
189-
}
190-
}
191-
192180
impl KeySizeUser for $name_enc {
193181
type KeySize = $key_size;
194182
}
@@ -248,13 +236,6 @@ macro_rules! define_aes_impl {
248236
backend: $name_back_dec,
249237
}
250238

251-
impl $name_dec {
252-
#[inline(always)]
253-
pub(crate) fn get_dec_backend(&self) -> &$name_back_dec {
254-
&self.backend
255-
}
256-
}
257-
258239
impl KeySizeUser for $name_dec {
259240
type KeySize = $key_size;
260241
}

‎aes/src/autodetect.rs‎

Lines changed: 42 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,10 @@ use core::fmt;
1212
use core::mem::ManuallyDrop;
1313

1414
#[cfg(target_arch = "aarch64")]
15-
use crate::armv8 as intrinsics;
15+
use crate::armv8 as arch;
1616

1717
#[cfg(any(target_arch = "x86_64", target_arch = "x86"))]
18-
use crate::ni as intrinsics;
19-
20-
cpufeatures::new!(aes_intrinsics, "aes");
18+
use crate::x86 as arch;
2119

2220
macro_rules! define_aes_impl {
2321
(
@@ -29,21 +27,21 @@ macro_rules! define_aes_impl {
2927
doc = $doc:expr,
3028
) => {
3129
mod $module {
32-
use super::{intrinsics, soft};
30+
use super::{arch, soft};
3331
use core::mem::ManuallyDrop;
3432

3533
pub(super) union Inner {
36-
pub(super) intrinsics: ManuallyDrop<intrinsics::$name>,
34+
pub(super) arch: ManuallyDrop<arch::$name>,
3735
pub(super) soft: ManuallyDrop<soft::$name>,
3836
}
3937

4038
pub(super) union InnerEnc {
41-
pub(super) intrinsics: ManuallyDrop<intrinsics::$name_enc>,
39+
pub(super) arch: ManuallyDrop<arch::$name_enc>,
4240
pub(super) soft: ManuallyDrop<soft::$name_enc>,
4341
}
4442

4543
pub(super) union InnerDec {
46-
pub(super) intrinsics: ManuallyDrop<intrinsics::$name_dec>,
44+
pub(super) arch: ManuallyDrop<arch::$name_dec>,
4745
pub(super) soft: ManuallyDrop<soft::$name_dec>,
4846
}
4947
}
@@ -52,7 +50,7 @@ macro_rules! define_aes_impl {
5250
#[doc = "block cipher"]
5351
pub struct $name {
5452
inner: $module::Inner,
55-
token: aes_intrinsics::InitToken,
53+
token: arch::features::aes::InitToken,
5654
}
5755

5856
impl KeySizeUser for $name {
@@ -70,9 +68,7 @@ macro_rules! define_aes_impl {
7068
use core::ops::Deref;
7169
let inner = if enc.token.get() {
7270
$module::Inner {
73-
intrinsics: ManuallyDrop::new(unsafe {
74-
enc.inner.intrinsics.deref().into()
75-
}),
71+
arch: ManuallyDrop::new(unsafe { enc.inner.arch.deref().into() }),
7672
}
7773
} else {
7874
$module::Inner {
@@ -90,11 +86,11 @@ macro_rules! define_aes_impl {
9086
impl KeyInit for $name {
9187
#[inline]
9288
fn new(key: &Key<Self>) -> Self {
93-
let (token, aesni_present) = aes_intrinsics::init_get();
89+
let (token, aes_features) = arch::features::aes::init_get();
9490

95-
let inner = if aesni_present {
91+
let inner = if aes_features {
9692
$module::Inner {
97-
intrinsics: ManuallyDrop::new(intrinsics::$name::new(key)),
93+
arch: ManuallyDrop::new(arch::$name::new(key)),
9894
}
9995
} else {
10096
$module::Inner {
@@ -115,7 +111,7 @@ macro_rules! define_aes_impl {
115111
fn clone(&self) -> Self {
116112
let inner = if self.token.get() {
117113
$module::Inner {
118-
intrinsics: unsafe { self.inner.intrinsics.clone() },
114+
arch: unsafe { self.inner.arch.clone() },
119115
}
120116
} else {
121117
$module::Inner {
@@ -136,38 +132,20 @@ macro_rules! define_aes_impl {
136132

137133
impl BlockCipherEncrypt for $name {
138134
fn encrypt_with_backend(&self, f: impl BlockCipherEncClosure<BlockSize = U16>) {
139-
unsafe {
140-
if self.token.get() {
141-
#[target_feature(enable = "aes")]
142-
unsafe fn inner(
143-
state: &intrinsics::$name,
144-
f: impl BlockCipherEncClosure<BlockSize = U16>,
145-
) {
146-
f.call(state.get_enc_backend());
147-
}
148-
inner(&self.inner.intrinsics, f);
149-
} else {
150-
f.call(&self.inner.soft.get_enc_backend());
151-
}
135+
if self.token.get() {
136+
unsafe { &self.inner.arch }.encrypt_with_backend(f)
137+
} else {
138+
unsafe { &self.inner.soft }.encrypt_with_backend(f)
152139
}
153140
}
154141
}
155142

156143
impl BlockCipherDecrypt for $name {
157144
fn decrypt_with_backend(&self, f: impl BlockCipherDecClosure<BlockSize = U16>) {
158-
unsafe {
159-
if self.token.get() {
160-
#[target_feature(enable = "aes")]
161-
unsafe fn inner(
162-
state: &intrinsics::$name,
163-
f: impl BlockCipherDecClosure<BlockSize = U16>,
164-
) {
165-
f.call(state.get_dec_backend());
166-
}
167-
inner(&self.inner.intrinsics, f);
168-
} else {
169-
f.call(&self.inner.soft.get_dec_backend());
170-
}
145+
if self.token.get() {
146+
unsafe { &self.inner.arch }.decrypt_with_backend(f)
147+
} else {
148+
unsafe { &self.inner.soft }.decrypt_with_backend(f)
171149
}
172150
}
173151
}
@@ -188,7 +166,7 @@ macro_rules! define_aes_impl {
188166
#[inline]
189167
fn drop(&mut self) {
190168
if self.token.get() {
191-
unsafe { ManuallyDrop::drop(&mut self.inner.intrinsics) };
169+
unsafe { ManuallyDrop::drop(&mut self.inner.arch) };
192170
} else {
193171
unsafe { ManuallyDrop::drop(&mut self.inner.soft) };
194172
};
@@ -202,7 +180,7 @@ macro_rules! define_aes_impl {
202180
#[doc = "block cipher (encrypt-only)"]
203181
pub struct $name_enc {
204182
inner: $module::InnerEnc,
205-
token: aes_intrinsics::InitToken,
183+
token: arch::features::aes::InitToken,
206184
}
207185

208186
impl KeySizeUser for $name_enc {
@@ -212,11 +190,11 @@ macro_rules! define_aes_impl {
212190
impl KeyInit for $name_enc {
213191
#[inline]
214192
fn new(key: &Key<Self>) -> Self {
215-
let (token, aesni_present) = aes_intrinsics::init_get();
193+
let (token, aes_features) = arch::features::aes::init_get();
216194

217-
let inner = if aesni_present {
195+
let inner = if aes_features {
218196
$module::InnerEnc {
219-
intrinsics: ManuallyDrop::new(intrinsics::$name_enc::new(key)),
197+
arch: ManuallyDrop::new(arch::$name_enc::new(key)),
220198
}
221199
} else {
222200
$module::InnerEnc {
@@ -237,7 +215,7 @@ macro_rules! define_aes_impl {
237215
fn clone(&self) -> Self {
238216
let inner = if self.token.get() {
239217
$module::InnerEnc {
240-
intrinsics: unsafe { self.inner.intrinsics.clone() },
218+
arch: unsafe { self.inner.arch.clone() },
241219
}
242220
} else {
243221
$module::InnerEnc {
@@ -258,19 +236,10 @@ macro_rules! define_aes_impl {
258236

259237
impl BlockCipherEncrypt for $name_enc {
260238
fn encrypt_with_backend(&self, f: impl BlockCipherEncClosure<BlockSize = U16>) {
261-
unsafe {
262-
if self.token.get() {
263-
#[target_feature(enable = "aes")]
264-
unsafe fn inner(
265-
state: &intrinsics::$name_enc,
266-
f: impl BlockCipherEncClosure<BlockSize = U16>,
267-
) {
268-
f.call(state.get_enc_backend());
269-
}
270-
inner(&self.inner.intrinsics, f);
271-
} else {
272-
f.call(&self.inner.soft.get_enc_backend());
273-
}
239+
if self.token.get() {
240+
unsafe { &self.inner.arch }.encrypt_with_backend(f)
241+
} else {
242+
unsafe { &self.inner.soft }.encrypt_with_backend(f)
274243
}
275244
}
276245
}
@@ -291,7 +260,7 @@ macro_rules! define_aes_impl {
291260
#[inline]
292261
fn drop(&mut self) {
293262
if self.token.get() {
294-
unsafe { ManuallyDrop::drop(&mut self.inner.intrinsics) };
263+
unsafe { ManuallyDrop::drop(&mut self.inner.arch) };
295264
} else {
296265
unsafe { ManuallyDrop::drop(&mut self.inner.soft) };
297266
};
@@ -305,7 +274,7 @@ macro_rules! define_aes_impl {
305274
#[doc = "block cipher (decrypt-only)"]
306275
pub struct $name_dec {
307276
inner: $module::InnerDec,
308-
token: aes_intrinsics::InitToken,
277+
token: arch::features::aes::InitToken,
309278
}
310279

311280
impl KeySizeUser for $name_dec {
@@ -324,9 +293,7 @@ macro_rules! define_aes_impl {
324293
use core::ops::Deref;
325294
let inner = if enc.token.get() {
326295
$module::InnerDec {
327-
intrinsics: ManuallyDrop::new(unsafe {
328-
enc.inner.intrinsics.deref().into()
329-
}),
296+
arch: ManuallyDrop::new(unsafe { enc.inner.arch.deref().into() }),
330297
}
331298
} else {
332299
$module::InnerDec {
@@ -344,11 +311,11 @@ macro_rules! define_aes_impl {
344311
impl KeyInit for $name_dec {
345312
#[inline]
346313
fn new(key: &Key<Self>) -> Self {
347-
let (token, aesni_present) = aes_intrinsics::init_get();
314+
let (token, aes_features) = arch::features::aes::init_get();
348315

349-
let inner = if aesni_present {
316+
let inner = if aes_features {
350317
$module::InnerDec {
351-
intrinsics: ManuallyDrop::new(intrinsics::$name_dec::new(key)),
318+
arch: ManuallyDrop::new(arch::$name_dec::new(key)),
352319
}
353320
} else {
354321
$module::InnerDec {
@@ -369,7 +336,7 @@ macro_rules! define_aes_impl {
369336
fn clone(&self) -> Self {
370337
let inner = if self.token.get() {
371338
$module::InnerDec {
372-
intrinsics: unsafe { self.inner.intrinsics.clone() },
339+
arch: unsafe { self.inner.arch.clone() },
373340
}
374341
} else {
375342
$module::InnerDec {
@@ -390,19 +357,10 @@ macro_rules! define_aes_impl {
390357

391358
impl BlockCipherDecrypt for $name_dec {
392359
fn decrypt_with_backend(&self, f: impl BlockCipherDecClosure<BlockSize = U16>) {
393-
unsafe {
394-
if self.token.get() {
395-
#[target_feature(enable = "aes")]
396-
unsafe fn inner(
397-
state: &intrinsics::$name_dec,
398-
f: impl BlockCipherDecClosure<BlockSize = U16>,
399-
) {
400-
f.call(state.get_dec_backend());
401-
}
402-
inner(&self.inner.intrinsics, f);
403-
} else {
404-
f.call(&self.inner.soft.get_dec_backend());
405-
}
360+
if self.token.get() {
361+
unsafe { &self.inner.arch }.decrypt_with_backend(f)
362+
} else {
363+
unsafe { &self.inner.soft }.decrypt_with_backend(f)
406364
}
407365
}
408366
}
@@ -423,7 +381,7 @@ macro_rules! define_aes_impl {
423381
#[inline]
424382
fn drop(&mut self) {
425383
if self.token.get() {
426-
unsafe { ManuallyDrop::drop(&mut self.inner.intrinsics) };
384+
unsafe { ManuallyDrop::drop(&mut self.inner.arch) };
427385
} else {
428386
unsafe { ManuallyDrop::drop(&mut self.inner.soft) };
429387
};

‎aes/src/hazmat.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ pub type Block8 = cipher::array::Array<Block, cipher::consts::U8>;
2121
use crate::armv8::hazmat as intrinsics;
2222

2323
#[cfg(all(any(target_arch = "x86_64", target_arch = "x86"), not(aes_force_soft)))]
24-
use crate::ni::hazmat as intrinsics;
24+
use crate::x86::ni::hazmat as intrinsics;
2525

2626
#[cfg(all(
2727
any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"),

0 commit comments

Comments
 (0)