Skip to content
Open
2 changes: 2 additions & 0 deletions crypto/chacha/asm/chacha-x86_64.pl
Original file line number Diff line number Diff line change
Expand Up @@ -1263,6 +1263,7 @@ sub AVX2_lane_ROUND {
my $xframe = $win64 ? 0xa8 : 8;

$code.=<<___;
#ifndef MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX
.globl ChaCha20_ctr32_avx2
.type ChaCha20_ctr32_avx2,\@function,5
.align 32
Expand Down Expand Up @@ -1781,6 +1782,7 @@ sub AVX2_lane_ROUND {
ret
.cfi_endproc
.size ChaCha20_ctr32_avx2,.-ChaCha20_ctr32_avx2
#endif
___
}

Expand Down
14 changes: 13 additions & 1 deletion crypto/chacha/chacha.c
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ static void ChaCha20_ctr32(uint8_t *out, const uint8_t *in, size_t in_len,
return;
}
#endif
#if defined(CHACHA20_ASM_AVX2) && !defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX)
#if defined(CHACHA20_ASM_AVX2)
if (ChaCha20_ctr32_avx2_capable(in_len)) {
ChaCha20_ctr32_avx2(out, in, in_len, key, counter);
return;
Expand Down Expand Up @@ -245,3 +245,15 @@ void CRYPTO_chacha_20(uint8_t *out, const uint8_t *in, size_t in_len,
}

#endif

#if defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX)

#if defined(CHACHA20_ASM_AVX2)
void ChaCha20_ctr32_avx2(uint8_t *out, const uint8_t *in, size_t in_len,
const uint32_t key[8], const uint32_t counter[4]) {
perror("ChaCha20_ctr32_avx2");
abort();
}
#endif // defined(CHACHA20_ASM_AVX2)

#endif // defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX)
2 changes: 1 addition & 1 deletion crypto/chacha/chacha_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -356,7 +356,7 @@ static void check_abi(uint8_t *out, const uint8_t *in, size_t in_len,
CHECK_ABI(ChaCha20_ctr32_neon, out, in, in_len, key, counter);
}
#endif
#if defined(CHACHA20_ASM_AVX2) && !defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX)
#if defined(CHACHA20_ASM_AVX2)
if (ChaCha20_ctr32_avx2_capable(in_len)) {
CHECK_ABI(ChaCha20_ctr32_avx2, out, in, in_len, key, counter);
}
Expand Down
3 changes: 3 additions & 0 deletions crypto/chacha/internal.h
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@ void ChaCha20_ctr32_neon(uint8_t *out, const uint8_t *in, size_t in_len,

#define CHACHA20_ASM_AVX2
OPENSSL_INLINE int ChaCha20_ctr32_avx2_capable(size_t len) {
#if defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX)
return 0;
#endif // defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX)
return len > 128 && CRYPTO_is_AVX2_capable();
}
void ChaCha20_ctr32_avx2(uint8_t *out, const uint8_t *in, size_t in_len,
Expand Down
2 changes: 2 additions & 0 deletions crypto/fipsmodule/sha/asm/sha1-x86_64.pl
Original file line number Diff line number Diff line change
Expand Up @@ -1318,6 +1318,7 @@ ()
my $frame="%r13";

$code.=<<___;
#ifndef MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX
.globl sha1_block_data_order_avx2
.type sha1_block_data_order_avx2,\@function,3
.align 16
Expand Down Expand Up @@ -1792,6 +1793,7 @@ ()
ret
.cfi_endproc
.size sha1_block_data_order_avx2,.-sha1_block_data_order_avx2
#endif
___
}
}
Expand Down
18 changes: 18 additions & 0 deletions crypto/fipsmodule/sha/internal.h
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,9 @@ void sha1_block_data_order_ssse3(uint32_t state[5], const uint8_t *data,

#define SHA1_ASM_AVX
OPENSSL_INLINE int sha1_avx_capable(void) {
#if defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_AVX)
return 0;
#endif // defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_AVX)
// Pre-Zen AMD CPUs had slow SHLD/SHRD; Zen added the SHA extension; see the
// discussion in sha1-586.pl.
//
Expand All @@ -215,6 +218,9 @@ void sha256_block_data_order_ssse3(uint32_t state[8], const uint8_t *data,

#define SHA256_ASM_AVX
OPENSSL_INLINE int sha256_avx_capable(void) {
#if defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_AVX)
return 0;
#endif // defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_AVX)
// Pre-Zen AMD CPUs had slow SHLD/SHRD; Zen added the SHA extension; see the
// discussion in sha1-586.pl.
//
Expand Down Expand Up @@ -245,6 +251,9 @@ OPENSSL_INLINE int sha1_hw_capable(void) {

#define SHA1_ASM_AVX2
OPENSSL_INLINE int sha1_avx2_capable(void) {
#if defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_AVX)
return 0;
#endif // defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_AVX)
// TODO: Simplify this logic, which was extracted from the assembly:
// * Does AVX2 imply SSSE3?
// * sha1_block_data_order_avx2 does not seem to use SSSE3 instructions.
Expand All @@ -256,6 +265,9 @@ void sha1_block_data_order_avx2(uint32_t state[5], const uint8_t *data,

#define SHA1_ASM_AVX
OPENSSL_INLINE int sha1_avx_capable(void) {
#if defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_AVX)
return 0;
#endif // defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_AVX)
// TODO: Simplify this logic, which was extracted from the assembly:
// * Does AVX imply SSSE3?
// * sha1_block_data_order_avx does not seem to use SSSE3 instructions.
Expand All @@ -281,6 +293,9 @@ OPENSSL_INLINE int sha256_hw_capable(void) {

#define SHA256_ASM_AVX
OPENSSL_INLINE int sha256_avx_capable(void) {
#if defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_AVX)
return 0;
#endif // defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_AVX)
// TODO: Simplify this logic, which was extracted from the assembly:
// * Does AVX imply SSSE3?
// * sha256_block_data_order_avx does not seem to use SSSE3 instructions.
Expand All @@ -301,6 +316,9 @@ void sha256_block_data_order_ssse3(uint32_t state[8], const uint8_t *data,

#define SHA512_ASM_AVX
OPENSSL_INLINE int sha512_avx_capable(void) {
#if defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_AVX)
return 0;
#endif // defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_AVX)
// TODO: Simplify this logic, which was extracted from the assembly:
// * Does AVX imply SSSE3?
// * sha512_block_data_order_avx does not seem to use SSSE3 instructions.
Expand Down
16 changes: 14 additions & 2 deletions crypto/fipsmodule/sha/sha1.c
Original file line number Diff line number Diff line change
Expand Up @@ -396,13 +396,13 @@ static void sha1_block_data_order(uint32_t state[5], const uint8_t *data,
return;
}
#endif
#if defined(SHA1_ASM_AVX2) && !defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_AVX)
#if defined(SHA1_ASM_AVX2)
if (sha1_avx2_capable()) {
sha1_block_data_order_avx2(state, data, num);
return;
}
#endif
#if defined(SHA1_ASM_AVX) && !defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_AVX)
#if defined(SHA1_ASM_AVX)
if (sha1_avx_capable()) {
sha1_block_data_order_avx(state, data, num);
return;
Expand Down Expand Up @@ -441,3 +441,15 @@ static void sha1_block_data_order(uint32_t state[5], const uint8_t *data,
#undef BODY_40_59
#undef BODY_60_79
#undef X

#if defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX)

#if defined(SHA1_ASM_AVX2)
void sha1_block_data_order_avx2(uint32_t state[5], const uint8_t *data,
size_t num) {
perror("sha1_block_data_order_avx2");
abort();
}
#endif // defined(ASH1_ASM_AVX2)

#endif // defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX)
14 changes: 13 additions & 1 deletion crypto/fipsmodule/sha/sha256.c
Original file line number Diff line number Diff line change
Expand Up @@ -392,7 +392,7 @@ static void sha256_block_data_order(uint32_t state[8], const uint8_t *data,
return;
}
#endif
#if defined(SHA256_ASM_AVX) && !defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_AVX)
#if defined(SHA256_ASM_AVX)
if (sha256_avx_capable()) {
sha256_block_data_order_avx(state, data, num);
return;
Expand Down Expand Up @@ -429,3 +429,15 @@ void SHA256_TransformBlocks(uint32_t state[8], const uint8_t *data,
#undef Maj
#undef ROUND_00_15
#undef ROUND_16_63

#if defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_AVX)

#if defined(SHA256_ASM_AVX)
void sha256_block_data_order_avx(uint32_t state[8], const uint8_t *data,
size_t num) {
perror("sha256_block_data_order_avx");
abort();
}
#endif // defined(SHA256_ASM_AVX)

#endif // defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_AVX)
14 changes: 13 additions & 1 deletion crypto/fipsmodule/sha/sha512.c
Original file line number Diff line number Diff line change
Expand Up @@ -689,7 +689,7 @@ static void sha512_block_data_order(uint64_t state[8], const uint8_t *data,
return;
}
#endif
#if defined(SHA512_ASM_AVX) && !defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_AVX)
#if defined(SHA512_ASM_AVX)
if (sha512_avx_capable()) {
sha512_block_data_order_avx(state, data, num);
return;
Expand All @@ -714,3 +714,15 @@ static void sha512_block_data_order(uint64_t state[8], const uint8_t *data,
#undef Maj
#undef ROUND_00_15
#undef ROUND_16_80

#if defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_AVX)

#if defined(SHA512_ASM_AVX)
void sha512_block_data_order_avx(uint64_t state[8], const uint8_t *data,
size_t num) {
perror("sha512_block_data_order_avx");
abort();
}
#endif // defined(SHA512_ASM_AVX)

#endif // defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_AVX)
8 changes: 4 additions & 4 deletions crypto/fipsmodule/sha/sha_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,12 @@ TEST(SHATest, SHA1ABI) {
CHECK_ABI(sha1_block_data_order_hw, ctx.h, kBuf, blocks);
}
#endif
#if defined(SHA1_ASM_AVX2) && !defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_AVX)
#if defined(SHA1_ASM_AVX2)
if (sha1_avx2_capable()) {
CHECK_ABI(sha1_block_data_order_avx2, ctx.h, kBuf, blocks);
}
#endif
#if defined(SHA1_ASM_AVX) && !defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_AVX)
#if defined(SHA1_ASM_AVX)
if (sha1_avx_capable()) {
CHECK_ABI(sha1_block_data_order_avx, ctx.h, kBuf, blocks);
}
Expand Down Expand Up @@ -76,7 +76,7 @@ TEST(SHATest, SHA256ABI) {
CHECK_ABI(sha256_block_data_order_hw, ctx.h, kBuf, blocks);
}
#endif
#if defined(SHA256_ASM_AVX) && !defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_AVX)
#if defined(SHA256_ASM_AVX)
if (sha256_avx_capable()) {
CHECK_ABI(sha256_block_data_order_avx, ctx.h, kBuf, blocks);
}
Expand Down Expand Up @@ -111,7 +111,7 @@ TEST(SHATest, SHA512ABI) {
CHECK_ABI(sha512_block_data_order_hw, ctx.h, kBuf, blocks);
}
#endif
#if defined(SHA512_ASM_AVX) && !defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_AVX)
#if defined(SHA512_ASM_AVX)
if (sha512_avx_capable()) {
CHECK_ABI(sha512_block_data_order_avx, ctx.h, kBuf, blocks);
}
Expand Down
2 changes: 2 additions & 0 deletions generated-src/linux-x86_64/crypto/chacha/chacha-x86_64.S
Original file line number Diff line number Diff line change
Expand Up @@ -998,6 +998,7 @@ _CET_ENDBR
.byte 0xf3,0xc3
.cfi_endproc
.size ChaCha20_ctr32_ssse3_4x,.-ChaCha20_ctr32_ssse3_4x
#ifndef MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX
.globl ChaCha20_ctr32_avx2
.hidden ChaCha20_ctr32_avx2
.type ChaCha20_ctr32_avx2,@function
Expand Down Expand Up @@ -1607,3 +1608,4 @@ _CET_ENDBR
.cfi_endproc
.size ChaCha20_ctr32_avx2,.-ChaCha20_ctr32_avx2
#endif
#endif
2 changes: 2 additions & 0 deletions generated-src/linux-x86_64/crypto/fipsmodule/sha1-x86_64.S
Original file line number Diff line number Diff line change
Expand Up @@ -3735,6 +3735,7 @@ _CET_ENDBR
.byte 0xf3,0xc3
.cfi_endproc
.size sha1_block_data_order_avx,.-sha1_block_data_order_avx
#ifndef MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX
.globl sha1_block_data_order_avx2
.hidden sha1_block_data_order_avx2
.type sha1_block_data_order_avx2,@function
Expand Down Expand Up @@ -5430,6 +5431,7 @@ _CET_ENDBR
.byte 0xf3,0xc3
.cfi_endproc
.size sha1_block_data_order_avx2,.-sha1_block_data_order_avx2
#endif
.section .rodata
.align 64
K_XX_XX:
Expand Down
2 changes: 2 additions & 0 deletions generated-src/mac-x86_64/crypto/chacha/chacha-x86_64.S
Original file line number Diff line number Diff line change
Expand Up @@ -992,6 +992,7 @@ L$4x_epilogue:
.byte 0xf3,0xc3


#ifndef MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX
.globl _ChaCha20_ctr32_avx2
.private_extern _ChaCha20_ctr32_avx2

Expand Down Expand Up @@ -1601,3 +1602,4 @@ L$8x_epilogue:


#endif
#endif
2 changes: 2 additions & 0 deletions generated-src/mac-x86_64/crypto/fipsmodule/sha1-x86_64.S
Original file line number Diff line number Diff line change
Expand Up @@ -3735,6 +3735,7 @@ L$epilogue_avx:
.byte 0xf3,0xc3


#ifndef MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX
.globl _sha1_block_data_order_avx2
.private_extern _sha1_block_data_order_avx2

Expand Down Expand Up @@ -5430,6 +5431,7 @@ L$epilogue_avx2:
.byte 0xf3,0xc3


#endif
.section __DATA,__const
.p2align 6
K_XX_XX:
Expand Down
2 changes: 2 additions & 0 deletions generated-src/win-x86_64/crypto/chacha/chacha-x86_64.asm
Original file line number Diff line number Diff line change
Expand Up @@ -1065,6 +1065,7 @@ $L$4x_epilogue:
DB 0F3h,0C3h ;repret

$L$SEH_end_ChaCha20_ctr32_ssse3_4x:
%ifndef MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX
global ChaCha20_ctr32_avx2

ALIGN 32
Expand Down Expand Up @@ -1706,6 +1707,7 @@ $L$8x_epilogue:
DB 0F3h,0C3h ;repret

$L$SEH_end_ChaCha20_ctr32_avx2:
%endif
EXTERN __imp_RtlVirtualUnwind

ALIGN 16
Expand Down
2 changes: 2 additions & 0 deletions generated-src/win-x86_64/crypto/fipsmodule/sha1-x86_64.asm
Original file line number Diff line number Diff line change
Expand Up @@ -3819,6 +3819,7 @@ $L$epilogue_avx:
DB 0F3h,0C3h ;repret

$L$SEH_end_sha1_block_data_order_avx:
%ifndef MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX
global sha1_block_data_order_avx2

ALIGN 16
Expand Down Expand Up @@ -5538,6 +5539,7 @@ $L$epilogue_avx2:
DB 0F3h,0C3h ;repret

$L$SEH_end_sha1_block_data_order_avx2:
%endif
section .rdata rdata align=8
ALIGN 64
K_XX_XX:
Expand Down
Loading