Skip to content

Commit 3c40cdb

Browse files
torvaldstorvalds
committed
Merge branch 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6
Pull crypto fixes from Herbert Xu: "This fixes a potential scheduling latency problem for the algorithms used by WireGuard" * 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: crypto: arch/nhpoly1305 - process in explicit 4k chunks crypto: arch/lib - limit simd usage to 4k chunks
2 parents dc56c5a + a9a8ba9 commit 3c40cdb

File tree

11 files changed

+69
-34
lines changed

11 files changed

+69
-34
lines changed

‎arch/arm/crypto/chacha-glue.c

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -91,9 +91,17 @@ void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes,
9191
return;
9292
}
9393

94-
kernel_neon_begin();
95-
chacha_doneon(state, dst, src, bytes, nrounds);
96-
kernel_neon_end();
94+
do {
95+
unsigned int todo = min_t(unsigned int, bytes, SZ_4K);
96+
97+
kernel_neon_begin();
98+
chacha_doneon(state, dst, src, todo, nrounds);
99+
kernel_neon_end();
100+
101+
bytes -= todo;
102+
src += todo;
103+
dst += todo;
104+
} while (bytes);
97105
}
98106
EXPORT_SYMBOL(chacha_crypt_arch);
99107

‎arch/arm/crypto/nhpoly1305-neon-glue.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ static int nhpoly1305_neon_update(struct shash_desc *desc,
3030
return crypto_nhpoly1305_update(desc, src, srclen);
3131

3232
do {
33-
unsigned int n = min_t(unsigned int, srclen, PAGE_SIZE);
33+
unsigned int n = min_t(unsigned int, srclen, SZ_4K);
3434

3535
kernel_neon_begin();
3636
crypto_nhpoly1305_update_helper(desc, src, n, _nh_neon);

‎arch/arm/crypto/poly1305-glue.c

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -160,13 +160,20 @@ void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,
160160
unsigned int len = round_down(nbytes, POLY1305_BLOCK_SIZE);
161161

162162
if (static_branch_likely(&have_neon) && do_neon) {
163-
kernel_neon_begin();
164-
poly1305_blocks_neon(&dctx->h, src, len, 1);
165-
kernel_neon_end();
163+
do {
164+
unsigned int todo = min_t(unsigned int, len, SZ_4K);
165+
166+
kernel_neon_begin();
167+
poly1305_blocks_neon(&dctx->h, src, todo, 1);
168+
kernel_neon_end();
169+
170+
len -= todo;
171+
src += todo;
172+
} while (len);
166173
} else {
167174
poly1305_blocks_arm(&dctx->h, src, len, 1);
175+
src += len;
168176
}
169-
src += len;
170177
nbytes %= POLY1305_BLOCK_SIZE;
171178
}
172179

‎arch/arm64/crypto/chacha-neon-glue.c

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -87,9 +87,17 @@ void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes,
8787
!crypto_simd_usable())
8888
return chacha_crypt_generic(state, dst, src, bytes, nrounds);
8989

90-
kernel_neon_begin();
91-
chacha_doneon(state, dst, src, bytes, nrounds);
92-
kernel_neon_end();
90+
do {
91+
unsigned int todo = min_t(unsigned int, bytes, SZ_4K);
92+
93+
kernel_neon_begin();
94+
chacha_doneon(state, dst, src, todo, nrounds);
95+
kernel_neon_end();
96+
97+
bytes -= todo;
98+
src += todo;
99+
dst += todo;
100+
} while (bytes);
93101
}
94102
EXPORT_SYMBOL(chacha_crypt_arch);
95103

‎arch/arm64/crypto/nhpoly1305-neon-glue.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ static int nhpoly1305_neon_update(struct shash_desc *desc,
3030
return crypto_nhpoly1305_update(desc, src, srclen);
3131

3232
do {
33-
unsigned int n = min_t(unsigned int, srclen, PAGE_SIZE);
33+
unsigned int n = min_t(unsigned int, srclen, SZ_4K);
3434

3535
kernel_neon_begin();
3636
crypto_nhpoly1305_update_helper(desc, src, n, _nh_neon);

‎arch/arm64/crypto/poly1305-glue.c

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -143,13 +143,20 @@ void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,
143143
unsigned int len = round_down(nbytes, POLY1305_BLOCK_SIZE);
144144

145145
if (static_branch_likely(&have_neon) && crypto_simd_usable()) {
146-
kernel_neon_begin();
147-
poly1305_blocks_neon(&dctx->h, src, len, 1);
148-
kernel_neon_end();
146+
do {
147+
unsigned int todo = min_t(unsigned int, len, SZ_4K);
148+
149+
kernel_neon_begin();
150+
poly1305_blocks_neon(&dctx->h, src, todo, 1);
151+
kernel_neon_end();
152+
153+
len -= todo;
154+
src += todo;
155+
} while (len);
149156
} else {
150157
poly1305_blocks(&dctx->h, src, len, 1);
158+
src += len;
151159
}
152-
src += len;
153160
nbytes %= POLY1305_BLOCK_SIZE;
154161
}
155162

‎arch/x86/crypto/blake2s-glue.c

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,16 +32,16 @@ void blake2s_compress_arch(struct blake2s_state *state,
3232
const u32 inc)
3333
{
3434
/* SIMD disables preemption, so relax after processing each page. */
35-
BUILD_BUG_ON(PAGE_SIZE / BLAKE2S_BLOCK_SIZE < 8);
35+
BUILD_BUG_ON(SZ_4K / BLAKE2S_BLOCK_SIZE < 8);
3636

3737
if (!static_branch_likely(&blake2s_use_ssse3) || !crypto_simd_usable()) {
3838
blake2s_compress_generic(state, block, nblocks, inc);
3939
return;
4040
}
4141

42-
for (;;) {
42+
do {
4343
const size_t blocks = min_t(size_t, nblocks,
44-
PAGE_SIZE / BLAKE2S_BLOCK_SIZE);
44+
SZ_4K / BLAKE2S_BLOCK_SIZE);
4545

4646
kernel_fpu_begin();
4747
if (IS_ENABLED(CONFIG_AS_AVX512) &&
@@ -52,10 +52,8 @@ void blake2s_compress_arch(struct blake2s_state *state,
5252
kernel_fpu_end();
5353

5454
nblocks -= blocks;
55-
if (!nblocks)
56-
break;
5755
block += blocks * BLAKE2S_BLOCK_SIZE;
58-
}
56+
} while (nblocks);
5957
}
6058
EXPORT_SYMBOL(blake2s_compress_arch);
6159

‎arch/x86/crypto/chacha_glue.c

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -153,9 +153,17 @@ void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes,
153153
bytes <= CHACHA_BLOCK_SIZE)
154154
return chacha_crypt_generic(state, dst, src, bytes, nrounds);
155155

156-
kernel_fpu_begin();
157-
chacha_dosimd(state, dst, src, bytes, nrounds);
158-
kernel_fpu_end();
156+
do {
157+
unsigned int todo = min_t(unsigned int, bytes, SZ_4K);
158+
159+
kernel_fpu_begin();
160+
chacha_dosimd(state, dst, src, todo, nrounds);
161+
kernel_fpu_end();
162+
163+
bytes -= todo;
164+
src += todo;
165+
dst += todo;
166+
} while (bytes);
159167
}
160168
EXPORT_SYMBOL(chacha_crypt_arch);
161169

‎arch/x86/crypto/nhpoly1305-avx2-glue.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ static int nhpoly1305_avx2_update(struct shash_desc *desc,
2929
return crypto_nhpoly1305_update(desc, src, srclen);
3030

3131
do {
32-
unsigned int n = min_t(unsigned int, srclen, PAGE_SIZE);
32+
unsigned int n = min_t(unsigned int, srclen, SZ_4K);
3333

3434
kernel_fpu_begin();
3535
crypto_nhpoly1305_update_helper(desc, src, n, _nh_avx2);

‎arch/x86/crypto/nhpoly1305-sse2-glue.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ static int nhpoly1305_sse2_update(struct shash_desc *desc,
2929
return crypto_nhpoly1305_update(desc, src, srclen);
3030

3131
do {
32-
unsigned int n = min_t(unsigned int, srclen, PAGE_SIZE);
32+
unsigned int n = min_t(unsigned int, srclen, SZ_4K);
3333

3434
kernel_fpu_begin();
3535
crypto_nhpoly1305_update_helper(desc, src, n, _nh_sse2);

0 commit comments

Comments
 (0)