Skip to content

Commit d887dec

Browse files
herbertxherbertx
committed
crypto: octeontx2 - Set DMA alignment explicitly
This driver has been implicitly relying on kmalloc alignment to be sufficient for DMA. This may no longer be the case with upcoming arm64 changes. This patch changes it to explicitly request DMA alignment from the Crypto API. Signed-off-by: Herbert Xu <[email protected]>
1 parent 0a55f4e commit d887dec

File tree

1 file changed

+40
-39
lines changed

1 file changed

+40
-39
lines changed

‎drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c‎

Lines changed: 40 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ static inline int validate_hmac_cipher_null(struct otx2_cpt_req_info *cpt_req)
8787

8888
req = container_of(cpt_req->areq, struct aead_request, base);
8989
tfm = crypto_aead_reqtfm(req);
90-
rctx = aead_request_ctx(req);
90+
rctx = aead_request_ctx_dma(req);
9191
if (memcmp(rctx->fctx.hmac.s.hmac_calc,
9292
rctx->fctx.hmac.s.hmac_recv,
9393
crypto_aead_authsize(tfm)) != 0)
@@ -137,7 +137,7 @@ static void output_iv_copyback(struct crypto_async_request *areq)
137137
ctx = crypto_skcipher_ctx(stfm);
138138
if (ctx->cipher_type == OTX2_CPT_AES_CBC ||
139139
ctx->cipher_type == OTX2_CPT_DES3_CBC) {
140-
rctx = skcipher_request_ctx(sreq);
140+
rctx = skcipher_request_ctx_dma(sreq);
141141
req_info = &rctx->cpt_req;
142142
ivsize = crypto_skcipher_ivsize(stfm);
143143
start = sreq->cryptlen - ivsize;
@@ -219,7 +219,7 @@ static inline int create_ctx_hdr(struct skcipher_request *req, u32 enc,
219219
u32 *argcnt)
220220
{
221221
struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
222-
struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
222+
struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
223223
struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
224224
struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
225225
struct otx2_cpt_fc_ctx *fctx = &rctx->fctx;
@@ -288,7 +288,7 @@ static inline int create_ctx_hdr(struct skcipher_request *req, u32 enc,
288288
static inline int create_input_list(struct skcipher_request *req, u32 enc,
289289
u32 enc_iv_len)
290290
{
291-
struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
291+
struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
292292
struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
293293
u32 argcnt = 0;
294294
int ret;
@@ -306,7 +306,7 @@ static inline int create_input_list(struct skcipher_request *req, u32 enc,
306306
static inline void create_output_list(struct skcipher_request *req,
307307
u32 enc_iv_len)
308308
{
309-
struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
309+
struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
310310
struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
311311
u32 argcnt = 0;
312312

@@ -325,7 +325,7 @@ static inline void create_output_list(struct skcipher_request *req,
325325
static int skcipher_do_fallback(struct skcipher_request *req, bool is_enc)
326326
{
327327
struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
328-
struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
328+
struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
329329
struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
330330
int ret;
331331

@@ -348,7 +348,7 @@ static int skcipher_do_fallback(struct skcipher_request *req, bool is_enc)
348348
static inline int cpt_enc_dec(struct skcipher_request *req, u32 enc)
349349
{
350350
struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
351-
struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
351+
struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
352352
struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
353353
struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
354354
u32 enc_iv_len = crypto_skcipher_ivsize(stfm);
@@ -537,8 +537,9 @@ static int otx2_cpt_enc_dec_init(struct crypto_skcipher *stfm)
537537
* allocated since the cryptd daemon uses
538538
* this memory for request_ctx information
539539
*/
540-
crypto_skcipher_set_reqsize(stfm, sizeof(struct otx2_cpt_req_ctx) +
541-
sizeof(struct skcipher_request));
540+
crypto_skcipher_set_reqsize_dma(
541+
stfm, sizeof(struct otx2_cpt_req_ctx) +
542+
sizeof(struct skcipher_request));
542543

543544
return cpt_skcipher_fallback_init(ctx, alg);
544545
}
@@ -572,7 +573,7 @@ static int cpt_aead_fallback_init(struct otx2_cpt_aead_ctx *ctx,
572573

573574
static int cpt_aead_init(struct crypto_aead *atfm, u8 cipher_type, u8 mac_type)
574575
{
575-
struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(atfm);
576+
struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(atfm);
576577
struct crypto_tfm *tfm = crypto_aead_tfm(atfm);
577578
struct crypto_alg *alg = tfm->__crt_alg;
578579

@@ -629,7 +630,7 @@ static int cpt_aead_init(struct crypto_aead *atfm, u8 cipher_type, u8 mac_type)
629630
ctx->enc_align_len = 1;
630631
break;
631632
}
632-
crypto_aead_set_reqsize(atfm, sizeof(struct otx2_cpt_req_ctx));
633+
crypto_aead_set_reqsize_dma(atfm, sizeof(struct otx2_cpt_req_ctx));
633634

634635
return cpt_aead_fallback_init(ctx, alg);
635636
}
@@ -681,7 +682,7 @@ static int otx2_cpt_aead_gcm_aes_init(struct crypto_aead *tfm)
681682

682683
static void otx2_cpt_aead_exit(struct crypto_aead *tfm)
683684
{
684-
struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
685+
struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
685686

686687
kfree(ctx->ipad);
687688
kfree(ctx->opad);
@@ -698,7 +699,7 @@ static void otx2_cpt_aead_exit(struct crypto_aead *tfm)
698699
static int otx2_cpt_aead_gcm_set_authsize(struct crypto_aead *tfm,
699700
unsigned int authsize)
700701
{
701-
struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
702+
struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
702703

703704
if (crypto_rfc4106_check_authsize(authsize))
704705
return -EINVAL;
@@ -722,7 +723,7 @@ static int otx2_cpt_aead_set_authsize(struct crypto_aead *tfm,
722723
static int otx2_cpt_aead_null_set_authsize(struct crypto_aead *tfm,
723724
unsigned int authsize)
724725
{
725-
struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
726+
struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
726727

727728
ctx->is_trunc_hmac = true;
728729
tfm->authsize = authsize;
@@ -794,7 +795,7 @@ static int copy_pad(u8 mac_type, u8 *out_pad, u8 *in_pad)
794795

795796
static int aead_hmac_init(struct crypto_aead *cipher)
796797
{
797-
struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
798+
struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher);
798799
int state_size = crypto_shash_statesize(ctx->hashalg);
799800
int ds = crypto_shash_digestsize(ctx->hashalg);
800801
int bs = crypto_shash_blocksize(ctx->hashalg);
@@ -892,7 +893,7 @@ static int otx2_cpt_aead_cbc_aes_sha_setkey(struct crypto_aead *cipher,
892893
const unsigned char *key,
893894
unsigned int keylen)
894895
{
895-
struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
896+
struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher);
896897
struct crypto_authenc_key_param *param;
897898
int enckeylen = 0, authkeylen = 0;
898899
struct rtattr *rta = (void *)key;
@@ -944,7 +945,7 @@ static int otx2_cpt_aead_ecb_null_sha_setkey(struct crypto_aead *cipher,
944945
const unsigned char *key,
945946
unsigned int keylen)
946947
{
947-
struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
948+
struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher);
948949
struct crypto_authenc_key_param *param;
949950
struct rtattr *rta = (void *)key;
950951
int enckeylen = 0;
@@ -979,7 +980,7 @@ static int otx2_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher,
979980
const unsigned char *key,
980981
unsigned int keylen)
981982
{
982-
struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
983+
struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher);
983984

984985
/*
985986
* For aes gcm we expect to get encryption key (16, 24, 32 bytes)
@@ -1012,9 +1013,9 @@ static int otx2_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher,
10121013
static inline int create_aead_ctx_hdr(struct aead_request *req, u32 enc,
10131014
u32 *argcnt)
10141015
{
1015-
struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1016+
struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
10161017
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1017-
struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
1018+
struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
10181019
struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
10191020
struct otx2_cpt_fc_ctx *fctx = &rctx->fctx;
10201021
int mac_len = crypto_aead_authsize(tfm);
@@ -1103,9 +1104,9 @@ static inline int create_aead_ctx_hdr(struct aead_request *req, u32 enc,
11031104
static inline void create_hmac_ctx_hdr(struct aead_request *req, u32 *argcnt,
11041105
u32 enc)
11051106
{
1106-
struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1107+
struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
11071108
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1108-
struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
1109+
struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
11091110
struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
11101111

11111112
req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG;
@@ -1127,7 +1128,7 @@ static inline void create_hmac_ctx_hdr(struct aead_request *req, u32 *argcnt,
11271128

11281129
static inline int create_aead_input_list(struct aead_request *req, u32 enc)
11291130
{
1130-
struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1131+
struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
11311132
struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
11321133
u32 inputlen = req->cryptlen + req->assoclen;
11331134
u32 status, argcnt = 0;
@@ -1144,7 +1145,7 @@ static inline int create_aead_input_list(struct aead_request *req, u32 enc)
11441145
static inline void create_aead_output_list(struct aead_request *req, u32 enc,
11451146
u32 mac_len)
11461147
{
1147-
struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1148+
struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
11481149
struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
11491150
u32 argcnt = 0, outputlen = 0;
11501151

@@ -1160,7 +1161,7 @@ static inline void create_aead_output_list(struct aead_request *req, u32 enc,
11601161
static inline void create_aead_null_input_list(struct aead_request *req,
11611162
u32 enc, u32 mac_len)
11621163
{
1163-
struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1164+
struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
11641165
struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
11651166
u32 inputlen, argcnt = 0;
11661167

@@ -1177,7 +1178,7 @@ static inline void create_aead_null_input_list(struct aead_request *req,
11771178
static inline int create_aead_null_output_list(struct aead_request *req,
11781179
u32 enc, u32 mac_len)
11791180
{
1180-
struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1181+
struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
11811182
struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
11821183
struct scatterlist *dst;
11831184
u8 *ptr = NULL;
@@ -1257,9 +1258,9 @@ static inline int create_aead_null_output_list(struct aead_request *req,
12571258

12581259
static int aead_do_fallback(struct aead_request *req, bool is_enc)
12591260
{
1260-
struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1261+
struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
12611262
struct crypto_aead *aead = crypto_aead_reqtfm(req);
1262-
struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(aead);
1263+
struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(aead);
12631264
int ret;
12641265

12651266
if (ctx->fbk_cipher) {
@@ -1281,10 +1282,10 @@ static int aead_do_fallback(struct aead_request *req, bool is_enc)
12811282

12821283
static int cpt_aead_enc_dec(struct aead_request *req, u8 reg_type, u8 enc)
12831284
{
1284-
struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1285+
struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
12851286
struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
12861287
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1287-
struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
1288+
struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
12881289
struct pci_dev *pdev;
12891290
int status, cpu_num;
12901291

@@ -1458,7 +1459,7 @@ static struct aead_alg otx2_cpt_aeads[] = { {
14581459
.cra_driver_name = "cpt_hmac_sha1_cbc_aes",
14591460
.cra_blocksize = AES_BLOCK_SIZE,
14601461
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1461-
.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1462+
.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
14621463
.cra_priority = 4001,
14631464
.cra_alignmask = 0,
14641465
.cra_module = THIS_MODULE,
@@ -1477,7 +1478,7 @@ static struct aead_alg otx2_cpt_aeads[] = { {
14771478
.cra_driver_name = "cpt_hmac_sha256_cbc_aes",
14781479
.cra_blocksize = AES_BLOCK_SIZE,
14791480
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1480-
.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1481+
.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
14811482
.cra_priority = 4001,
14821483
.cra_alignmask = 0,
14831484
.cra_module = THIS_MODULE,
@@ -1496,7 +1497,7 @@ static struct aead_alg otx2_cpt_aeads[] = { {
14961497
.cra_driver_name = "cpt_hmac_sha384_cbc_aes",
14971498
.cra_blocksize = AES_BLOCK_SIZE,
14981499
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1499-
.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1500+
.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
15001501
.cra_priority = 4001,
15011502
.cra_alignmask = 0,
15021503
.cra_module = THIS_MODULE,
@@ -1515,7 +1516,7 @@ static struct aead_alg otx2_cpt_aeads[] = { {
15151516
.cra_driver_name = "cpt_hmac_sha512_cbc_aes",
15161517
.cra_blocksize = AES_BLOCK_SIZE,
15171518
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1518-
.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1519+
.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
15191520
.cra_priority = 4001,
15201521
.cra_alignmask = 0,
15211522
.cra_module = THIS_MODULE,
@@ -1534,7 +1535,7 @@ static struct aead_alg otx2_cpt_aeads[] = { {
15341535
.cra_driver_name = "cpt_hmac_sha1_ecb_null",
15351536
.cra_blocksize = 1,
15361537
.cra_flags = CRYPTO_ALG_ASYNC,
1537-
.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1538+
.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
15381539
.cra_priority = 4001,
15391540
.cra_alignmask = 0,
15401541
.cra_module = THIS_MODULE,
@@ -1553,7 +1554,7 @@ static struct aead_alg otx2_cpt_aeads[] = { {
15531554
.cra_driver_name = "cpt_hmac_sha256_ecb_null",
15541555
.cra_blocksize = 1,
15551556
.cra_flags = CRYPTO_ALG_ASYNC,
1556-
.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1557+
.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
15571558
.cra_priority = 4001,
15581559
.cra_alignmask = 0,
15591560
.cra_module = THIS_MODULE,
@@ -1572,7 +1573,7 @@ static struct aead_alg otx2_cpt_aeads[] = { {
15721573
.cra_driver_name = "cpt_hmac_sha384_ecb_null",
15731574
.cra_blocksize = 1,
15741575
.cra_flags = CRYPTO_ALG_ASYNC,
1575-
.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1576+
.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
15761577
.cra_priority = 4001,
15771578
.cra_alignmask = 0,
15781579
.cra_module = THIS_MODULE,
@@ -1591,7 +1592,7 @@ static struct aead_alg otx2_cpt_aeads[] = { {
15911592
.cra_driver_name = "cpt_hmac_sha512_ecb_null",
15921593
.cra_blocksize = 1,
15931594
.cra_flags = CRYPTO_ALG_ASYNC,
1594-
.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1595+
.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
15951596
.cra_priority = 4001,
15961597
.cra_alignmask = 0,
15971598
.cra_module = THIS_MODULE,
@@ -1610,7 +1611,7 @@ static struct aead_alg otx2_cpt_aeads[] = { {
16101611
.cra_driver_name = "cpt_rfc4106_gcm_aes",
16111612
.cra_blocksize = 1,
16121613
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1613-
.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1614+
.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
16141615
.cra_priority = 4001,
16151616
.cra_alignmask = 0,
16161617
.cra_module = THIS_MODULE,

0 commit comments

Comments
 (0)