Skip to content

Commit 600a385

Browse files
JerryShihpalmer-dabbelt
JerryShih
authored and
palmer-dabbelt
committed
crypto: riscv - add vector crypto accelerated GHASH
Add an implementation of GHASH using the zvkg extension. The assembly code is derived from OpenSSL code (openssl/openssl#21923) that was dual-licensed so that it could be reused in the kernel. Nevertheless, the assembly has been significantly reworked for integration with the kernel, for example by using a regular .S file instead of the so-called perlasm, using the assembler instead of bare '.inst', reducing code duplication, and eliminating unnecessary endianness conversions. Co-developed-by: Christoph Müllner <[email protected]> Signed-off-by: Christoph Müllner <[email protected]> Co-developed-by: Heiko Stuebner <[email protected]> Signed-off-by: Heiko Stuebner <[email protected]> Signed-off-by: Jerry Shih <[email protected]> Co-developed-by: Eric Biggers <[email protected]> Signed-off-by: Eric Biggers <[email protected]> Link: https://lore.kernel.org/r/[email protected] Signed-off-by: Palmer Dabbelt <[email protected]>
1 parent bb54668 commit 600a385

File tree

4 files changed

+253
-0
lines changed

4 files changed

+253
-0
lines changed

‎arch/riscv/crypto/Kconfig

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,4 +29,14 @@ config CRYPTO_CHACHA_RISCV64
2929
Architecture: riscv64 using:
3030
- Zvkb vector crypto extension
3131

32+
config CRYPTO_GHASH_RISCV64
33+
tristate "Hash functions: GHASH"
34+
depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
35+
select CRYPTO_GCM
36+
help
37+
GCM GHASH function (NIST SP 800-38D)
38+
39+
Architecture: riscv64 using:
40+
- Zvkg vector crypto extension
41+
3242
endmenu

‎arch/riscv/crypto/Makefile

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,6 @@ aes-riscv64-y := aes-riscv64-glue.o aes-riscv64-zvkned.o \
66

77
obj-$(CONFIG_CRYPTO_CHACHA_RISCV64) += chacha-riscv64.o
88
chacha-riscv64-y := chacha-riscv64-glue.o chacha-riscv64-zvkb.o
9+
10+
obj-$(CONFIG_CRYPTO_GHASH_RISCV64) += ghash-riscv64.o
11+
ghash-riscv64-y := ghash-riscv64-glue.o ghash-riscv64-zvkg.o

‎arch/riscv/crypto/ghash-riscv64-glue.c

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
// SPDX-License-Identifier: GPL-2.0-only
2+
/*
3+
* GHASH using the RISC-V vector crypto extensions
4+
*
5+
* Copyright (C) 2023 VRULL GmbH
6+
* Author: Heiko Stuebner <[email protected]>
7+
*
8+
* Copyright (C) 2023 SiFive, Inc.
9+
* Author: Jerry Shih <[email protected]>
10+
*/
11+
12+
#include <asm/simd.h>
13+
#include <asm/vector.h>
14+
#include <crypto/ghash.h>
15+
#include <crypto/internal/hash.h>
16+
#include <crypto/internal/simd.h>
17+
#include <linux/linkage.h>
18+
#include <linux/module.h>
19+
20+
asmlinkage void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data,
21+
size_t len);
22+
23+
struct riscv64_ghash_tfm_ctx {
24+
be128 key;
25+
};
26+
27+
struct riscv64_ghash_desc_ctx {
28+
be128 accumulator;
29+
u8 buffer[GHASH_BLOCK_SIZE];
30+
u32 bytes;
31+
};
32+
33+
static int riscv64_ghash_setkey(struct crypto_shash *tfm, const u8 *key,
34+
unsigned int keylen)
35+
{
36+
struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(tfm);
37+
38+
if (keylen != GHASH_BLOCK_SIZE)
39+
return -EINVAL;
40+
41+
memcpy(&tctx->key, key, GHASH_BLOCK_SIZE);
42+
43+
return 0;
44+
}
45+
46+
static int riscv64_ghash_init(struct shash_desc *desc)
47+
{
48+
struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
49+
50+
*dctx = (struct riscv64_ghash_desc_ctx){};
51+
52+
return 0;
53+
}
54+
55+
static inline void
56+
riscv64_ghash_blocks(const struct riscv64_ghash_tfm_ctx *tctx,
57+
struct riscv64_ghash_desc_ctx *dctx,
58+
const u8 *src, size_t srclen)
59+
{
60+
/* The srclen is nonzero and a multiple of 16. */
61+
if (crypto_simd_usable()) {
62+
kernel_vector_begin();
63+
ghash_zvkg(&dctx->accumulator, &tctx->key, src, srclen);
64+
kernel_vector_end();
65+
} else {
66+
do {
67+
crypto_xor((u8 *)&dctx->accumulator, src,
68+
GHASH_BLOCK_SIZE);
69+
gf128mul_lle(&dctx->accumulator, &tctx->key);
70+
src += GHASH_BLOCK_SIZE;
71+
srclen -= GHASH_BLOCK_SIZE;
72+
} while (srclen);
73+
}
74+
}
75+
76+
static int riscv64_ghash_update(struct shash_desc *desc, const u8 *src,
77+
unsigned int srclen)
78+
{
79+
const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
80+
struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
81+
unsigned int len;
82+
83+
if (dctx->bytes) {
84+
if (dctx->bytes + srclen < GHASH_BLOCK_SIZE) {
85+
memcpy(dctx->buffer + dctx->bytes, src, srclen);
86+
dctx->bytes += srclen;
87+
return 0;
88+
}
89+
memcpy(dctx->buffer + dctx->bytes, src,
90+
GHASH_BLOCK_SIZE - dctx->bytes);
91+
riscv64_ghash_blocks(tctx, dctx, dctx->buffer,
92+
GHASH_BLOCK_SIZE);
93+
src += GHASH_BLOCK_SIZE - dctx->bytes;
94+
srclen -= GHASH_BLOCK_SIZE - dctx->bytes;
95+
dctx->bytes = 0;
96+
}
97+
98+
len = round_down(srclen, GHASH_BLOCK_SIZE);
99+
if (len) {
100+
riscv64_ghash_blocks(tctx, dctx, src, len);
101+
src += len;
102+
srclen -= len;
103+
}
104+
105+
if (srclen) {
106+
memcpy(dctx->buffer, src, srclen);
107+
dctx->bytes = srclen;
108+
}
109+
110+
return 0;
111+
}
112+
113+
static int riscv64_ghash_final(struct shash_desc *desc, u8 *out)
114+
{
115+
const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
116+
struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
117+
int i;
118+
119+
if (dctx->bytes) {
120+
for (i = dctx->bytes; i < GHASH_BLOCK_SIZE; i++)
121+
dctx->buffer[i] = 0;
122+
123+
riscv64_ghash_blocks(tctx, dctx, dctx->buffer,
124+
GHASH_BLOCK_SIZE);
125+
}
126+
127+
memcpy(out, &dctx->accumulator, GHASH_DIGEST_SIZE);
128+
return 0;
129+
}
130+
131+
static struct shash_alg riscv64_ghash_alg = {
132+
.init = riscv64_ghash_init,
133+
.update = riscv64_ghash_update,
134+
.final = riscv64_ghash_final,
135+
.setkey = riscv64_ghash_setkey,
136+
.descsize = sizeof(struct riscv64_ghash_desc_ctx),
137+
.digestsize = GHASH_DIGEST_SIZE,
138+
.base = {
139+
.cra_blocksize = GHASH_BLOCK_SIZE,
140+
.cra_ctxsize = sizeof(struct riscv64_ghash_tfm_ctx),
141+
.cra_priority = 300,
142+
.cra_name = "ghash",
143+
.cra_driver_name = "ghash-riscv64-zvkg",
144+
.cra_module = THIS_MODULE,
145+
},
146+
};
147+
148+
static int __init riscv64_ghash_mod_init(void)
149+
{
150+
if (riscv_isa_extension_available(NULL, ZVKG) &&
151+
riscv_vector_vlen() >= 128)
152+
return crypto_register_shash(&riscv64_ghash_alg);
153+
154+
return -ENODEV;
155+
}
156+
157+
static void __exit riscv64_ghash_mod_exit(void)
158+
{
159+
crypto_unregister_shash(&riscv64_ghash_alg);
160+
}
161+
162+
module_init(riscv64_ghash_mod_init);
163+
module_exit(riscv64_ghash_mod_exit);
164+
165+
MODULE_DESCRIPTION("GHASH (RISC-V accelerated)");
166+
MODULE_AUTHOR("Heiko Stuebner <[email protected]>");
167+
MODULE_LICENSE("GPL");
168+
MODULE_ALIAS_CRYPTO("ghash");

‎arch/riscv/crypto/ghash-riscv64-zvkg.S

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */
2+
//
3+
// This file is dual-licensed, meaning that you can use it under your
4+
// choice of either of the following two licenses:
5+
//
6+
// Copyright 2023 The OpenSSL Project Authors. All Rights Reserved.
7+
//
8+
// Licensed under the Apache License 2.0 (the "License"). You can obtain
9+
// a copy in the file LICENSE in the source distribution or at
10+
// https://www.openssl.org/source/license.html
11+
//
12+
// or
13+
//
14+
// Copyright (c) 2023, Christoph Müllner <[email protected]>
15+
// Copyright (c) 2023, Jerry Shih <[email protected]>
16+
// Copyright 2024 Google LLC
17+
// All rights reserved.
18+
//
19+
// Redistribution and use in source and binary forms, with or without
20+
// modification, are permitted provided that the following conditions
21+
// are met:
22+
// 1. Redistributions of source code must retain the above copyright
23+
// notice, this list of conditions and the following disclaimer.
24+
// 2. Redistributions in binary form must reproduce the above copyright
25+
// notice, this list of conditions and the following disclaimer in the
26+
// documentation and/or other materials provided with the distribution.
27+
//
28+
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
29+
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30+
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
31+
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
32+
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
33+
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
34+
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
35+
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
36+
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
37+
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
38+
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
39+
40+
// The generated code of this file depends on the following RISC-V extensions:
41+
// - RV64I
42+
// - RISC-V Vector ('V') with VLEN >= 128
43+
// - RISC-V Vector GCM/GMAC extension ('Zvkg')
44+
45+
#include <linux/linkage.h>
46+
47+
.text
48+
.option arch, +zvkg
49+
50+
#define ACCUMULATOR a0
51+
#define KEY a1
52+
#define DATA a2
53+
#define LEN a3
54+
55+
// void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data,
56+
// size_t len);
57+
//
58+
// |len| must be nonzero and a multiple of 16 (GHASH_BLOCK_SIZE).
59+
SYM_FUNC_START(ghash_zvkg)
60+
vsetivli zero, 4, e32, m1, ta, ma
61+
vle32.v v1, (ACCUMULATOR)
62+
vle32.v v2, (KEY)
63+
.Lnext_block:
64+
vle32.v v3, (DATA)
65+
vghsh.vv v1, v2, v3
66+
addi DATA, DATA, 16
67+
addi LEN, LEN, -16
68+
bnez LEN, .Lnext_block
69+
70+
vse32.v v1, (ACCUMULATOR)
71+
ret
72+
SYM_FUNC_END(ghash_zvkg)

0 commit comments

Comments
 (0)