Skip to content

Commit 34fcc28

Browse files
smuellerDDsmuellerDD
committed
Linux kernel: add RVV and RISCV64 ASM ML-DSA support
Signed-off-by: Stephan Mueller <smueller@chronox.de>
1 parent 6138d02 commit 34fcc28

File tree

88 files changed

+218
-6
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

88 files changed

+218
-6
lines changed

‎CHANGES.md‎

Lines changed: 2 additions & 0 deletions

‎TODO‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,6 @@
1919

2020
- RISCV64: update internal/src/cpufeatures_riscv_auxv.c once kernels report ZBB correctly
2121

22-
- RISCV64 ASM / ZBB / ML-DSA: add to Linux kernel
23-
2422
- X.509/PKCS7 parser: add to the Linux kernel
2523

2624
- SHA2 Neon: check whether it works on ARMv7

‎internal/api/ext_headers_riscv.h‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/*
2+
* Copyright (C) 2023 - 2024, Stephan Mueller <smueller@chronox.de>
3+
*
4+
* License: see LICENSE file in root directory
5+
*
6+
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
7+
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
8+
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
9+
* WHICH ARE HEREBY DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE
10+
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
11+
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
12+
* OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
13+
* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
14+
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
15+
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
16+
* USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
17+
* DAMAGE.
18+
*/
19+
20+
#ifndef EXT_HEADERS_RISCV_H
21+
#define EXT_HEADERS_RISCV_H
22+
23+
#ifdef LINUX_KERNEL
24+
25+
#include <asm/vector.h>
26+
#include <asm/simd.h>
27+
28+
#define LC_VECTOR_ENABLE kernel_vector_begin()
29+
#define LC_VECTOR_DISABLE kernel_vector_end()
30+
31+
#else /* LINUX_KERNEL */
32+
33+
#define LC_VECTOR_ENABLE
34+
#define LC_VECTOR_DISABLE
35+
36+
#endif /* LINUX_KERNEL */
37+
38+
#endif /* EXT_HEADERS_RISCV_H */

‎internal/src/cpufeatures_riscv_native.c‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
#include "cpufeatures.h"
2121
#include "ext_headers.h"
22+
#include "ext_headers_riscv.h"
2223
#include "visibility.h"
2324

2425
static enum lc_cpu_features features = LC_CPU_FEATURE_UNSET;
@@ -39,7 +40,8 @@ LC_INTERFACE_FUNCTION(enum lc_cpu_features, lc_cpu_feature_available, void)
3940
#ifdef LINUX_KERNEL
4041
if (riscv_isa_extension_available(NULL, ZBB))
4142
features |= LC_CPU_FEATURE_RISCV_ASM_ZBB;
42-
if (riscv_isa_extension_available(NULL, ZVBB))
43+
if (riscv_isa_extension_available(NULL, ZVE64D) &&
44+
may_use_simd())
4345
features |= LC_CPU_FEATURE_RISCV_ASM_RVV;
4446
#endif
4547
}

‎internal/src/status.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,7 @@ LC_INTERFACE_FUNCTION(void, lc_status, char *outbuf, size_t outlen)
185185
"",
186186
armv7, armv8, riscv64,
187187
(lc_cpu_feature_available() & LC_CPU_FEATURE_RISCV_ASM_RVV) ?
188-
"RISCV64 RVV " :
188+
"RISCV64-RVV " :
189189
"",
190190

191191
/* Curve25519 */

‎linux_kernel/Kbuild.ml-dsa‎

Lines changed: 78 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,17 +46,22 @@ leancrypto-$(CONFIG_LEANCRYPTO_DILITHIUM_44) \
4646
dilithium44/dilithium_poly.o \
4747
dilithium44/dilithium_rounding.o \
4848
dilithium44/dilithium_selftest.o \
49-
dilithium44/dilithium_signature_api_c.o \
5049
dilithium44/dilithium_signature_c.o \
5150
dilithium44/dilithium_signature_helper.o
5251
CFLAGS_dilithium44/dilithium_ntt.o := -DLC_DILITHIUM_TYPE_44
5352
CFLAGS_dilithium44/dilithium_poly.o := -DLC_DILITHIUM_TYPE_44
5453
CFLAGS_dilithium44/dilithium_rounding.o := -DLC_DILITHIUM_TYPE_44
5554
CFLAGS_dilithium44/dilithium_selftest.o := -DLC_DILITHIUM_TYPE_44
56-
CFLAGS_dilithium44/dilithium_signature_api_c.o := -DLC_DILITHIUM_TYPE_44
5755
CFLAGS_dilithium44/dilithium_signature_c.o := -DLC_DILITHIUM_TYPE_44
5856
CFLAGS_dilithium44/dilithium_signature_helper.o := -DLC_DILITHIUM_TYPE_44
5957

58+
# RISCV64 has its own Dilithium 44 API
59+
ifeq ($(and $(CONFIG_RISCV),$(CONFIG_64BIT)),)
60+
leancrypto-$(CONFIG_LEANCRYPTO_DILITHIUM_44) \
61+
+= dilithium44/dilithium_signature_api_c.o
62+
CFLAGS_dilithium44/dilithium_signature_api_c.o := -DLC_DILITHIUM_TYPE_44
63+
endif
64+
6065
# Dilithium-87-ED25519 implementation
6166
leancrypto-$(CONFIG_LEANCRYPTO_DILITHIUM_ED25519) \
6267
+= leancrypto_kernel_dilithium_ed25519.o
@@ -189,6 +194,77 @@ AFLAGS_dilithium65/armv8/dilithium_poly_armv8.o := -DLC_DILITHIUM_TYPE_65
189194
CFLAGS_dilithium65/armv8/dilithium_signature_api_armv8.o:= -DLC_DILITHIUM_TYPE_65
190195
CFLAGS_dilithium65/armv8/dilithium_signature_armv8.o := -DLC_DILITHIUM_TYPE_65
191196

197+
else ifneq ($(and $(CONFIG_RISCV),$(CONFIG_64BIT)),)
198+
################################################################################
199+
# RISCV64 assembler and RVV implementations
200+
################################################################################
201+
# Dilithium Common
202+
leancrypto-$(CONFIG_LEANCRYPTO_DILITHIUM) \
203+
+= ../ml-dsa/src/riscv64/dilithium_consts_rvv.o \
204+
../ml-dsa/src/riscv64/dilithium_zetas_riscv64.o
205+
206+
# Dilithium 87 RISCV64 ASM
207+
leancrypto-$(CONFIG_LEANCRYPTO_DILITHIUM_87) \
208+
+= ../ml-dsa/src/riscv64/dilithium_signature_api_riscv64.o\
209+
../ml-dsa/src/riscv64/dilithium_signature_riscv64.o \
210+
../ml-dsa/src/riscv64/ntt_8l_dualissue_plant_rv64im.o
211+
AFLAGS../ml-dsa/src/riscv64/ntt_8l_dualissue_plant_rv64im.o := -march=rv64gc \
212+
-mcmodel=medany
213+
214+
# Dilithium 65 RISCV64 ASM
215+
leancrypto-$(CONFIG_LEANCRYPTO_DILITHIUM_65) \
216+
+= dilithium65/riscv64/dilithium_signature_api_riscv64.o\
217+
dilithium65/riscv64/dilithium_signature_riscv64.o \
218+
dilithium65/riscv64/ntt_8l_dualissue_plant_rv64im.o
219+
CFLAGS_dilithium65/riscv64/dilithium_signature_api_riscv64.o := -DLC_DILITHIUM_TYPE_65
220+
CFLAGS_dilithium65/riscv64/dilithium_signature_riscv64.o := -DLC_DILITHIUM_TYPE_65
221+
AFLAGS_dilithium65/riscv64/ntt_8l_dualissue_plant_rv64im.o := -march=rv64gc \
222+
-mcmodel=medany \
223+
-DLC_DILITHIUM_TYPE_65
224+
225+
# Dilithium 44 RISCV64 ASM
226+
leancrypto-$(CONFIG_LEANCRYPTO_DILITHIUM_44) \
227+
+= dilithium44/riscv64/dilithium_signature_api_riscv64.o\
228+
dilithium44/riscv64/dilithium_signature_riscv64.o \
229+
dilithium44/riscv64/ntt_8l_dualissue_plant_rv64im.o
230+
CFLAGS_dilithium44/riscv64/dilithium_signature_api_riscv64.o := -DLC_DILITHIUM_TYPE_44
231+
CFLAGS_dilithium44/riscv64/dilithium_signature_riscv64.o := -DLC_DILITHIUM_TYPE_44
232+
AFLAGS_dilithium44/riscv64/ntt_8l_dualissue_plant_rv64im.o := -march=rv64gc \
233+
-mcmodel=medany \
234+
-DLC_DILITHIUM_TYPE_44
235+
236+
# Dilithium 87 RISCV64 RVV
237+
leancrypto-$(CONFIG_LEANCRYPTO_DILITHIUM_87) \
238+
+= dilithium87/riscv64_rvv/dilithium_signature_riscv64_rvv.o\
239+
dilithium87/riscv64_rvv/ntt_rvv.o
240+
CFLAGS_dilithium87/riscv64_rvv/dilithium_signature_riscv64_rvv.o := \
241+
-march=rv64imadcv -mcmodel=medany \
242+
-DLC_DILITHIUM_RISCV64_RVV
243+
AFLAGS_dilithium87/riscv64_rvv/ntt_rvv.o := -march=rv64imadcv -mcmodel=medany \
244+
-DLC_DILITHIUM_RISCV64_RVV
245+
246+
# Dilithium 65 RISCV64 RVV
247+
leancrypto-$(CONFIG_LEANCRYPTO_DILITHIUM_65) \
248+
+= dilithium65/riscv64_rvv/dilithium_signature_riscv64_rvv.o\
249+
dilithium65/riscv64_rvv/ntt_rvv.o
250+
CFLAGS_dilithium65/riscv64_rvv/dilithium_signature_riscv64_rvv.o := \
251+
-march=rv64imadcv -mcmodel=medany -DLC_DILITHIUM_TYPE_65\
252+
-DLC_DILITHIUM_RISCV64_RVV
253+
AFLAGS_dilithium65/riscv64_rvv/ntt_rvv.o := -march=rv64imadcv -mcmodel=medany \
254+
-DLC_DILITHIUM_TYPE_65 \
255+
-DLC_DILITHIUM_RISCV64_RVV
256+
257+
# Dilithium 44 RISCV64 RVV
258+
leancrypto-$(CONFIG_LEANCRYPTO_DILITHIUM_44) \
259+
+= dilithium44/riscv64_rvv/dilithium_signature_riscv64_rvv.o\
260+
dilithium44/riscv64_rvv/ntt_rvv.o
261+
CFLAGS_dilithium44/riscv64_rvv/dilithium_signature_riscv64_rvv.o := \
262+
-march=rv64imadcv -mcmodel=medany -DLC_DILITHIUM_TYPE_44\
263+
-DLC_DILITHIUM_RISCV64_RVV
264+
AFLAGS_dilithium44/riscv64_rvv/ntt_rvv.o := -march=rv64imadcv -mcmodel=medany \
265+
-DLC_DILITHIUM_TYPE_44 \
266+
-DLC_DILITHIUM_RISCV64_RVV
267+
192268

193269
################################################################################
194270
# Interfaces for any non-accelerated implementation

‎linux_kernel/dilithium44/riscv64/dilithium_consts_rvv.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../../../ml-dsa/src/riscv64/dilithium_consts_rvv.c

‎linux_kernel/dilithium44/riscv64/dilithium_consts_rvv.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../../../ml-dsa/src/riscv64/dilithium_consts_rvv.h

‎linux_kernel/dilithium44/riscv64/dilithium_ntt_rv64im.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../../../ml-dsa/src/riscv64/dilithium_ntt_rv64im.h

‎linux_kernel/dilithium44/riscv64/dilithium_ntt_rvv.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../../../ml-dsa/src/riscv64/dilithium_ntt_rvv.h

0 commit comments

Comments
 (0)