Skip to content

Commit e2e519a

Browse files
smuellerDDsmuellerDD
committed
RISCV64: Add RVV ML-DSA support
Signed-off-by: Stephan Mueller <smueller@chronox.de>
1 parent 4ba263a commit e2e519a

21 files changed

+3058
-7
lines changed

‎CHANGES.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,10 @@ Changes 1.2.0
2121

2222
* leancrypto passes X.509 IETF-Hackathon tests: https://ietf-hackathon.github.io/pqc-certificates/pqc_hackathon_results_certs_r4_automated_tests.html
2323

24+
* Add compilation support for (U)EFI environment
25+
26+
* RISCV64 RVV: ML-DSA - add assembler implementation using RVV support
27+
2428
Changes 1.1.0
2529
* ML-KEM remove modulus check of decapsulation key (not required by FIPS 203)
2630

‎LICENSE‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -369,6 +369,10 @@ ml-dsa/src/riscv64/dilithium_ntt_rv64im.h
369369
ml-dsa/src/riscv64/ntt_8l_dualissue_plant_rv64im.S
370370
ml-dsa/src/riscv64/dilithium_poly_riscv64.h
371371
ml-dsa/src/riscv64/dilithium_polyvec_riscv64.h
372+
ml-dsa/src/riscv64/dilithium_consts_rvv.c
373+
ml-dsa/src/riscv64/dilithium_consts_rvv.h
374+
ml-dsa/src/riscv64/dilithium_ntt_rvv.h
375+
ml-dsa/src/riscv64/ntt_rvv.S
372376

373377
The MIT license, the text of which is below, applies to PQRV in general.
374378

‎meson.build‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -503,6 +503,18 @@ if (not get_option('disable-asm'))
503503
'-mcmodel=medany'
504504
]
505505

506+
cc_riscv64_asm_rvv_args = [
507+
'-march=rv64imadcv',
508+
'-mabi=lp64d',
509+
'-mcmodel=medany'
510+
]
511+
512+
cc_riscv64_asm_rvv_zbb_args = [
513+
'-march=rv64imadcv_zba_zbb',
514+
'-mabi=lp64d',
515+
'-mcmodel=medany'
516+
]
517+
506518
# elif (host_machine.cpu_family() == 'riscv' and
507519
# cc.get_id() == 'gcc')
508520
#

‎ml-dsa/api/dilithium_type.h‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -263,7 +263,7 @@ extern "C" {
263263
#define lc_dilithium_verify_update_armv7 DILITHIUM_F(verify_update_armv7)
264264
#define lc_dilithium_verify_final_armv7 DILITHIUM_F(verify_final_armv7)
265265

266-
/* RISCV 64 Implementation */
266+
/* RISCV 64 ASM Implementation */
267267
#define lc_dilithium_keypair_riscv64 DILITHIUM_F(keypair_riscv64)
268268
#define lc_dilithium_keypair_from_seed_riscv64 \
269269
DILITHIUM_F(keypair_from_seed_riscv64)
@@ -288,6 +288,28 @@ extern "C" {
288288
#define dilithium_poly_basemul_8l_rv64im DILITHIUM_F(poly_basemul_8l_rv64im)
289289
#define dilithium_poly_reduce_rv64im DILITHIUM_F(poly_reduce_rv64im)
290290

291+
/* RISCV 64 RVV Implementation */
292+
#define lc_dilithium_keypair_riscv64_rvv DILITHIUM_F(keypair_riscv64_rvv)
293+
#define lc_dilithium_keypair_from_seed_riscv64_rvv \
294+
DILITHIUM_F(keypair_from_seed_riscv64_rvv)
295+
#define lc_dilithium_sign_riscv64_rvv DILITHIUM_F(sign_riscv64_rvv)
296+
#define lc_dilithium_sign_ctx_riscv64_rvv DILITHIUM_F(sign_ctx_riscv64_rvv)
297+
#define lc_dilithium_sign_init_riscv64_rvv DILITHIUM_F(sign_init_riscv64_rvv)
298+
#define lc_dilithium_sign_update_riscv64_rvv DILITHIUM_F(sign_update_riscv64_rvv)
299+
#define lc_dilithium_sign_final_riscv64_rvv DILITHIUM_F(sign_final_riscv64_rvv)
300+
#define lc_dilithium_verify_riscv64_rvv DILITHIUM_F(verify_riscv64_rvv)
301+
#define lc_dilithium_verify_ctx_riscv64_rvv DILITHIUM_F(verify_ctx_riscv64_rvv)
302+
#define lc_dilithium_verify_init_riscv64_rvv DILITHIUM_F(verify_init_riscv64_rvv)
303+
#define lc_dilithium_verify_update_riscv64_rvv DILITHIUM_F(verify_update_riscv64_rvv)
304+
#define lc_dilithium_verify_final_riscv64_rvv DILITHIUM_F(verify_final_riscv64_rvv)
305+
#define dilithium_ntt_8l_rvv DILITHIUM_F(ntt_8l_rvv)
306+
#define dilithium_intt_8l_rvv DILITHIUM_F(intt_8l_rvv)
307+
#define dilithium_poly_basemul_8l_rvv DILITHIUM_F(poly_basemul_8l_rvv)
308+
#define dilithium_poly_basemul_acc_8l_rvv DILITHIUM_F(poly_basemul_acc_8l_rvv)
309+
#define dilithium_ntt2normal_order_8l_rvv DILITHIUM_F(ntt2normal_order_8l_rvv)
310+
#define dilithium_normal2ntt_order_8l_rvv DILITHIUM_F(normal2ntt_order_8l_rvv)
311+
#define dilithium_poly_reduce_rvv DILITHIUM_F(poly_reduce_rvv)
312+
291313
#ifdef __cplusplus
292314
}
293315
#endif

‎ml-dsa/src/riscv64/dilithium_consts_rvv.c‎

Lines changed: 372 additions & 0 deletions
Large diffs are not rendered by default.

‎ml-dsa/src/riscv64/dilithium_consts_rvv.h‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/*
2+
* Copyright (C) 2024, Stephan Mueller <smueller@chronox.de>
3+
*
4+
* License: see LICENSE file in root directory
5+
*
6+
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
7+
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
8+
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
9+
* WHICH ARE HEREBY DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE
10+
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
11+
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
12+
* OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
13+
* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
14+
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
15+
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
16+
* USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
17+
* DAMAGE.
18+
*/
19+
/*
20+
* This file is derived from https://github.com/Ji-Peng/PQRV which uses the
21+
* following license.
22+
*
23+
* The MIT license, the text of which is below, applies to PQRV in general.
24+
*
25+
* Copyright (c) 2024 Jipeng Zhang (jp-zhang@outlook.com)
26+
* SPDX-License-Identifier: MIT
27+
*
28+
* Permission is hereby granted, free of charge, to any person obtaining a copy
29+
* of this software and associated documentation files (the "Software"), to deal
30+
* in the Software without restriction, including without limitation the rights
31+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
32+
* copies of the Software, and to permit persons to whom the Software is
33+
* furnished to do so, subject to the following conditions:
34+
*
35+
* The above copyright notice and this permission notice shall be included in all
36+
* copies or substantial portions of the Software.
37+
*
38+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
39+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
40+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
41+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
42+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
43+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
44+
* SOFTWARE.
45+
*/
46+
47+
#ifndef DILITHIUM_CONSTS_RVV_H
48+
#define DILITHIUM_CONSTS_RVV_H
49+
50+
#include "dilithium_type.h"
51+
#include "ext_headers.h"
52+
53+
extern const int32_t dilithium_qdata_rvv[1296];
54+
55+
#endif /* DILITHIUM_CONSTS_RVV_H */

‎ml-dsa/src/riscv64/dilithium_ntt_rvv.h‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
/*
2+
* Copyright (C) 2024, Stephan Mueller <smueller@chronox.de>
3+
*
4+
* License: see LICENSE file in root directory
5+
*
6+
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
7+
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
8+
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
9+
* WHICH ARE HEREBY DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE
10+
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
11+
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
12+
* OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
13+
* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
14+
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
15+
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
16+
* USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
17+
* DAMAGE.
18+
*/
19+
/*
20+
* This file is derived from https://github.com/Ji-Peng/PQRV which uses the
21+
* following license.
22+
*
23+
* The MIT license, the text of which is below, applies to PQRV in general.
24+
*
25+
* Copyright (c) 2024 Jipeng Zhang (jp-zhang@outlook.com)
26+
* SPDX-License-Identifier: MIT
27+
*
28+
* Permission is hereby granted, free of charge, to any person obtaining a copy
29+
* of this software and associated documentation files (the "Software"), to deal
30+
* in the Software without restriction, including without limitation the rights
31+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
32+
* copies of the Software, and to permit persons to whom the Software is
33+
* furnished to do so, subject to the following conditions:
34+
*
35+
* The above copyright notice and this permission notice shall be included in all
36+
* copies or substantial portions of the Software.
37+
*
38+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
39+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
40+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
41+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
42+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
43+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
44+
* SOFTWARE.
45+
*/
46+
47+
#ifndef DILITHIUM_NTT_RVV_H
48+
#define DILITHIUM_NTT_RVV_H
49+
50+
#include "dilithium_consts_rvv.h"
51+
52+
extern void dilithium_ntt_8l_rvv(int32_t *r, const int32_t *qdata);
53+
extern void dilithium_intt_8l_rvv(int32_t *r, const int32_t *qdata);
54+
extern void dilithium_poly_basemul_8l_rvv(int32_t *r, const int32_t *a,
55+
const int32_t *b);
56+
extern void dilithium_poly_basemul_acc_8l_rvv(int32_t *r, const int32_t *a,
57+
const int32_t *b);
58+
extern void dilithium_ntt2normal_order_8l_rvv(int32_t *r, const int32_t *qdata);
59+
extern void dilithium_normal2ntt_order_8l_rvv(int32_t *r, const int32_t *qdata);
60+
extern void dilithium_poly_reduce_rvv(int32_t *r);
61+
62+
#endif /* DILITHIUM_NTT_RVV_H */

‎ml-dsa/src/riscv64/dilithium_poly_riscv64.h‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949

5050
#include "dilithium_type.h"
5151
#include "dilithium_ntt_rv64im.h"
52+
#include "dilithium_ntt_rvv.h"
5253
#include "dilithium_zetas_riscv64.h"
5354

5455
#ifdef __cplusplus
@@ -65,7 +66,11 @@ typedef struct {
6566

6667
static inline void poly_reduce(poly *a)
6768
{
69+
#ifdef LC_DILITHIUM_RISCV64_RVV
70+
dilithium_poly_reduce_rvv(a->coeffs);
71+
#else
6872
dilithium_poly_reduce_rv64im(a->coeffs);
73+
#endif
6974
}
7075

7176
static inline void poly_basemul_init(poly_double *r, const poly *a,
@@ -87,20 +92,37 @@ static inline void poly_basemul_acc_end(poly *r, const poly *a, const poly *b,
8792
b->coeffs, r_double->coeffs);
8893
}
8994

95+
static inline void poly_basemul_acc_rvv(poly *c, const poly *a, const poly *b)
96+
{
97+
dilithium_poly_basemul_acc_8l_rvv(c->coeffs, a->coeffs, b->coeffs);
98+
}
99+
90100
static inline void poly_pointwise_montgomery(poly *c, const poly *a,
91101
const poly *b)
92102
{
103+
#ifdef LC_DILITHIUM_RISCV64_RVV
104+
dilithium_poly_basemul_8l_rvv(c->coeffs, a->coeffs, b->coeffs);
105+
#else
93106
dilithium_poly_basemul_8l_rv64im(c->coeffs, a->coeffs, b->coeffs);
107+
#endif
94108
}
95109

96110
static inline void poly_ntt(poly *a)
97111
{
112+
#ifdef LC_DILITHIUM_RISCV64_RVV
113+
dilithium_ntt_8l_rvv(a->coeffs, dilithium_qdata_rvv);
114+
#else
98115
dilithium_ntt_8l_rv64im(a->coeffs, zetas_ntt_8l_rv64im);
116+
#endif
99117
}
100118

101119
static inline void poly_invntt_tomont(poly *a)
102120
{
121+
#ifdef LC_DILITHIUM_RISCV64_RVV
122+
dilithium_intt_8l_rvv(a->coeffs, dilithium_qdata_rvv);
123+
#else
103124
dilithium_intt_8l_rv64im(a->coeffs, zetas_intt_8l_rv64im);
125+
#endif
104126
}
105127

106128
/**

‎ml-dsa/src/riscv64/dilithium_polyvec_riscv64.h‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,13 +102,36 @@ polyvec_matrix_expand(polyvecl mat[LC_DILITHIUM_K],
102102
unsigned int i, j;
103103

104104
for (i = 0; i < LC_DILITHIUM_K; ++i)
105-
for (j = 0; j < LC_DILITHIUM_L; ++j)
105+
for (j = 0; j < LC_DILITHIUM_L; ++j) {
106106
poly_uniform(
107107
&mat[i].vec[j], rho,
108108
le_bswap16((uint16_t)(i << 8) + (uint16_t)j),
109109
ws_buf);
110+
#ifdef LC_DILITHIUM_RISCV64_RVV
111+
dilithium_normal2ntt_order_8l_rvv(mat[i].vec[j].coeffs,
112+
dilithium_qdata_rvv);
113+
#endif
114+
}
115+
}
116+
117+
#ifdef LC_DILITHIUM_RISCV64_RVV
118+
119+
static inline void polyvecl_pointwise_acc_montgomery(poly *w, const polyvecl *u,
120+
const polyvecl *v,
121+
void *ws_buf)
122+
{
123+
unsigned int i;
124+
125+
(void)ws_buf;
126+
127+
poly_pointwise_montgomery(w, &u->vec[0], &v->vec[0]);
128+
for (i = 1; i < LC_DILITHIUM_L; ++i) {
129+
poly_basemul_acc_rvv(w, &u->vec[i], &v->vec[i]);
130+
}
110131
}
111132

133+
#else /* LC_DILITHIUM_RISCV64_RVV */
134+
112135
static inline void polyvecl_pointwise_acc_montgomery(poly *w, const polyvecl *u,
113136
const polyvecl *v,
114137
void *ws_buf)
@@ -122,8 +145,11 @@ static inline void polyvecl_pointwise_acc_montgomery(poly *w, const polyvecl *u,
122145
for (i = 1; i < LC_DILITHIUM_L - 1; ++i)
123146
poly_basemul_acc(&w_double, &u->vec[i], &v->vec[i]);
124147
poly_basemul_acc_end(w, &u->vec[i], &v->vec[i], &w_double);
148+
lc_memset_secure(&w_double, 0, sizeof(w_double));
125149
}
126150

151+
#endif /* LC_DILITHIUM_RISCV64_RVV */
152+
127153
static inline void
128154
polyvec_matrix_pointwise_montgomery(polyveck *t,
129155
const polyvecl mat[LC_DILITHIUM_K],

0 commit comments

Comments
 (0)