Skip to content

Commit 67f153b

Browse files
yinlenreeyinlenree
committed
crc: add RISC-V implementation
The CRC module of ISA-L has been accelerated using RISC-V's V, Zbc and Zvbc, instruction sets, implementing data folding and Barrett reduction optimizations. Signed-off-by: Ji Dong <[email protected]>
1 parent 73c5044 commit 67f153b

36 files changed

+3485
-16
lines changed

‎configure.ac‎

Lines changed: 72 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,9 @@ AM_CONDITIONAL([CPU_PPC64LE], [test "$CPU" = "ppc64le"])
3838
AM_CONDITIONAL([CPU_RISCV64], [test "$CPU" = "riscv64"])
3939
AM_CONDITIONAL([CPU_UNDEFINED], [test "x$CPU" = "x"])
4040
AM_CONDITIONAL([HAVE_RVV], [false])
41+
AM_CONDITIONAL([HAVE_ZBC], [false])
42+
AM_CONDITIONAL([HAVE_ZVBC], [false])
43+
AM_CONDITIONAL([HAVE_HWPROBE_H], [false])
4144

4245
# Check for programs
4346
AC_PROG_CC_STDC
@@ -57,24 +60,78 @@ case "${CPU}" in
5760

5861
riscv64)
5962

60-
AC_MSG_CHECKING([checking RVV support])
61-
AC_COMPILE_IFELSE(
62-
[AC_LANG_PROGRAM([], [
63-
__asm__ volatile(
64-
".option arch, +v\n"
65-
"vsetivli zero, 0, e8, m1, ta, ma\n"
66-
);
67-
])],
68-
[AC_DEFINE([HAVE_RVV], [1], [Enable RVV instructions])
69-
AM_CONDITIONAL([HAVE_RVV], [true]) rvv=yes],
70-
[AC_DEFINE([HAVE_RVV], [0], [Disable RVV instructions])
71-
AM_CONDITIONAL([HAVE_RVV], [false]) rvv=no]
63+
AC_CHECK_HEADER([asm/hwprobe.h],
64+
[AC_DEFINE([HAVE_HWPROBE_H], [1], [Define if asm/hwprobe.h exists])
65+
AM_CONDITIONAL([HAVE_HWPROBE_H], [true]) hwprobe_h=yes],
66+
[AC_DEFINE([HAVE_HWPROBE_H], [0], [Define if asm/hwprobe.h not exists])
67+
AM_CONDITIONAL([HAVE_HWPROBE_H], [false]) hwprobe_h=no]
7268
)
73-
if test "x$rvv" = "xyes"; then
74-
CFLAGS+=" -march=rv64gcv"
75-
CCASFLAGS+=" -march=rv64gcv"
69+
if test "x$hwprobe_h" = "xyes"; then
70+
AC_MSG_CHECKING([ZBC support])
71+
AC_COMPILE_IFELSE(
72+
[AC_LANG_PROGRAM([#include <asm/hwprobe.h>], [
73+
int a = RISCV_HWPROBE_EXT_ZBC;
74+
__asm__ volatile(
75+
".option arch, +zbc\n"
76+
"clmul zero, zero, zero\n"
77+
"clmulh zero, zero, zero\n"
78+
);
79+
])],
80+
[AC_DEFINE([HAVE_ZBC], [1], [Enable ZBC instructions])
81+
AM_CONDITIONAL([HAVE_ZBC], [true]) zbc=yes],
82+
[AC_DEFINE([HAVE_ZBC], [0], [Disable ZBC instructions])
83+
AM_CONDITIONAL([HAVE_ZBC], [false]) zbc=no]
84+
)
85+
AC_MSG_RESULT([$zbc])
86+
AC_MSG_CHECKING([ZVBC support])
87+
AC_COMPILE_IFELSE(
88+
[AC_LANG_PROGRAM([#include <asm/hwprobe.h>], [
89+
int a = RISCV_HWPROBE_EXT_ZVBC;
90+
__asm__ volatile(
91+
".option arch, +v, +zvbc\n"
92+
"vsetivli zero, 2, e64, m1, ta, ma\n"
93+
"vclmul.vv v0, v0, v0\n"
94+
"vclmulh.vv v0, v0, v0\n"
95+
);
96+
])],
97+
[AC_DEFINE([HAVE_ZVBC], [1], [Enable ZVBC instructions])
98+
AM_CONDITIONAL([HAVE_ZVBC], [true]) zvbc=yes],
99+
[AC_DEFINE([HAVE_ZVBC], [0], [Disable ZVBC instructions])
100+
AM_CONDITIONAL([HAVE_ZVBC], [false]) zvbc=no]
101+
)
102+
AC_MSG_RESULT([$zvbc])
76103
fi
104+
AC_MSG_CHECKING([RVV support])
105+
AS_IF([test "x$zvbc" = "xno"],
106+
[
107+
AC_COMPILE_IFELSE(
108+
[AC_LANG_PROGRAM([], [
109+
__asm__ volatile(
110+
".option arch, +v\n"
111+
"vsetivli zero, 0, e8, m1, ta, ma\n"
112+
);
113+
])],
114+
[AC_DEFINE([HAVE_RVV], [1], [Enable RVV instructions])
115+
AM_CONDITIONAL([HAVE_RVV], [true]) rvv=yes],
116+
[AC_DEFINE([HAVE_RVV], [0], [Disable RVV instructions])
117+
AM_CONDITIONAL([HAVE_RVV], [false]) rvv=no]
118+
)
119+
],
120+
[AC_DEFINE([HAVE_RVV], [1], [Enable RVV instructions])
121+
AM_CONDITIONAL([HAVE_RVV], [true]) rvv=yes]
122+
)
77123
AC_MSG_RESULT([$rvv])
124+
AS_IF([test "x$zvbc" = "xyes" && test "x$zbc" = "xyes"],
125+
[
126+
CFLAGS="$CFLAGS -march=rv64gcv_zbc_zvbc"
127+
CCASFLAGS="$CCASFLAGS -march=rv64gcv_zbc_zvbc"
128+
],
129+
[test "x$rvv" = "xyes"],
130+
[
131+
CFLAGS="$CFLAGS -march=rv64gcv"
132+
CCASFLAGS="$CCASFLAGS -march=rv64gcv"
133+
]
134+
)
78135
;;
79136

80137
*)

‎crc/Makefile.am‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,14 @@
2828
########################################################################
2929

3030
include crc/aarch64/Makefile.am
31+
include crc/riscv64/Makefile.am
3132

3233
lsrc += \
3334
crc/crc_base.c \
3435
crc/crc64_base.c
3536

3637
lsrc_base_aliases += crc/crc_base_aliases.c
3738
lsrc_ppc64le += crc/crc_base_aliases.c
38-
lsrc_riscv64 += crc/crc_base_aliases.c
3939

4040
lsrc_x86_64 += \
4141
crc/crc16_t10dif_01.asm \

‎crc/riscv64/Makefile.am‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
########################################################################
2+
# Copyright(c) 2025 ZTE Corporation All rights reserved.
3+
#
4+
# Redistribution and use in source and binary forms, with or without
5+
# modification, are permitted provided that the following conditions
6+
# are met:
7+
# * Redistributions of source code must retain the above copyright
8+
# notice, this list of conditions and the following disclaimer.
9+
# * Redistributions in binary form must reproduce the above copyright
10+
# notice, this list of conditions and the following disclaimer in
11+
# the documentation and/or other materials provided with the
12+
# distribution.
13+
# * Neither the name of ZTE Corporation nor the names of its
14+
# contributors may be used to endorse or promote products derived
15+
# from this software without specific prior written permission.
16+
#
17+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18+
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19+
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20+
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21+
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22+
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23+
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24+
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25+
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26+
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28+
#########################################################################
29+
lsrc_riscv64 += \
30+
crc/riscv64/crc_multibinary_riscv.S \
31+
crc/riscv64/crc_riscv64_dispatcher.c
32+
33+
lsrc_riscv64 += \
34+
crc/riscv64/crc16_t10dif_vclmul.S \
35+
crc/riscv64/crc16_t10dif_copy_vclmul.S \
36+
crc/riscv64/crc32_ieee_norm_vclmul.S \
37+
crc/riscv64/crc32_iscsi_refl_vclmul.S \
38+
crc/riscv64/crc32_gzip_refl_vclmul.S \
39+
crc/riscv64/crc64_ecma_refl_vclmul.S \
40+
crc/riscv64/crc64_ecma_norm_vclmul.S \
41+
crc/riscv64/crc64_iso_refl_vclmul.S \
42+
crc/riscv64/crc64_iso_norm_vclmul.S \
43+
crc/riscv64/crc64_jones_refl_vclmul.S \
44+
crc/riscv64/crc64_jones_norm_vclmul.S \
45+
crc/riscv64/crc64_rocksoft_refl_vclmul.S \
46+
crc/riscv64/crc64_rocksoft_norm_vclmul.S

‎crc/riscv64/crc16_t10dif_copy_vclmul.S‎

Lines changed: 220 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,220 @@
1+
########################################################################
2+
# Copyright (c) 2025 ZTE Corporation.
3+
#
4+
# Redistribution and use in source and binary forms, with or without
5+
# modification, are permitted provided that the following conditions
6+
# are met:
7+
# * Redistributions of source code must retain the above copyright
8+
# notice, this list of conditions and the following disclaimer.
9+
# * Redistributions in binary form must reproduce the above copyright
10+
# notice, this list of conditions and the following disclaimer in
11+
# the documentation and/or other materials provided with the
12+
# distribution.
13+
# * Neither the name of ZTE Corporation nor the names of its
14+
# contributors may be used to endorse or promote products derived
15+
# from this software without specific prior written permission.
16+
#
17+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18+
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19+
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20+
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21+
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22+
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23+
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24+
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25+
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26+
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28+
#########################################################################
29+
#if HAVE_ZBC && HAVE_ZVBC
30+
#include "crc16_t10dif_vclmul.h"
31+
32+
.section .text
33+
.align 2
34+
.global crc16_t10dif_copy_vclmul
35+
36+
# Arguments:
37+
# a0: uint16_t crc (seed)
38+
# a1: uint8_t *dst (pointer to data)
39+
# a2: uint8_t *src (pointer to data)
40+
# a3: size_t len (data length)
41+
42+
crc16_t10dif_copy_vclmul:
43+
# initialize seed for calculation in CRC32 format
44+
slli a0, a0, 16
45+
li t1, 64
46+
bgeu a3, t1, .crc_fold
47+
48+
.crc_table_loop_pre:
49+
beq a3, zero, .end
50+
la a7, .lanchor_crc_tab
51+
add a3, a3, a2
52+
53+
.crc_table_loop:
54+
lbu a4, 0(a2)
55+
sb a4, 0(a1)
56+
addi a2, a2, 1
57+
addi a1, a1, 1
58+
slliw a4, a4, 0x18
59+
xor a4, a4, a0
60+
srliw a5, a4, 0x18
61+
slli a5, a5, 0x2
62+
add a5, a5, a7
63+
lw a0, 0(a5)
64+
slliw a4, a4, 0x8
65+
xor a0, a0, a4
66+
bne a2, a3, .crc_table_loop
67+
68+
.end:
69+
slli a0, a0, 32
70+
srli a0, a0, 48
71+
ret
72+
73+
.crc_fold:
74+
vsetivli zero, 2, e64, m1, ta, ma
75+
vl4re64.v v4, 0(a2)
76+
addi a2, a2, 64
77+
addi a3, a3, -64
78+
79+
vs4r.v v4, (a1)
80+
addi a1, a1, 64
81+
82+
la t0, .shuffle_data_mask
83+
vsetivli zero, 16, e8, m1, ta, ma
84+
vle8.v v13, 0(t0)
85+
slli a0, a0, 32
86+
vrgather.vv v0, v4, v13
87+
vrgather.vv v1, v5, v13
88+
vrgather.vv v2, v6, v13
89+
vrgather.vv v3, v7, v13
90+
vsetivli zero, 2, e64, m1, ta, ma
91+
92+
vmv.v.x v5, a0
93+
vmv.s.x v4, zero
94+
vslideup.vi v4, v5, 1
95+
la t2, .crc_loop_const
96+
vle64.v v5, 0(t2)
97+
vxor.vv v0, v0, v4
98+
bltu a3, t1, crc_fold_finalization
99+
100+
li t0, 64
101+
102+
crc_fold_loop:
103+
vl4re64.v v8, (a2)
104+
addi a2, a2, 64
105+
vs4r.v v8, (a1)
106+
addi a1, a1, 64
107+
108+
vclmul.vv v4, v0, v5
109+
vclmulh.vv v0, v0, v5
110+
vslidedown.vi v15, v4, 1
111+
vslidedown.vi v14, v0, 1
112+
vxor.vv v15, v15, v4
113+
vxor.vv v14, v14, v0
114+
vslideup.vi v15, v14, 1
115+
116+
vclmul.vv v4, v1, v5
117+
vclmulh.vv v1, v1, v5
118+
vslidedown.vi v16, v4, 1
119+
vslidedown.vi v14, v1, 1
120+
vxor.vv v16, v16, v4
121+
vxor.vv v14, v14, v1
122+
vslideup.vi v16, v14, 1
123+
124+
vclmul.vv v4, v2, v5
125+
vclmulh.vv v2, v2, v5
126+
vslidedown.vi v17, v4, 1
127+
vslidedown.vi v14, v2, 1
128+
vxor.vv v17, v17, v4
129+
vxor.vv v14, v14, v2
130+
vslideup.vi v17, v14, 1
131+
132+
vclmul.vv v4, v3, v5
133+
vclmulh.vv v3, v3, v5
134+
vslidedown.vi v18, v4, 1
135+
vslidedown.vi v14, v3, 1
136+
vxor.vv v18, v18, v4
137+
vxor.vv v14, v14, v3
138+
vslideup.vi v18, v14, 1
139+
140+
vsetivli zero, 16, e8, m1, ta, ma
141+
vrgather.vv v0, v8, v13
142+
vrgather.vv v1, v9, v13
143+
vrgather.vv v2, v10, v13
144+
vrgather.vv v3, v11, v13
145+
vsetivli zero, 2, e64, m1, ta, ma
146+
vxor.vv v0, v0, v15
147+
vxor.vv v1, v1, v16
148+
vxor.vv v2, v2, v17
149+
vxor.vv v3, v3, v18
150+
151+
addi a3, a3, -64
152+
bge a3, t0, crc_fold_loop
153+
154+
crc_fold_finalization:
155+
# 512bit -> 128bit folding
156+
addi t2, t2, 16
157+
vle64.v v5, 0(t2)
158+
vclmul.vv v6, v0, v5
159+
vclmulh.vv v7, v0, v5
160+
vslidedown.vi v8, v6, 1
161+
vslidedown.vi v9, v7, 1
162+
vxor.vv v8, v8, v6
163+
vxor.vv v9, v9, v7
164+
vslideup.vi v8, v9, 1
165+
vxor.vv v0, v8, v1
166+
167+
vclmul.vv v6, v0, v5
168+
vclmulh.vv v7, v0, v5
169+
vslidedown.vi v8, v6, 1
170+
vslidedown.vi v9, v7, 1
171+
vxor.vv v8, v8, v6
172+
vxor.vv v9, v9, v7
173+
vslideup.vi v8, v9, 1
174+
vxor.vv v0, v8, v2
175+
176+
vclmul.vv v6, v0, v5
177+
vclmulh.vv v7, v0, v5
178+
vslidedown.vi v8, v6, 1
179+
vslidedown.vi v9, v7, 1
180+
vxor.vv v8, v8, v6
181+
vxor.vv v9, v9, v7
182+
vslideup.vi v8, v9, 1
183+
vxor.vv v0, v8, v3
184+
185+
# 128bit -> 64bit folding
186+
vmv.x.s t0, v0
187+
vslidedown.vi v0, v0, 1
188+
vmv.x.s t1, v0
189+
li t2, const_low
190+
li t3, const_high
191+
clmul a4, t1, t2
192+
clmulh a5, t1, t2
193+
slli a6, t0, 32
194+
srli a7, t0, 32
195+
xor a4, a4, a6
196+
xor a5, a5, a7
197+
clmul a5, a5, t3
198+
xor a4, a4, a5
199+
200+
# Barrett reduction
201+
srli a5, a4, 32
202+
li t2, const_quo
203+
clmul a5, t2, a5
204+
srli a5, a5, 32
205+
li t3, const_poly
206+
clmul a5, a5, t3
207+
xor a0, a5, a4
208+
209+
tail_processing:
210+
beqz a3, .end
211+
jal x0, .crc_table_loop_pre
212+
213+
.shuffle_data_mask = . + 0
214+
.type shuffle_data, %object
215+
.size shuffle_data, 16
216+
shuffle_data:
217+
.byte 15, 14, 13, 12, 11, 10, 9, 8
218+
.byte 7, 6, 5, 4, 3, 2, 1, 0
219+
220+
#endif

0 commit comments

Comments
 (0)