The CRC module of ISA-L has been accelerated using RISC-V's V, Zbc, Zvbc, and Zvbb instruction sets, implementing data folding and Barrett reduction optimizations.

Signed-off-by: Ji Dong <[email protected]>

Author: yinlenree

configure.ac

@@ -71,8 +71,8 @@ case "${CPU}" in
 			AM_CONDITIONAL([HAVE_RVV], [false]) rvv=no]
 		)
 		if test "x$rvv" = "xyes"; then
-		  CFLAGS+=" -march=rv64gcv"
-		  CCASFLAGS+=" -march=rv64gcv"
+		  CFLAGS+=" -march=rv64gcv_zbc_zvbc_zvbb"
+		  CCASFLAGS+=" -march=rv64gcv_zbc_zvbc_zvbb"
 		fi
 		AC_MSG_RESULT([$rvv])
 		;;

crc/Makefile.am

@@ -28,14 +28,14 @@
 ########################################################################
 
 include crc/aarch64/Makefile.am
+include crc/riscv64/Makefile.am
 
 lsrc  += \
 	crc/crc_base.c \
 	crc/crc64_base.c
 
 lsrc_base_aliases += crc/crc_base_aliases.c
 lsrc_ppc64le      += crc/crc_base_aliases.c
-lsrc_riscv64      += crc/crc_base_aliases.c
 
 lsrc_x86_64 += \
 	crc/crc16_t10dif_01.asm \

crc/riscv64/Makefile.am

@@ -0,0 +1,46 @@
+########################################################################
+#  Copyright(c) 2025 ZTE Corporation All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of ZTE Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#########################################################################
+lsrc_riscv64 += \
+	crc/riscv64/crc_multibinary_riscv.S	\
+	crc/riscv64/crc_riscv64_dispatcher.c
+
+lsrc_riscv64 += \
+	crc/riscv64/crc16_t10dif_vclmul.S \
+	crc/riscv64/crc16_t10dif_copy_vclmul.S \
+	crc/riscv64/crc32_ieee_norm_vclmul.S \
+	crc/riscv64/crc32_iscsi_refl_vclmul.S \
+	crc/riscv64/crc32_gzip_refl_vclmul.S \
+	crc/riscv64/crc64_ecma_refl_vclmul.S \
+	crc/riscv64/crc64_ecma_norm_vclmul.S \
+	crc/riscv64/crc64_iso_refl_vclmul.S \
+	crc/riscv64/crc64_iso_norm_vclmul.S \
+	crc/riscv64/crc64_jones_refl_vclmul.S \
+	crc/riscv64/crc64_jones_norm_vclmul.S \
+	crc/riscv64/crc64_rocksoft_refl_vclmul.S \
+	crc/riscv64/crc64_rocksoft_norm_vclmul.S

crc/riscv64/crc16_t10dif_copy_vclmul.S

@@ -0,0 +1,297 @@
+########################################################################
+#  Copyright (c) 2025 ZTE Corporation.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of ZTE Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#########################################################################
+.section .text
+.align 2
+.global crc16_t10dif_copy_vclmul
+
+# Arguments:
+# a0: uint16_t crc (seed)
+# a1: uint8_t *dst (pointer to data)
+# a2: uint8_t *src (pointer to data)
+# a3: size_t len (data length)
+
+# Register usage:
+
+crc16_t10dif_copy_vclmul:
+    # Initialize state
+    slli a0, a0, 16
+    # Check if length >= 64
+    li t1, 64
+    bgeu a3, t1, .crc_fold
+
+.crc_table_loop_pre:
+    beq a3, zero, .end
+    la a7, .LANCHOR0
+    add a3, a3, a2
+
+.crc_table_loop:
+    lbu a4, 0(a2)
+    sb a4, 0(a1)
+    addi a2, a2, 1
+    addi a1, a1, 1
+    sllw a4, a4, 0x18
+    xor a4, a4, a0
+    srlw a5, a4, 0x18
+    sll a5, a5, 0x2
+    add a5, a5, a7
+    lw a0, 0(a5)
+    sllw a4, a4, 0x8
+    xor a0, a0, a4
+    bne a2, a3, .crc_table_loop
+
+.end:
+    slli a0, a0, 32
+    srli a0, a0, 48
+    ret
+    
+.crc_fold:
+    # Initialize vector registers
+    vsetivli zero, 2, e64, m1, ta, ma
+    vle64.v v6, 0(a2)
+    addi a2, a2, 16
+    vle64.v v7, 0(a2)
+    addi a2, a2, 16
+    vle64.v v8, 0(a2)
+    addi a2, a2, 16
+    vle64.v v9, 0(a2)
+    addi a2, a2, 16
+    addi a3, a3, -64
+
+    vse64.v v6, (a1)
+    addi a1, a1, 16
+    vse64.v v7, (a1)
+    addi a1, a1, 16
+    vse64.v v8, (a1)
+    addi a1, a1, 16
+    vse64.v v9, (a1)
+    addi a1, a1, 16
+
+    # Prepare initial vector
+    slli a0, a0, 32
+    vmv.s.x v4, zero
+    vrev8.v v6, v6
+    vrev8.v v7, v7
+    vrev8.v v8, v8
+    vrev8.v v9, v9
+    vslidedown.vi v0, v6, 1
+    vslidedown.vi v1, v7, 1
+    vslidedown.vi v2, v8, 1
+    vslidedown.vi v3, v9, 1
+    vslideup.vi v0, v6, 1
+    vslideup.vi v1, v7, 1
+    vslideup.vi v2, v8, 1
+    vslideup.vi v3, v9, 1
+
+    vmv.v.x v5, a0
+    vslideup.vi v4, v5, 1
+    
+    vxor.vv v0, v0, v4
+    
+    vmv.v.x v8, zero
+    bltu a3, t1, final_fold
+
+    # Load constants
+    la t0, k_const1
+    vle64.v v5, 0(t0)
+
+    li t0, 64
+    
+    # Main processing loop
+loop_start:
+    vle64.v v9, (a2)
+    addi a2, a2, 16
+    vle64.v v10, (a2)
+    addi a2, a2, 16
+    vle64.v v11, (a2)
+    addi a2, a2, 16
+    vle64.v v12, (a2)
+    addi a2, a2, 16
+    vse64.v v9, (a1)
+    addi a1, a1, 16
+    vse64.v v10, (a1)
+    addi a1, a1, 16
+    vse64.v v11, (a1)
+    addi a1, a1, 16
+    vse64.v v12, (a1)
+    addi a1, a1, 16
+
+    vclmul.vv v4, v0, v5
+    vclmulh.vv v0, v0, v5
+    vredxor.vs v0, v0, v8
+    vredxor.vs v4, v4, v8
+    vslideup.vi v4, v0, 1
+    vrev8.v v9, v9
+    vslidedown.vi v6, v9, 1
+    vslideup.vi v6, v9, 1
+    vxor.vv v0, v4, v6
+
+    # Process v1-v3 (similar to v0)
+    
+    vclmul.vv v4, v1, v5
+    vclmulh.vv v1, v1, v5
+    vredxor.vs v1, v1, v8
+    vredxor.vs v4, v4, v8
+    vslideup.vi v4, v1, 1
+    vrev8.v v10, v10
+    vslidedown.vi v6, v10, 1
+    vslideup.vi v6, v10, 1
+    vxor.vv v1, v4, v6
+
+    vclmul.vv v4, v2, v5
+    vclmulh.vv v2, v2, v5
+    vredxor.vs v2, v2, v8
+    vredxor.vs v4, v4, v8
+    vslideup.vi v4, v2, 1
+    vrev8.v v11, v11
+    vslidedown.vi v6, v11, 1
+    vslideup.vi v6, v11, 1
+    vxor.vv v2, v4, v6
+
+    vclmul.vv v4, v3, v5
+    vclmulh.vv v3, v3, v5
+    vredxor.vs v3, v3, v8
+    vredxor.vs v4, v4, v8
+    vslideup.vi v4, v3, 1
+    vrev8.v v12, v12
+    vslidedown.vi v6, v12, 1
+    vslideup.vi v6, v12, 1
+    vxor.vv v3, v4, v6
+
+    addi a3, a3, -64
+    bge a3, t0, loop_start
+
+final_fold:
+    la t0, k_const2
+    vle64.v v5, 0(t0)
+    vclmul.vv v6, v0, v5
+    vclmulh.vv v7, v0, v5
+    vredxor.vs v6, v6, v8
+    vredxor.vs v7, v7, v8
+    vslideup.vi v6, v7, 1
+    vxor.vv v0, v6, v1
+    vclmul.vv v6, v0, v5
+    vclmulh.vv v7, v0, v5
+    vredxor.vs v6, v6, v8
+    vredxor.vs v7, v7, v8
+    vslideup.vi v6, v7, 1
+    vxor.vv v0, v6, v2
+    vclmul.vv v6, v0, v5
+    vclmulh.vv v7, v0, v5
+    vredxor.vs v6, v6, v8
+    vredxor.vs v7, v7, v8
+    vslideup.vi v6, v7, 1
+    vxor.vv v0, v6, v3
+    
+    # Store result
+    addi sp, sp, -16
+    vse64.v v0, (sp)
+
+    # 128b -> 64b folding
+    ld t0, 0(sp)
+    ld t1, 8(sp)
+    li t2, 0x2d560000
+    li t3, 0x13680000
+    clmul a4, t1, t2
+    clmulh a5, t1, t2
+    slli a6, t0, 32
+    srli a7, t0, 32
+    xor a4, a4, a6
+    xor a5, a5, a7
+    clmul a5, a5, t3
+    xor a4, a4, a5
+    
+    # Barrett reduction
+    srli a5, a4, 32
+    li t2, 0x1f65a57f9    # x_quo
+    clmul a5, t2, a5
+    srli a5, a5, 32
+    li t3, 0x18bb70000    # x_poly
+    clmul a5, a5, t3
+    xor a0, a5, a4
+    addi sp, sp, 16
+    
+tail_processing:
+    # Process remaining bytes
+    beqz a3, .end
+    
+    # Call crc16_t10dif_generic equivalent for remaining bytes
+    jal x0, .crc_table_loop_pre
+
+.section .rodata
+.align 4
+initial_vector:
+    .quad 0x00000000
+    .quad 0x00000000
+
+k_const1:
+    .quad 0x87e70000
+    .quad 0x371d0000
+
+k_const2:
+    .quad 0xfb0b0000
+    .quad 0x4c1a0000
+
+.LANCHOR0 = . + 0
+    .type	crc16tab, %object
+	.size	crc16tab, 1024
+crc16tab:
+    .word 0x00000000, 0x8bb70000, 0x9cd90000, 0x176e0000, 0xb2050000, 0x39b20000, 0x2edc0000, 0xa56b0000
+    .word 0xefbd0000, 0x640a0000, 0x73640000, 0xf8d30000, 0x5db80000, 0xd60f0000, 0xc1610000, 0x4ad60000
+    .word 0x54cd0000, 0xdf7a0000, 0xc8140000, 0x43a30000, 0xe6c80000, 0x6d7f0000, 0x7a110000, 0xf1a60000
+    .word 0xbb700000, 0x30c70000, 0x27a90000, 0xac1e0000, 0x09750000, 0x82c20000, 0x95ac0000, 0x1e1b0000
+    .word 0xa99a0000, 0x222d0000, 0x35430000, 0xbef40000, 0x1b9f0000, 0x90280000, 0x87460000, 0x0cf10000
+    .word 0x46270000, 0xcd900000, 0xdafe0000, 0x51490000, 0xf4220000, 0x7f950000, 0x68fb0000, 0xe34c0000
+    .word 0xfd570000, 0x76e00000, 0x618e0000, 0xea390000, 0x4f520000, 0xc4e50000, 0xd38b0000, 0x583c0000
+    .word 0x12ea0000, 0x995d0000, 0x8e330000, 0x05840000, 0xa0ef0000, 0x2b580000, 0x3c360000, 0xb7810000
+    .word 0xd8830000, 0x53340000, 0x445a0000, 0xcfed0000, 0x6a860000, 0xe1310000, 0xf65f0000, 0x7de80000
+    .word 0x373e0000, 0xbc890000, 0xabe70000, 0x20500000, 0x853b0000, 0x0e8c0000, 0x19e20000, 0x92550000
+    .word 0x8c4e0000, 0x07f90000, 0x10970000, 0x9b200000, 0x3e4b0000, 0xb5fc0000, 0xa2920000, 0x29250000
+    .word 0x63f30000, 0xe8440000, 0xff2a0000, 0x749d0000, 0xd1f60000, 0x5a410000, 0x4d2f0000, 0xc6980000
+    .word 0x71190000, 0xfaae0000, 0xedc00000, 0x66770000, 0xc31c0000, 0x48ab0000, 0x5fc50000, 0xd4720000
+    .word 0x9ea40000, 0x15130000, 0x027d0000, 0x89ca0000, 0x2ca10000, 0xa7160000, 0xb0780000, 0x3bcf0000
+    .word 0x25d40000, 0xae630000, 0xb90d0000, 0x32ba0000, 0x97d10000, 0x1c660000, 0x0b080000, 0x80bf0000
+    .word 0xca690000, 0x41de0000, 0x56b00000, 0xdd070000, 0x786c0000, 0xf3db0000, 0xe4b50000, 0x6f020000
+    .word 0x3ab10000, 0xb1060000, 0xa6680000, 0x2ddf0000, 0x88b40000, 0x03030000, 0x146d0000, 0x9fda0000
+    .word 0xd50c0000, 0x5ebb0000, 0x49d50000, 0xc2620000, 0x67090000, 0xecbe0000, 0xfbd00000, 0x70670000
+    .word 0x6e7c0000, 0xe5cb0000, 0xf2a50000, 0x79120000, 0xdc790000, 0x57ce0000, 0x40a00000, 0xcb170000
+    .word 0x81c10000, 0x0a760000, 0x1d180000, 0x96af0000, 0x33c40000, 0xb8730000, 0xaf1d0000, 0x24aa0000
+    .word 0x932b0000, 0x189c0000, 0x0ff20000, 0x84450000, 0x212e0000, 0xaa990000, 0xbdf70000, 0x36400000
+    .word 0x7c960000, 0xf7210000, 0xe04f0000, 0x6bf80000, 0xce930000, 0x45240000, 0x524a0000, 0xd9fd0000
+    .word 0xc7e60000, 0x4c510000, 0x5b3f0000, 0xd0880000, 0x75e30000, 0xfe540000, 0xe93a0000, 0x628d0000
+    .word 0x285b0000, 0xa3ec0000, 0xb4820000, 0x3f350000, 0x9a5e0000, 0x11e90000, 0x06870000, 0x8d300000
+    .word 0xe2320000, 0x69850000, 0x7eeb0000, 0xf55c0000, 0x50370000, 0xdb800000, 0xccee0000, 0x47590000
+    .word 0x0d8f0000, 0x86380000, 0x91560000, 0x1ae10000, 0xbf8a0000, 0x343d0000, 0x23530000, 0xa8e40000
+    .word 0xb6ff0000, 0x3d480000, 0x2a260000, 0xa1910000, 0x04fa0000, 0x8f4d0000, 0x98230000, 0x13940000
+    .word 0x59420000, 0xd2f50000, 0xc59b0000, 0x4e2c0000, 0xeb470000, 0x60f00000, 0x779e0000, 0xfc290000
+    .word 0x4ba80000, 0xc01f0000, 0xd7710000, 0x5cc60000, 0xf9ad0000, 0x721a0000, 0x65740000, 0xeec30000
+    .word 0xa4150000, 0x2fa20000, 0x38cc0000, 0xb37b0000, 0x16100000, 0x9da70000, 0x8ac90000, 0x017e0000
+    .word 0x1f650000, 0x94d20000, 0x83bc0000, 0x080b0000, 0xad600000, 0x26d70000, 0x31b90000, 0xba0e0000
+    .word 0xf0d80000, 0x7b6f0000, 0x6c010000, 0xe7b60000, 0x42dd0000, 0xc96a0000, 0xde040000, 0x55b30000
+
+