erasure_code: add prefetch and vsetvli optimization

Signed-off-by: Shuo Lv <[email protected]>

Author: lvshuo2016

configure.ac

@@ -71,8 +71,8 @@ case "${CPU}" in
 			AM_CONDITIONAL([HAVE_RVV], [false]) rvv=no]
 		)
 		if test "x$rvv" = "xyes"; then
-		  CFLAGS+=" -march=rv64gcv"
-		  CCASFLAGS+=" -march=rv64gcv"
+		  CFLAGS+=" -march=rv64gcv_zicbop"
+		  CCASFLAGS+=" -march=rv64gcv_zicbop"
 		fi
 		AC_MSG_RESULT([$rvv])
 		;;

erasure_code/riscv64/gf_2vect_dot_prod_rvv.S

@@ -74,9 +74,10 @@ gf_2vect_dot_prod_rvv:
     li t6, 16
     blt x_len, t6, .return_fail
 
-    vsetvli a5, x0, e8, m1  /* Set vector length to maximum */
-
+    vsetvli a5, x0, e8, m1,ta,ma  /* Set vector length to maximum */
     li x_pos, 0
+    slli x_vec, x_vec, 3
+
     ld x_dest1, 0(x_dest)
     ld x_dest2, 8(x_dest)
 
@@ -92,15 +93,12 @@ gf_2vect_dot_prod_rvv:
 
     /* gf_tbl base = (x_tbl + dest_idx * x_vec * 32) */
     mv x_tbl1, x_tbl           /* reset x_tbl1 */
-    slli t6, x_vec, 5
+    slli t6, x_vec, 2
     add x_tbl2, x_tbl1, t6     /* reset x_tbl2 */
 
 /* Loop 2: x_vec, number of source vectors (ie. data blocks) */
 .Llooprvv_vl_vects:
     /* load src data */
-    slli a6, x_vec_i, 3
-    add a6,x_src,a6
-    ld x_ptr, 0(a6)
     add x_ptr,x_ptr,x_pos
 
     vle8.v v_src, (x_ptr)      /* load from: src base + pos offset */
@@ -120,6 +118,14 @@ gf_2vect_dot_prod_rvv:
     vle8.v v_gft2_hi, (x_tbl2)
     addi x_tbl2, x_tbl2, 16
 
+    prefetch.r 0(x_tbl1)
+    prefetch.r 0(x_tbl2)
+
+    /* calc for next */
+    addi x_vec_i, x_vec_i, 8   /* move x_vec_i to next */
+    add a6,x_src,x_vec_i
+    ld x_ptr, 0(a6)
+
     /* dest 1 */
     /* table indexing, ie. gf(2^8) multiplication */
     vrgather.vv v26, v_gft1_lo, v_src_lo
@@ -134,8 +140,6 @@ gf_2vect_dot_prod_rvv:
     vxor.vv v_dest2, v_dest2, v26
     vxor.vv v_dest2, v_dest2, v27
 
-    /* calc for next */
-    addi x_vec_i, x_vec_i, 1   /* move x_vec_i to next */
     blt x_vec_i, x_vec, .Llooprvv_vl_vects
 /* end of Loop 2 */
 

erasure_code/riscv64/gf_2vect_mad_rvv.S

@@ -71,7 +71,7 @@ gf_2vect_mad_rvv:
     li t3, 16
     blt x_len, t3, .return_fail
 
-    vsetvli t4, x0, e8, m1
+    vsetvli t4, x0, e8, m1,ta,ma
 
     /* load table 1 */
     slli t3, x_vec_i, 5

erasure_code/riscv64/gf_3vect_dot_prod_rvv.S

@@ -84,10 +84,11 @@ gf_3vect_dot_prod_rvv:
     sd s0, 0(sp)
     sd s1, 8(sp)
 
-    vsetvli a7, x0, e8, m1  /* Set vector length to maximum */
-
+    vsetvli a7, x0, e8, m1,ta,ma  /* Set vector length to maximum */
     li x_pos, 0
-    slli t_offset, x_vec, 5
+    slli x_vec, x_vec, 3
+
+    slli t_offset, x_vec, 2
     ld x_dest1, 0(x_dest)
     ld x_dest2, 8(x_dest)
     ld x_dest3, 16(x_dest)
@@ -101,20 +102,19 @@ gf_3vect_dot_prod_rvv:
     vmv.v.i v_dest2, 0
     vmv.v.i v_dest3, 0
 
+    /* Loop 2: x_vec, number of source vectors (ie. data blocks) */
+    li x_vec_i, 0
+    /* load source pointer */
+    ld x_ptr, 0(x_src)
+
     /* Reset table pointers */
     mv x_tbl1, x_tbl
     add x_tbl2, x_tbl1, t_offset
     add x_tbl3, x_tbl2, t_offset
 
-    /* Loop 2: x_vec, number of source vectors (ie. data blocks) */
-    li x_vec_i, 0
 .Lloop_rvv_vl_vects:
-    /* Load source data */
-    slli t0, x_vec_i, 3
-    add t0,x_src,t0
-    ld x_ptr, 0(t0)
+    /* load source data */
     add x_ptr,x_ptr,x_pos
-
     vle8.v v_src, (x_ptr)
 
     /* Split 4-bit lo; 4-bit hi */
@@ -131,14 +131,22 @@ gf_3vect_dot_prod_rvv:
     vle8.v v_gft2_hi, (x_tbl2)
     addi x_tbl2, x_tbl2, 16
 
+    /* Move to next source vector */
+    addi x_vec_i, x_vec_i, 8
+    add t0,x_src,x_vec_i
+    ld x_ptr, 0(t0)
+
+    prefetch.r 0(x_tbl1)
+    prefetch.r 0(x_tbl2)
 
     /* Load next gf_table's */
     vle8.v v_gft3_lo, (x_tbl3)
     addi x_tbl3, x_tbl3, 16
     vle8.v v_gft3_hi, (x_tbl3)
     addi x_tbl3, x_tbl3, 16
+    prefetch.r 0(x_tbl3)
 
-/* dest 1 */
+    /* dest 1 */
     vrgather.vv v26, v_gft1_lo, v_src_lo
     vrgather.vv v27, v_gft1_hi, v_src_hi
     vxor.vv v_dest1, v_dest1, v26
@@ -156,9 +164,6 @@ gf_3vect_dot_prod_rvv:
     vxor.vv v_dest3, v_dest3, v26
     vxor.vv v_dest3, v_dest3, v27
 
-    /* Move to next source vector */
-    addi x_vec_i, x_vec_i, 1
-
     /* Check if we have processed all vectors */
     blt x_vec_i, x_vec, .Lloop_rvv_vl_vects
 

erasure_code/riscv64/gf_3vect_mad_rvv.S

@@ -75,7 +75,7 @@ gf_3vect_mad_rvv:
     li t4, 16
     blt x_len, t4, .return_fail
 
-    vsetvli t5, x0, e8, m1
+    vsetvli t5, x0, e8, m1,ta,ma
 
     /* Load table 1 */
     slli t4, x_vec_i, 5

erasure_code/riscv64/gf_4vect_dot_prod_rvv.S

@@ -91,10 +91,13 @@ gf_4vect_dot_prod_rvv:
     sd s2, 16(sp)
     sd s3, 24(sp)
 
-    vsetvli t0, x0, e8, m1  /* Set vector length to maximum */
-
+  #  vsetvli t0, x0, e8, m1  /* Set vector length to maximum */
+    vsetvli t0, x0, e8, m1,ta,ma
     li x_pos, 0
-    slli t_offset, x_vec, 5
+
+    slli x_vec, x_vec, 3
+    slli t_offset, x_vec, 2
+
     ld x_dest1, 0(x_dest)
     ld x_dest2, 8(x_dest)
     ld x_dest3, 16(x_dest)
@@ -111,19 +114,20 @@ gf_4vect_dot_prod_rvv:
     vmv.v.i v_dest3, 0
     vmv.v.i v_dest4, 0
 
+    /*  x_vec, number of source vectors (ie. data blocks) */
+    li x_vec_i, 0
+
+    /* load source pointer */
+    ld x_ptr, 0(x_src)
+
     /* Reset table pointers */
     mv x_tbl1, x_tbl
     add x_tbl2, x_tbl1, t_offset
     add x_tbl3, x_tbl2, t_offset
     add x_tbl4, x_tbl3, t_offset
 
-    /* Loop 2: x_vec, number of source vectors (ie. data blocks) */
-    li x_vec_i, 0
 .Lloop_rvv_vl_vects:
     /* Load source data */
-    slli a6, x_vec_i, 3
-    add a6,x_src,a6
-    ld x_ptr, 0(a6)
     add x_ptr,x_ptr,x_pos
 
     vle8.v v_src, (x_ptr)
@@ -142,6 +146,13 @@ gf_4vect_dot_prod_rvv:
     vle8.v v_gft2_hi, (x_tbl2)
     addi x_tbl2, x_tbl2, 16
 
+    prefetch.r 0(x_tbl1)
+    prefetch.r 0(x_tbl2)
+
+    /* Move to next source vector */
+    addi x_vec_i, x_vec_i, 8
+    add a6,x_src,x_vec_i
+    ld x_ptr, 0(a6)
 
     /* Load next gf_table's */
     vle8.v v_gft3_lo, (x_tbl3)
@@ -153,6 +164,8 @@ gf_4vect_dot_prod_rvv:
     addi x_tbl4, x_tbl4, 16
     vle8.v v_gft4_hi, (x_tbl4)
     addi x_tbl4, x_tbl4, 16
+    prefetch.r 0(x_tbl3)
+    prefetch.r 0(x_tbl4)
 
     /* dest 1 */
     vrgather.vv v26, v_gft1_lo, v_src_lo
@@ -178,9 +191,6 @@ gf_4vect_dot_prod_rvv:
     vxor.vv v_dest4, v_dest4, v26
     vxor.vv v_dest4, v_dest4, v27
 
-    /* Move to next source vector */
-    addi x_vec_i, x_vec_i, 1
-
     /* Check if we have processed all vectors */
     blt x_vec_i, x_vec, .Lloop_rvv_vl_vects
 
@@ -198,7 +208,7 @@ gf_4vect_dot_prod_rvv:
     j .Lloop_rvv_vl
 
 .return_pass:
-/* restore callee-saved registers */
+    /* restore callee-saved registers */
     ld s0, 0(sp)
     ld s1, 8(sp)
     ld s2, 16(sp)

erasure_code/riscv64/gf_4vect_mad_rvv.S

@@ -79,7 +79,7 @@ gf_4vect_mad_rvv:
 	li	t5, 16
 	blt	x_len, t5, .return_fail
 
-	vsetvli	t6, x0, e8, m1
+	vsetvli	t6, x0, e8, m1,ta,ma
 
 	/* load table 1 */
 	slli	t5, x_vec_i, 5

erasure_code/riscv64/gf_5vect_dot_prod_rvv.S

@@ -95,8 +95,7 @@ gf_5vect_dot_prod_rvv:
     sd s4, 32(sp)
     sd s5, 40(sp)
 
-    vsetvli a5, x0, e8, m1
-
+    vsetvli a5, x0, e8, m1,ta,ma
     /* Initialize position */
     li x_pos, 0
 
@@ -151,6 +150,9 @@ gf_5vect_dot_prod_rvv:
     vle8.v v_gft2_hi, (x_tbl2)
     addi x_tbl2, x_tbl2, 16
 
+    prefetch.r 0(x_tbl1)
+    prefetch.r 0(x_tbl2)
+
     /* Move to next source vector */
     addi x_vec_i, x_vec_i, 1
 
@@ -171,6 +173,9 @@ gf_5vect_dot_prod_rvv:
     vle8.v v_gft4_hi, (x_tbl4)
     addi x_tbl4, x_tbl4, 16
 
+    prefetch.r 0(x_tbl3)
+    prefetch.r 0(x_tbl4)
+
     /* dest 2 */
     vrgather.vv v26, v_gft2_lo, v_src_lo
     vrgather.vv v27, v_gft2_hi, v_src_hi
@@ -188,6 +193,7 @@ gf_5vect_dot_prod_rvv:
     addi x_tbl5, x_tbl5, 16
     vle8.v v_gft5_hi, (x_tbl5)
     addi x_tbl5, x_tbl5, 16
+    prefetch.r 0(x_tbl5)
 
     /* dest 4 */
     vrgather.vv v26, v_gft4_lo, v_src_lo

erasure_code/riscv64/gf_5vect_mad_rvv.S

@@ -83,7 +83,7 @@ gf_5vect_mad_rvv:
     li t6, 16
     blt x_len, t6, .return_fail
 
-    vsetvli a7, x0, e8, m1
+    vsetvli a7, x0, e8, m1,ta,ma
 
     /* Load table 1 */
     slli a6, x_vec_i, 5

erasure_code/riscv64/gf_6vect_dot_prod_rvv.S

@@ -102,11 +102,13 @@ gf_6vect_dot_prod_rvv:
     sd s7, 56(sp)
 
     li t0, 0x0F
-    vsetvli a5, x0, e8, m1
+    vsetvli a5, x0, e8, m1,ta,ma
 
     /* initialize position */
     li x_pos, 0
 
+    slli x_vec, x_vec, 3
+
     /* load destination pointers */
     ld x_dest1, 0(x14)  #  a4 is also x14
     ld x_dest2, 8(x_dest)
@@ -136,7 +138,7 @@ gf_6vect_dot_prod_rvv:
     /* initialize table pointers */
     /* gf_tbl base = (x_tbl + dest_idx * x_vec * 32) */
     mv x_tbl1, x_tbl
-    slli t0, x_vec, 5
+    slli t0, x_vec, 2
     add x_tbl2, x_tbl1, t0
     add x_tbl3, x_tbl2, t0
     add x_tbl4, x_tbl3, t0
@@ -145,11 +147,7 @@ gf_6vect_dot_prod_rvv:
 
 .Llooprvv_vl_vects:
     /* load source data */
-    slli a6, x_vec_i, 3
-    add a6,x_src,a6
-    ld x_ptr, 0(a6)
     add x_ptr,x_ptr,x_pos
-
     vle8.v v_src, (x_ptr)
 
 
@@ -167,6 +165,13 @@ gf_6vect_dot_prod_rvv:
     addi x_tbl2, x_tbl2, 16
     vle8.v v_gft2_hi, (x_tbl2)
     addi x_tbl2, x_tbl2, 16
+    prefetch.r 0(x_tbl1)
+    prefetch.r 0(x_tbl2)
+
+    /* load next source pointer */
+    addi x_vec_i, x_vec_i,8
+    add a6,x_src,x_vec_i
+    ld x_ptr, 0(a6)
 
     vle8.v v_gft3_lo, (x_tbl3)
     addi x_tbl3, x_tbl3, 16
@@ -178,6 +183,9 @@ gf_6vect_dot_prod_rvv:
     vle8.v v_gft4_hi, (x_tbl4)
     addi x_tbl4, x_tbl4, 16
 
+    prefetch.r 0(x_tbl3)
+    prefetch.r 0(x_tbl4)
+
     vle8.v v_gft5_lo, (x_tbl5)
     addi x_tbl5, x_tbl5, 16
     vle8.v v_gft5_hi, (x_tbl5)
@@ -188,6 +196,8 @@ gf_6vect_dot_prod_rvv:
     vle8.v v_gft6_hi, (x_tbl6)
     addi x_tbl6, x_tbl6, 16
 
+    prefetch.r 0(x_tbl5)
+    prefetch.r 0(x_tbl6)
 
     /* dest 1 */
     vrgather.vv v26, v_gft1_lo, v_src_lo
@@ -225,10 +235,6 @@ gf_6vect_dot_prod_rvv:
     vxor.vv v_dest6, v_dest6, v26
     vxor.vv v_dest6, v_dest6, v27
 
-
-    /* load next source pointer */
-    addi x_vec_i, x_vec_i,1
-
     /* check if we have processed all vectors */
     blt x_vec_i, x_vec, .Llooprvv_vl_vects