crc: only prefetch data that will be consumed for non-VPCLMUL functions

Also, use only 2 prefetch instructions for 128B.

Signed-off-by: Pablo de Lara <[email protected]>

Author: pablodelara

crc/crc16_t10dif_01.asm

@@ -147,10 +147,14 @@ crc16_t10dif_01:
 	; at this section of the code, there is 128*x+y (0<=y<128) bytes of buffer. The _fold_128_B_loop
 	; loop will fold 128B at a time until we have 128+y Bytes of buffer
 
+%if fetch_dist != 0
+	; check if there is at least 4kb (fetch distance) + 128b in the buffer
+        cmp     arg3, (fetch_dist + 128)
+        jb     _fold_128_B_loop
 
 	; fold 128B at a time. This section of the code folds 8 xmm registers in parallel
-_fold_128_B_loop:
-
+align 16
+_fold_and_prefetch_128_B_loop:
 	; update the buffer pointer
 	add	arg2, 128		;    buf += 128;
 
@@ -170,7 +174,6 @@ _fold_128_B_loop:
 	pxor	xmm1, xmm12
 	xorps	xmm1, xmm13
 
-	PREFETCH [arg2+fetch_dist+32]
 	movdqu	xmm9, [arg2+16*2]
 	movdqu	xmm12, [arg2+16*3]
 	pshufb	xmm9, xmm11
@@ -202,7 +205,79 @@ _fold_128_B_loop:
 	pxor	xmm5, xmm12
 	xorps	xmm5, xmm13
 
-	PREFETCH [arg2+fetch_dist+96]
+	movdqu	xmm9, [arg2+16*6]
+	movdqu	xmm12, [arg2+16*7]
+	pshufb	xmm9, xmm11
+	pshufb	xmm12, xmm11
+	movdqa	xmm8, xmm6
+	movdqa	xmm13, xmm7
+	pclmulqdq	xmm6, xmm10, 0x0
+	pclmulqdq	xmm8, xmm10 , 0x11
+	pclmulqdq	xmm7, xmm10, 0x0
+	pclmulqdq	xmm13, xmm10 , 0x11
+	pxor	xmm6, xmm9
+	xorps	xmm6, xmm8
+	pxor	xmm7, xmm12
+	xorps	xmm7, xmm13
+
+	sub	arg3, 128
+
+	; check if there is another 128B in the buffer to be able to fold
+        cmp     arg3, (fetch_dist + 128)
+	jge	_fold_and_prefetch_128_B_loop
+%endif ; fetch_dist != 0
+
+	; fold 128B at a time. This section of the code folds 8 xmm registers in parallel
+align 16
+_fold_128_B_loop:
+	; update the buffer pointer
+	add	arg2, 128		;    buf += 128;
+
+	movdqu	xmm9, [arg2+16*0]
+	movdqu	xmm12, [arg2+16*1]
+	pshufb	xmm9, xmm11
+	pshufb	xmm12, xmm11
+	movdqa	xmm8, xmm0
+	movdqa	xmm13, xmm1
+	pclmulqdq	xmm0, xmm10, 0x0
+	pclmulqdq	xmm8, xmm10 , 0x11
+	pclmulqdq	xmm1, xmm10, 0x0
+	pclmulqdq	xmm13, xmm10 , 0x11
+	pxor	xmm0, xmm9
+	xorps	xmm0, xmm8
+	pxor	xmm1, xmm12
+	xorps	xmm1, xmm13
+
+	movdqu	xmm9, [arg2+16*2]
+	movdqu	xmm12, [arg2+16*3]
+	pshufb	xmm9, xmm11
+	pshufb	xmm12, xmm11
+	movdqa	xmm8, xmm2
+	movdqa	xmm13, xmm3
+	pclmulqdq	xmm2, xmm10, 0x0
+	pclmulqdq	xmm8, xmm10 , 0x11
+	pclmulqdq	xmm3, xmm10, 0x0
+	pclmulqdq	xmm13, xmm10 , 0x11
+	pxor	xmm2, xmm9
+	xorps	xmm2, xmm8
+	pxor	xmm3, xmm12
+	xorps	xmm3, xmm13
+
+	movdqu	xmm9, [arg2+16*4]
+	movdqu	xmm12, [arg2+16*5]
+	pshufb	xmm9, xmm11
+	pshufb	xmm12, xmm11
+	movdqa	xmm8, xmm4
+	movdqa	xmm13, xmm5
+	pclmulqdq	xmm4, xmm10, 0x0
+	pclmulqdq	xmm8, xmm10 , 0x11
+	pclmulqdq	xmm5, xmm10, 0x0
+	pclmulqdq	xmm13, xmm10 , 0x11
+	pxor	xmm4, xmm9
+	xorps	xmm4, xmm8
+	pxor	xmm5, xmm12
+	xorps	xmm5, xmm13
+
 	movdqu	xmm9, [arg2+16*6]
 	movdqu	xmm12, [arg2+16*7]
 	pshufb	xmm9, xmm11

crc/crc16_t10dif_02.asm

@@ -148,10 +148,14 @@ crc16_t10dif_02:
 	; at this section of the code, there is 128*x+y (0<=y<128) bytes of buffer. The _fold_128_B_loop
 	; loop will fold 128B at a time until we have 128+y Bytes of buffer
 
+%if fetch_dist != 0
+	; check if there is at least 4kb (fetch distance) + 128b in the buffer
+        cmp     arg3, (fetch_dist + 128)
+        jb     _fold_128_B_loop
 
 	; fold 128B at a time. This section of the code folds 8 xmm registers in parallel
-_fold_128_B_loop:
-
+align 16
+_fold_and_prefetch_128_B_loop:
 	; update the buffer pointer
 	add	arg2, 128		;    buf += 128;
 
@@ -171,7 +175,6 @@ _fold_128_B_loop:
 	vpxor	xmm1, xmm12
 	vxorps	xmm1, xmm13
 
-	PREFETCH [arg2+fetch_dist+32]
 	vmovdqu	xmm9, [arg2+16*2]
 	vmovdqu	xmm12, [arg2+16*3]
 	vpshufb	xmm9, xmm11
@@ -203,7 +206,81 @@ _fold_128_B_loop:
 	vpxor	xmm5, xmm12
 	vxorps	xmm5, xmm13
 
-	PREFETCH [arg2+fetch_dist+96]
+	vmovdqu	xmm9, [arg2+16*6]
+	vmovdqu	xmm12, [arg2+16*7]
+	vpshufb	xmm9, xmm11
+	vpshufb	xmm12, xmm11
+	vmovdqa	xmm8, xmm6
+	vmovdqa	xmm13, xmm7
+	vpclmulqdq	xmm6, xmm10, 0x0
+	vpclmulqdq	xmm8, xmm10 , 0x11
+	vpclmulqdq	xmm7, xmm10, 0x0
+	vpclmulqdq	xmm13, xmm10 , 0x11
+	vpxor	xmm6, xmm9
+	vxorps	xmm6, xmm8
+	vpxor	xmm7, xmm12
+	vxorps	xmm7, xmm13
+
+	sub	arg3, 128
+
+	; check if there is another 128B in the buffer to be able to fold
+        cmp     arg3, (fetch_dist + 128)
+	jge	_fold_and_prefetch_128_B_loop
+%endif ; fetch_dist != 0
+	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+	; fold 128B at a time. This section of the code folds 8 xmm registers in parallel
+align 16
+_fold_128_B_loop:
+
+	; update the buffer pointer
+	add	arg2, 128		;    buf += 128;
+
+	vmovdqu	xmm9, [arg2+16*0]
+	vmovdqu	xmm12, [arg2+16*1]
+	vpshufb	xmm9, xmm11
+	vpshufb	xmm12, xmm11
+	vmovdqa	xmm8, xmm0
+	vmovdqa	xmm13, xmm1
+	vpclmulqdq	xmm0, xmm10, 0x0
+	vpclmulqdq	xmm8, xmm10 , 0x11
+	vpclmulqdq	xmm1, xmm10, 0x0
+	vpclmulqdq	xmm13, xmm10 , 0x11
+	vpxor	xmm0, xmm9
+	vxorps	xmm0, xmm8
+	vpxor	xmm1, xmm12
+	vxorps	xmm1, xmm13
+
+	vmovdqu	xmm9, [arg2+16*2]
+	vmovdqu	xmm12, [arg2+16*3]
+	vpshufb	xmm9, xmm11
+	vpshufb	xmm12, xmm11
+	vmovdqa	xmm8, xmm2
+	vmovdqa	xmm13, xmm3
+	vpclmulqdq	xmm2, xmm10, 0x0
+	vpclmulqdq	xmm8, xmm10 , 0x11
+	vpclmulqdq	xmm3, xmm10, 0x0
+	vpclmulqdq	xmm13, xmm10 , 0x11
+	vpxor	xmm2, xmm9
+	vxorps	xmm2, xmm8
+	vpxor	xmm3, xmm12
+	vxorps	xmm3, xmm13
+
+	vmovdqu	xmm9, [arg2+16*4]
+	vmovdqu	xmm12, [arg2+16*5]
+	vpshufb	xmm9, xmm11
+	vpshufb	xmm12, xmm11
+	vmovdqa	xmm8, xmm4
+	vmovdqa	xmm13, xmm5
+	vpclmulqdq	xmm4, xmm10, 0x0
+	vpclmulqdq	xmm8, xmm10 , 0x11
+	vpclmulqdq	xmm5, xmm10, 0x0
+	vpclmulqdq	xmm13, xmm10 , 0x11
+	vpxor	xmm4, xmm9
+	vxorps	xmm4, xmm8
+	vpxor	xmm5, xmm12
+	vxorps	xmm5, xmm13
+
 	vmovdqu	xmm9, [arg2+16*6]
 	vmovdqu	xmm12, [arg2+16*7]
 	vpshufb	xmm9, xmm11

crc/crc16_t10dif_by4.asm

@@ -130,10 +130,15 @@ crc16_t10dif_by4:
 	; buffer. The _fold_64_B_loop
 	; loop will fold 64B at a time until we have 64+y Bytes of buffer
 
+%if fetch_dist != 0
+	; check if there is at least 4KB (fetch distance) + 64B in the buffer
+        cmp     arg3, (fetch_dist + 64)
+        jb     _fold_64_B_loop
 
 	; fold 64B at a time. This section of the code folds 4 xmm
 	; registers in parallel
-_fold_64_B_loop:
+align 16
+_fold_and_prefetch_64_B_loop:
 
 	; update the buffer pointer
 	add	arg2, 64		;    buf += 64;
@@ -151,7 +156,61 @@ _fold_64_B_loop:
 	pxor	xmm0, xmm4
    	pxor	xmm1, xmm5
 
-	PREFETCH [arg2+fetch_dist+32]
+	movdqu	xmm4, xmm2
+	movdqu	xmm5, xmm3
+
+	pclmulqdq	xmm2, xmm6, 0x11
+	pclmulqdq	xmm3, xmm6, 0x11
+
+	pclmulqdq	xmm4, xmm6, 0x0
+	pclmulqdq	xmm5, xmm6, 0x0
+
+	pxor	xmm2, xmm4
+	pxor	xmm3, xmm5
+
+	movdqu	xmm4, [arg2]
+	movdqu	xmm5, [arg2+16]
+	pshufb	xmm4, xmm7
+	pshufb	xmm5, xmm7
+	pxor	xmm0, xmm4
+	pxor	xmm1, xmm5
+
+	movdqu	xmm4, [arg2+32]
+	movdqu	xmm5, [arg2+48]
+	pshufb	xmm4, xmm7
+	pshufb	xmm5, xmm7
+
+	pxor	xmm2, xmm4
+	pxor	xmm3, xmm5
+
+	sub	arg3, 64
+
+	; check if there is another 64B in the buffer to be able to fold
+        cmp     arg3, (fetch_dist + 64)
+	jge	_fold_and_prefetch_64_B_loop
+%endif ; fetch_dist != 0
+
+	; fold 64B at a time. This section of the code folds 4 xmm
+	; registers in parallel
+
+align 16
+_fold_64_B_loop:
+
+	; update the buffer pointer
+	add	arg2, 64		;    buf += 64;
+
+	movdqu	xmm4, xmm0
+	movdqu	xmm5, xmm1
+
+	pclmulqdq	xmm0, xmm6 , 0x11
+	pclmulqdq	xmm1, xmm6 , 0x11
+
+	pclmulqdq	xmm4, xmm6, 0x0
+	pclmulqdq	xmm5, xmm6, 0x0
+
+	pxor	xmm0, xmm4
+   	pxor	xmm1, xmm5
+
 	movdqu	xmm4, xmm2
 	movdqu	xmm5, xmm3
 

crc/crc16_t10dif_copy_by4.asm

@@ -139,10 +139,15 @@ crc16_t10dif_copy_by4:
 	; buffer. The _fold_64_B_loop
 	; loop will fold 64B at a time until we have 64+y Bytes of buffer
 
+%if fetch_dist != 0
+	; check if there is at least 4KB (fetch distance) + 64B in the buffer
+        cmp     arg4, (fetch_dist + 64)
+        jb     _fold_64_B_loop
 
 	; fold 64B at a time. This section of the code folds 4 xmm
 	; registers in parallel
-_fold_64_B_loop:
+align 16
+_fold_and_prefetch_64_B_loop:
 
 	; update the buffer pointer
 	add	arg3, 64		;    buf += 64;
@@ -161,7 +166,67 @@ _fold_64_B_loop:
 	pxor	xmm0, xmm4
 	pxor	xmm1, xmm5
 
-	PREFETCH [arg3+fetch_dist+32]
+	movdqu	xmm4, xmm2
+	movdqu	xmm5, xmm3
+
+	pclmulqdq	xmm2, xmm6, 0x11
+	pclmulqdq	xmm3, xmm6, 0x11
+
+	pclmulqdq	xmm4, xmm6, 0x0
+	pclmulqdq	xmm5, xmm6, 0x0
+
+	pxor	xmm2, xmm4
+	pxor	xmm3, xmm5
+
+	movdqu	xmm4, [arg3]
+	movdqu	xmm5, [arg3+16]
+	movdqu	[arg2], xmm4
+	movdqu	[arg2+16], xmm5
+	pshufb	xmm4, xmm7
+	pshufb	xmm5, xmm7
+	pxor	xmm0, xmm4
+	pxor	xmm1, xmm5
+
+	movdqu	xmm4, [arg3+32]
+	movdqu	xmm5, [arg3+48]
+	movdqu	[arg2+32], xmm4
+	movdqu	[arg2+48], xmm5
+	pshufb	xmm4, xmm7
+	pshufb	xmm5, xmm7
+
+	pxor	xmm2, xmm4
+	pxor	xmm3, xmm5
+
+	sub	arg4, 64
+
+	; check if there is another 4KB (fetch distance) + 64B in the buffer
+        cmp     arg4, (fetch_dist + 64)
+	jge	_fold_and_prefetch_64_B_loop
+	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+%endif ; fetch_dist != 0
+
+	; fold 64B at a time. This section of the code folds 4 xmm
+	; registers in parallel
+align 16
+_fold_64_B_loop:
+
+	; update the buffer pointer
+	add	arg3, 64		;    buf += 64;
+	add	arg2, 64
+
+	movdqu	xmm4, xmm0
+	movdqu	xmm5, xmm1
+
+	pclmulqdq	xmm0, xmm6 , 0x11
+	pclmulqdq	xmm1, xmm6 , 0x11
+
+	pclmulqdq	xmm4, xmm6, 0x0
+	pclmulqdq	xmm5, xmm6, 0x0
+
+	pxor	xmm0, xmm4
+	pxor	xmm1, xmm5
+
 	movdqu	xmm4, xmm2
 	movdqu	xmm5, xmm3