mem: Move new mem_zero_detect function to avx2

New mem_zero_detect function will fail on avx only machines.

Change-Id: I3bca49bff886f9c130c89e8c74b31110e9bac76b
Signed-off-by: Greg Tucker <[email protected]>

Author: gbtucker

Makefile.nmake

@@ -159,6 +159,7 @@ objs = \
 	bin\igzip_set_long_icf_fg_04.obj \
 	bin\igzip_set_long_icf_fg_06.obj \
 	bin\mem_zero_detect_avx.obj \
+	bin\mem_zero_detect_avx2.obj \
 	bin\mem_zero_detect_sse.obj \
 	bin\mem_multibinary.obj
 

mem/Makefile.am

@@ -35,6 +35,7 @@ lsrc_base_aliases += mem/mem_zero_detect_base_aliases.c
 lsrc_ppc64le      += mem/mem_zero_detect_base_aliases.c
 
 lsrc_x86_64 += 	mem/mem_zero_detect_avx.asm \
+		mem/mem_zero_detect_avx2.asm \
 		mem/mem_zero_detect_sse.asm \
 		mem/mem_multibinary.asm
 

mem/mem_multibinary.asm

@@ -33,10 +33,11 @@
 default rel
 [bits 64]
 
+extern mem_zero_detect_avx2
 extern mem_zero_detect_avx
 extern mem_zero_detect_sse
 extern mem_zero_detect_base
 
 mbin_interface isal_zero_detect
 
-mbin_dispatch_init5 isal_zero_detect, mem_zero_detect_base, mem_zero_detect_sse, mem_zero_detect_avx, mem_zero_detect_avx
+mbin_dispatch_init5 isal_zero_detect, mem_zero_detect_base, mem_zero_detect_sse, mem_zero_detect_avx, mem_zero_detect_avx2

mem/mem_zero_detect_avx.asm

@@ -64,105 +64,126 @@
 
 %define src arg0
 %define	len arg1
-%define tmp0 arg2
-%define tmp1 arg3
+%define ptr arg2
+%define pos return
 
-%use smartalign
-ALIGNMODE P6
 default rel
 
 [bits 64]
 section .text
-align 32	; maximize mu-ops cache usage
+
+align 16
 mk_global  mem_zero_detect_avx, function
 func(mem_zero_detect_avx)
 	FUNC_SAVE
-	cmp	len, 127
-	jbe	.mem_z_small_block
-	; check the first 128 bytes
-	vpxor	xmm2, xmm2, xmm2
-	vmovdqu ymm0, [src]
-	vpor	ymm0, ymm0, [src+32]
-	vmovdqu	ymm1, [src+64]
-	vpor	ymm1, ymm1, [src+96]
-	vpor	ymm0, ymm0, ymm1
-	vpcmpeqb ymm0, ymm2, ymm0
-	vpmovmskb DWORD(tmp0), ymm0
-	not	DWORD(tmp0)
-	mov	DWORD(tmp1), DWORD(len)
-	and	DWORD(tmp1), 127
-	add	src, tmp1
-	xor	eax, eax
-	shr	len, 7	; len/128
-	test	len, len; break partial flag stall
-	setz	al	; if len < 128, eax != 0
-	add	eax, DWORD(tmp0) ; jump if (edx OR eax) !=0, use add for macrofusion
-	jnz .return
-	xor	eax, eax
+	mov	pos, 0
+	sub	len, 4*32
+	jle	.mem_z_small_block
 
-align 16
 .mem_z_loop:
-	vmovdqu	ymm0, [src]
-	vpor	ymm0, ymm0,[src+32]
-	vmovdqu	ymm1, [src+64]
-	vpor	ymm1, ymm1, [src+96]
-	add	src, 128
-	xor	DWORD(tmp1), DWORD(tmp1)
-	sub	len, 1
-	setz	BYTE(tmp1)
-	vpor	ymm0, ymm0, ymm1
-	vpcmpeqb ymm0, ymm2, ymm0
-	vpmovmskb DWORD(tmp0), ymm0
-	not	DWORD(tmp0)
-	add	DWORD(tmp1), DWORD(tmp0)
-	jz	.mem_z_loop
-
-.return:
-	xor	eax, eax
-	test	tmp0, tmp0
-	setnz	al
+	vmovdqu	ymm0, [src+pos]
+	vmovdqu	ymm1, [src+pos+1*32]
+	vmovdqu	ymm2, [src+pos+2*32]
+	vmovdqu	ymm3, [src+pos+3*32]
+	vptest	ymm0, ymm0
+	jnz	.return_fail
+	vptest	ymm1, ymm1
+	jnz	.return_fail
+	vptest	ymm2, ymm2
+	jnz	.return_fail
+	vptest	ymm3, ymm3
+	jnz	.return_fail
+	add	pos, 4*32
+	cmp	pos, len
+	jl	.mem_z_loop
+
+.mem_z_last_block:
+	vmovdqu	ymm0, [src+len]
+	vmovdqu	ymm1, [src+len+1*32]
+	vmovdqu	ymm2, [src+len+2*32]
+	vmovdqu	ymm3, [src+len+3*32]
+	vptest	ymm0, ymm0
+	jnz	.return_fail
+	vptest	ymm1, ymm1
+	jnz	.return_fail
+	vptest	ymm2, ymm2
+	jnz	.return_fail
+	vptest	ymm3, ymm3
+	jnz	.return_fail
+
+.return_pass:
+	mov	return, 0
 	FUNC_RESTORE
 	ret
 
 
-align 16
 .mem_z_small_block:
-	;len < 128
-	xor	DWORD(tmp0), DWORD(tmp0)
-	movzx	DWORD(tmp1), BYTE(len)
-	cmp	DWORD(len), 16
-	jb     .mem_z_small_check_zero
-	;17 < len < 128
-	shr	DWORD(len), 4
-	xor	eax, eax ; alignment
-.mem_z_small_block_loop:
-	xor	eax, eax
-	mov	tmp0, [src]
-	or	tmp0, [src+8]
-	sub	DWORD(len), 1
-	setz	al
-	add	src, 16
-	add	rax, tmp0
-	jz	.mem_z_small_block_loop
-
-	test	tmp0, tmp0
-	jnz	.return_small
-	movzx	DWORD(len), BYTE(tmp1)
-
-.mem_z_small_check_zero:
-	xor	DWORD(tmp0), DWORD(tmp0)
-	and	DWORD(len), 15
-	jz	.return_small
-.mem_z_small_byte_loop:
-	movzx	eax, byte [src]
-	add	src, 1
-	or	DWORD(tmp0), eax
-	sub	DWORD(len), 1
-	jnz	.mem_z_small_byte_loop
-.return_small:
-	xor	eax, eax
-	test	tmp0, tmp0
-	setnz	al
+	add	len, 4*32
+	cmp	len, 2*32
+	jl	.mem_z_lt64
+	vmovdqu	ymm0, [src]
+	vmovdqu	ymm1, [src+32]
+	vmovdqu	ymm2, [src+len-2*32]
+	vmovdqu	ymm3, [src+len-1*32]
+	vptest	ymm0, ymm0
+	jnz	.return_fail
+	vptest	ymm1, ymm1
+	jnz	.return_fail
+	vptest	ymm2, ymm2
+	jnz	.return_fail
+	vptest	ymm3, ymm3
+	jnz	.return_fail
+	jmp	.return_pass
+
+.mem_z_lt64:
+	cmp	len, 32
+	jl	.mem_z_lt32
+	vmovdqu	ymm0, [src]
+	vmovdqu	ymm1, [src+len-32]
+	vptest	ymm0, ymm0
+	jnz	.return_fail
+	vptest	ymm1, ymm1
+	jnz	.return_fail
+	jmp	.return_pass
+
+
+.mem_z_lt32:
+	cmp	len, 16
+	jl	.mem_z_lt16
+	vmovdqu	xmm0, [src]
+	vmovdqu	xmm1, [src+len-16]
+	vptest	xmm0, xmm0
+	jnz	.return_fail
+	vptest	xmm1, xmm1
+	jnz	.return_fail
+	jmp	.return_pass
+
+
+.mem_z_lt16:
+	cmp	len, 8
+	jl	.mem_z_lt8
+	mov	tmp, [src]
+	mov	tmp3,[src+len-8]
+	or	tmp, tmp3
+	test	tmp, tmp
+	jnz	.return_fail
+	jmp	.return_pass
+
+.mem_z_lt8:
+	cmp	len, 0
+	je	.return_pass
+.mem_z_1byte_loop:
+	mov	tmpb, [src+pos]
+	cmp	tmpb, 0
+	jnz	.return_fail
+	add	pos, 1
+	cmp	pos, len
+	jl	.mem_z_1byte_loop
+	jmp	.return_pass
+
+.return_fail:
+	mov	return, 1
+	FUNC_RESTORE
 	ret
 
 endproc_frame