Skip to content

Commit 97c5cd7

Browse files
author
Jatin Bhateja
committed
8278508: Enable X86 maskAll instruction pattern for 32 bit JVM.
Reviewed-by: kvn, sviswanathan
1 parent 9ee3ccf commit 97c5cd7

File tree

10 files changed

+119
-56
lines changed

10 files changed

+119
-56
lines changed

‎src/hotspot/cpu/x86/assembler_x86.cpp‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2788,6 +2788,15 @@ void Assembler::kshiftlbl(KRegister dst, KRegister src, int imm8) {
27882788
emit_int8(imm8);
27892789
}
27902790

2791+
void Assembler::kshiftlql(KRegister dst, KRegister src, int imm8) {
2792+
assert(VM_Version::supports_avx512bw(), "");
2793+
InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2794+
int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
2795+
emit_int16(0x33, (0xC0 | encode));
2796+
emit_int8(imm8);
2797+
}
2798+
2799+
27912800
void Assembler::kshiftrbl(KRegister dst, KRegister src, int imm8) {
27922801
assert(VM_Version::supports_avx512dq(), "");
27932802
InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
@@ -2819,6 +2828,13 @@ void Assembler::kshiftrql(KRegister dst, KRegister src, int imm8) {
28192828
emit_int8(imm8);
28202829
}
28212830

2831+
void Assembler::kunpckdql(KRegister dst, KRegister src1, KRegister src2) {
2832+
assert(VM_Version::supports_avx512bw(), "");
2833+
InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2834+
int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2835+
emit_int16(0x4B, (0xC0 | encode));
2836+
}
2837+
28222838
void Assembler::movb(Address dst, int imm8) {
28232839
InstructionMark im(this);
28242840
prefix(dst);

‎src/hotspot/cpu/x86/assembler_x86.hpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1510,12 +1510,15 @@ class Assembler : public AbstractAssembler {
15101510

15111511
void kxnorbl(KRegister dst, KRegister src1, KRegister src2);
15121512
void kshiftlbl(KRegister dst, KRegister src, int imm8);
1513+
void kshiftlql(KRegister dst, KRegister src, int imm8);
15131514
void kshiftrbl(KRegister dst, KRegister src, int imm8);
15141515
void kshiftrwl(KRegister dst, KRegister src, int imm8);
15151516
void kshiftrdl(KRegister dst, KRegister src, int imm8);
15161517
void kshiftrql(KRegister dst, KRegister src, int imm8);
15171518
void ktestq(KRegister src1, KRegister src2);
15181519
void ktestd(KRegister src1, KRegister src2);
1520+
void kunpckdql(KRegister dst, KRegister src1, KRegister src2);
1521+
15191522

15201523
void ktestql(KRegister dst, KRegister src);
15211524
void ktestdl(KRegister dst, KRegister src);

‎src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4273,3 +4273,30 @@ void C2_MacroAssembler::vector_mask_operation(int opc, Register dst, XMMRegister
42734273
vector_mask_operation_helper(opc, dst, tmp, masklen);
42744274
}
42754275
#endif
4276+
4277+
void C2_MacroAssembler::vector_maskall_operation(KRegister dst, Register src, int mask_len) {
4278+
if (VM_Version::supports_avx512bw()) {
4279+
if (mask_len > 32) {
4280+
kmovql(dst, src);
4281+
} else {
4282+
kmovdl(dst, src);
4283+
if (mask_len != 32) {
4284+
kshiftrdl(dst, dst, 32 - mask_len);
4285+
}
4286+
}
4287+
} else {
4288+
assert(mask_len <= 16, "");
4289+
kmovwl(dst, src);
4290+
if (mask_len != 16) {
4291+
kshiftrwl(dst, dst, 16 - mask_len);
4292+
}
4293+
}
4294+
}
4295+
4296+
#ifndef _LP64
4297+
void C2_MacroAssembler::vector_maskall_operation32(KRegister dst, Register src, KRegister tmp, int mask_len) {
4298+
assert(VM_Version::supports_avx512bw(), "");
4299+
kmovdl(tmp, src);
4300+
kunpckdql(dst, tmp, tmp);
4301+
}
4302+
#endif

‎src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,13 @@
231231
void vector_mask_operation(int opc, Register dst, XMMRegister mask, XMMRegister xtmp,
232232
Register tmp, int masklen, BasicType bt, int vec_enc);
233233
#endif
234+
235+
void vector_maskall_operation(KRegister dst, Register src, int mask_len);
236+
237+
#ifndef _LP64
238+
void vector_maskall_operation32(KRegister dst, Register src, KRegister ktmp, int mask_len);
239+
#endif
240+
234241
void string_indexof_char(Register str1, Register cnt1, Register ch, Register result,
235242
XMMRegister vec1, XMMRegister vec2, XMMRegister vec3, Register tmp);
236243

‎src/hotspot/cpu/x86/x86.ad‎

Lines changed: 7 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -1827,7 +1827,7 @@ const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType
18271827
}
18281828
break;
18291829
case Op_MaskAll:
1830-
if (!is_LP64 || !VM_Version::supports_evex()) {
1830+
if (!VM_Version::supports_evex()) {
18311831
return false;
18321832
}
18331833
if ((vlen > 16 || is_subword_type(bt)) && !VM_Version::supports_avx512bw()) {
@@ -9452,64 +9452,18 @@ instruct evcmp_masked(kReg dst, vec src1, vec src2, immI8 cond, kReg mask, rRegP
94529452
ins_pipe( pipe_slow );
94539453
%}
94549454

9455-
#ifdef _LP64
9456-
instruct mask_all_evexI_imm(kReg dst, immI cnt, rRegL tmp) %{
9457-
match(Set dst (MaskAll cnt));
9458-
effect(TEMP_DEF dst, TEMP tmp);
9459-
format %{ "mask_all_evexI $dst, $cnt \t! using $tmp as TEMP" %}
9460-
ins_encode %{
9461-
int vec_len = Matcher::vector_length(this);
9462-
if (VM_Version::supports_avx512bw()) {
9463-
__ movq($tmp$$Register, $cnt$$constant);
9464-
__ kmovql($dst$$KRegister, $tmp$$Register);
9465-
__ kshiftrql($dst$$KRegister, $dst$$KRegister, 64 - vec_len);
9466-
} else {
9467-
assert(vec_len <= 16, "");
9468-
__ movq($tmp$$Register, $cnt$$constant);
9469-
__ kmovwl($dst$$KRegister, $tmp$$Register);
9470-
__ kshiftrwl($dst$$KRegister, $dst$$KRegister, 16 - vec_len);
9471-
}
9472-
%}
9473-
ins_pipe( pipe_slow );
9474-
%}
9475-
9476-
instruct mask_all_evexI(kReg dst, rRegI src, rRegL tmp) %{
9477-
match(Set dst (MaskAll src));
9478-
effect(TEMP_DEF dst, TEMP tmp);
9479-
format %{ "mask_all_evexI $dst, $src \t! using $tmp as TEMP" %}
9480-
ins_encode %{
9481-
int vec_len = Matcher::vector_length(this);
9482-
if (VM_Version::supports_avx512bw()) {
9483-
__ movslq($tmp$$Register, $src$$Register);
9484-
__ kmovql($dst$$KRegister, $tmp$$Register);
9485-
__ kshiftrql($dst$$KRegister, $dst$$KRegister, 64 - vec_len);
9486-
} else {
9487-
assert(vec_len <= 16, "");
9488-
__ kmovwl($dst$$KRegister, $src$$Register);
9489-
__ kshiftrwl($dst$$KRegister, $dst$$KRegister, 16 - vec_len);
9490-
}
9491-
%}
9492-
ins_pipe( pipe_slow );
9493-
%}
9494-
9495-
instruct mask_all_evexL(kReg dst, rRegL src) %{
9455+
instruct mask_all_evexI_LE32(kReg dst, rRegI src) %{
9456+
predicate(Matcher::vector_length(n) <= 32);
94969457
match(Set dst (MaskAll src));
9497-
effect(TEMP_DEF dst);
9498-
format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
9458+
format %{ "mask_all_evexI_LE32 $dst, $src \t" %}
94999459
ins_encode %{
9500-
int vec_len = Matcher::vector_length(this);
9501-
if (VM_Version::supports_avx512bw()) {
9502-
__ kmovql($dst$$KRegister, $src$$Register);
9503-
__ kshiftrql($dst$$KRegister, $dst$$KRegister, 64 - vec_len);
9504-
} else {
9505-
assert(vec_len <= 16, "");
9506-
__ kmovwl($dst$$KRegister, $src$$Register);
9507-
__ kshiftrwl($dst$$KRegister, $dst$$KRegister, 16 - vec_len);
9508-
}
9460+
int mask_len = Matcher::vector_length(this);
9461+
__ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
95099462
%}
95109463
ins_pipe( pipe_slow );
95119464
%}
95129465

9466+
#ifdef _LP64
95139467
instruct mask_not_immLT8(kReg dst, kReg src, rRegI rtmp, kReg ktmp, immI_M1 cnt) %{
95149468
predicate(Matcher::vector_length(n) < 8 && VM_Version::supports_avx512dq());
95159469
match(Set dst (XorVMask src (MaskAll cnt)));

‎src/hotspot/cpu/x86/x86_32.ad‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13847,7 +13847,40 @@ instruct cmpFastUnlock(eFlagsReg cr, eRegP object, eAXRegP box, eRegP tmp ) %{
1384713847
ins_pipe(pipe_slow);
1384813848
%}
1384913849

13850+
instruct mask_all_evexL_LT32(kReg dst, eRegL src) %{
13851+
predicate(Matcher::vector_length(n) <= 32);
13852+
match(Set dst (MaskAll src));
13853+
format %{ "mask_all_evexL_LE32 $dst, $src \t" %}
13854+
ins_encode %{
13855+
int mask_len = Matcher::vector_length(this);
13856+
__ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
13857+
%}
13858+
ins_pipe( pipe_slow );
13859+
%}
1385013860

13861+
instruct mask_all_evexL_GT32(kReg dst, eRegL src, kReg ktmp) %{
13862+
predicate(Matcher::vector_length(n) > 32);
13863+
match(Set dst (MaskAll src));
13864+
effect(TEMP ktmp);
13865+
format %{ "mask_all_evexL_GT32 $dst, $src \t! using $ktmp as TEMP " %}
13866+
ins_encode %{
13867+
int mask_len = Matcher::vector_length(this);
13868+
__ vector_maskall_operation32($dst$$KRegister, $src$$Register, $ktmp$$KRegister, mask_len);
13869+
%}
13870+
ins_pipe( pipe_slow );
13871+
%}
13872+
13873+
instruct mask_all_evexI_GT32(kReg dst, rRegI src, kReg ktmp) %{
13874+
predicate(Matcher::vector_length(n) > 32);
13875+
match(Set dst (MaskAll src));
13876+
effect(TEMP ktmp);
13877+
format %{ "mask_all_evexI_GT32 $dst, $src \t! using $ktmp as TEMP" %}
13878+
ins_encode %{
13879+
int mask_len = Matcher::vector_length(this);
13880+
__ vector_maskall_operation32($dst$$KRegister, $src$$Register, $ktmp$$KRegister, mask_len);
13881+
%}
13882+
ins_pipe( pipe_slow );
13883+
%}
1385113884

1385213885
// ============================================================================
1385313886
// Safepoint Instruction

‎src/hotspot/cpu/x86/x86_64.ad‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13011,6 +13011,29 @@ instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
1301113011
ins_pipe(ialu_reg_mem);
1301213012
%}
1301313013

13014+
instruct mask_all_evexL(kReg dst, rRegL src) %{
13015+
match(Set dst (MaskAll src));
13016+
format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
13017+
ins_encode %{
13018+
int mask_len = Matcher::vector_length(this);
13019+
__ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
13020+
%}
13021+
ins_pipe( pipe_slow );
13022+
%}
13023+
13024+
instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
13025+
predicate(Matcher::vector_length(n) > 32);
13026+
match(Set dst (MaskAll src));
13027+
effect(TEMP tmp);
13028+
format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
13029+
ins_encode %{
13030+
int mask_len = Matcher::vector_length(this);
13031+
__ movslq($tmp$$Register, $src$$Register);
13032+
__ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
13033+
%}
13034+
ins_pipe( pipe_slow );
13035+
%}
13036+
1301413037
// ============================================================================
1301513038
// Procedure Call/Return Instructions
1301613039
// Call Java Static Instruction

‎test/jdk/jdk/incubator/vector/Byte512VectorTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
/*
2525
* @test
2626
* @modules jdk.incubator.vector
27-
* @run testng/othervm -ea -esa -Xbatch -XX:-TieredCompilation Byte512VectorTests
27+
* @run testng/othervm/timeout=240 -ea -esa -Xbatch -XX:-TieredCompilation Byte512VectorTests
2828
*/
2929

3030
// -- This file was mechanically generated: Do not edit! -- //

‎test/jdk/jdk/incubator/vector/ByteMaxVectorTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
/*
2525
* @test
2626
* @modules jdk.incubator.vector
27-
* @run testng/othervm -ea -esa -Xbatch -XX:-TieredCompilation ByteMaxVectorTests
27+
* @run testng/othervm/timeout=240 -ea -esa -Xbatch -XX:-TieredCompilation ByteMaxVectorTests
2828
*/
2929

3030
// -- This file was mechanically generated: Do not edit! -- //

‎test/jdk/jdk/incubator/vector/VectorReshapeTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@
3939
* @test
4040
* @modules jdk.incubator.vector
4141
* @modules java.base/jdk.internal.vm.annotation
42-
* @run testng/othervm --add-opens jdk.incubator.vector/jdk.incubator.vector=ALL-UNNAMED
42+
* @run testng/othervm/timeout=240 --add-opens jdk.incubator.vector/jdk.incubator.vector=ALL-UNNAMED
4343
* -XX:-TieredCompilation VectorReshapeTests
4444
*/
4545

0 commit comments

Comments
 (0)