@@ -52,11 +52,11 @@ static void x_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node
5252%}
5353
5454// Load Pointer
55- instruct xLoadP(iRegPNoSp dst, memory mem)
55+ instruct xLoadP(iRegPNoSp dst, memory mem, iRegPNoSp tmp )
5656%{
5757 match(Set dst (LoadP mem));
5858 predicate(UseZGC && !ZGenerational && (n->as_Load()->barrier_data() != 0));
59- effect(TEMP dst);
59+ effect(TEMP dst, TEMP tmp );
6060
6161 ins_cost(4 * DEFAULT_COST);
6262
@@ -65,17 +65,17 @@ instruct xLoadP(iRegPNoSp dst, memory mem)
6565 ins_encode %{
6666 const Address ref_addr (as_Register($mem$$base), $mem$$disp);
6767 __ ld($dst$$Register, ref_addr);
68- x_load_barrier(_masm, this, ref_addr, $dst$$Register, t0 /* tmp */, barrier_data());
68+ x_load_barrier(_masm, this, ref_addr, $dst$$Register, $tmp$$Register /* tmp */, barrier_data());
6969 %}
7070
7171 ins_pipe(iload_reg_mem);
7272%}
7373
74- instruct xCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr ) %{
74+ instruct xCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp ) %{
7575 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
7676 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
7777 predicate(UseZGC && !ZGenerational && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong);
78- effect(KILL cr, TEMP_DEF res );
78+ effect(TEMP_DEF res, TEMP tmp );
7979
8080 ins_cost(2 * VOLATILE_REF_COST);
8181
@@ -86,17 +86,15 @@ instruct xCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newva
8686 Label failed;
8787 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
8888 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
89- Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register,
90- true /* result_as_bool */);
91- __ beqz($res$$Register, failed);
92- __ mv(t0, $oldval$$Register);
93- __ bind(failed);
89+ Assembler::relaxed /* acquire */, Assembler::rl /* release */, $tmp$$Register);
90+ __ sub(t0, $tmp$$Register, $oldval$$Register);
91+ __ seqz($res$$Register, t0);
9492 if (barrier_data() != XLoadBarrierElided) {
9593 Label good;
96- __ ld(t1 , Address(xthread, XThreadLocalData::address_bad_mask_offset()), t1 /* tmp */ );
97- __ andr(t1, t1, t0 );
98- __ beqz(t1 , good);
99- x_load_barrier_slow_path(_masm, this, Address($mem$$Register), t0 /* ref */, t1 /* tmp */);
94+ __ ld(t0 , Address(xthread, XThreadLocalData::address_bad_mask_offset()));
95+ __ andr(t0, t0, $tmp$$Register );
96+ __ beqz(t0 , good);
97+ x_load_barrier_slow_path(_masm, this, Address($mem$$Register), $tmp$$Register /* ref */, $res$$Register /* tmp */);
10098 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
10199 Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register,
102100 true /* result_as_bool */);
@@ -107,11 +105,11 @@ instruct xCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newva
107105 ins_pipe(pipe_slow);
108106%}
109107
110- instruct xCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr ) %{
108+ instruct xCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp ) %{
111109 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
112110 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
113111 predicate(UseZGC && !ZGenerational && needs_acquiring_load_reserved(n) && (n->as_LoadStore()->barrier_data() == XLoadBarrierStrong));
114- effect(KILL cr, TEMP_DEF res );
112+ effect(TEMP_DEF res, TEMP tmp );
115113
116114 ins_cost(2 * VOLATILE_REF_COST);
117115
@@ -122,17 +120,15 @@ instruct xCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP ne
122120 Label failed;
123121 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
124122 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
125- Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register,
126- true /* result_as_bool */);
127- __ beqz($res$$Register, failed);
128- __ mv(t0, $oldval$$Register);
129- __ bind(failed);
123+ Assembler::aq /* acquire */, Assembler::rl /* release */, $tmp$$Register);
124+ __ sub(t0, $tmp$$Register, $oldval$$Register);
125+ __ seqz($res$$Register, t0);
130126 if (barrier_data() != XLoadBarrierElided) {
131127 Label good;
132- __ ld(t1 , Address(xthread, XThreadLocalData::address_bad_mask_offset()), t1 /* tmp */ );
133- __ andr(t1, t1, t0 );
134- __ beqz(t1 , good);
135- x_load_barrier_slow_path(_masm, this, Address($mem$$Register), t0 /* ref */, t1 /* tmp */);
128+ __ ld(t0 , Address(xthread, XThreadLocalData::address_bad_mask_offset()));
129+ __ andr(t0, t0, $tmp$$Register );
130+ __ beqz(t0 , good);
131+ x_load_barrier_slow_path(_masm, this, Address($mem$$Register), $tmp$$Register /* ref */, $res$$Register /* tmp */);
136132 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
137133 Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register,
138134 true /* result_as_bool */);
@@ -143,10 +139,10 @@ instruct xCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP ne
143139 ins_pipe(pipe_slow);
144140%}
145141
146- instruct xCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval) %{
142+ instruct xCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp ) %{
147143 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
148144 predicate(UseZGC && !ZGenerational && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong);
149- effect(TEMP_DEF res);
145+ effect(TEMP_DEF res, TEMP tmp );
150146
151147 ins_cost(2 * VOLATILE_REF_COST);
152148
@@ -161,7 +157,7 @@ instruct xCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP n
161157 __ ld(t0, Address(xthread, XThreadLocalData::address_bad_mask_offset()));
162158 __ andr(t0, t0, $res$$Register);
163159 __ beqz(t0, good);
164- x_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, t0 /* tmp */);
160+ x_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, $tmp$$Register /* tmp */);
165161 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
166162 Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register);
167163 __ bind(good);
@@ -171,10 +167,10 @@ instruct xCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP n
171167 ins_pipe(pipe_slow);
172168%}
173169
174- instruct xCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval) %{
170+ instruct xCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp ) %{
175171 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
176172 predicate(UseZGC && !ZGenerational && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong);
177- effect(TEMP_DEF res);
173+ effect(TEMP_DEF res, TEMP tmp );
178174
179175 ins_cost(2 * VOLATILE_REF_COST);
180176
@@ -189,7 +185,7 @@ instruct xCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iReg
189185 __ ld(t0, Address(xthread, XThreadLocalData::address_bad_mask_offset()));
190186 __ andr(t0, t0, $res$$Register);
191187 __ beqz(t0, good);
192- x_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, t0 /* tmp */);
188+ x_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, $tmp$$Register /* tmp */);
193189 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
194190 Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register);
195191 __ bind(good);
@@ -199,35 +195,35 @@ instruct xCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iReg
199195 ins_pipe(pipe_slow);
200196%}
201197
202- instruct xGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr ) %{
198+ instruct xGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, iRegPNoSp tmp ) %{
203199 match(Set prev (GetAndSetP mem newv));
204200 predicate(UseZGC && !ZGenerational && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
205- effect(TEMP_DEF prev, KILL cr );
201+ effect(TEMP_DEF prev, TEMP tmp );
206202
207203 ins_cost(2 * VOLATILE_REF_COST);
208204
209205 format %{ "atomic_xchg $prev, $newv, [$mem], #@zGetAndSetP" %}
210206
211207 ins_encode %{
212208 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
213- x_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, t0 /* tmp */, barrier_data());
209+ x_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, $tmp$$Register /* tmp */, barrier_data());
214210 %}
215211
216212 ins_pipe(pipe_serial);
217213%}
218214
219- instruct xGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr ) %{
215+ instruct xGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, iRegPNoSp tmp ) %{
220216 match(Set prev (GetAndSetP mem newv));
221217 predicate(UseZGC && !ZGenerational && needs_acquiring_load_reserved(n) && (n->as_LoadStore()->barrier_data() != 0));
222- effect(TEMP_DEF prev, KILL cr );
218+ effect(TEMP_DEF prev, TEMP tmp );
223219
224220 ins_cost(VOLATILE_REF_COST);
225221
226222 format %{ "atomic_xchg_acq $prev, $newv, [$mem], #@zGetAndSetPAcq" %}
227223
228224 ins_encode %{
229225 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
230- x_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, t0 /* tmp */, barrier_data());
226+ x_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, $tmp$$Register /* tmp */, barrier_data());
231227 %}
232228 ins_pipe(pipe_serial);
233229%}
0 commit comments