Skip to content

Commit 6700baa

Browse files
author
Hamlin Li
committed
8357551: RISC-V: support CMoveF/D vectorization
Reviewed-by: fyang, luhenry
1 parent b83bf07 commit 6700baa

File tree

14 files changed

+3777
-316
lines changed

14 files changed

+3777
-316
lines changed

‎src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2067,6 +2067,83 @@ void C2_MacroAssembler::enc_cmove_cmp_fp(int cmpFlag, FloatRegister op1, FloatRe
20672067
}
20682068
}
20692069

2070+
void C2_MacroAssembler::enc_cmove_fp_cmp(int cmpFlag, Register op1, Register op2,
2071+
FloatRegister dst, FloatRegister src, bool is_single) {
2072+
bool is_unsigned = (cmpFlag & unsigned_branch_mask) == unsigned_branch_mask;
2073+
int op_select = cmpFlag & (~unsigned_branch_mask);
2074+
2075+
switch (op_select) {
2076+
case BoolTest::eq:
2077+
cmov_fp_eq(op1, op2, dst, src, is_single);
2078+
break;
2079+
case BoolTest::ne:
2080+
cmov_fp_ne(op1, op2, dst, src, is_single);
2081+
break;
2082+
case BoolTest::le:
2083+
if (is_unsigned) {
2084+
cmov_fp_leu(op1, op2, dst, src, is_single);
2085+
} else {
2086+
cmov_fp_le(op1, op2, dst, src, is_single);
2087+
}
2088+
break;
2089+
case BoolTest::ge:
2090+
if (is_unsigned) {
2091+
cmov_fp_geu(op1, op2, dst, src, is_single);
2092+
} else {
2093+
cmov_fp_ge(op1, op2, dst, src, is_single);
2094+
}
2095+
break;
2096+
case BoolTest::lt:
2097+
if (is_unsigned) {
2098+
cmov_fp_ltu(op1, op2, dst, src, is_single);
2099+
} else {
2100+
cmov_fp_lt(op1, op2, dst, src, is_single);
2101+
}
2102+
break;
2103+
case BoolTest::gt:
2104+
if (is_unsigned) {
2105+
cmov_fp_gtu(op1, op2, dst, src, is_single);
2106+
} else {
2107+
cmov_fp_gt(op1, op2, dst, src, is_single);
2108+
}
2109+
break;
2110+
default:
2111+
assert(false, "unsupported compare condition");
2112+
ShouldNotReachHere();
2113+
}
2114+
}
2115+
2116+
void C2_MacroAssembler::enc_cmove_fp_cmp_fp(int cmpFlag,
2117+
FloatRegister op1, FloatRegister op2,
2118+
FloatRegister dst, FloatRegister src,
2119+
bool cmp_single, bool cmov_single) {
2120+
int op_select = cmpFlag & (~unsigned_branch_mask);
2121+
2122+
switch (op_select) {
2123+
case BoolTest::eq:
2124+
cmov_fp_cmp_fp_eq(op1, op2, dst, src, cmp_single, cmov_single);
2125+
break;
2126+
case BoolTest::ne:
2127+
cmov_fp_cmp_fp_ne(op1, op2, dst, src, cmp_single, cmov_single);
2128+
break;
2129+
case BoolTest::le:
2130+
cmov_fp_cmp_fp_le(op1, op2, dst, src, cmp_single, cmov_single);
2131+
break;
2132+
case BoolTest::ge:
2133+
cmov_fp_cmp_fp_ge(op1, op2, dst, src, cmp_single, cmov_single);
2134+
break;
2135+
case BoolTest::lt:
2136+
cmov_fp_cmp_fp_lt(op1, op2, dst, src, cmp_single, cmov_single);
2137+
break;
2138+
case BoolTest::gt:
2139+
cmov_fp_cmp_fp_gt(op1, op2, dst, src, cmp_single, cmov_single);
2140+
break;
2141+
default:
2142+
assert(false, "unsupported compare condition");
2143+
ShouldNotReachHere();
2144+
}
2145+
}
2146+
20702147
// Set dst to NaN if any NaN input.
20712148
void C2_MacroAssembler::minmax_fp(FloatRegister dst, FloatRegister src1, FloatRegister src2,
20722149
FLOAT_TYPE ft, bool is_min) {

‎src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,13 @@
132132
FloatRegister op1, FloatRegister op2,
133133
Register dst, Register src, bool is_single);
134134

135+
void enc_cmove_fp_cmp(int cmpFlag, Register op1, Register op2,
136+
FloatRegister dst, FloatRegister src, bool is_single);
137+
138+
void enc_cmove_fp_cmp_fp(int cmpFlag, FloatRegister op1, FloatRegister op2,
139+
FloatRegister dst, FloatRegister src,
140+
bool cmp_single, bool cmov_single);
141+
135142
void spill(Register r, bool is64, int offset) {
136143
is64 ? sd(r, Address(sp, offset))
137144
: sw(r, Address(sp, offset));

‎src/hotspot/cpu/riscv/macroAssembler_riscv.cpp‎

Lines changed: 255 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1233,7 +1233,119 @@ void MacroAssembler::cmov_gtu(Register cmp1, Register cmp2, Register dst, Regist
12331233
bind(no_set);
12341234
}
12351235

1236-
// ----------- cmove, compare float -----------
1236+
// ----------- cmove float/double -----------
1237+
1238+
void MacroAssembler::cmov_fp_eq(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single) {
1239+
Label no_set;
1240+
bne(cmp1, cmp2, no_set);
1241+
if (is_single) {
1242+
fmv_s(dst, src);
1243+
} else {
1244+
fmv_d(dst, src);
1245+
}
1246+
bind(no_set);
1247+
}
1248+
1249+
void MacroAssembler::cmov_fp_ne(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single) {
1250+
Label no_set;
1251+
beq(cmp1, cmp2, no_set);
1252+
if (is_single) {
1253+
fmv_s(dst, src);
1254+
} else {
1255+
fmv_d(dst, src);
1256+
}
1257+
bind(no_set);
1258+
}
1259+
1260+
void MacroAssembler::cmov_fp_le(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single) {
1261+
Label no_set;
1262+
bgt(cmp1, cmp2, no_set);
1263+
if (is_single) {
1264+
fmv_s(dst, src);
1265+
} else {
1266+
fmv_d(dst, src);
1267+
}
1268+
bind(no_set);
1269+
}
1270+
1271+
void MacroAssembler::cmov_fp_leu(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single) {
1272+
Label no_set;
1273+
bgtu(cmp1, cmp2, no_set);
1274+
if (is_single) {
1275+
fmv_s(dst, src);
1276+
} else {
1277+
fmv_d(dst, src);
1278+
}
1279+
bind(no_set);
1280+
}
1281+
1282+
void MacroAssembler::cmov_fp_ge(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single) {
1283+
Label no_set;
1284+
blt(cmp1, cmp2, no_set);
1285+
if (is_single) {
1286+
fmv_s(dst, src);
1287+
} else {
1288+
fmv_d(dst, src);
1289+
}
1290+
bind(no_set);
1291+
}
1292+
1293+
void MacroAssembler::cmov_fp_geu(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single) {
1294+
Label no_set;
1295+
bltu(cmp1, cmp2, no_set);
1296+
if (is_single) {
1297+
fmv_s(dst, src);
1298+
} else {
1299+
fmv_d(dst, src);
1300+
}
1301+
bind(no_set);
1302+
}
1303+
1304+
void MacroAssembler::cmov_fp_lt(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single) {
1305+
Label no_set;
1306+
bge(cmp1, cmp2, no_set);
1307+
if (is_single) {
1308+
fmv_s(dst, src);
1309+
} else {
1310+
fmv_d(dst, src);
1311+
}
1312+
bind(no_set);
1313+
}
1314+
1315+
void MacroAssembler::cmov_fp_ltu(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single) {
1316+
Label no_set;
1317+
bgeu(cmp1, cmp2, no_set);
1318+
if (is_single) {
1319+
fmv_s(dst, src);
1320+
} else {
1321+
fmv_d(dst, src);
1322+
}
1323+
bind(no_set);
1324+
}
1325+
1326+
void MacroAssembler::cmov_fp_gt(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single) {
1327+
Label no_set;
1328+
ble(cmp1, cmp2, no_set);
1329+
if (is_single) {
1330+
fmv_s(dst, src);
1331+
} else {
1332+
fmv_d(dst, src);
1333+
}
1334+
bind(no_set);
1335+
}
1336+
1337+
void MacroAssembler::cmov_fp_gtu(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single) {
1338+
Label no_set;
1339+
bleu(cmp1, cmp2, no_set);
1340+
if (is_single) {
1341+
fmv_s(dst, src);
1342+
} else {
1343+
fmv_d(dst, src);
1344+
}
1345+
bind(no_set);
1346+
}
1347+
1348+
// ----------- cmove, compare float/double -----------
12371349
//
12381350
// For CmpF/D + CMoveI/L, ordered ones are quite straight and simple,
12391351
// so, just list behaviour of unordered ones as follow.
@@ -1391,6 +1503,148 @@ void MacroAssembler::cmov_cmp_fp_gt(FloatRegister cmp1, FloatRegister cmp2, Regi
13911503
bind(no_set);
13921504
}
13931505

1506+
// ----------- cmove float/double, compare float/double -----------
1507+
1508+
// Move src to dst only if cmp1 == cmp2,
1509+
// otherwise leave dst unchanged, including the case where one of them is NaN.
1510+
// Clarification:
1511+
// java code : cmp1 != cmp2 ? dst : src
1512+
// transformed to : CMove dst, (cmp1 eq cmp2), dst, src
1513+
void MacroAssembler::cmov_fp_cmp_fp_eq(FloatRegister cmp1, FloatRegister cmp2,
1514+
FloatRegister dst, FloatRegister src,
1515+
bool cmp_single, bool cmov_single) {
1516+
Label no_set;
1517+
if (cmp_single) {
1518+
// jump if cmp1 != cmp2, including the case of NaN
1519+
// not jump (i.e. move src to dst) if cmp1 == cmp2
1520+
float_bne(cmp1, cmp2, no_set);
1521+
} else {
1522+
double_bne(cmp1, cmp2, no_set);
1523+
}
1524+
if (cmov_single) {
1525+
fmv_s(dst, src);
1526+
} else {
1527+
fmv_d(dst, src);
1528+
}
1529+
bind(no_set);
1530+
}
1531+
1532+
// Keep dst unchanged only if cmp1 == cmp2,
1533+
// otherwise move src to dst, including the case where one of them is NaN.
1534+
// Clarification:
1535+
// java code : cmp1 == cmp2 ? dst : src
1536+
// transformed to : CMove dst, (cmp1 ne cmp2), dst, src
1537+
void MacroAssembler::cmov_fp_cmp_fp_ne(FloatRegister cmp1, FloatRegister cmp2,
1538+
FloatRegister dst, FloatRegister src,
1539+
bool cmp_single, bool cmov_single) {
1540+
Label no_set;
1541+
if (cmp_single) {
1542+
// jump if cmp1 == cmp2
1543+
// not jump (i.e. move src to dst) if cmp1 != cmp2, including the case of NaN
1544+
float_beq(cmp1, cmp2, no_set);
1545+
} else {
1546+
double_beq(cmp1, cmp2, no_set);
1547+
}
1548+
if (cmov_single) {
1549+
fmv_s(dst, src);
1550+
} else {
1551+
fmv_d(dst, src);
1552+
}
1553+
bind(no_set);
1554+
}
1555+
1556+
// When cmp1 <= cmp2 or any of them is NaN then dst = src, otherwise, dst = dst
1557+
// Clarification
1558+
// scenario 1:
1559+
// java code : cmp2 < cmp1 ? dst : src
1560+
// transformed to : CMove dst, (cmp1 le cmp2), dst, src
1561+
// scenario 2:
1562+
// java code : cmp1 > cmp2 ? dst : src
1563+
// transformed to : CMove dst, (cmp1 le cmp2), dst, src
1564+
void MacroAssembler::cmov_fp_cmp_fp_le(FloatRegister cmp1, FloatRegister cmp2,
1565+
FloatRegister dst, FloatRegister src,
1566+
bool cmp_single, bool cmov_single) {
1567+
Label no_set;
1568+
if (cmp_single) {
1569+
// jump if cmp1 > cmp2
1570+
// not jump (i.e. move src to dst) if cmp1 <= cmp2 or either is NaN
1571+
float_bgt(cmp1, cmp2, no_set);
1572+
} else {
1573+
double_bgt(cmp1, cmp2, no_set);
1574+
}
1575+
if (cmov_single) {
1576+
fmv_s(dst, src);
1577+
} else {
1578+
fmv_d(dst, src);
1579+
}
1580+
bind(no_set);
1581+
}
1582+
1583+
void MacroAssembler::cmov_fp_cmp_fp_ge(FloatRegister cmp1, FloatRegister cmp2,
1584+
FloatRegister dst, FloatRegister src,
1585+
bool cmp_single, bool cmov_single) {
1586+
Label no_set;
1587+
if (cmp_single) {
1588+
// jump if cmp1 < cmp2 or either is NaN
1589+
// not jump (i.e. move src to dst) if cmp1 >= cmp2
1590+
float_blt(cmp1, cmp2, no_set, false, true);
1591+
} else {
1592+
double_blt(cmp1, cmp2, no_set, false, true);
1593+
}
1594+
if (cmov_single) {
1595+
fmv_s(dst, src);
1596+
} else {
1597+
fmv_d(dst, src);
1598+
}
1599+
bind(no_set);
1600+
}
1601+
1602+
// When cmp1 < cmp2 or any of them is NaN then dst = src, otherwise, dst = dst
1603+
// Clarification
1604+
// scenario 1:
1605+
// java code : cmp2 <= cmp1 ? dst : src
1606+
// transformed to : CMove dst, (cmp1 lt cmp2), dst, src
1607+
// scenario 2:
1608+
// java code : cmp1 >= cmp2 ? dst : src
1609+
// transformed to : CMove dst, (cmp1 lt cmp2), dst, src
1610+
void MacroAssembler::cmov_fp_cmp_fp_lt(FloatRegister cmp1, FloatRegister cmp2,
1611+
FloatRegister dst, FloatRegister src,
1612+
bool cmp_single, bool cmov_single) {
1613+
Label no_set;
1614+
if (cmp_single) {
1615+
// jump if cmp1 >= cmp2
1616+
// not jump (i.e. move src to dst) if cmp1 < cmp2 or either is NaN
1617+
float_bge(cmp1, cmp2, no_set);
1618+
} else {
1619+
double_bge(cmp1, cmp2, no_set);
1620+
}
1621+
if (cmov_single) {
1622+
fmv_s(dst, src);
1623+
} else {
1624+
fmv_d(dst, src);
1625+
}
1626+
bind(no_set);
1627+
}
1628+
1629+
void MacroAssembler::cmov_fp_cmp_fp_gt(FloatRegister cmp1, FloatRegister cmp2,
1630+
FloatRegister dst, FloatRegister src,
1631+
bool cmp_single, bool cmov_single) {
1632+
Label no_set;
1633+
if (cmp_single) {
1634+
// jump if cmp1 <= cmp2 or either is NaN
1635+
// not jump (i.e. move src to dst) if cmp1 > cmp2
1636+
float_ble(cmp1, cmp2, no_set, false, true);
1637+
} else {
1638+
double_ble(cmp1, cmp2, no_set, false, true);
1639+
}
1640+
if (cmov_single) {
1641+
fmv_s(dst, src);
1642+
} else {
1643+
fmv_d(dst, src);
1644+
}
1645+
bind(no_set);
1646+
}
1647+
13941648
// Float compare branch instructions
13951649

13961650
#define INSN(NAME, FLOATCMP, BRANCH) \

‎src/hotspot/cpu/riscv/macroAssembler_riscv.hpp‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -665,6 +665,24 @@ class MacroAssembler: public Assembler {
665665
void cmov_cmp_fp_lt(FloatRegister cmp1, FloatRegister cmp2, Register dst, Register src, bool is_single);
666666
void cmov_cmp_fp_gt(FloatRegister cmp1, FloatRegister cmp2, Register dst, Register src, bool is_single);
667667

668+
void cmov_fp_eq(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single);
669+
void cmov_fp_ne(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single);
670+
void cmov_fp_le(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single);
671+
void cmov_fp_leu(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single);
672+
void cmov_fp_ge(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single);
673+
void cmov_fp_geu(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single);
674+
void cmov_fp_lt(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single);
675+
void cmov_fp_ltu(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single);
676+
void cmov_fp_gt(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single);
677+
void cmov_fp_gtu(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single);
678+
679+
void cmov_fp_cmp_fp_eq(FloatRegister cmp1, FloatRegister cmp2, FloatRegister dst, FloatRegister src, bool cmp_single, bool cmov_single);
680+
void cmov_fp_cmp_fp_ne(FloatRegister cmp1, FloatRegister cmp2, FloatRegister dst, FloatRegister src, bool cmp_single, bool cmov_single);
681+
void cmov_fp_cmp_fp_le(FloatRegister cmp1, FloatRegister cmp2, FloatRegister dst, FloatRegister src, bool cmp_single, bool cmov_single);
682+
void cmov_fp_cmp_fp_ge(FloatRegister cmp1, FloatRegister cmp2, FloatRegister dst, FloatRegister src, bool cmp_single, bool cmov_single);
683+
void cmov_fp_cmp_fp_lt(FloatRegister cmp1, FloatRegister cmp2, FloatRegister dst, FloatRegister src, bool cmp_single, bool cmov_single);
684+
void cmov_fp_cmp_fp_gt(FloatRegister cmp1, FloatRegister cmp2, FloatRegister dst, FloatRegister src, bool cmp_single, bool cmov_single);
685+
668686
public:
669687
// We try to follow risc-v asm menomics.
670688
// But as we don't layout a reachable GOT,

0 commit comments

Comments
 (0)