Skip to content

Commit ba658a7

Browse files
author
Xiaohong Gong
committed
8349522: AArch64: Add backend implementation for new unsigned and saturating vector operations
Reviewed-by: epeter, haosun, bkilambi
1 parent 5625b43 commit ba658a7

File tree

8 files changed

+1146
-477
lines changed

8 files changed

+1146
-477
lines changed

‎src/hotspot/cpu/aarch64/aarch64_vector.ad

Lines changed: 284 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
//
2-
// Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
2+
// Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
33
// Copyright (c) 2020, 2024, Arm Limited. All rights reserved.
44
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
55
//
@@ -254,6 +254,13 @@ source %{
254254
case Op_CompressBitsV:
255255
case Op_ExpandBitsV:
256256
return false;
257+
case Op_SaturatingAddV:
258+
case Op_SaturatingSubV:
259+
// Only SVE2 supports the predicated saturating instructions.
260+
if (UseSVE < 2) {
261+
return false;
262+
}
263+
break;
257264
// We use Op_LoadVectorMasked to implement the predicated Op_LoadVector.
258265
// Hence we turn to check whether Op_LoadVectorMasked is supported. The
259266
// same as vector store/gather/scatter.
@@ -1539,6 +1546,142 @@ instruct vand_notL_masked(vReg dst_src1, vReg src2, immL_M1 m1, pRegGov pg) %{
15391546
ins_pipe(pipe_slow);
15401547
%}
15411548

1549+
// ------------------------- Vector saturating add -----------------------------
1550+
1551+
// Signed saturating add
1552+
1553+
instruct vsqadd(vReg dst, vReg src1, vReg src2) %{
1554+
predicate(!n->as_SaturatingVector()->is_unsigned());
1555+
match(Set dst (SaturatingAddV src1 src2));
1556+
format %{ "vsqadd $dst, $src1, $src2" %}
1557+
ins_encode %{
1558+
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
1559+
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
1560+
__ sqaddv($dst$$FloatRegister, get_arrangement(this),
1561+
$src1$$FloatRegister, $src2$$FloatRegister);
1562+
} else {
1563+
assert(UseSVE > 0, "must be sve");
1564+
BasicType bt = Matcher::vector_element_basic_type(this);
1565+
__ sve_sqadd($dst$$FloatRegister, __ elemType_to_regVariant(bt),
1566+
$src1$$FloatRegister, $src2$$FloatRegister);
1567+
}
1568+
%}
1569+
ins_pipe(pipe_slow);
1570+
%}
1571+
1572+
instruct vsqadd_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1573+
predicate(UseSVE == 2 && !n->as_SaturatingVector()->is_unsigned());
1574+
match(Set dst_src1 (SaturatingAddV (Binary dst_src1 src2) pg));
1575+
format %{ "vsqadd_masked $dst_src1, $pg, $dst_src1, $src2" %}
1576+
ins_encode %{
1577+
BasicType bt = Matcher::vector_element_basic_type(this);
1578+
__ sve_sqadd($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
1579+
$pg$$PRegister, $src2$$FloatRegister);
1580+
%}
1581+
ins_pipe(pipe_slow);
1582+
%}
1583+
1584+
// Unsigned saturating add
1585+
1586+
instruct vuqadd(vReg dst, vReg src1, vReg src2) %{
1587+
predicate(n->as_SaturatingVector()->is_unsigned());
1588+
match(Set dst (SaturatingAddV src1 src2));
1589+
format %{ "vuqadd $dst, $src1, $src2" %}
1590+
ins_encode %{
1591+
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
1592+
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
1593+
__ uqaddv($dst$$FloatRegister, get_arrangement(this),
1594+
$src1$$FloatRegister, $src2$$FloatRegister);
1595+
} else {
1596+
assert(UseSVE > 0, "must be sve");
1597+
BasicType bt = Matcher::vector_element_basic_type(this);
1598+
__ sve_uqadd($dst$$FloatRegister, __ elemType_to_regVariant(bt),
1599+
$src1$$FloatRegister, $src2$$FloatRegister);
1600+
}
1601+
%}
1602+
ins_pipe(pipe_slow);
1603+
%}
1604+
1605+
instruct vuqadd_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1606+
predicate(UseSVE == 2 && n->as_SaturatingVector()->is_unsigned());
1607+
match(Set dst_src1 (SaturatingAddV (Binary dst_src1 src2) pg));
1608+
format %{ "vuqadd_masked $dst_src1, $pg, $dst_src1, $src2" %}
1609+
ins_encode %{
1610+
BasicType bt = Matcher::vector_element_basic_type(this);
1611+
__ sve_uqadd($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
1612+
$pg$$PRegister, $src2$$FloatRegister);
1613+
%}
1614+
ins_pipe(pipe_slow);
1615+
%}
1616+
1617+
// ------------------------- Vector saturating sub -----------------------------
1618+
1619+
// Signed saturating sub
1620+
1621+
instruct vsqsub(vReg dst, vReg src1, vReg src2) %{
1622+
predicate(!n->as_SaturatingVector()->is_unsigned());
1623+
match(Set dst (SaturatingSubV src1 src2));
1624+
format %{ "vsqsub $dst, $src1, $src2" %}
1625+
ins_encode %{
1626+
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
1627+
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
1628+
__ sqsubv($dst$$FloatRegister, get_arrangement(this),
1629+
$src1$$FloatRegister, $src2$$FloatRegister);
1630+
} else {
1631+
assert(UseSVE > 0, "must be sve");
1632+
BasicType bt = Matcher::vector_element_basic_type(this);
1633+
__ sve_sqsub($dst$$FloatRegister, __ elemType_to_regVariant(bt),
1634+
$src1$$FloatRegister, $src2$$FloatRegister);
1635+
}
1636+
%}
1637+
ins_pipe(pipe_slow);
1638+
%}
1639+
1640+
instruct vsqsub_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1641+
predicate(UseSVE == 2 && !n->as_SaturatingVector()->is_unsigned());
1642+
match(Set dst_src1 (SaturatingSubV (Binary dst_src1 src2) pg));
1643+
format %{ "vsqsub_masked $dst_src1, $pg, $dst_src1, $src2" %}
1644+
ins_encode %{
1645+
BasicType bt = Matcher::vector_element_basic_type(this);
1646+
__ sve_sqsub($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
1647+
$pg$$PRegister, $src2$$FloatRegister);
1648+
%}
1649+
ins_pipe(pipe_slow);
1650+
%}
1651+
1652+
// Unsigned saturating sub
1653+
1654+
instruct vuqsub(vReg dst, vReg src1, vReg src2) %{
1655+
predicate(n->as_SaturatingVector()->is_unsigned());
1656+
match(Set dst (SaturatingSubV src1 src2));
1657+
format %{ "vuqsub $dst, $src1, $src2" %}
1658+
ins_encode %{
1659+
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
1660+
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
1661+
__ uqsubv($dst$$FloatRegister, get_arrangement(this),
1662+
$src1$$FloatRegister, $src2$$FloatRegister);
1663+
} else {
1664+
assert(UseSVE > 0, "must be sve");
1665+
BasicType bt = Matcher::vector_element_basic_type(this);
1666+
__ sve_uqsub($dst$$FloatRegister, __ elemType_to_regVariant(bt),
1667+
$src1$$FloatRegister, $src2$$FloatRegister);
1668+
}
1669+
%}
1670+
ins_pipe(pipe_slow);
1671+
%}
1672+
1673+
instruct vuqsub_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1674+
predicate(UseSVE == 2 && n->as_SaturatingVector()->is_unsigned());
1675+
match(Set dst_src1 (SaturatingSubV (Binary dst_src1 src2) pg));
1676+
format %{ "vuqsub_masked $dst_src1, $pg, $dst_src1, $src2" %}
1677+
ins_encode %{
1678+
BasicType bt = Matcher::vector_element_basic_type(this);
1679+
__ sve_uqsub($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
1680+
$pg$$PRegister, $src2$$FloatRegister);
1681+
%}
1682+
ins_pipe(pipe_slow);
1683+
%}
1684+
15421685
// ------------------------------ Vector abs -----------------------------------
15431686

15441687
// vector abs
@@ -1993,6 +2136,76 @@ instruct vmin_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
19932136
ins_pipe(pipe_slow);
19942137
%}
19952138

2139+
// vector unsigned min - LONG
2140+
2141+
instruct vuminL_neon(vReg dst, vReg src1, vReg src2) %{
2142+
predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n) == T_LONG);
2143+
match(Set dst (UMinV src1 src2));
2144+
effect(TEMP_DEF dst);
2145+
format %{ "vuminL_neon $dst, $src1, $src2\t# 2L" %}
2146+
ins_encode %{
2147+
__ cm(Assembler::HI, $dst$$FloatRegister, __ T2D, $src1$$FloatRegister, $src2$$FloatRegister);
2148+
__ bsl($dst$$FloatRegister, __ T16B, $src2$$FloatRegister, $src1$$FloatRegister);
2149+
%}
2150+
ins_pipe(pipe_slow);
2151+
%}
2152+
2153+
instruct vuminL_sve(vReg dst_src1, vReg src2) %{
2154+
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_LONG);
2155+
match(Set dst_src1 (UMinV dst_src1 src2));
2156+
format %{ "vuminL_sve $dst_src1, $dst_src1, $src2" %}
2157+
ins_encode %{
2158+
__ sve_umin($dst_src1$$FloatRegister, __ D, ptrue, $src2$$FloatRegister);
2159+
%}
2160+
ins_pipe(pipe_slow);
2161+
%}
2162+
2163+
// vector unsigned min - B/S/I
2164+
2165+
instruct vumin_neon(vReg dst, vReg src1, vReg src2) %{
2166+
predicate(Matcher::vector_element_basic_type(n) != T_LONG &&
2167+
VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
2168+
match(Set dst (UMinV src1 src2));
2169+
format %{ "vumin_neon $dst, $src1, $src2\t# B/S/I" %}
2170+
ins_encode %{
2171+
BasicType bt = Matcher::vector_element_basic_type(this);
2172+
assert(is_integral_type(bt) && bt != T_LONG, "unsupported type");
2173+
__ uminv($dst$$FloatRegister, get_arrangement(this),
2174+
$src1$$FloatRegister, $src2$$FloatRegister);
2175+
%}
2176+
ins_pipe(pipe_slow);
2177+
%}
2178+
2179+
instruct vumin_sve(vReg dst_src1, vReg src2) %{
2180+
predicate(Matcher::vector_element_basic_type(n) != T_LONG &&
2181+
!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
2182+
match(Set dst_src1 (UMinV dst_src1 src2));
2183+
format %{ "vumin_sve $dst_src1, $dst_src1, $src2\t# B/S/I" %}
2184+
ins_encode %{
2185+
assert(UseSVE > 0, "must be sve");
2186+
BasicType bt = Matcher::vector_element_basic_type(this);
2187+
assert(is_integral_type(bt) && bt != T_LONG, "unsupported type");
2188+
__ sve_umin($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
2189+
ptrue, $src2$$FloatRegister);
2190+
%}
2191+
ins_pipe(pipe_slow);
2192+
%}
2193+
2194+
// vector unsigned min - predicated
2195+
2196+
instruct vumin_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
2197+
predicate(UseSVE > 0);
2198+
match(Set dst_src1 (UMinV (Binary dst_src1 src2) pg));
2199+
format %{ "vumin_masked $dst_src1, $pg, $dst_src1, $src2" %}
2200+
ins_encode %{
2201+
BasicType bt = Matcher::vector_element_basic_type(this);
2202+
assert(is_integral_type(bt), "unsupported type");
2203+
__ sve_umin($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
2204+
$pg$$PRegister, $src2$$FloatRegister);
2205+
%}
2206+
ins_pipe(pipe_slow);
2207+
%}
2208+
19962209
// ------------------------------ Vector max -----------------------------------
19972210

19982211
// vector max - LONG
@@ -2080,6 +2293,76 @@ instruct vmax_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
20802293
ins_pipe(pipe_slow);
20812294
%}
20822295

2296+
// vector unsigned max - LONG
2297+
2298+
instruct vumaxL_neon(vReg dst, vReg src1, vReg src2) %{
2299+
predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n) == T_LONG);
2300+
match(Set dst (UMaxV src1 src2));
2301+
effect(TEMP_DEF dst);
2302+
format %{ "vumaxL_neon $dst, $src1, $src2\t# 2L" %}
2303+
ins_encode %{
2304+
__ cm(Assembler::HI, $dst$$FloatRegister, __ T2D, $src1$$FloatRegister, $src2$$FloatRegister);
2305+
__ bsl($dst$$FloatRegister, __ T16B, $src1$$FloatRegister, $src2$$FloatRegister);
2306+
%}
2307+
ins_pipe(pipe_slow);
2308+
%}
2309+
2310+
instruct vumaxL_sve(vReg dst_src1, vReg src2) %{
2311+
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_LONG);
2312+
match(Set dst_src1 (UMaxV dst_src1 src2));
2313+
format %{ "vumaxL_sve $dst_src1, $dst_src1, $src2" %}
2314+
ins_encode %{
2315+
__ sve_umax($dst_src1$$FloatRegister, __ D, ptrue, $src2$$FloatRegister);
2316+
%}
2317+
ins_pipe(pipe_slow);
2318+
%}
2319+
2320+
// vector unsigned max - B/S/I
2321+
2322+
instruct vumax_neon(vReg dst, vReg src1, vReg src2) %{
2323+
predicate(Matcher::vector_element_basic_type(n) != T_LONG &&
2324+
VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
2325+
match(Set dst (UMaxV src1 src2));
2326+
format %{ "vumax_neon $dst, $src1, $src2\t# B/S/I" %}
2327+
ins_encode %{
2328+
BasicType bt = Matcher::vector_element_basic_type(this);
2329+
assert(is_integral_type(bt) && bt != T_LONG, "unsupported type");
2330+
__ umaxv($dst$$FloatRegister, get_arrangement(this),
2331+
$src1$$FloatRegister, $src2$$FloatRegister);
2332+
%}
2333+
ins_pipe(pipe_slow);
2334+
%}
2335+
2336+
instruct vumax_sve(vReg dst_src1, vReg src2) %{
2337+
predicate(Matcher::vector_element_basic_type(n) != T_LONG &&
2338+
!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
2339+
match(Set dst_src1 (UMaxV dst_src1 src2));
2340+
format %{ "vumax_sve $dst_src1, $dst_src1, $src2\t# B/S/I" %}
2341+
ins_encode %{
2342+
assert(UseSVE > 0, "must be sve");
2343+
BasicType bt = Matcher::vector_element_basic_type(this);
2344+
assert(is_integral_type(bt) && bt != T_LONG, "unsupported type");
2345+
__ sve_umax($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
2346+
ptrue, $src2$$FloatRegister);
2347+
%}
2348+
ins_pipe(pipe_slow);
2349+
%}
2350+
2351+
// vector unsigned max - predicated
2352+
2353+
instruct vumax_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
2354+
predicate(UseSVE > 0);
2355+
match(Set dst_src1 (UMaxV (Binary dst_src1 src2) pg));
2356+
format %{ "vumax_masked $dst_src1, $pg, $dst_src1, $src2" %}
2357+
ins_encode %{
2358+
BasicType bt = Matcher::vector_element_basic_type(this);
2359+
assert(is_integral_type(bt), "unsupported type");
2360+
__ sve_umax($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
2361+
$pg$$PRegister, $src2$$FloatRegister);
2362+
%}
2363+
ins_pipe(pipe_slow);
2364+
%}
2365+
20832366
// ------------------------------ MLA RELATED ----------------------------------
20842367

20852368
// vector mla

0 commit comments

Comments
 (0)