8278644: riscv: Intrinsify mulAdd

Co-authored-by: Taiping Guo <[email protected]>
Reviewed-by: fyang

Author: 2 people

src/hotspot/cpu/riscv/macroAssembler_riscv.cpp

@@ -3025,6 +3025,72 @@ void MacroAssembler::compute_match_mask(Register src, Register pattern, Register
   andr(match_mask, match_mask, src);
 }
 
+#ifdef COMPILER2
+// Code for BigInteger::mulAdd instrinsic
+// out     = x10
+// in      = x11
+// offset  = x12  (already out.length-offset)
+// len     = x13
+// k       = x14
+// tmp     = x28
+//
+// pseudo code from java implementation:
+// carry = 0;
+// offset = out.length-offset - 1;
+// for (int j = len - 1; j >= 0; j--) {
+//     product = (in[j] & LONG_MASK) * kLong + (out[offset] & LONG_MASK) + carry;
+//     out[offset--] = (int)product;
+//     carry = product >>> 32;
+// }
+// return (int)carry;
+void MacroAssembler::mul_add(Register out, Register in, Register offset,
+                             Register len, Register k, Register tmp) {
+  Label L_tail_loop, L_unroll, L_end;
+  mv(tmp, out);
+  mv(out, zr);
+  beqz(len, L_end);
+  zero_ext(k, k, 32);
+  slli(t0, offset, LogBytesPerInt);
+  add(offset, tmp, t0);
+  slli(t0, len, LogBytesPerInt);
+  add(in, in, t0);
+
+  const int unroll = 8;
+  li(tmp, unroll);
+  blt(len, tmp, L_tail_loop);
+  bind(L_unroll);
+  for (int i = 0; i < unroll; i++) {
+    sub(in, in, BytesPerInt);
+    lwu(t0, Address(in, 0));
+    mul(t1, t0, k);
+    add(t0, t1, out);
+    sub(offset, offset, BytesPerInt);
+    lwu(t1, Address(offset, 0));
+    add(t0, t0, t1);
+    sw(t0, Address(offset, 0));
+    srli(out, t0, 32);
+  }
+  sub(len, len, tmp);
+  bge(len, tmp, L_unroll);
+
+  bind(L_tail_loop);
+  beqz(len, L_end);
+  sub(in, in, BytesPerInt);
+  lwu(t0, Address(in, 0));
+  mul(t1, t0, k);
+  add(t0, t1, out);
+  sub(offset, offset, BytesPerInt);
+  lwu(t1, Address(offset, 0));
+  add(t0, t0, t1);
+  sw(t0, Address(offset, 0));
+  srli(out, t0, 32);
+  sub(len, len, 1);
+  j(L_tail_loop);
+
+  bind(L_end);
+}
+#endif
+
 // Count bits of trailing zero chars from lsb to msb until first non-zero element.
 // For LL case, one byte for one element, so shift 8 bits once, and for other case,
 // shift 16 bits once.

src/hotspot/cpu/riscv/macroAssembler_riscv.hpp

@@ -656,6 +656,11 @@ class MacroAssembler: public Assembler {
   void compute_match_mask(Register src, Register pattern, Register match_mask,
                           Register mask1, Register mask2);
 
+#ifdef COMPILER2
+  void mul_add(Register out, Register in, Register offset,
+               Register len, Register k, Register tmp);
+#endif
+
   void inflate_lo32(Register Rd, Register Rs, Register Rtmp1 = t0, Register Rtmp2 = t1);
   void inflate_hi32(Register Rd, Register Rs, Register Rtmp1 = t0, Register Rtmp2 = t1);
 

src/hotspot/cpu/riscv/stubGenerator_riscv.cpp

@@ -2770,6 +2770,31 @@ class StubGenerator: public StubCodeGenerator {
     StubRoutines::riscv64::_string_indexof_linear_ul = generate_string_indexof_linear(true, false);
   }
 
+#ifdef COMPILER2
+  address generate_mulAdd()
+  {
+    __ align(CodeEntryAlignment);
+    StubCodeMark mark(this, "StubRoutines", "mulAdd");
+
+    address entry = __ pc();
+
+    const Register out     = x10;
+    const Register in      = x11;
+    const Register offset  = x12;
+    const Register len     = x13;
+    const Register k       = x14;
+    const Register tmp     = x28;
+
+    BLOCK_COMMENT("Entry:");
+    __ enter();
+    __ mul_add(out, in, offset, len, k, tmp);
+    __ leave();
+    __ ret();
+
+    return entry;
+  }
+#endif
+
   // Continuation point for throwing of implicit exceptions that are
   // not handled in the current activation. Fabricates an exception
   // oop and initiates normal exception dispatching in this
@@ -2940,6 +2965,12 @@ class StubGenerator: public StubCodeGenerator {
     // arraycopy stubs used by compilers
     generate_arraycopy_stubs();
 
+#ifdef COMPILER2
+    if (UseMulAddIntrinsic) {
+      StubRoutines::_mulAdd = generate_mulAdd();
+    }
+#endif
+
     generate_compare_long_strings();
 
     generate_string_indexof_stubs();

src/hotspot/cpu/riscv/vm_version_riscv.cpp

@@ -174,6 +174,10 @@ void VM_Version::initialize_c2() {
   if (FLAG_IS_DEFAULT(AllocatePrefetchStyle)) {
     FLAG_SET_DEFAULT(AllocatePrefetchStyle, 0);
   }
+
+  if (FLAG_IS_DEFAULT(UseMulAddIntrinsic)) {
+    FLAG_SET_DEFAULT(UseMulAddIntrinsic, true);
+  }
 }
 #endif // COMPILER2