Valhalla JVMCI update

make/test/BuildMicrobenchmark.gmk

@@ -99,14 +99,20 @@ $(eval $(call SetupJavaCompilation, BUILD_JDK_MICROBENCHMARK, \
         --add-exports java.base/sun.security.util=ALL-UNNAMED \
         --add-exports java.base/sun.security.util.math=ALL-UNNAMED \
         --add-exports java.base/sun.security.util.math.intpoly=ALL-UNNAMED \
+        --add-exports java.base/jdk.internal.vm.annotation=ALL-UNNAMED \
         --enable-preview \
+        --patch-module java.base=$(JAVA_HOME)/lib/valueclasses/java.base-valueclasses.jar \
+        --add-exports java.base/jdk.internal.value=ALL-UNNAMED \
         -XDsuppressNotes \
         -processor org.openjdk.jmh.generators.BenchmarkProcessor \
         -s $(MICROBENCHMARK_GENSRC), \
     JAVA_FLAGS := \
         --add-exports java.base/jdk.internal.vm=ALL-UNNAMED \
         --add-modules jdk.unsupported \
+        --add-modules java.base \
+        --add-exports java.base/jdk.internal.vm.annotation=ALL-UNNAMED \
         --enable-preview \
+        --add-exports java.base/jdk.internal.value=ALL-UNNAMED \
         --limit-modules java.management, \
 ))
 

src/hotspot/cpu/aarch64/jvmciCodeInstaller_aarch64.cpp

@@ -133,20 +133,23 @@ void CodeInstaller::pd_relocate_JavaMethod(CodeBuffer &cbuf, methodHandle& metho
     case INVOKEINTERFACE: {
       assert(!method->is_static(), "cannot call static method with invokeinterface");
       call = nativeCall_at(_instructions->start() + pc_offset);
+      // TODO: attach method for valhalla calling convention see jvmciCodeInstaller_x86.cpp
       _instructions->relocate(call->instruction_address(), virtual_call_Relocation::spec(_invoke_mark_pc));
       call->trampoline_jump(cbuf, SharedRuntime::get_resolve_virtual_call_stub(), JVMCI_CHECK);
       break;
     }
     case INVOKESTATIC: {
       assert(method->is_static(), "cannot call non-static method with invokestatic");
       call = nativeCall_at(_instructions->start() + pc_offset);
+      // TODO: attach method for valhalla calling convention see jvmciCodeInstaller_x86.cpp
       _instructions->relocate(call->instruction_address(), relocInfo::static_call_type);
       call->trampoline_jump(cbuf, SharedRuntime::get_resolve_static_call_stub(), JVMCI_CHECK);
       break;
     }
     case INVOKESPECIAL: {
       assert(!method->is_static(), "cannot call static method with invokespecial");
       call = nativeCall_at(_instructions->start() + pc_offset);
+      // TODO: attach method for valhalla calling convention see jvmciCodeInstaller_x86.cpp
       _instructions->relocate(call->instruction_address(), relocInfo::opt_virtual_call_type);
       call->trampoline_jump(cbuf, SharedRuntime::get_resolve_opt_virtual_call_stub(), JVMCI_CHECK);
       break;

src/hotspot/cpu/x86/jvmciCodeInstaller_x86.cpp

@@ -164,26 +164,49 @@ void CodeInstaller::pd_relocate_JavaMethod(CodeBuffer &, methodHandle& method, j
 
       call = nativeCall_at(_instructions->start() + pc_offset);
       call->set_destination(SharedRuntime::get_resolve_virtual_call_stub());
+      if (method->has_scalarized_args() && !method->mismatch()) {
       _instructions->relocate(call->instruction_address(),
-                                             virtual_call_Relocation::spec(_invoke_mark_pc),
+                                             virtual_call_Relocation::spec(_invoke_mark_pc, _oop_recorder->find_index(method())),
                                              Assembler::call32_operand);
+      } else {
+          _instructions->relocate(call->instruction_address(),
+                                          virtual_call_Relocation::spec(_invoke_mark_pc),
+                                          Assembler::call32_operand);
+      }
       break;
     }
     case INVOKESTATIC: {
       assert(method->is_static(), "cannot call non-static method with invokestatic");
 
       call = nativeCall_at(_instructions->start() + pc_offset);
       call->set_destination(SharedRuntime::get_resolve_static_call_stub());
-      _instructions->relocate(call->instruction_address(),
+      // calls to the ValueObjectMethods class do not exist in bytecode, need to attach them
+      if (method->has_scalarized_args() || method() == Universe::is_substitutable_method() || method() == Universe::value_object_hash_code_method()) {
+        _instructions->relocate(call->instruction_address(),
+                                             relocInfo::static_call_type, Assembler::call32_operand,_oop_recorder->find_index(method()));
+      }else{
+        _instructions->relocate(call->instruction_address(),
                                              relocInfo::static_call_type, Assembler::call32_operand);
+      }
+
       break;
     }
     case INVOKESPECIAL: {
       assert(!method->is_static(), "cannot call static method with invokespecial");
       call = nativeCall_at(_instructions->start() + pc_offset);
       call->set_destination(SharedRuntime::get_resolve_opt_virtual_call_stub());
-      _instructions->relocate(call->instruction_address(),
+
+      // attach the target method only for scalarized args to avoid a null check on the receiver if receiver was optimized from non-scalarized to scalarized
+      // attach only for scalarized args otherwise assert(attached_method->has_scalarized_args(), "invalid use of attached method"); will trigger
+      // see resolved_method_index in machnode.hpp
+      if (method->has_scalarized_args() && !method->mismatch()) {
+        _instructions->relocate(call->instruction_address(),
+                              relocInfo::opt_virtual_call_type, Assembler::call32_operand, _oop_recorder->find_index(method()));
+      } else {
+        _instructions->relocate(call->instruction_address(),
                               relocInfo::opt_virtual_call_type, Assembler::call32_operand);
+      }
+
       break;
     }
     default:

src/hotspot/share/code/nmethod.cpp

@@ -727,7 +727,7 @@ void nmethod::preserve_callee_argument_oops(frame fr, const RegisterMap *reg_map
 
       // If inline types are passed as fields, use the extended signature
       // which contains the types of all (oop) fields of the inline type.
-      if (is_compiled_by_c2() && callee->has_scalarized_args()) {
+      if ((is_compiled_by_c2() || is_compiled_by_jvmci()) && callee->has_scalarized_args()) {
         const GrowableArray<SigEntry>* sig = callee->adapter()->get_sig_cc();
         assert(sig != nullptr, "sig should never be null");
         TempNewSymbol tmp_sig = SigEntry::create_symbol(sig);

src/hotspot/share/code/nmethod.hpp

@@ -685,7 +685,7 @@ class nmethod : public CodeBlob {
   bool  needs_stack_repair() const {
     if (is_compiled_by_c1()) {
       return method()->c1_needs_stack_repair();
-    } else if (is_compiled_by_c2()) {
+    } else if (is_compiled_by_c2() || is_compiled_by_jvmci()) {
       return method()->c2_needs_stack_repair();
     } else {
       return false;

src/hotspot/share/interpreter/linkResolver.cpp

@@ -1769,7 +1769,7 @@ void LinkResolver::resolve_invokevirtual(CallInfo& result, Handle recv,
 void LinkResolver::resolve_invokeinterface(CallInfo& result, Handle recv, const constantPoolHandle& pool, int index, TRAPS) {
   LinkInfo link_info(pool, index, Bytecodes::_invokeinterface, CHECK);
   Klass* recvrKlass = recv.is_null() ? (Klass*)nullptr : recv->klass();
-  resolve_interface_call(result, recv, recvrKlass, link_info, true, CHECK);
+  resolve_interface_call(result, recv, recvrKlass, link_info, /*check_null_or_abstract*/true, CHECK);
 }
 
 bool LinkResolver::resolve_previously_linked_invokehandle(CallInfo& result, const LinkInfo& link_info, const constantPoolHandle& pool, int index, TRAPS) {

src/hotspot/share/interpreter/linkResolver.hpp

@@ -357,7 +357,7 @@ class LinkResolver: AllStatic {
   // runtime resolving from attached method
   static void resolve_invoke(CallInfo& result, Handle& recv,
                              const methodHandle& attached_method,
-                             Bytecodes::Code byte, bool check_null_and_abstract, TRAPS);
+                             Bytecodes::Code byte, bool check_null_or_abstract, TRAPS);
 
   // Only resolved method known.
   static void throw_abstract_method_error(const methodHandle& resolved_method, TRAPS) {

src/hotspot/share/jvmci/jvmciCodeInstaller.cpp

@@ -41,6 +41,9 @@
 #include "runtime/os.hpp"
 #include "runtime/sharedRuntime.hpp"
 #include "utilities/align.hpp"
+#include "ci/ciSignature.hpp"
+#include "oops/inlineKlass.hpp"
+#include "runtime/globals.hpp"
 
 // frequently used constants
 // Allocate them with new so they are never destroyed (otherwise, a
@@ -778,6 +781,14 @@ JVMCI::CodeInstallResult CodeInstaller::install(JVMCICompiler* compiler,
       JVMCI_THROW_MSG_(IllegalArgumentException, "nmethod entry barrier is missing", JVMCI::ok);
     }
 
+    if(_offsets.value(CodeOffsets::Verified_Inline_Entry) == -1) {
+      _offsets.set_value(CodeOffsets::Verified_Inline_Entry, _offsets.value(CodeOffsets::Verified_Entry));
+    }
+
+    if(_offsets.value(CodeOffsets::Verified_Inline_Entry_RO) == -1) {
+      _offsets.set_value(CodeOffsets::Verified_Inline_Entry_RO, _offsets.value(CodeOffsets::Verified_Entry));
+    }
+
     JVMCIObject mirror = installed_code;
     nmethod* nm = nullptr; // nm is an out parameter of register_method
     result = runtime()->register_method(jvmci_env(),
@@ -1098,9 +1109,19 @@ void CodeInstaller::read_virtual_objects(HotSpotCompiledCodeStream* stream, JVMC
     if (is_auto_box) {
       _has_auto_box = true;
     }
+    // see code in output.cpp line 812 (PhaseOutput::FillLocArray line)
+    bool check_is_not_null = stream->read_bool("nonNull");
+    ScopeValue *is_init = nullptr;
+    if (check_is_not_null) {
+      ScopeValue* cur_second = nullptr;
+      BasicType type = (BasicType) stream->read_u1("basicType");
+      ScopeValue* value;
+      u1 tag = stream->read_u1("tag");
+      is_init = get_scope_value(stream, tag, type, cur_second, JVMCI_CHECK);
+    }
     oop javaMirror = klass->java_mirror();
     ScopeValue *klass_sv = new ConstantOopWriteValue(JNIHandles::make_local(javaMirror));
-    ObjectValue* sv = is_auto_box ? new AutoBoxObjectValue(id, klass_sv) : new ObjectValue(id, klass_sv);
+    ObjectValue* sv = is_auto_box ? new AutoBoxObjectValue(id, klass_sv) : new ObjectValue(id, klass_sv, true, is_init);
     objects->at_put(id, sv);
   }
   // All the values which could be referenced by the VirtualObjects
@@ -1128,6 +1149,18 @@ int CodeInstaller::map_jvmci_bci(int bci) {
   return bci;
 }
 
+bool has_scalarized_return(methodHandle& methodHandle){
+  if (!InlineTypeReturnedAsFields) {
+    return false;
+  }
+  Method* method = methodHandle();
+  InlineKlass* klass = method->returns_inline_type(Thread::current());
+  if (klass != nullptr) {
+    return !method->is_native() && klass->can_be_returned_as_fields();
+  }
+  return false;
+}
+
 void CodeInstaller::record_scope(jint pc_offset, HotSpotCompiledCodeStream* stream, u1 debug_info_flags, bool full_info, bool is_mh_invoke, bool return_oop, JVMCI_TRAPS) {
   if (full_info) {
     read_virtual_objects(stream, JVMCI_CHECK);
@@ -1169,7 +1202,7 @@ void CodeInstaller::record_scope(jint pc_offset, HotSpotCompiledCodeStream* stre
       }
 
       // has_ea_local_in_scope and arg_escape should be added to JVMCI
-      const bool return_scalarized     = false;
+      const bool return_scalarized     = has_scalarized_return(method);
       const bool has_ea_local_in_scope = false;
       const bool arg_escape            = false;
       _debug_recorder->describe_scope(pc_offset, method, nullptr, bci, reexecute, rethrow_exception, is_mh_invoke, return_oop,
@@ -1315,9 +1348,16 @@ void CodeInstaller::site_Mark(CodeBuffer& buffer, jint pc_offset, HotSpotCompile
     case UNVERIFIED_ENTRY:
       _offsets.set_value(CodeOffsets::Entry, pc_offset);
       break;
+    case INLINE_ENTRY:
+      _offsets.set_value(CodeOffsets::Inline_Entry, pc_offset);
+      break;
     case VERIFIED_ENTRY:
       _offsets.set_value(CodeOffsets::Verified_Entry, pc_offset);
+      break;
+    case VERIFIED_INLINE_ENTRY:
       _offsets.set_value(CodeOffsets::Verified_Inline_Entry, pc_offset);
+      break;
+    case VERIFIED_INLINE_ENTRY_RO:
       _offsets.set_value(CodeOffsets::Verified_Inline_Entry_RO, pc_offset);
       break;
     case OSR_ENTRY:

src/hotspot/share/jvmci/jvmciCodeInstaller.hpp

@@ -147,7 +147,10 @@ class CodeInstaller : public StackObj {
   enum MarkId {
     INVALID_MARK,
     VERIFIED_ENTRY,
+    VERIFIED_INLINE_ENTRY,
+    VERIFIED_INLINE_ENTRY_RO,
     UNVERIFIED_ENTRY,
+    INLINE_ENTRY,
     OSR_ENTRY,
     EXCEPTION_HANDLER_ENTRY,
     DEOPT_HANDLER_ENTRY,