Initial support for inline types

src/hotspot/cpu/x86/jvmciCodeInstaller_x86.cpp

@@ -154,6 +154,14 @@ void CodeInstaller::pd_relocate_ForeignCall(NativeInstruction* inst, jlong forei
 }
 
 void CodeInstaller::pd_relocate_JavaMethod(CodeBuffer &, methodHandle& method, jint pc_offset, JVMCI_TRAPS) {
+  // attach the target method only for scalarized args to avoid a null check on the receiver if receiver was optimized from non-scalarized to scalarized
+  // attach only for scalarized args otherwise assert(attached_method->has_scalarized_args(), "invalid use of attached method"); will trigger
+  // see resolved_method_index in machnode.hpp
+  // TODO integrate https://github.com/graalvm/labs-openjdk/commit/c3da3483f3a04a0e77db4420fac48ac2d9155c47 instead of this logic.
+  int method_index = 0;
+  if ((method->has_scalarized_args() && !method->mismatch()) || method() == Universe::is_substitutable_method()) {
+    method_index = _oop_recorder->find_index(method());
+  }
   NativeCall* call = nullptr;
   switch (_next_call_type) {
     case INLINE_INVOKE:
@@ -165,7 +173,7 @@ void CodeInstaller::pd_relocate_JavaMethod(CodeBuffer &, methodHandle& method, j
       call = nativeCall_at(_instructions->start() + pc_offset);
       call->set_destination(SharedRuntime::get_resolve_virtual_call_stub());
       _instructions->relocate(call->instruction_address(),
-                                             virtual_call_Relocation::spec(_invoke_mark_pc),
+                                             virtual_call_Relocation::spec(_invoke_mark_pc, method_index),
                                              Assembler::call32_operand);
       break;
     }
@@ -175,15 +183,16 @@ void CodeInstaller::pd_relocate_JavaMethod(CodeBuffer &, methodHandle& method, j
       call = nativeCall_at(_instructions->start() + pc_offset);
       call->set_destination(SharedRuntime::get_resolve_static_call_stub());
       _instructions->relocate(call->instruction_address(),
-                                             relocInfo::static_call_type, Assembler::call32_operand);
+                                             relocInfo::static_call_type, Assembler::call32_operand, method_index);
+
       break;
     }
     case INVOKESPECIAL: {
       assert(!method->is_static(), "cannot call static method with invokespecial");
       call = nativeCall_at(_instructions->start() + pc_offset);
       call->set_destination(SharedRuntime::get_resolve_opt_virtual_call_stub());
       _instructions->relocate(call->instruction_address(),
-                              relocInfo::opt_virtual_call_type, Assembler::call32_operand);
+                              relocInfo::opt_virtual_call_type, Assembler::call32_operand, method_index);
       break;
     }
     default:

src/hotspot/share/code/nmethod.cpp

@@ -723,7 +723,7 @@ void nmethod::preserve_callee_argument_oops(frame fr, const RegisterMap *reg_map
 
       // If inline types are passed as fields, use the extended signature
       // which contains the types of all (oop) fields of the inline type.
-      if (is_compiled_by_c2() && callee->has_scalarized_args()) {
+      if ((is_compiled_by_c2() || is_compiled_by_jvmci()) && callee->has_scalarized_args()) {
         const GrowableArray<SigEntry>* sig = callee->adapter()->get_sig_cc();
         assert(sig != nullptr, "sig should never be null");
         TempNewSymbol tmp_sig = SigEntry::create_symbol(sig);

src/hotspot/share/code/nmethod.hpp

@@ -763,7 +763,7 @@ class nmethod : public CodeBlob {
   bool  needs_stack_repair() const {
     if (is_compiled_by_c1()) {
       return method()->c1_needs_stack_repair();
-    } else if (is_compiled_by_c2()) {
+    } else if (is_compiled_by_c2() || is_compiled_by_jvmci()) {
       return method()->c2_needs_stack_repair();
     } else {
       return false;

src/hotspot/share/jvmci/jvmciCodeInstaller.cpp

@@ -41,6 +41,9 @@
 #include "runtime/os.hpp"
 #include "runtime/sharedRuntime.hpp"
 #include "utilities/align.hpp"
+#include "ci/ciSignature.hpp"
+#include "oops/inlineKlass.hpp"
+#include "runtime/globals.hpp"
 
 // frequently used constants
 // Allocate them with new so they are never destroyed (otherwise, a
@@ -785,6 +788,14 @@ JVMCI::CodeInstallResult CodeInstaller::install(JVMCICompiler* compiler,
       JVMCI_THROW_MSG_(IllegalArgumentException, "nmethod entry barrier is missing", JVMCI::ok);
     }
 
+    if(_offsets.value(CodeOffsets::Verified_Inline_Entry) == -1) {
+      _offsets.set_value(CodeOffsets::Verified_Inline_Entry, _offsets.value(CodeOffsets::Verified_Entry));
+    }
+
+    if(_offsets.value(CodeOffsets::Verified_Inline_Entry_RO) == -1) {
+      _offsets.set_value(CodeOffsets::Verified_Inline_Entry_RO, _offsets.value(CodeOffsets::Verified_Entry));
+    }
+
     JVMCIObject mirror = installed_code;
     nmethod* nm = nullptr; // nm is an out parameter of register_method
     result = runtime()->register_method(jvmci_env(),
@@ -1111,9 +1122,34 @@ void CodeInstaller::read_virtual_objects(HotSpotCompiledCodeStream* stream, JVMC
     if (is_auto_box) {
       _has_auto_box = true;
     }
+    // see code in output.cpp (PhaseOutput::FillLocArray)
+    bool check_non_null = stream->read_bool("nonNull");
+    ScopeValue *properties = nullptr;
+    if (check_non_null) {
+      ScopeValue* cur_second = nullptr;
+      BasicType type = (BasicType) stream->read_u1("basicType");
+      ScopeValue* value;
+      u1 tag = stream->read_u1("tag");
+      properties = get_scope_value(stream, tag, type, cur_second, JVMCI_CHECK);
+    }
+    if (klass->is_array_klass() && !klass->is_typeArray_klass()) {
+      jint props = ArrayKlass::ArrayProperties::DEFAULT;
+      ObjArrayKlass* ak = ObjArrayKlass::cast(klass);
+      if (ak->element_klass()->is_inline_klass()) {
+        if (klass->is_null_free_array_klass()) {
+          props |= ArrayKlass::ArrayProperties::NULL_RESTRICTED;
+        }
+        bool is_atomic = (ak->properties() & ArrayKlass::ArrayProperties::INVALID) == 0 &&
+                            (ak->properties() & ArrayKlass::ArrayProperties::NON_ATOMIC) == 0;
+        if (!is_atomic) {
+          props |= ArrayKlass::ArrayProperties::NON_ATOMIC;
+        }
+      }
+      properties = new ConstantIntValue(props);
+    }
     oop javaMirror = klass->java_mirror();
     ScopeValue *klass_sv = new ConstantOopWriteValue(JNIHandles::make_local(javaMirror));
-    ObjectValue* sv = is_auto_box ? new AutoBoxObjectValue(id, klass_sv) : new ObjectValue(id, klass_sv);
+    ObjectValue* sv = is_auto_box ? new AutoBoxObjectValue(id, klass_sv) : new ObjectValue(id, klass_sv, true, properties);
     objects->at_put(id, sv);
   }
   // All the values which could be referenced by the VirtualObjects
@@ -1141,6 +1177,18 @@ int CodeInstaller::map_jvmci_bci(int bci) {
   return bci;
 }
 
+bool has_scalarized_return(methodHandle& methodHandle){
+  if (!InlineTypeReturnedAsFields) {
+    return false;
+  }
+  Method* method = methodHandle();
+  InlineKlass* klass = method->returns_inline_type();
+  if (klass != nullptr) {
+    return !method->is_native() && klass->can_be_returned_as_fields();
+  }
+  return false;
+}
+
 void CodeInstaller::record_scope(jint pc_offset, HotSpotCompiledCodeStream* stream, u1 debug_info_flags, bool full_info, bool is_mh_invoke, bool return_oop, JVMCI_TRAPS) {
   if (full_info) {
     read_virtual_objects(stream, JVMCI_CHECK);
@@ -1182,7 +1230,7 @@ void CodeInstaller::record_scope(jint pc_offset, HotSpotCompiledCodeStream* stre
       }
 
       // has_ea_local_in_scope and arg_escape should be added to JVMCI
-      const bool return_scalarized     = false;
+      const bool return_scalarized     = has_scalarized_return(method);
       const bool has_ea_local_in_scope = false;
       const bool arg_escape            = false;
       _debug_recorder->describe_scope(pc_offset, method, nullptr, bci, reexecute, rethrow_exception, is_mh_invoke, return_oop,
@@ -1328,9 +1376,16 @@ void CodeInstaller::site_Mark(CodeBuffer& buffer, jint pc_offset, HotSpotCompile
     case UNVERIFIED_ENTRY:
       _offsets.set_value(CodeOffsets::Entry, pc_offset);
       break;
+    case INLINE_ENTRY:
+      _offsets.set_value(CodeOffsets::Inline_Entry, pc_offset);
+      break;
     case VERIFIED_ENTRY:
       _offsets.set_value(CodeOffsets::Verified_Entry, pc_offset);
+      break;
+    case VERIFIED_INLINE_ENTRY:
       _offsets.set_value(CodeOffsets::Verified_Inline_Entry, pc_offset);
+      break;
+    case VERIFIED_INLINE_ENTRY_RO:
       _offsets.set_value(CodeOffsets::Verified_Inline_Entry_RO, pc_offset);
       break;
     case OSR_ENTRY:

src/hotspot/share/jvmci/jvmciCodeInstaller.hpp

@@ -147,7 +147,10 @@ class CodeInstaller : public StackObj {
   enum MarkId {
     INVALID_MARK,
     VERIFIED_ENTRY,
+    VERIFIED_INLINE_ENTRY,
+    VERIFIED_INLINE_ENTRY_RO,
     UNVERIFIED_ENTRY,
+    INLINE_ENTRY,
     OSR_ENTRY,
     EXCEPTION_HANDLER_ENTRY,
     DEOPT_HANDLER_ENTRY,