diff --git a/src/hotspot/share/ci/ciArray.cpp b/src/hotspot/share/ci/ciArray.cpp index 82d8b04e9f9..4ef0fa781e9 100644 --- a/src/hotspot/share/ci/ciArray.cpp +++ b/src/hotspot/share/ci/ciArray.cpp @@ -27,6 +27,8 @@ #include "ci/ciConstant.hpp" #include "ci/ciKlass.hpp" #include "ci/ciUtilities.inline.hpp" +#include "oops/flatArrayKlass.hpp" +#include "oops/layoutKind.hpp" #include "oops/objArrayOop.inline.hpp" #include "oops/oop.inline.hpp" #include "oops/typeArrayOop.inline.hpp" @@ -128,16 +130,7 @@ bool ciArray::is_null_free() { bool ciArray::is_atomic() { VM_ENTRY_MARK; arrayOop oop = get_arrayOop(); - if (oop->is_refArray()) { - return oop->klass()->is_inline_klass(); - } - if (oop->is_flatArray()) { - FlatArrayKlass* fak = FlatArrayKlass::cast(oop->klass()); - if (fak->element_klass()->is_naturally_atomic() || fak->layout_kind() == LayoutKind::ATOMIC_FLAT || fak->layout_kind() == LayoutKind::NULLABLE_ATOMIC_FLAT) { - return true; - } - } - return false; + return !oop->is_flatArray() || FlatArrayKlass::cast(oop->klass())->layout_kind() != LayoutKind::NON_ATOMIC_FLAT; } // ------------------------------------------------------------------ diff --git a/src/hotspot/share/ci/ciArrayKlass.cpp b/src/hotspot/share/ci/ciArrayKlass.cpp index 7f61f7a85ef..794b413bdc7 100644 --- a/src/hotspot/share/ci/ciArrayKlass.cpp +++ b/src/hotspot/share/ci/ciArrayKlass.cpp @@ -29,6 +29,7 @@ #include "ci/ciTypeArrayKlass.hpp" #include "ci/ciUtilities.inline.hpp" #include "memory/universe.hpp" +#include "oops/arrayKlass.hpp" #include "oops/inlineKlass.inline.hpp" // ciArrayKlass @@ -121,14 +122,15 @@ ciInstance* ciArrayKlass::component_mirror_instance() const { } bool ciArrayKlass::is_elem_null_free() const { - GUARDED_VM_ENTRY(return is_loaded() && get_Klass()->is_null_free_array_klass();) + ArrayKlass::ArrayProperties props = properties(); + assert(props != ArrayKlass::INVALID, "meaningless"); + return ArrayKlass::is_null_restricted(props); } -bool ciArrayKlass::is_elem_atomic() { - ciKlass* elem = element_klass(); - GUARDED_VM_ENTRY(return elem != nullptr && elem->is_inlinetype() && - (ArrayKlass::cast(get_Klass())->properties() & ArrayKlass::ArrayProperties::INVALID) == 0 && - (ArrayKlass::cast(get_Klass())->properties() & ArrayKlass::ArrayProperties::NON_ATOMIC) == 0;) +bool ciArrayKlass::is_elem_atomic() const { + ArrayKlass::ArrayProperties props = properties(); + assert(props != ArrayKlass::INVALID, "meaningless"); + return !ArrayKlass::is_non_atomic(props); } ArrayKlass::ArrayProperties ciArrayKlass::properties() const { diff --git a/src/hotspot/share/ci/ciArrayKlass.hpp b/src/hotspot/share/ci/ciArrayKlass.hpp index bed653b1360..ddd268c6406 100644 --- a/src/hotspot/share/ci/ciArrayKlass.hpp +++ b/src/hotspot/share/ci/ciArrayKlass.hpp @@ -53,6 +53,8 @@ class ciArrayKlass : public ciKlass { ciType* base_element_type(); // JLS calls this the "element type" bool is_leaf_type(); // No subtypes of this array type. + bool is_refined() const { return !is_type_array_klass() && properties() != ArrayKlass::INVALID; } + // What kind of vmObject is this? bool is_array_klass() const { return true; } @@ -65,7 +67,7 @@ class ciArrayKlass : public ciKlass { ciInstance* component_mirror_instance() const; bool is_elem_null_free() const; - bool is_elem_atomic(); + bool is_elem_atomic() const; ArrayKlass::ArrayProperties properties() const; }; diff --git a/src/hotspot/share/ci/ciObjArrayKlass.cpp b/src/hotspot/share/ci/ciObjArrayKlass.cpp index 6b66e4c98e7..33dab779fcb 100644 --- a/src/hotspot/share/ci/ciObjArrayKlass.cpp +++ b/src/hotspot/share/ci/ciObjArrayKlass.cpp @@ -191,22 +191,5 @@ ciArrayKlass* ciObjArrayKlass::make(ciKlass* element_klass, int dims) { } ciKlass* ciObjArrayKlass::exact_klass() { - if (!is_loaded()) { - return nullptr; - } - ciType* base = base_element_type(); - if (base->is_instance_klass()) { - ciInstanceKlass* ik = base->as_instance_klass(); - // Even though MyValue is final, [LMyValue is only exact if the array - // is null-free due to null-free [LMyValue <: null-able [LMyValue. - if (ik->is_inlinetype() && !is_elem_null_free()) { - return nullptr; - } - if (ik->exact_klass() != nullptr) { - return this; - } - } else if (base->is_primitive_type()) { - return this; - } - return nullptr; + return (is_loaded() && is_refined()) ? this : nullptr; } diff --git a/src/hotspot/share/oops/objArrayKlass.cpp b/src/hotspot/share/oops/objArrayKlass.cpp index 94d7814c614..b80bba57656 100644 --- a/src/hotspot/share/oops/objArrayKlass.cpp +++ b/src/hotspot/share/oops/objArrayKlass.cpp @@ -137,6 +137,7 @@ ArrayKlass(name, kind, props, mk) { Klass* bk; if (element_klass->is_objArray_klass()) { + assert(!element_klass->is_refined_objArray_klass(), "no such mechanism yet"); bk = ObjArrayKlass::cast(element_klass)->bottom_klass(); } else { assert(!element_klass->is_refArray_klass(), "Sanity"); diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index 4256923a695..ab26b736515 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -1388,19 +1388,6 @@ const TypePtr *Compile::flatten_alias_type( const TypePtr *tj ) const { // Array pointers need some flattening const TypeAryPtr* ta = tj->isa_aryptr(); - if (ta && ta->is_stable()) { - // Erase stability property for alias analysis. - tj = ta = ta->cast_to_stable(false); - } - if (ta && ta->is_not_flat()) { - // Erase not flat property for alias analysis. - tj = ta = ta->cast_to_not_flat(false); - } - if (ta && ta->is_not_null_free()) { - // Erase not null free property for alias analysis. - tj = ta = ta->cast_to_not_null_free(false); - } - if( ta && is_known_inst ) { if ( offset != Type::OffsetBot && offset > arrayOopDesc::length_offset_in_bytes() ) { @@ -1411,78 +1398,46 @@ const TypePtr *Compile::flatten_alias_type( const TypePtr *tj ) const { with_offset(offset); } } else if (ta) { - // For arrays indexed by constant indices, we flatten the alias - // space to include all of the array body. Only the header, klass - // and array length can be accessed un-aliased. - // For flat inline type array, each field has its own slice so - // we must include the field offset. - if( offset != Type::OffsetBot ) { - if( ta->const_oop() ) { // MethodData* or Method* - offset = Type::OffsetBot; // Flatten constant access into array body - tj = ta = ta-> - remove_speculative()-> - cast_to_ptr_type(ptr)-> - cast_to_exactness(false)-> - with_offset(offset); - } else if( offset == arrayOopDesc::length_offset_in_bytes() ) { - // range is OK as-is. - tj = ta = TypeAryPtr::RANGE; - } else if( offset == oopDesc::klass_offset_in_bytes() ) { - tj = TypeInstPtr::KLASS; // all klass loads look alike - ta = TypeAryPtr::RANGE; // generic ignored junk - ptr = TypePtr::BotPTR; - } else if( offset == oopDesc::mark_offset_in_bytes() ) { - tj = TypeInstPtr::MARK; - ta = TypeAryPtr::RANGE; // generic ignored junk - ptr = TypePtr::BotPTR; - } else { // Random constant offset into array body - offset = Type::OffsetBot; // Flatten constant access into array body - tj = ta = ta-> - remove_speculative()-> - cast_to_ptr_type(ptr)-> - cast_to_exactness(false)-> - with_offset(offset); - } - } - // Arrays of fixed size alias with arrays of unknown size. - if (ta->size() != TypeInt::POS) { - const TypeAry *tary = TypeAry::make(ta->elem(), TypeInt::POS); - tj = ta = ta-> - remove_speculative()-> - cast_to_ptr_type(ptr)-> - with_ary(tary)-> - cast_to_exactness(false); - } - // Arrays of known objects become arrays of unknown objects. - if (ta->elem()->isa_narrowoop() && ta->elem() != TypeNarrowOop::BOTTOM) { - const TypeAry *tary = TypeAry::make(TypeNarrowOop::BOTTOM, ta->size()); - tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,nullptr,false,Type::Offset(offset), ta->field_offset()); - } - if (ta->elem()->isa_oopptr() && ta->elem() != TypeInstPtr::BOTTOM) { - const TypeAry *tary = TypeAry::make(TypeInstPtr::BOTTOM, ta->size()); - tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,nullptr,false,Type::Offset(offset), ta->field_offset()); - } - // Initially all flattened array accesses share a single slice - if (ta->is_flat() && ta->elem() != TypeInstPtr::BOTTOM && _flat_accesses_share_alias) { - const TypeAry* tary = TypeAry::make(TypeInstPtr::BOTTOM, ta->size(), /* stable= */ false, /* flat= */ true); - tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,nullptr,false,Type::Offset(offset), Type::Offset(Type::OffsetBot)); + // Common slices + if (offset == arrayOopDesc::length_offset_in_bytes()) { + return TypeAryPtr::RANGE; + } else if (offset == oopDesc::klass_offset_in_bytes()) { + return TypeInstPtr::KLASS; + } else if (offset == oopDesc::mark_offset_in_bytes()) { + return TypeInstPtr::MARK; + } + + // Remove size and stability + const TypeAry* normalized_ary = TypeAry::make(ta->elem(), TypeInt::POS, false, ta->is_flat(), ta->is_not_flat(), ta->is_not_null_free(), ta->is_atomic()); + // Remove ptr, const_oop, and offset + if (ta->elem()->make_oopptr() != nullptr) { + // Object arrays, keep field_offset + tj = ta = TypeAryPtr::make(TypePtr::BotPTR, nullptr, normalized_ary, nullptr, ta->klass_is_exact(), Type::Offset::bottom, Type::Offset(ta->field_offset())); + } else { + // Primitive arrays + tj = ta = TypeAryPtr::make(TypePtr::BotPTR, nullptr, normalized_ary, ta->exact_klass(), true, Type::Offset::bottom); } + // Arrays of bytes and of booleans both use 'bastore' and 'baload' so // cannot be distinguished by bytecode alone. if (ta->elem() == TypeInt::BOOL) { - const TypeAry *tary = TypeAry::make(TypeInt::BYTE, ta->size()); - ciKlass* aklass = ciTypeArrayKlass::make(T_BYTE); - tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,aklass,false,Type::Offset(offset), ta->field_offset()); - } - // During the 2nd round of IterGVN, NotNull castings are removed. - // Make sure the Bottom and NotNull variants alias the same. - // Also, make sure exact and non-exact variants alias the same. - if (ptr == TypePtr::NotNull || ta->klass_is_exact() || ta->speculative() != nullptr) { - tj = ta = ta-> - remove_speculative()-> - cast_to_ptr_type(TypePtr::BotPTR)-> - cast_to_exactness(false)-> - with_offset(offset); + tj = ta = TypeAryPtr::BYTES; + } + + // All arrays of references share the same slice + if (!ta->is_flat() && ta->elem()->make_oopptr() != nullptr) { + const TypeAry* tary = TypeAry::make(TypeInstPtr::BOTTOM, TypeInt::POS, false, false, true, true, true); + tj = ta = TypeAryPtr::make(TypePtr::BotPTR, nullptr, tary, nullptr, false, Type::Offset::bottom); + } + + if (ta->is_flat()) { + if (_flat_accesses_share_alias) { + // Initially all flattened array accesses share a single slice + tj = ta = TypeAryPtr::INLINES; + } else { + // Flat accesses are always exact + tj = ta = ta->cast_to_exactness(true); + } } } diff --git a/src/hotspot/share/opto/graphKit.cpp b/src/hotspot/share/opto/graphKit.cpp index 156e91a9b09..91290d3c351 100644 --- a/src/hotspot/share/opto/graphKit.cpp +++ b/src/hotspot/share/opto/graphKit.cpp @@ -1876,30 +1876,36 @@ Node* GraphKit::array_element_address(Node* ary, Node* idx, BasicType elembt, return basic_plus_adr(ary, base, scale); } -Node* GraphKit::cast_to_flat_array(Node* array, ciInlineKlass* vk, bool is_null_free, bool is_not_null_free, bool is_atomic) { - assert(vk->maybe_flat_in_array(), "element of type %s cannot be flat in array", vk->name()->as_utf8()); - if (!vk->has_nullable_atomic_layout()) { +Node* GraphKit::cast_to_flat_array(Node* array, ciInlineKlass* elem_vk) { + assert(elem_vk->maybe_flat_in_array(), "no flat array for %s", elem_vk->name()->as_utf8()); + if (!elem_vk->has_atomic_layout() && !elem_vk->has_nullable_atomic_layout()) { + return cast_to_flat_array_exact(array, elem_vk, true, false); + } else if (!elem_vk->has_nullable_atomic_layout() && !elem_vk->has_non_atomic_layout()) { + return cast_to_flat_array_exact(array, elem_vk, true, true); + } else if (!elem_vk->has_atomic_layout() && !elem_vk->has_non_atomic_layout()) { + return cast_to_flat_array_exact(array, elem_vk, false, true); + } + + bool is_null_free = false; + if (!elem_vk->has_nullable_atomic_layout()) { // Element does not have a nullable flat layout, cannot be nullable is_null_free = true; } - if (!vk->has_atomic_layout() && !vk->has_non_atomic_layout()) { - // Element does not have a null-free flat layout, cannot be null-free - is_not_null_free = true; - } - if (is_null_free) { - // TODO 8350865 Impossible type - is_not_null_free = false; - } - bool is_exact = is_null_free || is_not_null_free; - ciArrayKlass* array_klass = ciArrayKlass::make(vk, is_null_free, is_atomic, true); - assert(array_klass->is_elem_null_free() == is_null_free, "inconsistency"); - assert(array_klass->is_elem_atomic() == is_atomic, "inconsistency"); + ciArrayKlass* array_klass = ciObjArrayKlass::make(elem_vk, false); + const TypeAryPtr* arytype = TypeOopPtr::make_from_klass(array_klass)->isa_aryptr(); + arytype = arytype->cast_to_flat(true)->cast_to_null_free(is_null_free); + return _gvn.transform(new CastPPNode(control(), array, arytype, ConstraintCastNode::StrongDependency)); +} + +Node* GraphKit::cast_to_flat_array_exact(Node* array, ciInlineKlass* elem_vk, bool is_null_free, bool is_atomic) { + assert(is_null_free || is_atomic, "nullable arrays must be atomic"); + ciArrayKlass* array_klass = ciObjArrayKlass::make(elem_vk, true, is_null_free, is_atomic); const TypeAryPtr* arytype = TypeOopPtr::make_from_klass(array_klass)->isa_aryptr(); - arytype = arytype->cast_to_exactness(is_exact); - arytype = arytype->cast_to_not_null_free(is_not_null_free); + assert(arytype->klass_is_exact(), "inconsistency"); + assert(arytype->is_flat(), "inconsistency"); assert(arytype->is_null_free() == is_null_free, "inconsistency"); - assert(arytype->is_not_null_free() == is_not_null_free, "inconsistency"); + assert(arytype->is_not_null_free() == !is_null_free, "inconsistency"); assert(arytype->is_atomic() == is_atomic, "inconsistency"); return _gvn.transform(new CastPPNode(control(), array, arytype, ConstraintCastNode::StrongDependency)); } @@ -4676,7 +4682,7 @@ Node* GraphKit::load_String_value(Node* str, bool set_ctrl) { false, nullptr, Type::Offset(0)); const TypePtr* value_field_type = string_type->add_offset(value_offset); const TypeAryPtr* value_type = TypeAryPtr::make(TypePtr::NotNull, - TypeAry::make(TypeInt::BYTE, TypeInt::POS, false, false, true, true), + TypeAry::make(TypeInt::BYTE, TypeInt::POS, false, false, true, true, true), ciTypeArrayKlass::make(T_BYTE), true, Type::Offset(0)); Node* p = basic_plus_adr(str, str, value_offset); Node* load = access_load_at(str, p, value_field_type, value_type, T_OBJECT, diff --git a/src/hotspot/share/opto/graphKit.hpp b/src/hotspot/share/opto/graphKit.hpp index 9d69401c8be..f3850d67ffd 100644 --- a/src/hotspot/share/opto/graphKit.hpp +++ b/src/hotspot/share/opto/graphKit.hpp @@ -660,7 +660,8 @@ class GraphKit : public Phase { const TypeInt* sizetype = nullptr, // Optional control dependency (for example, on range check) Node* ctrl = nullptr); - Node* cast_to_flat_array(Node* array, ciInlineKlass* elem_vk, bool is_null_free, bool is_not_null_free, bool is_atomic); + Node* cast_to_flat_array(Node* array, ciInlineKlass* elem_vk); + Node* cast_to_flat_array_exact(Node* array, ciInlineKlass* elem_vk, bool is_null_free, bool is_atomic); // Return a load of array element at idx. Node* load_array_element(Node* ary, Node* idx, const TypeAryPtr* arytype, bool set_ctrl); diff --git a/src/hotspot/share/opto/inlinetypenode.cpp b/src/hotspot/share/opto/inlinetypenode.cpp index ae2bfa7f561..d6da996a7fb 100644 --- a/src/hotspot/share/opto/inlinetypenode.cpp +++ b/src/hotspot/share/opto/inlinetypenode.cpp @@ -554,7 +554,7 @@ void InlineTypeNode::store_flat_array(GraphKit* kit, Node* base, Node* idx) { if (!kit->stopped()) { assert(vk->has_nullable_atomic_layout(), "element type %s does not have a nullable flat layout", vk->name()->as_utf8()); kit->set_all_memory(input_memory_state); - Node* cast = kit->cast_to_flat_array(base, vk, false, true, true); + Node* cast = kit->cast_to_flat_array_exact(base, vk, false, true); Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT); store_flat(kit, cast, ptr, true, false, false, decorators); @@ -576,7 +576,7 @@ void InlineTypeNode::store_flat_array(GraphKit* kit, Node* base, Node* idx) { if (!kit->stopped()) { assert(vk->has_atomic_layout(), "element type %s does not have a null-free atomic flat layout", vk->name()->as_utf8()); kit->set_all_memory(input_memory_state); - Node* cast = kit->cast_to_flat_array(base, vk, true, false, true); + Node* cast = kit->cast_to_flat_array_exact(base, vk, true, true); Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT); store_flat(kit, cast, ptr, true, false, true, decorators); @@ -590,7 +590,7 @@ void InlineTypeNode::store_flat_array(GraphKit* kit, Node* base, Node* idx) { if (!kit->stopped()) { assert(vk->has_non_atomic_layout(), "element type %s does not have a null-free non-atomic flat layout", vk->name()->as_utf8()); kit->set_all_memory(input_memory_state); - Node* cast = kit->cast_to_flat_array(base, vk, true, false, false); + Node* cast = kit->cast_to_flat_array_exact(base, vk, true, false); Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT); store_flat(kit, cast, ptr, false, false, true, decorators); @@ -1081,7 +1081,7 @@ InlineTypeNode* InlineTypeNode::make_from_flat_array(GraphKit* kit, ciInlineKlas if (!kit->stopped()) { assert(vk->has_nullable_atomic_layout(), "element type %s does not have a nullable flat layout", vk->name()->as_utf8()); kit->set_all_memory(input_memory_state); - Node* cast = kit->cast_to_flat_array(base, vk, false, true, true); + Node* cast = kit->cast_to_flat_array_exact(base, vk, false, true); Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT); vt_nullable = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, true, false, false, decorators); @@ -1103,7 +1103,7 @@ InlineTypeNode* InlineTypeNode::make_from_flat_array(GraphKit* kit, ciInlineKlas if (!kit->stopped()) { assert(vk->has_atomic_layout(), "element type %s does not have a null-free atomic flat layout", vk->name()->as_utf8()); kit->set_all_memory(input_memory_state); - Node* cast = kit->cast_to_flat_array(base, vk, true, false, true); + Node* cast = kit->cast_to_flat_array_exact(base, vk, true, true); Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT); vt_null_free = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, true, false, true, decorators); @@ -1117,7 +1117,7 @@ InlineTypeNode* InlineTypeNode::make_from_flat_array(GraphKit* kit, ciInlineKlas if (!kit->stopped()) { assert(vk->has_non_atomic_layout(), "element type %s does not have a null-free non-atomic flat layout", vk->name()->as_utf8()); kit->set_all_memory(input_memory_state); - Node* cast = kit->cast_to_flat_array(base, vk, true, false, false); + Node* cast = kit->cast_to_flat_array_exact(base, vk, true, false); Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT); vt_non_atomic = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, false, false, true, decorators); diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp index e719beca418..76d7234a0bc 100644 --- a/src/hotspot/share/opto/library_call.cpp +++ b/src/hotspot/share/opto/library_call.cpp @@ -2885,7 +2885,7 @@ bool LibraryCallKit::inline_unsafe_flat_access(bool is_store, AccessKind kind) { // Flat array must have an exact type bool is_null_free = layout != LayoutKind::NULLABLE_ATOMIC_FLAT; bool is_atomic = layout != LayoutKind::NON_ATOMIC_FLAT; - Node* new_base = cast_to_flat_array(base, value_klass, is_null_free, !is_null_free, is_atomic); + Node* new_base = cast_to_flat_array_exact(base, value_klass, is_null_free, is_atomic); replace_in_map(base, new_base); base = new_base; ptr = basic_plus_adr(base, ConvL2X(offset)); @@ -4198,7 +4198,7 @@ bool LibraryCallKit::inline_native_setCurrentThread() { const Type* LibraryCallKit::scopedValueCache_type() { ciKlass* objects_klass = ciObjArrayKlass::make(env()->Object_klass()); const TypeOopPtr* etype = TypeOopPtr::make_from_klass(env()->Object_klass()); - const TypeAry* arr0 = TypeAry::make(etype, TypeInt::POS, /* stable= */ false, /* flat= */ false, /* not_flat= */ true, /* not_null_free= */ true); + const TypeAry* arr0 = TypeAry::make(etype, TypeInt::POS, /* stable= */ false, /* flat= */ false, /* not_flat= */ true, /* not_null_free= */ true, true); // Because we create the scopedValue cache lazily we have to make the // type of the result BotPTR. @@ -4813,7 +4813,7 @@ bool LibraryCallKit::inline_newArray(bool null_free, bool atomic) { } if (array_klass->is_loaded() && array_klass->element_klass()->as_inline_klass()->is_initialized()) { - const TypeAryKlassPtr* array_klass_type = TypeAryKlassPtr::make(array_klass, Type::trust_interfaces, true); + const TypeAryKlassPtr* array_klass_type = TypeAryKlassPtr::make(array_klass, Type::trust_interfaces); if (null_free) { if (init_val->is_InlineType()) { if (array_klass_type->is_flat() && init_val->as_InlineType()->is_all_zero(&gvn(), /* flat */ true)) { @@ -5088,7 +5088,7 @@ bool LibraryCallKit::inline_array_copyOf(bool is_copyOfRange) { if (not_objArray != nullptr) { // Improve the klass node's type from the new optimistic assumption: ciKlass* ak = ciArrayKlass::make(env()->Object_klass()); - const Type* akls = TypeKlassPtr::make(TypePtr::NotNull, ak, Type::Offset(0)); + const Type* akls = TypeAryKlassPtr::make(TypePtr::NotNull, ak, Type::Offset(0), Type::trust_interfaces, false, false, false, false, false, true); Node* cast = new CastPPNode(control(), refined_klass_node, akls); refined_klass_node = _gvn.transform(cast); } @@ -6713,7 +6713,7 @@ bool LibraryCallKit::inline_arraycopy() { assert(stopped(), "Should be stopped"); } - const TypeKlassPtr* dest_klass_t = _gvn.type(refined_dest_klass)->is_klassptr(); + const TypeKlassPtr* dest_klass_t = _gvn.type(dest_klass)->is_klassptr(); const Type* toop = dest_klass_t->cast_to_exactness(false)->as_instance_type(); src = _gvn.transform(new CheckCastPPNode(control(), src, toop)); arraycopy_move_allocation_here(alloc, dest, saved_jvms_before_guards, saved_reexecute_sp, new_idx); diff --git a/src/hotspot/share/opto/memnode.cpp b/src/hotspot/share/opto/memnode.cpp index ed6145a62f7..1766ff31470 100644 --- a/src/hotspot/share/opto/memnode.cpp +++ b/src/hotspot/share/opto/memnode.cpp @@ -24,6 +24,7 @@ */ #include "ci/ciFlatArrayKlass.hpp" +#include "ci/ciInlineKlass.hpp" #include "classfile/javaClasses.hpp" #include "classfile/systemDictionary.hpp" #include "compiler/compileLog.hpp" @@ -36,6 +37,7 @@ #include "oops/objArrayKlass.hpp" #include "opto/addnode.hpp" #include "opto/arraycopynode.hpp" +#include "opto/callnode.hpp" #include "opto/cfgnode.hpp" #include "opto/compile.hpp" #include "opto/connode.hpp" @@ -1236,6 +1238,7 @@ static Node* see_through_inline_type(PhaseValues* phase, const MemNode* load, No // same time (uses the Oracle model of aliasing), then some // LoadXNode::Identity will fold things back to the equivalence-class model // of aliasing. +// This method may find an unencoded node instead of the corresponding encoded one. Node* MemNode::can_see_stored_value(Node* st, PhaseValues* phase) const { Node* ld_adr = in(MemNode::Address); intptr_t ld_off = 0; @@ -1352,9 +1355,31 @@ Node* MemNode::can_see_stored_value(Node* st, PhaseValues* phase) const { // virtually pre-existing constants.) Node* init_value = ld_alloc->in(AllocateNode::InitValue); if (init_value != nullptr) { - // TODO 8350865 Scalar replacement does not work well for flat arrays. - // Is this correct for non-all-zero init values? Don't we need field_value_by_offset? - return init_value; + const TypeAryPtr* ld_adr_type = phase->type(ld_adr)->isa_aryptr(); + if (ld_adr_type == nullptr) { + return nullptr; + } + + // We know that this is not a flat array, the load should return the whole oop + if (ld_adr_type->is_not_flat()) { + return init_value; + } + + // If this is a flat array, try to see through init_value + if (init_value->is_EncodeP()) { + init_value = init_value->in(1); + } + if (!init_value->is_InlineType() || ld_adr_type->field_offset() == Type::Offset::bottom) { + return nullptr; + } + + ciInlineKlass* vk = phase->type(init_value)->inline_klass(); + int field_offset_in_payload = ld_adr_type->field_offset().get(); + if (field_offset_in_payload == vk->null_marker_offset_in_payload()) { + return init_value->as_InlineType()->get_null_marker(); + } else { + return init_value->as_InlineType()->field_value_by_offset(field_offset_in_payload + vk->payload_offset(), true); + } } assert(ld_alloc->in(AllocateNode::RawInitValue) == nullptr, "init value may not be null"); if (value_basic_type() != T_VOID) { @@ -1439,6 +1464,10 @@ Node* LoadNode::Identity(PhaseGVN* phase) { if (!phase->type(value)->higher_equal(phase->type(this))) return this; } + + if (phase->type(value)->isa_ptr() && phase->type(this)->isa_narrowoop()) { + return this; + } // (This works even when value is a Con, but LoadNode::Value // usually runs first, producing the singleton type of the Con.) if (!has_pinned_control_dependency() || value->is_Con()) { @@ -2171,8 +2200,12 @@ const Type* LoadNode::Value(PhaseGVN* phase) const { bool is_instance = (tinst != nullptr) && tinst->is_known_instance_field(); Node* value = can_see_stored_value(mem, phase); if (value != nullptr && value->is_Con()) { - assert(value->bottom_type()->higher_equal(_type), "sanity"); - return value->bottom_type(); + if (phase->type(value)->isa_ptr() && _type->isa_narrowoop()) { + return phase->type(value)->make_narrowoop(); + } else { + assert(value->bottom_type()->higher_equal(_type), "sanity"); + return phase->type(value); + } } // Try to guess loaded type from pointer type @@ -2589,6 +2622,13 @@ Node* LoadNNode::Ideal(PhaseGVN* phase, bool can_reshape) { return new EncodePNode(value, type()); } + // Can see the corresponding value, may need to add an EncodeP + value = can_see_stored_value(in(Memory), phase); + if (value != nullptr && phase->type(value)->isa_ptr() && type()->isa_narrowoop()) { + return new EncodePNode(value, type()); + } + + // Identity call will handle the case where EncodeP is unnecessary return LoadNode::Ideal(phase, can_reshape); } @@ -2668,9 +2708,7 @@ const Type* LoadNode::klass_value_common(PhaseGVN* phase) const { const TypeAryPtr* tary = tp->isa_aryptr(); if (tary != nullptr && tary->offset() == oopDesc::klass_offset_in_bytes()) { - const TypeAryKlassPtr* res = tary->as_klass_type(true)->is_aryklassptr(); - // The klass of an array object must be a refined array klass - return res->cast_to_refined_array_klass_ptr(); + return tary->as_klass_type(true)->is_aryklassptr(); } // Check for loading klass from an array klass @@ -2693,7 +2731,7 @@ const Type* LoadNode::klass_value_common(PhaseGVN* phase) const { tkls->offset() == in_bytes(Klass::super_offset())) { // We are loading the super klass of a refined array klass, return the non-refined klass pointer assert(tkls->is_aryklassptr()->is_refined_type(), "Must be a refined array klass pointer"); - return tkls->is_aryklassptr()->cast_to_refined_array_klass_ptr(false); + return tkls->is_aryklassptr()->with_offset(0)->cast_to_non_refined(); } if (tkls->isa_instklassptr() != nullptr && tkls->klass_is_exact() && tkls->offset() == in_bytes(Klass::super_offset())) { diff --git a/src/hotspot/share/opto/parse.hpp b/src/hotspot/share/opto/parse.hpp index d6f7b3d01a5..85a3f16b1ff 100644 --- a/src/hotspot/share/opto/parse.hpp +++ b/src/hotspot/share/opto/parse.hpp @@ -443,7 +443,7 @@ class Parse : public GraphKit { // OSR helpers Node* fetch_interpreter_state(int index, const Type* type, Node* local_addrs, Node* local_addrs_base); - Node* check_interpreter_type(Node* l, const Type* type, SafePointNode* &bad_type_exit, bool is_larval); + Node* check_interpreter_type(Node* l, const Type* type, const TypeKlassPtr* klass_type, SafePointNode* &bad_type_exit, bool is_larval); void load_interpreter_state(Node* osr_buf); // Functions for managing basic blocks: diff --git a/src/hotspot/share/opto/parse1.cpp b/src/hotspot/share/opto/parse1.cpp index 4309e0deaf4..19e822fd175 100644 --- a/src/hotspot/share/opto/parse1.cpp +++ b/src/hotspot/share/opto/parse1.cpp @@ -152,7 +152,7 @@ Node* Parse::fetch_interpreter_state(int index, // The type is the type predicted by ciTypeFlow. Note that it is // not a general type, but can only come from Type::get_typeflow_type. // The safepoint is a map which will feed an uncommon trap. -Node* Parse::check_interpreter_type(Node* l, const Type* type, +Node* Parse::check_interpreter_type(Node* l, const Type* type, const TypeKlassPtr* klass_type, SafePointNode* &bad_type_exit, bool is_early_larval) { const TypeOopPtr* tp = type->isa_oopptr(); @@ -184,7 +184,7 @@ Node* Parse::check_interpreter_type(Node* l, const Type* type, bad_type_exit->control()->add_req(bad_type_ctrl); } - l = gen_checkcast(l, makecon(tp->as_klass_type()->cast_to_exactness(true)), &bad_type_ctrl, false, is_early_larval); + l = gen_checkcast(l, makecon(klass_type), &bad_type_ctrl, false, is_early_larval); bad_type_exit->control()->add_req(bad_type_ctrl); } @@ -375,17 +375,27 @@ void Parse::load_interpreter_state(Node* osr_buf) { // value and the expected type is a constant. continue; } + const TypeKlassPtr* klass_type = nullptr; + if (type->isa_oopptr()) { + klass_type = TypeKlassPtr::make(osr_block->flow()->local_type_at(index)->unwrap()->as_klass(), Type::ignore_interfaces); + klass_type = klass_type->try_improve(); + } bool is_early_larval = osr_block->flow()->local_type_at(index)->is_early_larval(); - set_local(index, check_interpreter_type(l, type, bad_type_exit, is_early_larval)); + set_local(index, check_interpreter_type(l, type, klass_type, bad_type_exit, is_early_larval)); } for (index = 0; index < sp(); index++) { if (stopped()) break; Node* l = stack(index); if (l->is_top()) continue; // nothing here - const Type *type = osr_block->stack_type_at(index); + const Type* type = osr_block->stack_type_at(index); + const TypeKlassPtr* klass_type = nullptr; + if (type->isa_oopptr()) { + klass_type = TypeKlassPtr::make(osr_block->flow()->stack_type_at(index)->unwrap()->as_klass(), Type::ignore_interfaces); + klass_type = klass_type->try_improve(); + } bool is_early_larval = osr_block->flow()->stack_type_at(index)->is_early_larval(); - set_stack(index, check_interpreter_type(l, type, bad_type_exit, is_early_larval)); + set_stack(index, check_interpreter_type(l, type, klass_type, bad_type_exit, is_early_larval)); } if (bad_type_exit->control()->req() > 1) { diff --git a/src/hotspot/share/opto/parse2.cpp b/src/hotspot/share/opto/parse2.cpp index 20e8d1fbf65..a36af03fa62 100644 --- a/src/hotspot/share/opto/parse2.cpp +++ b/src/hotspot/share/opto/parse2.cpp @@ -116,7 +116,7 @@ void Parse::array_load(BasicType bt) { if (!array_type->is_not_flat()) { if (element_ptr->is_inlinetypeptr()) { ciInlineKlass* vk = element_ptr->inline_klass(); - Node* flat_array = cast_to_flat_array(array, vk, false, false, false); + Node* flat_array = cast_to_flat_array(array, vk); Node* vt = InlineTypeNode::make_from_flat_array(this, vk, flat_array, array_index); ideal.set(res, vt); } else { @@ -267,7 +267,7 @@ void Parse::array_store(BasicType bt) { if (vk != nullptr) { // Element type is known, cast and store to flat array layout. - Node* flat_array = cast_to_flat_array(array, vk, false, false, false); + Node* flat_array = cast_to_flat_array(array, vk); // Re-execute flat array store if buffering triggers deoptimization PreserveReexecuteState preexecs(this); diff --git a/src/hotspot/share/opto/subnode.cpp b/src/hotspot/share/opto/subnode.cpp index 72ade247113..42883200030 100644 --- a/src/hotspot/share/opto/subnode.cpp +++ b/src/hotspot/share/opto/subnode.cpp @@ -22,6 +22,7 @@ * */ +#include "ci/ciObjArrayKlass.hpp" #include "compiler/compileLog.hpp" #include "gc/shared/barrierSet.hpp" #include "gc/shared/c2/barrierSetC2.hpp" @@ -1192,10 +1193,11 @@ static inline Node* isa_const_java_mirror(PhaseGVN* phase, Node* n, bool& might_ // return the ConP(Foo.klass) ciKlass* mirror_klass = mirror_type->as_klass(); - if (mirror_klass->is_array_klass()) { + if (mirror_klass->is_array_klass() && !mirror_klass->is_type_array_klass()) { if (!mirror_klass->can_be_inline_array_klass()) { // Special case for non-value arrays: They only have one (default) refined class, use it - return phase->makecon(TypeAryKlassPtr::make(mirror_klass, Type::trust_interfaces, true)); + ciArrayKlass* refined_mirror_klass = ciObjArrayKlass::make(mirror_klass->as_array_klass()->element_klass(), true); + return phase->makecon(TypeAryKlassPtr::make(refined_mirror_klass, Type::trust_interfaces)); } might_be_an_array |= true; } @@ -1324,8 +1326,7 @@ Node* CmpPNode::Ideal(PhaseGVN *phase, bool can_reshape) { superklass = t2->exact_klass(); assert(!superklass->is_flat_array_klass(), "Unexpected flat array klass"); if (superklass->is_obj_array_klass()) { - if (!superklass->as_array_klass()->is_elem_null_free() && - superklass->as_array_klass()->element_klass()->is_inlinetype()) { + if (superklass->as_array_klass()->element_klass()->is_inlinetype() && !superklass->as_array_klass()->is_refined()) { return nullptr; } else { // Special case for non-value arrays: They only have one (default) refined class, use it diff --git a/src/hotspot/share/opto/type.cpp b/src/hotspot/share/opto/type.cpp index fb338c888c4..00870b02e74 100644 --- a/src/hotspot/share/opto/type.cpp +++ b/src/hotspot/share/opto/type.cpp @@ -26,6 +26,7 @@ #include "ci/ciFlatArrayKlass.hpp" #include "ci/ciInlineKlass.hpp" #include "ci/ciMethodData.hpp" +#include "ci/ciObjArrayKlass.hpp" #include "ci/ciTypeFlow.hpp" #include "classfile/javaClasses.hpp" #include "classfile/symbolTable.hpp" @@ -46,6 +47,7 @@ #include "opto/rangeinference.hpp" #include "opto/runtime.hpp" #include "opto/type.hpp" +#include "runtime/globals.hpp" #include "runtime/stubRoutines.hpp" #include "utilities/checkedCast.hpp" #include "utilities/globalDefinitions.hpp" @@ -653,10 +655,10 @@ void Type::Initialize_shared(Compile* current) { TypeAryPtr::_array_interfaces = TypeInterfaces::make(&array_interfaces); TypeAryKlassPtr::_array_interfaces = TypeAryPtr::_array_interfaces; - TypeAryPtr::BOTTOM = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(Type::BOTTOM, TypeInt::POS), nullptr, false, Offset::bottom); - TypeAryPtr::RANGE = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(Type::BOTTOM,TypeInt::POS), nullptr /* current->env()->Object_klass() */, false, Offset(arrayOopDesc::length_offset_in_bytes())); + TypeAryPtr::BOTTOM = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(Type::BOTTOM, TypeInt::POS, false, false, false, false, false), nullptr, false, Offset::bottom); + TypeAryPtr::RANGE = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(Type::BOTTOM,TypeInt::POS, false, false, false, false, false), nullptr /* current->env()->Object_klass() */, false, Offset(arrayOopDesc::length_offset_in_bytes())); - TypeAryPtr::NARROWOOPS = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeNarrowOop::BOTTOM, TypeInt::POS), nullptr /*ciArrayKlass::make(o)*/, false, Offset::bottom); + TypeAryPtr::NARROWOOPS = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeNarrowOop::BOTTOM, TypeInt::POS, false, false, false, false, false), nullptr /*ciArrayKlass::make(o)*/, false, Offset::bottom); #ifdef _LP64 if (UseCompressedOops) { @@ -666,16 +668,16 @@ void Type::Initialize_shared(Compile* current) { #endif { // There is no shared klass for Object[]. See note in TypeAryPtr::klass(). - TypeAryPtr::OOPS = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeInstPtr::BOTTOM,TypeInt::POS), nullptr /*ciArrayKlass::make(o)*/, false, Offset::bottom); + TypeAryPtr::OOPS = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeInstPtr::BOTTOM,TypeInt::POS, false, false, false, false, false), nullptr /*ciArrayKlass::make(o)*/, false, Offset::bottom); } - TypeAryPtr::BYTES = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeInt::BYTE ,TypeInt::POS), ciTypeArrayKlass::make(T_BYTE), true, Offset::bottom); - TypeAryPtr::SHORTS = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeInt::SHORT ,TypeInt::POS), ciTypeArrayKlass::make(T_SHORT), true, Offset::bottom); - TypeAryPtr::CHARS = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeInt::CHAR ,TypeInt::POS), ciTypeArrayKlass::make(T_CHAR), true, Offset::bottom); - TypeAryPtr::INTS = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeInt::INT ,TypeInt::POS), ciTypeArrayKlass::make(T_INT), true, Offset::bottom); - TypeAryPtr::LONGS = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeLong::LONG ,TypeInt::POS), ciTypeArrayKlass::make(T_LONG), true, Offset::bottom); - TypeAryPtr::FLOATS = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(Type::FLOAT ,TypeInt::POS), ciTypeArrayKlass::make(T_FLOAT), true, Offset::bottom); - TypeAryPtr::DOUBLES = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(Type::DOUBLE ,TypeInt::POS), ciTypeArrayKlass::make(T_DOUBLE), true, Offset::bottom); - TypeAryPtr::INLINES = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeInstPtr::BOTTOM,TypeInt::POS, /* stable= */ false, /* flat= */ true), nullptr, false, Offset::bottom); + TypeAryPtr::BYTES = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeInt::BYTE ,TypeInt::POS, false, false, true, true, true), ciTypeArrayKlass::make(T_BYTE), true, Offset::bottom); + TypeAryPtr::SHORTS = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeInt::SHORT ,TypeInt::POS, false, false, true, true, true), ciTypeArrayKlass::make(T_SHORT), true, Offset::bottom); + TypeAryPtr::CHARS = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeInt::CHAR ,TypeInt::POS, false, false, true, true, true), ciTypeArrayKlass::make(T_CHAR), true, Offset::bottom); + TypeAryPtr::INTS = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeInt::INT ,TypeInt::POS, false, false, true, true, true), ciTypeArrayKlass::make(T_INT), true, Offset::bottom); + TypeAryPtr::LONGS = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeLong::LONG ,TypeInt::POS, false, false, true, true, true), ciTypeArrayKlass::make(T_LONG), true, Offset::bottom); + TypeAryPtr::FLOATS = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(Type::FLOAT ,TypeInt::POS, false, false, true, true, true), ciTypeArrayKlass::make(T_FLOAT), true, Offset::bottom); + TypeAryPtr::DOUBLES = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(Type::DOUBLE ,TypeInt::POS, false, false, true, true, true), ciTypeArrayKlass::make(T_DOUBLE), true, Offset::bottom); + TypeAryPtr::INLINES = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeInstPtr::BOTTOM,TypeInt::POS, /* stable= */ false, /* flat= */ true, false, false, false), nullptr, false, Offset::bottom); // Nobody should ask _array_body_type[T_NARROWOOP]. Use null as assert. TypeAryPtr::_array_body_type[T_NARROWOOP] = nullptr; @@ -3832,8 +3834,9 @@ const TypeOopPtr* TypeOopPtr::make_from_klass_common(ciKlass *klass, bool klass_ } else if (klass->is_obj_array_klass()) { // Element is an object or inline type array. Recursively call ourself. const TypeOopPtr* etype = TypeOopPtr::make_from_klass_common(klass->as_array_klass()->element_klass(), /* klass_change= */ false, try_for_exact, interface_handling); + bool xk = klass->is_loaded() && klass->as_obj_array_klass()->is_refined(); // Determine null-free/flat properties - const bool is_null_free = klass->as_array_klass()->is_elem_null_free(); + const bool is_null_free = xk && klass->as_array_klass()->is_elem_null_free(); if (is_null_free) { etype = etype->join_speculative(NOTNULL)->is_oopptr(); } @@ -3843,11 +3846,9 @@ const TypeOopPtr* TypeOopPtr::make_from_klass_common(ciKlass *klass, bool klass_ exact_etype = TypeOopPtr::make_from_klass_common(klass->as_array_klass()->element_klass(), /* klass_change= */ true, /* try_for_exact= */ true, interface_handling); } bool not_inline = !exact_etype->can_be_inline_type(); - bool not_null_free = not_inline; - bool not_flat = !UseArrayFlattening || not_inline || (exact_etype->is_inlinetypeptr() && !exact_etype->inline_klass()->maybe_flat_in_array()); - bool atomic = klass->as_array_klass()->is_elem_atomic(); - // Even though MyValue is final, [LMyValue is not exact because null-free [LMyValue is a subtype. - bool xk = etype->klass_is_exact() && !etype->is_inlinetypeptr(); + bool not_null_free = xk ? !is_null_free : not_inline; + bool not_flat = xk || !UseArrayFlattening || not_inline || (exact_etype->is_inlinetypeptr() && !exact_etype->inline_klass()->maybe_flat_in_array()); + bool atomic = not_flat; const TypeAry* arr0 = TypeAry::make(etype, TypeInt::POS, /* stable= */ false, /* flat= */ false, not_flat, not_null_free, atomic); // We used to pass NotNull in here, asserting that the sub-arrays // are all not-null. This is not true in generally, as code can @@ -3858,7 +3859,7 @@ const TypeOopPtr* TypeOopPtr::make_from_klass_common(ciKlass *klass, bool klass_ // Element is an typeArray const Type* etype = get_const_basic_type(klass->as_type_array_klass()->element_type()); const TypeAry* arr0 = TypeAry::make(etype, TypeInt::POS, - /* stable= */ false, /* flat= */ false, /* not_flat= */ true, /* not_null_free= */ true); + /* stable= */ false, /* flat= */ false, /* not_flat= */ true, /* not_null_free= */ true, true); // We used to pass NotNull in here, asserting that the array pointer // is not-null. That was not true in general. const TypeAryPtr* arr = TypeAryPtr::make(TypePtr::BotPTR, arr0, klass, true, Offset(0)); @@ -3870,9 +3871,8 @@ const TypeOopPtr* TypeOopPtr::make_from_klass_common(ciKlass *klass, bool klass_ etype = etype->join_speculative(NOTNULL)->is_oopptr(); } bool atomic = klass->as_array_klass()->is_elem_atomic(); - const TypeAry* arr0 = TypeAry::make(etype, TypeInt::POS, /* stable= */ false, /* flat= */ true, /* not_flat= */ false, /* not_null_free= */ false, atomic); - const bool exact = is_null_free; // Only exact if null-free because "null-free [LMyValue <: null-able [LMyValue". - const TypeAryPtr* arr = TypeAryPtr::make(TypePtr::BotPTR, arr0, klass, exact, Offset(0)); + const TypeAry* arr0 = TypeAry::make(etype, TypeInt::POS, /* stable= */ false, /* flat= */ true, /* not_flat= */ false, /* not_null_free= */ !is_null_free, atomic); + const TypeAryPtr* arr = TypeAryPtr::make(TypePtr::BotPTR, arr0, klass, true, Offset(0)); return arr; } else { ShouldNotReachHere(); @@ -3918,7 +3918,7 @@ const TypeOopPtr* TypeOopPtr::make_from_constant(ciObject* o, bool require_const // Element is an typeArray const Type* etype = (Type*)get_const_basic_type(klass->as_type_array_klass()->element_type()); const TypeAry* arr0 = TypeAry::make(etype, TypeInt::make(o->as_array()->length()), /* stable= */ false, /* flat= */ false, - /* not_flat= */ true, /* not_null_free= */ true); + /* not_flat= */ true, /* not_null_free= */ true, true); // We used to pass NotNull in here, asserting that the array pointer // is not-null. That was not true in general. if (make_constant) { @@ -5062,6 +5062,19 @@ const TypeAryPtr* TypeAryPtr::cast_to_size(const TypeInt* new_size) const { return make(ptr(), const_oop(), new_ary, klass(), klass_is_exact(), _offset, _field_offset, _instance_id, _speculative, _inline_depth, _is_autobox_cache); } +const TypeAryPtr* TypeAryPtr::cast_to_flat(bool flat) const { + if (flat == is_flat()) { + return this; + } + assert(!flat || !is_not_flat(), "inconsistency"); + const TypeAry* new_ary = TypeAry::make(elem(), size(), is_stable(), flat, is_not_flat(), is_not_null_free(), is_atomic()); + const TypeAryPtr* res = make(ptr(), const_oop(), new_ary, klass(), klass_is_exact(), _offset, _field_offset, _instance_id, _speculative, _inline_depth, _is_autobox_cache); + if (res->speculative() == res->remove_speculative()) { + return res->remove_speculative(); + } + return res; +} + //-------------------------------cast_to_not_flat------------------------------ const TypeAryPtr* TypeAryPtr::cast_to_not_flat(bool not_flat) const { if (not_flat == is_not_flat()) { @@ -5078,6 +5091,27 @@ const TypeAryPtr* TypeAryPtr::cast_to_not_flat(bool not_flat) const { return res; } +const TypeAryPtr* TypeAryPtr::cast_to_null_free(bool null_free) const { + if (null_free == is_null_free()) { + return this; + } + assert(!null_free || !is_not_null_free(), "inconsistency"); + const Type* elem = this->elem(); + const Type* new_elem = elem->make_ptr(); + if (null_free) { + new_elem = new_elem->join_speculative(TypePtr::NOTNULL); + } else { + new_elem = new_elem->meet_speculative(TypePtr::NULL_PTR); + } + new_elem = elem->isa_narrowoop() ? new_elem->make_narrowoop() : new_elem; + const TypeAry* new_ary = TypeAry::make(new_elem, size(), is_stable(), is_flat(), is_not_flat(), is_not_null_free(), is_atomic()); + const TypeAryPtr* res = make(ptr(), const_oop(), new_ary, klass(), klass_is_exact(), _offset, _field_offset, _instance_id, _speculative, _inline_depth, _is_autobox_cache); + if (res->speculative() == res->remove_speculative()) { + return res->remove_speculative(); + } + return res; +} + //-------------------------------cast_to_not_null_free------------------------- const TypeAryPtr* TypeAryPtr::cast_to_not_null_free(bool not_null_free) const { if (not_null_free == is_not_null_free()) { @@ -6062,18 +6096,15 @@ const TypeMetadataPtr* TypeMetadataPtr::make(PTR ptr, ciMetadata* m, Offset offs const TypeKlassPtr* TypeAryPtr::as_klass_type(bool try_for_exact) const { const Type* elem = _ary->_elem; bool xk = klass_is_exact(); + bool is_refined = false; if (elem->make_oopptr() != nullptr) { + is_refined = true; elem = elem->make_oopptr()->as_klass_type(try_for_exact); - if (elem->is_klassptr()->klass_is_exact() && - // Even though MyValue is final, [LMyValue is only exact if the array - // is (not) null-free due to null-free [LMyValue <: null-able [LMyValue. - // TODO 8350865 If we know that the array can't be null-free, it's allowed to be exact, right? - // If so, we should add '|| is_not_null_free()' - (is_null_free() || !_ary->_elem->make_oopptr()->is_inlinetypeptr())) { - xk = true; + if (elem->isa_aryklassptr() && elem->is_aryklassptr()->is_refined_type()) { + elem = elem->is_aryklassptr()->cast_to_non_refined(); } } - return TypeAryKlassPtr::make(xk ? TypePtr::Constant : TypePtr::NotNull, elem, klass(), Offset(0), is_not_flat(), is_not_null_free(), is_flat(), is_null_free(), is_atomic(), is_flat() || is_null_free()); + return TypeAryKlassPtr::make(xk ? TypePtr::Constant : TypePtr::NotNull, elem, klass(), Offset(0), is_not_flat(), is_not_null_free(), is_flat(), is_null_free(), is_atomic(), is_refined); } const TypeKlassPtr* TypeKlassPtr::make(ciKlass* klass, InterfaceHandling interface_handling) { @@ -6083,14 +6114,6 @@ const TypeKlassPtr* TypeKlassPtr::make(ciKlass* klass, InterfaceHandling interfa return TypeAryKlassPtr::make(klass, interface_handling); } -const TypeKlassPtr* TypeKlassPtr::make(PTR ptr, ciKlass* klass, Offset offset, InterfaceHandling interface_handling) { - if (klass->is_instance_klass()) { - const TypeInterfaces* interfaces = TypePtr::interfaces(klass, true, true, false, interface_handling); - return TypeInstKlassPtr::make(ptr, klass, interfaces, offset); - } - return TypeAryKlassPtr::make(ptr, klass, offset, interface_handling); -} - TypeKlassPtr::TypeKlassPtr(TYPES t, PTR ptr, ciKlass* klass, const TypeInterfaces* interfaces, Offset offset) : TypePtr(t, ptr, offset), _klass(klass), _interfaces(interfaces) { assert(klass == nullptr || !klass->is_loaded() || (klass->is_instance_klass() && !klass->is_interface()) || @@ -6558,19 +6581,16 @@ const TypeKlassPtr* TypeInstKlassPtr::try_improve() const { Compile* C = Compile::current(); Dependencies* deps = C->dependencies(); assert((deps != nullptr) == (C->method() != nullptr && C->method()->code_size() > 0), "sanity"); - const TypeInterfaces* interfaces = _interfaces; if (k->is_loaded()) { ciInstanceKlass* ik = k->as_instance_klass(); - bool klass_is_exact = ik->is_final(); - if (!klass_is_exact && - deps != nullptr) { + if (deps != nullptr) { ciInstanceKlass* sub = ik->unique_concrete_subklass(); if (sub != nullptr) { - if (_interfaces->eq(sub)) { + bool improve_to_exact = sub->is_final() && _ptr == NotNull; + const TypeInstKlassPtr* improved = TypeInstKlassPtr::make(improve_to_exact ? Constant : _ptr, sub, _offset); + if (improved->_interfaces->contains(_interfaces)) { deps->assert_abstract_with_unique_concrete_subtype(ik, sub); - k = ik = sub; - klass_is_exact = sub->is_final(); - return TypeKlassPtr::make(klass_is_exact ? Constant : _ptr, k, _offset); + return improved; } } } @@ -6599,42 +6619,45 @@ const TypeAryKlassPtr *TypeAryKlassPtr::make(PTR ptr, const Type* elem, ciKlass* } const TypeAryKlassPtr* TypeAryKlassPtr::make(PTR ptr, ciKlass* k, Offset offset, InterfaceHandling interface_handling, bool not_flat, bool not_null_free, bool flat, bool null_free, bool atomic, bool refined_type) { + const Type* etype; if (k->is_obj_array_klass()) { // Element is an object array. Recursively call ourself. ciKlass* eklass = k->as_obj_array_klass()->element_klass(); - const TypeKlassPtr* etype = TypeKlassPtr::make(eklass, interface_handling)->cast_to_exactness(false); - return TypeAryKlassPtr::make(ptr, etype, nullptr, offset, not_flat, not_null_free, flat, null_free, atomic, refined_type); + etype = TypeKlassPtr::make(eklass, interface_handling)->cast_to_exactness(false); + k = nullptr; } else if (k->is_type_array_klass()) { // Element is an typeArray - const Type* etype = get_const_basic_type(k->as_type_array_klass()->element_type()); - return TypeAryKlassPtr::make(ptr, etype, k, offset, not_flat, not_null_free, flat, null_free, atomic); + etype = get_const_basic_type(k->as_type_array_klass()->element_type()); } else if (k->is_flat_array_klass()) { ciKlass* eklass = k->as_flat_array_klass()->element_klass(); - const TypeKlassPtr* etype = TypeKlassPtr::make(eklass, interface_handling)->cast_to_exactness(false); - return TypeAryKlassPtr::make(ptr, etype, k, offset, not_flat, not_null_free, flat, null_free, atomic, refined_type); + etype = TypeKlassPtr::make(eklass, interface_handling)->cast_to_exactness(false); + k = nullptr; } else { ShouldNotReachHere(); - return nullptr; } -} -const TypeAryKlassPtr* TypeAryKlassPtr::make(PTR ptr, ciKlass* k, Offset offset, InterfaceHandling interface_handling, bool refined_type) { - bool flat = k->is_flat_array_klass(); - bool null_free = k->as_array_klass()->is_elem_null_free(); - bool atomic = k->as_array_klass()->is_elem_atomic(); - - bool not_inline = k->is_type_array_klass() || !k->as_array_klass()->element_klass()->can_be_inline_klass(false); - bool not_null_free = (ptr == Constant) ? !null_free : not_inline; - bool not_flat = (ptr == Constant) ? !flat : (!UseArrayFlattening || not_inline || - (k->as_array_klass()->element_klass() != nullptr && - k->as_array_klass()->element_klass()->is_inlinetype() && - !k->as_array_klass()->element_klass()->maybe_flat_in_array())); + return TypeAryKlassPtr::make(ptr, etype, k, offset, not_flat, not_null_free, flat, null_free, atomic, refined_type); +} - return TypeAryKlassPtr::make(ptr, k, offset, interface_handling, not_flat, not_null_free, flat, null_free, atomic, refined_type); +const TypeAryKlassPtr* TypeAryKlassPtr::make(ciKlass* klass, InterfaceHandling interface_handling) { + ciArrayKlass* k = klass->as_array_klass(); + if (k->is_refined()) { + return TypeAryKlassPtr::make(Constant, k, Offset(0), interface_handling, !k->is_flat_array_klass(), !k->is_elem_null_free(), + k->is_flat_array_klass(), k->is_elem_null_free(), k->is_elem_atomic(), true); + } else { + // Use the default combination to canonicalize all non-refined klass pointers + return TypeAryKlassPtr::make(Constant, k, Offset(0), interface_handling, true, true, false, false, true, false); + } } -const TypeAryKlassPtr* TypeAryKlassPtr::make(ciKlass* klass, InterfaceHandling interface_handling, bool refined_type) { - return TypeAryKlassPtr::make(Constant, klass, Offset(0), interface_handling, refined_type); +const TypeAryKlassPtr* TypeAryKlassPtr::cast_to_non_refined() const { + assert(is_refined_type(), "must be a refined type"); + PTR ptr = _ptr; + // There can be multiple refined array types corresponding to a single unrefined type + if (ptr == NotNull && elem()->is_klassptr()->klass_is_exact()) { + ptr = Constant; + } + return make(ptr, elem(), nullptr, _offset, true, true, false, false, true, false); } // Get the (non-)refined array klass ptr @@ -6642,12 +6665,13 @@ const TypeAryKlassPtr* TypeAryKlassPtr::cast_to_refined_array_klass_ptr(bool ref if ((refined == is_refined_type()) || !klass_is_exact() || (!exact_klass()->is_obj_array_klass() && !exact_klass()->is_flat_array_klass())) { return this; } - ciKlass* eklass = elem()->is_klassptr()->exact_klass_helper(); - if (elem()->isa_aryklassptr()) { - eklass = exact_klass()->as_obj_array_klass()->element_klass(); + ciArrayKlass* k = exact_klass()->as_array_klass(); + if (refined) { + k = ciObjArrayKlass::make(k->element_klass(), true); + } else { + k = ciObjArrayKlass::make(k->element_klass(), false); } - ciKlass* array_klass = ciArrayKlass::make(eklass, eklass->is_inlinetype() ? is_null_free() : false, eklass->is_inlinetype() ? is_atomic() : true, refined); - return make(_ptr, array_klass, Offset(0), trust_interfaces, refined); + return make(k, trust_interfaces); } //------------------------------eq--------------------------------------------- @@ -6743,8 +6767,16 @@ ciKlass* TypeAryPtr::exact_klass_helper() const { if (k == nullptr) { return nullptr; } - k = ciArrayKlass::make(k, is_null_free(), is_atomic(), is_flat() || is_null_free()); - return k; + if (k->is_array_klass() && k->as_array_klass()->is_refined()) { + // We have no mechanism to create an array of refined arrays + k = ciObjArrayKlass::make(k->as_array_klass()->element_klass(), false); + } + if (klass_is_exact()) { + return ciObjArrayKlass::make(k, true, is_null_free(), is_atomic()); + } else { + // We may reach here if called recursively, must be an unrefined type then + return ciObjArrayKlass::make(k, false); + } } return klass(); @@ -6778,51 +6810,47 @@ const TypeAryKlassPtr* TypeAryKlassPtr::cast_to_ptr_type(PTR ptr) const { } bool TypeAryKlassPtr::must_be_exact() const { - if (_elem == Type::BOTTOM) return false; - if (_elem == Type::TOP ) return false; - const TypeKlassPtr* tk = _elem->isa_klassptr(); - if (!tk) return true; // a primitive type, like int - // Even though MyValue is final, [LMyValue is only exact if the array - // is (not) null-free due to null-free [LMyValue <: null-able [LMyValue. - // TODO 8350865 If we know that the array can't be null-free, it's allowed to be exact, right? - // If so, we should add '&& !is_not_null_free()' - if (tk->isa_instklassptr() && tk->klass()->is_inlinetype() && !is_null_free()) { + assert(klass_is_exact(), "precondition"); + if (_elem == Type::BOTTOM || _elem == Type::TOP) { return false; } - return tk->must_be_exact(); -} + const TypeKlassPtr* elem = _elem->isa_klassptr(); + if (elem == nullptr) { + // primitive arrays + return true; + } + // refined types are final + return _refined_type; +} //-----------------------------cast_to_exactness------------------------------- const TypeKlassPtr *TypeAryKlassPtr::cast_to_exactness(bool klass_is_exact) const { - if (must_be_exact() && !klass_is_exact) return this; // cannot clear xk if (klass_is_exact == this->klass_is_exact()) { return this; } - ciKlass* k = _klass; + if (!klass_is_exact && must_be_exact()) { + return this; + } const Type* elem = this->elem(); if (elem->isa_klassptr() && !klass_is_exact) { elem = elem->is_klassptr()->cast_to_exactness(klass_is_exact); } - bool not_flat = is_not_flat(); - bool not_null_free = is_not_null_free(); - if (_elem->isa_klassptr()) { - if (klass_is_exact || _elem->isa_aryklassptr()) { - assert((!is_null_free() && !is_flat()) || - _elem->is_klassptr()->klass()->is_abstract() || _elem->is_klassptr()->klass()->is_java_lang_Object(), - "null-free (or flat) concrete inline type arrays should always be exact"); - // An array can't be null-free (or flat) if the klass is exact - not_null_free = true; - not_flat = true; - } else { - // Klass is not exact (anymore), re-compute null-free/flat properties - const TypeOopPtr* exact_etype = TypeOopPtr::make_from_klass_unique(_elem->is_instklassptr()->instance_klass()); - bool not_inline = !exact_etype->can_be_inline_type(); - not_null_free = not_inline; - not_flat = !UseArrayFlattening || not_inline || (exact_etype->is_inlinetypeptr() && !exact_etype->inline_klass()->maybe_flat_in_array()); - } + + if (klass_is_exact) { + // cast_to_exactness(true) really means get the LCA of all values represented by this + // TypeAryKlassPtr. As a result, it must be an unrefined klass pointer. + return make(Constant, elem, nullptr, _offset, true, true, false, false, true, false); + } else { + // cast_to_exactness(false) means get the TypeAryKlassPtr representing all values that subtype + // this value + bool not_inline = !_elem->isa_instklassptr() || !_elem->is_instklassptr()->instance_klass()->can_be_inline_klass(); + bool not_flat = !UseArrayFlattening || not_inline || + (_elem->isa_instklassptr() && _elem->is_instklassptr()->instance_klass()->is_inlinetype() && !_elem->is_instklassptr()->instance_klass()->maybe_flat_in_array()); + bool not_null_free = not_inline; + bool atomic = not_flat; + return make(NotNull, elem, nullptr, _offset, not_flat, not_null_free, false, false, atomic, false); } - return make(klass_is_exact ? Constant : NotNull, elem, k, _offset, not_flat, not_null_free, _flat, _null_free, _atomic, _refined_type); } //-----------------------------as_instance_type-------------------------------- @@ -7153,7 +7181,8 @@ ciKlass* TypeAryKlassPtr::exact_klass_helper() const { if (k == nullptr) { return nullptr; } - k = ciArrayKlass::make(k, k->is_inlinetype() ? is_null_free() : false, k->is_inlinetype() ? is_atomic() : true, _refined_type); + assert(!k->is_array_klass() || !k->as_array_klass()->is_refined(), "no mechanism to create an array of refined arrays %s", k->name()->as_utf8()); + k = ciArrayKlass::make(k, is_null_free(), is_atomic(), _refined_type); return k; } diff --git a/src/hotspot/share/opto/type.hpp b/src/hotspot/share/opto/type.hpp index 52b73b8f67c..5dc8b745793 100644 --- a/src/hotspot/share/opto/type.hpp +++ b/src/hotspot/share/opto/type.hpp @@ -1031,8 +1031,8 @@ class TypeAry : public Type { friend class TypeAryPtr; public: - static const TypeAry* make(const Type* elem, const TypeInt* size, bool stable = false, - bool flat = false, bool not_flat = false, bool not_null_free = false, bool atomic = false); + static const TypeAry* make(const Type* elem, const TypeInt* size, bool stable, + bool flat, bool not_flat, bool not_null_free, bool atomic); virtual const Type *xmeet( const Type *t ) const; virtual const Type *xdual() const; // Compute dual right now. @@ -1773,7 +1773,9 @@ class TypeAryPtr : public TypeOopPtr { virtual const Type *xdual() const; // Compute dual right now. // Inline type array properties + const TypeAryPtr* cast_to_flat(bool flat) const; const TypeAryPtr* cast_to_not_flat(bool not_flat = true) const; + const TypeAryPtr* cast_to_null_free(bool null_free) const; const TypeAryPtr* cast_to_not_null_free(bool not_null_free = true) const; const TypeAryPtr* update_properties(const TypeAryPtr* new_type) const; jint flat_layout_helper() const; @@ -1910,7 +1912,6 @@ class TypeKlassPtr : public TypePtr { virtual bool klass_is_exact() const { return _ptr == Constant; } static const TypeKlassPtr* make(ciKlass* klass, InterfaceHandling interface_handling = ignore_interfaces); - static const TypeKlassPtr *make(PTR ptr, ciKlass* klass, Offset offset, InterfaceHandling interface_handling = ignore_interfaces); virtual bool is_loaded() const { return _klass->is_loaded(); } @@ -2103,7 +2104,7 @@ class TypeAryKlassPtr : public TypeKlassPtr { // returns base element type, an instance klass (and not interface) for object arrays const Type* base_element_type(int& dims) const; - static const TypeAryKlassPtr* make(PTR ptr, ciKlass* k, Offset offset, InterfaceHandling interface_handling, bool not_flat, bool not_null_free, bool flat, bool null_free, bool atomic, bool refined_type = false); + static const TypeAryKlassPtr* make(PTR ptr, ciKlass* k, Offset offset, InterfaceHandling interface_handling, bool not_flat, bool not_null_free, bool flat, bool null_free, bool atomic, bool refined_type); bool is_same_java_type_as_helper(const TypeKlassPtr* other) const; bool is_java_subtype_of_helper(const TypeKlassPtr* other, bool this_exact, bool other_exact) const; @@ -2111,10 +2112,10 @@ class TypeAryKlassPtr : public TypeKlassPtr { bool is_loaded() const { return (_elem->isa_klassptr() ? _elem->is_klassptr()->is_loaded() : true); } - static const TypeAryKlassPtr* make(PTR ptr, const Type* elem, ciKlass* k, Offset offset, bool not_flat, bool not_null_free, bool flat, bool null_free, bool atomic, bool refined_type = false); - static const TypeAryKlassPtr* make(PTR ptr, ciKlass* k, Offset offset, InterfaceHandling interface_handling, bool refined_type = false); - static const TypeAryKlassPtr* make(ciKlass* klass, InterfaceHandling interface_handling, bool refined_type = false); + static const TypeAryKlassPtr* make(PTR ptr, const Type* elem, ciKlass* k, Offset offset, bool not_flat, bool not_null_free, bool flat, bool null_free, bool atomic, bool refined_type); + static const TypeAryKlassPtr* make(ciKlass* klass, InterfaceHandling interface_handling); + const TypeAryKlassPtr* cast_to_non_refined() const; const TypeAryKlassPtr* cast_to_refined_array_klass_ptr(bool refined = true) const; const Type *elem() const { return _elem; } diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestBasicFunctionality.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestBasicFunctionality.java index 9c6ea0119e9..2719336a6c2 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestBasicFunctionality.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestBasicFunctionality.java @@ -378,7 +378,7 @@ public void test11_verifier() { // Test loop with uncommon trap referencing a value object @Test @IR(applyIf = {"UseArrayFlattening", "true"}, - counts = {SCOPE_OBJECT, ">= 1", LOAD_OF_ANY_KLASS, "<= 12"}) // TODO 8227588 (loads should be removed) + counts = {SCOPE_OBJECT, ">= 1"}) // LOAD_OF_ANY_KLASS, "<= 12"}) // TODO 8372332, 8227588 (loads should be removed) public long test12(boolean b) { MyValue1 v = MyValue1.createWithFieldsInline(rI, rL); MyValue1[] va = (MyValue1[])ValueClass.newNullRestrictedNonAtomicArray(MyValue1.class, Math.abs(rI) % 10, MyValue1.DEFAULT);