8372113: [lworld] Fix various issues with TypeAryPtr and TypeAryKlassPtr

src/hotspot/share/ci/ciArray.cpp

@@ -27,6 +27,8 @@
 #include "ci/ciConstant.hpp"
 #include "ci/ciKlass.hpp"
 #include "ci/ciUtilities.inline.hpp"
+#include "oops/flatArrayKlass.hpp"
+#include "oops/layoutKind.hpp"
 #include "oops/objArrayOop.inline.hpp"
 #include "oops/oop.inline.hpp"
 #include "oops/typeArrayOop.inline.hpp"
@@ -128,16 +130,7 @@ bool ciArray::is_null_free() {
 bool ciArray::is_atomic() {
   VM_ENTRY_MARK;
   arrayOop oop = get_arrayOop();
-  if (oop->is_refArray()) {
-    return oop->klass()->is_inline_klass();
-  }
-  if (oop->is_flatArray()) {
-    FlatArrayKlass* fak = FlatArrayKlass::cast(oop->klass());
-    if (fak->element_klass()->is_naturally_atomic() || fak->layout_kind() == LayoutKind::ATOMIC_FLAT || fak->layout_kind() == LayoutKind::NULLABLE_ATOMIC_FLAT) {
-      return true;
-    }
-  }
-  return false;
+  return !oop->is_flatArray() || FlatArrayKlass::cast(oop->klass())->layout_kind() != LayoutKind::NON_ATOMIC_FLAT;
 }
 
 // ------------------------------------------------------------------

src/hotspot/share/ci/ciArrayKlass.cpp

@@ -29,6 +29,7 @@
 #include "ci/ciTypeArrayKlass.hpp"
 #include "ci/ciUtilities.inline.hpp"
 #include "memory/universe.hpp"
+#include "oops/arrayKlass.hpp"
 #include "oops/inlineKlass.inline.hpp"
 
 // ciArrayKlass
@@ -121,14 +122,15 @@ ciInstance* ciArrayKlass::component_mirror_instance() const {
 }
 
 bool ciArrayKlass::is_elem_null_free() const {
-  GUARDED_VM_ENTRY(return is_loaded() && get_Klass()->is_null_free_array_klass();)
+  ArrayKlass::ArrayProperties props = properties();
+  assert(props != ArrayKlass::INVALID, "meaningless");
+  return ArrayKlass::is_null_restricted(props);
 }
 
-bool ciArrayKlass::is_elem_atomic() {
-  ciKlass* elem = element_klass();
-  GUARDED_VM_ENTRY(return elem != nullptr && elem->is_inlinetype() &&
-                          (ArrayKlass::cast(get_Klass())->properties() & ArrayKlass::ArrayProperties::INVALID) == 0 &&
-                          (ArrayKlass::cast(get_Klass())->properties() & ArrayKlass::ArrayProperties::NON_ATOMIC) == 0;)
+bool ciArrayKlass::is_elem_atomic() const {
+  ArrayKlass::ArrayProperties props = properties();
+  assert(props != ArrayKlass::INVALID, "meaningless");
+  return !ArrayKlass::is_non_atomic(props);
 }
 
 ArrayKlass::ArrayProperties ciArrayKlass::properties() const {

src/hotspot/share/ci/ciArrayKlass.hpp

@@ -53,6 +53,8 @@ class ciArrayKlass : public ciKlass {
   ciType* base_element_type();  // JLS calls this the "element type"
   bool is_leaf_type();          // No subtypes of this array type.
 
+  bool is_refined() const { return !is_type_array_klass() && properties() != ArrayKlass::INVALID; }
+
   // What kind of vmObject is this?
   bool is_array_klass() const { return true; }
 
@@ -65,7 +67,7 @@ class ciArrayKlass : public ciKlass {
   ciInstance* component_mirror_instance() const;
 
   bool is_elem_null_free() const;
-  bool is_elem_atomic();
+  bool is_elem_atomic() const;
 
   ArrayKlass::ArrayProperties properties() const;
 };

src/hotspot/share/ci/ciObjArrayKlass.cpp

@@ -191,22 +191,5 @@ ciArrayKlass* ciObjArrayKlass::make(ciKlass* element_klass, int dims) {
 }
 
 ciKlass* ciObjArrayKlass::exact_klass() {
-  if (!is_loaded()) {
-    return nullptr;
-  }
-  ciType* base = base_element_type();
-  if (base->is_instance_klass()) {
-    ciInstanceKlass* ik = base->as_instance_klass();
-    // Even though MyValue is final, [LMyValue is only exact if the array
-    // is null-free due to null-free [LMyValue <: null-able [LMyValue.
-    if (ik->is_inlinetype() && !is_elem_null_free()) {
-      return nullptr;
-    }
-    if (ik->exact_klass() != nullptr) {
-      return this;
-    }
-  } else if (base->is_primitive_type()) {
-    return this;
-  }
-  return nullptr;
+  return (is_loaded() && is_refined()) ? this : nullptr;
 }

src/hotspot/share/oops/objArrayKlass.cpp

@@ -137,6 +137,7 @@ ArrayKlass(name, kind, props, mk) {
 
   Klass* bk;
   if (element_klass->is_objArray_klass()) {
+    assert(!element_klass->is_refined_objArray_klass(), "no such mechanism yet");
     bk = ObjArrayKlass::cast(element_klass)->bottom_klass();
   } else {
     assert(!element_klass->is_refArray_klass(), "Sanity");

src/hotspot/share/opto/compile.cpp

@@ -1388,19 +1388,6 @@ const TypePtr *Compile::flatten_alias_type( const TypePtr *tj ) const {
 
   // Array pointers need some flattening
   const TypeAryPtr* ta = tj->isa_aryptr();
-  if (ta && ta->is_stable()) {
-    // Erase stability property for alias analysis.
-    tj = ta = ta->cast_to_stable(false);
-  }
-  if (ta && ta->is_not_flat()) {
-    // Erase not flat property for alias analysis.
-    tj = ta = ta->cast_to_not_flat(false);
-  }
-  if (ta && ta->is_not_null_free()) {
-    // Erase not null free property for alias analysis.
-    tj = ta = ta->cast_to_not_null_free(false);
-  }
-
   if( ta && is_known_inst ) {
     if ( offset != Type::OffsetBot &&
          offset > arrayOopDesc::length_offset_in_bytes() ) {
@@ -1411,78 +1398,46 @@ const TypePtr *Compile::flatten_alias_type( const TypePtr *tj ) const {
               with_offset(offset);
     }
   } else if (ta) {
-    // For arrays indexed by constant indices, we flatten the alias
-    // space to include all of the array body.  Only the header, klass
-    // and array length can be accessed un-aliased.
-    // For flat inline type array, each field has its own slice so
-    // we must include the field offset.
-    if( offset != Type::OffsetBot ) {
-      if( ta->const_oop() ) { // MethodData* or Method*
-        offset = Type::OffsetBot;   // Flatten constant access into array body
-        tj = ta = ta->
-                remove_speculative()->
-                cast_to_ptr_type(ptr)->
-                cast_to_exactness(false)->
-                with_offset(offset);
-      } else if( offset == arrayOopDesc::length_offset_in_bytes() ) {
-        // range is OK as-is.
-        tj = ta = TypeAryPtr::RANGE;
-      } else if( offset == oopDesc::klass_offset_in_bytes() ) {
-        tj = TypeInstPtr::KLASS; // all klass loads look alike
-        ta = TypeAryPtr::RANGE; // generic ignored junk
-        ptr = TypePtr::BotPTR;
-      } else if( offset == oopDesc::mark_offset_in_bytes() ) {
-        tj = TypeInstPtr::MARK;
-        ta = TypeAryPtr::RANGE; // generic ignored junk
-        ptr = TypePtr::BotPTR;
-      } else {                  // Random constant offset into array body
-        offset = Type::OffsetBot;   // Flatten constant access into array body
-        tj = ta = ta->
-                remove_speculative()->
-                cast_to_ptr_type(ptr)->
-                cast_to_exactness(false)->
-                with_offset(offset);
-      }
-    }
-    // Arrays of fixed size alias with arrays of unknown size.
-    if (ta->size() != TypeInt::POS) {
-      const TypeAry *tary = TypeAry::make(ta->elem(), TypeInt::POS);
-      tj = ta = ta->
-              remove_speculative()->
-              cast_to_ptr_type(ptr)->
-              with_ary(tary)->
-              cast_to_exactness(false);
-    }
-    // Arrays of known objects become arrays of unknown objects.
-    if (ta->elem()->isa_narrowoop() && ta->elem() != TypeNarrowOop::BOTTOM) {
-      const TypeAry *tary = TypeAry::make(TypeNarrowOop::BOTTOM, ta->size());
-      tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,nullptr,false,Type::Offset(offset), ta->field_offset());
-    }
-    if (ta->elem()->isa_oopptr() && ta->elem() != TypeInstPtr::BOTTOM) {
-      const TypeAry *tary = TypeAry::make(TypeInstPtr::BOTTOM, ta->size());
-      tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,nullptr,false,Type::Offset(offset), ta->field_offset());
-    }
-    // Initially all flattened array accesses share a single slice
-    if (ta->is_flat() && ta->elem() != TypeInstPtr::BOTTOM && _flat_accesses_share_alias) {
-      const TypeAry* tary = TypeAry::make(TypeInstPtr::BOTTOM, ta->size(), /* stable= */ false, /* flat= */ true);
-      tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,nullptr,false,Type::Offset(offset), Type::Offset(Type::OffsetBot));
+    // Common slices
+    if (offset == arrayOopDesc::length_offset_in_bytes()) {
+      return TypeAryPtr::RANGE;
+    } else if (offset == oopDesc::klass_offset_in_bytes()) {
+      return TypeInstPtr::KLASS;
+    } else if (offset == oopDesc::mark_offset_in_bytes()) {
+      return TypeInstPtr::MARK;
+    }
+
+    // Remove size and stability
+    const TypeAry* normalized_ary = TypeAry::make(ta->elem(), TypeInt::POS, false, ta->is_flat(), ta->is_not_flat(), ta->is_not_null_free(), ta->is_atomic());
+    // Remove ptr, const_oop, and offset
+    if (ta->elem()->make_oopptr() != nullptr) {
+      // Object arrays, keep field_offset
+      tj = ta = TypeAryPtr::make(TypePtr::BotPTR, nullptr, normalized_ary, nullptr, ta->klass_is_exact(), Type::Offset::bottom, Type::Offset(ta->field_offset()));
+    } else {
+      // Primitive arrays
+      tj = ta = TypeAryPtr::make(TypePtr::BotPTR, nullptr, normalized_ary, ta->exact_klass(), true, Type::Offset::bottom);
     }
+
     // Arrays of bytes and of booleans both use 'bastore' and 'baload' so
     // cannot be distinguished by bytecode alone.
     if (ta->elem() == TypeInt::BOOL) {
-      const TypeAry *tary = TypeAry::make(TypeInt::BYTE, ta->size());
-      ciKlass* aklass = ciTypeArrayKlass::make(T_BYTE);
-      tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,aklass,false,Type::Offset(offset), ta->field_offset());
-    }
-    // During the 2nd round of IterGVN, NotNull castings are removed.
-    // Make sure the Bottom and NotNull variants alias the same.
-    // Also, make sure exact and non-exact variants alias the same.
-    if (ptr == TypePtr::NotNull || ta->klass_is_exact() || ta->speculative() != nullptr) {
-      tj = ta = ta->
-              remove_speculative()->
-              cast_to_ptr_type(TypePtr::BotPTR)->
-              cast_to_exactness(false)->
-              with_offset(offset);
+      tj = ta = TypeAryPtr::BYTES;
+    }
+
+    // All arrays of references share the same slice
+    if (!ta->is_flat() && ta->elem()->make_oopptr() != nullptr) {
+      const TypeAry* tary = TypeAry::make(TypeInstPtr::BOTTOM, TypeInt::POS, false, false, true, true, true);
+      tj = ta = TypeAryPtr::make(TypePtr::BotPTR, nullptr, tary, nullptr, false, Type::Offset::bottom);
+    }
+
+    if (ta->is_flat()) {
+      if (_flat_accesses_share_alias) {
+        // Initially all flattened array accesses share a single slice
+        tj = ta = TypeAryPtr::INLINES;
+      } else {
+        // Flat accesses are always exact
+        tj = ta = ta->cast_to_exactness(true);
+      }
     }
   }
 

src/hotspot/share/opto/graphKit.cpp

@@ -1876,30 +1876,36 @@ Node* GraphKit::array_element_address(Node* ary, Node* idx, BasicType elembt,
   return basic_plus_adr(ary, base, scale);
 }
 
-Node* GraphKit::cast_to_flat_array(Node* array, ciInlineKlass* vk, bool is_null_free, bool is_not_null_free, bool is_atomic) {
-  assert(vk->maybe_flat_in_array(), "element of type %s cannot be flat in array", vk->name()->as_utf8());
-  if (!vk->has_nullable_atomic_layout()) {
+Node* GraphKit::cast_to_flat_array(Node* array, ciInlineKlass* elem_vk) {
+  assert(elem_vk->maybe_flat_in_array(), "no flat array for %s", elem_vk->name()->as_utf8());
+  if (!elem_vk->has_atomic_layout() && !elem_vk->has_nullable_atomic_layout()) {
+    return cast_to_flat_array_exact(array, elem_vk, true, false);
+  } else if (!elem_vk->has_nullable_atomic_layout() && !elem_vk->has_non_atomic_layout()) {
+    return cast_to_flat_array_exact(array, elem_vk, true, true);
+  } else if (!elem_vk->has_atomic_layout() && !elem_vk->has_non_atomic_layout()) {
+    return cast_to_flat_array_exact(array, elem_vk, false, true);
+  }
+
+  bool is_null_free = false;
+  if (!elem_vk->has_nullable_atomic_layout()) {
     // Element does not have a nullable flat layout, cannot be nullable
     is_null_free = true;
   }
-  if (!vk->has_atomic_layout() && !vk->has_non_atomic_layout()) {
-    // Element does not have a null-free flat layout, cannot be null-free
-    is_not_null_free = true;
-  }
-  if (is_null_free) {
-    // TODO 8350865 Impossible type
-    is_not_null_free = false;
-  }
 
-  bool is_exact = is_null_free || is_not_null_free;
-  ciArrayKlass* array_klass = ciArrayKlass::make(vk, is_null_free, is_atomic, true);
-  assert(array_klass->is_elem_null_free() == is_null_free, "inconsistency");
-  assert(array_klass->is_elem_atomic() == is_atomic, "inconsistency");
+  ciArrayKlass* array_klass = ciObjArrayKlass::make(elem_vk, false);
+  const TypeAryPtr* arytype = TypeOopPtr::make_from_klass(array_klass)->isa_aryptr();
+  arytype = arytype->cast_to_flat(true)->cast_to_null_free(is_null_free);
+  return _gvn.transform(new CastPPNode(control(), array, arytype, ConstraintCastNode::StrongDependency));
+}
+
+Node* GraphKit::cast_to_flat_array_exact(Node* array, ciInlineKlass* elem_vk, bool is_null_free, bool is_atomic) {
+  assert(is_null_free || is_atomic, "nullable arrays must be atomic");
+  ciArrayKlass* array_klass = ciObjArrayKlass::make(elem_vk, true, is_null_free, is_atomic);
   const TypeAryPtr* arytype = TypeOopPtr::make_from_klass(array_klass)->isa_aryptr();
-  arytype = arytype->cast_to_exactness(is_exact);
-  arytype = arytype->cast_to_not_null_free(is_not_null_free);
+  assert(arytype->klass_is_exact(), "inconsistency");
+  assert(arytype->is_flat(), "inconsistency");
   assert(arytype->is_null_free() == is_null_free, "inconsistency");
-  assert(arytype->is_not_null_free() == is_not_null_free, "inconsistency");
+  assert(arytype->is_not_null_free() == !is_null_free, "inconsistency");
   assert(arytype->is_atomic() == is_atomic, "inconsistency");
   return _gvn.transform(new CastPPNode(control(), array, arytype, ConstraintCastNode::StrongDependency));
 }
@@ -4676,7 +4682,7 @@ Node* GraphKit::load_String_value(Node* str, bool set_ctrl) {
                                                      false, nullptr, Type::Offset(0));
   const TypePtr* value_field_type = string_type->add_offset(value_offset);
   const TypeAryPtr* value_type = TypeAryPtr::make(TypePtr::NotNull,
-                                                  TypeAry::make(TypeInt::BYTE, TypeInt::POS, false, false, true, true),
+                                                  TypeAry::make(TypeInt::BYTE, TypeInt::POS, false, false, true, true, true),
                                                   ciTypeArrayKlass::make(T_BYTE), true, Type::Offset(0));
   Node* p = basic_plus_adr(str, str, value_offset);
   Node* load = access_load_at(str, p, value_field_type, value_type, T_OBJECT,

src/hotspot/share/opto/graphKit.hpp

@@ -660,7 +660,8 @@ class GraphKit : public Phase {
                               const TypeInt* sizetype = nullptr,
                               // Optional control dependency (for example, on range check)
                               Node* ctrl = nullptr);
-  Node* cast_to_flat_array(Node* array, ciInlineKlass* elem_vk, bool is_null_free, bool is_not_null_free, bool is_atomic);
+  Node* cast_to_flat_array(Node* array, ciInlineKlass* elem_vk);
+  Node* cast_to_flat_array_exact(Node* array, ciInlineKlass* elem_vk, bool is_null_free, bool is_atomic);
 
   // Return a load of array element at idx.
   Node* load_array_element(Node* ary, Node* idx, const TypeAryPtr* arytype, bool set_ctrl);

src/hotspot/share/opto/inlinetypenode.cpp

@@ -554,7 +554,7 @@ void InlineTypeNode::store_flat_array(GraphKit* kit, Node* base, Node* idx) {
   if (!kit->stopped()) {
     assert(vk->has_nullable_atomic_layout(), "element type %s does not have a nullable flat layout", vk->name()->as_utf8());
     kit->set_all_memory(input_memory_state);
-    Node* cast = kit->cast_to_flat_array(base, vk, false, true, true);
+    Node* cast = kit->cast_to_flat_array_exact(base, vk, false, true);
     Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
     store_flat(kit, cast, ptr, true, false, false, decorators);
 
@@ -576,7 +576,7 @@ void InlineTypeNode::store_flat_array(GraphKit* kit, Node* base, Node* idx) {
     if (!kit->stopped()) {
       assert(vk->has_atomic_layout(), "element type %s does not have a null-free atomic flat layout", vk->name()->as_utf8());
       kit->set_all_memory(input_memory_state);
-      Node* cast = kit->cast_to_flat_array(base, vk, true, false, true);
+      Node* cast = kit->cast_to_flat_array_exact(base, vk, true, true);
       Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
       store_flat(kit, cast, ptr, true, false, true, decorators);
 
@@ -590,7 +590,7 @@ void InlineTypeNode::store_flat_array(GraphKit* kit, Node* base, Node* idx) {
     if (!kit->stopped()) {
       assert(vk->has_non_atomic_layout(), "element type %s does not have a null-free non-atomic flat layout", vk->name()->as_utf8());
       kit->set_all_memory(input_memory_state);
-      Node* cast = kit->cast_to_flat_array(base, vk, true, false, false);
+      Node* cast = kit->cast_to_flat_array_exact(base, vk, true, false);
       Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
       store_flat(kit, cast, ptr, false, false, true, decorators);
 
@@ -1081,7 +1081,7 @@ InlineTypeNode* InlineTypeNode::make_from_flat_array(GraphKit* kit, ciInlineKlas
   if (!kit->stopped()) {
     assert(vk->has_nullable_atomic_layout(), "element type %s does not have a nullable flat layout", vk->name()->as_utf8());
     kit->set_all_memory(input_memory_state);
-    Node* cast = kit->cast_to_flat_array(base, vk, false, true, true);
+    Node* cast = kit->cast_to_flat_array_exact(base, vk, false, true);
     Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
     vt_nullable = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, true, false, false, decorators);
 
@@ -1103,7 +1103,7 @@ InlineTypeNode* InlineTypeNode::make_from_flat_array(GraphKit* kit, ciInlineKlas
     if (!kit->stopped()) {
       assert(vk->has_atomic_layout(), "element type %s does not have a null-free atomic flat layout", vk->name()->as_utf8());
       kit->set_all_memory(input_memory_state);
-      Node* cast = kit->cast_to_flat_array(base, vk, true, false, true);
+      Node* cast = kit->cast_to_flat_array_exact(base, vk, true, true);
       Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
       vt_null_free = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, true, false, true, decorators);
 
@@ -1117,7 +1117,7 @@ InlineTypeNode* InlineTypeNode::make_from_flat_array(GraphKit* kit, ciInlineKlas
     if (!kit->stopped()) {
       assert(vk->has_non_atomic_layout(), "element type %s does not have a null-free non-atomic flat layout", vk->name()->as_utf8());
       kit->set_all_memory(input_memory_state);
-      Node* cast = kit->cast_to_flat_array(base, vk, true, false, false);
+      Node* cast = kit->cast_to_flat_array_exact(base, vk, true, false);
       Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
       vt_non_atomic = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, false, false, true, decorators);