From 23828cc7b14d5a3fde4b24d8972cef91a4a990b9 Mon Sep 17 00:00:00 2001 From: Quan Anh Mai Date: Mon, 28 Jul 2025 21:10:31 +0700 Subject: [PATCH 1/3] add LoadFlatNode and StoreFlatNode --- src/hotspot/share/opto/classes.hpp | 2 + src/hotspot/share/opto/compile.cpp | 39 ++ src/hotspot/share/opto/compile.hpp | 13 + src/hotspot/share/opto/escape.cpp | 84 ++- src/hotspot/share/opto/escape.hpp | 2 + src/hotspot/share/opto/inlinetypenode.cpp | 607 ++++++++++++------ src/hotspot/share/opto/inlinetypenode.hpp | 89 ++- src/hotspot/share/opto/loopopts.cpp | 3 +- src/hotspot/share/opto/macro.cpp | 3 + src/hotspot/share/opto/memnode.cpp | 15 +- src/hotspot/share/opto/node.cpp | 3 + src/hotspot/share/runtime/deoptimization.cpp | 15 +- .../TestScalarReplaceFlatFields.java | 102 +++ 13 files changed, 768 insertions(+), 209 deletions(-) create mode 100644 test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestScalarReplaceFlatFields.java diff --git a/src/hotspot/share/opto/classes.hpp b/src/hotspot/share/opto/classes.hpp index 76162cdc781..a30332fd3d4 100644 --- a/src/hotspot/share/opto/classes.hpp +++ b/src/hotspot/share/opto/classes.hpp @@ -385,6 +385,8 @@ macro(URShiftL) macro(XorI) macro(XorL) macro(InlineType) +macro(LoadFlat) +macro(StoreFlat) macro(Vector) macro(AddVB) macro(AddVS) diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index b9bc8c4120c..ced0ca58e38 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -408,6 +408,9 @@ void Compile::remove_useless_node(Node* dead) { if (dead->is_InlineType()) { remove_inline_type(dead); } + if (dead->Opcode() == Op_LoadFlat || dead->Opcode() == Op_StoreFlat) { + remove_flat_access(dead); + } if (dead->for_merge_stores_igvn()) { remove_from_merge_stores_igvn(dead); } @@ -467,6 +470,7 @@ void Compile::disconnect_useless_nodes(Unique_Node_List& useful, Unique_Node_Lis remove_useless_nodes(_expensive_nodes, useful); // remove useless expensive nodes remove_useless_nodes(_for_post_loop_igvn, useful); // remove useless node recorded for post loop opts IGVN pass remove_useless_nodes(_inline_type_nodes, useful); // remove useless inline type nodes + remove_useless_nodes(_flat_access_nodes, useful); // remove useless flat access nodes #ifdef ASSERT if (_modified_nodes != nullptr) { _modified_nodes->remove_useless_nodes(useful.member_set()); @@ -674,6 +678,7 @@ Compile::Compile(ciEnv* ci_env, ciMethod* target, int osr_bci, _expensive_nodes(comp_arena(), 8, 0, nullptr), _for_post_loop_igvn(comp_arena(), 8, 0, nullptr), _inline_type_nodes (comp_arena(), 8, 0, nullptr), + _flat_access_nodes(comp_arena(), 8, 0, nullptr), _for_merge_stores_igvn(comp_arena(), 8, 0, nullptr), _unstable_if_traps(comp_arena(), 8, 0, nullptr), _coarsened_locks(comp_arena(), 8, 0, nullptr), @@ -2080,6 +2085,35 @@ void Compile::process_inline_types(PhaseIterGVN &igvn, bool remove) { igvn.optimize(); } +void Compile::add_flat_access(Node* n) { + assert(n != nullptr && (n->Opcode() == Op_LoadFlat || n->Opcode() == Op_StoreFlat), "unexpected node %s", n == nullptr ? "nullptr" : n->Name()); + assert(!_flat_access_nodes.contains(n), "duplicate insertion"); + _flat_access_nodes.push(n); +} + +void Compile::remove_flat_access(Node* n) { + assert(n != nullptr && (n->Opcode() == Op_LoadFlat || n->Opcode() == Op_StoreFlat), "unexpected node %s", n == nullptr ? "nullptr" : n->Name()); + _flat_access_nodes.remove_if_existing(n); +} + +void Compile::process_flat_accesses(PhaseIterGVN& igvn) { + assert(igvn._worklist.size() == 0, "should be empty"); + igvn.set_delay_transform(true); + for (int i = _flat_access_nodes.length() - 1; i >= 0; i--) { + Node* n = _flat_access_nodes.at(i); + assert(n != nullptr, "unexpected nullptr"); + if (n->Opcode() == Op_LoadFlat) { + static_cast(n)->expand_atomic(igvn); + } else { + assert(n->Opcode() == Op_StoreFlat, "unexpected node %s", n->Name()); + static_cast(n)->expand_atomic(igvn); + } + } + _flat_access_nodes.clear_and_deallocate(); + igvn.set_delay_transform(false); + igvn.optimize(); +} + void Compile::adjust_flat_array_access_aliases(PhaseIterGVN& igvn) { if (!_has_flat_accesses) { return; @@ -2928,6 +2962,11 @@ void Compile::Optimize() { } while (progress); } + process_flat_accesses(igvn); + if (failing()) { + return; + } + // Loop transforms on the ideal graph. Range Check Elimination, // peeling, unrolling, etc. diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index 8114504a072..26730dea30a 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -44,6 +44,7 @@ #include "runtime/sharedRuntime.hpp" #include "runtime/timerTrace.hpp" #include "runtime/vmThread.hpp" +#include "utilities/growableArray.hpp" #include "utilities/ticks.hpp" #include "utilities/vmEnums.hpp" #include "opto/printinlining.hpp" @@ -383,6 +384,7 @@ class Compile : public Phase { GrowableArray _expensive_nodes; // List of nodes that are expensive to compute and that we'd better not let the GVN freely common GrowableArray _for_post_loop_igvn; // List of nodes for IGVN after loop opts are over GrowableArray _inline_type_nodes; // List of InlineType nodes + GrowableArray _flat_access_nodes; // List of LoadFlat and StoreFlat nodes GrowableArray _for_merge_stores_igvn; // List of nodes for IGVN merge stores GrowableArray _unstable_if_traps; // List of ifnodes after IGVN GrowableArray _coarsened_locks; // List of coarsened Lock and Unlock nodes @@ -791,6 +793,17 @@ class Compile : public Phase { void remove_inline_type(Node* n); void process_inline_types(PhaseIterGVN &igvn, bool remove = false); + void add_flat_access(Node* n); + void remove_flat_access(Node* n); + void process_flat_accesses(PhaseIterGVN& igvn); + + template + void for_each_flat_access(F consumer) { + for (int i = _flat_access_nodes.length() - 1; i >= 0; i--) { + consumer(_flat_access_nodes.at(i)); + } + } + void adjust_flat_array_access_aliases(PhaseIterGVN& igvn); void record_unstable_if_trap(UnstableIfTrap* trap); diff --git a/src/hotspot/share/opto/escape.cpp b/src/hotspot/share/opto/escape.cpp index 0a035b7f499..01ecffee21d 100644 --- a/src/hotspot/share/opto/escape.cpp +++ b/src/hotspot/share/opto/escape.cpp @@ -39,6 +39,7 @@ #include "opto/inlinetypenode.hpp" #include "opto/macro.hpp" #include "opto/locknode.hpp" +#include "opto/opcodes.hpp" #include "opto/phaseX.hpp" #include "opto/movenode.hpp" #include "opto/narrowptrnode.hpp" @@ -425,7 +426,15 @@ bool ConnectionGraph::compute_escape() { #endif } - // 6. Reduce allocation merges used as debug information. This is done after + // 6. Expand flat accesses if the object does not escape. This adds nodes to + // the graph, so it has to be after split_unique_types. This expands atomic + // mismatched accesses (though encapsulated in LoadFlats and StoreFlats) into + // non-mismatched accesses, so it is better before reduce allocation merges. + if (has_non_escaping_obj) { + optimize_flat_accesses(); + } + + // 7. Reduce allocation merges used as debug information. This is done after // split_unique_types because the methods used to create SafePointScalarObject // need to traverse the memory graph to find values for object fields. We also // set to null the scalarized inputs of reducible Phis so that the Allocate @@ -1679,6 +1688,14 @@ void ConnectionGraph::add_node_to_connection_graph(Node *n, Unique_Node_List *de } break; } + case Op_LoadFlat: + // Treat LoadFlat similar to an unknown call that receives nothing and produces its results + map_ideal_node(n, phantom_obj); + break; + case Op_StoreFlat: + // Treat StoreFlat similar to a call that escapes the stored flattened fields + delayed_worklist->push(n); + break; case Op_Proj: { // we are only interested in the oop result projection from a call if (n->as_Proj()->_con >= TypeFunc::Parms && n->in(0)->is_Call() && @@ -1686,6 +1703,9 @@ void ConnectionGraph::add_node_to_connection_graph(Node *n, Unique_Node_List *de assert((n->as_Proj()->_con == TypeFunc::Parms && n->in(0)->as_Call()->returns_pointer()) || n->in(0)->as_Call()->tf()->returns_inline_type_as_fields(), "what kind of oop return is it?"); add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0), delayed_worklist); + } else if (n->as_Proj()->_con >= TypeFunc::Parms && n->in(0)->Opcode() == Op_LoadFlat && igvn->type(n)->isa_ptr()) { + // Treat LoadFlat outputs similar to a call return value + add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0), delayed_worklist); } break; } @@ -1770,7 +1790,7 @@ void ConnectionGraph::add_final_edges(Node *n) { process_call_arguments(n->as_Call()); return; } - assert(n->is_Store() || n->is_LoadStore() || + assert(n->is_Store() || n->is_LoadStore() || n->Opcode() == Op_StoreFlat || ((n_ptn != nullptr) && (n_ptn->ideal_node() != nullptr)), "node should be registered already"); int opcode = n->Opcode(); @@ -1839,11 +1859,32 @@ void ConnectionGraph::add_final_edges(Node *n) { } break; } + case Op_StoreFlat: { + // StoreFlat globally escapes its stored flattened fields + InlineTypeNode* value = static_cast(n)->value(); + ciInlineKlass* vk = _igvn->type(value)->inline_klass(); + for (int i = 0; i < vk->nof_nonstatic_fields(); i++) { + ciField* field = vk->nonstatic_field_at(i); + if (field->type()->is_primitive_type()) { + continue; + } + + Node* field_value = value->field_value_by_offset(field->offset_in_bytes(), true); + PointsToNode* field_value_ptn = ptnode_adr(field_value->_idx); + set_escape_state(field_value_ptn, PointsToNode::GlobalEscape, "store into a flat field"); + } + break; + } case Op_Proj: { - // we are only interested in the oop result projection from a call - assert((n->as_Proj()->_con == TypeFunc::Parms && n->in(0)->as_Call()->returns_pointer()) || - n->in(0)->as_Call()->tf()->returns_inline_type_as_fields(), "what kind of oop return is it?"); - add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0), nullptr); + if (n->in(0)->is_Call()) { + // we are only interested in the oop result projection from a call + assert((n->as_Proj()->_con == TypeFunc::Parms && n->in(0)->as_Call()->returns_pointer()) || + n->in(0)->as_Call()->tf()->returns_inline_type_as_fields(), "what kind of oop return is it?"); + add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0), nullptr); + } else if (n->in(0)->Opcode() == Op_LoadFlat) { + // Treat LoadFlat outputs similar to a call return value + add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0), nullptr); + } break; } case Op_Rethrow: // Exception object escapes @@ -3342,6 +3383,35 @@ void ConnectionGraph::optimize_ideal_graph(GrowableArray& ptr_cmp_worklis } } +// Atomic flat accesses on non-escape objects can be optimized to non-atomic accesses +void ConnectionGraph::optimize_flat_accesses() { + PhaseIterGVN& igvn = *_igvn; + bool delay = igvn.delay_transform(); + igvn.set_delay_transform(true); + igvn.C->for_each_flat_access([&](Node* n) { + Node* ptr = n->in(TypeFunc::Parms); + if (!ptr->is_AddP()) { + return; + } + + if (!not_global_escape(ptr->as_AddP()->base_node())) { + return; + } + + bool expanded; + if (n->Opcode() == Op_LoadFlat) { + expanded = static_cast(n)->expand_non_atomic(igvn); + } else { + assert(n->Opcode() == Op_StoreFlat, "unexpected node %s", n->Name()); + expanded = static_cast(n)->expand_non_atomic(igvn); + } + if (expanded) { + igvn.C->remove_flat_access(n); + } + }); + igvn.set_delay_transform(delay); +} + // Optimize objects compare. const TypeInt* ConnectionGraph::optimize_ptr_compare(Node* left, Node* right) { assert(OptimizePtrCompare, "sanity"); @@ -4709,6 +4779,8 @@ void ConnectionGraph::split_unique_types(GrowableArray &alloc_worklist, if (m->is_MergeMem()) { assert(mergemem_worklist.contains(m->as_MergeMem()), "EA: missing MergeMem node in the worklist"); } + } else if (use->Opcode() == Op_LoadFlat || use->Opcode() == Op_StoreFlat) { + // These nodes consume and output whole memory states so there is nothing to do here } else if (use->Opcode() == Op_EncodeISOArray) { if (use->in(MemNode::Memory) == n || use->in(3) == n) { // EncodeISOArray overwrites destination array diff --git a/src/hotspot/share/opto/escape.hpp b/src/hotspot/share/opto/escape.hpp index 0b8cd3aa138..33d5c3175df 100644 --- a/src/hotspot/share/opto/escape.hpp +++ b/src/hotspot/share/opto/escape.hpp @@ -481,6 +481,8 @@ class ConnectionGraph: public ArenaObj { // Optimize ideal graph. void optimize_ideal_graph(GrowableArray& ptr_cmp_worklist, GrowableArray& storestore_worklist); + // Expand flat accesses to accesses to each component if the object does not escape + void optimize_flat_accesses(); // Optimize objects compare. const TypeInt* optimize_ptr_compare(Node* left, Node* right); diff --git a/src/hotspot/share/opto/inlinetypenode.cpp b/src/hotspot/share/opto/inlinetypenode.cpp index 11d32de77ee..2b61c5679bf 100644 --- a/src/hotspot/share/opto/inlinetypenode.cpp +++ b/src/hotspot/share/opto/inlinetypenode.cpp @@ -29,10 +29,13 @@ #include "oops/accessDecorators.hpp" #include "opto/addnode.hpp" #include "opto/castnode.hpp" +#include "opto/compile.hpp" #include "opto/convertnode.hpp" #include "opto/graphKit.hpp" #include "opto/inlinetypenode.hpp" +#include "opto/memnode.hpp" #include "opto/movenode.hpp" +#include "opto/multnode.hpp" #include "opto/narrowptrnode.hpp" #include "opto/opcodes.hpp" #include "opto/rootnode.hpp" @@ -487,151 +490,7 @@ void InlineTypeNode::load(GraphKit* kit, Node* base, Node* ptr, bool immutable_m } } -// Get a field value from the payload by shifting it according to the offset -static Node* get_payload_value(PhaseGVN* gvn, Node* payload, BasicType bt, BasicType val_bt, int offset) { - // Shift to the right position in the long value - assert((offset + type2aelembytes(val_bt)) <= type2aelembytes(bt), "Value does not fit into payload"); - Node* value = nullptr; - Node* shift_val = gvn->intcon(offset << LogBitsPerByte); - if (bt == T_LONG) { - value = gvn->transform(new URShiftLNode(payload, shift_val)); - value = gvn->transform(new ConvL2INode(value)); - } else { - value = gvn->transform(new URShiftINode(payload, shift_val)); - } - - if (val_bt == T_INT || val_bt == T_OBJECT || val_bt == T_ARRAY) { - return value; - } else { - // Make sure to zero unused bits in the 32-bit value - return Compile::narrow_value(val_bt, value, nullptr, gvn, true); - } -} - -// Convert a payload value to field values -void InlineTypeNode::convert_from_payload(GraphKit* kit, BasicType bt, Node* payload, int holder_offset, bool null_free, bool trust_null_free_oop) { - PhaseGVN* gvn = &kit->gvn(); - ciInlineKlass* vk = inline_klass(); - Node* value = nullptr; - if (!null_free) { - // Get the null marker - value = get_payload_value(gvn, payload, bt, T_BOOLEAN, holder_offset + vk->null_marker_offset_in_payload()); - set_req(NullMarker, value); - } - // Iterate over the fields and get their values from the payload - for (uint i = 0; i < field_count(); ++i) { - ciType* ft = field_type(i); - bool field_null_free = field_is_null_free(i); - int offset = holder_offset + field_offset(i) - vk->payload_offset(); - if (field_is_flat(i)) { - InlineTypeNode* vt = make_uninitialized(*gvn, ft->as_inline_klass(), field_null_free); - vt->convert_from_payload(kit, bt, payload, offset, field_null_free, trust_null_free_oop && field_null_free); - value = gvn->transform(vt); - } else { - value = get_payload_value(gvn, payload, bt, ft->basic_type(), offset); - if (!ft->is_primitive_type()) { - // Narrow oop field - assert(UseCompressedOops && bt == T_LONG, "Naturally atomic"); - const Type* val_type = Type::get_const_type(ft); - if (trust_null_free_oop && field_null_free) { - val_type = val_type->join_speculative(TypePtr::NOTNULL); - } - value = gvn->transform(new CastI2NNode(kit->control(), value, val_type->make_narrowoop())); - value = gvn->transform(new DecodeNNode(value, val_type->make_narrowoop())); - - // Similar to CheckCastPP nodes with raw input, CastI2N nodes require special handling in 'PhaseCFG::schedule_late' to ensure the - // register allocator does not move the CastI2N below a safepoint. This is necessary to avoid having the raw pointer span a safepoint, - // making it opaque to the GC. Unlike CheckCastPPs, which need extra handling in 'Scheduling::ComputeRegisterAntidependencies' due to - // scalarization, CastI2N nodes are always used by a load if scalarization happens which inherently keeps them pinned above the safepoint. - - if (ft->is_inlinetype()) { - GrowableArray visited; - value = make_from_oop_impl(kit, value, ft->as_inline_klass(), visited); - } - } - } - set_field_value(i, value); - } -} - -// Set a field value in the payload by shifting it according to the offset -static Node* set_payload_value(PhaseGVN* gvn, Node* payload, BasicType bt, Node* value, BasicType val_bt, int offset) { - assert((offset + type2aelembytes(val_bt)) <= type2aelembytes(bt), "Value does not fit into payload"); - - // Make sure to zero unused bits in the 32-bit value - if (val_bt == T_BYTE || val_bt == T_BOOLEAN) { - value = gvn->transform(new AndINode(value, gvn->intcon(0xFF))); - } else if (val_bt == T_CHAR || val_bt == T_SHORT) { - value = gvn->transform(new AndINode(value, gvn->intcon(0xFFFF))); - } else if (val_bt == T_FLOAT) { - value = gvn->transform(new MoveF2INode(value)); - } else { - assert(val_bt == T_INT, "Unsupported type: %s", type2name(val_bt)); - } - - Node* shift_val = gvn->intcon(offset << LogBitsPerByte); - if (bt == T_LONG) { - // Convert to long and remove the sign bit (the backend will fold this and emit a zero extend i2l) - value = gvn->transform(new ConvI2LNode(value)); - value = gvn->transform(new AndLNode(value, gvn->longcon(0xFFFFFFFF))); - - Node* shift_value = gvn->transform(new LShiftLNode(value, shift_val)); - payload = new OrLNode(shift_value, payload); - } else { - Node* shift_value = gvn->transform(new LShiftINode(value, shift_val)); - payload = new OrINode(shift_value, payload); - } - return gvn->transform(payload); -} - -// Convert the field values to a payload value of type 'bt' -Node* InlineTypeNode::convert_to_payload(GraphKit* kit, BasicType bt, Node* payload, int holder_offset, bool null_free, int null_marker_offset, int& oop_off_1, int& oop_off_2) const { - PhaseGVN* gvn = &kit->gvn(); - Node* value = nullptr; - if (!null_free) { - // Set the null marker - value = get_null_marker(); - payload = set_payload_value(gvn, payload, bt, value, T_BOOLEAN, null_marker_offset); - } - // Iterate over the fields and add their values to the payload - for (uint i = 0; i < field_count(); ++i) { - value = field_value(i); - int inner_offset = field_offset(i) - inline_klass()->payload_offset(); - int offset = holder_offset + inner_offset; - if (field_is_flat(i)) { - null_marker_offset = holder_offset + field_null_marker_offset(i) - inline_klass()->payload_offset(); - payload = value->as_InlineType()->convert_to_payload(kit, bt, payload, offset, field_is_null_free(i), null_marker_offset, oop_off_1, oop_off_2); - } else { - ciType* ft = field_type(i); - BasicType field_bt = ft->basic_type(); - if (!ft->is_primitive_type()) { - // Narrow oop field - assert(UseCompressedOops && bt == T_LONG, "Naturally atomic"); - assert(inner_offset != -1, "sanity"); - if (oop_off_1 == -1) { - oop_off_1 = inner_offset; - } else { - assert(oop_off_2 == -1, "already set"); - oop_off_2 = inner_offset; - } - const Type* val_type = Type::get_const_type(ft)->make_narrowoop(); - if (value->is_InlineType()) { - PreserveReexecuteState preexecs(kit); - kit->jvms()->set_should_reexecute(true); - value = value->as_InlineType()->buffer(kit, false); - } - value = gvn->transform(new EncodePNode(value, val_type)); - value = gvn->transform(new CastP2XNode(kit->control(), value)); - value = gvn->transform(new ConvL2INode(value)); - field_bt = T_INT; - } - payload = set_payload_value(gvn, payload, bt, value, field_bt, offset); - } - } - return payload; -} - -void InlineTypeNode::store_flat(GraphKit* kit, Node* base, Node* ptr, bool atomic, bool immutable_memory, bool null_free, DecoratorSet decorators) const { +void InlineTypeNode::store_flat(GraphKit* kit, Node* base, Node* ptr, bool atomic, bool immutable_memory, bool null_free, DecoratorSet decorators) { ciInlineKlass* vk = inline_klass(); bool do_atomic = atomic; // With immutable memory, a non-atomic load and an atomic load are the same @@ -654,42 +513,10 @@ void InlineTypeNode::store_flat(GraphKit* kit, Node* base, Node* ptr, bool atomi return; } - // Convert to a payload value <= 64-bit and write atomically. - // The payload might contain at most two oop fields that must be narrow because otherwise they would be 64-bit - // in size and would then be written by a "normal" oop store. If the payload contains oops, its size is always - // 64-bit because the next smaller (power-of-two) size would be 32-bit which could only hold one narrow oop that - // would then be written by a normal narrow oop store. These properties are asserted in 'convert_to_payload'. - assert(!immutable_memory, "immutable memory does not need explicit atomic access"); - BasicType store_bt = vk->atomic_size_to_basic_type(null_free); - Node* payload = (store_bt == T_LONG) ? kit->longcon(0) : kit->intcon(0); - int oop_off_1 = -1; - int oop_off_2 = -1; - payload = convert_to_payload(kit, store_bt, payload, 0, null_free, vk->null_marker_offset_in_payload(), oop_off_1, oop_off_2); - if (!UseG1GC || oop_off_1 == -1) { - // No oop fields or no late barrier expansion. Emit an atomic store of the payload and add GC barriers if needed. - assert(oop_off_2 == -1 || !UseG1GC, "sanity"); - // ZGC does not support compressed oops, so only one oop can be in the payload which is written by a "normal" oop store. - assert((oop_off_1 == -1 && oop_off_2 == -1) || !UseZGC, "ZGC does not support embedded oops in flat fields"); - const Type* val_type = Type::get_const_basic_type(store_bt); - kit->insert_mem_bar(Op_MemBarCPUOrder); - kit->access_store_at(base, ptr, TypeRawPtr::BOTTOM, payload, val_type, store_bt, decorators | C2_MISMATCHED, true, this); - kit->insert_mem_bar(Op_MemBarCPUOrder); - } else { - // Contains oops and requires late barrier expansion. Emit a special store node that allows to emit GC barriers in the backend. - assert(UseG1GC, "Unexpected GC"); - assert(store_bt == T_LONG, "Unexpected payload type"); - // If one oop, set the offset (if no offset is set, two oops are assumed by the backend) - Node* oop_offset = (oop_off_2 == -1) ? kit->intcon(oop_off_1) : nullptr; - kit->insert_mem_bar(Op_MemBarCPUOrder); - Node* mem = kit->reset_memory(); - kit->set_all_memory(mem); - Node* st = kit->gvn().transform(new StoreLSpecialNode(kit->control(), mem, ptr, TypeRawPtr::BOTTOM, payload, oop_offset, MemNode::unordered)); - kit->set_memory(st, TypeRawPtr::BOTTOM); - kit->insert_mem_bar(Op_MemBarCPUOrder); - } + StoreFlatNode::store(kit, ptr, this, null_free, (decorators & C2_MISMATCHED) != 0); } -void InlineTypeNode::store_flat_array(GraphKit* kit, Node* base, Node* idx) const { +void InlineTypeNode::store_flat_array(GraphKit* kit, Node* base, Node* idx) { PhaseGVN& gvn = kit->gvn(); DecoratorSet decorators = IN_HEAP | IS_ARRAY | MO_UNORDERED; kit->C->set_flat_accesses(); @@ -937,7 +764,7 @@ void InlineTypeNode::replace_field_projs(Compile* C, CallNode* call, uint& proj_ } } -Node* InlineTypeNode::allocate_fields(GraphKit* kit) { +InlineTypeNode* InlineTypeNode::allocate_fields(GraphKit* kit) { InlineTypeNode* vt = clone_if_required(&kit->gvn(), kit->map()); for (uint i = 0; i < field_count(); i++) { Node* value = field_value(i); @@ -1211,15 +1038,7 @@ InlineTypeNode* InlineTypeNode::make_from_flat_impl(GraphKit* kit, ciInlineKlass } assert(!immutable_memory, "immutable memory does not need explicit atomic access"); - InlineTypeNode* vt = make_uninitialized(kit->gvn(), vk, null_free); - BasicType load_bt = vk->atomic_size_to_basic_type(null_free); - decorators |= C2_MISMATCHED | C2_CONTROL_DEPENDENT_LOAD; - const Type* val_type = Type::get_const_basic_type(load_bt); - kit->insert_mem_bar(Op_MemBarCPUOrder); - Node* payload = kit->access_load_at(base, ptr, TypeRawPtr::BOTTOM, val_type, load_bt, decorators, kit->control()); - kit->insert_mem_bar(Op_MemBarCPUOrder); - vt->convert_from_payload(kit, load_bt, kit->gvn().transform(payload), 0, null_free, trust_null_free_oop); - return gvn.transform(vt)->as_InlineType(); + return LoadFlatNode::load(kit, vk, ptr, null_free, trust_null_free_oop, (decorators & C2_MISMATCHED) != 0); } InlineTypeNode* InlineTypeNode::make_from_flat_array(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* idx) { @@ -1650,3 +1469,413 @@ const Type* InlineTypeNode::Value(PhaseGVN* phase) const { } return t; } + +InlineTypeNode* LoadFlatNode::load(GraphKit* kit, ciInlineKlass* vk, Node* ptr, bool null_free, bool trust_null_free_oop, bool mismatched) { + int output_type_size = vk->nof_nonstatic_fields() + (null_free ? 0 : 1); + const Type** output_types = TypeTuple::fields(output_type_size); + collect_field_types(vk, output_types + TypeFunc::Parms, 0, output_type_size, null_free, trust_null_free_oop); + const TypeTuple* type = TypeTuple::make(output_type_size + TypeFunc::Parms, output_types); + + LoadFlatNode* load = new LoadFlatNode(vk, type, null_free, mismatched); + load->init_req(TypeFunc::Control, kit->control()); + load->init_req(TypeFunc::Memory, kit->reset_memory()); + load->init_req(TypeFunc::Parms, ptr); + load = static_cast(kit->gvn().transform(load)); + + kit->set_control(kit->gvn().transform(new ProjNode(load, TypeFunc::Control))); + kit->set_all_memory(kit->gvn().transform(new ProjNode(load, TypeFunc::Memory))); + return load->collect_projs(kit, vk, TypeFunc::Parms, null_free); +} + +bool LoadFlatNode::expand_non_atomic(PhaseIterGVN& igvn) { + if (_mismatched) { + return false; + } + + Node* ctrl = in(TypeFunc::Control); + Node* mem = in(TypeFunc::Memory); + AddPNode* ptr = this->ptr()->as_AddP(); + Node* base = ptr->base_node(); + + for (int i = 0; i < _vk->nof_nonstatic_fields(); i++) { + ProjNode* proj_out = proj_out_or_null(TypeFunc::Parms + i); + if (proj_out == nullptr) { + continue; + } + + ciField* field = _vk->nonstatic_field_at(i); + Node* field_ptr = igvn.transform(new AddPNode(base, ptr, igvn.MakeConX(field->offset_in_bytes() - _vk->payload_offset()))); + const TypePtr* field_ptr_type = field_ptr->Value(&igvn)->is_ptr(); + igvn.set_type(field_ptr, field_ptr_type); + Node* field_value = LoadNode::make(igvn, ctrl, mem, field_ptr, field_ptr_type, igvn.type(proj_out), field->type()->basic_type(), MemNode::unordered, LoadNode::UnknownControl); + field_value = igvn.transform(field_value); + igvn.replace_node(proj_out, field_value); + } + + if (!_null_free) { + ProjNode* proj_out = proj_out_or_null(TypeFunc::Parms + _vk->nof_nonstatic_fields()); + if (proj_out != nullptr) { + Node* null_marker_ptr = igvn.transform(new AddPNode(base, ptr, igvn.MakeConX(_vk->null_marker_offset_in_payload()))); + const TypePtr* null_marker_ptr_type = null_marker_ptr->Value(&igvn)->is_ptr(); + igvn.set_type(null_marker_ptr, null_marker_ptr_type); + Node* null_marker_value = LoadNode::make(igvn, ctrl, mem, null_marker_ptr, null_marker_ptr_type, TypeInt::BOOL, T_BOOLEAN, MemNode::unordered, LoadNode::UnknownControl); + null_marker_value = igvn.transform(null_marker_value); + igvn.replace_node(proj_out, null_marker_value); + } + } + + Node* old_ctrl = proj_out_or_null(TypeFunc::Control); + if (old_ctrl != nullptr) { + igvn.replace_node(old_ctrl, ctrl); + } + Node* old_mem = proj_out_or_null(TypeFunc::Memory); + if (old_mem != nullptr) { + igvn.replace_node(old_mem, mem); + } + return true; +} + +void LoadFlatNode::expand_atomic(PhaseIterGVN& igvn) { + Node* ctrl = in(TypeFunc::Control); + Node* mem = in(TypeFunc::Memory); + Node* ptr = this->ptr(); + + BasicType payload_bt = _vk->atomic_size_to_basic_type(_null_free); + Node* in_membar = MemBarNode::make(igvn.C, Op_MemBarCPUOrder); + in_membar->init_req(TypeFunc::Control, ctrl); + in_membar->init_req(TypeFunc::Memory, mem); + in_membar = igvn.transform(in_membar); + + ctrl = igvn.transform(new ProjNode(in_membar, TypeFunc::Control)); + mem = igvn.transform(new ProjNode(in_membar, TypeFunc::Memory)); + Node* payload = LoadNode::make(igvn, ctrl, mem, ptr, TypeRawPtr::BOTTOM, Type::get_const_basic_type(payload_bt), payload_bt, MemNode::unordered, LoadNode::Pinned, true, false, true); + payload = igvn.transform(payload); + + Node* out_membar = MemBarNode::make(igvn.C, Op_MemBarCPUOrder); + out_membar->init_req(TypeFunc::Control, ctrl); + out_membar->init_req(TypeFunc::Memory, mem); + out_membar = igvn.transform(out_membar); + + ctrl = igvn.transform(new ProjNode(out_membar, TypeFunc::Control)); + Node* old_ctrl = proj_out_or_null(TypeFunc::Control); + if (old_ctrl != nullptr) { + igvn.replace_node(old_ctrl, ctrl); + } + Node* old_mem = proj_out_or_null(TypeFunc::Memory); + if (old_mem != nullptr) { + mem = igvn.transform(new ProjNode(out_membar, TypeFunc::Memory)); + igvn.replace_node(old_mem, mem); + } + + expand_projs_atomic(igvn, ctrl, payload); +} + +void LoadFlatNode::collect_field_types(ciInlineKlass* vk, const Type** field_types, int idx, int limit, bool null_free, bool trust_null_free_oop) { + assert(null_free || !trust_null_free_oop, "cannot trust null-free oop when the holder object is not null-free"); + for (int i = 0; i < vk->nof_declared_nonstatic_fields(); i++) { + ciField* field = vk->declared_nonstatic_field_at(i); + if (field->is_flat()) { + ciInlineKlass* field_klass = field->type()->as_inline_klass(); + collect_field_types(field_klass, field_types, idx, limit, field->is_null_free(), trust_null_free_oop && field->is_null_free()); + idx += field_klass->nof_nonstatic_fields() + (field->is_null_free() ? 0 : 1); + continue; + } + + const Type* field_type = Type::get_const_type(field->type()); + if (trust_null_free_oop && field->is_null_free()) { + field_type = field_type->filter(TypePtr::NOTNULL); + } + + assert(idx >= 0 && idx < limit, "field type out of bounds, %d - %d", idx, limit); + field_types[idx] = field_type; + idx++; + } + + if (!null_free) { + assert(idx >= 0 && idx < limit, "field type out of bounds, %d - %d", idx, limit); + field_types[idx] = TypeInt::BOOL; + } +} + +// Create an InlineTypeNode from a LoadFlatNode with its fields being extracted from the +// LoadFlatNode +InlineTypeNode* LoadFlatNode::collect_projs(GraphKit* kit, ciInlineKlass* vk, int proj_con, bool null_free) { + PhaseGVN& gvn = kit->gvn(); + InlineTypeNode* res = InlineTypeNode::make_uninitialized(gvn, vk, null_free); + for (int i = 0; i < vk->nof_declared_nonstatic_fields(); i++) { + ciField* field = vk->declared_nonstatic_field_at(i); + Node* field_value; + if (field->is_flat()) { + ciInlineKlass* field_klass = field->type()->as_inline_klass(); + field_value = collect_projs(kit, field_klass, proj_con, field->is_null_free()); + proj_con += field_klass->nof_nonstatic_fields() + (field->is_null_free() ? 0 : 1); + } else { + field_value = gvn.transform(new ProjNode(this, proj_con)); + if (field->type()->is_inlinetype()) { + field_value = InlineTypeNode::make_from_oop(kit, field_value, field->type()->as_inline_klass()); + } + proj_con++; + } + res->set_field_value(i, field_value); + } + + if (null_free) { + res->set_null_marker(gvn); + } else { + res->set_null_marker(gvn, gvn.transform(new ProjNode(this, proj_con))); + } + return gvn.transform(res)->as_InlineType(); +} + +// Extract the values of the flattened fields from the loaded payload +void LoadFlatNode::expand_projs_atomic(PhaseIterGVN& igvn, Node* ctrl, Node* payload) { + BasicType payload_bt = _vk->atomic_size_to_basic_type(_null_free); + for (int i = 0; i < _vk->nof_nonstatic_fields(); i++) { + ProjNode* proj_out = proj_out_or_null(TypeFunc::Parms + i); + if (proj_out == nullptr) { + continue; + } + + ciField* field = _vk->nonstatic_field_at(i); + int field_offset = field->offset_in_bytes() - _vk->payload_offset(); + const Type* field_type = igvn.type(proj_out); + Node* field_value = get_payload_value(igvn, ctrl, payload_bt, payload, field_type, field->type()->basic_type(), field_offset); + igvn.replace_node(proj_out, field_value); + } + + if (!_null_free) { + ProjNode* proj_out = proj_out_or_null(TypeFunc::Parms + _vk->nof_nonstatic_fields()); + if (proj_out == nullptr) { + return; + } + + int null_marker_offset = _vk->null_marker_offset_in_payload(); + Node* null_marker_value = get_payload_value(igvn, ctrl, payload_bt, payload, TypeInt::BOOL, T_BOOLEAN, null_marker_offset); + igvn.replace_node(proj_out, null_marker_value); + } +} + +Node* LoadFlatNode::get_payload_value(PhaseIterGVN& igvn, Node* ctrl, BasicType payload_bt, Node* payload, const Type* value_type, BasicType value_bt, int offset) { + assert((offset + type2aelembytes(value_bt)) <= type2aelembytes(payload_bt), "Value does not fit into payload"); + Node* value = nullptr; + // Shift to the right position in the long value + Node* shift_val = igvn.intcon(offset << LogBitsPerByte); + if (payload_bt == T_LONG) { + value = igvn.transform(new URShiftLNode(payload, shift_val)); + value = igvn.transform(new ConvL2INode(value)); + } else { + value = igvn.transform(new URShiftINode(payload, shift_val)); + } + + if (value_bt == T_INT) { + return value; + } else if (!is_java_primitive(value_bt)) { + assert(UseCompressedOops && payload_bt == T_LONG, "Naturally atomic"); + value = igvn.transform(new CastI2NNode(ctrl, value, value_type->make_narrowoop())); + value = igvn.transform(new DecodeNNode(value, value_type)); + + // Similar to CheckCastPP nodes with raw input, CastI2N nodes require special handling in 'PhaseCFG::schedule_late' to ensure the + // register allocator does not move the CastI2N below a safepoint. This is necessary to avoid having the raw pointer span a safepoint, + // making it opaque to the GC. Unlike CheckCastPPs, which need extra handling in 'Scheduling::ComputeRegisterAntidependencies' due to + // scalarization, CastI2N nodes are always used by a load if scalarization happens which inherently keeps them pinned above the safepoint. + return value; + } else { + // Make sure to zero unused bits in the 32-bit value + return Compile::narrow_value(value_bt, value, nullptr, &igvn, true); + } +} + +void StoreFlatNode::store(GraphKit* kit, Node* ptr, InlineTypeNode* value, bool null_free, bool mismatched) { + value = value->allocate_fields(kit); + Node* store = new StoreFlatNode(null_free, mismatched); + store->init_req(TypeFunc::Control, kit->control()); + store->init_req(TypeFunc::Memory, kit->reset_memory()); + store->init_req(TypeFunc::Parms, ptr); + store->init_req(TypeFunc::Parms + 1, value); + + store = kit->gvn().transform(store); + kit->set_control(kit->gvn().transform(new ProjNode(store, TypeFunc::Control))); + kit->set_all_memory(kit->gvn().transform(new ProjNode(store, TypeFunc::Memory))); +} + +bool StoreFlatNode::expand_non_atomic(PhaseIterGVN& igvn) { + if (_mismatched) { + return false; + } + + Node* ctrl = in(TypeFunc::Control); + Node* mem = in(TypeFunc::Memory); + AddPNode* ptr = this->ptr()->as_AddP(); + Node* base = ptr->base_node(); + InlineTypeNode* value = this->value(); + + ciInlineKlass* vk = igvn.type(value)->inline_klass(); + MergeMemNode* new_mem = MergeMemNode::make(mem); + for (int i = 0; i < vk->nof_nonstatic_fields(); i++) { + ciField* field = vk->nonstatic_field_at(i); + Node* field_ptr = igvn.transform(new AddPNode(base, ptr, igvn.MakeConX(field->offset_in_bytes() - vk->payload_offset()))); + const TypePtr* field_ptr_type = field_ptr->Value(&igvn)->is_ptr(); + igvn.set_type(field_ptr, field_ptr_type); + Node* field_value = value->field_value_by_offset(field->offset_in_bytes(), true); + Node* store = StoreNode::make(igvn, ctrl, mem, field_ptr, field_ptr_type, field_value, field->type()->basic_type(), MemNode::unordered); + store = igvn.transform(store); + new_mem->set_memory_at(igvn.C->get_alias_index(field_ptr_type), store); + } + + if (!_null_free) { + Node* null_marker_ptr = igvn.transform(new AddPNode(base, ptr, igvn.MakeConX(vk->null_marker_offset_in_payload()))); + const TypePtr* null_marker_ptr_type = null_marker_ptr->Value(&igvn)->is_ptr(); + igvn.set_type(null_marker_ptr, null_marker_ptr_type); + Node* null_marker_value = value->get_null_marker(); + Node* store = StoreNode::make(igvn, ctrl, mem, null_marker_ptr, null_marker_ptr_type, null_marker_value, T_BOOLEAN, MemNode::unordered); + store = igvn.transform(store); + new_mem->set_memory_at(igvn.C->get_alias_index(null_marker_ptr_type), store); + } + + mem = igvn.transform(new_mem); + Node* old_ctrl = proj_out_or_null(TypeFunc::Control); + if (old_ctrl != nullptr) { + igvn.replace_node(old_ctrl, ctrl); + } + Node* old_mem = proj_out_or_null(TypeFunc::Memory); + if (old_mem != nullptr) { + igvn.replace_node(old_mem, mem); + } + return true; +} + +void StoreFlatNode::expand_atomic(PhaseIterGVN& igvn) { + // Convert to a payload value <= 64-bit and write atomically. + // The payload might contain at most two oop fields that must be narrow because otherwise they would be 64-bit + // in size and would then be written by a "normal" oop store. If the payload contains oops, its size is always + // 64-bit because the next smaller (power-of-two) size would be 32-bit which could only hold one narrow oop that + // would then be written by a normal narrow oop store. These properties are asserted in 'convert_to_payload'. + Node* ctrl = in(TypeFunc::Control); + Node* mem = in(TypeFunc::Memory); + Node* ptr = this->ptr(); + InlineTypeNode* value = this->value(); + + int oop_off_1 = -1; + int oop_off_2 = -1; + Node* payload = convert_to_payload(igvn, ctrl, value, _null_free, oop_off_1, oop_off_2); + + ciInlineKlass* vk = igvn.type(value)->inline_klass(); + BasicType payload_bt = vk->atomic_size_to_basic_type(_null_free); + Node* in_membar = MemBarNode::make(igvn.C, Op_MemBarCPUOrder); + in_membar->init_req(TypeFunc::Control, ctrl); + in_membar->init_req(TypeFunc::Memory, mem); + in_membar = igvn.transform(in_membar); + + ctrl = igvn.transform(new ProjNode(in_membar, TypeFunc::Control)); + mem = igvn.transform(new ProjNode(in_membar, TypeFunc::Memory)); + if (!UseG1GC || oop_off_1 == -1) { + // No oop fields or no late barrier expansion. Emit an atomic store of the payload and add GC barriers if needed. + assert(oop_off_2 == -1 || !UseG1GC, "sanity"); + // ZGC does not support compressed oops, so only one oop can be in the payload which is written by a "normal" oop store. + assert((oop_off_1 == -1 && oop_off_2 == -1) || !UseZGC, "ZGC does not support embedded oops in flat fields"); + const Type* val_type = Type::get_const_basic_type(payload_bt); + Node* store = StoreNode::make(igvn, ctrl, mem, ptr, TypeRawPtr::BOTTOM, payload, payload_bt, MemNode::unordered, true); + store = igvn.transform(store); + mem = MergeMemNode::make(mem); + mem->set_req(Compile::AliasIdxRaw, store); + mem = igvn.transform(mem); + } else { + // Contains oops and requires late barrier expansion. Emit a special store node that allows to emit GC barriers in the backend. + assert(UseG1GC, "Unexpected GC"); + assert(payload_bt == T_LONG, "Unexpected payload type"); + // If one oop, set the offset (if no offset is set, two oops are assumed by the backend) + Node* oop_offset = (oop_off_2 == -1) ? igvn.intcon(oop_off_1) : nullptr; + Node* store = igvn.transform(new StoreLSpecialNode(ctrl, mem, ptr, TypeRawPtr::BOTTOM, payload, oop_offset, MemNode::unordered)); + mem = MergeMemNode::make(mem); + mem->set_req(Compile::AliasIdxRaw, store); + mem = igvn.transform(mem); + } + + Node* out_membar = MemBarNode::make(igvn.C, Op_MemBarCPUOrder); + out_membar->init_req(TypeFunc::Control, ctrl); + out_membar->init_req(TypeFunc::Memory, mem); + out_membar = igvn.transform(out_membar); + + Node* old_ctrl = proj_out_or_null(TypeFunc::Control); + if (old_ctrl != nullptr) { + ctrl = igvn.transform(new ProjNode(out_membar, TypeFunc::Control)); + igvn.replace_node(old_ctrl, ctrl); + } + Node* old_mem = proj_out_or_null(TypeFunc::Memory); + if (old_mem != nullptr) { + mem = igvn.transform(new ProjNode(out_membar, TypeFunc::Memory)); + igvn.replace_node(old_mem, mem); + } +} + +// Convert the field values to a payload value of type 'bt' +Node* StoreFlatNode::convert_to_payload(PhaseIterGVN& igvn, Node* ctrl, InlineTypeNode* value, bool null_free, int& oop_off_1, int& oop_off_2) { + ciInlineKlass* vk = igvn.type(value)->inline_klass(); + BasicType payload_bt = vk->atomic_size_to_basic_type(null_free); + Node* payload = igvn.zerocon(payload_bt); + if (!null_free) { + // Set the null marker + payload = set_payload_value(igvn, payload_bt, payload, T_BOOLEAN, value->get_null_marker(), vk->null_marker_offset_in_payload()); + } + + // Iterate over the fields and add their values to the payload + for (int i = 0; i < vk->nof_nonstatic_fields(); i++) { + ciField* field = vk->nonstatic_field_at(i); + Node* field_value = value->field_value_by_offset(field->offset_in_bytes(), true); + ciType* field_klass = field->type(); + BasicType field_bt = field_klass->basic_type(); + int field_offset_in_payload = field->offset_in_bytes() - vk->payload_offset(); + if (!field_klass->is_primitive_type()) { + // Narrow oop field + assert(UseCompressedOops && payload_bt == T_LONG, "Naturally atomic"); + if (oop_off_1 == -1) { + oop_off_1 = field_offset_in_payload; + } else { + assert(oop_off_2 == -1, "already set"); + oop_off_2 = field_offset_in_payload; + } + + const Type* val_type = Type::get_const_type(field_klass)->make_narrowoop(); + if (field_value->is_InlineType()) { + assert(field_value->as_InlineType()->is_allocated(&igvn), "must be allocated"); + } + + field_value = igvn.transform(new EncodePNode(field_value, val_type)); + field_value = igvn.transform(new CastP2XNode(ctrl, field_value)); + field_value = igvn.transform(new ConvL2INode(field_value)); + field_bt = T_INT; + } + payload = set_payload_value(igvn, payload_bt, payload, field_bt, field_value, field_offset_in_payload); + } + + return payload; +} + +Node* StoreFlatNode::set_payload_value(PhaseIterGVN& igvn, BasicType payload_bt, Node* payload, BasicType val_bt, Node* value, int offset) { + assert((offset + type2aelembytes(val_bt)) <= type2aelembytes(payload_bt), "Value does not fit into payload"); + + // Make sure to zero unused bits in the 32-bit value + if (val_bt == T_BYTE || val_bt == T_BOOLEAN) { + value = igvn.transform(new AndINode(value, igvn.intcon(0xFF))); + } else if (val_bt == T_CHAR || val_bt == T_SHORT) { + value = igvn.transform(new AndINode(value, igvn.intcon(0xFFFF))); + } else if (val_bt == T_FLOAT) { + value = igvn.transform(new MoveF2INode(value)); + } else { + assert(val_bt == T_INT, "Unsupported type: %s", type2name(val_bt)); + } + + Node* shift_val = igvn.intcon(offset << LogBitsPerByte); + if (payload_bt == T_LONG) { + // Convert to long and remove the sign bit (the backend will fold this and emit a zero extend i2l) + value = igvn.transform(new ConvI2LNode(value)); + value = igvn.transform(new AndLNode(value, igvn.longcon(0xFFFFFFFF))); + + Node* shift_value = igvn.transform(new LShiftLNode(value, shift_val)); + payload = new OrLNode(shift_value, payload); + } else { + Node* shift_value = igvn.transform(new LShiftINode(value, shift_val)); + payload = new OrINode(shift_value, payload); + } + return igvn.transform(payload); +} diff --git a/src/hotspot/share/opto/inlinetypenode.hpp b/src/hotspot/share/opto/inlinetypenode.hpp index 902ec28e54b..6f5855cfbec 100644 --- a/src/hotspot/share/opto/inlinetypenode.hpp +++ b/src/hotspot/share/opto/inlinetypenode.hpp @@ -28,8 +28,9 @@ #include "ci/ciInlineKlass.hpp" #include "gc/shared/c2/barrierSetC2.hpp" #include "oops/accessDecorators.hpp" -#include "opto/connode.hpp" +#include "opto/compile.hpp" #include "opto/loopnode.hpp" +#include "opto/multnode.hpp" #include "opto/node.hpp" class GraphKit; @@ -80,9 +81,6 @@ class InlineTypeNode : public TypeNode { static InlineTypeNode* make_from_flat_impl(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* ptr, bool atomic, bool immutable_memory, bool null_free, bool trust_null_free_oop, DecoratorSet decorators, GrowableArray& visited); - void convert_from_payload(GraphKit* kit, BasicType bt, Node* payload, int holder_offset, bool null_free, bool trust_null_free_oop); - Node* convert_to_payload(GraphKit* kit, BasicType bt, Node* payload, int holder_offset, bool null_free, int null_marker_offset, int& oop_off_1, int& oop_off_2) const; - public: // Create with all-zero field values static InlineTypeNode* make_all_zero(PhaseGVN& gvn, ciInlineKlass* vk); @@ -135,9 +133,9 @@ class InlineTypeNode : public TypeNode { void make_scalar_in_safepoints(PhaseIterGVN* igvn, bool allow_oop = true); // Store the inline type as a flat (headerless) representation - void store_flat(GraphKit* kit, Node* base, Node* ptr, bool atomic, bool immutable_memory, bool null_free, DecoratorSet decorators) const; + void store_flat(GraphKit* kit, Node* base, Node* ptr, bool atomic, bool immutable_memory, bool null_free, DecoratorSet decorators); // Store the inline type as a flat (headerless) representation into an array - void store_flat_array(GraphKit* kit, Node* base, Node* idx) const; + void store_flat_array(GraphKit* kit, Node* base, Node* idx); // Make sure that inline type is fully scalarized InlineTypeNode* adjust_scalarization_depth(GraphKit* kit); @@ -149,7 +147,7 @@ class InlineTypeNode : public TypeNode { void replace_field_projs(Compile* C, CallNode* call, uint& proj_idx); // Allocate all non-flat inline type fields - Node* allocate_fields(GraphKit* kit); + InlineTypeNode* allocate_fields(GraphKit* kit); Node* tagged_klass(PhaseGVN& gvn) { return tagged_klass(inline_klass(), gvn); @@ -170,4 +168,81 @@ class InlineTypeNode : public TypeNode { virtual int Opcode() const; }; +// Load from a flat element, the node produces 1 Proj output for each flattened field of the flat +// element. The order of the Proj node is the same as that of _vk->_nonstatic_fields, and the null +// marker if existing will be the last Proj output. This node acts as if the load happens +// atomically and will be expanded to loading the whole payload and extracting the flattened fields +// from the loaded payload. In special cases, such as when the object from which this load read +// does not escape, this node can be expanded to multiple loads from each flattened field. +// This node allows us to replace its results with the value from a matching store because the +// payload value cannot be directly propagated if it contains oops. This effect, in turns, allows +// objects with atomic flat fields to be scalar replaced. +class LoadFlatNode final : public MultiNode { +private: + ciInlineKlass* _vk; + const TypeTuple* _type; + bool _null_free; + bool _mismatched; + +public: + static InlineTypeNode* load(GraphKit* kit, ciInlineKlass* vk, Node* ptr, bool null_free, bool trust_null_free_oop, bool mismatched); + Node* ptr() { return in(TypeFunc::Parms); } + bool expand_non_atomic(PhaseIterGVN& igvn); + void expand_atomic(PhaseIterGVN& igvn); + +private: + LoadFlatNode(ciInlineKlass* vk, const TypeTuple* type, bool null_free, bool mismatched) + : MultiNode(TypeFunc::Parms + 1), _vk(vk), _type(type), _null_free(null_free), _mismatched(mismatched) { + Compile::current()->add_flat_access(this); + } + + virtual int Opcode() const override; + virtual const Type* bottom_type() const override { return _type; } + virtual const TypePtr* adr_type() const override { return TypePtr::BOTTOM; } + virtual uint size_of() const override { return sizeof(LoadFlatNode); } + virtual uint hash() const override { return MultiNode::hash() + _vk->hash() + _null_free; } + virtual bool cmp(const Node& other) const override { + const LoadFlatNode& f = static_cast(other); + return MultiNode::cmp(f) && _vk == f._vk && _null_free == f._null_free; + } + + static void collect_field_types(ciInlineKlass* vk, const Type** field_types, int idx, int limit, bool null_free, bool trust_null_free_oop); + InlineTypeNode* collect_projs(GraphKit* kit, ciInlineKlass* vk, int proj_con, bool null_free); + void expand_projs_atomic(PhaseIterGVN& gvn, Node* ctrl, Node* payload); + static Node* get_payload_value(PhaseIterGVN& igvn, Node* ctrl, BasicType payload_bt, Node* payload, const Type* value_type, BasicType value_bt, int offset); +}; + +// Store an InlineTypeNode to a flat element, the store acts as if it is atomic. Similar to +// LoadFlatNode, this node is expanded to storing a payload creating from the field values of the +// InlineTypeNode, and under special circumstances, when there is no racing access to the field, +// this node can be expanded to multiple stores to each flattened field. +// The purposes of this node complement those of LoadFlatNode. +class StoreFlatNode final : public MultiNode { +private: + bool _null_free; + bool _mismatched; + +public: + static void store(GraphKit* kit, Node* ptr, InlineTypeNode* obj, bool null_free, bool mismatched); + Node* ptr() { return in(TypeFunc::Parms); } + InlineTypeNode* value() { return in(TypeFunc::Parms + 1)->as_InlineType(); } + bool expand_non_atomic(PhaseIterGVN& igvn); + void expand_atomic(PhaseIterGVN& igvn); + +private: + StoreFlatNode(bool null_free, bool mismatched) : MultiNode(TypeFunc::Parms + 2), _null_free(null_free), _mismatched(mismatched) { + Compile::current()->add_flat_access(this); + } + + virtual int Opcode() const override; + virtual const Type* bottom_type() const override { return TypeTuple::MEMBAR; } + virtual const TypePtr* adr_type() const override { return TypePtr::BOTTOM; } + virtual uint size_of() const override { return sizeof(LoadFlatNode); } + virtual uint hash() const override { return MultiNode::hash() + _null_free; } + virtual bool cmp(const Node& other) const override { return MultiNode::cmp(other) && _null_free == static_cast(other)._null_free; } + + static Node* convert_to_payload(PhaseIterGVN& igvn, Node* ctrl, InlineTypeNode* value, bool null_free, int& oop_off_1, int& oop_off_2); + static Node* set_payload_value(PhaseIterGVN& igvn, BasicType payload_bt, Node* payload, BasicType val_bt, Node* value, int offset); +}; + #endif // SHARE_VM_OPTO_INLINETYPENODE_HPP diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp index c48a3cd6d6f..865add2be6a 100644 --- a/src/hotspot/share/opto/loopopts.cpp +++ b/src/hotspot/share/opto/loopopts.cpp @@ -38,6 +38,7 @@ #include "opto/mulnode.hpp" #include "opto/movenode.hpp" #include "opto/opaquenode.hpp" +#include "opto/opcodes.hpp" #include "opto/rootnode.hpp" #include "opto/subnode.hpp" #include "opto/subtypenode.hpp" @@ -1126,7 +1127,7 @@ void PhaseIdealLoop::move_flat_array_check_out_of_loop(Node* n) { mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw); } else if (mem->is_Proj()) { mem = mem->in(0); - } else if (mem->is_MemBar() || mem->is_SafePoint()) { + } else if (mem->is_MemBar() || mem->is_SafePoint() || mem->Opcode() == Op_LoadFlat || mem->Opcode() == Op_StoreFlat) { mem = mem->in(TypeFunc::Memory); } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) { mem = mem->in(MemNode::Memory); diff --git a/src/hotspot/share/opto/macro.cpp b/src/hotspot/share/opto/macro.cpp index ef96a73f4b5..2ab9f96045e 100644 --- a/src/hotspot/share/opto/macro.cpp +++ b/src/hotspot/share/opto/macro.cpp @@ -46,6 +46,7 @@ #include "opto/narrowptrnode.hpp" #include "opto/node.hpp" #include "opto/opaquenode.hpp" +#include "opto/opcodes.hpp" #include "opto/phaseX.hpp" #include "opto/rootnode.hpp" #include "opto/runtime.hpp" @@ -191,6 +192,8 @@ static Node *scan_mem_chain(Node *mem, int alias_idx, int offset, Node *start_me } } mem = in->in(TypeFunc::Memory); + } else if (in->Opcode() == Op_LoadFlat || in->Opcode() == Op_StoreFlat) { + mem = in->in(TypeFunc::Memory); } else { #ifdef ASSERT in->dump(); diff --git a/src/hotspot/share/opto/memnode.cpp b/src/hotspot/share/opto/memnode.cpp index 36feef02724..2b509642b63 100644 --- a/src/hotspot/share/opto/memnode.cpp +++ b/src/hotspot/share/opto/memnode.cpp @@ -36,6 +36,7 @@ #include "opto/addnode.hpp" #include "opto/arraycopynode.hpp" #include "opto/cfgnode.hpp" +#include "opto/opcodes.hpp" #include "opto/regalloc.hpp" #include "opto/compile.hpp" #include "opto/connode.hpp" @@ -55,6 +56,7 @@ #include "opto/vectornode.hpp" #include "utilities/align.hpp" #include "utilities/copy.hpp" +#include "utilities/globalDefinitions.hpp" #include "utilities/macros.hpp" #include "utilities/powerOfTwo.hpp" #include "utilities/vmError.hpp" @@ -302,10 +304,15 @@ Node* MemNode::optimize_simple_memory_chain(Node* mchain, const TypeOopPtr* t_oo break; } result = proj_in->in(TypeFunc::Memory); + } else if (proj_in->Opcode() == Op_LoadFlat || proj_in->Opcode() == Op_StoreFlat) { + if (is_strict_final_load) { + // LoadFlat and StoreFlat cannot happen to strict final fields + result = proj_in->in(TypeFunc::Memory); + } } else if (proj_in->is_top()) { break; // dead code } else { - assert(false, "unexpected projection"); + assert(false, "unexpected projection of %s", proj_in->Name()); } } else if (result->is_ClearArray()) { if (!is_instance || !ClearArrayNode::step_through(&result, instance_id, phase)) { @@ -1086,6 +1093,7 @@ Node* LoadNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypeP case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break; case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic_access); break; case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break; + case T_ARRAY: case T_OBJECT: case T_NARROWOOP: #ifdef _LP64 @@ -1099,7 +1107,7 @@ Node* LoadNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypeP } break; default: - ShouldNotReachHere(); + assert(false, "unexpected basic type %s", type2name(bt)); break; } assert(load != nullptr, "LoadNode should have been created"); @@ -2902,6 +2910,7 @@ StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const case T_METADATA: case T_ADDRESS: case T_OBJECT: + case T_ARRAY: #ifdef _LP64 if (adr->bottom_type()->is_ptr_to_narrowoop()) { val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop())); @@ -2917,7 +2926,7 @@ StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const return new StorePNode(ctl, mem, adr, adr_type, val, mo); } default: - ShouldNotReachHere(); + assert(false, "unexpected basic type %s", type2name(bt)); return (StoreNode*)nullptr; } } diff --git a/src/hotspot/share/opto/node.cpp b/src/hotspot/share/opto/node.cpp index 7645c380dc5..5ba16477af9 100644 --- a/src/hotspot/share/opto/node.cpp +++ b/src/hotspot/share/opto/node.cpp @@ -567,6 +567,9 @@ Node *Node::clone() const { if (n->is_InlineType()) { C->add_inline_type(n); } + if (n->Opcode() == Op_LoadFlat || n->Opcode() == Op_StoreFlat) { + C->add_flat_access(n); + } Compile::current()->record_modified_node(n); return n; // Return the clone } diff --git a/src/hotspot/share/runtime/deoptimization.cpp b/src/hotspot/share/runtime/deoptimization.cpp index ff1a09c789f..52a7a73f833 100644 --- a/src/hotspot/share/runtime/deoptimization.cpp +++ b/src/hotspot/share/runtime/deoptimization.cpp @@ -1646,12 +1646,21 @@ void Deoptimization::reassign_flat_array_elements(frame* fr, RegisterMap* reg_ma InlineKlass* vk = vak->element_klass(); assert(vk->maybe_flat_in_array(), "should only be used for flat inline type arrays"); // Adjust offset to omit oop header - int base_offset = arrayOopDesc::base_offset_in_bytes(T_FLAT_ELEMENT) - InlineKlass::cast(vk)->payload_offset(); + int base_offset = arrayOopDesc::base_offset_in_bytes(T_FLAT_ELEMENT) - vk->payload_offset(); // Initialize all elements of the flat inline type array for (int i = 0; i < sv->field_size(); i++) { - ScopeValue* val = sv->field_at(i); + ObjectValue* val = sv->field_at(i)->as_ObjectValue(); int offset = base_offset + (i << Klass::layout_helper_log2_element_size(vak->layout_helper())); - reassign_fields_by_klass(vk, fr, reg_map, val->as_ObjectValue(), 0, (oop)obj, is_jvmci, offset, CHECK); + reassign_fields_by_klass(vk, fr, reg_map, val, 0, (oop)obj, is_jvmci, offset, CHECK); + if (!obj->is_null_free_array()) { + jboolean null_marker_value; + if (val->maybe_null()) { + null_marker_value = StackValue::create_stack_value(fr, reg_map, val->null_marker())->get_jint() & 1; + } else { + null_marker_value = 1; + } + obj->bool_field_put(offset + vk->null_marker_offset(), null_marker_value); + } } } diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestScalarReplaceFlatFields.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestScalarReplaceFlatFields.java new file mode 100644 index 00000000000..d8acfe6a02d --- /dev/null +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestScalarReplaceFlatFields.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package compiler.valhalla.inlinetypes; + +import compiler.lib.ir_framework.*; +import jdk.internal.vm.annotation.NullRestricted; +import jdk.internal.vm.annotation.Strict; + +/* + * @test + * @bug 8364191 + * @summary Test the removal of allocations of objects with atomic flat fields + * @library /test/lib / + * @enablePreview + * @modules java.base/jdk.internal.value + * java.base/jdk.internal.vm.annotation + * @run main compiler.valhalla.inlinetypes.TestScalarReplaceFlatFields + */ +public class TestScalarReplaceFlatFields { + @DontInline + private static void call() {} + + static value class V0 { + byte v1; + byte v2; + + V0(int v1, int v2) { + this.v1 = (byte) v1; + this.v2 = (byte) v2; + } + } + + static value class V1 { + V0 v; + short s; + + V1(V0 v, int s) { + this.v = v; + this.s = (short) s; + } + } + + static class Holder { + @Strict + @NullRestricted + V1 v1; + V1 v2; + + Holder(V1 v1, V1 v2) { + this.v1 = v1; + this.v2 = v2; + } + } + + @Test + @IR(failOn = IRNode.ALLOC) + @Arguments(values = {Argument.RANDOM_EACH}) + private static int testField(int v) { + V1 v1 = new V1(null, v); + V1 v2 = new V1(new V0(v, v), v); + Holder h = new Holder(v1, v2); + call(); + return h.v1.s; + } + + @Test + @IR(failOn = IRNode.ALLOC) + @Arguments(values = {Argument.RANDOM_EACH, Argument.RANDOM_EACH}) + private static int testArray(int v1, int v2) { + V1[] array = new V1[2]; + array[0] = new V1(null, v1); + array[1] = new V1(new V0(v1, v2), v2); + call(); + return array[1].v.v1; + } + + public static void main(String[] args) { + InlineTypes.getFramework() + .addScenarios(InlineTypes.DEFAULT_SCENARIOS) + .start(); + } +} From 428b656c4bb2c23f5dbd12c4136244844a3830dc Mon Sep 17 00:00:00 2001 From: Quan Anh Mai Date: Fri, 1 Aug 2025 16:47:22 +0700 Subject: [PATCH 2/3] small fix StoreFlatNode::size_of --- src/hotspot/share/opto/inlinetypenode.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/opto/inlinetypenode.hpp b/src/hotspot/share/opto/inlinetypenode.hpp index 6f5855cfbec..7b0cbd3e977 100644 --- a/src/hotspot/share/opto/inlinetypenode.hpp +++ b/src/hotspot/share/opto/inlinetypenode.hpp @@ -237,7 +237,7 @@ class StoreFlatNode final : public MultiNode { virtual int Opcode() const override; virtual const Type* bottom_type() const override { return TypeTuple::MEMBAR; } virtual const TypePtr* adr_type() const override { return TypePtr::BOTTOM; } - virtual uint size_of() const override { return sizeof(LoadFlatNode); } + virtual uint size_of() const override { return sizeof(StoreFlatNode); } virtual uint hash() const override { return MultiNode::hash() + _null_free; } virtual bool cmp(const Node& other) const override { return MultiNode::cmp(other) && _null_free == static_cast(other)._null_free; } From eb23f4e5a6a2db7abf77da701f46d2ac6c4d20e7 Mon Sep 17 00:00:00 2001 From: Quan Anh Mai Date: Thu, 7 Aug 2025 20:31:10 +0700 Subject: [PATCH 3/3] fix release build --- src/hotspot/share/opto/escape.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/opto/escape.cpp b/src/hotspot/share/opto/escape.cpp index 01ecffee21d..f14ccfa9a6f 100644 --- a/src/hotspot/share/opto/escape.cpp +++ b/src/hotspot/share/opto/escape.cpp @@ -1871,7 +1871,7 @@ void ConnectionGraph::add_final_edges(Node *n) { Node* field_value = value->field_value_by_offset(field->offset_in_bytes(), true); PointsToNode* field_value_ptn = ptnode_adr(field_value->_idx); - set_escape_state(field_value_ptn, PointsToNode::GlobalEscape, "store into a flat field"); + set_escape_state(field_value_ptn, PointsToNode::GlobalEscape NOT_PRODUCT(COMMA "store into a flat field")); } break; }