From c6c6beb3121e8d14b748fd18f0f1e74ea52484f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20H=C3=BCbner?= Date: Fri, 12 Sep 2025 10:24:05 +0200 Subject: [PATCH 1/8] Follow JEP450 MarkWord specification. --- src/hotspot/share/oops/markWord.hpp | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/src/hotspot/share/oops/markWord.hpp b/src/hotspot/share/oops/markWord.hpp index 7a6660c90b0..62980e6111d 100644 --- a/src/hotspot/share/oops/markWord.hpp +++ b/src/hotspot/share/oops/markWord.hpp @@ -111,10 +111,9 @@ // -------- // hash:24 ------------>| larval:1 age:4 inline_type:1 lock:2 // -// 64 bits: +// 64 bits (follows ocmpact object header): // -------- -// unused:1 | <-- hash:31 -->| unused:22 larval:1 age:4 flat_array:1 null_free_array:1 inline_type:1 lock:2 -// klass:22 hash:31 -->| larval:1 age:4 flat_array:1 null_free_array:1 inline_type:1 self-fwd:1 lock:2 (normal object) +// klass:22 hash:31 -->| larval:1 flat_array:1 null_free_array:1 inline_type:1 age:4 self-fwd:1 lock:2 (normal object) // // The "fast" static type bits (flat_array, null_free_array, and inline_type) // are placed lowest next to lock bits to more easily decode forwarding pointers. @@ -183,44 +182,43 @@ class markWord { // Constants, in least significant bit order static const int lock_bits = 2; static const int self_fwd_bits = 1; + // instance state + static const int age_bits = 4; // EnableValhalla: static prototype header bits (fast path instead of klass layout_helper) static const int inline_type_bits = 1; static const int null_free_array_bits = LP64_ONLY(1) NOT_LP64(0); static const int flat_array_bits = LP64_ONLY(1) NOT_LP64(0); - // instance state - static const int age_bits = 4; static const int larval_bits = 1; static const int max_hash_bits = BitsPerWord - age_bits - lock_bits - inline_type_bits - larval_bits - flat_array_bits - null_free_array_bits - self_fwd_bits; static const int hash_bits = max_hash_bits > 31 ? 31 : max_hash_bits; static const int lock_shift = 0; - static const int self_fwd_shift = lock_bits ; - static const int inline_type_shift = self_fwd_shift + self_fwd_bits; + static const int self_fwd_shift = lock_shift + lock_bits; + static const int age_shift = self_fwd_shift + self_fwd_bits; + static const int inline_type_shift = age_shift + age_bits; static const int null_free_array_shift = inline_type_shift + inline_type_bits; static const int flat_array_shift = null_free_array_shift + null_free_array_bits; - static const int age_shift = flat_array_shift + flat_array_bits; - static const int larval_shift = age_shift + age_bits; + static const int larval_shift = flat_array_shift + flat_array_bits; static const int hash_shift = larval_shift + larval_bits; static const uintptr_t lock_mask = right_n_bits(lock_bits); static const uintptr_t lock_mask_in_place = lock_mask << lock_shift; static const uintptr_t self_fwd_mask = right_n_bits(self_fwd_bits); static const uintptr_t self_fwd_mask_in_place = self_fwd_mask << self_fwd_shift; - static const uintptr_t inline_type_bit_in_place = 1 << inline_type_shift; - static const uintptr_t inline_type_mask = inline_type_bit_in_place + lock_mask; - static const uintptr_t inline_type_mask_in_place = inline_type_mask << lock_shift; + static const uintptr_t inline_type_bit_in_place = right_n_bits(inline_type_bits) << inline_type_shift; + static const uintptr_t inline_type_mask_in_place = inline_type_bit_in_place + lock_mask; static const uintptr_t null_free_array_mask = right_n_bits(null_free_array_bits); static const uintptr_t null_free_array_mask_in_place = (null_free_array_mask << null_free_array_shift) | lock_mask_in_place; - static const uintptr_t null_free_array_bit_in_place = (1 << null_free_array_shift); + static const uintptr_t null_free_array_bit_in_place = (right_n_bits(null_free_array_bits) << null_free_array_shift); static const uintptr_t flat_array_mask = right_n_bits(flat_array_bits); static const uintptr_t flat_array_mask_in_place = (flat_array_mask << flat_array_shift) | null_free_array_mask_in_place | lock_mask_in_place; - static const uintptr_t flat_array_bit_in_place = (1 << flat_array_shift); + static const uintptr_t flat_array_bit_in_place = right_n_bits(flat_array_bits) << flat_array_shift; static const uintptr_t age_mask = right_n_bits(age_bits); static const uintptr_t age_mask_in_place = age_mask << age_shift; static const uintptr_t larval_mask = right_n_bits(larval_bits); static const uintptr_t larval_mask_in_place = (larval_mask << larval_shift) | inline_type_mask_in_place; - static const uintptr_t larval_bit_in_place = (1 << larval_shift); + static const uintptr_t larval_bit_in_place = right_n_bits(larval_bits) << larval_shift; static const uintptr_t hash_mask = right_n_bits(hash_bits); static const uintptr_t hash_mask_in_place = hash_mask << hash_shift; From 2464706cdec791930ca5cf2b1908de150da028f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20H=C3=BCbner?= Date: Fri, 12 Sep 2025 11:43:14 +0200 Subject: [PATCH 2/8] Fix typo. --- src/hotspot/share/oops/markWord.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/oops/markWord.hpp b/src/hotspot/share/oops/markWord.hpp index 62980e6111d..9139e3f0102 100644 --- a/src/hotspot/share/oops/markWord.hpp +++ b/src/hotspot/share/oops/markWord.hpp @@ -111,7 +111,7 @@ // -------- // hash:24 ------------>| larval:1 age:4 inline_type:1 lock:2 // -// 64 bits (follows ocmpact object header): +// 64 bits (follows compact object header): // -------- // klass:22 hash:31 -->| larval:1 flat_array:1 null_free_array:1 inline_type:1 age:4 self-fwd:1 lock:2 (normal object) // From 1c9ebf79f5b523cb8c79150087aa06bc6a4a6331 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20H=C3=BCbner?= Date: Thu, 18 Sep 2025 14:21:19 +0200 Subject: [PATCH 3/8] Emulate old Valhalla decode_pointer behavior. --- src/hotspot/share/oops/markWord.hpp | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/hotspot/share/oops/markWord.hpp b/src/hotspot/share/oops/markWord.hpp index 9139e3f0102..5034a21887a 100644 --- a/src/hotspot/share/oops/markWord.hpp +++ b/src/hotspot/share/oops/markWord.hpp @@ -247,11 +247,6 @@ class markWord { static const uintptr_t null_free_flat_array_pattern = flat_array_bit_in_place | null_free_array_pattern; static const uintptr_t nullable_flat_array_pattern = flat_array_bit_in_place | unlocked_value; - // Has static klass prototype, used for decode/encode pointer - static const uintptr_t static_prototype_mask = LP64_ONLY(right_n_bits(inline_type_bits + flat_array_bits + null_free_array_bits)) NOT_LP64(right_n_bits(inline_type_bits)); - static const uintptr_t static_prototype_mask_in_place = static_prototype_mask << lock_bits; - static const uintptr_t static_prototype_value_max = (1 << age_shift) - 1; - static const uintptr_t larval_pattern = larval_bit_in_place | inline_type_pattern; static const uintptr_t no_hash = 0 ; // no hash value assigned @@ -460,10 +455,23 @@ class markWord { // Recover address of oop from encoded form used in mark inline void* decode_pointer() const { - return (EnableValhalla && _value < static_prototype_value_max) ? nullptr : + // Get the current age and 3 Valhalla bits. + // Note that the larval bit is ignored in this case. + uint n_valhalla = 3; + uintptr_t age = (_value >> age_shift) & age_mask; + uintptr_t valhalla = (_value >> inline_type_shift) & right_n_bits(n_valhalla); + // Use the legacy indices: age = 6, Valhalla = 3. + uint old_age = 6, old_valhalla = 3; + // Swap into the legacy format. + uintptr_t tmp = write_bits(_value, valhalla, old_valhalla, old_valhalla + n_valhalla); + tmp = write_bits(tmp, age, old_age, old_age + age_bits); + // No idea what this achives, but it is needed to work. + uintptr_t static_prototype_value_max = (1 << old_age) - 1; + return (EnableValhalla && tmp < static_prototype_value_max) ? nullptr : (void*) (clear_lock_bits().value()); } + inline bool is_self_forwarded() const { NOT_LP64(assert(LockingMode != LM_LEGACY, "incorrect with LM_LEGACY on 32 bit");) return mask_bits(value(), self_fwd_mask_in_place) != 0; @@ -482,6 +490,13 @@ class markWord { inline oop forwardee() const { return cast_to_oop(decode_pointer()); } + + private: + inline uintptr_t write_bits(uintptr_t input, uintptr_t val, uint start, uint end) const { + uintptr_t mask = ((1 << start) - 1) ^ ((1 << (end + 1)) - 1); + input &= ~mask; + return input | (val << start); + } }; // Support atomic operations. From 908d146363dc0c675e5061bc8d16eb497864c045 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20H=C3=BCbner?= Date: Thu, 18 Sep 2025 18:07:15 +0200 Subject: [PATCH 4/8] Chuck out weird markWord code. --- src/hotspot/share/oops/markWord.hpp | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/src/hotspot/share/oops/markWord.hpp b/src/hotspot/share/oops/markWord.hpp index 5034a21887a..3721e93ef63 100644 --- a/src/hotspot/share/oops/markWord.hpp +++ b/src/hotspot/share/oops/markWord.hpp @@ -453,25 +453,10 @@ class markWord { // Prepare address of oop for placement into mark inline static markWord encode_pointer_as_mark(void* p) { return from_pointer(p).set_marked(); } - // Recover address of oop from encoded form used in mark inline void* decode_pointer() const { - // Get the current age and 3 Valhalla bits. - // Note that the larval bit is ignored in this case. - uint n_valhalla = 3; - uintptr_t age = (_value >> age_shift) & age_mask; - uintptr_t valhalla = (_value >> inline_type_shift) & right_n_bits(n_valhalla); - // Use the legacy indices: age = 6, Valhalla = 3. - uint old_age = 6, old_valhalla = 3; - // Swap into the legacy format. - uintptr_t tmp = write_bits(_value, valhalla, old_valhalla, old_valhalla + n_valhalla); - tmp = write_bits(tmp, age, old_age, old_age + age_bits); - // No idea what this achives, but it is needed to work. - uintptr_t static_prototype_value_max = (1 << old_age) - 1; - return (EnableValhalla && tmp < static_prototype_value_max) ? nullptr : - (void*) (clear_lock_bits().value()); + return (void*) (clear_lock_bits().value()); } - inline bool is_self_forwarded() const { NOT_LP64(assert(LockingMode != LM_LEGACY, "incorrect with LM_LEGACY on 32 bit");) return mask_bits(value(), self_fwd_mask_in_place) != 0; From ec71f71b55b7cb557dda15d434aabac46e32d2be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20H=C3=BCbner?= Date: Fri, 19 Sep 2025 13:13:23 +0200 Subject: [PATCH 5/8] Fix outdated markWord test semantics. --- test/hotspot/gtest/oops/test_markWord.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/hotspot/gtest/oops/test_markWord.cpp b/test/hotspot/gtest/oops/test_markWord.cpp index 4139fa958be..1eb3ef09038 100644 --- a/test/hotspot/gtest/oops/test_markWord.cpp +++ b/test/hotspot/gtest/oops/test_markWord.cpp @@ -157,7 +157,6 @@ TEST_VM(markWord, prototype) { EXPECT_TRUE(mark.has_no_hash()); EXPECT_FALSE(mark.is_marked()); - EXPECT_TRUE(mark.decode_pointer() == nullptr); assert_copy_set_hash(mark); assert_type(mark); @@ -180,7 +179,6 @@ TEST_VM(markWord, inline_type_prototype) { EXPECT_TRUE(mark.has_no_hash()); EXPECT_FALSE(mark.is_marked()); - EXPECT_TRUE(mark.decode_pointer() == nullptr); markWord larval = mark.enter_larval_state(); EXPECT_TRUE(larval.is_larval_state()); @@ -193,7 +191,6 @@ TEST_VM(markWord, inline_type_prototype) { EXPECT_TRUE(mark.has_no_hash()); EXPECT_FALSE(mark.is_marked()); - EXPECT_TRUE(mark.decode_pointer() == nullptr); } #if _LP64 @@ -214,7 +211,6 @@ TEST_VM(markWord, null_free_flat_array_prototype) { EXPECT_TRUE(mark.has_no_hash()); EXPECT_FALSE(mark.is_marked()); - EXPECT_TRUE(mark.decode_pointer() == nullptr); assert_copy_set_hash(mark); assert_flat_array_type(mark); @@ -233,7 +229,6 @@ TEST_VM(markWord, nullable_flat_array_prototype) { EXPECT_TRUE(mark.has_no_hash()); EXPECT_FALSE(mark.is_marked()); - EXPECT_TRUE(mark.decode_pointer() == nullptr); assert_copy_set_hash(mark); assert_flat_array_type(mark); @@ -258,7 +253,6 @@ TEST_VM(markWord, null_free_array_prototype) { EXPECT_TRUE(mark.has_no_hash()); EXPECT_FALSE(mark.is_marked()); - EXPECT_TRUE(mark.decode_pointer() == nullptr); assert_copy_set_hash(mark); assert_null_free_array_type(mark); From 7deea69d869b9f2b754a2e98684e28bf15d17e70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20H=C3=BCbner?= Date: Wed, 1 Oct 2025 09:57:57 +0200 Subject: [PATCH 6/8] Fix hashCode intrinsic miscompilation. --- src/hotspot/share/opto/library_call.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp index 1fe4a2d2a41..49cc7bd6299 100644 --- a/src/hotspot/share/opto/library_call.cpp +++ b/src/hotspot/share/opto/library_call.cpp @@ -5364,8 +5364,8 @@ bool LibraryCallKit::inline_native_hashcode(bool is_virtual, bool is_static) { if (!UseObjectMonitorTable) { // Test the header to see if it is safe to read w.r.t. locking. - // This also serves as guard against inline types - Node *lock_mask = _gvn.MakeConX(markWord::inline_type_mask_in_place); + // We guard against inline types at this point, + Node *lock_mask = _gvn.MakeConX(markWord::lock_mask_in_place); Node *lmasked_header = _gvn.transform(new AndXNode(header, lock_mask)); if (LockingMode == LM_LIGHTWEIGHT) { Node *monitor_val = _gvn.MakeConX(markWord::monitor_value); From 8fe33b7986354e526ad58d5d84a3a2a29ad95611 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20H=C3=BCbner?= Date: Wed, 1 Oct 2025 13:56:01 +0200 Subject: [PATCH 7/8] Revise documentation. --- src/hotspot/share/oops/markWord.hpp | 68 +++---------------------- src/hotspot/share/opto/library_call.cpp | 5 +- 2 files changed, 11 insertions(+), 62 deletions(-) diff --git a/src/hotspot/share/oops/markWord.hpp b/src/hotspot/share/oops/markWord.hpp index 3721e93ef63..003d05aa379 100644 --- a/src/hotspot/share/oops/markWord.hpp +++ b/src/hotspot/share/oops/markWord.hpp @@ -44,11 +44,11 @@ // // 64 bits: // -------- -// unused:22 hash:31 -->| unused_gap:4 age:4 self-fwd:1 lock:2 (normal object) +// unused:22 hash:31 -->| valhalla:4 age:4 self-fwd:1 lock:2 (normal object) // // 64 bits (with compact headers): // ------------------------------- -// klass:22 hash:31 -->| unused_gap:4 age:4 self-fwd:1 lock:2 (normal object) +// klass:22 hash:31 -->| valhalla:4 age:4 self-fwd:1 lock:2 (normal object) // // - hash contains the identity hash value: largest value is // 31 bits, see os::random(). Also, 64-bit vm's require @@ -72,75 +72,23 @@ // used when inflating an existing stack-lock into an ObjectMonitor. // See below for is_being_inflated() and INFLATING(). // +// VALHALLA EXTENSIONS: // +// N.B.: 32 bit mode is not supported, this section assumes 64 bit systems. // -// Valhalla -// -// -// -// Project Valhalla has mark word encoding requirements for the following oops: -// +// Project Valhalla uses markWord bits to denote the following oops (listed least to most significant): // * inline types: have alternative bytecode behavior, e.g. can not be locked -// - "larval state": mutable state, but only during object init, observable -// by only by a single thread (generally do not mutate markWord) -// // * flat arrays: load/decode of klass layout helper is expensive for aaload -// // * "null free" arrays: load/decode of klass layout helper again for aaload +// * inline type: "larval state": mutable state, but only during object init, observable +// by only by a single thread (generally do not mutate markWord) // -// EnableValhalla -// -// Formerly known as "biased lock bit", "unused_gap" is free to use: using this -// bit to indicate inline type, combined with "unlocked" lock bits, means we -// will not interfere with lock encodings (displaced, inflating, and monitor), -// since inline types can't be locked. -// -// Further state encoding -// -// 32 bit plaforms currently have no further room for encoding. No room for -// "denormalized layout helper bits", these fast mark word tests can only be made on -// 64 bit platforms. 32-bit platforms need to load the klass->_layout_helper. This -// said, the larval state bit is still required for operation, stealing from the hash -// code is simplest mechanism. -// -// Valhalla specific encodings -// -// Revised Bit-format of an object header (most significant first, big endian layout below): -// -// 32 bits: -// -------- -// hash:24 ------------>| larval:1 age:4 inline_type:1 lock:2 -// -// 64 bits (follows compact object header): -// -------- -// klass:22 hash:31 -->| larval:1 flat_array:1 null_free_array:1 inline_type:1 age:4 self-fwd:1 lock:2 (normal object) -// -// The "fast" static type bits (flat_array, null_free_array, and inline_type) -// are placed lowest next to lock bits to more easily decode forwarding pointers. -// G1 for example, implicitly clears age bits ("G1FullGCCompactionPoint::forward()") -// using "oopDesc->forwardee()", so it necessary for "markWord::decode_pointer()" -// to return a non-nullptr for this case, but not confuse the static type bits for -// a pointer. +// Inline types cannot be locked, monitored or inflating. // // Note the position of 'self-fwd' is not by accident. When forwarding an // object to a new heap position, HeapWord alignment guarantees the lower // bits, including 'self-fwd' are 0. "is_self_forwarded()" will be correctly // set to false. Otherwise encode_pointer_as_mark() may have 'self-fwd' set. -// -// -// Static types bits are recorded in the "klass->prototype_header()", displaced -// mark should simply use the prototype header as "slow path", rather chasing -// monitor or stack lock races. -// -// Lock patterns (note inline types can't be locked/monitor/inflating)... -// -// [ptr | 000] locked ptr points to real header on stack -// [header | ?01] unlocked regular object header -// [ptr | 010] monitor inflated lock (header is wapped out) -// [ptr | ?11] marked used to mark an object -// [0 ............ | 000] inflating inflation in progress -// -// class BasicLock; class ObjectMonitor; diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp index 49cc7bd6299..5b161c0c76d 100644 --- a/src/hotspot/share/opto/library_call.cpp +++ b/src/hotspot/share/opto/library_call.cpp @@ -5310,7 +5310,7 @@ bool LibraryCallKit::inline_native_hashcode(bool is_virtual, bool is_static) { Node* obj = argument(0); // Don't intrinsify hashcode on inline types for now. - // The "is locked" runtime check below also serves as inline type check and goes to the slow path. + // The "is locked" runtime check also subsumes the inline type check (as inline types cannot be locked) and goes to the slow path. if (gvn().type(obj)->is_inlinetypeptr()) { return false; } @@ -5364,7 +5364,8 @@ bool LibraryCallKit::inline_native_hashcode(bool is_virtual, bool is_static) { if (!UseObjectMonitorTable) { // Test the header to see if it is safe to read w.r.t. locking. - // We guard against inline types at this point, + // We cannot use the inline type mask as this may check bits that are overriden + // by an object monitor's pointer when inflating locking. Node *lock_mask = _gvn.MakeConX(markWord::lock_mask_in_place); Node *lmasked_header = _gvn.transform(new AndXNode(header, lock_mask)); if (LockingMode == LM_LIGHTWEIGHT) { From 5b0b70ee4ab5101ff3e32d2148f28270d130426d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20H=C3=BCbner?= Date: Wed, 1 Oct 2025 15:30:08 +0200 Subject: [PATCH 8/8] Remove unused prototype method. --- src/hotspot/share/oops/markWord.hpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/hotspot/share/oops/markWord.hpp b/src/hotspot/share/oops/markWord.hpp index 003d05aa379..d5adec184de 100644 --- a/src/hotspot/share/oops/markWord.hpp +++ b/src/hotspot/share/oops/markWord.hpp @@ -423,13 +423,6 @@ class markWord { inline oop forwardee() const { return cast_to_oop(decode_pointer()); } - - private: - inline uintptr_t write_bits(uintptr_t input, uintptr_t val, uint start, uint end) const { - uintptr_t mask = ((1 << start) - 1) ^ ((1 << (end + 1)) - 1); - input &= ~mask; - return input | (val << start); - } }; // Support atomic operations.