8325937: runtime/handshake/HandshakeDirectTest.java causes "monitor end should be strictly below the frame pointer" assertion failure on AArch64

Reviewed-by: aph, rrich
Backport-of: 83e9e482b181e76ca9f645e8cc83cfa9337df498

Author: GoeLin

src/hotspot/share/runtime/handshake.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -245,6 +245,10 @@ class VM_HandshakeAllThreads: public VM_Operation {
       number_of_threads_issued++;
     }
 
+    // Separate the arming of the poll in add_operation() above from
+    // the read of JavaThread state in the try_process() call below.
+    OrderAccess::fence();
+
     if (number_of_threads_issued < 1) {
       log_handshake_info(start_time_ns, _op->name(), 0, 0, "no threads alive");
       return;
@@ -357,6 +361,10 @@ void Handshake::execute(HandshakeClosure* hs_cl, JavaThread* target) {
     return;
   }
 
+  // Separate the arming of the poll in add_operation() above from
+  // the read of JavaThread state in the try_process() call below.
+  OrderAccess::fence();
+
   // Keeps count on how many of own emitted handshakes
   // this thread execute.
   int emitted_handshakes_executed = 0;
@@ -481,6 +489,10 @@ bool HandshakeState::process_by_self(bool allow_suspend) {
   // It by-passes the NSV by manually doing the transition.
   NoSafepointVerifier nsv;
 
+  // Separate all the writes above for other threads reading state
+  // set by this thread in case the operation is ThreadSuspendHandshake.
+  OrderAccess::fence();
+
   while (has_operation()) {
     MutexLocker ml(&_lock, Mutex::_no_safepoint_check_flag);