8352299: GenShen: Young cycles that interrupt old cycles cannot be cancelled

Reviewed-by: shade

Author: William Kemper

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -2171,8 +2171,8 @@ size_t ShenandoahHeap::tlab_used(Thread* thread) const {
 }
 
 bool ShenandoahHeap::try_cancel_gc(GCCause::Cause cause) {
-  jbyte prev = _cancelled_gc.cmpxchg(cause, GCCause::_no_gc);
-  return prev == GCCause::_no_gc;
+  const GCCause::Cause prev = _cancelled_gc.xchg(cause);
+  return prev == GCCause::_no_gc || prev == GCCause::_shenandoah_concurrent_gc;
 }
 
 void ShenandoahHeap::cancel_concurrent_mark() {

src/hotspot/share/gc/shenandoah/shenandoahSharedVariables.hpp

@@ -212,8 +212,9 @@ typedef struct ShenandoahSharedBitmap {
 
 template<class T>
 struct ShenandoahSharedEnumFlag {
+  typedef uint32_t EnumValueType;
   shenandoah_padding(0);
-  volatile ShenandoahSharedValue value;
+  volatile EnumValueType value;
   shenandoah_padding(1);
 
   ShenandoahSharedEnumFlag() {
@@ -222,8 +223,8 @@ struct ShenandoahSharedEnumFlag {
 
   void set(T v) {
     assert (v >= 0, "sanity");
-    assert (v < (sizeof(ShenandoahSharedValue) * CHAR_MAX), "sanity");
-    Atomic::release_store_fence(&value, (ShenandoahSharedValue)v);
+    assert (v < (sizeof(EnumValueType) * CHAR_MAX), "sanity");
+    Atomic::release_store_fence(&value, (EnumValueType)v);
   }
 
   T get() const {
@@ -232,11 +233,17 @@ struct ShenandoahSharedEnumFlag {
 
   T cmpxchg(T new_value, T expected) {
     assert (new_value >= 0, "sanity");
-    assert (new_value < (sizeof(ShenandoahSharedValue) * CHAR_MAX), "sanity");
-    return (T)Atomic::cmpxchg(&value, (ShenandoahSharedValue)expected, (ShenandoahSharedValue)new_value);
+    assert (new_value < (sizeof(EnumValueType) * CHAR_MAX), "sanity");
+    return (T)Atomic::cmpxchg(&value, (EnumValueType)expected, (EnumValueType)new_value);
   }
 
-  volatile ShenandoahSharedValue* addr_of() {
+  T xchg(T new_value) {
+    assert (new_value >= 0, "sanity");
+    assert (new_value < (sizeof(EnumValueType) * CHAR_MAX), "sanity");
+    return (T)Atomic::xchg(&value, (EnumValueType)new_value);
+  }
+
+  volatile EnumValueType* addr_of() {
     return &value;
   }