fix: Address a number of liveness-related issues (#6366)

* Clarify liveness requirements for interacting with TableLocationProvider
* Address liveness guarantees for LiveSuppliers in AbstractTableLocationProvider
* Address transient liveness errors in static SourcePartitionedTables for the UnderlyingTableMaintainer the underlying table itself
* Ensure that WeakCleanupReferences returned by the underlying LivenessReferent of a DelegatingLivenessReferent are held by the delegating implementation's getWeakReference() result
* Clean up a few instances where we invoke super.destroy() in a different order than the usual from ReferenceCountedLivenessNode subclasses
* Annotate ReferenceCountedLivenessNode.destroy() with @OverridingMethodsMustInvokeSuper, and ensure that this is followed in all but one documented exceptional case
* RetainedReferenceTracker should only do immediate CleanupReference.cleanup() for its own (idempotent) implementation
* Rename RetainedReferenceTracker.DropState to be less ambiguous
* Address reachability during deferred drop for LivenessReferents retained by a RetainedReferenceTracker with enforceStrongReachability=true
* Remove a few redundant LivenessScopes in QueryTableAjTest
* Preserve reachability at time of enqueue when not done via cleanup. Use enforceStrongReachability more places in the testing framework and QueryTableAjTest to avoid future issues.
* Separate enqueueing references to be dropped (now done before destroy) from dropping enqueued references
* Make LivenessScopeStack play better with other LivenessManager implementations, not just LivenessScope

Author: rcaudy

engine/table/src/main/java/io/deephaven/engine/table/impl/AddOnlyToBlinkTableAdapter.java

@@ -14,6 +14,7 @@
 import org.apache.commons.lang3.mutable.MutableObject;
 import org.jetbrains.annotations.NotNull;
 
+import javax.annotation.OverridingMethodsMustInvokeSuper;
 import java.util.List;
 import java.util.Map;
 
@@ -121,10 +122,11 @@ public void run() {
             }
         }
 
+        @OverridingMethodsMustInvokeSuper
         @Override
         protected void destroy() {
-            getUpdateGraph().removeSource(this);
             super.destroy();
+            getUpdateGraph().removeSource(this);
         }
     }
 }

engine/table/src/main/java/io/deephaven/engine/table/impl/AsOfJoinHelper.java

@@ -38,6 +38,7 @@
 import io.deephaven.util.SafeCloseableList;
 import org.jetbrains.annotations.NotNull;
 
+import javax.annotation.OverridingMethodsMustInvokeSuper;
 import java.util.*;
 import java.util.function.Function;
 import java.util.function.Supplier;
@@ -1335,6 +1336,7 @@ public void onUpdate(TableUpdate upstream) {
                         result.notifyListeners(downstream);
                     }
 
+                    @OverridingMethodsMustInvokeSuper
                     @Override
                     protected void destroy() {
                         super.destroy();
@@ -1516,6 +1518,7 @@ public void onUpdate(TableUpdate upstream) {
                                     }
                                 }
 
+                                @OverridingMethodsMustInvokeSuper
                                 @Override
                                 protected void destroy() {
                                     super.destroy();

engine/table/src/main/java/io/deephaven/engine/table/impl/BaseTable.java

@@ -44,6 +44,7 @@
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
+import javax.annotation.OverridingMethodsMustInvokeSuper;
 import java.io.*;
 import java.util.*;
 import java.util.concurrent.TimeUnit;
@@ -923,6 +924,7 @@ public boolean canExecute(final long step) {
             return parent.satisfied(step);
         }
 
+        @OverridingMethodsMustInvokeSuper
         @Override
         protected void destroy() {
             super.destroy();
@@ -980,6 +982,7 @@ public boolean canExecute(final long step) {
             return parent.satisfied(step);
         }
 
+        @OverridingMethodsMustInvokeSuper
         @Override
         protected void destroy() {
             super.destroy();
@@ -1321,6 +1324,7 @@ public <T extends OperationSnapshotControl> T createSnapshotControlIfRefreshing(
     // Reference Counting
     // ------------------------------------------------------------------------------------------------------------------
 
+    @OverridingMethodsMustInvokeSuper
     @Override
     protected void destroy() {
         super.destroy();

engine/table/src/main/java/io/deephaven/engine/table/impl/InstrumentedTableUpdateListenerAdapter.java

@@ -17,6 +17,7 @@
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
+import javax.annotation.OverridingMethodsMustInvokeSuper;
 import java.io.IOException;
 
 /**
@@ -92,8 +93,10 @@ public boolean canExecute(final long step) {
         return source.satisfied(step);
     }
 
+    @OverridingMethodsMustInvokeSuper
     @Override
     protected void destroy() {
+        super.destroy();
         source.removeUpdateListener(this);
         if (retain) {
             RETENTION_CACHE.forget(this);

engine/table/src/main/java/io/deephaven/engine/table/impl/ListenerRecorder.java

@@ -14,6 +14,8 @@
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
+import javax.annotation.OverridingMethodsMustInvokeSuper;
+
 /**
  * A listener recorder stores references to added, removed, modified, and shifted indices; and then notifies a
  * {@link MergedListener} that a change has occurred. The combination of a {@link ListenerRecorder} and
@@ -90,6 +92,7 @@ public boolean canExecute(final long step) {
         return parent.satisfied(step);
     }
 
+    @OverridingMethodsMustInvokeSuper
     @Override
     protected void destroy() {
         super.destroy();

engine/table/src/main/java/io/deephaven/engine/table/impl/MergedListener.java

@@ -24,6 +24,7 @@
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
+import javax.annotation.OverridingMethodsMustInvokeSuper;
 import java.io.IOException;
 import java.lang.ref.WeakReference;
 import java.util.Collection;
@@ -174,11 +175,13 @@ protected void propagateError(
     }
 
     protected boolean systemicResult() {
-        return result == null ? false : SystemicObjectTracker.isSystemic(result);
+        return result != null && SystemicObjectTracker.isSystemic(result);
     }
 
+    @OverridingMethodsMustInvokeSuper
     @Override
     protected void destroy() {
+        super.destroy();
         recorders.forEach(ListenerRecorder::forceReferenceCountToZero);
     }
 

engine/table/src/main/java/io/deephaven/engine/table/impl/ShiftObliviousInstrumentedListenerAdapter.java

@@ -16,6 +16,7 @@
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
+import javax.annotation.OverridingMethodsMustInvokeSuper;
 import java.io.IOException;
 
 /**
@@ -90,8 +91,10 @@ public boolean canExecute(final long step) {
         return source.satisfied(step);
     }
 
+    @OverridingMethodsMustInvokeSuper
     @Override
     protected void destroy() {
+        super.destroy();
         source.removeUpdateListener(this);
         if (retain) {
             RETENTION_CACHE.forget(this);

engine/table/src/main/java/io/deephaven/engine/table/impl/SourcePartitionedTable.java

@@ -174,11 +174,20 @@ protected void instrumentedRefresh() {
                 tableLocationProvider.refresh();
 
                 final Collection<TableLocation> locations = new ArrayList<>();
-                tableLocationProvider.getTableLocationKeys(
-                        tlk -> locations.add(tableLocationProvider.getTableLocation(tlk.get())),
-                        locationKeyMatcher);
-                try (final RowSet added = sortAndAddLocations(locations.stream())) {
-                    resultRows.insert(added);
+                try {
+                    retainReference();
+                    tableLocationProvider.getTableLocationKeys(
+                            (final LiveSupplier<ImmutableTableLocationKey> lstlk) -> {
+                                final TableLocation tableLocation = tableLocationProvider.getTableLocation(lstlk.get());
+                                manage(tableLocation);
+                                locations.add(tableLocation);
+                            },
+                            locationKeyMatcher);
+                    try (final RowSet added = sortAndAddLocations(locations.stream())) {
+                        resultRows.insert(added);
+                    }
+                } finally {
+                    dropReference();
                 }
             }
 
@@ -204,7 +213,9 @@ private RowSet sortAndAddLocations(@NotNull final Stream<TableLocation> location
                 resultLocationTables.ensureCapacity(constituentRowKey + 1);
                 resultLocationTables.set(constituentRowKey, constituentTable);
 
-                result.manage(constituentTable);
+                if (result.isRefreshing()) {
+                    result.manage(constituentTable);
+                }
             });
             return initialLastRowKey == lastInsertedRowKey.get()
                     ? RowSetFactory.empty()

engine/table/src/main/java/io/deephaven/engine/table/impl/SourceTable.java

@@ -25,6 +25,7 @@
 import org.apache.commons.lang3.mutable.MutableObject;
 import org.jetbrains.annotations.NotNull;
 
+import javax.annotation.OverridingMethodsMustInvokeSuper;
 import java.util.ArrayList;
 import java.util.Collection;
 
@@ -310,9 +311,11 @@ public boolean subscribeForUpdates(@NotNull final TableUpdateListener listener)
             }
 
             if (snapshotControl != null) {
+                // noinspection MethodDoesntCallSuperMethod
                 final ListenerImpl listener =
                         new ListenerImpl("SourceTable.coalesce", this, resultTable) {
 
+                            @OverridingMethodsMustInvokeSuper
                             @Override
                             protected void destroy() {
                                 // This impl cannot call super.destroy() because we must unsubscribe from the actual
@@ -330,6 +333,7 @@ protected void destroy() {
         return result.getValue();
     }
 
+    @OverridingMethodsMustInvokeSuper
     @Override
     protected void destroy() {
         super.destroy();

engine/table/src/main/java/io/deephaven/engine/table/impl/TimeTable.java

@@ -29,6 +29,7 @@
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
+import javax.annotation.OverridingMethodsMustInvokeSuper;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.Collections;
@@ -215,6 +216,7 @@ private void refresh(final boolean notifyListeners) {
         }
     }
 
+    @OverridingMethodsMustInvokeSuper
     @Override
     protected void destroy() {
         super.destroy();