Skip to content

Commit 9e3e639

Browse files
jasonstackdriftx
authored andcommitted
CNDB-14722: handle OOM error for compaction task (#1856)
### What is the issue CNDB-14722: OOM error is not handled by error handler https://github.com/riptano/cndb/pull/14726/files ### What does this PR fix and why was it fixed Handle OOM error for compaction tasks
1 parent cdf443e commit 9e3e639

File tree

2 files changed

+28
-0
lines changed

2 files changed

+28
-0
lines changed

src/java/org/apache/cassandra/db/compaction/BackgroundCompactionRunner.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -487,6 +487,11 @@ else if (Throwables.isCausedBy(t, IOException.class) && t.toString().contains(NO
487487
t = t instanceof FSError ? t : new FSWriteError(t);
488488
JVMStabilityInspector.inspectThrowable(t);
489489
}
490+
else if (Throwables.isCausedBy(t, OutOfMemoryError.class))
491+
{
492+
logger.error("Encountered out of memory error on {}", cfs, t);
493+
JVMStabilityInspector.inspectThrowable(t);
494+
}
490495
else if (t instanceof CompactionInterruptedException)
491496
{
492497
logger.warn(String.format("Aborting background compaction of %s due to interruption", cfs), Throwables.unwrapped(t));

test/unit/org/apache/cassandra/db/compaction/BackgroundCompactionRunnerTest.java

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,10 @@
2828
import com.google.common.collect.ImmutableSet;
2929
import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
3030
import org.apache.cassandra.concurrent.WrappedExecutorPlus;
31+
import org.apache.cassandra.config.Config;
32+
import org.apache.cassandra.utils.JVMKiller;
33+
import org.apache.cassandra.utils.JVMStabilityInspector;
34+
import org.apache.cassandra.utils.KillerForTests;
3135
import org.apache.cassandra.utils.concurrent.Promise;
3236
import org.junit.After;
3337
import org.junit.AfterClass;
@@ -515,6 +519,25 @@ public void handleTaskFailure() throws Exception
515519
verifyCFSWasMarkedForCompaction();
516520
}
517521

522+
@Test
523+
public void handleOOMError()
524+
{
525+
JVMKiller originalKiller = JVMStabilityInspector.replaceKiller(new KillerForTests());
526+
Config.DiskFailurePolicy originalPolicy = DatabaseDescriptor.getDiskFailurePolicy();
527+
try
528+
{
529+
DatabaseDescriptor.setDiskFailurePolicy(Config.DiskFailurePolicy.die);
530+
531+
OutOfMemoryError oomError = new OutOfMemoryError("oom");
532+
assertThatThrownBy(() -> BackgroundCompactionRunner.handleCompactionError(oomError, cfs))
533+
.isInstanceOf(OutOfMemoryError.class);
534+
}
535+
finally
536+
{
537+
DatabaseDescriptor.setDiskFailurePolicy(originalPolicy);
538+
JVMStabilityInspector.replaceKiller(originalKiller);
539+
}
540+
}
518541

519542
private void verifyTaskScheduled(Executor executor)
520543
{

0 commit comments

Comments
 (0)