Skip to content

Commit 205f21c

Browse files
bsideuprnorth
bsideup
and
rnorth
authored
Prune from JVM hook if Ryuk is disabled (#4960)
If Ryuk is disabled, singleton containers and other uncontrolled containers were remaining dangling. This change adds best-effort Ryuk-like functionality that will use a JVM hook to clean by labels. Co-authored-by: Richard North <[email protected]>
1 parent 242dfde commit 205f21c

File tree

3 files changed

+82
-24
lines changed

3 files changed

+82
-24
lines changed

‎core/src/main/java/org/testcontainers/DockerClientFactory.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,9 @@ public void close() {
225225
} else {
226226
log.debug("Ryuk is disabled");
227227
ryukContainerId = null;
228+
// best-efforts cleanup at JVM shutdown, without using the Ryuk container
229+
//noinspection deprecation
230+
ResourceReaper.instance().setHook();
228231
}
229232

230233
boolean checksEnabled = !TestcontainersConfiguration.getInstance().isDisableChecks();

‎core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,6 @@
4141

4242
import java.io.File;
4343
import java.time.Duration;
44-
import java.util.AbstractMap.SimpleEntry;
4544
import java.util.ArrayList;
4645
import java.util.Arrays;
4746
import java.util.Collections;
@@ -56,7 +55,6 @@
5655
import java.util.concurrent.TimeUnit;
5756
import java.util.concurrent.atomic.AtomicInteger;
5857
import java.util.function.Consumer;
59-
import java.util.stream.Collectors;
6058
import java.util.stream.Stream;
6159

6260
import static com.google.common.base.Preconditions.checkArgument;
@@ -310,8 +308,8 @@ private void runWithCompose(String cmd) {
310308
}
311309

312310
private void registerContainersForShutdown() {
313-
ResourceReaper.instance().registerFilterForCleanup(Arrays.asList(
314-
new SimpleEntry<>("label", "com.docker.compose.project=" + project)
311+
ResourceReaper.instance().registerLabelsFilterForCleanup(Collections.singletonMap(
312+
"com.docker.compose.project", project
315313
));
316314
}
317315

‎core/src/main/java/org/testcontainers/utility/ResourceReaper.java‎

Lines changed: 77 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,14 @@
55
import com.github.dockerjava.api.command.InspectContainerResponse;
66
import com.github.dockerjava.api.exception.NotFoundException;
77
import com.github.dockerjava.api.model.Bind;
8+
import com.github.dockerjava.api.model.Container;
89
import com.github.dockerjava.api.model.ExposedPort;
910
import com.github.dockerjava.api.model.Frame;
1011
import com.github.dockerjava.api.model.HostConfig;
1112
import com.github.dockerjava.api.model.Network;
1213
import com.github.dockerjava.api.model.PortBinding;
1314
import com.github.dockerjava.api.model.Ports;
15+
import com.github.dockerjava.api.model.PruneType;
1416
import com.github.dockerjava.api.model.Volume;
1517
import com.google.common.annotations.VisibleForTesting;
1618
import com.google.common.base.Throwables;
@@ -36,6 +38,7 @@
3638
import java.nio.charset.StandardCharsets;
3739
import java.util.AbstractMap.SimpleEntry;
3840
import java.util.ArrayList;
41+
import java.util.Arrays;
3942
import java.util.Collections;
4043
import java.util.List;
4144
import java.util.Map;
@@ -58,24 +61,28 @@ public final class ResourceReaper {
5861

5962
private static final Logger LOGGER = LoggerFactory.getLogger(ResourceReaper.class);
6063

61-
private static final List<List<Map.Entry<String, String>>> DEATH_NOTE = new ArrayList<>();
64+
private static final List<List<Map.Entry<String, String>>> DEATH_NOTE = new ArrayList<>(
65+
Arrays.asList(
66+
DockerClientFactory.DEFAULT_LABELS.entrySet().stream()
67+
.<Map.Entry<String, String>>map(it -> new SimpleEntry<>("label", it.getKey() + "=" + it.getValue()))
68+
.collect(Collectors.toList())
69+
)
70+
);
71+
6272
private static final RateLimiter RYUK_ACK_RATE_LIMITER = RateLimiterBuilder
6373
.newBuilder()
6474
.withRate(4, TimeUnit.SECONDS)
6575
.withConstantThroughput()
6676
.build();
6777

6878
private static ResourceReaper instance;
69-
private final DockerClient dockerClient;
79+
private static AtomicBoolean ryukStarted = new AtomicBoolean(false);
80+
private final DockerClient dockerClient = DockerClientFactory.lazyClient();
7081
private Map<String, String> registeredContainers = new ConcurrentHashMap<>();
7182
private Set<String> registeredNetworks = Sets.newConcurrentHashSet();
7283
private Set<String> registeredImages = Sets.newConcurrentHashSet();
7384
private AtomicBoolean hookIsSet = new AtomicBoolean(false);
7485

75-
private ResourceReaper() {
76-
dockerClient = DockerClientFactory.instance().client();
77-
}
78-
7986

8087
/**
8188
*
@@ -173,14 +180,6 @@ public InspectContainerResponse getContainerInfo() {
173180

174181
CountDownLatch ryukScheduledLatch = new CountDownLatch(1);
175182

176-
synchronized (DEATH_NOTE) {
177-
DEATH_NOTE.add(
178-
DockerClientFactory.DEFAULT_LABELS.entrySet().stream()
179-
.<Map.Entry<String, String>>map(it -> new SimpleEntry<>("label", it.getKey() + "=" + it.getValue()))
180-
.collect(Collectors.toList())
181-
);
182-
}
183-
184183
String host = containerState.getHost();
185184
Integer ryukPort = containerState.getFirstMappedPort();
186185
Thread kiraThread = new Thread(
@@ -238,6 +237,7 @@ public InspectContainerResponse getContainerInfo() {
238237
}
239238
}
240239

240+
ryukStarted.set(true);
241241
return ryukContainerId;
242242
}
243243

@@ -253,7 +253,7 @@ public synchronized static ResourceReaper instance() {
253253
* Perform a cleanup.
254254
*/
255255
public synchronized void performCleanup() {
256-
registeredContainers.forEach(this::stopContainer);
256+
registeredContainers.forEach(this::removeContainer);
257257
registeredNetworks.forEach(this::removeNetwork);
258258
registeredImages.forEach(this::removeImage);
259259
}
@@ -262,14 +262,29 @@ public synchronized void performCleanup() {
262262
* Register a filter to be cleaned up.
263263
*
264264
* @param filter the filter
265+
* @deprecated only label filter is supported by the prune API, use {@link #registerLabelsFilterForCleanup(Map)}
265266
*/
267+
@Deprecated
266268
public void registerFilterForCleanup(List<Map.Entry<String, String>> filter) {
267269
synchronized (DEATH_NOTE) {
268270
DEATH_NOTE.add(filter);
269271
DEATH_NOTE.notifyAll();
270272
}
271273
}
272274

275+
/**
276+
* Register a label to be cleaned up.
277+
*
278+
* @param labels the filter
279+
*/
280+
public void registerLabelsFilterForCleanup(Map<String, String> labels) {
281+
registerFilterForCleanup(
282+
labels.entrySet().stream()
283+
.map(it -> new SimpleEntry<>("label", it.getKey() + "=" + it.getValue()))
284+
.collect(Collectors.toList())
285+
);
286+
}
287+
273288
/**
274289
* Register a container to be cleaned up, either on explicit call to stopAndRemoveContainer, or at JVM shutdown.
275290
*
@@ -287,7 +302,7 @@ public void registerContainerForCleanup(String containerId, String imageName) {
287302
* @param containerId the ID of the container
288303
*/
289304
public void stopAndRemoveContainer(String containerId) {
290-
stopContainer(containerId, registeredContainers.get(containerId));
305+
removeContainer(containerId, registeredContainers.get(containerId));
291306

292307
registeredContainers.remove(containerId);
293308
}
@@ -299,12 +314,12 @@ public void stopAndRemoveContainer(String containerId) {
299314
* @param imageName the image name of the container (used for logging)
300315
*/
301316
public void stopAndRemoveContainer(String containerId, String imageName) {
302-
stopContainer(containerId, imageName);
317+
removeContainer(containerId, imageName);
303318

304319
registeredContainers.remove(containerId);
305320
}
306321

307-
private void stopContainer(String containerId, String imageName) {
322+
private void removeContainer(String containerId, String imageName) {
308323
boolean running;
309324
try {
310325
InspectContainerResponse containerInfo = dockerClient.inspectContainerCmd(containerId).exec();
@@ -444,10 +459,52 @@ private void removeImage(String dockerImageName) {
444459
}
445460
}
446461

447-
private void setHook() {
462+
private void prune(PruneType pruneType, List<Map.Entry<String, String>> filters) {
463+
String[] labels = filters.stream()
464+
.filter(it -> "label".equals(it.getKey()))
465+
.map(Map.Entry::getValue)
466+
.toArray(String[]::new);
467+
switch (pruneType) {
468+
// Docker only prunes stopped containers, so we have to do it manually
469+
case CONTAINERS:
470+
List<Container> containers = dockerClient.listContainersCmd()
471+
.withFilter("label", Arrays.asList(labels))
472+
.withShowAll(true)
473+
.exec();
474+
475+
containers.parallelStream().forEach(container -> {
476+
removeContainer(container.getId(), container.getImage());
477+
});
478+
break;
479+
default:
480+
dockerClient.pruneCmd(pruneType).withLabelFilter(labels).exec();
481+
break;
482+
}
483+
}
484+
485+
/**
486+
* @deprecated internal API, not intended for public usage
487+
*/
488+
@Deprecated
489+
public void setHook() {
448490
if (hookIsSet.compareAndSet(false, true)) {
449491
// If the JVM stops without containers being stopped, try and stop the container.
450-
Runtime.getRuntime().addShutdownHook(new Thread(DockerClientFactory.TESTCONTAINERS_THREAD_GROUP, this::performCleanup));
492+
Runtime.getRuntime().addShutdownHook(
493+
new Thread(DockerClientFactory.TESTCONTAINERS_THREAD_GROUP,
494+
() -> {
495+
performCleanup();
496+
497+
if (!ryukStarted.get()) {
498+
synchronized (DEATH_NOTE) {
499+
DEATH_NOTE.forEach(filters -> prune(PruneType.CONTAINERS, filters));
500+
DEATH_NOTE.forEach(filters -> prune(PruneType.NETWORKS, filters));
501+
DEATH_NOTE.forEach(filters -> prune(PruneType.VOLUMES, filters));
502+
DEATH_NOTE.forEach(filters -> prune(PruneType.IMAGES, filters));
503+
}
504+
}
505+
}
506+
)
507+
);
451508
}
452509
}
453510

0 commit comments

Comments
 (0)