Update Hazelcast health indicator to verify that Hazelcast is running

This is needed because when the default Out-Of-Memory handler in Hazelcast detects a OOM error, and terminates
Hazelcast, it does so on the _node_ but not via the Hazelcast lifecycle manager - and the results is that it is
still possible to successfully execute transactions in Hazelcast after this termination. The health indicator
should not report UP after Hazelcast has detected an OOM and terminated itself.

Signed-off-by: Tommy Karlsson <[email protected]>

Author: tommyk-gears

module/spring-boot-hazelcast/src/main/java/org/springframework/boot/hazelcast/health/HazelcastHealthIndicator.java

@@ -28,6 +28,7 @@
  *
  * @author Dmytro Nosan
  * @author Stephane Nicoll
+ * @author Tommy Karlsson
  * @since 4.0.0
  */
 public class HazelcastHealthIndicator extends AbstractHealthIndicator {
@@ -42,6 +43,10 @@ public HazelcastHealthIndicator(HazelcastInstance hazelcast) {
 
 	@Override
 	protected void doHealthCheck(Health.Builder builder) {
+		if (!this.hazelcast.getLifecycleService().isRunning()) {
+			builder.down();
+			return;
+		}
 		this.hazelcast.executeTransaction((context) -> {
 			String uuid = this.hazelcast.getLocalEndpoint().getUuid().toString();
 			builder.up().withDetail("name", this.hazelcast.getName()).withDetail("uuid", uuid);

module/spring-boot-hazelcast/src/test/java/org/springframework/boot/hazelcast/health/HazelcastHealthIndicatorTests.java

@@ -18,6 +18,8 @@
 
 import com.hazelcast.core.HazelcastException;
 import com.hazelcast.core.HazelcastInstance;
+import com.hazelcast.instance.impl.HazelcastInstanceProxy;
+import com.hazelcast.instance.impl.OutOfMemoryHandlerHelper;
 import org.junit.jupiter.api.Test;
 
 import org.springframework.boot.autoconfigure.AutoConfigurations;
@@ -37,25 +39,26 @@
  *
  * @author Dmytro Nosan
  * @author Stephane Nicoll
+ * @author Tommy Karlsson
  */
+@WithResource(name = "hazelcast.xml", content = """
+		<hazelcast xmlns="http://www.hazelcast.com/schema/config"
+				   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+				   xsi:schemaLocation="http://www.hazelcast.com/schema/config
+		           http://www.hazelcast.com/schema/config/hazelcast-config-5.0.xsd">
+			<instance-name>actuator-hazelcast</instance-name>
+			<map name="defaultCache" />
+			<network>
+				<join>
+					<auto-detection enabled="false"/>
+					<multicast enabled="false"/>
+				</join>
+			</network>
+		</hazelcast>
+		""")
 class HazelcastHealthIndicatorTests {
 
 	@Test
-	@WithResource(name = "hazelcast.xml", content = """
-			<hazelcast xmlns="http://www.hazelcast.com/schema/config"
-					   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-					   xsi:schemaLocation="http://www.hazelcast.com/schema/config
-			           http://www.hazelcast.com/schema/config/hazelcast-config-5.0.xsd">
-				<instance-name>actuator-hazelcast</instance-name>
-				<map name="defaultCache" />
-				<network>
-					<join>
-						<auto-detection enabled="false"/>
-						<multicast enabled="false"/>
-					</join>
-				</network>
-			</hazelcast>
-			""")
 	void hazelcastUp() {
 		new ApplicationContextRunner().withConfiguration(AutoConfigurations.of(HazelcastAutoConfiguration.class))
 			.withPropertyValues("spring.hazelcast.config=hazelcast.xml")
@@ -69,6 +72,32 @@ void hazelcastUp() {
 			});
 	}
 
+	@Test
+	void hazelcastShutdown() {
+		new ApplicationContextRunner().withConfiguration(AutoConfigurations.of(HazelcastAutoConfiguration.class))
+			.withPropertyValues("spring.hazelcast.config=hazelcast.xml")
+			.run((context) -> {
+				HazelcastInstance hazelcast = context.getBean(HazelcastInstance.class);
+				hazelcast.shutdown();
+				Health health = new HazelcastHealthIndicator(hazelcast).health();
+				assertThat(health.getStatus()).isEqualTo(Status.DOWN);
+			});
+	}
+
+	@Test
+	void hazelcastOOMShutdown() {
+		new ApplicationContextRunner().withConfiguration(AutoConfigurations.of(HazelcastAutoConfiguration.class))
+			.withPropertyValues("spring.hazelcast.config=hazelcast.xml")
+			.run((context) -> {
+				HazelcastInstance hazelcast = context.getBean(HazelcastInstance.class);
+				HazelcastInstance original = ((HazelcastInstanceProxy) hazelcast).getOriginal();
+				OutOfMemoryHandlerHelper.tryCloseConnections(original);
+				OutOfMemoryHandlerHelper.tryShutdown(original);
+				Health health = new HazelcastHealthIndicator(hazelcast).health();
+				assertThat(health.getStatus()).isEqualTo(Status.DOWN);
+			});
+	}
+
 	@Test
 	void hazelcastDown() {
 		HazelcastInstance hazelcast = mock(HazelcastInstance.class);