[routers] distiction between fatal failure and warning or unknown on healthchecks

DaanHoogland · DaanHoogland · commit 82cc21f5bcc6 · 2025-04-14T16:26:23.000+02:00
diff --git a/api/src/main/java/com/cloud/network/RouterHealthCheckResult.java b/api/src/main/java/com/cloud/network/RouterHealthCheckResult.java
@@ -26,7 +26,7 @@ public interface RouterHealthCheckResult {
 
     String getCheckType();
 
-    boolean getCheckResult();
+    VirtualNetworkApplianceService.RouterHealthStatus getCheckResult();
 
     Date getLastUpdateTime();
 
diff --git a/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java b/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java
@@ -87,4 +87,9 @@ void startRouterForHA(VirtualMachine vm, Map<VirtualMachineProfile.Param, Object
     Pair<Boolean, String> performRouterHealthChecks(long routerId);
 
     <T extends VirtualRouter> void collectNetworkStatistics(T router, Nic nic);
+
+    enum RouterHealthStatus{
+        UNKNOWN, SUCCESS, WARNING, FAILURE;
+    }
+
 }
diff --git a/api/src/main/java/org/apache/cloudstack/api/response/RouterHealthCheckResultResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/RouterHealthCheckResultResponse.java
@@ -19,6 +19,7 @@
 
 import java.util.Date;
 
+import com.cloud.network.VirtualNetworkApplianceService;
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.BaseResponse;
 
@@ -36,7 +37,7 @@ public class RouterHealthCheckResultResponse extends BaseResponse {
 
     @SerializedName(ApiConstants.RESULT)
     @Param(description = "result of the health check")
-    private boolean result;
+    private VirtualNetworkApplianceService.RouterHealthStatus result;
 
     @SerializedName(ApiConstants.LAST_UPDATED)
     @Param(description = "the date this VPC was created")
@@ -54,7 +55,7 @@ public String getCheckType() {
         return checkType;
     }
 
-    public boolean getResult() {
+    public VirtualNetworkApplianceService.RouterHealthStatus getResult() {
         return result;
     }
 
@@ -74,7 +75,7 @@ public void setCheckType(String checkType) {
         this.checkType = checkType;
     }
 
-    public void setResult(boolean result) {
+    public void setResult(VirtualNetworkApplianceService.RouterHealthStatus result) {
         this.result = result;
     }
 
diff --git a/engine/schema/src/main/java/com/cloud/network/dao/RouterHealthCheckResultVO.java b/engine/schema/src/main/java/com/cloud/network/dao/RouterHealthCheckResultVO.java
@@ -29,6 +29,7 @@
 import javax.persistence.TemporalType;
 
 import com.cloud.network.RouterHealthCheckResult;
+import com.cloud.network.VirtualNetworkApplianceService;
 import com.cloud.utils.StringUtils;
 
 @Entity
@@ -49,7 +50,7 @@ public class RouterHealthCheckResultVO implements RouterHealthCheckResult {
     private String checkType;
 
     @Column(name = "check_result")
-    private boolean checkResult;
+    private VirtualNetworkApplianceService.RouterHealthStatus checkResult;
 
     @Temporal(TemporalType.TIMESTAMP)
     @Column(name = "last_update", updatable = true, nullable = true)
@@ -87,7 +88,7 @@ public String getCheckType() {
     }
 
     @Override
-    public boolean getCheckResult() {
+    public VirtualNetworkApplianceService.RouterHealthStatus getCheckResult() {
         return checkResult;
     }
 
@@ -105,7 +106,7 @@ public byte[] getCheckDetails() {
         return checkDetails;
     }
 
-    public void setCheckResult(boolean checkResult) {
+    public void setCheckResult(VirtualNetworkApplianceService.RouterHealthStatus checkResult) {
         this.checkResult = checkResult;
     }
 
diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41910to41920.sql b/engine/schema/src/main/resources/META-INF/db/schema-41910to41920.sql
@@ -43,3 +43,7 @@ CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Read-Only Admin - Default', 'va
 
 CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Support Admin - Default', 'setupUserTwoFactorAuthentication', 'ALLOW');
 CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Support Admin - Default', 'validateUserTwoFactorAuthenticationCode', 'ALLOW');
+
+-- add status warn and unknown to router health checks
+
+CALL `cloud`.`IDEMPOTENT_CHANGE_COLUMN`('cloud.router_health_check', 'check_result', 'check_result', 'VACHAR(16) NOT NULL COMMENT "check executions for success or (fatal) failure"')
diff --git a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java
@@ -1172,19 +1172,19 @@ private List<String> getFailingChecks(DomainRouterVO router, GetRouterMonitorRes
 
         if (answer == null) {
             s_logger.warn("Unable to fetch monitor results for router " + router);
-            resetRouterHealthChecksAndConnectivity(router.getId(), false, false, "Communication failed");
+            resetRouterHealthChecksAndConnectivity(router.getId(), RouterHealthStatus.UNKNOWN, RouterHealthStatus.UNKNOWN, "Communication failed");
             return Arrays.asList(CONNECTIVITY_TEST);
         } else if (!answer.getResult()) {
             s_logger.warn("Failed to fetch monitor results from router " + router + " with details: " + answer.getDetails());
             if (StringUtils.isNotBlank(answer.getDetails()) && answer.getDetails().equalsIgnoreCase(READONLY_FILESYSTEM_ERROR)) {
-                resetRouterHealthChecksAndConnectivity(router.getId(), true, false, "Failed to write: " + answer.getDetails());
+                resetRouterHealthChecksAndConnectivity(router.getId(), RouterHealthStatus.SUCCESS, RouterHealthStatus.FAILURE, "Failed to write: " + answer.getDetails());
                 return Arrays.asList(FILESYSTEM_WRITABLE_TEST);
             } else {
-                resetRouterHealthChecksAndConnectivity(router.getId(), false, false, "Failed to fetch results with details: " + answer.getDetails());
+                resetRouterHealthChecksAndConnectivity(router.getId(), RouterHealthStatus.FAILURE, RouterHealthStatus.UNKNOWN, "Failed to fetch results with details: " + answer.getDetails());
                 return Arrays.asList(CONNECTIVITY_TEST);
             }
         } else {
-            resetRouterHealthChecksAndConnectivity(router.getId(), true, true, "Successfully fetched data");
+            resetRouterHealthChecksAndConnectivity(router.getId(), RouterHealthStatus.SUCCESS, RouterHealthStatus.SUCCESS, "Successfully fetched data");
             updateDbHealthChecksFromRouterResponse(router.getId(), answer.getMonitoringResults());
             return answer.getFailingChecks();
         }
@@ -1322,13 +1322,13 @@ private Map<String, Map<String, RouterHealthCheckResultVO>> getHealthChecksFromD
         return healthCheckResults;
     }
 
-    private void resetRouterHealthChecksAndConnectivity(final long routerId, boolean connected, boolean writable, String message) {
+    private void resetRouterHealthChecksAndConnectivity(final long routerId, VirtualNetworkApplianceService.RouterHealthStatus connected, VirtualNetworkApplianceService.RouterHealthStatus writable, String message) {
         routerHealthCheckResultDao.expungeHealthChecks(routerId);
-        updateRouterHealthCheckResult(routerId, CONNECTIVITY_TEST, "basic", connected, connected ? "Successfully connected to router" : message);
-        updateRouterHealthCheckResult(routerId, FILESYSTEM_WRITABLE_TEST, "basic", writable, writable ? "Successfully written to file system" : message);
+        updateRouterHealthCheckResult(routerId, CONNECTIVITY_TEST, "basic", connected, connected.equals(RouterHealthStatus.SUCCESS) ? "Successfully connected to router" : message);
+        updateRouterHealthCheckResult(routerId, FILESYSTEM_WRITABLE_TEST, "basic", writable, writable.equals(RouterHealthStatus.SUCCESS) ? "Successfully written to file system" : message);
     }
 
-    private void updateRouterHealthCheckResult(final long routerId, String checkName, String checkType, boolean checkResult, String checkMessage) {
+    private void updateRouterHealthCheckResult(final long routerId, String checkName, String checkType, VirtualNetworkApplianceService.RouterHealthStatus checkResult, String checkMessage) {
         boolean newHealthCheckEntry = false;
         RouterHealthCheckResultVO connectivityVO = routerHealthCheckResultDao.getRouterHealthCheckResult(routerId, checkName, checkType);
         if (connectivityVO == null) {
@@ -1352,7 +1352,7 @@ private void updateRouterHealthCheckResult(final long routerId, String checkName
     private RouterHealthCheckResultVO parseHealthCheckVOFromJson(final long routerId,
             final String checkName, final String checkType, final Map<String, String> checkData,
             final Map<String, Map<String, RouterHealthCheckResultVO>> checksInDb) {
-        boolean success = Boolean.parseBoolean(checkData.get("success"));
+        VirtualNetworkApplianceService.RouterHealthStatus success = RouterHealthStatus.valueOf(checkData.get("success"));
         Date lastUpdate = new Date(Long.parseLong(checkData.get("lastUpdate")));
         double lastRunDuration = Double.parseDouble(checkData.get("lastRunDuration"));
         String message = checkData.get("message");

Original file line number	Diff line number	Diff line change
`@@ -87,4 +87,9 @@ void startRouterForHA(VirtualMachine vm, Map<VirtualMachineProfile.Param, Object`
`87`	`87`	`Pair<Boolean, String> performRouterHealthChecks(long routerId);`
`88`	`88`
`89`	`89`	`<T extends VirtualRouter> void collectNetworkStatistics(T router, Nic nic);`
	`90`	`+`
	`91`	`+ enum RouterHealthStatus{`
	`92`	`+ UNKNOWN, SUCCESS, WARNING, FAILURE;`
	`93`	`+ }`
	`94`	`+`
`90`	`95`	`}`
Original file line number	Diff line number	Diff line change
`@@ -19,6 +19,7 @@`
`19`	`19`
`20`	`20`	`import java.util.Date;`
`21`	`21`
	`22`	`+import com.cloud.network.VirtualNetworkApplianceService;`
`22`	`23`	`import org.apache.cloudstack.api.ApiConstants;`
`23`	`24`	`import org.apache.cloudstack.api.BaseResponse;`
`24`	`25`
`@@ -36,7 +37,7 @@ public class RouterHealthCheckResultResponse extends BaseResponse {`
`36`	`37`
`37`	`38`	`@SerializedName(ApiConstants.RESULT)`
`38`	`39`	`@Param(description = "result of the health check")`
`39`		`- private boolean result;`
	`40`	`+ private VirtualNetworkApplianceService.RouterHealthStatus result;`
`40`	`41`
`41`	`42`	`@SerializedName(ApiConstants.LAST_UPDATED)`
`42`	`43`	`@Param(description = "the date this VPC was created")`
`@@ -54,7 +55,7 @@ public String getCheckType() {`
`54`	`55`	`return checkType;`
`55`	`56`	`}`
`56`	`57`
`57`		`- public boolean getResult() {`
	`58`	`+ public VirtualNetworkApplianceService.RouterHealthStatus getResult() {`
`58`	`59`	`return result;`
`59`	`60`	`}`
`60`	`61`
`@@ -74,7 +75,7 @@ public void setCheckType(String checkType) {`
`74`	`75`	`this.checkType = checkType;`
`75`	`76`	`}`
`76`	`77`
`77`		`- public void setResult(boolean result) {`
	`78`	`+ public void setResult(VirtualNetworkApplianceService.RouterHealthStatus result) {`
`78`	`79`	`this.result = result;`
`79`	`80`	`}`
`80`	`81`
Original file line number	Diff line number	Diff line change
`@@ -29,6 +29,7 @@`
`29`	`29`	`import javax.persistence.TemporalType;`
`30`	`30`
`31`	`31`	`import com.cloud.network.RouterHealthCheckResult;`
	`32`	`+import com.cloud.network.VirtualNetworkApplianceService;`
`32`	`33`	`import com.cloud.utils.StringUtils;`
`33`	`34`
`34`	`35`	`@Entity`
`@@ -49,7 +50,7 @@ public class RouterHealthCheckResultVO implements RouterHealthCheckResult {`
`49`	`50`	`private String checkType;`
`50`	`51`
`51`	`52`	`@Column(name = "check_result")`
`52`		`- private boolean checkResult;`
	`53`	`+ private VirtualNetworkApplianceService.RouterHealthStatus checkResult;`
`53`	`54`
`54`	`55`	`@Temporal(TemporalType.TIMESTAMP)`
`55`	`56`	`@Column(name = "last_update", updatable = true, nullable = true)`
`@@ -87,7 +88,7 @@ public String getCheckType() {`
`87`	`88`	`}`
`88`	`89`
`89`	`90`	`@Override`
`90`		`- public boolean getCheckResult() {`
	`91`	`+ public VirtualNetworkApplianceService.RouterHealthStatus getCheckResult() {`
`91`	`92`	`return checkResult;`
`92`	`93`	`}`
`93`	`94`
`@@ -105,7 +106,7 @@ public byte[] getCheckDetails() {`
`105`	`106`	`return checkDetails;`
`106`	`107`	`}`
`107`	`108`
`108`		`- public void setCheckResult(boolean checkResult) {`
	`109`	`+ public void setCheckResult(VirtualNetworkApplianceService.RouterHealthStatus checkResult) {`
`109`	`110`	`this.checkResult = checkResult;`
`110`	`111`	`}`
`111`	`112`