Shutdown expunged resources cleanup executor properly, and allow other components to configure/start/stop on error (#9723)

Author: sureshanaparti

framework/cluster/src/main/java/com/cloud/cluster/ClusterManagerImpl.java

@@ -294,7 +294,7 @@ private void onSendingClusterPdu() {
                     }
                 }
             } catch (final Throwable e) {
-                logger.error("Unexcpeted exception: ", e);
+                logger.error("Unexpected exception: ", e);
             }
         }
     }
@@ -346,7 +346,7 @@ protected void runInContext() {
                     }
                 });
             } catch (final Throwable e) {
-                logger.error("Unexcpeted exception: ", e);
+                logger.error("Unexpected exception: ", e);
             }
         }
     }
@@ -384,7 +384,7 @@ public void broadcast(final long agentId, final String cmds) {
                 }
                 executeAsync(peerName, agentId, cmds, true);
             } catch (final Exception e) {
-                logger.warn("Caught exception while talkign to " + peer.getMsid());
+                logger.warn("Caught exception while talking to " + peer.getMsid());
             }
         }
     }

framework/spring/lifecycle/src/main/java/org/apache/cloudstack/spring/lifecycle/CloudStackExtendedLifeCycle.java

@@ -69,7 +69,12 @@ public void startBeans() {
         with(new WithComponentLifeCycle() {
             @Override
             public void with(ComponentLifecycle lifecycle) {
-                lifecycle.start();
+                logger.info("starting bean {}.", lifecycle.getName());
+                try {
+                    lifecycle.start();
+                } catch (Exception e) {
+                    logger.error("Error on starting bean {} - {}", lifecycle.getName(), e.getMessage(), e);
+                }
 
                 if (lifecycle instanceof ManagementBean) {
                     ManagementBean mbean = (ManagementBean)lifecycle;
@@ -93,13 +98,21 @@ public void with(ComponentLifecycle lifecycle) {
     }
 
     public void stopBeans() {
+        logger.info("Stopping CloudStack Components");
+
         with(new WithComponentLifeCycle() {
             @Override
             public void with(ComponentLifecycle lifecycle) {
-                logger.info("stopping bean " + lifecycle.getName());
-                lifecycle.stop();
+                logger.info("stopping bean {}.", lifecycle.getName());
+                try {
+                    lifecycle.stop();
+                } catch (Exception e) {
+                    logger.error("Error on stopping bean {} - {}", lifecycle.getName(), e.getMessage(), e);
+                }
             }
         });
+
+        logger.info("Done Stopping CloudStack Components");
     }
 
     private void configure() {
@@ -109,10 +122,13 @@ private void configure() {
             @Override
             public void with(ComponentLifecycle lifecycle) {
                 try {
+                    logger.info("configuring bean {}.", lifecycle.getName());
                     lifecycle.configure(lifecycle.getName(), lifecycle.getConfigParams());
                 } catch (ConfigurationException e) {
                     logger.error("Failed to configure " +  lifecycle.getName(), e);
                     throw new CloudRuntimeException(e);
+                } catch (Exception e) {
+                    logger.error("Error on configuring bean {} - {}", lifecycle.getName(), e.getMessage(), e);
                 }
             }
         });

framework/spring/lifecycle/src/main/java/org/apache/cloudstack/spring/lifecycle/CloudStackExtendedLifeCycleStart.java

@@ -45,5 +45,4 @@ public void setLifeCycle(CloudStackExtendedLifeCycle lifeCycle) {
     public void run() {
         lifeCycle.startBeans();
     }
-
 }

framework/spring/module/src/main/java/org/apache/cloudstack/spring/module/factory/CloudStackSpringContext.java

@@ -77,8 +77,10 @@ public void registerShutdownHook() {
         for (String appName : contextMap.keySet()) {
             ApplicationContext contex = contextMap.get(appName);
             if (contex instanceof ConfigurableApplicationContext) {
-                logger.trace("registering shutdown hook for bean "+ appName);
+                logger.trace("Registering shutdown hook for bean {}.", appName);
                 ((ConfigurableApplicationContext)contex).registerShutdownHook();
+            } else {
+                logger.warn("Shutdown hook not registered for bean {}.", appName);
             }
         }
     }

server/src/main/java/org/apache/cloudstack/resource/ResourceCleanupServiceImpl.java

@@ -579,7 +579,9 @@ public boolean start() {
     @Override
     public boolean stop() {
         purgeExpungedResourcesJobExecutor.shutdown();
-        expungedResourcesCleanupExecutor.shutdownNow();
+        if (expungedResourcesCleanupExecutor != null) {
+            expungedResourcesCleanupExecutor.shutdownNow();
+        }
         return true;
     }