Add protection for dynamic config change propagate chain. (#12899)

Author: weixiang1862

docs/en/changes/changes.md

@@ -37,6 +37,7 @@
 * BanyanDB: Support `@EnableSort` on the column to enable sorting for `IndexRule` and set the default to false.
 * Support `Get Effective TTL Configurations` API.
 * Fix `ServerStatusService.statusWatchers` concurrent modification.
+* Add protection for dynamic config change propagate chain.
 
 #### UI
 

oap-server/server-configuration/configuration-api/src/main/java/org/apache/skywalking/oap/server/configuration/api/ConfigWatcherRegister.java

@@ -37,17 +37,25 @@ protected void notifySingleValue(final ConfigChangeWatcher watcher, ConfigTable.
         if (newItemValue == null) {
             if (watcher.value() != null) {
                 // Notify watcher, the new value is null with delete event type.
-                watcher.notify(
-                    new ConfigChangeWatcher.ConfigChangeEvent(null, ConfigChangeWatcher.EventType.DELETE));
+                try {
+                    watcher.notify(
+                        new ConfigChangeWatcher.ConfigChangeEvent(null, ConfigChangeWatcher.EventType.DELETE));
+                } catch (Exception e) {
+                    log.error("notify config change watcher {} failed", watcher, e);
+                }
             } else {
                 // Don't need to notify, stay in null.
             }
         } else {
             if (!newItemValue.equals(watcher.value())) {
-                watcher.notify(new ConfigChangeWatcher.ConfigChangeEvent(
-                    newItemValue,
-                    ConfigChangeWatcher.EventType.MODIFY
-                ));
+                try {
+                    watcher.notify(new ConfigChangeWatcher.ConfigChangeEvent(
+                        newItemValue,
+                        ConfigChangeWatcher.EventType.MODIFY
+                    ));
+                } catch (Exception e) {
+                    log.error("notify config change watcher {} failed", watcher, e);
+                }
             } else {
                 // Don't need to notify, stay in the same config value.
             }
@@ -99,7 +107,11 @@ protected void notifyGroupValues(final GroupConfigChangeWatcher watcher,
         });
 
         if (changedGroupItems.size() > 0) {
-            watcher.notifyGroup(changedGroupItems);
+            try {
+                watcher.notifyGroup(changedGroupItems);
+            } catch (Exception e) {
+                log.error("notify config change watcher {} failed", watcher, e);
+            }
         }
     }