[#4813] Adjust the log information about scheduled tasks and limiting to improve product experience (#4814) (#4817) (#4847)

Author: chengyouling

clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterClient.java

@@ -127,6 +127,11 @@ public QueryConfigurationsResponse queryConfigurations(QueryConfigurationsReques
         queryConfigurationsResponse.setChanged(false);
         addressManager.recordSuccessState(address);
         return queryConfigurationsResponse;
+      } else if (httpResponse.getStatusCode() == HttpStatus.SC_TOO_MANY_REQUESTS) {
+        LOGGER.warn("rate limited, keep the local dimension [{}] configs unchanged.", dimensionsInfo);
+        queryConfigurationsResponse.setChanged(false);
+        addressManager.recordSuccessState(address);
+        return queryConfigurationsResponse;
       } else if (httpResponse.getStatusCode() == HttpStatus.SC_BAD_REQUEST) {
         throw new OperationException("Bad request for query configurations.");
       } else {

clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterManager.java

@@ -89,7 +89,8 @@ public void execute() {
         }
         startTask(new BackOffSleepTask(configCenterConfiguration.getRefreshIntervalInMillis(), new PollConfigurationTask(0)));
       } catch (Exception e) {
-        LOGGER.error("get configurations from ConfigCenter failed, and will try again.", e);
+        LOGGER.warn("get configurations from ConfigCenter failed, and will try again, cause message: {}. current "
+            + "fail does not affect the obtained historical configuration.", e.getCause().getMessage());
         startTask(new BackOffSleepTask(failCount + 1, new PollConfigurationTask(failCount + 1)));
       }
     }

clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieClient.java

@@ -111,6 +111,12 @@ public ConfigurationsResponse queryConfigurations(ConfigurationsRequest request,
         addressManager.recordSuccessState(address);
         return configurationsResponse;
       }
+      if (httpResponse.getStatusCode() == HttpStatus.SC_TOO_MANY_REQUESTS) {
+        LOGGER.warn("rate limited, keep the local dimension [{}] configs unchanged.", request.getLabelsQuery());
+        configurationsResponse.setChanged(false);
+        addressManager.recordSuccessState(address);
+        return configurationsResponse;
+      }
       addressManager.recordFailState(address);
       throw new OperationException(
           "read response failed. status:" + httpResponse.getStatusCode() + "; message:" +

clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigManager.java

@@ -155,7 +155,8 @@ public void execute() {
           startTask(new BackOffSleepTask(kieConfiguration.getRefreshIntervalInMillis(), new PollConfigurationTask(0, this.configurationsRequest)));
         }
       } catch (Exception e) {
-        LOGGER.error("get configurations from KieConfigCenter failed, and will try again.", e);
+        LOGGER.warn("get configurations from KieConfigCenter failed, and will try again, cause message: {}. current "
+            + "fail does not affect the obtained historical configuration.", e.getCause().getMessage());
         startTask(
             new BackOffSleepTask(failCount + 1, new PollConfigurationTask(failCount + 1, this.configurationsRequest)));
       }

clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterClient.java

@@ -249,6 +249,11 @@ public FindMicroserviceInstancesResponse findMicroserviceInstance(String consume
         result.setModified(false);
         return result;
       }
+      if (response.getStatusCode() == HttpStatus.SC_TOO_MANY_REQUESTS) {
+        LOGGER.warn("rate limited, keep the local service {}#{} instance cache unchanged.", appId, serviceName);
+        result.setModified(false);
+        return result;
+      }
       sendUnAuthorizedEvent(response);
       throw new OperationException(
           "get service instances list fails, statusCode = " + response.getStatusCode() + "; message = " + response

clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterDiscovery.java

@@ -186,11 +186,14 @@ private List<SubscriptionKey> pullInstance(SubscriptionKey k, SubscriptionValue
         }
       }
     } catch (Exception e) {
-      LOGGER.error("find service {}#{} instance failed.", k.appId, k.serviceName, e);
       if (!(e.getCause() instanceof IOException)) {
         // for IOException, do not remove cache, or when service center
         // not available, invocation between microservices will fail.
         failedKeys.add(k);
+        LOGGER.error("find service {}#{} instance failed and remove local cache.", k.appId, k.serviceName, e);
+      } else {
+        LOGGER.warn("find service {}#{} instance failed, remaining local instances cache, cause message: {}",
+            k.appId, k.serviceName, e.getMessage());
       }
     }
     return failedKeys;

clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRegistration.java

@@ -277,7 +277,7 @@ public void execute() {
         }
 
         if (!serviceCenterClient.sendHeartBeat(microservice.getServiceId(), microserviceInstance.getInstanceId())) {
-          LOGGER.error("send heart failed, and will try again.");
+          LOGGER.warn("send heart failed, and will try again.");
           eventBus.post(new HeartBeatEvent(false, microservice, microserviceInstance));
           startTask(new BackOffSleepTask(failedCount + 1, new SendHeartBeatTask(failedCount + 1)));
         } else {
@@ -287,7 +287,12 @@ public void execute() {
               new BackOffSleepTask(Math.max(heartBeatInterval, heartBeatRequestTimeout), new SendHeartBeatTask(0)));
         }
       } catch (Exception e) {
-        LOGGER.error("send heart failed, and will try again.", e);
+        // If heartbeat failures three times, log error stack help troubleshooting. Others just log message as warn.
+        if (failedCount == 2) {
+          LOGGER.error("send heart failed, and will try again.", e);
+        } else {
+          LOGGER.warn("send heart failed, and will try again. message [{}]", e.getCause().getMessage());
+        }
         eventBus.post(new HeartBeatEvent(false, microservice, microserviceInstance));
         startTask(new BackOffSleepTask(failedCount + 1, new SendHeartBeatTask(failedCount + 1)));
       }