Add more clear logs when oap-cluster-internal data format is inconcsistent (#13059)

Author: wu-sheng

docs/en/changes/changes.md

@@ -73,6 +73,7 @@
 * Add `Get Alarm Runtime Status` API.
 * Add `lock` when query the Alarm metrics window values.
 * Add a fail-safe mechanism to prevent traffic metrics inconsistent between in-memory and database server.
+* Add more clear logs when oap-cluster-internal data(metrics/traffic) format is inconsistent.
 
 #### UI
 

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/RemoteServiceHandler.java

@@ -117,24 +117,33 @@ public void onNext(RemoteMessage message) {
                     String nextWorkerName = message.getNextWorkerName();
                     RemoteData remoteData = message.getRemoteData();
 
-                    try {
-                        RemoteHandleWorker handleWorker = workerInstanceGetter.get(nextWorkerName);
-                        if (handleWorker != null) {
-                            AbstractWorker nextWorker = handleWorker.getWorker();
-                            StreamData streamData = handleWorker.getStreamDataClass().newInstance();
+                    RemoteHandleWorker handleWorker = workerInstanceGetter.get(nextWorkerName);
+                    if (handleWorker != null) {
+                        AbstractWorker nextWorker = handleWorker.getWorker();
+                        StreamData streamData;
+                        try {
+                            streamData = handleWorker.getStreamDataClass().newInstance();
+                        } catch (Throwable t) {
+                            remoteInErrorCounter.inc();
+                            LOGGER.error(t.getMessage(), t);
+                            return;
+                        }
+                        try {
                             streamData.deserialize(remoteData);
-                            nextWorker.in(streamData);
-                        } else {
-                            remoteInTargetNotFoundCounter.inc();
-                            LOGGER.warn(
-                                "Work name [{}] not found. Check OAL script, make sure they are same in the whole cluster.",
-                                nextWorkerName
-                            );
+                        } catch (Throwable t) {
+                            remoteInErrorCounter.inc();
+                            LOGGER.error("Can't deserialize data {}, this data is discarded.", message, t);
+                            return;
                         }
-                    } catch (Throwable t) {
-                        remoteInErrorCounter.inc();
-                        LOGGER.error(t.getMessage(), t);
+                        nextWorker.in(streamData);
+                    } else {
+                        remoteInTargetNotFoundCounter.inc();
+                        LOGGER.warn(
+                            "Data is discarded due to worker not found. Check OAL/MAL script, make sure they are aligned in the whole cluster. The data is {}",
+                            message
+                        );
                     }
+
                 } finally {
                     timer.finish();
                 }