Adapt new Browser data upload protocol (#13009)

mrproliu · web-flow · commit 8ebb52dd190d · 2025-01-24T16:41:30.000+10:00
diff --git a/apm-protocol/apm-network/src/main/proto b/apm-protocol/apm-network/src/main/proto
@@ -1 +1 @@
-Subproject commit 385fb8b0057b98c9350f7a94d228b977f5154d4b
+Subproject commit 1717f5c753a4d6c644f056d69fed85873482648f
diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
@@ -58,7 +58,7 @@
 * MQE: Support `top_n_of` function for merging multiple metrics topn query.
 * Support `labelAvg` function in the OAL engine.
 * Added `maxLabelCount` parameter in the `labelCount` function of OAL to limit the number of labels can be counted.
-* Adapt the new Browser API(`/browser/perfData/webVitals`, `/browser/perfData/resources`) protocol.
+* Adapt the new Browser API(`/browser/perfData/webVitals`, `/browser/perfData/webInteractions`, `/browser/perfData/resources`) protocol.
 * Add Circuit Breaking mechanism.
 * BanyanDB: Add support for compatibility checks based on the BanyanDB server's API version.
 * MQE: Support `&&(and)`, `||(or)` bool operators.
diff --git a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4
@@ -75,6 +75,7 @@ SRC_BROWSER_APP_PAGE_TRAFFIC: 'BrowserAppPageTraffic';
 SRC_BROWSER_APP_SINGLE_VERSION_TRAFFIC: 'BrowserAppSingleVersionTraffic';
 SRC_BROWSER_APP_RESOURCE_PERF: 'BrowserAppResourcePerf';
 SRC_BROWSER_APP_WEB_VITALS_PERF: 'BrowserAppWebVitalsPerf';
+SRC_BROWSER_APP_WEB_INTERACTION_PERF: 'BrowserAppWebInteractionPerf';
 
 // Constructors symbols
 
diff --git a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4
@@ -59,7 +59,7 @@ source
       SRC_SERVICE_INSTANCE_CLR_CPU | SRC_SERVICE_INSTANCE_CLR_GC | SRC_SERVICE_INSTANCE_CLR_THREAD |
       SRC_SERVICE_INSTANCE_JVM_CPU | SRC_SERVICE_INSTANCE_JVM_MEMORY | SRC_SERVICE_INSTANCE_JVM_MEMORY_POOL | SRC_SERVICE_INSTANCE_JVM_GC | SRC_SERVICE_INSTANCE_JVM_THREAD | SRC_SERVICE_INSTANCE_JVM_CLASS |// JVM source of service instance
       SRC_ENVOY_INSTANCE_METRIC |
-      SRC_BROWSER_APP_PERF | SRC_BROWSER_APP_PAGE_PERF | SRC_BROWSER_APP_SINGLE_VERSION_PERF | SRC_BROWSER_APP_RESOURCE_PERF | SRC_BROWSER_APP_WEB_VITALS_PERF |
+      SRC_BROWSER_APP_PERF | SRC_BROWSER_APP_PAGE_PERF | SRC_BROWSER_APP_SINGLE_VERSION_PERF | SRC_BROWSER_APP_RESOURCE_PERF | SRC_BROWSER_APP_WEB_VITALS_PERF | SRC_BROWSER_APP_WEB_INTERACTION_PERF |
       SRC_BROWSER_APP_TRAFFIC | SRC_BROWSER_APP_PAGE_TRAFFIC | SRC_BROWSER_APP_SINGLE_VERSION_TRAFFIC |
       SRC_EVENT | SRC_MQ_ACCESS | SRC_MQ_ENDPOINT_ACCESS |
       SRC_K8S_SERVICE | SRC_K8S_SERVICE_INSTANCE | SRC_K8S_ENDPOINT | SRC_K8S_SERVICE_RELATION | SRC_K8S_SERVICE_INSTANCE_RELATION |
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppWebInteractionPerf.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppWebInteractionPerf.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.browser.source;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.skywalking.oap.server.core.analysis.IDManager;
+import org.apache.skywalking.oap.server.core.source.ScopeDeclaration;
+import org.apache.skywalking.oap.server.core.source.ScopeDefaultColumn;
+import org.apache.skywalking.oap.server.core.source.Source;
+
+import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.BROWSER_APP_WEB_INTERACTION_PAGE_PERF;
+import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.ENDPOINT_CATALOG_NAME;
+
+@ScopeDeclaration(id = BROWSER_APP_WEB_INTERACTION_PAGE_PERF, name = "BrowserAppWebInteractionPerf", catalog = ENDPOINT_CATALOG_NAME)
+@ScopeDefaultColumn.VirtualColumnDefinition(fieldName = "entityId", columnName = "entity_id", isID = true, type = String.class)
+@Setter
+@Getter
+public class BrowserAppWebInteractionPerf extends Source {
+    @Override
+    public int scope() {
+        return BROWSER_APP_WEB_INTERACTION_PAGE_PERF;
+    }
+
+    @Override
+    public String getEntityId() {
+        return IDManager.EndpointID.buildId(serviceId, path);
+    }
+
+    @ScopeDefaultColumn.DefinedByField(columnName = "service_id")
+    @ScopeDefaultColumn.BanyanDB(groupByCondInTopN = true)
+    private String serviceId;
+    @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true)
+    private String serviceName;
+    private String path;
+    private int inpTime;
+
+    @Override
+    public void prepare() {
+        serviceId = IDManager.ServiceID.buildId(serviceName, true);
+    }
+}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java
@@ -149,6 +149,7 @@ public class DefaultScopeDefine {
 
     public static final int BROWSER_APP_WEB_VITALS_PAGE_PERF = 87;
     public static final int BROWSER_APP_RESOURCE_PERF = 88;
+    public static final int BROWSER_APP_WEB_INTERACTION_PAGE_PERF = 89;
 
     /**
      * Catalog of scope, the metrics processor could use this to group all generated metrics by oal rt.
diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/BrowserModuleProvider.java b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/BrowserModuleProvider.java
@@ -38,8 +38,10 @@
 import org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.PerfDataParserListenerManager;
 import org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.decorators.BrowserPerfDataDecorator;
 import org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.decorators.BrowserResourcePerfDataDecorator;
+import org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.decorators.BrowserWebInteractionPerfDataDecorator;
 import org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.decorators.BrowserWebVitalsPerfDataDecorator;
 import org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.listener.BrowserPerfDataAnalysisListener;
+import org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.listener.BrowserWebInteractionPerfDataAnalysisListener;
 import org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.listener.BrowserWebResourcePerfDataAnalysisListener;
 import org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.listener.BrowserWebVitalsPerfDataAnalysisListener;
 import org.apache.skywalking.oap.server.receiver.sharing.server.SharingServerModule;
@@ -126,6 +128,7 @@ private PerfDataParserListenerManager perfDataListenerManager() {
         listenerManager.add(BrowserPerfDataDecorator.class, new BrowserPerfDataAnalysisListener.Factory(getManager(), moduleConfig));
         listenerManager.add(BrowserWebVitalsPerfDataDecorator.class, new BrowserWebVitalsPerfDataAnalysisListener.Factory(getManager(), moduleConfig));
         listenerManager.add(BrowserResourcePerfDataDecorator.class, new BrowserWebResourcePerfDataAnalysisListener.Factory(getManager(), moduleConfig));
+        listenerManager.add(BrowserWebInteractionPerfDataDecorator.class, new BrowserWebInteractionPerfDataAnalysisListener.Factory(getManager(), moduleConfig));
         return listenerManager;
     }
 
diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/handler/grpc/BrowserPerfServiceHandler.java b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/handler/grpc/BrowserPerfServiceHandler.java
@@ -24,6 +24,7 @@
 import org.apache.skywalking.apm.network.language.agent.v3.BrowserPerfData;
 import org.apache.skywalking.apm.network.language.agent.v3.BrowserPerfServiceGrpc;
 import org.apache.skywalking.apm.network.language.agent.v3.BrowserResourcePerfData;
+import org.apache.skywalking.apm.network.language.agent.v3.BrowserWebInteractionsPerfData;
 import org.apache.skywalking.apm.network.language.agent.v3.BrowserWebVitalsPerfData;
 import org.apache.skywalking.oap.server.library.module.ModuleManager;
 import org.apache.skywalking.oap.server.library.server.grpc.GRPCHandler;
@@ -34,6 +35,7 @@
 import org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.PerfDataAnalyzer;
 import org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.PerfDataParserListenerManager;
 import org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.decorators.BrowserResourcePerfDataDecorator;
+import org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.decorators.BrowserWebInteractionPerfDataDecorator;
 import org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.decorators.BrowserWebVitalsPerfDataDecorator;
 import org.apache.skywalking.oap.server.telemetry.TelemetryModule;
 import org.apache.skywalking.oap.server.telemetry.api.CounterMetrics;
@@ -150,6 +152,25 @@ public void collectWebVitalsPerfData(BrowserWebVitalsPerfData request, StreamObs
         }
     }
 
+    @Override
+    public void collectWebInteractionsPerfData(BrowserWebInteractionsPerfData request, StreamObserver<Commands> responseObserver) {
+        if (log.isDebugEnabled()) {
+            log.debug("receive browser web interaction performance data");
+        }
+        HistogramMetrics.Timer timer = perfHistogram.createTimer();
+        try {
+            PerfDataAnalyzer analyzer = new PerfDataAnalyzer(perfDataListenerManager);
+            analyzer.doAnalysis(new BrowserWebInteractionPerfDataDecorator(request));
+        } catch (Throwable e) {
+            log.error(e.getMessage(), e);
+            perfErrorCounter.inc();
+        } finally {
+            timer.finish();
+            responseObserver.onNext(Commands.newBuilder().build());
+            responseObserver.onCompleted();
+        }
+    }
+
     @Override
     public StreamObserver<BrowserErrorLog> collectErrorLogs(final StreamObserver<Commands> responseObserver) {
         return new StreamObserver<BrowserErrorLog>() {
diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/handler/rest/BrowserPerfServiceHTTPHandler.java b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/handler/rest/BrowserPerfServiceHTTPHandler.java
@@ -24,6 +24,7 @@
 import org.apache.skywalking.apm.network.language.agent.v3.BrowserErrorLog;
 import org.apache.skywalking.apm.network.language.agent.v3.BrowserPerfData;
 import org.apache.skywalking.apm.network.language.agent.v3.BrowserResourcePerfData;
+import org.apache.skywalking.apm.network.language.agent.v3.BrowserWebInteractionsPerfData;
 import org.apache.skywalking.apm.network.language.agent.v3.BrowserWebVitalsPerfData;
 import org.apache.skywalking.oap.server.library.module.ModuleManager;
 import org.apache.skywalking.oap.server.receiver.browser.provider.BrowserServiceModuleConfig;
@@ -33,6 +34,7 @@
 import org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.PerfDataAnalyzer;
 import org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.PerfDataParserListenerManager;
 import org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.decorators.BrowserResourcePerfDataDecorator;
+import org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.decorators.BrowserWebInteractionPerfDataDecorator;
 import org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.decorators.BrowserWebVitalsPerfDataDecorator;
 import org.apache.skywalking.oap.server.telemetry.TelemetryModule;
 import org.apache.skywalking.oap.server.telemetry.api.CounterMetrics;
@@ -159,4 +161,19 @@ public Commands collectResourcesPerfData(final List<BrowserResourcePerfData> res
             throw e;
         }
     }
+
+    @Post("/browser/perfData/webInteractions")
+    public Commands collectWebInteractionsPerfData(final List<BrowserWebInteractionsPerfData> perfDataList) {
+        try (HistogramMetrics.Timer ignored = perfHistogram.createTimer()) {
+            perfDataList.forEach(perfData -> {
+                final PerfDataAnalyzer analyzer = new PerfDataAnalyzer(perfDataListenerManager);
+                analyzer.doAnalysis(new BrowserWebInteractionPerfDataDecorator(perfData));
+            });
+            return Commands.newBuilder().build();
+        } catch (Throwable e) {
+            log.error(e.getMessage(), e);
+            perfErrorCounter.inc();
+            throw e;
+        }
+    }
 }
diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/performance/decorators/BrowserWebInteractionPerfDataDecorator.java b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/performance/decorators/BrowserWebInteractionPerfDataDecorator.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.decorators;
+
+import org.apache.skywalking.apm.network.language.agent.v3.BrowserWebInteractionsPerfData;
+
+public class BrowserWebInteractionPerfDataDecorator implements BrowserPerfDecorator {
+    private BrowserWebInteractionsPerfData.Builder builder;
+
+    public BrowserWebInteractionPerfDataDecorator(BrowserWebInteractionsPerfData data) {
+        this.builder = data.toBuilder();
+    }
+
+    @Override
+    public String getService() {
+        return builder.getService();
+    }
+
+    @Override
+    public void setTime(long time) {
+        builder.setTime(time);
+    }
+
+    @Override
+    public String getServiceVersion() {
+        return builder.getServiceVersion();
+    }
+
+    @Override
+    public void setServiceVersion(String serviceVersion) {
+        builder.setServiceVersion(serviceVersion);
+    }
+
+    @Override
+    public String getPagePath() {
+        return builder.getPagePath();
+    }
+
+    @Override
+    public void setPagePath(String pagePath) {
+        builder.setPagePath(pagePath);
+    }
+
+    public int getInpTime() {
+        return builder.getInpTime();
+    }
+
+    public long getTime() {
+        return builder.getTime();
+    }
+}
diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/performance/listener/BrowserWebInteractionPerfDataAnalysisListener.java b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/performance/listener/BrowserWebInteractionPerfDataAnalysisListener.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.listener;
+
+import org.apache.skywalking.oap.server.core.CoreModule;
+import org.apache.skywalking.oap.server.core.analysis.TimeBucket;
+import org.apache.skywalking.oap.server.core.browser.source.BrowserAppWebInteractionPerf;
+import org.apache.skywalking.oap.server.core.config.NamingControl;
+import org.apache.skywalking.oap.server.core.source.SourceReceiver;
+import org.apache.skywalking.oap.server.library.module.ModuleManager;
+import org.apache.skywalking.oap.server.receiver.browser.provider.BrowserServiceModuleConfig;
+import org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.decorators.BrowserWebInteractionPerfDataDecorator;
+
+public class BrowserWebInteractionPerfDataAnalysisListener implements PerfDataAnalysisListener<BrowserWebInteractionPerfDataDecorator> {
+    private final SourceReceiver sourceReceiver;
+    private final NamingControl namingControl;
+    private BrowserAppWebInteractionPerf browserAppWebInteractionPerf;
+
+    public BrowserWebInteractionPerfDataAnalysisListener(SourceReceiver sourceReceiver, NamingControl namingControl) {
+        this.sourceReceiver = sourceReceiver;
+        this.namingControl = namingControl;
+    }
+
+    @Override
+    public void build() {
+        sourceReceiver.receive(browserAppWebInteractionPerf);
+    }
+
+    @Override
+    public void parse(BrowserWebInteractionPerfDataDecorator decorator) {
+        browserAppWebInteractionPerf = new BrowserAppWebInteractionPerf();
+        browserAppWebInteractionPerf.setTimeBucket(TimeBucket.getMinuteTimeBucket(decorator.getTime()));
+        browserAppWebInteractionPerf.setServiceName(namingControl.formatServiceName(decorator.getService()));
+        browserAppWebInteractionPerf.setPath(namingControl.formatEndpointName(browserAppWebInteractionPerf.getServiceName(), decorator.getPagePath()));
+        browserAppWebInteractionPerf.setInpTime(decorator.getInpTime());
+    }
+
+    public static class Factory implements PerfDataListenerFactory<BrowserWebInteractionPerfDataDecorator> {
+
+        private final SourceReceiver sourceReceiver;
+        private final NamingControl namingControl;
+
+        public Factory(ModuleManager moduleManager, BrowserServiceModuleConfig moduleConfig) {
+            this.sourceReceiver = moduleManager.find(CoreModule.NAME)
+                .provider()
+                .getService(SourceReceiver.class);
+
+            this.namingControl = moduleManager.find(CoreModule.NAME)
+                .provider()
+                .getService(NamingControl.class);
+        }
+
+        @Override
+        public PerfDataAnalysisListener<BrowserWebInteractionPerfDataDecorator> create(ModuleManager moduleManager, BrowserServiceModuleConfig moduleConfig) {
+            return new BrowserWebInteractionPerfDataAnalysisListener(sourceReceiver, namingControl);
+        }
+    }
+}
diff --git a/oap-server/server-starter/src/main/resources/oal/browser.oal b/oap-server/server-starter/src/main/resources/oal/browser.oal
@@ -68,6 +68,8 @@ browser_app_web_vitals_fmp_avg = from(BrowserAppWebVitalsPerf.fmpTime).filter(fm
 browser_app_web_vitals_cls_avg = from(BrowserAppWebVitalsPerf.clsTime).filter(clsTime > 0).longAvg();
 browser_app_web_vitals_lcp_avg = from(BrowserAppWebVitalsPerf.lcpTime).filter(lcpTime > 0).longAvg();
 
+browser_app_web_interaction_inp_percentile = from(BrowserAppWebInteractionPerf.inpTime).percentile2(10);
+
 // Disable unnecessary hard core stream, targeting @Stream#name
 /////////
 //disable(browser_error_log);
diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/browser/browser-page.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/browser/browser-page.json
