Add doc_values for fields that need to be sorted or aggregated in ElasticSearch, and disable all others. (#12782)

Author: kezhenxu94

docs/en/changes/changes.md

@@ -1,5 +1,12 @@
 ## 10.2.0
 
+#### Project
+* Add [`doc_values`](https://www.elastic.co/guide/en/elasticsearch/reference/current/doc-values.html) for fields
+  that need to be sorted or aggregated in Elasticsearch, and disable all others.
+  * This change would not impact the existing deployment and its feature for our official release users.
+  * **Warning** If there are custom query plugins for our Elasticsearch indices, this change could break them as
+    sort queries and aggregation queries which used the unexpected fields are being blocked.
+
 #### OAP Server
 
 * Skip processing OTLP metrics data points with flag `FLAG_NO_RECORDED_VALUE`, which causes exceptional result.

oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/util/OALClassGenerator.java

@@ -58,6 +58,7 @@
 import org.apache.skywalking.oap.server.core.storage.StorageBuilderFactory;
 import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB;
 import org.apache.skywalking.oap.server.core.storage.annotation.Column;
+import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearch;
 import org.apache.skywalking.oap.server.library.util.StringUtil;
 
 import java.io.DataOutputStream;
@@ -205,6 +206,10 @@ private Class generateMetricsClass(AnalysisResult metricsStmt) throws OALCompile
                     Annotation banyanShardingKeyAnnotation = new Annotation(BanyanDB.SeriesID.class.getName(), constPool);
                     banyanShardingKeyAnnotation.addMemberValue("index", new IntegerMemberValue(constPool, 0));
                     annotationsAttribute.addAnnotation(banyanShardingKeyAnnotation);
+
+                    // Entity id field should enable doc values.
+                    final var enableDocValuesAnnotation = new Annotation(ElasticSearch.EnableDocValues.class.getName(), constPool);
+                    annotationsAttribute.addAnnotation(enableDocValuesAnnotation);
                 }
 
                 if (field.isGroupByCondInTopN()) {

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java

@@ -78,6 +78,7 @@ public StorageID id() {
     private String id0;
     @Column(name = ID1, storageOnly = true)
     private String id1;
+    @ElasticSearch.EnableDocValues
     @Column(name = START_TIME)
     private long startTime;
     @Column(name = ALARM_MESSAGE, length = 512)

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java

@@ -89,6 +89,7 @@ public abstract class AbstractLogRecord extends Record {
     private LongText content;
     @Setter
     @Getter
+    @ElasticSearch.EnableDocValues
     @Column(name = TIMESTAMP)
     private long timestamp;
 

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/endpoint/EndpointRelationServerSideMetrics.java

@@ -30,6 +30,7 @@
 import org.apache.skywalking.oap.server.core.storage.StorageID;
 import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB;
 import org.apache.skywalking.oap.server.core.storage.annotation.Column;
+import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearch;
 import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
 import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
 import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder;
@@ -58,10 +59,12 @@ public class EndpointRelationServerSideMetrics extends Metrics {
     private String destEndpoint;
     @Setter
     @Getter
+    @ElasticSearch.EnableDocValues
     @Column(name = COMPONENT_ID, storageOnly = true)
     private int componentId;
     @Setter
     @Getter
+    @ElasticSearch.EnableDocValues
     @Column(name = ENTITY_ID, length = 512)
     @BanyanDB.SeriesID(index = 0)
     private String entityId;

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationClientSideMetrics.java

@@ -30,6 +30,7 @@
 import org.apache.skywalking.oap.server.core.storage.StorageID;
 import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB;
 import org.apache.skywalking.oap.server.core.storage.annotation.Column;
+import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearch;
 import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
 import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
 import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder;
@@ -67,6 +68,7 @@ public class ServiceInstanceRelationClientSideMetrics extends Metrics {
     private String destServiceInstanceId;
     @Setter
     @Getter
+    @ElasticSearch.EnableDocValues
     @Column(name = ENTITY_ID, length = 512)
     @BanyanDB.SeriesID(index = 0)
     private String entityId;

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationServerSideMetrics.java

@@ -30,6 +30,7 @@
 import org.apache.skywalking.oap.server.core.storage.StorageID;
 import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB;
 import org.apache.skywalking.oap.server.core.storage.annotation.Column;
+import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearch;
 import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
 import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
 import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder;
@@ -67,6 +68,7 @@ public class ServiceInstanceRelationServerSideMetrics extends Metrics {
     private String destServiceInstanceId;
     @Setter
     @Getter
+    @ElasticSearch.EnableDocValues
     @Column(name = ENTITY_ID, length = 512)
     @BanyanDB.SeriesID(index = 0)
     private String entityId;

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/process/ProcessRelationClientSideMetrics.java

@@ -31,6 +31,7 @@
 import org.apache.skywalking.oap.server.core.storage.StorageID;
 import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB;
 import org.apache.skywalking.oap.server.core.storage.annotation.Column;
+import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearch;
 import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
 import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
 import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder;
@@ -65,11 +66,13 @@ public class ProcessRelationClientSideMetrics extends Metrics {
     private String destProcessId;
     @Setter
     @Getter
+    @ElasticSearch.EnableDocValues
     @Column(name = ENTITY_ID, length = 512)
     @BanyanDB.SeriesID(index = 0)
     private String entityId;
     @Setter
     @Getter
+    @ElasticSearch.EnableDocValues
     @Column(name = COMPONENT_ID, storageOnly = true)
     @BanyanDB.SeriesID(index = 1)
     private int componentId;

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/process/ProcessRelationServerSideMetrics.java

@@ -31,6 +31,7 @@
 import org.apache.skywalking.oap.server.core.storage.StorageID;
 import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB;
 import org.apache.skywalking.oap.server.core.storage.annotation.Column;
+import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearch;
 import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
 import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
 import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder;
@@ -64,11 +65,13 @@ public class ProcessRelationServerSideMetrics extends Metrics {
     private String destProcessId;
     @Setter
     @Getter
+    @ElasticSearch.EnableDocValues
     @Column(name = ENTITY_ID, length = 512)
     @BanyanDB.SeriesID(index = 0)
     private String entityId;
     @Setter
     @Getter
+    @ElasticSearch.EnableDocValues
     @Column(name = COMPONENT_ID, storageOnly = true)
     @BanyanDB.SeriesID(index = 1)
     private int componentId;

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationClientSideMetrics.java

@@ -60,9 +60,11 @@ public class ServiceRelationClientSideMetrics extends Metrics {
     @Getter
     @Column(name = COMPONENT_IDS, storageOnly = true)
     @ElasticSearch.Keyword
+    @ElasticSearch.EnableDocValues
     private IntList componentIds = new IntList(3);
     @Setter
     @Getter
+    @ElasticSearch.EnableDocValues
     @Column(name = ENTITY_ID, length = 512)
     @BanyanDB.SeriesID(index = 0)
     private String entityId;