BanyanDB: support new Index rule type SKIPPING/TREE, and update the record log's trace_id indexType to SKIPPING (#13366)

Author: wankai123

docs/en/changes/changes.md

@@ -39,6 +39,8 @@
 * OAP gRPC-Client support `Health Check`.
 * [Break Change] `health_check_xx` metrics make response 1 represents healthy, 0 represents unhealthy.
 * Bump up grpc to 1.70.0.
+* BanyanDB: support new Index rule type `SKIPPING/TREE`, and update the record `log`'s `trace_id` indexType to `SKIPPING`
+* BanyanDB: remove `index-only` from tag setting.
 
 #### UI
 

oap-server-bom/pom.xml

@@ -72,7 +72,7 @@
         <httpcore.version>4.4.16</httpcore.version>
         <httpasyncclient.version>4.1.5</httpasyncclient.version>
         <commons-compress.version>1.21</commons-compress.version>
-        <banyandb-java-client.version>0.9.0-rc2</banyandb-java-client.version>
+        <banyandb-java-client.version>0.9.0-rc3</banyandb-java-client.version>
         <kafka-clients.version>3.4.0</kafka-clients.version>
         <spring-kafka-test.version>2.4.6.RELEASE</spring-kafka-test.version>
         <consul.client.version>1.5.3</consul.client.version>

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java

@@ -68,6 +68,7 @@ public abstract class AbstractLogRecord extends Record {
     @Setter
     @Getter
     @Column(name = TRACE_ID, length = 150)
+    @BanyanDB.IndexRule(indexType = BanyanDB.IndexRule.IndexType.SKIPPING)
     private String traceId;
     @Setter
     @Getter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDB.java

@@ -24,6 +24,7 @@
 import java.lang.annotation.Target;
 
 import lombok.Getter;
+import org.apache.skywalking.oap.server.core.analysis.manual.log.LogRecord;
 import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
 import org.apache.skywalking.oap.server.core.analysis.record.Record;
 import org.apache.skywalking.oap.server.core.storage.StorageID;
@@ -132,6 +133,16 @@ enum IndexType {
              * It's suitable for most tag indexing due to a better memory usage ratio and query performance.
              */
             INVERTED,
+            /**
+             * The `SKIPPING` index is optimized for the majority of stream tags, which prioritizes efficient space utilization.
+             * Such as the `trace_id` in the {@link LogRecord}.
+             */
+            SKIPPING,
+            /**
+             * The `TREE` index is designed for storing hierarchical data.
+             * Such as Trace Span.
+             */
+            TREE
         }
     }
 

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBExtension.java

@@ -54,9 +54,8 @@ public class BanyanDBExtension {
      * indexType is the type of index built for a {@link ModelColumn} in BanyanDB.
      *
      * @since 9.3.0
-     * @deprecated since 10.2. Only support {@link BanyanDB.IndexRule.IndexType#INVERTED} now. There was IndexType#TREE,
-     * but removed.
      */
+    @Getter
     private final BanyanDB.IndexRule.IndexType indexType;
 
     /**

oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/MetadataRegistry.java

@@ -81,6 +81,10 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import static org.apache.skywalking.banyandb.database.v1.BanyandbDatabase.IndexRule.Type.TYPE_INVERTED;
+import static org.apache.skywalking.banyandb.database.v1.BanyandbDatabase.IndexRule.Type.TYPE_SKIPPING;
+import static org.apache.skywalking.banyandb.database.v1.BanyandbDatabase.IndexRule.Type.TYPE_TREE;
+
 @Slf4j
 public enum MetadataRegistry {
     INSTANCE;
@@ -160,7 +164,7 @@ public MeasureModel registerMeasureModel(Model model, BanyanDBStorageConfig conf
                 .collect(Collectors.toList());
 
         if (model.getBanyanDBModelExtension().isStoreIDTag()) {
-            indexRules.add(indexRule(schemaMetadata.group, BanyanDBConverter.ID, false, null));
+            indexRules.add(indexRule(schemaMetadata.group, BanyanDBConverter.ID, false, null, null));
         }
 
         final Measure.Builder builder = Measure.newBuilder();
@@ -368,12 +372,35 @@ Duration downSamplingDuration(DownSampling downSampling) {
         }
     }
 
-    IndexRule indexRule(String group, String tagName, boolean enableSort,  BanyanDB.MatchQuery.AnalyzerType analyzer) {
+    IndexRule indexRule(String group,
+                        String tagName,
+                        boolean enableSort,
+                        BanyanDB.MatchQuery.AnalyzerType analyzer,
+                        BanyanDB.IndexRule.IndexType type) {
         IndexRule.Builder builder = IndexRule.newBuilder()
                                              .setMetadata(Metadata.newBuilder().setName(tagName).setGroup(group))
-                                             .setType(IndexRule.Type.TYPE_INVERTED).addTags(tagName);
+                                             .addTags(tagName);
         // *Notice*: here is a reverse logic, if enableSort is true, then setNoSort is false
         builder.setNoSort(!enableSort);
+
+        if (type != null) {
+            switch (type) {
+                case INVERTED:
+                    builder.setType(TYPE_INVERTED);
+                    break;
+                case TREE:
+                    builder.setType(TYPE_TREE);
+                    break;
+                case SKIPPING:
+                    builder.setType(TYPE_SKIPPING);
+                    break;
+                default:
+                    throw new UnsupportedOperationException("unsupported index type: " + type);
+            }
+        } else {
+            builder.setType(TYPE_INVERTED);
+        }
+
         if (analyzer != null) {
             switch (analyzer) {
                 case KEYWORD:
@@ -449,7 +476,8 @@ List<TagMetadata> parseTagMetadata(Model model, Schema.SchemaBuilder builder, Li
                     tagMetadataList.add(new TagMetadata(
                         indexRule(
                             group, tagSpec.getName(), col.getBanyanDBExtension().isEnableSort(),
-                            col.getBanyanDBExtension().getAnalyzer()
+                            col.getBanyanDBExtension().getAnalyzer(),
+                            col.getBanyanDBExtension().getIndexType()
                         ), tagSpec));
                 } else {
                     tagMetadataList.add(new TagMetadata(null, tagSpec));
@@ -499,7 +527,8 @@ MeasureMetadata parseTagAndFieldMetadata(Model model, Schema.SchemaBuilder build
                     result.tag(new TagMetadata(
                         indexRule(
                             group, tagSpec.getName(), col.getBanyanDBExtension().isEnableSort(),
-                            col.getBanyanDBExtension().getAnalyzer()
+                            col.getBanyanDBExtension().getAnalyzer(),
+                            col.getBanyanDBExtension().getIndexType()
                         ), tagSpec));
                 } else {
                     result.tag(new TagMetadata(null, tagSpec));
@@ -544,10 +573,6 @@ private TagSpec parseTagSpec(ModelColumn modelColumn) {
         } else {
             throw new IllegalStateException("type " + modelColumn.getType().toString() + " is not supported");
         }
-
-        if (modelColumn.isIndexOnly()) {
-            tagSpec.setIndexedOnly(true);
-        }
         return tagSpec.build();
     }
 

oap-server/server-telemetry/telemetry-api/src/main/java/org/apache/skywalking/oap/server/telemetry/api/MetricsCreator.java

@@ -53,7 +53,7 @@ HistogramMetrics createHistogramMetric(String name, String tips, MetricsTag.Keys
     default HealthCheckMetrics createHealthCheckerGauge(String name, MetricsTag.Keys tagKeys, MetricsTag.Values tagValues) {
         Preconditions.checkArgument(!Strings.isNullOrEmpty(name), "Require non-null or empty metric name");
         return new HealthCheckMetrics(createGauge(Strings.lenientFormat("%s%s", HEALTH_METRIC_PREFIX, name),
-            Strings.lenientFormat("%s health check. 1 health, 0 not health, -1 unknown", name),
+            Strings.lenientFormat("%s health check. 1 healthy, 0 not healthy, -1 unknown", name),
             tagKeys, tagValues));
     }
 

test/e2e-v2/script/env

@@ -23,7 +23,7 @@ SW_AGENT_CLIENT_JS_COMMIT=af0565a67d382b683c1dbd94c379b7080db61449
 SW_AGENT_CLIENT_JS_TEST_COMMIT=4f1eb1dcdbde3ec4a38534bf01dded4ab5d2f016
 SW_KUBERNETES_COMMIT_SHA=6fe5e6f0d3b7686c6be0457733e825ee68cb9b35
 SW_ROVER_COMMIT=79292fe07f17f98f486e0c4471213e1961fb2d1d
-SW_BANYANDB_COMMIT=b90a0ff31a4d8560422c04d152e7e7188d72ea89
+SW_BANYANDB_COMMIT=0f3b90d9b4f628d2de02cb39ef678d636358df44
 SW_AGENT_PHP_COMMIT=d1114e7be5d89881eec76e5b56e69ff844691e35
 SW_PREDICTOR_COMMIT=54a0197654a3781a6f73ce35146c712af297c994