Support hot/warm/cold stages TTL query in the status API. (#13156)

Author: wankai123

docs/en/changes/changes.md

@@ -6,6 +6,7 @@
 
 * BanyanDB: Support `hot/warm/cold` stages configuration.
 * Fix query continues profiling policies error when the policy is already in the cache.
+* Support `hot/warm/cold` stages TTL query in the status API.
 
 #### UI
 

docs/en/status/query_ttl_setup.md

@@ -3,7 +3,8 @@
 Time To Live (TTL) mechanism has different behaviors according to different storage implementations. By default, the
 core module provides two TTL configurations: [`recordDataTTL` and `metricsDataTTL`](../setup/backend/ttl.md).
 But some storage implementations could override these settings and provide its own TTL configurations, for example,
-BanyanDB provides its native TTL mechanism to support [progressive TTL](../banyandb/ttl.md) feature.
+BanyanDB provides its native TTL mechanism to support [progressive TTL](../banyandb/ttl.md) feature and [Data Lifecycle Stages(Hot/Warm/Cold)](../banyandb/stages.md) feature.
+
 
 This API is used to get the unified and effective TTL configurations.
 - URL, `http://{core restHost}:{core restPort}/status/config/ttl`
@@ -15,17 +16,30 @@ This API is used to get the unified and effective TTL configurations.
 # e.g.
 # 1. The metadata of the service, instance, endpoint, topology map, etc.
 # 2. Generated metrics data from OAL and MAL engines.
+# 3. Banyandb storage provides Data Lifecycle Stages(Hot/Warm/Cold).
 #
 # TTLs for each granularity metrics are listed separately.
+#
+# Cover hot and warm data for BanyanDB.
 metrics.minute=7
-metrics.hour=7
-metrics.day=7
+metrics.hour=15
+metrics.day=15
+# Cold data, '-1' represents no cold stage data.
+metrics.minute.cold=-1
+metrics.hour.cold=-1
+metrics.day.cold=-1
 
 # Records TTL includes the definition of the TTL of the records data in the storage,
 # Records include traces, logs, sampled slow SQL statements, HTTP requests(by Rover), alarms, etc.
 # Super dataset of records are traces and logs, which volume should be much larger.
+#
+# Cover hot and warm data for BanyanDB.
 records.default=3
 records.superDataset=3
+# Cold data, '-1' represents no cold stage data.
+records.default.cold=-1
+records.superDataset.cold=-1
+
 ```
 
 This API also provides the response in JSON format, which is more friendly for programmatic usage.
@@ -37,12 +51,17 @@ This API also provides the response in JSON format, which is more friendly for p
 {
     "metrics": {
         "minute": 7,
-        "hour": 7,
-        "day": 7
+        "hour": 15,
+        "day": 15,
+        "coldMinute": -1,
+        "coldHour": -1,
+        "coldDay": -1
     },
     "records": {
         "default": 3,
-        "superDataset": 3
+        "superDataset": 3,
+        "coldValue": -1,
+        "coldSuperDataset": -1
     }
 }
-```
+```

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/MetricsTTL.java

@@ -33,4 +33,8 @@ public class MetricsTTL {
     private final int minute;
     private final int hour;
     private final int day;
+    // -1 means no cold stage.
+    private int coldMinute = -1;
+    private int coldHour = -1;
+    private int coldDay = -1;
 }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/RecordsTTL.java

@@ -31,4 +31,7 @@ public class RecordsTTL {
     @SerializedName("default")
     private final int value;
     private final int superDataset;
+    // -1 means no cold stage.
+    private int coldValue = -1;
+    private int coldSuperDataset = -1;
 }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/TTLDefinition.java

@@ -36,17 +36,29 @@ public String generateTTLDefinition() {
         ttlDefinition.append("# e.g.\n");
         ttlDefinition.append("# 1. The metadata of the service, instance, endpoint, topology map, etc.\n");
         ttlDefinition.append("# 2. Generated metrics data from OAL and MAL engines.\n");
+        ttlDefinition.append("# 3. Banyandb storage provides Data Lifecycle Stages(Hot/Warm/Cold).\n");
         ttlDefinition.append("#\n");
         ttlDefinition.append("# TTLs for each granularity metrics are listed separately.\n");
+        ttlDefinition.append("#\n");
+        ttlDefinition.append("# Cover hot and warm data for BanyanDB.\n");
         ttlDefinition.append("metrics.minute=").append(metrics.getMinute()).append("\n");
         ttlDefinition.append("metrics.hour=").append(metrics.getHour()).append("\n");
         ttlDefinition.append("metrics.day=").append(metrics.getDay()).append("\n");
+        ttlDefinition.append("# Cold data, '-1' represents no cold stage data.\n");
+        ttlDefinition.append("metrics.minute.cold=").append(metrics.getColdMinute()).append("\n");
+        ttlDefinition.append("metrics.hour.cold=").append(metrics.getColdHour()).append("\n");
+        ttlDefinition.append("metrics.day.cold=").append(metrics.getColdDay()).append("\n");
         ttlDefinition.append("\n");
         ttlDefinition.append("# Records TTL includes the definition of the TTL of the records data in the storage,\n");
         ttlDefinition.append("# Records include traces, logs, sampled slow SQL statements, HTTP requests(by Rover), alarms, etc.\n");
         ttlDefinition.append("# Super dataset of records are traces and logs, which volume should be much larger.\n");
+        ttlDefinition.append("#\n");
+        ttlDefinition.append("# Cover hot and warm data for BanyanDB.\n");
         ttlDefinition.append("records.default=").append(records.getValue()).append("\n");
         ttlDefinition.append("records.superDataset=").append(records.getSuperDataset()).append("\n");
+        ttlDefinition.append("# Cold data, '-1' represents no cold stage data.\n");
+        ttlDefinition.append("records.default.cold=").append(records.getColdValue()).append("\n");
+        ttlDefinition.append("records.superDataset.cold=").append(records.getColdSuperDataset()).append("\n");
         return ttlDefinition.toString();
     }
 

oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBTTLStatusQuery.java

@@ -24,25 +24,72 @@
 import org.apache.skywalking.oap.server.core.storage.ttl.TTLDefinition;
 
 public class BanyanDBTTLStatusQuery implements StorageTTLStatusQuery {
-    private final int grNormalTTLDays;
-    private final int grSuperTTLDays;
-    private final int gmMinuteTTLDays;
-    private final int gmHourTTLDays;
-    private final int gmDayTTLDays;
+    private int grNormalTTLDays;
+    private int grSuperTTLDays;
+    // -1 means no cold stage.
+    private int grColdNormalTTLDays = -1;
+    private int grColdSuperTTLDays = -1;
+    private int gmMinuteTTLDays;
+    private int gmHourTTLDays;
+    private int gmDayTTLDays;
+    private int gmColdMinuteTTLDays = -1;
+    private int gmColdHourTTLDays = -1;
+    private int gmColdDayTTLDays = -1;
 
     public BanyanDBTTLStatusQuery(BanyanDBStorageConfig config) {
         grNormalTTLDays = config.getRecordsNormal().getTtl();
         grSuperTTLDays = config.getRecordsSuper().getTtl();
         gmMinuteTTLDays = config.getMetricsMin().getTtl();
         gmHourTTLDays = config.getMetricsHour().getTtl();
         gmDayTTLDays = config.getMetricsDay().getTtl();
+        config.getRecordsNormal().getAdditionalLifecycleStages().forEach(stage -> {
+            if (stage.getName().equals(BanyanDBStorageConfig.StageName.warm)) {
+                grNormalTTLDays = grNormalTTLDays + stage.getTtl();
+            } else if (stage.getName().equals(BanyanDBStorageConfig.StageName.cold)) {
+                grColdNormalTTLDays = stage.getTtl();
+            }
+        });
+        config.getRecordsSuper().getAdditionalLifecycleStages().forEach(stage -> {
+            if (stage.getName().equals(BanyanDBStorageConfig.StageName.warm)) {
+                grSuperTTLDays = grSuperTTLDays + stage.getTtl();
+            } else if (stage.getName().equals(BanyanDBStorageConfig.StageName.cold)) {
+                grColdSuperTTLDays = stage.getTtl();
+            }
+        });
+        config.getMetricsMin().getAdditionalLifecycleStages().forEach(stage -> {
+            if (stage.getName().equals(BanyanDBStorageConfig.StageName.warm)) {
+                gmMinuteTTLDays = gmMinuteTTLDays + stage.getTtl();
+            } else if (stage.getName().equals(BanyanDBStorageConfig.StageName.cold)) {
+                gmColdMinuteTTLDays = stage.getTtl();
+            }
+        });
+        config.getMetricsHour().getAdditionalLifecycleStages().forEach(stage -> {
+            if (stage.getName().equals(BanyanDBStorageConfig.StageName.warm)) {
+                gmHourTTLDays = gmHourTTLDays + stage.getTtl();
+            } else if (stage.getName().equals(BanyanDBStorageConfig.StageName.cold)) {
+                gmColdHourTTLDays = stage.getTtl();
+            }
+        });
+        config.getMetricsDay().getAdditionalLifecycleStages().forEach(stage -> {
+            if (stage.getName().equals(BanyanDBStorageConfig.StageName.warm)) {
+                gmDayTTLDays = gmDayTTLDays + stage.getTtl();
+            } else if (stage.getName().equals(BanyanDBStorageConfig.StageName.cold)) {
+                gmColdDayTTLDays = stage.getTtl();
+            }
+        });
     }
 
     @Override
     public TTLDefinition getTTL() {
-        return new TTLDefinition(
+        TTLDefinition definition = new TTLDefinition(
             new MetricsTTL(gmMinuteTTLDays, gmHourTTLDays, gmDayTTLDays),
             new RecordsTTL(grNormalTTLDays, grSuperTTLDays)
         );
+        definition.getRecords().setColdValue(grColdNormalTTLDays);
+        definition.getRecords().setColdSuperDataset(grColdSuperTTLDays);
+        definition.getMetrics().setColdMinute(gmColdMinuteTTLDays);
+        definition.getMetrics().setColdHour(gmColdHourTTLDays);
+        definition.getMetrics().setColdDay(gmColdDayTTLDays);
+        return definition;
     }
 }