feat(backend): doris新增DOS存储层管理【saas部分】 TencentBlueKing#14367

# Reviewed, transaction id: 65259

Author: v_ycggyao

dbm-ui/backend/configuration/constants.py

@@ -148,6 +148,8 @@ class SystemSettingsEnum(StrStructuredEnum):
     DORIS_COS_REGION = EnumField("DORIS_COS_REGION", _("Doris使用COS地域映射"))
     # Doris使用COS标签
     DORIS_COS_TAGGING = EnumField("DORIS_COS_TAGGING", _("Doris使用COS标签"))
+    # Doris 低频存储开关
+    DORIS_COS_SWITCH = EnumField("DORIS_COS_SWITCH", _("Doris低频存储开关"))
     # 小额绿通单据申请
     QUICK_MINOR_POAA = EnumField("QUICK_MINOR_POAA", _("小额绿通单据申请"))
     # 资源池独立业务,如果配置该变量,需要联动修改 MANAGE_TOPO::resource.idle.module的值

dbm-ui/backend/configuration/views/system.py

@@ -94,6 +94,18 @@ def common_sqls(self, request, *args, **kwargs):
     def device_classes(self, request, *args, **kwargs):
         return Response(SystemSettings.get_setting_value(SystemSettingsEnum.DEVICE_CLASSES.value, default=[]))
 
+    @common_swagger_auto_schema(
+        operation_summary=_("查询doris 低频存储开关"),
+        tags=tags,
+    )
+    @action(methods=["GET"], detail=False)
+    def doris_cos_switch(self, request, *args, **kwargs):
+        return Response(
+            SystemSettings.get_setting_value(
+                SystemSettingsEnum.DORIS_COS_SWITCH.value, default={"DORIS_COS_SWITCH": False}
+            )
+        )
+
     @common_swagger_auto_schema(operation_summary=_("查询环境变量"), tags=tags)
     @action(detail=False, methods=["get"])
     def environ(self, request):

dbm-ui/backend/db_services/bigdata/doris/query.py

@@ -10,7 +10,7 @@
 """
 import json
 
-from django.core import cache
+from django.core.cache import cache
 from django.forms import model_to_dict
 from django.utils.translation import gettext_lazy as _
 
@@ -53,7 +53,7 @@ class DorisListRetrieveResource(BigDataBaseListRetrieveResource, DorisExportQuer
         InstanceRole.DORIS_FOLLOWER.value,
         InstanceRole.DORIS_OBSERVER.value,
         InstanceRole.DORIS_BACKEND_HOT.value,
-        InstanceRole.DORIS_BACKEND_COLD.value,
+        InstanceRole.DORIS_BACKEND_WARM.value,
     ]
     fields = [
         *BigDataBaseListRetrieveResource.fields,
@@ -117,3 +117,11 @@ def get_clusters_master(cls, bk_biz_id: int, cluster_ids: list) -> dict:
         }
 
         return cluster_stat_map
+
+    @classmethod
+    def retrieve_cluster(cls, bk_biz_id: int, cluster_id: int) -> dict:
+        """查询集群详情"""
+        cluster_details = cls.list_clusters(bk_biz_id, {"id": cluster_id}, limit=1, offset=0).data[0]
+        details = cls._retrieve_cluster(cluster_details, cluster_id)
+        details["cold_resource"] = cls.get_cold_resource(bk_biz_id, cluster_id)
+        return details

dbm-ui/backend/tests/mock_data/ticket/doris_flow.py

@@ -85,6 +85,7 @@
         "old_nodes": {
             "hot": [],
             "cold": [],
+            "warm": [],
             "observer": [
                 {"ip": "127.0.0.5", "bk_host_id": 1005, "bk_cloud_id": 0},
                 {"ip": "127.0.1.4", "bk_host_id": 1004, "bk_cloud_id": 0},

dbm-ui/backend/ticket/builders/common/constants.py

@@ -21,7 +21,7 @@
 DORIS_FOLLOWER_NEED = 3
 DORIS_OBSERVER_ZERO = 0
 DORIS_OBSERVER_MIN = 2
-DORIS_HOT_COLD_LIMIT = 2
+DORIS_HOT_WARM_LIMIT = 2
 
 HDFS_DATANODE_MIN = 2
 HDFS_NAMENODE_NEED = 2
@@ -68,7 +68,7 @@ class Pulsar(StrStructuredEnum):
 
     class Doris(StrStructuredEnum):
         HOT = EnumField("hot", _("hot"))
-        COLD = EnumField("cold", _("cold"))
+        WARM = EnumField("warm", _("warm"))
         FOLLOWER = EnumField("follower", _("follower"))
         OBSERVER = EnumField("observer", _("observer"))
 

dbm-ui/backend/ticket/builders/doris/doris_apply.py

@@ -33,6 +33,7 @@
 class DorisApplyDetailSerializer(BigDataApplyDetailsSerializer):
     http_port = serializers.IntegerField(help_text=_("http端口"), default=DORIS_DEFAULT_HTTP_PORT)
     query_port = serializers.IntegerField(help_text=_("输入端口"), default=DORIS_DEFAULT_QUERY_PORT)
+    enable_cold_storage = serializers.BooleanField(help_text=_("是否建立低频存储"), default=False)
 
     def validate(self, attrs):
         """
@@ -75,8 +76,8 @@ def format_ticket_data(self):
 class DorisApplyResourceParamBuilder(builders.ResourceApplyParamBuilder):
     @classmethod
     def fill_instance_num(cls, next_flow_data, ticket_data, nodes_key):
-        """对doris的hot和cold角色填充实例数"""
-        for role in ["hot", "cold"]:
+        """对doris的hot和warm角色填充实例数"""
+        for role in ["hot", "warm"]:
             if role not in next_flow_data[nodes_key]:
                 continue
 

dbm-ui/backend/ticket/builders/doris/doris_scale_up.py

@@ -28,6 +28,8 @@
 
 
 class DorisScaleUpDetailSerializer(BigDataScaleDetailSerializer):
+    enable_cold_storage = serializers.BooleanField(help_text=_("是否建立低频存储"), default=False)
+
     def validate(self, attrs):
         attrs = super().validate(attrs)
 
@@ -75,7 +77,7 @@ def format_ticket_data(self):
                         "ip": "127.0.0.7"
                     }
                 ],
-                "cold": [
+                "warm": [
                     {
                         "bk_cloud_id": 0,
                         "bk_host_id": 0,

dbm-ui/backend/ticket/builders/doris/doris_shrink.py

@@ -24,10 +24,10 @@
 
 
 class DorisShrinkDetailSerializer(BigDataSingleClusterOpsDetailsSerializer):
-    # 目前只支持hot/cold/observer节点缩容，不支持follower节点缩容
+    # 目前只支持hot/warm/observer节点缩容，不支持follower节点缩容
     class NodesSerializer(serializers.Serializer):
         hot = serializers.ListField(help_text=_("hot信息列表"), child=serializers.DictField())
-        cold = serializers.ListField(help_text=_("cold信息列表"), child=serializers.DictField())
+        warm = serializers.ListField(help_text=_("warm信息列表"), child=serializers.DictField())
         observer = serializers.ListField(help_text=_("observer信息列表"), child=serializers.DictField())
 
     old_nodes = NodesSerializer(help_text=_("nodes节点列表"))