Modern-SelectDB-Lakehouse-Platform.yaml

ROSTemplateFormatVersion: '2015-09-01'
Description:
  zh-cn: 基于SelectDB低成本、高性能的湖仓一体化架构。
  en: Low cost, high performance lake warehouse integrated architecture based on SelectDB .
Parameters:
  CommonName:
    Type: String
    Default: Lakehouse
  ZoneId:
    Type: String
    AssociationProperty: ALIYUN::ECS::Instance::ZoneId
    Label:
      zh-cn: 可用区
      en: Available Zone
  InstanceClass:
    Type: String
    Label:
      zh-cn: 实例规格
      en: Instance class
    AllowedValues:
      - selectdb.xlarge
      - selectdb.2xlarge
      - selectdb.4xlarge
      - selectdb.8xlarge
      - selectdb.16xlarge
      - selectdb.24xlarge
      - selectdb.32xlarge
    AssociationPropertyMetadata:
      ValueLabelMapping:
        selectdb.xlarge:
          zh-cn: 4 核 16GB
          en: 4 cores 16GB
        selectdb.2xlarge:
          zh-cn: 8 核 32GB
          en: 8 cores 32GB
        selectdb.4xlarge:
          zh-cn: 16 核 64GB
          en: 16 cores 64GB
        selectdb.8xlarge:
          zh-cn: 32 核 128GB
          en: 32 cores 128GB
        selectdb.16xlarge:
          zh-cn: 64 核 256GB
          en: 64 cores 256GB
        selectdb.24xlarge:
          zh-cn: 96 核 384GB
          en: 96 cores 384GB
        selectdb.32xlarge:
          zh-cn: 128 核 512GB
          en: 128 cores 512GB
    Default: selectdb.2xlarge
  SelectDBEngineVersion:
    Type: String
    Label:
      zh-cn: 内核版本
      en: Engine Version
    AllowedValues:
      - '3.0'
      - '4.0'
    Default: '4.0'
  AccountPassword:
    Type: String
    NoEcho: true
    Label:
      zh-cn: admin 账号密码
      en: admin account password
    Description:
      zh-cn: <br><b>注： <font color='blue'>长度8-32，输入密码至少满足其中三种条件:大写字母、小写字母、数字、特殊字符(!@#$%^&*()_+-=)。</font></b>
      en: <br><b>note： <font color='blue'>The 8-32 long login password of instance, The password must meet at least three conditions:uppercase letters, lowercase letters, digits, and special characters (!). @#$%^&*()_+-=).</font></b>
    AssociationProperty: ALIYUN::RDS::Instance::AccountPassword
  RdsAccountName:
    Type: String
    Label:
      zh-cn: 高权限账号
      en: Super Account
    AllowedPattern: ^(?!dbo\b|login\b|admin\b|administrator\b|adminsys\b|alimail\b|aliyun\b|apache\b|appadmin\b|apsara\b|aurora\b|bulkadmin\b|cangjie\b|cdn\b|client\b|cm\b|dataengine\b|dayu\b|dba\b|dbcreator\b|developer\b|diskadmin\b|distribution\b|dns\b|download\b|eagleye\b|f5\b|faq\b|fuxi\b|galaxy\b|gongcao\b|gongming\b|groupon\b|ha\b|help\b|host\b|hostmaster\b|houyi\b|hr\b|info\b|information_schema\b|kuafu\b|lvs\b|manager\b|master\b|meituan\b|model\b|monitor\b|msdb\b|mssqld\b|mssqlsystemresource\b|mysql\b|nas\b|net\b|netops\b|netweb\b|news\b|no-reply\b|ntp\b|nuwa\b|nvwa\b|operator\b|opr\b|ops\b|opsdb\b|oracle\b|pangu\b|pe\b|post\b|postmaster\b|processadmin\b|public\b|qq\b|replicator\b|reply\b|root\b|sa\b|sales\b|san\b|security\b|securityadmin\b|serveradmin\b|services\b|setupadmin\b|shennong\b|siteops\b|sqlengine\b|sqlonline\b|squid\b|ssladmin\b|support\b|sys\b|sysadmin\b|syslog\b|system\b|taoyun\b|tempdb\b|test\b|tianyun\b|wangwang\b|eb\b|webmaster\b|webnet\b|xtrabak\b|youchao\b|yum\b|yunti\b|zhongkui\b|database\b|add\b|except\b|percent\b|all\b|exec\b|plan\b|alter\b|execute\b|precision\b|and\b|exists\b|primary\b|any\b|exit\b|print\b|as\b|fetch\b|proc\b|asc\b|file\b|procedure\b|authorization\b|fillfactor\b|public\b|backup\b|for\b|raiserror\b|begin\b|foreign\b|read\b|between\b|freetext\b|readtext\b|break\b|freetexttable\b|reconfigure\b|browse\b|from\b|references\b|bulk\b|full\b|replication\b|by\b|function\b|restore\b|cascade\b|goto\b|restrict\b|case\b|grant\b|return\b|check\b|group\b|revoke\b|checkpoint\b|having\b|right\b|close\b|holdlock\b|rollback\b|clustered\b|identity\b|rowcount\b|coalesce\b|identity_insert\b|rowguidcol\b|collate\b|identitycol\b|rule\b|column\b|if\b|save\b|commit\b|in\b|schema\b|compute\b|index\b|select\b|constraint\b|inner\b|session_user\b|contains\b|insert\b|set\b|containstable\b|intersect\b|setuser\b|continue\b|into\b|shutdown\b|convert\b|is\b|some\b|create\b|join\b|statistics\b|cross\b|key\b|system_user\b|current\b|kill\b|table\b|current_date\b|left\b|textsize\b|current_time\b|like\b|then\b|current_timestamp\b|lineno\b|to\b|current_user\b|load\b|top\b|cursor\b|national\b|tran\b|database\b|nocheck\b|transaction\b|dbcc\b|nonclustered\b|trigger\b|deallocate\b|not\b|truncate\b|declare\b|null\b|tsequal\b|default\b|nullif\b|union\b|delete\b|of\b|unique\b|deny\b|off\b|update\b|desc\b|offsets\b|updatetext\b|disk\b|on\b|use\b|distinct\b|open\b|user\b|distributed\b|opendatasource\b|values\b|double\b|openquery\b|varying\b|drop\b|openrowset\b|view\b|dummy\b|openxml\b|waitfor\b|dump\b|option\b|when\b|else\b|or\b|where\b|end\b|order\b|while\b|errlvl\b|outer\b|with\b|escape\b|over\b|writetext\b|add\b|analyze\b|asc\b|between\b|blob\b|call\b|change\b|check\b|condition\b|continue\b|cross\b|current_timestamp\b|database\b|day_microsecond\b|dec\b|default\b|desc\b|distinct\b|double\b|each\b|enclosed\b|exit\b|fetch\b|float8\b|foreign\b|goto\b|having\b|hour_minute\b|ignore\b|infile\b|insensitive\b|int1\b|int4\b|interval\b|iterate\b|keys\b|leading\b|like\b|lines\b|localtimestamp\b|longblob\b|low_priority\b|mediumint\b|minute_microsecond\b|modifies\b|no_write_to_binlog\b|on\b|optionally\b|out\b|precision\b|purge\b|read\b|references\b|rename\b|require\b|revoke\b|schema\b|select\b|set\b|spatial\b|sqlexception\b|sql_big_result\b|ssl\b|table\b|tinyblob\b|to\b|true\b|unique\b|update\b|using\b|utc_timestamp\b|varchar\b|when\b|with\b|xor\b|all\b|and\b|asensitive\b|bigint\b|both\b|cascade\b|char\b|collate\b|connection\b|convert\b|current_date\b|current_user\b|databases\b|day_minute\b|decimal\b|delayed\b|describe\b|distinctrow\b|drop\b|else\b|escaped\b|explain\b|float\b|for\b|from\b|grant\b|high_priority\b|hour_second\b|in\b|inner\b|insert\b|int2\b|int8\b|into\b|join\b|kill\b|leave\b|limit\b|load\b|lock\b|longtext\b|match\b|mediumtext\b|minute_second\b|natural\b|null\b|optimize\b|or\b|outer\b|primary\b|raid0\b|reads\b|regexp\b|repeat\b|restrict\b|right\b|schemas\b|sensitive\b|show\b|specific\b|sqlstate\b|sql_calc_found_rows\b|starting\b|terminated\b|tinyint\b|trailing\b|undo\b|unlock\b|usage\b|utc_date\b|values\b|varcharacter\b|where\b|write\b|year_month\b|alter\b|as\b|before\b|binary\b|by\b|case\b|character\b|column\b|constraint\b|create\b|current_time\b|cursor\b|day_hour\b|day_second\b|declare\b|delete\b|deterministic\b|div\b|dual\b|elseif\b|exists\b|false\b|float4\b|force\b|fulltext\b|group\b|hour_microsecond\b|if\b|index\b|inout\b|int\b|int3\b|integer\b|is\b|key\b|label\b|left\b|linear\b|localtime\b|long\b|loop\b|mediumblob\b|middleint\b|mod\b|not\b|numeric\b|option\b|order\b|outfile\b|procedure\b|range\b|real\b|release\b|replace\b|return\b|rlike\b|second_microsecond\b|separator\b|smallint\b|sql\b|sqlwarning\b|sql_small_result\b|straight_join\b|then\b|tinytext\b|trigger\b|union\b|unsigned\b|use\b|utc_time\b|varbinary\b|varying\b|while\b|x509\b|zerofill)([a-zA-Z][a-zA-Z0-9_]{0,30}[a-zA-Z0-9])$
    ConstraintDescription:
      en: The value can be up to 32 characters in length and can contain letters, digits, and underscores (_). The value must start with a letter and end with a letter or digit.  For more information about invalid characters, see 详见<a href="https://help.aliyun.com/zh/rds/developer-reference/forbidden-keywords" target="_blank">Forbidden keywords table</a>.
      zh-cn: 由小写字母、大写字母、数字、下划线（_）组成，以字母开头，以字母或数字结尾，最多32个字符。其他非法字符，详见<a href="https://help.aliyun.com/zh/rds/developer-reference/forbidden-keywords" target="_blank">禁用关键字表</a>。
    Default: db_user
  RdsAccountPwd:
    Type: String
    NoEcho: true
    Label:
      zh-cn: 高权限账号密码
      en: Super Account Password
    Description:
      zh-cn: <br><b>注： <font color='blue'>长度8-32，输入密码至少满足其中三种条件:大写字母、小写字母、数字、特殊字符(!@#$%^&*()_+-=)。</font></b>
      en: <br><b>note： <font color='blue'>The 8-32 long login password of instance, The password must meet at least three conditions:uppercase letters, lowercase letters, digits, and special characters (!). @#$%^&*()_+-=).</font></b>
    AssociationProperty: ALIYUN::RDS::Instance::AccountPassword
  DBInstanceClass:
    Type: String
    Label:
      en: Instance Class
      zh-cn: 实例规格
    AssociationProperty: ALIYUN::RDS::Instance::InstanceType
    AssociationPropertyMetadata:
      ZoneId: ${ZoneId}
      EngineVersion: "8.0"
      DBInstanceStorageType: cloud_essd
      Engine: MySQL
      Category: Basic
      CommodityCode: bards
  BucketName:
    Type: String
    Label:
      en: OSS Bucket Name
      zh-cn: OSS Bucket 名字
    ConstraintDescription:
      zh-cn: 3 到 63 个字符，不以连字符 (-) 开头和结尾，可以包含小写字母、数字和连字符 (-)
      en: >-
        3 to 63 characters, not beginning and ending with a hyphen (-), can
        contain lowercase letters, Numbers and hyphens (-)
    Description:
      zh-cn: >-
        3到63个字符，不以连字符(-)开头和结尾，可以包含小写字母、数字和连字符(-)；<br>注：<font
        color='blue'><b>需要保证bucket名称的唯一性，已经存在的无法创建。</b></font>
      en: >-
        3 to 63 characters, not beginning and ending with a hyphen (-), can
        contain lowercase letters, Numbers and hyphens (-);<br>Note: <font
        color='blue'><b>bucket name must be unique,existing one cannot be created again.</b></font>
    AllowedPattern: '^[a-z0-9]+[a-z0-9\-]*[a-z0-9]+$'
    MinLength: 3
    MaxLength: 63
    AssociationProperty: AutoCompleteInput
    AssociationPropertyMetadata:
      Length: 5
      Prefix: iceberg-
      CharacterClasses:
        - Class: lowercase
          min: 1
  DtsJobName:
    Type: String
    Label:
      en: Dts Job Name
      zh-cn: 同步任务名称
    Description:
      en: You are advised to configure a name with service significance (without unique requirement) for easy identification.
      zh-cn: 建议配置具有业务意义的名称（无唯一性要求），便于后续识别。
    Default: RDS2SelectDB
Resources:
  Vpc:
    Type: 'ALIYUN::ECS::VPC'
    Properties:
      CidrBlock: 192.168.0.0/16
      VpcName:
        Fn::Sub: VPC_${CommonName}
  VSwitch:
    Type: 'ALIYUN::ECS::VSwitch'
    Properties:
      VpcId:
        Ref: Vpc
      CidrBlock: 192.168.0.0/24
      ZoneId:
        Ref: ZoneId
      VSwitchName:
        Fn::Sub: VSW_${CommonName}
  Rds:
    Type: ALIYUN::RDS::DBInstance
    Properties:
      EngineVersion: "8.0"
      DBInstanceClass:
        Ref: DBInstanceClass
      SecurityIPList: 192.168.0.0/16
      Category: Basic
      DBInstanceStorageType: cloud_essd
      DBInstanceStorage: 50
      Engine: MySQL
      VpcId:
        Ref: Vpc
      VSwitchId:
        Ref: VSwitch
      AllocatePublicConnection: true
  RdsDatabase1:
    Type: ALIYUN::RDS::Database
    Properties:
      CharacterSetName: utf8
      DBInstanceId:
        Ref: Rds
      DBName: tpch100
  RdsDatabase2:
    Type: ALIYUN::RDS::Database
    Properties:
      CharacterSetName: utf8
      DBInstanceId:
        Ref: Rds
      DBName: workshop
  RdsAccount:
    DependsOn:
      - Rds
    Type: ALIYUN::RDS::Account
    Properties:
      DBInstanceId:
        Ref: Rds
      AccountPassword:
        Ref: RdsAccountPwd
      AccountType: Super
      AccountName:
        Ref: RdsAccountName
  SelectDBInstance:
    Type: ALIYUN::SELECTDB::DBInstance
    Properties:
      VpcId:
        Ref: Vpc
      ZoneId:
        Ref: ZoneId
      ChargeType: Postpaid
      Engine: selectdb
      EngineVersion:
        Ref: SelectDBEngineVersion
      CacheSize: 100
      VSwitchId:
        Ref: VSwitch
      DBInstanceClass:
        Ref: InstanceClass
      SecurityIPList:
        - 192.168.0.0/16
      AccountPassword:
        Ref: AccountPassword
  Bucket:
    Type: 'ALIYUN::OSS::Bucket'
    Properties:
      BucketName:
        Ref: BucketName
      RedundancyType: ZRS
      Policy:
        Version: '1'
        Statement:
          - Principal:
              - Ref: ALIYUN::AccountId
              - Ref: ALIYUN::TenantId
            Effect: Allow
            Resource:
              - Fn::Sub: acs:oss:*:${ALIYUN::TenantId}:${BucketName}/*
              - Fn::Sub: acs:oss:*:${ALIYUN::TenantId}:${BucketName}
            Condition:
              StringEquals:
                acs:SourceVpc:
                  - Ref: Vpc
            Action:
              - oss:*
      DeletionForce: true
  MigrationJob:
    Type: ALIYUN::DTS::SynchronizationJob2
    Properties:
      DtsJobName:
        Ref: DtsJobName
      SourceEndpoint:
        InstanceType: RDS
        InstanceID:
          Ref: Rds
        EngineName: MYSQL
        Region:
          Ref: ALIYUN::Region
        UserName:
          Ref: RdsAccountName
        Password:
          Ref: RdsAccountPwd
      DestinationEndpoint:
        InstanceType: SelectDB
        InstanceID:
          Ref: SelectDBInstance
        EngineName: SelectDB
        Region:
          Ref: ALIYUN::Region
        UserName: admin
        Password:
          Ref: AccountPassword
      StructureInitialization: true
      DelayNotice: true
      ErrorNotice: true
      DelayRuleTime: 60
      DtsBisLabel: normal
      DataInitialization: true
      DataSynchronization: true
      DbList:
        workshop:
          name: workshop
          all: true
  DtsInstance:
    Type: ALIYUN::DTS::Instance
    Properties:
      JobId:
        Fn::GetAtt:
          - MigrationJob
          - DtsJobId
      InstanceClass: small
      PayType: PostPaid
      AutoStart: true
      Type: SYNC
      SourceRegion:
        Ref: ALIYUN::Region
      DestinationRegion:
        Ref: ALIYUN::Region
      SourceEndpointEngineName: MySQL
      DestinationEndpointEngineName: SelectDB
Outputs:
  SelectDBAccount:
    Label:
      en: SelectDB login user
      zh-cn: SelectDB 登录用户
    Description:
      en: SelectDB login user.
      zh-cn: SelectDB 登录用户。
    Value: admin
  SelectDBLogingPassword:
    Label:
      en: SelectDB user admin password
      zh-cn: SelectDB 用户admin密码
    Description:
      en: SelectDB user admin password.
      zh-cn: SelectDB 用户admin密码。
    NoEcho: true
    Value:
      Ref: AccountPassword
  SelectDBVpcConnectionString:
    Label:
      en: SelectDB VPC address
      zh-cn: SelectDB VPC 地址
    Description:
      en: SelectDB VPC address.
      zh-cn: SelectDB VPC 地址。
    Value:
      Fn::GetAtt:
        - SelectDBInstance
        - VpcConnectionString
  RdsInnerConnectionString:
    Label:
      en: RDS Intranet address
      zh-cn: RDS 内网地址
    Description:
      en: RDS Intranet address
      zh-cn: RDS 内网地址。
    Value:
      Fn::GetAtt:
        - Rds
        - InnerConnectionString
  RdsAccountName:
    Label:
      en: Super Account
      zh-cn: 高权限账号
    Description:
      en: RDS super Account.
      zh-cn: RDS 高权限账号。
    Value:
      Ref: RdsAccountName
  RdsAccountPwd:
    Label:
      zh-cn: RDS 高权限账号密码
      en: RDS super Account Password
    Description:
      zh-cn: RDS 高权限账号密码。
      en: RDS super Account Password.
    NoEcho: true
    Value:
      Ref: RdsAccountPwd
  VpcId:
    Label:
      en: VPC
      zh-cn: 专有网络（VPC）
    Description:
      en: VPC
      zh-cn: 专有网络（VPC）
    Value:
      Ref: Vpc
  CidrBlock:
    Label:
      en: VPC CIDR IPv4 Block
      zh-cn: 专有网络 IPv4 网段
    Description:
      en: VPC CIDR IPv4 Block
      zh-cn: 专有网络 IPv4 网段
    Value: 192.168.0.0/16
  BucketName:
    Label:
      en: Bucket Name
      zh-cn: Bucket 名称
    Description:
      en: Bucket Name
      zh-cn: Bucket 名称
    Value:
      Ref: BucketName
Metadata:
  ALIYUN::ROS::Interface:
    Outputs:
      - SelectDBAccount
      - SelectDBLogingPassword
      - SelectDBVpcConnectionString
      - RdsAccountName
      - RdsAccountPwd
      - RdsInnerConnectionString
      - VpcId
      - CidrBlock
      - BucketName
    ParameterGroups:
      - Parameters:
          - ZoneId
        Label:
          default:
            en: Network basic configuration
            zh-cn: 网络基础配置
      - Parameters:
          - BucketName
        Label:
          default:
            en: OSS Configuration
            zh-cn: OSS 配置
      - Parameters:
          - DBInstanceClass
          - RdsAccountName
          - RdsAccountPwd
        Label:
          default:
            en: RDS Configuration
            zh-cn: RDS 配置
      - Parameters:
          - InstanceClass
          - SelectDBEngineVersion
          - AccountPassword
        Label:
          default:
            en: Cloud database SelectDB edition
            zh-cn: 云数据库 SelectDB 版
      - Parameters:
          - DtsJobName
        Label:
          default:
            en: DTS Configuration
            zh-cn: 数据传输服务 DTS 配置
    TemplateTags:
      - acs:document-help:ecs:基于SelectDB低成本、高性能的湖仓一体化架构-tech_solu_239
    Hidden:
      - CommonName
  ALIYUN::ROS::Composer:
    '92299008':
      Parent: '97090508'
      Edge:
        - '104524e3'
        - 315788d7
      Line: 0:0:0:gray:0
    '97090508':
      Parent: bd55c35a
      Rect:
        - 640
        - 472
        - 60
        - 148
        - 2
        - 0
      ResT: Composer::ROSParameter::Region
    bd55c35a:
      Rect:
        - 680
        - 540
        - 40
        - 100
        - 1
        - 0
      ResT: Composer::ROSParameter::AlibabaCloud
    c22ba0c2:
      Res:
        - Vpc
      Parent: '97090508'
      Rect:
        - 540
        - 340
        - 80
        - 260
        - 3
        - 0
    315788d7:
      Res:
        - MigrationJob
      Parent: '97090508'
      Rect:
        - 40
        - 40
        - 640
        - 200
        - 3
        - 0
      Hidden: true
    '104524e3':
      Res:
        - DtsInstance
      Parent: '97090508'
      Rect:
        - 40
        - 40
        - 371
        - 188
        - 3
        - 0
    8a2b6345:
      Res:
        - Bucket
      Parent: c22ba0c2
      Rect:
        - 40
        - 40
        - 100
        - 310
        - 4
        - 0
      Hidden: true
    637986aa:
      Res:
        - ZoneId
      Parent: c22ba0c2
      Rect:
        - 440
        - 270
        - 160
        - 310
        - 4
        - 0
      ResT: Composer::ROSParameter::Zone
    d8db9793:
      Res:
        - VSwitch
      Parent: 637986aa
      Rect:
        - 400
        - 200
        - 180
        - 360
        - 5
        - 0
    13a72e98:
      Res:
        - Rds
        - RdsDatabase1
        - RdsDatabase2
        - RdsAccount
      Ref:
        - RdsDatabase1
        - RdsDatabase2
        - RdsAccount
      Parent: d8db9793
      Rect:
        - 40
        - 40
        - 289
        - 425
        - 6
        - 0
    3ee43873:
      Res:
        - SelectDBInstance
      Parent: d8db9793
      Rect:
        - 40
        - 40
        - 449
        - 425
        - 6
        - 0
      ResT: ALIYUN::SELECTDB::DBInstance
    55cbc4e1:
      Parent: '97090508'
      Edge:
        - 315788d7
        - 13a72e98
      Line: 0:0:0:gray:0
    237684c6:
      Parent: '97090508'
      Edge:
        - 315788d7
        - 3ee43873
      Line: 0:0:0:gray:0