controlplane.yaml

AWSTemplateFormatVersion: "2010-09-09"
Description: Kong Mesh Control Plane on ECS
Parameters:
  VPCStackName:
    Type: String
    Description:
      Name of the parent Fargate networking stack that you created. Necessary
      to locate and reference resources created by that stack.
  Image:
    Type: String
    Default: "docker.io/kong/kuma-cp:2.1.1"
    Description: Name of the control plane docker image
  AllowedCidr:
    Type: String
    Default: "10.0.0.0/8"
    Description: CIDR used to restrict control plane access
  DesiredCount:
    Type: Number
    Default: 1
    Description: Number of CP tasks to run

  ZoneName:
    Type: String
    Default: "ecs-demo"
    Description: Name of this zone
  GlobalKDSAddress:
    Type: String
    Default: ""
    Description: Address of the global KDS Service
  GlobalCPTokenSecret:
    Type: String
    Default: ""
    Description: Secret ARN holding the global control plane token
  KonnectCPId:
    Type: String
    Default: ""
    Description: cpId on the Konnect platform

  LicenseSecret:
    Type: String
    Description: Secret ARN holding a Kong Mesh license

  ServerCertSecret:
    Type: String
    Description: ARN of an AWS secret containing a TLS certificate for serving control plane traffic
  ServerKeySecret:
    Type: String
    Description: ARN of an AWS secret containing a TLS private key for serving control plane traffic
Conditions:
  IsMultizone: !And
    - !Not [!Equals [!Ref GlobalKDSAddress, ""]]
    - !Not [!Equals [!Ref GlobalCPTokenSecret, ""]]
Resources:
  # Database
  DBSubnetGroup:
    Type: "AWS::RDS::DBSubnetGroup"
    Properties:
      DBSubnetGroupDescription: !Ref AWS::StackName
      SubnetIds:
        - Fn::ImportValue: !Join [":", [!Ref VPCStackName, PublicSubnet]]
        - Fn::ImportValue: !Join [":", [!Ref VPCStackName, PublicSubnet2]]

  CPDatabasePassword:
    Type: AWS::SecretsManager::Secret
    Properties:
      # Using the StackId gives us a randomness to the name while allowing us to
      # prefix it with a path
      Name:
        !Join [
          "/",
          [
            !Ref AWS::StackName,
            "CPDatabasePassword",
            !Select [2, !Split ["/", !Ref AWS::StackId]],
          ],
        ]
      Description: Secret containing the password for the Kong Mesh CP DB backend
      GenerateSecretString:
        ExcludeCharacters: >-
          '"@/\
        PasswordLength: 16

  CPDatabase:
    Type: AWS::RDS::DBInstance
    # The default is snapshot which is a good default but, we use this in CI where snapshots don't make sense
    # If reusing this you probably don't want this deletionPolicy
    DeletionPolicy: Delete
    Properties:
      DBName: kongmesh
      Engine: postgres
      MasterUsername: kongmesh
      MasterUserPassword:
        !Join [
          "",
          [
            "{{resolve:secretsmanager:",
            !Ref CPDatabasePassword,
            ":SecretString}}",
          ],
        ]
      AllocatedStorage: 50
      DBInstanceClass: db.m5.large
      DBSubnetGroupName: !Ref DBSubnetGroup
      VPCSecurityGroups:
        - !Ref CPSecurityGroup

  # We use ECS service discovery to let the mesh data plane proxies
  # communicate with the control plane
  DiscoveryServiceCP:
    Type: AWS::ServiceDiscovery::Service
    Properties:
      Description: Discovery Service for Kong Mesh CP
      DnsConfig:
        RoutingPolicy: MULTIVALUE
        DnsRecords:
          - TTL: 60
            Type: A
          - TTL: 60
            Type: SRV
      HealthCheckCustomConfig:
        FailureThreshold: 1
      Name: controlplane
      NamespaceId:
        Fn::ImportValue: !Join [":", [!Ref VPCStackName, PrivateNamespace]]

  # Our NLB exposes the Kuma API on port 5682
  LoadBalancerTargetAPITLS:
    Type: AWS::ElasticLoadBalancingV2::TargetGroup
    Properties:
      TargetType: ip
      Port: 5682
      Protocol: TCP
      VpcId:
        Fn::ImportValue: !Join [":", [!Ref VPCStackName, VPCId]]

  LoadBalancerListenerAPITLS:
    Type: AWS::ElasticLoadBalancingV2::Listener
    Properties:
      LoadBalancerArn:
        Fn::ImportValue: !Join [":", [!Ref VPCStackName, LoadBalancer]]
      Port: 5682
      Protocol: TCP
      DefaultActions:
        - Type: forward
          ForwardConfig:
            TargetGroups:
              - TargetGroupArn: !Ref LoadBalancerTargetAPITLS

  # The control plane task populates this secret with the admin token
  APITokenSecret:
    Type: AWS::SecretsManager::Secret
    Properties:
      Name:
        !Join [
          "/",
          [
            !Ref AWS::StackName,
            "APITokenSecret",
            !Select [2, !Split ["/", !Ref AWS::StackId]],
          ],
        ]
      Description: Secret containing an admin Kuma API Token

  # IAM roles for the control plane task
  CPTaskExecutionRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Statement:
          - Effect: Allow
            Principal:
              Service: [ecs-tasks.amazonaws.com]
            Action: [sts:AssumeRole]
      Policies:
        - PolicyName: get-license-secret
          PolicyDocument:
            Statement:
              - Effect: Allow
                Action:
                  - secretsmanager:GetSecretValue
                Resource:
                  - !Ref LicenseSecret
                  - !Ref CPDatabasePassword
                  - !Ref ServerCertSecret
                  - !Ref ServerKeySecret
                  - !If
                    - IsMultizone
                    - !Ref GlobalCPTokenSecret
                    - !Ref AWS::NoValue
      Path: /
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly
        - arn:aws:iam::aws:policy/CloudWatchLogsFullAccess

  CPTaskRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument: |
        {
            "Statement": [{
                "Effect": "Allow",
                "Principal": { "Service": [ "ecs-tasks.amazonaws.com" ]},
                "Action": [ "sts:AssumeRole" ]
            }]
        }
      Policies:
        - PolicyName: put-api-token-secret
          PolicyDocument:
            Statement:
              - Effect: Allow
                Action:
                  - secretsmanager:PutSecretValue
                Resource:
                  - !Ref APITokenSecret
        - PolicyName: get-dataplane-roles
          PolicyDocument:
            Statement:
              - Effect: Allow
                Action:
                  - iam:GetRole
                Resource:
                  - "*"
        - PolicyName: update-route53
          PolicyDocument:
            Statement:
              - Effect: Allow
                Action:
                  - route53:ListResourceRecordSets
                  - route53:ChangeResourceRecordSets
                  - route53:GetHostedZone
                Resource:
                  - "*"
      Path: /
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/CloudWatchFullAccess
        - arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess
        - arn:aws:iam::aws:policy/AWSAppMeshEnvoyAccess

  # CP security group configuration
  CPSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: Access to Kong Mesh CP
      VpcId:
        Fn::ImportValue: !Join [":", [!Ref VPCStackName, VPCId]]

  CPIngressTCPFromSelf:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      Description: Ingress for control plane TCP from VPC
      GroupId: !Ref CPSecurityGroup
      CidrIp: "10.0.0.0/8"
      IpProtocol: tcp
      FromPort: 5676
      ToPort: 5682

  CPPostgresFromSelf:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      Description: Ingress for control plane to postgres
      GroupId: !Ref CPSecurityGroup
      CidrIp: "10.0.0.0/8"
      IpProtocol: tcp
      FromPort: 5432
      ToPort: 5432

  # ECS configuration
  CPLogGroup:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupName: !Ref AWS::StackName
      RetentionInDays: 7

  CPService:
    Type: AWS::ECS::Service
    Properties:
      LaunchType: FARGATE
      Cluster:
        Fn::ImportValue: !Join [":", [!Ref VPCStackName, ClusterName]]
      DesiredCount: !Ref DesiredCount
      TaskDefinition: !Ref CPTaskDefinition
      # Our Dataplane proxies reach the control plane through this service
      ServiceRegistries:
        - RegistryArn: !GetAtt DiscoveryServiceCP.Arn
          Port: 5678
      # We can reach the control plane API via the NLB
      LoadBalancers:
        - TargetGroupArn: !Ref LoadBalancerTargetAPITLS
          ContainerPort: 5682
          ContainerName: controlplane
      NetworkConfiguration:
        AwsvpcConfiguration:
          AssignPublicIp: ENABLED
          Subnets:
            - Fn::ImportValue: !Join [":", [!Ref VPCStackName, PublicSubnet]]
          SecurityGroups:
            - !Ref CPSecurityGroup
            - Fn::ImportValue:
                !Join [":", [!Ref VPCStackName, FargateContainerSecurityGroup]]

  CPTaskDefinition:
    Type: AWS::ECS::TaskDefinition
    Properties:
      Family: kong-mesh-cp-family
      NetworkMode: awsvpc
      RequiresCompatibilities:
        - FARGATE
      TaskRoleArn: !Ref CPTaskRole
      ExecutionRoleArn: !Ref CPTaskExecutionRole
      Cpu: 256
      Memory: 512
      Volumes:
        - Name: token
      ContainerDefinitions:
        - Name: init
          Essential: false
          Image: !Ref Image
          Secrets:
            - Name: KUMA_STORE_POSTGRES_PASSWORD
              ValueFrom: !Ref CPDatabasePassword
          Environment:
            - Name: KUMA_STORE_TYPE
              Value: postgres
            - Name: KUMA_STORE_POSTGRES_HOST
              Value:
                Fn::GetAtt: CPDatabase.Endpoint.Address
            - Name: KUMA_STORE_POSTGRES_PORT
              Value:
                Fn::Sub: "${CPDatabase.Endpoint.Port}" # convert number to string
            - Name: KUMA_STORE_POSTGRES_USER
              Value: kongmesh
            - Name: KUMA_STORE_POSTGRES_DB_NAME
              Value: kongmesh
            - Name: KUMA_STORE_POSTGRES_TLS_MODE
              Value: verifyNone
          Command: [migrate, up]
          LogConfiguration:
            LogDriver: awslogs
            Options:
              awslogs-group: !Ref CPLogGroup
              awslogs-region: !Ref AWS::Region
              awslogs-stream-prefix: init
        - Name: controlplane
          DependsOn:
            - Condition: SUCCESS
              ContainerName: init
          Essential: true
          Image: !Ref Image
          PortMappings:
            - ContainerPort: 5676
              Protocol: tcp
            - ContainerPort: 5678
              Protocol: tcp
            - ContainerPort: 5682
              Protocol: tcp
          User: root:root # needed for UDP port 53 binding
          Environment:
            - Name: KUMA_MODE
              Value: zone
            - Name: KMESH_LICENSE_PATH
              Value: /tmp/license.json
            - Name: KUMA_GENERAL_TLS_CERT_FILE
              Value: /tmp/crt.pem
            - Name: KUMA_GENERAL_TLS_KEY_FILE
              Value: /tmp/key.pem
            - Name: KUMA_STORE_TYPE
              Value: postgres
            - Name: KUMA_STORE_POSTGRES_HOST
              Value:
                Fn::GetAtt: CPDatabase.Endpoint.Address
            - Name: KUMA_STORE_POSTGRES_PORT
              Value:
                Fn::Sub: "${CPDatabase.Endpoint.Port}"
            - Name: KUMA_STORE_POSTGRES_USER
              Value: kongmesh
            - Name: KUMA_STORE_POSTGRES_DB_NAME
              Value: kongmesh
            - Name: KUMA_STORE_POSTGRES_TLS_MODE
              Value: verifyNone
            - Name: KUMA_DP_SERVER_AUTHN_DP_PROXY_TYPE
              Value: aws-iam
            - Name: KUMA_DP_SERVER_AUTHN_ZONE_PROXY_TYPE
              Value: aws-iam
            - Name: KUMA_DP_SERVER_AUTHN_ENABLE_RELOADABLE_TOKENS
              Value: "true"
            - Name: KMESH_AWSIAM_AUTHORIZEDACCOUNTIDS
              Value: !Ref AWS::AccountId
            - Name: KUMA_DEFAULTS_SKIP_HOSTNAME_GENERATORS
              Value: "true"
            - Name: KMESH_RUNTIME_AWS_ROUTE53_ENABLED
              Value: "true"
            - Name: KMESH_RUNTIME_AWS_ROUTE53_HOSTED_ZONE_ID
              Value: !ImportValue
                Fn::Sub: "${VPCStackName}:HostedZoneId"
            - Name: KUMA_DNS_SERVER_CIDR
              Value: "127.1.0.0/16"
            - Name: KUMA_IPAM_MESH_MULTI_ZONE_SERVICE_CIDR
              Value: "127.2.0.0/16"
            - Name: KUMA_IPAM_MESH_EXTERNAL_SERVICE_CIDR
              Value: "127.3.0.0/16"
            - Name: KUMA_IPAM_MESH_SERVICE_CIDR
              Value: "127.4.0.0/16"
            - Name: KUMA_DNS_SERVER_DOMAIN
              Value: "mesh.local"
            - Name: KUMA_DNS_SERVER_SERVICE_VIP_PORT
              Value: 8080
            - Name: KUMA_MULTIZONE_ZONE_NAME
              Value: !Ref ZoneName
            - Name: KMESH_MULTIZONE_ZONE_KDS_AUTH_CP_TOKEN_PATH
              Value: /tmp/cp-token
            - !If
              - IsMultizone
              - Name: KUMA_MULTIZONE_ZONE_GLOBAL_ADDRESS
                Value: !Ref GlobalKDSAddress
              - !Ref AWS::NoValue
            - !If
              - IsMultizone
              - Name: KUMA_MULTIZONE_ZONE_KDS_TLS_SKIP_VERIFY
                Value: "true"
              - !Ref AWS::NoValue
            - !If
              - IsMultizone
              - Name: KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED
                Value: "true"
              - !Ref AWS::NoValue
            - !If
              - IsMultizone
              - Name: KMESH_MULTIZONE_ZONE_KONNECT_CP_ID
                Value: !Ref KonnectCPId
              - !Ref AWS::NoValue
            - !If
              - IsMultizone
              - Name: KMESH_MULTIZONE_GLOBAL_KDS_AUTH_TYPE
                Value: "cpToken"
              - !Ref AWS::NoValue
          Secrets:
            - Name: KMESH_LICENSE
              ValueFrom: !Ref LicenseSecret
            - Name: KUMA_STORE_POSTGRES_PASSWORD
              ValueFrom: !Ref CPDatabasePassword
            - Name: TLS_CRT
              ValueFrom: !Ref ServerCertSecret
            - Name: TLS_KEY
              ValueFrom: !Ref ServerKeySecret
            - !If
              - IsMultizone
              - Name: KDS_AUTH_CP_TOKEN
                ValueFrom: !Ref GlobalCPTokenSecret
              - !Ref AWS::NoValue
          EntryPoint:
            - sh
            - -c
            - echo "${TLS_CRT}" > "${KUMA_GENERAL_TLS_CERT_FILE}" && echo "${TLS_KEY}" > "${KUMA_GENERAL_TLS_KEY_FILE}" && echo "${KMESH_LICENSE}" > "${KMESH_LICENSE_PATH}" && echo "${KDS_AUTH_CP_TOKEN}" > "${KMESH_MULTIZONE_ZONE_KDS_AUTH_CP_TOKEN_PATH}"; kuma-cp run
          HealthCheck:
            Command:
              [
                CMD, "/busybox/busybox", "sh", "-c",
                "wget -q -O - http://localhost:5680/healthy || exit 1",
              ]
            StartPeriod: 10
            Interval: 10
          LogConfiguration:
            LogDriver: awslogs
            Options:
              awslogs-group: !Ref CPLogGroup
              awslogs-region: !Ref AWS::Region
              awslogs-stream-prefix: controlplane
        # The next 2 containers fetch the admin token and save it to AWS secrets
        # manager if it hasn't been saved yet
        - Name: fetch-admin-token
          Essential: false
          DependsOn:
            - Condition: HEALTHY
              ContainerName: controlplane
          Image: nicolaka/netshoot
          User: root:root # needed for modifying mounted volume
          EntryPoint:
            - bash
            - -ec
            - |
              TOKEN_URL=http://localhost:5681/global-secrets/admin-user-token
              # Wait until our token exists
              while [[ $(curl -s -o /dev/null "${TOKEN_URL}" -w '%{http_code}') != "200" ]]; do
                echo "Waiting 5s for admin token"
                sleep 5s
              done
              curl -s -o - "${TOKEN_URL}" | jq -r .data | base64 -d > /var/token/api-token
          MountPoints:
            - ContainerPath: /var/token
              SourceVolume: token
          LogConfiguration:
            LogDriver: awslogs
            Options:
              awslogs-group: !Ref CPLogGroup
              awslogs-region: !Ref AWS::Region
              awslogs-stream-prefix: fetch-admin-token
        - Name: save-admin-token
          Essential: false
          DependsOn:
            - Condition: SUCCESS
              ContainerName: fetch-admin-token
          Image: amazon/aws-cli
          EntryPoint:
            - bash
            - -c
            - !Sub 'aws secretsmanager put-secret-value --client-request-token $(sha256sum /var/token/api-token | cut -d " " -f 1) --secret-id ${APITokenSecret} --secret-string file:///var/token/api-token'
          MountPoints:
            - ContainerPath: /var/token
              SourceVolume: token
              ReadOnly: true
          LogConfiguration:
            LogDriver: awslogs
            Options:
              awslogs-group: !Ref CPLogGroup
              awslogs-region: !Ref AWS::Region
              awslogs-stream-prefix: save-admin-token
Outputs:
  CPAddress:
    Description: Address of the control plane
    Value:
      Fn::Sub:
        - ${DiscoveryServiceCP.Name}.${TLD}
        - TLD:
            Fn::ImportValue: !Join [":", [!Ref VPCStackName, TLD]]
    Export:
      Name: !Join [":", [!Ref AWS::StackName, CPAddress]]
  APITokenSecret:
    Description: Secret ARN with an admin API token
    Value: !Ref APITokenSecret
    Export:
      Name: !Join [":", [!Ref AWS::StackName, APITokenSecret]]
  CPCASecret:
    Description: ARN of an AWS secret containing a TLS CA for verifying control plane traffic
    Value: !Ref ServerCertSecret
    Export:
      Name: !Join [":", [!Ref AWS::StackName, CPCASecret]]