Use New Fargate Profile For Each Run And Allow More Than One Tests At The Same Time (#416) (#430)

Author: sethAmazon

terraform/eks/container-insights-agent/aoc_service_fargate.yml

@@ -2,7 +2,7 @@ apiVersion: v1
 kind: Service
 metadata:
   name: adot-collector-service
-  namespace: default
+  namespace: ${Namespace}
   labels:
     app: aws-adot
     component: adot-collector

terraform/eks/container-insights-agent/cluster_role_binding_fargate.yml

@@ -5,7 +5,7 @@ metadata:
 subjects:
   - kind: ServiceAccount
     name: adot-collector-service-account
-    namespace: default
+    namespace: ${Namespace}
 roleRef:
   kind: ClusterRole
   name: adotcol-admin-role

terraform/eks/container-insights-agent/config_map_fargate.yml

@@ -2,7 +2,7 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: adot-collector-config
-  namespace: default
+  namespace: ${Namespace}
   labels:
     app: aws-adot
     component: adot-collector-config
@@ -39,19 +39,19 @@ data:
                 regex: (.+)
                 target_label: __metrics_path__
                 # Changes the default metrics path to kubelet's proxy cadvdisor metrics endpoint
-                replacement: /api/v1/nodes/$${1}/proxy/metrics/cadvisor
+                replacement: /api/v1/nodes/$$${1}/proxy/metrics/cadvisor
             metric_relabel_configs:
               # extract readable container/pod name from id field
               - action: replace
                 source_labels: [id]
                 regex: '^/machine\.slice/machine-rkt\\x2d([^\\]+)\\.+/([^/]+)\.service$'
                 target_label: rkt_container_name
-                replacement: '$${2}-$${1}'
+                replacement: '$$${2}-$$${1}'
               - action: replace
                 source_labels: [id]
                 regex: '^/system\.slice/(.+)\.service$'
                 target_label: systemd_service_name
-                replacement: '$${1}'
+                replacement: '$$${1}'
     processors:
       # rename labels which apply to all metrics and are used in metricstransform/rename processor
       metricstransform/label_1:

terraform/eks/container-insights-agent/logs_sample_fargate.yml

@@ -4,7 +4,7 @@ metadata:
   name: flblogproducer2eks
   labels:
     app: flblogproducer2eks
-  namespace: default
+  namespace: ${Namespace}
 spec:
   replicas: 1
   selector:

terraform/eks/container-insights-agent/service_account_fargate.tpl renamed to terraform/eks/container-insights-agent/service_account_fargate.yml

@@ -2,6 +2,6 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: adot-collector-service-account
-  namespace: default
+  namespace: ${Namespace}
   annotations:
     eks.amazonaws.com/role-arn: ${RoleArn}

terraform/eks/container-insights-agent/stateful_set_fargate.tpl renamed to terraform/eks/container-insights-agent/stateful_set_fargate.yml

@@ -2,7 +2,7 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: adot-collector
-  namespace: default
+  namespace: ${Namespace}
   labels:
     app: aws-adot
     component: adot-collector

terraform/eks/container_insights_agent.tf

@@ -1,43 +1,49 @@
 data "template_file" "cluster_role_file" {
   template = file("./container-insights-agent/cluster_role.tpl")
   vars = {
-    NAMESPACE = var.deployment_type == "fargate" ? "default" : kubernetes_namespace.aoc_ns.metadata[0].name
+    NAMESPACE = var.deployment_type == "fargate" ? tolist(aws_eks_fargate_profile.test_profile.selector)[0].namespace : kubernetes_namespace.aoc_ns.metadata[0].name
   }
+  depends_on = [aws_eks_fargate_profile.test_profile]
 }
 
 data "template_file" "cluster_role_binding_file" {
   template = file("./container-insights-agent/cluster_role_binding.tpl")
   vars = {
-    NAMESPACE       = var.deployment_type == "fargate" ? "default" : kubernetes_namespace.aoc_ns.metadata[0].name,
+    NAMESPACE       = var.deployment_type == "fargate" ? tolist(aws_eks_fargate_profile.test_profile.selector)[0].namespace : kubernetes_namespace.aoc_ns.metadata[0].name,
     SERVICE_ACCOUNT = kubernetes_service_account.aoc-agent-role.metadata[0].name
   }
+  depends_on = [aws_eks_fargate_profile.test_profile]
 }
 
 data "template_file" "config_map_file" {
   template = file("./container-insights-agent/config_map.tpl")
   vars = {
-    NAMESPACE = var.deployment_type == "fargate" ? "default" : kubernetes_namespace.aoc_ns.metadata[0].name
+    NAMESPACE = var.deployment_type == "fargate" ? tolist(aws_eks_fargate_profile.test_profile.selector)[0].namespace : kubernetes_namespace.aoc_ns.metadata[0].name
   }
+  depends_on = [aws_eks_fargate_profile.test_profile]
 }
 
 data "template_file" "daemonset_file" {
   template = file("./container-insights-agent/daemonset.tpl")
   vars = {
-    NAMESPACE       = var.deployment_type == "fargate" ? "default" : kubernetes_namespace.aoc_ns.metadata[0].name
+    NAMESPACE       = var.deployment_type == "fargate" ? tolist(aws_eks_fargate_profile.test_profile.selector)[0].namespace : kubernetes_namespace.aoc_ns.metadata[0].name
     SERVICE_ACCOUNT = kubernetes_service_account.aoc-agent-role.metadata[0].name
     OTELIMAGE       = module.common.aoc_image
     REGION          = var.region
   }
+  depends_on = [aws_eks_fargate_profile.test_profile]
 }
 
 resource "kubectl_manifest" "service_account" {
   count = var.aoc_base_scenario == "infra" && var.deployment_type == "fargate" ? 1 : 0
-  yaml_body = templatefile("./container-insights-agent/service_account_fargate.tpl",
+  yaml_body = templatefile("./container-insights-agent/service_account_fargate.yml",
     {
-      RoleArn : module.iam_assumable_role_admin.iam_role_arn
+      RoleArn : module.iam_assumable_role_admin.iam_role_arn,
+      Namespace : tolist(aws_eks_fargate_profile.test_profile.selector)[0].namespace
   })
   depends_on = [
-    module.iam_assumable_role_admin
+    module.iam_assumable_role_admin,
+    aws_eks_fargate_profile.test_profile
   ]
 }
 
@@ -46,14 +52,16 @@ resource "kubectl_manifest" "cluster_role" {
 }
 
 resource "kubectl_manifest" "cluster_role_binding" {
-  yaml_body = var.deployment_type == "fargate" ? file("./container-insights-agent/cluster_role_binding_fargate.yml") : data.template_file.cluster_role_binding_file.rendered
+  yaml_body = var.deployment_type == "fargate" ? templatefile("./container-insights-agent/cluster_role_binding_fargate.yml", { Namespace : tolist(aws_eks_fargate_profile.test_profile.selector)[0].namespace }) : data.template_file.cluster_role_binding_file.rendered
   depends_on = [
-    kubectl_manifest.cluster_role
+    kubectl_manifest.cluster_role,
+    aws_eks_fargate_profile.test_profile
   ]
 }
 
 resource "kubectl_manifest" "config_map" {
-  yaml_body = var.deployment_type == "fargate" ? file("./container-insights-agent/config_map_fargate.yml") : data.template_file.config_map_file.rendered
+  yaml_body  = var.deployment_type == "fargate" ? templatefile("./container-insights-agent/config_map_fargate.yml", { Namespace : tolist(aws_eks_fargate_profile.test_profile.selector)[0].namespace }) : data.template_file.config_map_file.rendered
+  depends_on = [aws_eks_fargate_profile.test_profile]
 }
 
 resource "kubectl_manifest" "daemonset" {
@@ -67,25 +75,28 @@ resource "kubectl_manifest" "daemonset" {
 
 resource "kubectl_manifest" "aoc_service_deploy" {
   count     = var.aoc_base_scenario == "infra" && var.deployment_type == "fargate" ? 1 : 0
-  yaml_body = file("./container-insights-agent/aoc_service_fargate.yml")
+  yaml_body = templatefile("./container-insights-agent/aoc_service_fargate.yml", { Namespace : tolist(aws_eks_fargate_profile.test_profile.selector)[0].namespace })
   depends_on = [
-    kubectl_manifest.config_map
+    kubectl_manifest.config_map,
+    aws_eks_fargate_profile.test_profile
   ]
 }
 
 resource "kubectl_manifest" "aoc_fargate_deploy" {
   count = var.aoc_base_scenario == "infra" && var.deployment_type == "fargate" ? 1 : 0
-  yaml_body = templatefile("./container-insights-agent/stateful_set_fargate.tpl",
-  { ClusterName : var.eks_cluster_name, AocRepo : var.aoc_image_repo, AocTag : var.aoc_version })
+  yaml_body = templatefile("./container-insights-agent/stateful_set_fargate.yml",
+  { ClusterName : var.eks_cluster_name, AocRepo : var.aoc_image_repo, AocTag : var.aoc_version, Namespace : tolist(aws_eks_fargate_profile.test_profile.selector)[0].namespace })
   depends_on = [
-    kubectl_manifest.aoc_service_deploy
+    kubectl_manifest.aoc_service_deploy,
+    aws_eks_fargate_profile.test_profile
   ]
 }
 
 resource "kubectl_manifest" "logs_sample_fargate_deploy" {
   count     = var.aoc_base_scenario == "infra" && var.deployment_type == "fargate" ? 1 : 0
-  yaml_body = file("./container-insights-agent/logs_sample_fargate.yml")
+  yaml_body = templatefile("./container-insights-agent/logs_sample_fargate.yml", { Namespace : tolist(aws_eks_fargate_profile.test_profile.selector)[0].namespace })
   depends_on = [
-    kubectl_manifest.aoc_fargate_deploy
+    kubectl_manifest.aoc_fargate_deploy,
+    aws_eks_fargate_profile.test_profile
   ]
 }

terraform/eks/main.tf

@@ -102,6 +102,54 @@ resource "kubernetes_namespace" "aoc_ns" {
   }
 }
 
+# create a unique fargate namespace for each run
+resource "kubernetes_namespace" "aoc_fargate_ns" {
+  metadata {
+    name = "aoc-fargate-ns-${module.common.testing_id}"
+  }
+}
+
+resource "aws_iam_role" "fargate_profile_file" {
+  name                = "fargate-profile-role-${module.common.testing_id}"
+  managed_policy_arns = ["arn:aws:iam::aws:policy/AmazonEKSFargatePodExecutionRolePolicy"]
+
+  # Terraform's "jsonencode" function converts a
+  # Terraform expression result to valid JSON syntax.
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = "sts:AssumeRole"
+        Effect = "Allow"
+        Principal = {
+          Service = "eks-fargate-pods.amazonaws.com"
+        }
+      },
+    ]
+  })
+}
+
+data "aws_subnet_ids" "private_subnets" {
+  vpc_id = data.aws_eks_cluster.testing_cluster.vpc_config[0].vpc_id
+  filter {
+    name   = "mapPublicIpOnLaunch"
+    values = ["false"] # insert values here
+  }
+}
+
+resource "aws_eks_fargate_profile" "test_profile" {
+  cluster_name           = var.eks_cluster_name
+  fargate_profile_name   = "fp-aoc-${module.common.testing_id}"
+  pod_execution_role_arn = aws_iam_role.fargate_profile_file.arn
+  subnet_ids             = data.aws_subnet_ids.private_subnets.ids
+
+  selector {
+    namespace = kubernetes_namespace.aoc_fargate_ns.metadata[0].name
+  }
+
+  depends_on = [aws_iam_role.fargate_profile_file, kubernetes_namespace.aoc_fargate_ns]
+}
+
 resource "kubernetes_service_account" "aoc-role" {
   metadata {
     name      = "aoc-role-${module.common.testing_id}"
@@ -114,14 +162,14 @@ resource "kubernetes_service_account" "aoc-role" {
 resource "kubernetes_service_account" "aoc-fargate-role" {
   metadata {
     name      = "aoc-fargate-role-${module.common.testing_id}"
-    namespace = "default"
+    namespace = tolist(aws_eks_fargate_profile.test_profile.selector)[0].namespace
     annotations = {
       "eks.amazonaws.com/role-arn" : module.iam_assumable_role_admin.iam_role_arn
     }
   }
 
   automount_service_account_token = true
-  depends_on                      = [module.iam_assumable_role_admin]
+  depends_on                      = [module.iam_assumable_role_admin, aws_eks_fargate_profile.test_profile]
 }
 
 module "iam_assumable_role_admin" {
@@ -152,17 +200,19 @@ resource "kubernetes_cluster_role_binding" "aoc-role-binding" {
   subject {
     kind      = "ServiceAccount"
     name      = var.deployment_type == "fargate" ? "aoc-fargate-role-${module.common.testing_id}" : "aoc-role-${module.common.testing_id}"
-    namespace = var.deployment_type == "fargate" ? "default" : kubernetes_namespace.aoc_ns.metadata[0].name
+    namespace = var.deployment_type == "fargate" ? tolist(aws_eks_fargate_profile.test_profile.selector)[0].namespace : kubernetes_namespace.aoc_ns.metadata[0].name
   }
+  depends_on = [aws_eks_fargate_profile.test_profile]
 }
 
 resource "kubernetes_service_account" "aoc-agent-role" {
   metadata {
     name      = "aoc-agent-${module.common.testing_id}"
-    namespace = var.deployment_type == "fargate" ? "default" : kubernetes_namespace.aoc_ns.metadata[0].name
+    namespace = var.deployment_type == "fargate" ? tolist(aws_eks_fargate_profile.test_profile.selector)[0].namespace : kubernetes_namespace.aoc_ns.metadata[0].name
   }
 
   automount_service_account_token = true
+  depends_on                      = [aws_eks_fargate_profile.test_profile]
 }
 
 module "adot_operator" {
@@ -222,5 +272,6 @@ module "validator" {
 
   depends_on = [
     module.aoc_oltp,
-  module.adot_operator]
+    module.adot_operator,
+  kubectl_manifest.logs_sample_fargate_deploy]
 }