Update iam policies for csi driver (#9)

* adding tolerations to chart

* adding tolerations in a different way

* fixing the name for tolerations

* fixing tolerations to be in the controller

* adding defaults

* updating iam policy for the driver

* updating to actions

* fmt

* updating docs

* simple code refactor

* simple code refactor

* adding tolerations to chart

* changing back tolerations

* fixing service account names

* Update iam.tf

Co-authored-by: Dustin Whited <dgwhited@gmail.com>

* Update variables.tf

Co-authored-by: Charles Bushong <bushong1@gmail.com>

* adding ingress rules

* making the tolerations dynamic

* refactoring

* fmting

---------

Co-authored-by: Arun Sanna <arunsanna@Aruns-MacBook-Pro.local>
Co-authored-by: Arun Sanna <arunchowdary.sanna@cms.hhs.gov>
Co-authored-by: Dustin Whited <dgwhited@gmail.com>
Co-authored-by: Charles Bushong <bushong1@gmail.com>

Author: 5 people

README.md

@@ -1,7 +1,7 @@
 # batcave-tf-efs
 ## Requirements
 
- * Aws elastic file system should be created and file system id shoud be supplied to this chart
+No requirements.
 
 ## Providers
 
@@ -12,29 +12,50 @@
 
 ## Modules
 
-No modules.
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_iam_assumable_role_admin"></a> [iam\_assumable\_role\_admin](#module\_iam\_assumable\_role\_admin) | terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc | n/a |
 
 ## Resources
 
 | Name | Type |
 |------|------|
+| [aws_efs_backup_policy.policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/efs_backup_policy) | resource |
+| [aws_efs_file_system.efs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/efs_file_system) | resource |
+| [aws_efs_mount_target.efs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/efs_mount_target) | resource |
 | [aws_iam_policy.batcave_efscsidriver](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
-| [aws_iam_role_policy_attachment.efsscidriver_policy_attachment](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
-| [helm_release.aws_efs_csi_diver](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
-| [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source |
+| [aws_security_group.efs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource |
+| [helm_release.aws-efs-csi-driver](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
+| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
+| [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source |
+| [aws_iam_policy_document.batcave_efscsidriver](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_kms_key.efs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/kms_key) | data source |
 
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
+| <a name="input_cluster_certificate_authority_data"></a> [cluster\_certificate\_authority\_data](#input\_cluster\_certificate\_authority\_data) | n/a | `any` | n/a | yes |
+| <a name="input_cluster_endpoint"></a> [cluster\_endpoint](#input\_cluster\_endpoint) | n/a | `any` | n/a | yes |
 | <a name="input_cluster_name"></a> [cluster\_name](#input\_cluster\_name) | n/a | `any` | n/a | yes |
+| <a name="input_cluster_oidc_issuer_url"></a> [cluster\_oidc\_issuer\_url](#input\_cluster\_oidc\_issuer\_url) | n/a | `any` | n/a | yes |
+| <a name="input_efsid"></a> [efsid](#input\_efsid) | n/a | `string` | `""` | no |
+| <a name="input_helm_name"></a> [helm\_name](#input\_helm\_name) | n/a | `string` | `"aws-efs-csi-driver"` | no |
 | <a name="input_helm_namespace"></a> [helm\_namespace](#input\_helm\_namespace) | ## Helm variables | `string` | `"kube-system"` | no |
 | <a name="input_iam_path"></a> [iam\_path](#input\_iam\_path) | n/a | `string` | `"/delegatedadmin/developer/"` | no |
-| <a name="input_permissions_boundary"></a> [permissions\_boundary](#input\_permissions\_boundary) | n/a | `string` | `"arn:aws:iam::373346310182:policy/cms-cloud-admin/developer-boundary-policy"` | no |
-| <a name="input_provider_url"></a> [provider\_url](#input\_provider\_url) | n/a | `string` | `""` | no |
-| <a name="input_efsid"></a> efsid | n/a | `elasticfilesystem` | n/a | yes |
-
+| <a name="input_imagerepo"></a> [imagerepo](#input\_imagerepo) | # Image repo | `string` | `"602401143452.dkr.ecr.us-west-2.amazonaws.com/eks/aws-efs-csi-driver"` | no |
+| <a name="input_kms_key_id"></a> [kms\_key\_id](#input\_kms\_key\_id) | n/a | `string` | `""` | no |
+| <a name="input_permissions_boundary"></a> [permissions\_boundary](#input\_permissions\_boundary) | n/a | `string` | `""` | no |
+| <a name="input_private_subnet_ids"></a> [private\_subnet\_ids](#input\_private\_subnet\_ids) | n/a | `list(any)` | `[]` | no |
+| <a name="input_toleration_effect"></a> [toleration\_effect](#input\_toleration\_effect) | toleration effect | `string` | `""` | no |
+| <a name="input_toleration_key"></a> [toleration\_key](#input\_toleration\_key) | toleration key | `string` | `""` | no |
+| <a name="input_toleration_operator"></a> [toleration\_operator](#input\_toleration\_operator) | toleration operator | `string` | `""` | no |
+| <a name="input_toleration_value"></a> [toleration\_value](#input\_toleration\_value) | toleration value | `string` | `""` | no |
+| <a name="input_vpc_id"></a> [vpc\_id](#input\_vpc\_id) | n/a | `string` | `""` | no |
 
 ## Outputs
 
-No outputs.
+| Name | Description |
+|------|-------------|
+| <a name="output_oidc_iam_role_arn"></a> [oidc\_iam\_role\_arn](#output\_oidc\_iam\_role\_arn) | n/a |
+<!-- END_TF_DOCS -->

csi-efs.tf

@@ -11,7 +11,11 @@ resource "helm_release" "aws-efs-csi-driver" {
   }
   set {
     name  = "controller.serviceAccount.name"
-    value = "efs-csi-controller-sa"
+    value = local.controller_service_account_name
+  }
+  set {
+    name  = "node.serviceAccount.name"
+    value = local.node_service_account_name
   }
   set {
     name  = "controller.serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn"
@@ -68,20 +72,32 @@ resource "helm_release" "aws-efs-csi-driver" {
     name  = "storageClasses[0].volumeBindingMode"
     value = "Immediate"
   }
-  set {
-    name  = "controller.tolerations[0].key"
-    value = var.toleration_key
-  }
-  set {
-    name  = "controller.tolerations[0].value"
-    value = var.toleration_value
-  }
-  set {
-    name  = "controller.tolerations[0].operator"
-    value = var.toleration_operator
-  }
-  set {
-    name  = "controller.tolerations[0].effect"
-    value = var.toleration_effect
+  dynamic "set" {
+    for_each = var.tolerations
+    content {
+      name  = "controller.tolerations[${set.key}].key"
+      value = set.value.key
+    }
+  }
+  dynamic "set" {
+    for_each = var.tolerations
+    content {
+      name  = "controller.tolerations[${set.key}].value"
+      value = try(set.value.value, "")
+    }
+  }
+  dynamic "set" {
+    for_each = var.tolerations
+    content {
+      name  = "controller.tolerations[${set.key}].operator"
+      value = set.value.operator
+    }
+  }
+  dynamic "set" {
+    for_each = var.tolerations
+    content {
+      name  = "controller.tolerations[${set.key}].effect"
+      value = try(set.value.effect, "NoSchedule")
+    }
   }
 }

efs.tf

@@ -3,6 +3,15 @@ resource "aws_security_group" "efs" {
   vpc_id      = var.vpc_id
 }
 
+resource "aws_security_group_rule" "efs_ingress" {
+  security_group_id        = aws_security_group.efs.id
+  type                     = "ingress"
+  from_port                = 2049
+  to_port                  = 2049
+  protocol                 = "tcp"
+  source_security_group_id = var.worker_security_group_id
+}
+
 data "aws_kms_key" "efs" {
   key_id = var.kms_key_id
 }
@@ -14,23 +23,25 @@ resource "aws_efs_file_system" "efs" {
   kms_key_id = data.aws_kms_key.efs.arn
 
   tags = {
-    Name = "${var.cluster_name}"
+    Name = var.cluster_name
   }
+
   lifecycle_policy {
     transition_to_ia = "AFTER_60_DAYS"
   }
 }
 
 resource "aws_efs_mount_target" "efs" {
-  for_each        = toset(var.private_subnet_ids)
+  count           = length(var.private_subnet_ids)
   file_system_id  = aws_efs_file_system.efs.id
-  subnet_id       = each.key
+  subnet_id       = var.private_subnet_ids[count.index]
   security_groups = [aws_security_group.efs.id]
 }
+
 resource "aws_efs_backup_policy" "policy" {
   file_system_id = aws_efs_file_system.efs.id
 
   backup_policy {
     status = "ENABLED"
   }
-}
+}

iam.tf

@@ -1,21 +1,51 @@
 locals {
-  k8s_service_account_namespace = "kube-system"
-  k8s_service_account_name      = "aws-efs-csi-driver"
+  k8s_service_account_namespace   = "kube-system"
+  controller_service_account_name = "efs-csi-controller-sa"
+  node_service_account_name       = "efs-csi-node-sa"
 }
 
 data "aws_caller_identity" "current" {}
 
 data "aws_iam_policy_document" "batcave_efscsidriver" {
+  # Allow EFS access
   statement {
+    effect = "Allow"
     actions = [
       "elasticfilesystem:DescribeAccessPoints",
       "elasticfilesystem:DescribeFileSystems",
+      "elasticfilesystem:DescribeMountTargets",
+      "ec2:DescribeAvailabilityZones"
     ]
-
     resources = ["*"]
-
   }
+  # Allow creating EFS access points with specific tags
   statement {
+    effect = "Allow"
+    actions = [
+      "elasticfilesystem:CreateAccessPoint",
+      "elasticfilesystem:TagResource"
+    ]
+    resources = ["*"]
+    condition {
+      test     = "StringLike"
+      variable = "aws:RequestTag/efs.csi.aws.com/cluster"
+      values   = ["true"]
+    }
+  }
+  # Allow deleting EFS access points with specific tags
+  statement {
+    effect    = "Allow"
+    actions   = ["elasticfilesystem:DeleteAccessPoint"]
+    resources = ["*"]
+    condition {
+      test     = "StringLike"
+      variable = "aws:ResourceTag/efs.csi.aws.com/cluster"
+      values   = ["true"]
+    }
+  }
+  # Allow creating and deleting EFS resources with specific ARNs
+  statement {
+    effect = "Allow"
     actions = [
       "elasticfilesystem:CreateAccessPoint",
       "elasticfilesystem:DeleteAccessPoint"
@@ -25,21 +55,24 @@ data "aws_iam_policy_document" "batcave_efscsidriver" {
       "arn:aws:elasticfilesystem:*:${data.aws_caller_identity.current.account_id}:access-point/*"
     ]
   }
-
 }
+
 resource "aws_iam_policy" "batcave_efscsidriver" {
   name   = "efscsidriver-policy-${var.cluster_name}"
   path   = var.iam_path
   policy = data.aws_iam_policy_document.batcave_efscsidriver.json
-
 }
+
 module "iam_assumable_role_admin" {
-  source                        = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc"
-  create_role                   = true
-  role_name                     = "${var.cluster_name}-cluster-efscsidriver"
-  provider_url                  = replace(var.cluster_oidc_issuer_url, "https://", "")
-  role_policy_arns              = [aws_iam_policy.batcave_efscsidriver.arn]
-  oidc_fully_qualified_subjects = ["system:serviceaccount:${local.k8s_service_account_namespace}:efs-csi-controller-sa", "system:serviceaccount:${local.k8s_service_account_namespace}:efs-csi-node-sa"]
+  source           = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc"
+  create_role      = true
+  role_name        = "${var.cluster_name}-cluster-efscsidriver"
+  provider_url     = replace(var.cluster_oidc_issuer_url, "https://", "")
+  role_policy_arns = [aws_iam_policy.batcave_efscsidriver.arn]
+  oidc_fully_qualified_subjects = [
+    "system:serviceaccount:${local.k8s_service_account_namespace}:${local.controller_service_account_name}",
+    "system:serviceaccount:${local.k8s_service_account_namespace}:${local.node_service_account_name}"
+  ]
   role_path                     = var.iam_path
   role_permissions_boundary_arn = var.permissions_boundary
 }

main.tf

@@ -1,13 +1,15 @@
 data "aws_eks_cluster_auth" "cluster" {
   name = var.cluster_name
 }
+
 provider "helm" {
   kubernetes {
     host                   = var.cluster_endpoint
     cluster_ca_certificate = base64decode(var.cluster_certificate_authority_data)
     token                  = data.aws_eks_cluster_auth.cluster.token
   }
 }
+
 provider "kubernetes" {
   host                   = var.cluster_endpoint
   cluster_ca_certificate = base64decode(var.cluster_certificate_authority_data)

variables.tf

@@ -1,72 +1,69 @@
-variable "cluster_name" {}
-variable "cluster_endpoint" {}
-variable "cluster_certificate_authority_data" {}
-
-
-
-variable "iam_path" {
-  default = "/delegatedadmin/developer/"
+variable "cluster_name" {
+  description = "Name of EKS cluster"
 }
 
-variable "permissions_boundary" {
-  default = ""
-  ## check this value in common.hcl file for dev
-  #arn:aws:iam::373346310182:policy/cms-cloud-admin/developer-boundary-policy
+variable "cluster_endpoint" {
+  description = "Endpoint for EKS cluster"
 }
 
-variable "toleration_key" {
-  type        = string
-  default     = ""
-  description = "toleration key"
+variable "cluster_certificate_authority_data" {
+  description = "CA certificate data for EKS cluster"
 }
 
-variable "toleration_value" {
-  type        = string
-  default     = ""
-  description = "toleration value"
+variable "iam_path" {
+  description = "Path for IAM roles"
+  default     = "/delegatedadmin/developer/"
 }
 
-variable "toleration_operator" {
-  type        = string
+variable "permissions_boundary" {
+  description = "Permissions boundary for IAM roles"
   default     = ""
-  description = "toleration operator"
 }
 
-variable "toleration_effect" {
-  type        = string
-  default     = ""
-  description = "toleration effect"
+### Helm variables
+variable "tolerations" {
+  type    = list(any)
+  default = []
 }
 
-### Helm variables
 variable "helm_namespace" {
-  default = "kube-system"
+  description = "Namespace for Helm chart"
+  default     = "kube-system"
 }
 
-## Image repo
 variable "imagerepo" {
-  default = "602401143452.dkr.ecr.us-west-2.amazonaws.com/eks/aws-efs-csi-driver"
+  description = "ECR repository for container images"
+  default     = "602401143452.dkr.ecr.us-west-2.amazonaws.com/eks/aws-efs-csi-driver"
 }
 
 variable "efsid" {
-  default = ""
+  description = "EFS filesystem ID"
+  default     = ""
 }
 
 variable "helm_name" {
-  default = "aws-efs-csi-driver"
+  description = "Name for Helm release"
+  default     = "aws-efs-csi-driver"
 }
 
-variable "cluster_oidc_issuer_url" {}
+variable "cluster_oidc_issuer_url" {
+  description = "OIDC issuer URL for EKS cluster"
+}
 
 variable "kms_key_id" {
-  default = ""
+  description = "KMS key ID for secrets encryption"
+  default     = ""
 }
 
 variable "vpc_id" {
-  default = ""
+  description = "VPC ID for EKS cluster"
 }
 
 variable "private_subnet_ids" {
   type    = list(any)
   default = []
 }
+
+variable "worker_security_group_id" {
+  type = string
+}