GitOps support with Grafana Operator, External Secrets and FluxCD (#151)

* Adding FluxCD and Grafana Operator Addons

* Adding Helm release for Grafana Operator

* Adding External Secrets

* removing spot options

* Fixing Documentation
---------

Author: elamaran11

docs/index.md

@@ -25,6 +25,9 @@ traces collection, dashboards and alerts for monitoring:
 - NGINX workloads (running on Amazon EKS)
 - Java/JMX workloads (running on Amazon EKS)
 - Amazon Managed Service for Prometheus workspaces with Amazon CloudWatch
+- Installs Grafana Operator to add AWS data sources and create Grafana Dashboards to Amazon Managed Grafana.
+- Installs FluxCD to perform GitOps sync of a Git Repo to EKS Cluster. We will use this later for creating Grafana Dashboards and AWS datasources to Amazon Managed Grafana.
+- Installs External Secrets Operator to retrieve and Sync the Grafana API keys.
 
 These modules can be directly configured in your existing Terraform
 configurations or ready to be deployed in our packaged

examples/existing-cluster-java/main.tf

@@ -69,6 +69,12 @@ module "eks_monitoring" {
   # enable java metrics collection, dashboards and alerts rules creation
   enable_java = true
 
+  # deploys external-secrets in to the cluster
+  enable_external_secrets = true
+  grafana_api_key         = var.grafana_api_key
+  target_secret_name      = "grafana-admin-credentials"
+  target_secret_namespace = "grafana-operator"
+
   eks_cluster_id = var.eks_cluster_id
 
   # control the publishing of dashboards by specifying the boolean value for the variable 'enable_dashboards', default is 'true'

examples/existing-cluster-nginx/main.tf

@@ -67,6 +67,12 @@ module "eks_monitoring" {
 
   eks_cluster_id = var.eks_cluster_id
 
+  # deploys external-secrets in to the cluster
+  enable_external_secrets = true
+  grafana_api_key         = var.grafana_api_key
+  target_secret_name      = "grafana-admin-credentials"
+  target_secret_namespace = "grafana-operator"
+
   # control the publishing of dashboards by specifying the boolean value for the variable 'enable_dashboards', default is 'true'
   # the intention to publish is overruled depending upon whether grafana dashboard folder is created by the observability accelerator
   enable_dashboards = module.aws_observability_accelerator.grafana_dashboard_folder_created ? var.enable_dashboards : false

examples/existing-cluster-with-base-and-infra/main.tf

@@ -77,6 +77,12 @@ module "eks_monitoring" {
   # reusing existing certificate manager? defaults to true
   enable_cert_manager = true
 
+  # deploys external-secrets in to the cluster
+  enable_external_secrets = true
+  grafana_api_key         = var.grafana_api_key
+  target_secret_name      = "grafana-admin-credentials"
+  target_secret_namespace = "grafana-operator"
+
   # control the publishing of dashboards by specifying the boolean value for the variable 'enable_dashboards', default is 'true'
   # the intention to publish is overruled depending upon whether grafana dashboard folder is created by the observability accelerator
   enable_dashboards = module.aws_observability_accelerator.grafana_dashboard_folder_created ? var.enable_dashboards : false

modules/eks-monitoring/README.md

@@ -4,7 +4,10 @@ This module provides EKS cluster monitoring with the following resources:
 
 - AWS Distro For OpenTelemetry Operator and Collector for Metrics and Traces
 - Logs with [AWS for FluentBit](https://github.com/aws/aws-for-fluent-bit)
-- AWS Managed Grafana Dashboard and data source
+- Installs Grafana Operator to add AWS data sources and create Grafana Dashboards to Amazon Managed Grafana.
+- Installs FluxCD to perform GitOps sync of a Git Repo to EKS Cluster. We will use this later for creating Grafana Dashboards and AWS datasources to Amazon Managed Grafana.
+- Installs External Secrets Operator to retrieve and Sync the Grafana API keys.
+- Amazon Managed Grafana Dashboard and data source
 - Alerts and recording rules with AWS Managed Service for Prometheus
 
 This module makes use of the open source [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack)
@@ -33,6 +36,7 @@ This module makes use of the open source [kube-prometheus-stack](https://github.
 
 | Name | Source | Version |
 |------|--------|---------|
+| <a name="module_external_secrets"></a> [external\_secrets](#module\_external\_secrets) | ./add-ons/external-secrets | n/a |
 | <a name="module_fluentbit_logs"></a> [fluentbit\_logs](#module\_fluentbit\_logs) | ./add-ons/aws-for-fluentbit | n/a |
 | <a name="module_helm_addon"></a> [helm\_addon](#module\_helm\_addon) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.26.0 |
 | <a name="module_java_monitoring"></a> [java\_monitoring](#module\_java\_monitoring) | ./patterns/java | n/a |
@@ -51,6 +55,8 @@ This module makes use of the open source [kube-prometheus-stack](https://github.
 | [grafana_dashboard.nodes](https://registry.terraform.io/providers/grafana/grafana/latest/docs/resources/dashboard) | resource |
 | [grafana_dashboard.nsworkload](https://registry.terraform.io/providers/grafana/grafana/latest/docs/resources/dashboard) | resource |
 | [grafana_dashboard.workloads](https://registry.terraform.io/providers/grafana/grafana/latest/docs/resources/dashboard) | resource |
+| [helm_release.fluxcd](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
+| [helm_release.grafana_operator](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
 | [helm_release.kube_state_metrics](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
 | [helm_release.prometheus_node_exporter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
 | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
@@ -70,13 +76,19 @@ This module makes use of the open source [kube-prometheus-stack](https://github.
 | <a name="input_enable_cert_manager"></a> [enable\_cert\_manager](#input\_enable\_cert\_manager) | Allow reusing an existing installation of cert-manager | `bool` | `true` | no |
 | <a name="input_enable_custom_metrics"></a> [enable\_custom\_metrics](#input\_enable\_custom\_metrics) | Allows additional metrics collection for config elements in the `custom_metrics_config` config object. Automatic dashboards are not included | `bool` | `false` | no |
 | <a name="input_enable_dashboards"></a> [enable\_dashboards](#input\_enable\_dashboards) | Enables or disables curated dashboards | `bool` | `true` | no |
+| <a name="input_enable_external_secrets"></a> [enable\_external\_secrets](#input\_enable\_external\_secrets) | Installs External Secrets to EKS Cluster | `bool` | `true` | no |
+| <a name="input_enable_fluxcd"></a> [enable\_fluxcd](#input\_enable\_fluxcd) | Enables or disables FluxCD. Disabling this might affect some data in the dashboards | `bool` | `true` | no |
+| <a name="input_enable_grafana_operator"></a> [enable\_grafana\_operator](#input\_enable\_grafana\_operator) | Deploys Grafana Operator to EKS Cluster | `bool` | `true` | no |
 | <a name="input_enable_java"></a> [enable\_java](#input\_enable\_java) | Enable Java workloads monitoring, alerting and default dashboards | `bool` | `false` | no |
 | <a name="input_enable_kube_state_metrics"></a> [enable\_kube\_state\_metrics](#input\_enable\_kube\_state\_metrics) | Enables or disables Kube State metrics exporter. Disabling this might affect some data in the dashboards | `bool` | `true` | no |
 | <a name="input_enable_logs"></a> [enable\_logs](#input\_enable\_logs) | Using AWS For FluentBit to collect cluster and application logs to Amazon CloudWatch | `bool` | `true` | no |
 | <a name="input_enable_nginx"></a> [enable\_nginx](#input\_enable\_nginx) | Enable NGINX workloads monitoring, alerting and default dashboards | `bool` | `false` | no |
 | <a name="input_enable_node_exporter"></a> [enable\_node\_exporter](#input\_enable\_node\_exporter) | Enables or disables Node exporter. Disabling this might affect some data in the dashboards | `bool` | `true` | no |
 | <a name="input_enable_recording_rules"></a> [enable\_recording\_rules](#input\_enable\_recording\_rules) | Enables or disables Managed Prometheus recording rules | `bool` | `true` | no |
 | <a name="input_enable_tracing"></a> [enable\_tracing](#input\_enable\_tracing) | (Experimental) Enables tracing with AWS X-Ray. This changes the deploy mode of the collector to daemon set. Requirement: adot add-on <= 0.58-build.0 | `bool` | `false` | no |
+| <a name="input_flux_config"></a> [flux\_config](#input\_flux\_config) | FluxCD configuration | <pre>object({<br>    create_namespace   = bool<br>    k8s_namespace      = string<br>    helm_chart_name    = string<br>    helm_chart_version = string<br>    helm_release_name  = string<br>    helm_repo_url      = string<br>    helm_settings      = map(string)<br>    helm_values        = map(any)<br>  })</pre> | <pre>{<br>  "create_namespace": true,<br>  "helm_chart_name": "flux2",<br>  "helm_chart_version": "2.7.0",<br>  "helm_release_name": "observability-fluxcd-addon",<br>  "helm_repo_url": "https://fluxcd-community.github.io/helm-charts",<br>  "helm_settings": {},<br>  "helm_values": {},<br>  "k8s_namespace": "flux-system"<br>}</pre> | no |
+| <a name="input_go_config"></a> [go\_config](#input\_go\_config) | Grafana Operator configuration | <pre>object({<br>    create_namespace   = bool<br>    helm_chart         = string<br>    helm_name          = string<br>    k8s_namespace      = string<br>    helm_release_name  = string<br>    helm_chart_version = string<br>  })</pre> | <pre>{<br>  "create_namespace": true,<br>  "helm_chart": "oci://ghcr.io/grafana-operator/helm-charts/grafana-operator",<br>  "helm_chart_version": "v5.0.0-rc1",<br>  "helm_name": "grafana-operator",<br>  "helm_release_name": "grafana-operator",<br>  "k8s_namespace": "grafana-operator"<br>}</pre> | no |
+| <a name="input_grafana_api_key"></a> [grafana\_api\_key](#input\_grafana\_api\_key) | Grafana API key for the Amazon Managed Grafana workspace | `string` | n/a | yes |
 | <a name="input_helm_config"></a> [helm\_config](#input\_helm\_config) | Helm Config for Prometheus | `any` | `{}` | no |
 | <a name="input_irsa_iam_permissions_boundary"></a> [irsa\_iam\_permissions\_boundary](#input\_irsa\_iam\_permissions\_boundary) | IAM permissions boundary for IRSA roles | `string` | `null` | no |
 | <a name="input_irsa_iam_role_path"></a> [irsa\_iam\_role\_path](#input\_irsa\_iam\_role\_path) | IAM role path for IRSA roles | `string` | `"/"` | no |
@@ -90,6 +102,8 @@ This module makes use of the open source [kube-prometheus-stack](https://github.
 | <a name="input_nginx_config"></a> [nginx\_config](#input\_nginx\_config) | Configuration object for NGINX monitoring | <pre>object({<br>    enable_alerting_rules       = bool<br>    scrape_sample_limit         = number<br>    prometheus_metrics_endpoint = string<br>  })</pre> | <pre>{<br>  "enable_alerting_rules": true,<br>  "prometheus_metrics_endpoint": "metrics",<br>  "scrape_sample_limit": 1000<br>}</pre> | no |
 | <a name="input_prometheus_config"></a> [prometheus\_config](#input\_prometheus\_config) | Controls default values such as scrape interval, timeouts and ports globally | <pre>object({<br>    global_scrape_interval = string<br>    global_scrape_timeout  = string<br>  })</pre> | <pre>{<br>  "global_scrape_interval": "60s",<br>  "global_scrape_timeout": "15s"<br>}</pre> | no |
 | <a name="input_tags"></a> [tags](#input\_tags) | Additional tags (e.g. `map('BusinessUnit`,`XYZ`) | `map(string)` | `{}` | no |
+| <a name="input_target_secret_name"></a> [target\_secret\_name](#input\_target\_secret\_name) | Target secret in Kubernetes to store the Grafana API Key Secret | `string` | `"grafana-admin-credentials"` | no |
+| <a name="input_target_secret_namespace"></a> [target\_secret\_namespace](#input\_target\_secret\_namespace) | Target namespace of secret in Kubernetes to store the Grafana API Key Secret | `string` | `"grafana-operator"` | no |
 | <a name="input_tracing_config"></a> [tracing\_config](#input\_tracing\_config) | Configuration object for traces collection to AWS X-Ray | <pre>object({<br>    otlp_grpc_endpoint = string<br>    otlp_http_endpoint = string<br>    send_batch_size    = number<br>    timeout            = string<br>  })</pre> | <pre>{<br>  "otlp_grpc_endpoint": "0.0.0.0:4317",<br>  "otlp_http_endpoint": "0.0.0.0:4318",<br>  "send_batch_size": 50,<br>  "timeout": "30s"<br>}</pre> | no |
 
 ## Outputs

modules/eks-monitoring/add-ons/external-secrets/README.md

@@ -0,0 +1,55 @@
+# External Secrets Operator Kubernetes addon
+
+This deploys an EKS Cluster with the External Secrets Operator. The cluster is populated with a ClusterSecretStore and ExternalSecret using Grafana API Key secret from AWS Secret Manager. A secret store for each AWS Secret Manager is created. Store use IRSA (IAM Roles For Service Account) to retrieve the secret values from AWS.
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 3.72 |
+| <a name="requirement_kubectl"></a> [kubectl](#requirement\_kubectl) | >= 1.14 |
+| <a name="requirement_kubernetes"></a> [kubernetes](#requirement\_kubernetes) | >= 2.10 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 3.72 |
+| <a name="provider_kubectl"></a> [kubectl](#provider\_kubectl) | >= 1.14 |
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_cluster_secretstore_role"></a> [cluster\_secretstore\_role](#module\_cluster\_secretstore\_role) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/irsa | v4.28.0 |
+| <a name="module_external_secrets"></a> [external\_secrets](#module\_external\_secrets) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/external-secrets | v4.28.0 |
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [aws_iam_policy.cluster_secretstore](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
+| [aws_kms_key.secrets](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource |
+| [aws_secretsmanager_secret.secret](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret) | resource |
+| [aws_secretsmanager_secret_version.secret](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret_version) | resource |
+| [kubectl_manifest.cluster_secretstore](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource |
+| [kubectl_manifest.secret](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource |
+| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_addon_context"></a> [addon\_context](#input\_addon\_context) | Input configuration for the addon | <pre>object({<br>    aws_caller_identity_account_id = string<br>    aws_caller_identity_arn        = string<br>    aws_eks_cluster_endpoint       = string<br>    aws_partition_id               = string<br>    aws_region_name                = string<br>    eks_cluster_id                 = string<br>    eks_oidc_issuer_url            = string<br>    eks_oidc_provider_arn          = string<br>    irsa_iam_role_path             = string<br>    irsa_iam_permissions_boundary  = string<br>    tags                           = map(string)<br>  })</pre> | n/a | yes |
+| <a name="input_enable_external_secrets"></a> [enable\_external\_secrets](#input\_enable\_external\_secrets) | Enable external-secrets | `bool` | `true` | no |
+| <a name="input_grafana_api_key"></a> [grafana\_api\_key](#input\_grafana\_api\_key) | Grafana API key for the Amazon Managed Grafana workspace | `string` | n/a | yes |
+| <a name="input_helm_config"></a> [helm\_config](#input\_helm\_config) | Helm provider config for external secrets | `any` | `{}` | no |
+| <a name="input_target_secret_name"></a> [target\_secret\_name](#input\_target\_secret\_name) | Name to store the secret for Grafana API Key | `string` | n/a | yes |
+| <a name="input_target_secret_namespace"></a> [target\_secret\_namespace](#input\_target\_secret\_namespace) | Namespace to store the secret for Grafana API Key | `string` | n/a | yes |
+
+## Outputs
+
+No outputs.
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

modules/eks-monitoring/add-ons/external-secrets/locals.tf

@@ -0,0 +1,6 @@
+locals {
+  name                     = "external-secrets"
+  namespace                = "external-secrets"
+  cluster_secretstore_name = "cluster-secretstore-sm"
+  cluster_secretstore_sa   = "cluster-secretstore-sa"
+}

modules/eks-monitoring/add-ons/external-secrets/main.tf

@@ -0,0 +1,109 @@
+module "external_secrets" {
+  source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/external-secrets?ref=v4.28.0"
+  count  = var.enable_external_secrets ? 1 : 0
+
+  helm_config   = var.helm_config
+  addon_context = var.addon_context
+}
+
+data "aws_region" "current" {}
+
+#---------------------------------------------------------------
+# External Secrets Operator - Secret
+#---------------------------------------------------------------
+
+resource "aws_kms_key" "secrets" {
+  enable_key_rotation = true
+}
+
+module "cluster_secretstore_role" {
+  source                      = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/irsa?ref=v4.28.0"
+  kubernetes_namespace        = local.namespace
+  create_kubernetes_namespace = false
+  kubernetes_service_account  = local.cluster_secretstore_sa
+  irsa_iam_policies           = [aws_iam_policy.cluster_secretstore.arn]
+  eks_cluster_id              = var.addon_context.eks_cluster_id
+  eks_oidc_provider_arn       = var.addon_context.eks_oidc_provider_arn
+  depends_on                  = [module.external_secrets]
+}
+
+resource "aws_iam_policy" "cluster_secretstore" {
+  name_prefix = local.cluster_secretstore_sa
+  policy      = <<POLICY
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "secretsmanager:GetResourcePolicy",
+        "secretsmanager:GetSecretValue",
+        "secretsmanager:DescribeSecret",
+        "secretsmanager:ListSecretVersionIds"
+      ],
+      "Resource": "${aws_secretsmanager_secret.secret.arn}"
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "kms:Decrypt"
+      ],
+      "Resource": "${aws_kms_key.secrets.arn}"
+    }
+  ]
+}
+POLICY
+}
+
+resource "kubectl_manifest" "cluster_secretstore" {
+  yaml_body  = <<YAML
+apiVersion: external-secrets.io/v1beta1
+kind: ClusterSecretStore
+metadata:
+  name: ${local.cluster_secretstore_name}
+spec:
+  provider:
+    aws:
+      service: SecretsManager
+      region: ${data.aws_region.current.name}
+      auth:
+        jwt:
+          serviceAccountRef:
+            name: ${local.cluster_secretstore_sa}
+            namespace: ${local.namespace}
+YAML
+  depends_on = [module.external_secrets]
+}
+
+resource "aws_secretsmanager_secret" "secret" {
+  recovery_window_in_days = 0
+  kms_key_id              = aws_kms_key.secrets.arn
+}
+
+resource "aws_secretsmanager_secret_version" "secret" {
+  secret_id = aws_secretsmanager_secret.secret.id
+  secret_string = jsonencode({
+    GF_SECURITY_ADMIN_APIKEY = var.grafana_api_key
+  })
+}
+
+resource "kubectl_manifest" "secret" {
+  yaml_body  = <<YAML
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: ${local.name}-sm
+  namespace: ${var.target_secret_namespace}
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: ${local.cluster_secretstore_name}
+    kind: ClusterSecretStore
+  target:
+    name: ${var.target_secret_name}
+  dataFrom:
+  - extract:
+      key: ${aws_secretsmanager_secret.secret.name}
+YAML
+  depends_on = [module.external_secrets]
+}