diff --git a/cloud/aws/README.md b/cloud/aws/README.md index f867cce..17854cf 100644 --- a/cloud/aws/README.md +++ b/cloud/aws/README.md @@ -1,5 +1,17 @@ # CLOUD AWS DataDog integrations +This Terraform module sets up the integration between AWS and Datadog monitoring service. +It creates an IAM role and policies in AWS that allow Datadog to access and collect metrics from your AWS resources, +and configures the Datadog AWS integration with customizable metric collection settings. + +Related documentation: [https://docs.datadoghq.com/integrations/amazon-web-services/#setup](https://docs.datadoghq.com/integrations/amazon-web-services/#setup) + +## Default behavior + +- Namespaces related to ElasticMapReduce, SQS, and Usage are excluded from monitoring to reduce noise +- EC2 and Lambda metrics are filtered to only include resources tagged with `claranet_monitored:true` to avoid unexpected costs +- Resource collection is enabled + ## How to use this module ```hcl @@ -7,23 +19,25 @@ module "datadog-integrations-cloud-aws" { source = "claranet/integrations/datadog//cloud/aws" version = "{revision}" - aws_account = var.aws_account + aws_account_id = var.aws_account } ``` + ## Requirements | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 0.12.31 | +| [terraform](#requirement\_terraform) | >= 1.11 | +| [aws](#requirement\_aws) | >= 6.0.0 | | [datadog](#requirement\_datadog) | >= 3.0.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | n/a | +| [aws](#provider\_aws) | >= 6.0.0 | | [datadog](#provider\_datadog) | >= 3.0.0 | ## Modules @@ -36,60 +50,30 @@ No modules. |------|------| | [aws_iam_policy.dd_integration_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_policy_attachment.allow_dd_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy_attachment) | resource | +| [aws_iam_policy_attachment.allow_security_audit_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy_attachment) | resource | | [aws_iam_role.dd_integration_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | -| [datadog_integration_aws.datadog_integration_aws](https://registry.terraform.io/providers/Datadog/datadog/latest/docs/resources/integration_aws) | resource | +| [datadog_integration_aws_account.main](https://registry.terraform.io/providers/Datadog/datadog/latest/docs/resources/integration_aws_account) | resource | | [aws_iam_policy_document.datadog_integration_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.dd_trust_relationship](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [datadog_integration_aws_external_id.main](https://registry.terraform.io/providers/Datadog/datadog/latest/docs/data-sources/integration_aws_external_id) | data source | ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [account\_specific\_namespace\_rules](#input\_account\_specific\_namespace\_rules) | Namespaces to limit metric collection for datadog aws integration | `map` | `{}` | no | -| [aws\_account](#input\_aws\_account) | n/a | `any` | n/a | yes | +| [aws\_account\_id](#input\_aws\_account\_id) | AWS account configuration for Datadog integration | `string` | n/a | yes | +| [aws\_iam\_role\_enabled](#input\_aws\_iam\_role\_enabled) | Enable IAM role deployment for Datadog AWS integration | `bool` | `true` | no | +| [aws\_partition](#input\_aws\_partition) | AWS partition for Datadog integration | `string` | `"aws"` | no | | [datadog\_aws\_account\_id](#input\_datadog\_aws\_account\_id) | AWS account\_id of Datadog | `string` | `"464622532012"` | no | -| [filter\_tags](#input\_filter\_tags) | Filters tags to limit metrics collection on EC2 for datadog aws integration | `list` | 
[
  "dd_monitoring:enabled"
]
| no | -| [host\_tags](#input\_host\_tags) | Tags to add all metrics retrieved from the datadog aws integration | `list` | `[]` | no | +| [metrics\_config](#input\_metrics\_config) | Metrics configuration for Datadog AWS integration | 
object({
    automute_enabled : optional(bool, true),
    collect_cloudwatch_alarms : optional(bool, false),
    collect_custom_metrics : optional(bool, false),
    enabled : optional(bool, true),
    namespace_filters : optional(object({
      exclude_only : optional(list(string), null),
      include_only : optional(list(string), null),
      }), {
      exclude_only = ["AWS/ElasticMapReduce", "AWS/SQS", "AWS/Usage"]
    }),
    tag_filters : optional(list(object({
      namespace : string,
      tags : list(string),
    })), [
      {
        namespace = "AWS/EC2"
        tags = ["claranet_monitored:true"]
      }, {
        namespace = "AWS/Lambda"
        tags = [
          "claranet_monitored:true",
        ]
      }
    ],
    )})
| `{}` | no | +| [metrics\_tags](#input\_metrics\_tags) | Tags to apply to metrics collected from AWS | `map(string)` | `{}` | no | +| [resource\_collection\_enabled](#input\_resource\_collection\_enabled) | Enable resource collection for Datadog AWS integration | `bool` | `true` | no | ## Outputs | Name | Description | |------|-------------| +| [aws\_integration\_id](#output\_aws\_integration\_id) | The ID of the DataDog AWS integration | | [aws\_role\_arn](#output\_aws\_role\_arn) | The role ARN of the DataDog integration | | [aws\_role\_name](#output\_aws\_role\_name) | The IAM role name of the DataDog integration | -## Related documentation - -DataDog documentation: [https://docs.datadoghq.com/integrations/amazon_web_services/#setup](https://docs.datadoghq.com/integrations/amazon_web_services/#setup) - -## Requirements - -You need to configure you AWS provider. -Credentials could be set in your `terraform.tfvars`. - -``` -variable "aws_region" { - type = string -} - -variable "aws_account" { - type = string -} - -variable "aws_access_key" { -} - -variable "aws_secret_key" { -} - -variable "aws_token" { -} - -provider "aws" { - region = var.aws_region - access_key = var.aws_access_key - secret_key = var.aws_secret_key - token = var.aws_token -} - -``` - + \ No newline at end of file diff --git a/cloud/aws/inputs.tf b/cloud/aws/inputs.tf deleted file mode 100644 index 77279d1..0000000 --- a/cloud/aws/inputs.tf +++ /dev/null @@ -1,24 +0,0 @@ -variable "aws_account" { -} - -variable "datadog_aws_account_id" { - description = "AWS account_id of Datadog" - type = string - default = "464622532012" -} - -variable "filter_tags" { - description = "Filters tags to limit metrics collection on EC2 for datadog aws integration" - default = ["dd_monitoring:enabled"] -} - -variable "host_tags" { - description = "Tags to add all metrics retrieved from the datadog aws integration" - default = [] -} - -variable "account_specific_namespace_rules" { - description = "Namespaces to limit metric collection for datadog aws integration" - default = {} -} - diff --git a/cloud/aws/integrations-aws.tf b/cloud/aws/integrations-aws.tf deleted file mode 100644 index 01f9506..0000000 --- a/cloud/aws/integrations-aws.tf +++ /dev/null @@ -1,8 +0,0 @@ -resource "datadog_integration_aws" "datadog_integration_aws" { - account_id = var.aws_account - role_name = local.role_name - filter_tags = var.filter_tags - host_tags = var.host_tags - account_specific_namespace_rules = var.account_specific_namespace_rules -} - diff --git a/cloud/aws/locals.tf b/cloud/aws/locals.tf index a24824a..23661ff 100644 --- a/cloud/aws/locals.tf +++ b/cloud/aws/locals.tf @@ -1,4 +1,4 @@ locals { - role_name = "DatadogAWSIntegrationRole" + policy_name = "claranet-datadog-integration-policy" + role_name = "claranet-datadog-integration-role" } - diff --git a/cloud/aws/outputs.tf b/cloud/aws/outputs.tf index 4a3b5de..ea99b93 100644 --- a/cloud/aws/outputs.tf +++ b/cloud/aws/outputs.tf @@ -1,10 +1,14 @@ output "aws_role_arn" { description = "The role ARN of the DataDog integration" - value = aws_iam_role.dd_integration_role.arn + value = try(aws_iam_role.dd_integration_role.arn, null) } output "aws_role_name" { description = "The IAM role name of the DataDog integration" - value = aws_iam_role.dd_integration_role.name + value = try(aws_iam_role.dd_integration_role.name, null) } +output "aws_integration_id" { + description = "The ID of the DataDog AWS integration" + value = datadog_integration_aws_account.main.id +} diff --git a/cloud/aws/policy.tf b/cloud/aws/policy.tf deleted file mode 100644 index 700e12f..0000000 --- a/cloud/aws/policy.tf +++ /dev/null @@ -1,88 +0,0 @@ -resource "aws_iam_policy" "dd_integration_policy" { - name = "DatadogAWSIntegrationPolicy" - path = "/" - description = "Datadog integration policy according to https://docs.datadoghq.com/integrations/aws/" - - policy = data.aws_iam_policy_document.datadog_integration_policy.json -} - -data "aws_iam_policy_document" "datadog_integration_policy" { - statement { - sid = "DatadogAWSIntegration" - effect = "Allow" - - actions = [ - "apigateway:GET", - "autoscaling:Describe*", - "budgets:ViewBudget", - "cloudfront:GetDistributionConfig", - "cloudfront:ListDistributions", - "cloudtrail:DescribeTrails", - "cloudtrail:GetTrailStatus", - "cloudwatch:Describe*", - "cloudwatch:Get*", - "cloudwatch:List*", - "codedeploy:BatchGet*", - "codedeploy:List*", - "directconnect:Describe*", - "dynamodb:Describe*", - "dynamodb:List*", - "ec2:Describe*", - "ecs:Describe*", - "ecs:List*", - "elasticache:Describe*", - "elasticache:List*", - "elasticfilesystem:DescribeAccessPoints", - "elasticfilesystem:DescribeFileSystems", - "elasticfilesystem:DescribeTags", - "elasticloadbalancing:Describe*", - "elasticmapreduce:Describe*", - "elasticmapreduce:List*", - "es:DescribeElasticsearchDomains", - "es:ListDomainNames", - "es:ListTags", - "health:DescribeAffectedEntities", - "health:DescribeEventDetails", - "health:DescribeEvents", - "kinesis:Describe*", - "kinesis:List*", - "lambda:AddPermission", - "lambda:GetPolicy", - "lambda:List*", - "lambda:RemovePermission", - "logs:DeleteSubscriptionFilter", - "logs:Describe*", - "logs:DescribeSubscriptionFilters", - "logs:FilterLogEvents", - "logs:Get*", - "logs:PutSubscriptionFilter", - "logs:TestMetricFilter", - "rds:Describe*", - "rds:List*", - "redshift:DescribeClusters", - "redshift:DescribeLoggingStatus", - "route53:List*", - "s3:GetBucketLocation", - "s3:GetBucketLogging", - "s3:GetBucketNotification", - "s3:GetBucketTagging", - "s3:ListAllMyBuckets", - "s3:PutBucketNotification", - "ses:Get*", - "sns:List*", - "sns:Publish", - "sqs:ListQueues", - "states:DescribeStateMachine", - "states:ListStateMachines", - "support:*", - "tag:GetResources", - "tag:GetTagKeys", - "tag:GetTagValues", - "xray:BatchGetTraces", - "xray:GetTraceSummaries", - ] - - resources = ["*"] - } -} - diff --git a/cloud/aws/r-aws-iam.tf b/cloud/aws/r-aws-iam.tf new file mode 100644 index 0000000..3a8ce51 --- /dev/null +++ b/cloud/aws/r-aws-iam.tf @@ -0,0 +1,200 @@ +data "datadog_integration_aws_external_id" "main" { + aws_account_id = var.aws_account_id + + lifecycle { + enabled = var.aws_iam_role_enabled + } +} + +resource "aws_iam_role" "dd_integration_role" { + name = local.role_name + description = "Datadog AWS Integration Role according to https://docs.datadoghq.com/integrations/aws" + + assume_role_policy = data.aws_iam_policy_document.dd_trust_relationship.json + + lifecycle { + enabled = var.aws_iam_role_enabled + } +} + +data "aws_iam_policy_document" "dd_trust_relationship" { + statement { + sid = "DatadogAWSTrustRelationship" + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "AWS" + + identifiers = [ + "arn:aws:iam::${var.datadog_aws_account_id}:root", + ] + } + + condition { + test = "StringEquals" + values = [data.datadog_integration_aws_external_id.main.external_id] + variable = "sts:ExternalId" + } + } + + lifecycle { + enabled = var.aws_iam_role_enabled + } +} + +resource "aws_iam_policy_attachment" "allow_dd_role" { + name = "Allow Datadog PolicyAccess via Role" + roles = [aws_iam_role.dd_integration_role.name] + policy_arn = aws_iam_policy.dd_integration_policy.arn + + lifecycle { + enabled = var.aws_iam_role_enabled + } +} + +resource "aws_iam_policy_attachment" "allow_security_audit_policy" { + name = "Allow SecurityAudit policy via Role" + roles = [aws_iam_role.dd_integration_role.name] + policy_arn = "arn:aws:iam::aws:policy/SecurityAudit" + + lifecycle { + enabled = var.aws_iam_role_enabled && var.resource_collection_enabled + } +} + +resource "aws_iam_policy" "dd_integration_policy" { + name = local.policy_name + path = "/" + description = "Datadog integration policy according to https://docs.datadoghq.com/integrations/aws/" + + policy = data.aws_iam_policy_document.datadog_integration_policy.json + + lifecycle { + enabled = var.aws_iam_role_enabled + } +} + +# Required permissions documented at https://docs.datadoghq.com/integrations/amazon-web-services/#aws-iam-permissions +data "aws_iam_policy_document" "datadog_integration_policy" { + statement { + sid = "DatadogAWSIntegration" + effect = "Allow" + + actions = [ + "account:GetAccountInformation", + "airflow:GetEnvironment", + "airflow:ListEnvironments", + "apigateway:GET", + "appsync:ListGraphqlApis", + "autoscaling:Describe*", + "backup:List*", + "batch:DescribeJobDefinitions", + "batch:DescribeJobQueues", + "batch:DescribeJobs", + "batch:ListJobs", + "bcm-data-exports:GetExport", + "bcm-data-exports:ListExports", + "budgets:ViewBudget", + "cloudfront:GetDistributionConfig", + "cloudfront:ListDistributions", + "cloudtrail:DescribeTrails", + "cloudtrail:GetTrail", + "cloudtrail:GetTrailStatus", + "cloudtrail:ListTrails", + "cloudtrail:LookupEvents", + "cloudwatch:Describe*", + "cloudwatch:Get*", + "cloudwatch:List*", + "codebuild:BatchGetProjects", + "codebuild:ListProjects", + "codedeploy:BatchGet*", + "codedeploy:List*", + "cur:DescribeReportDefinitions", + "directconnect:Describe*", + "dms:DescribeReplicationInstances", + "dynamodb:Describe*", + "dynamodb:List*", + "ec2:Describe*", + "ecs:Describe*", + "ecs:List*", + "eks:DescribeCluster", + "eks:ListClusters", + "elasticache:Describe*", + "elasticache:List*", + "elasticfilesystem:DescribeAccessPoints", + "elasticfilesystem:DescribeFileSystems", + "elasticfilesystem:DescribeTags", + "elasticloadbalancing:Describe*", + "elasticmapreduce:Describe*", + "elasticmapreduce:List*", + "es:DescribeElasticsearchDomains", + "es:ListDomainNames", + "es:ListTags", + "events:CreateEventBus", + "fsx:DescribeFileSystems", + "fsx:ListTagsForResource", + "health:DescribeAffectedEntities", + "health:DescribeEventDetails", + "health:DescribeEvents", + "iam:ListAccountAliases", + "kinesis:Describe*", + "kinesis:List*", + "lambda:List*", + "logs:DeleteSubscriptionFilter", + "logs:DescribeDeliveries", + "logs:DescribeDeliverySources", + "logs:DescribeLogGroups", + "logs:DescribeLogStreams", + "logs:DescribeSubscriptionFilters", + "logs:FilterLogEvents", + "logs:GetDeliveryDestination", + "logs:PutSubscriptionFilter", + "logs:TestMetricFilter", + "network-firewall:DescribeLoggingConfiguration", + "network-firewall:ListFirewalls", + "oam:ListAttachedLinks", + "oam:ListSinks", + "organizations:Describe*", + "organizations:List*", + "rds:Describe*", + "rds:List*", + "redshift-serverless:ListNamespaces", + "redshift:DescribeClusters", + "redshift:DescribeLoggingStatus", + "route53:List*", + "route53resolver:ListResolverQueryLogConfigs", + "s3:GetBucketLocation", + "s3:GetBucketLogging", + "s3:GetBucketNotification", + "s3:GetBucketTagging", + "s3:ListAllMyBuckets", + "s3:PutBucketNotification", + "ses:Get*", + "ses:List*", + "sns:GetSubscriptionAttributes", + "sns:List*", + "sns:Publish", + "sqs:ListQueues", + "ssm:GetServiceSetting", + "ssm:ListCommands", + "states:DescribeStateMachine", + "states:ListStateMachines", + "support:DescribeTrustedAdvisor*", + "support:RefreshTrustedAdvisorCheck", + "tag:GetResources", + "tag:GetTagKeys", + "tag:GetTagValues", + "timestream:DescribeEndpoints", + "wafv2:ListLoggingConfigurations", + "xray:BatchGetTraces", + "xray:GetTraceSummaries", + ] + + resources = ["*"] + } + + lifecycle { + enabled = var.aws_iam_role_enabled + } +} diff --git a/cloud/aws/r-datadog-integration.tf b/cloud/aws/r-datadog-integration.tf new file mode 100644 index 0000000..6e45407 --- /dev/null +++ b/cloud/aws/r-datadog-integration.tf @@ -0,0 +1,44 @@ +resource "datadog_integration_aws_account" "main" { + account_tags = [for k, v in var.metrics_tags : "${k}:${v}"] + aws_account_id = var.aws_account_id + aws_partition = var.aws_partition + + aws_regions { + include_all = true + } + + auth_config { + aws_auth_config_role { + role_name = local.role_name + } + } + + metrics_config { + automute_enabled = var.metrics_config.automute_enabled + collect_cloudwatch_alarms = var.metrics_config.collect_cloudwatch_alarms + collect_custom_metrics = var.metrics_config.collect_custom_metrics + enabled = var.metrics_config.enabled + namespace_filters { + exclude_only = var.metrics_config.namespace_filters.exclude_only + include_only = var.metrics_config.namespace_filters.include_only + } + dynamic "tag_filters" { + for_each = var.metrics_config.tag_filters[*] + content { + namespace = tag_filters.value.namespace + tags = tag_filters.value.tags + } + } + } + + resources_config { + extended_collection = var.resource_collection_enabled + } + + logs_config { + lambda_forwarder {} + } + traces_config { + xray_services {} + } +} diff --git a/cloud/aws/role.tf b/cloud/aws/role.tf deleted file mode 100644 index 47e22a4..0000000 --- a/cloud/aws/role.tf +++ /dev/null @@ -1,35 +0,0 @@ -resource "aws_iam_role" "dd_integration_role" { - name = local.role_name - description = "Datadog AWS Integration Role according to https://docs.datadoghq.com/integrations/aws" - - assume_role_policy = data.aws_iam_policy_document.dd_trust_relationship.json -} - -data "aws_iam_policy_document" "dd_trust_relationship" { - statement { - sid = "DatadogAWSTrustRelationship" - effect = "Allow" - actions = ["sts:AssumeRole"] - - principals { - type = "AWS" - - identifiers = [ - "arn:aws:iam::${var.datadog_aws_account_id}:root", - ] - } - - condition { - test = "StringEquals" - values = [datadog_integration_aws.datadog_integration_aws.external_id] - variable = "sts:ExternalId" - } - } -} - -resource "aws_iam_policy_attachment" "allow_dd_role" { - name = "Allow Datadog PolicyAccess via Role" - roles = [aws_iam_role.dd_integration_role.name] - policy_arn = aws_iam_policy.dd_integration_policy.arn -} - diff --git a/cloud/aws/test.tf.ci b/cloud/aws/test.tf.ci deleted file mode 100644 index 0db7bdf..0000000 --- a/cloud/aws/test.tf.ci +++ /dev/null @@ -1,20 +0,0 @@ -variable "aws_region" { - type = string -} - -variable "aws_access_key" { -} - -variable "aws_secret_key" { -} - -variable "aws_token" { -} - -provider "aws" { - region = var.aws_region - access_key = var.aws_access_key - secret_key = var.aws_secret_key - token = var.aws_token -} - diff --git a/cloud/aws/variables.tf b/cloud/aws/variables.tf new file mode 100644 index 0000000..68ac4ad --- /dev/null +++ b/cloud/aws/variables.tf @@ -0,0 +1,65 @@ +variable "aws_account_id" { + description = "AWS account configuration for Datadog integration" + type = string +} + +variable "aws_partition" { + description = "AWS partition for Datadog integration" + type = string + default = "aws" +} + +variable "datadog_aws_account_id" { + description = "AWS account_id of Datadog" + type = string + default = "464622532012" +} + +variable "metrics_config" { + description = "Metrics configuration for Datadog AWS integration" + type = object({ + automute_enabled : optional(bool, true), + collect_cloudwatch_alarms : optional(bool, false), + collect_custom_metrics : optional(bool, false), + enabled : optional(bool, true), + namespace_filters : optional(object({ + exclude_only : optional(list(string), null), + include_only : optional(list(string), null), + }), { + exclude_only = ["AWS/ElasticMapReduce", "AWS/SQS", "AWS/Usage"] + }), + tag_filters : optional(list(object({ + namespace : string, + tags : list(string), + })), [ + { + namespace = "AWS/EC2" + tags = ["claranet_monitored:true"] + }, { + namespace = "AWS/Lambda" + tags = [ + "claranet_monitored:true", + ] + } + ], + )}) + default = {} +} + +variable "aws_iam_role_enabled" { + description = "Enable IAM role deployment for Datadog AWS integration" + type = bool + default = true +} + +variable "resource_collection_enabled" { + description = "Enable resource collection for Datadog AWS integration" + type = bool + default = true +} + +variable "metrics_tags" { + description = "Tags to apply to metrics collected from AWS" + type = map(string) + default = {} +} diff --git a/cloud/aws/versions.tf b/cloud/aws/versions.tf index eec8740..6292f89 100644 --- a/cloud/aws/versions.tf +++ b/cloud/aws/versions.tf @@ -1,11 +1,11 @@ terraform { - required_version = ">= 0.12.31" + required_version = ">= 1.11" required_providers { aws = { - source = "hashicorp/aws" + source = "hashicorp/aws" + version = ">= 6.0.0" } - datadog = { source = "Datadog/datadog" version = ">= 3.0.0"