When expanding the plan for module.wafv2["allow"].aws_wafv2_web_acl.main to include new values learned so far during apply, provider

[dangarthwaite]

We were using a previous version of the module and want to upgrade to the latest. Using the latest aws provider terraform plan works, but terraform apply produces megabytes of error logs.

│ 
│ When expanding the plan for module.wafv2["allow"].aws_wafv2_web_acl.main to include new values learned so far during apply, provider
│ "registry.terraform.io/hashicorp/aws" produced an invalid new value for .rule: planned set element
│ cty.ObjectVal(map[string]cty.Value{"action":cty.ListValEmpty(cty.Object(map[string]cty.Type{"allow":cty.List(cty.Object(map[string]cty.Type{"custom_request_handling":cty.List(cty.Object(map[string]cty.Type{"insert_header":cty.Set(cty.Object(map[string]cty.Type{"name":cty.String,
│ "value":cty.String}))}))})),
│ "block":cty.List(cty.Object(map[string]cty.Type{"custom_response":cty.List(cty.Object(map[string]cty.Type{"custom_response_body_key":cty.String,
│ "response_code":cty.Number, "response_header":cty.Set(cty.Object(map[string]cty.Type{"name":cty.String, "value":cty.String}))}))})),
│ "captcha":cty.List(cty.Object(map[string]cty.Type{"custom_request_handling":cty.List(cty.Object(map[string]cty.Type{"insert_header":cty.Set(cty.Object(map[string]cty.Type{"name":cty.String,
│ "value":cty.String}))}))})),
│ "challenge":cty.List(cty.Object(map[string]cty.Type{"custom_request_handling":cty.List(cty.Object(map[string]cty.Type{"insert_header":cty.Set(cty.Object(map[string]cty.Type{"name":cty.String,
│ "value":cty.String}))}))})),
│ "count":cty.List(cty.Object(map[string]cty.Type{"custom_request_handling":cty.List(cty.Object(map[string]cty.Type{"insert_header":cty.Set(cty.Object(map[string]cty.Type{"name":cty.String,
│ "value":cty.String}))}))}))})),
│ "captcha_config":cty.ListValEmpty(cty.Object(map[string]cty.Type{"immunity_time_property":cty.List(cty.Object(map[string]cty.Type{"immunity_time":cty.Number}))})),
│ "name":cty.StringVal("AWSManagedRulesAmazonIpReputationList"),
│ "override_action":cty.ListVal([]cty.Value{cty.ObjectVal(map[string]cty.Value{"count":cty.ListVal([]cty.Value{cty.EmptyObjectVal}),
│ "none":cty.ListValEmpty(cty.EmptyObject)})}), "priority":cty.NumberIntVal(2),
│ "rule_label":cty.SetValEmpty(cty.Object(map[string]cty.Type{"name":cty.String})),
│ "statement":cty.ListVal([]cty.Value{cty.ObjectVal(map[string]cty.Value{"and_statement":cty.ListValEmpty(cty.Object(map[string]cty.Type{"statement":cty.List(cty.Object(map[string]cty.Type{"and_statement":cty.List(cty.Object(map[string]cty.Type{"statement":cty.List(cty.Object(map[string]cty.Type{"and_statement":cty.List(cty.Object(map[string]cty.Type{"statement":cty.List(cty.Object(map[string]cty.Type{"byte_match_statement":cty.List(cty.Object(map[string]cty.Type{"field_to_match":cty.List(cty.Object(map[string]cty.Type{"all_query_arguments":cty.List(cty.EmptyObject),
│ "body":cty.List(cty.Object(map[string]cty.Type{"oversize_handling":cty.String})),
│ "cookies":cty.List(cty.Object(map[string]cty.Type{"match_pattern":cty.List(cty.Object(map[string]cty.Type{"all":cty.List(cty.EmptyObject),
│ "excluded_cookies":cty.List(cty.String), "included_cookies":cty.List(cty.String)})), "match_scope":cty.String, "oversize_handling":cty.String})),
│ "headers":cty.List(cty.Object(map[string]cty.Type{"match_pattern":cty.List(cty.Object(map[string]cty.Type{"all":cty.List(cty.EmptyObject),
│ "excluded_headers":cty.List(cty.String), "included_headers":cty.List(cty.String)})), "match_scope":cty.String, "oversize_handling":cty.String})),
│ "json_body":cty.List(cty.Object(map[string]cty.Type{"invalid_fallback_behavior":cty.String,
│ "match_pattern":cty.List(cty.Object(map[string]cty.Type{"all":cty.List(cty.EmptyObject), "included_paths":cty.List(cty.String)})), "match_scope":cty.String,
│ "oversize_handling":cty.String})), "method":cty.List(cty.EmptyObject), "query_string":cty.List(cty.EmptyObject),
│ "single_header":cty.List(cty.Object(map[string]cty.Type{"name":cty.String})),
│ "single_query_argument":cty.List(cty.Object(map[string]cty.Type{"name":cty.String})), "uri_path":cty.List(cty.EmptyObject)})),
│ "positional_constraint":cty.String, "search_string":cty.String, "text_transformation":cty.Set(cty.Object(map[string]cty.Type{"priority":cty.Number,
│ "type":cty.String}))})), "geo_match_statement":cty.List(cty.Object(map[string]cty.Type{"country_codes":cty.List(cty.String),
│ "forwarded_ip_config":cty.List(cty.Object(map[string]cty.Type{"fallback_behavior":cty.String, "header_name":cty.String}))})),
│ "ip_set_reference_statement":cty.List(cty.Object(map[string]cty.Type{"arn":cty.String,
│ "ip_set_forwarded_ip_config":cty.List(cty.Object(map[string]cty.Type{"fallback_behavior":cty.String, "header_name":cty.String, "position":cty.String}))})),
│ "label_match_statement":cty.List(cty.Object(map[string]cty.Type{"key":cty.String, "scope":cty.String})),
│ "regex_match_statement":cty.List(cty.Object(map[string]cty.Type{"field_to_match":cty.List(cty.Object(map[string]cty.Type{"all_query_arguments":cty.List(cty.EmptyObject),

[.... snip - megabytes of logs ...]

│ "match_pattern":cty.List(cty.Object(map[string]cty.Type{"all":cty.List(cty.EmptyObject), "included_paths":cty.List(cty.String)})), "match_scope":cty.String,
│ "oversize_handling":cty.String})), "method":cty.List(cty.EmptyObject), "query_string":cty.List(cty.EmptyObject),
│ "single_header":cty.List(cty.Object(map[string]cty.Type{"name":cty.String})),
│ "single_query_argument":cty.List(cty.Object(map[string]cty.Type{"name":cty.String})), "uri_path":cty.List(cty.EmptyObject)})),
│ "text_transformation":cty.Set(cty.Object(map[string]cty.Type{"priority":cty.Number, "type":cty.String}))}))})}),
│ "visibility_config":cty.ListVal([]cty.Value{cty.ObjectVal(map[string]cty.Value{"cloudwatch_metrics_enabled":cty.True,
│ "metric_name":cty.StringVal("AWSManagedRulesCommonRuleSet"), "sampled_requests_enabled":cty.True})})}) does not correlate with any element in actual.
│ 
│ This is a bug in the provider, which should be reported in the provider's own issue tracker.

[dangarthwaite]

locals {
  log_group_name                = "aws-waf-logs-managed"
  max_requests_per_five_minutes = 1000000
  days_to_retain_waf_logs       = 7
  days_until_log_key_rotation   = 30
}

resource "aws_cloudwatch_log_group" "aws-managed-waf" {
  name              = local.log_group_name
  retention_in_days = local.days_to_retain_waf_logs
  kms_key_id        = aws_kms_key.aws-managed-waf.arn
}


resource "aws_wafv2_ip_set" "allow_all_ips" {
  name               = "AllowAllIPs"
  scope              = "REGIONAL"
  ip_address_version = "IPV4"
  addresses          = formatlist("%s.0.0.0/8", range(0, 256))
}

module "wafv2" {
  for_each       = toset(["allow", "block"])
  source         = "trussworks/wafv2/aws"
  version        = "2.9.0"
  name           = "aws-managed-waf-${each.key}"
  scope          = "REGIONAL"
  associate_alb  = false
  default_action = each.key

  ip_rate_based_rule = {
    name : "ip-rate-limit",
    priority : 0,
    action : "block",
    limit : local.max_requests_per_five_minutes
  }
  /* ALLOW ALL sink for everything that passes lower priority rules */
  ip_sets_rule = [{
    name       = "AllowAllIPs"
    priority   = 7
    action     = "allow"
    ip_set_arn = aws_wafv2_ip_set.allow_all_ips.arn
  }]

  managed_rules = [
    {
      "vendor_name" : "AWS"
      "excluded_rules" : []
      "name" : "AWSManagedRulesCommonRuleSet"
      "override_action" : each.key == "allow" ? "count" : "none"
      "rule_action_override" : [
        {"action_to_use" = "count", "name" = "SizeRestrictions_BODY"},
        {"action_to_use" = "count", "name" = "SizeRestrictions_QUERYSTRING"},
        {"action_to_use" = "count", "name" = "CrossSiteScripting_BODY"},
        {"action_to_use" = "count", "name" = "NoUserAgent_HEADER"},
        {"action_to_use" = "count", "name" = "EC2MetaDataSSRF_BODY"}
      ]
      "priority" : 1
    },
    {
      "vendor_name" : "AWS"
      "excluded_rules" : []
      "name" : "AWSManagedRulesAmazonIpReputationList"
      "override_action" : each.key == "allow" ? "count" : "none"
      "rule_action_override" : [
        {"action_to_use" = "count", "name" = "AWSManagedIPDDoSList"},
        {"action_to_use" = "count", "name" = "AWSManagedIPReputationList"}
      ]
      "priority" : 2
    },
    {
      "vendor_name" : "AWS"
      "excluded_rules" : []
      "name" : "AWSManagedRulesKnownBadInputsRuleSet"
      "override_action" : each.key == "allow" ? "count" : "none"
      "rule_action_override" : []
      "priority" : 3
    },
    {
      "vendor_name" : "AWS"
      "excluded_rules" : [
        "SQLiExtendedPatterns_QUERYARGUMENTS",
        "SQLi_QUERYARGUMENTS"
      ]
      "name" : "AWSManagedRulesSQLiRuleSet"
      "override_action" : each.key == "allow" ? "count" : "none"
      "rule_action_override" : []
      "priority" : 4
    },
    {
      "vendor_name" : "AWS"
      "excluded_rules" : []
      "name" : "AWSManagedRulesLinuxRuleSet"
      "override_action" : each.key == "allow" ? "count" : "none"
      "rule_action_override" : []
      "priority" : 5
    },
    {
      "vendor_name" : "AWS"
      "excluded_rules" : []
      "name" : "AWSManagedRulesUnixRuleSet"
      "override_action" : each.key == "allow" ? "count" : "none"
      "rule_action_override" : []
      "priority" : 6
    }
  ]
}

resource "aws_wafv2_web_acl_logging_configuration" "aws-managed-waf" {
  for_each                = module.wafv2
  log_destination_configs = [aws_cloudwatch_log_group.aws-managed-waf.arn]
  resource_arn            = each.value.web_acl_id
  redacted_fields {
    single_header {
      name = "cookie"
    }
  }
}

resource "aws_kms_key" "aws-managed-waf" {
  description             = "Encrypt WAF cloudwatch logs"
  deletion_window_in_days = local.days_until_log_key_rotation
  policy                  = <<-POLICY
  {
   "Version": "2012-10-17",
      "Id": "key-default-1",
      "Statement": [
          {
              "Sid": "Enable IAM User Permissions",
              "Effect": "Allow",
              "Principal": {
                  "AWS": "arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"
              },
              "Action": "kms:*",
              "Resource": "*"
          },
          {
              "Effect": "Allow",
              "Principal": {
                  "Service": "logs.${data.aws_region.current.name}.amazonaws.com"
              },
              "Action": [
                  "kms:Encrypt*",
                  "kms:Decrypt*",
                  "kms:ReEncrypt*",
                  "kms:GenerateDataKey*",
                  "kms:Describe*"
              ],
              "Resource": "*",
              "Condition": {
                  "ArnEquals": {
                      "kms:EncryptionContext:aws:logs:arn": "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:${local.log_group_name}"
                  }
              }
          }
      ]
  }
  POLICY
}

[dangarthwaite]

Terraform v1.3.3
on linux_amd64
+ provider registry.terraform.io/hashicorp/aws v4.67.0
+ provider registry.terraform.io/hashicorp/helm v2.9.0
+ provider registry.terraform.io/hashicorp/kubernetes v2.20.0
+ provider registry.terraform.io/hashicorp/tls v4.0.4

[stevewright82]

I updated my version of Terraform (not the provider), and this problem disappeared.

[jsclarridge]

@dangarthwaite I haven't encountered this issue myself when using this module. Based on the error log output you posted, it sounds like a bug in the provider. If upgrading the Terraform version (as @stevewright82 suggests) doesn't fix the issue or isn't an option, I recommend reviewing the open/closed issues in the Terraform AWS provider repo or opening a new issue there to see if they can provide any guidance.