diff --git a/README.md b/README.md index 1cc25fd7..dd73e870 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,10 @@ intended for Terraform 0.12.x is [v4.5.0](https://registry.terraform.io/modules/ ## Upgrading -The current version is 4.X. The following guides are available to assist with upgrades: +The current version is 8.X. The following guides are available to assist with upgrades: +- [8.0 -> 9.0](./docs/upgrading_to_bigquery_v9.0.md) +- [7.0 -> 8.0](./docs/upgrading_to_bigquery_v8.0.md) - [3.0 -> 4.0](./docs/upgrading_to_bigquery_v4.0.md) - [2.0 -> 3.0](./docs/upgrading_to_bigquery_v3.0.md) - [1.0 -> 2.0](./docs/upgrading_to_bigquery_v2.0.md) @@ -41,10 +43,10 @@ module "bigquery" { { table_id = "foo", schema = "", + require_partition_filter = false, time_partitioning = { type = "DAY", field = null, - require_partition_filter = false, expiration_ms = null, }, range_partitioning = null, @@ -116,10 +118,10 @@ The `tables` variable should be provided as a list of object with the following table_id = "some_id" # Unique table id (will be used as ID for table). table_name = "Friendly Name" # Optional friendly name for table. If not set, the "table_id" will be used by default. schema = file("path/to/schema.json") # Schema as JSON string. + require_partition_filter = false, # If set to true, queries over this table require a partition filter that can be used for partition elimination to be specified. Set it to `null` to omit configuration. time_partitioning = { # Set it to `null` to omit partitioning configuration for the table. type = "DAY", # The only type supported is DAY, which will generate one partition per day based on data loading time. field = null, # The field used to determine how to create a time-based partition. If time-based partitioning is enabled without this value, the table is partitioned based on the load time. Set it to `null` to omit configuration. - require_partition_filter = false, # If set to true, queries over this table require a partition filter that can be used for partition elimination to be specified. Set it to `null` to omit configuration. expiration_ms = null, # Number of milliseconds for which to keep the storage for a partition. }, range_partitioning = { # Set it to `null` to omit partitioning configuration for the table. @@ -197,7 +199,7 @@ This module provisions a dataset and a list of tables with associated JSON schem | encryption\_key | Default encryption key to apply to the dataset. Defaults to null (Google-managed). | `string` | `null` | no | | external\_tables | A list of objects which include table\_id, expiration\_time, external\_data\_configuration, and labels. | 
list(object({
    table_id              = string,
    description           = optional(string),
    autodetect            = bool,
    compression           = string,
    ignore_unknown_values = bool,
    max_bad_records       = number,
    schema                = string,
    source_format         = string,
    source_uris           = list(string),
    csv_options = object({
      quote                 = string,
      allow_jagged_rows     = bool,
      allow_quoted_newlines = bool,
      encoding              = string,
      field_delimiter       = string,
      skip_leading_rows     = number,
    }),
    google_sheets_options = object({
      range             = string,
      skip_leading_rows = number,
    }),
    hive_partitioning_options = object({
      mode              = string,
      source_uri_prefix = string,
    }),
    expiration_time     = string,
    max_staleness       = optional(string),
    deletion_protection = optional(bool),
    labels              = map(string),
  }))
| `[]` | no | | location | The regional location for the dataset only US and EU are allowed in module | `string` | `"US"` | no | -| materialized\_views | A list of objects which includes view\_id, view\_query, clustering, time\_partitioning, range\_partitioning, expiration\_time and labels | 
list(object({
    view_id             = string,
    description         = optional(string),
    query               = string,
    enable_refresh      = bool,
    refresh_interval_ms = string,
    clustering          = list(string),
    time_partitioning = object({
      expiration_ms            = string,
      field                    = string,
      type                     = string,
      require_partition_filter = bool,
    }),
    range_partitioning = object({
      field = string,
      range = object({
        start    = string,
        end      = string,
        interval = string,
      }),
    }),
    expiration_time = string,
    max_staleness   = optional(string),
    labels          = map(string),
  }))
| `[]` | no | +| materialized\_views | A list of objects which includes view\_id, view\_query, clustering, time\_partitioning, range\_partitioning, expiration\_time and labels | 
list(object({
    view_id             = string,
    description         = optional(string),
    query               = string,
    enable_refresh      = bool,
    refresh_interval_ms = string,
    clustering          = list(string),
    require_partition_filter = optional(bool),
    time_partitioning = object({
      expiration_ms            = string,
      field                    = string,
      type                     = string,
    }),
    range_partitioning = object({
      field = string,
      range = object({
        start    = string,
        end      = string,
        interval = string,
      }),
    }),
    expiration_time = string,
    max_staleness   = optional(string),
    labels          = map(string),
  }))
| `[]` | no | | max\_time\_travel\_hours | Defines the time travel window in hours | `number` | `null` | no | | project\_id | Project where the dataset and table are created | `string` | n/a | yes | | resource\_tags | A map of resource tags to add to the dataset | `map(string)` | `{}` | no | diff --git a/docs/upgrading_to_bigquery_v9.0.md b/docs/upgrading_to_bigquery_v9.0.md new file mode 100644 index 00000000..58e289ad --- /dev/null +++ b/docs/upgrading_to_bigquery_v9.0.md @@ -0,0 +1,49 @@ +# Upgrading to BigQuery v9.0 + +- The supported provider has been updated to v5.3 ([terraform-provider-google/releases/tag/v5.3.0](https://github.com/hashicorp/terraform-provider-google/releases/tag/v5.3.0)) +- `require_partition_filter` has been deprecated under the `time_partitioning` block and can be used at the top level with the same name instead. (hashicorp/terraform-provider-google#16238) + +## Migration Instructions + +In the previous release, `require_partition_filter` was a part of the `time_partitioning` block on a `materialized_views` object. + + ```hcl + module "bigquery" { + source = "terraform-google-modules/bigquery/google" + version = "~> 8.0" + .... + materialized_views = [ + { + view_id = "foo", + .... + time_partitioning = { + .... + require_partition_filter = true, + .... + }, + } + ], + ... + } + ``` + +In the new release, `require_partition_filter` is a top-level field in the `materialized_views` object. + + ```hcl + module "bigquery" { + source = "terraform-google-modules/bigquery/google" + version = "~> 9.0" + .... + materialized_views = [ + { + view_id = "foo", + .... + require_partition_filter = true, + time_partitioning = { + .... + }, + }, + ], + ... + } + ``` diff --git a/main.tf b/main.tf index aa692341..37ea5749 100644 --- a/main.tf +++ b/main.tf @@ -130,25 +130,25 @@ resource "google_bigquery_table" "view" { } resource "google_bigquery_table" "materialized_view" { - for_each = local.materialized_views - dataset_id = google_bigquery_dataset.main.dataset_id - friendly_name = each.key - table_id = each.key - description = each.value["description"] - labels = each.value["labels"] - clustering = each.value["clustering"] - expiration_time = each.value["expiration_time"] != null ? each.value["expiration_time"] : 0 - max_staleness = each.value["max_staleness"] - project = var.project_id - deletion_protection = false + for_each = local.materialized_views + dataset_id = google_bigquery_dataset.main.dataset_id + friendly_name = each.key + table_id = each.key + description = each.value["description"] + labels = each.value["labels"] + clustering = each.value["clustering"] + expiration_time = each.value["expiration_time"] != null ? each.value["expiration_time"] : 0 + max_staleness = each.value["max_staleness"] + project = var.project_id + deletion_protection = false + require_partition_filter = each.value["require_partition_filter"] dynamic "time_partitioning" { for_each = each.value["time_partitioning"] != null ? [each.value["time_partitioning"]] : [] content { - type = time_partitioning.value["type"] - expiration_ms = time_partitioning.value["expiration_ms"] != null ? time_partitioning.value["expiration_ms"] : 0 - field = time_partitioning.value["field"] - require_partition_filter = time_partitioning.value["require_partition_filter"] + type = time_partitioning.value["type"] + expiration_ms = time_partitioning.value["expiration_ms"] != null ? time_partitioning.value["expiration_ms"] : 0 + field = time_partitioning.value["field"] } } diff --git a/metadata.yaml b/metadata.yaml index b5f8cecb..6bc11d91 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -149,11 +149,11 @@ spec: enable_refresh = bool, refresh_interval_ms = string, clustering = list(string), + require_partition_filter = optional(bool), time_partitioning = object({ expiration_ms = string, field = string, type = string, - require_partition_filter = bool, }), range_partitioning = object({ field = string, diff --git a/variables.tf b/variables.tf index 02dcc6f6..30ad2882 100644 --- a/variables.tf +++ b/variables.tf @@ -145,17 +145,17 @@ variable "materialized_views" { description = "A list of objects which includes view_id, view_query, clustering, time_partitioning, range_partitioning, expiration_time and labels" default = [] type = list(object({ - view_id = string, - description = optional(string), - query = string, - enable_refresh = bool, - refresh_interval_ms = string, - clustering = list(string), + view_id = string, + description = optional(string), + query = string, + enable_refresh = bool, + refresh_interval_ms = string, + clustering = list(string), + require_partition_filter = optional(bool), time_partitioning = object({ - expiration_ms = string, - field = string, - type = string, - require_partition_filter = bool, + expiration_ms = string, + field = string, + type = string, }), range_partitioning = object({ field = string,