feat: add storage_billing_model and default_partition_expiration_ms (#367)

Co-authored-by: Sam Poole <[email protected]>

Author: SamuZad

README.md

@@ -190,6 +190,7 @@ This module provisions a dataset and a list of tables with associated JSON schem
 | dataset\_id | Unique ID for the dataset being provisioned. | `string` | n/a | yes |
 | dataset\_labels | Key value pairs in a map for dataset labels | `map(string)` | `{}` | no |
 | dataset\_name | Friendly name for the dataset being provisioned. | `string` | `null` | no |
+| default\_partition\_expiration\_ms | The default partition expiration for all partitioned tables in the dataset, in MS | `number` | `null` | no |
 | default\_table\_expiration\_ms | TTL of tables using the dataset in MS | `number` | `null` | no |
 | delete\_contents\_on\_destroy | (Optional) If set to true, delete all the tables in the dataset when destroying the resource; otherwise, destroying the resource will fail if tables are present. | `bool` | `null` | no |
 | deletion\_protection | Whether or not to allow deletion of tables and external tables defined by this module. Can be overriden by table-level deletion\_protection configuration. | `bool` | `false` | no |
@@ -202,6 +203,7 @@ This module provisions a dataset and a list of tables with associated JSON schem
 | project\_id | Project where the dataset and table are created | `string` | n/a | yes |
 | resource\_tags | A map of resource tags to add to the dataset | `map(string)` | `{}` | no |
 | routines | A list of objects which include routine\_id, routine\_type, routine\_language, definition\_body, return\_type, routine\_description and arguments. | <pre>list(object({<br>    routine_id      = string,<br>    routine_type    = string,<br>    language        = string,<br>    definition_body = string,<br>    return_type     = string,<br>    description     = string,<br>    arguments = list(object({<br>      name          = string,<br>      data_type     = string,<br>      argument_kind = string,<br>      mode          = string,<br>    })),<br>  }))</pre> | `[]` | no |
+| storage\_billing\_model | Specifies the storage billing model for the dataset. Set this flag value to LOGICAL to use logical bytes for storage billing, or to PHYSICAL to use physical bytes instead. LOGICAL is the default if this flag isn't specified. | `string` | `null` | no |
 | tables | A list of objects which include table\_id, table\_name, schema, clustering, time\_partitioning, range\_partitioning, expiration\_time and labels. | <pre>list(object({<br>    table_id                 = string,<br>    description              = optional(string),<br>    table_name               = optional(string),<br>    schema                   = string,<br>    clustering               = list(string),<br>    require_partition_filter = optional(bool),<br>    time_partitioning = object({<br>      expiration_ms = string,<br>      field         = string,<br>      type          = string,<br>    }),<br>    range_partitioning = object({<br>      field = string,<br>      range = object({<br>        start    = string,<br>        end      = string,<br>        interval = string,<br>      }),<br>    }),<br>    expiration_time     = string,<br>    deletion_protection = optional(bool),<br>    labels              = map(string),<br>  }))</pre> | `[]` | no |
 | views | A list of objects which include view\_id and view query | <pre>list(object({<br>    view_id        = string,<br>    description    = optional(string),<br>    query          = string,<br>    use_legacy_sql = bool,<br>    labels         = map(string),<br>  }))</pre> | `[]` | no |
 

main.tf

@@ -29,16 +29,18 @@ locals {
 }
 
 resource "google_bigquery_dataset" "main" {
-  dataset_id                  = var.dataset_id
-  friendly_name               = var.dataset_name
-  description                 = var.description
-  location                    = var.location
-  delete_contents_on_destroy  = var.delete_contents_on_destroy
-  default_table_expiration_ms = var.default_table_expiration_ms
-  max_time_travel_hours       = var.max_time_travel_hours
-  project                     = var.project_id
-  labels                      = var.dataset_labels
-  resource_tags               = var.resource_tags
+  dataset_id                      = var.dataset_id
+  friendly_name                   = var.dataset_name
+  description                     = var.description
+  location                        = var.location
+  delete_contents_on_destroy      = var.delete_contents_on_destroy
+  default_table_expiration_ms     = var.default_table_expiration_ms
+  max_time_travel_hours           = var.max_time_travel_hours
+  storage_billing_model           = var.storage_billing_model
+  project                         = var.project_id
+  labels                          = var.dataset_labels
+  resource_tags                   = var.resource_tags
+  default_partition_expiration_ms = var.default_partition_expiration_ms
 
   dynamic "default_encryption_configuration" {
     for_each = var.encryption_key == null ? [] : [var.encryption_key]

metadata.display.yaml

@@ -39,6 +39,9 @@ spec:
         dataset_name:
           name: dataset_name
           title: Dataset Name
+        default_partition_expiration_ms:
+          name: default_partition_expiration_ms
+          title: Default Partition Expiration Ms
         default_table_expiration_ms:
           name: default_table_expiration_ms
           title: Default Table Expiration Ms
@@ -75,6 +78,9 @@ spec:
         routines:
           name: routines
           title: Routines
+        storage_billing_model:
+          name: storage_billing_model
+          title: Storage Billing Model
         tables:
           name: tables
           title: Tables

metadata.yaml

@@ -76,9 +76,15 @@ spec:
       - name: default_table_expiration_ms
         description: TTL of tables using the dataset in MS
         varType: number
+      - name: default_partition_expiration_ms
+        description: The default partition expiration for all partitioned tables in the dataset, in MS
+        varType: number
       - name: max_time_travel_hours
         description: Defines the time travel window in hours
         varType: number
+      - name: storage_billing_model
+        description: Specifies the storage billing model for the dataset. Set this flag value to LOGICAL to use logical bytes for storage billing, or to PHYSICAL to use physical bytes instead. LOGICAL is the default if this flag isn't specified.
+        varType: string
       - name: project_id
         description: Project where the dataset and table are created
         varType: string

variables.tf

@@ -55,12 +55,29 @@ variable "default_table_expiration_ms" {
   default     = null
 }
 
+variable "default_partition_expiration_ms" {
+  description = "The default partition expiration for all partitioned tables in the dataset, in MS"
+  type        = number
+  default     = null
+}
+
 variable "max_time_travel_hours" {
   description = "Defines the time travel window in hours"
   type        = number
   default     = null
 }
 
+variable "storage_billing_model" {
+  description = "Specifies the storage billing model for the dataset. Set this flag value to LOGICAL to use logical bytes for storage billing, or to PHYSICAL to use physical bytes instead. LOGICAL is the default if this flag isn't specified."
+  type        = string
+  default     = null
+
+  validation {
+    condition     = var.storage_billing_model == null || var.storage_billing_model == "LOGICAL" || var.storage_billing_model == "PHYSICAL"
+    error_message = "storage_billing_model must be null, \"LOGICAL\" or \"PHYSICAL\"."
+  }
+}
+
 variable "project_id" {
   description = "Project where the dataset and table are created"
   type        = string