Skip to content

Commit 5ef1f01

Browse files
m-habibm-habib
authored
Introduce Customization Rules to google_datastream_stream (#15856)
1 parent 07012a5 commit 5ef1f01

File tree

2 files changed

+241
-0
lines changed

2 files changed

+241
-0
lines changed

‎mmv1/products/datastream/Stream.yaml‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,19 @@ examples:
238238
skip_vcr: true
239239
# Involves complex dependency creation, which makes it impractical in this context
240240
exclude_test: true
241+
- name: 'datastream_stream_rule_sets_bigquery'
242+
primary_resource_id: 'default'
243+
vars:
244+
stream_id: 'rules-stream'
245+
source_connection_profile_id: 'rules-source-profile'
246+
destination_connection_profile_id: 'rules-dest-profile'
247+
dataset_id: 'rules-project:rules-dataset'
248+
deletion_protection: 'true'
249+
test_vars_overrides:
250+
'deletion_protection': 'false'
251+
skip_vcr: true
252+
# Involves complex dependency creation, which makes it impractical in this context
253+
exclude_test: true
241254
- name: 'datastream_stream_salesforce'
242255
primary_resource_id: 'default'
243256
vars:
@@ -1986,3 +1999,153 @@ properties:
19861999
will be encrypted using an internal Stream-specific encryption key provisioned through KMS.
19872000
immutable: true
19882001
is_missing_in_cai: true
2002+
- name: 'ruleSets'
2003+
type: Array
2004+
description: 'Rule sets to apply to the stream.'
2005+
item_type:
2006+
type: NestedObject
2007+
properties:
2008+
- name: 'customizationRules'
2009+
type: Array
2010+
description: 'List of customization rules to apply.'
2011+
required: true
2012+
item_type:
2013+
type: NestedObject
2014+
properties:
2015+
- name: 'bigqueryPartitioning'
2016+
type: NestedObject
2017+
description: 'BigQuery partitioning rule.'
2018+
properties:
2019+
- name: 'integerRangePartition'
2020+
type: NestedObject
2021+
properties:
2022+
- name: 'column'
2023+
type: String
2024+
description: 'The partitioning column.'
2025+
required: true
2026+
- name: 'start'
2027+
type: Integer
2028+
description: 'The starting value for range partitioning (inclusive).'
2029+
required: true
2030+
- name: 'end'
2031+
type: Integer
2032+
description: 'The ending value for range partitioning (exclusive).'
2033+
required: true
2034+
- name: 'interval'
2035+
type: Integer
2036+
description: 'The interval of each range within the partition.'
2037+
required: true
2038+
- name: 'timeUnitPartition'
2039+
type: NestedObject
2040+
properties:
2041+
- name: 'column'
2042+
type: String
2043+
description: 'The partitioning column.'
2044+
required: true
2045+
- name: 'partitioningTimeGranularity'
2046+
type: Enum
2047+
description: 'Partition granularity.'
2048+
enum_values:
2049+
- 'PARTITIONING_TIME_GRANULARITY_UNSPECIFIED'
2050+
- 'PARTITIONING_TIME_GRANULARITY_HOUR'
2051+
- 'PARTITIONING_TIME_GRANULARITY_DAY'
2052+
- 'PARTITIONING_TIME_GRANULARITY_MONTH'
2053+
- 'PARTITIONING_TIME_GRANULARITY_YEAR'
2054+
- name: 'ingestionTimePartition'
2055+
type: NestedObject
2056+
allow_empty_object: true
2057+
send_empty_value: true
2058+
properties:
2059+
- name: 'partitioningTimeGranularity'
2060+
type: Enum
2061+
description: 'Partition granularity.'
2062+
enum_values:
2063+
- 'PARTITIONING_TIME_GRANULARITY_UNSPECIFIED'
2064+
- 'PARTITIONING_TIME_GRANULARITY_HOUR'
2065+
- 'PARTITIONING_TIME_GRANULARITY_DAY'
2066+
- 'PARTITIONING_TIME_GRANULARITY_MONTH'
2067+
- 'PARTITIONING_TIME_GRANULARITY_YEAR'
2068+
- name: 'requirePartitionFilter'
2069+
type: Boolean
2070+
description: 'If true, queries over the table require a partition filter.'
2071+
- name: 'bigqueryClustering'
2072+
type: NestedObject
2073+
description: 'BigQuery clustering rule.'
2074+
properties:
2075+
- name: 'columns'
2076+
type: Array
2077+
description: 'Column names to set as clustering columns.'
2078+
required: true
2079+
item_type:
2080+
type: String
2081+
- name: 'objectFilter'
2082+
type: NestedObject
2083+
description: 'Object filter to apply the customization rules to.'
2084+
required: true
2085+
properties:
2086+
- name: 'sourceObjectIdentifier'
2087+
type: NestedObject
2088+
description: 'Specific source object identifier.'
2089+
properties:
2090+
- name: 'oracleIdentifier'
2091+
type: NestedObject
2092+
properties:
2093+
- name: 'schema'
2094+
type: String
2095+
description: 'The schema name.'
2096+
required: true
2097+
- name: 'table'
2098+
type: String
2099+
description: 'The table name.'
2100+
required: true
2101+
- name: 'mysqlIdentifier'
2102+
type: NestedObject
2103+
properties:
2104+
- name: 'database'
2105+
type: String
2106+
description: 'The database name.'
2107+
required: true
2108+
- name: 'table'
2109+
type: String
2110+
description: 'The table name.'
2111+
required: true
2112+
- name: 'postgresqlIdentifier'
2113+
type: NestedObject
2114+
properties:
2115+
- name: 'schema'
2116+
type: String
2117+
description: 'The schema name.'
2118+
required: true
2119+
- name: 'table'
2120+
type: String
2121+
description: 'The table name.'
2122+
required: true
2123+
- name: 'sqlServerIdentifier'
2124+
type: NestedObject
2125+
properties:
2126+
- name: 'schema'
2127+
type: String
2128+
description: 'The schema name.'
2129+
required: true
2130+
- name: 'table'
2131+
type: String
2132+
description: 'The table name.'
2133+
required: true
2134+
- name: 'salesforceIdentifier'
2135+
type: NestedObject
2136+
properties:
2137+
- name: 'objectName'
2138+
type: String
2139+
description: 'The Salesforce object name.'
2140+
required: true
2141+
- name: 'mongodbIdentifier'
2142+
type: NestedObject
2143+
properties:
2144+
- name: 'database'
2145+
type: String
2146+
description: 'The MongoDB database name.'
2147+
required: true
2148+
- name: 'collection'
2149+
type: String
2150+
description: 'The MongoDB collection name.'
2151+
required: true

‎mmv1/templates/terraform/examples/datastream_stream_rule_sets_bigquery.tf.tmpl‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
data "google_project" "project" {
2+
}
3+
4+
resource "google_datastream_stream" "stream" {
5+
stream_id = "{{index .Vars "stream_id"}}"
6+
location = "us-central1"
7+
display_name = "BigQuery Stream with Rules"
8+
9+
source_config {
10+
source_connection_profile = "{{index $.Vars "source_connection_profile_id"}}"
11+
mysql_source_config {
12+
include_objects {
13+
mysql_databases {
14+
database = "my_database"
15+
}
16+
}
17+
binary_log_position {}
18+
}
19+
}
20+
21+
destination_config {
22+
destination_connection_profile = "{{index $.Vars "destination_connection_profile_id"}}"
23+
bigquery_destination_config {
24+
single_target_dataset {
25+
dataset_id = "{{index $.Vars "dataset_id"}}"
26+
}
27+
}
28+
}
29+
30+
backfill_none {}
31+
32+
rule_sets {
33+
object_filter {
34+
source_object_identifier {
35+
mysql_identifier {
36+
database = "test_database"
37+
table = "test_table_1"
38+
}
39+
}
40+
}
41+
customization_rules {
42+
bigquery_clustering {
43+
columns = ["user_id"]
44+
}
45+
}
46+
customization_rules {
47+
bigquery_partitioning {
48+
ingestion_time_partition {
49+
}
50+
}
51+
}
52+
}
53+
54+
rule_sets {
55+
object_filter {
56+
source_object_identifier {
57+
mysql_identifier {
58+
database = "test_database"
59+
table = "test_table_2"
60+
}
61+
}
62+
}
63+
customization_rules {
64+
bigquery_clustering {
65+
columns = ["event_time"]
66+
}
67+
}
68+
customization_rules {
69+
bigquery_partitioning {
70+
time_unit_partition {
71+
column = "event_time"
72+
partitioning_time_granularity = "PARTITIONING_TIME_GRANULARITY_DAY"
73+
}
74+
}
75+
}
76+
}
77+
}
78+

0 commit comments

Comments
 (0)