Skip to content

Commit 9fccf56

Browse files
modular-magicianmodular-magician
authored
dataproc metastore - add BQ support (#6631) (#4753)
Signed-off-by: Modular Magician <[email protected]> Signed-off-by: Modular Magician <[email protected]>
1 parent d53d378 commit 9fccf56

File tree

5 files changed

+146
-33
lines changed

5 files changed

+146
-33
lines changed

‎.changelog/6631.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
```release-note:enhancement
2+
metastore: add bigquery support for `google_dataproc_metastore_service`
3+
```

‎google-beta/iam_dataproc_metastore_federation_generated_test.go‎

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -94,10 +94,10 @@ resource "google_dataproc_metastore_federation" "default" {
9494
version = "3.1.2"
9595
9696
backend_metastores {
97-
rank = "1"
98-
name = google_dataproc_metastore_service.default.id
99-
metastore_type = "DATAPROC_METASTORE"
100-
}
97+
rank = "1"
98+
name = google_dataproc_metastore_service.default.id
99+
metastore_type = "DATAPROC_METASTORE"
100+
}
101101
}
102102
103103
resource "google_dataproc_metastore_service" "default" {
@@ -133,10 +133,10 @@ resource "google_dataproc_metastore_federation" "default" {
133133
version = "3.1.2"
134134
135135
backend_metastores {
136-
rank = "1"
137-
name = google_dataproc_metastore_service.default.id
138-
metastore_type = "DATAPROC_METASTORE"
139-
}
136+
rank = "1"
137+
name = google_dataproc_metastore_service.default.id
138+
metastore_type = "DATAPROC_METASTORE"
139+
}
140140
}
141141
142142
resource "google_dataproc_metastore_service" "default" {
@@ -179,10 +179,10 @@ resource "google_dataproc_metastore_federation" "default" {
179179
version = "3.1.2"
180180
181181
backend_metastores {
182-
rank = "1"
183-
name = google_dataproc_metastore_service.default.id
184-
metastore_type = "DATAPROC_METASTORE"
185-
}
182+
rank = "1"
183+
name = google_dataproc_metastore_service.default.id
184+
metastore_type = "DATAPROC_METASTORE"
185+
}
186186
}
187187
188188
resource "google_dataproc_metastore_service" "default" {
@@ -221,10 +221,10 @@ resource "google_dataproc_metastore_federation" "default" {
221221
version = "3.1.2"
222222
223223
backend_metastores {
224-
rank = "1"
225-
name = google_dataproc_metastore_service.default.id
226-
metastore_type = "DATAPROC_METASTORE"
227-
}
224+
rank = "1"
225+
name = google_dataproc_metastore_service.default.id
226+
metastore_type = "DATAPROC_METASTORE"
227+
}
228228
}
229229
230230
resource "google_dataproc_metastore_service" "default" {
@@ -260,10 +260,10 @@ resource "google_dataproc_metastore_federation" "default" {
260260
version = "3.1.2"
261261
262262
backend_metastores {
263-
rank = "1"
264-
name = google_dataproc_metastore_service.default.id
265-
metastore_type = "DATAPROC_METASTORE"
266-
}
263+
rank = "1"
264+
name = google_dataproc_metastore_service.default.id
265+
metastore_type = "DATAPROC_METASTORE"
266+
}
267267
}
268268
269269
resource "google_dataproc_metastore_service" "default" {

‎google-beta/resource_dataproc_metastore_federation.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,13 +55,13 @@ func resourceDataprocMetastoreFederation() *schema.Resource {
5555
"metastore_type": {
5656
Type: schema.TypeString,
5757
Required: true,
58-
ValidateFunc: validateEnum([]string{"METASTORE_TYPE_UNSPECIFIED", "DATAPROC_METASTORE"}),
59-
Description: `The type of the backend metastore. Possible values: ["METASTORE_TYPE_UNSPECIFIED", "DATAPROC_METASTORE"]`,
58+
ValidateFunc: validateEnum([]string{"METASTORE_TYPE_UNSPECIFIED", "DATAPROC_METASTORE", "BIGQUERY"}),
59+
Description: `The type of the backend metastore. Possible values: ["METASTORE_TYPE_UNSPECIFIED", "DATAPROC_METASTORE", "BIGQUERY"]`,
6060
},
6161
"name": {
6262
Type: schema.TypeString,
6363
Required: true,
64-
Description: `The relative resource name of the metastore that is being federated.`,
64+
Description: `The relative resource name of the metastore that is being federated. The formats of the relative resource names for the currently supported metastores are listed below: Dataplex: projects/{projectId}/locations/{location}/lakes/{lake_id} BigQuery: projects/{projectId} Dataproc Metastore: projects/{projectId}/locations/{location}/services/{serviceId}`,
6565
},
6666
},
6767
},

‎google-beta/resource_dataproc_metastore_federation_generated_test.go‎

Lines changed: 69 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -57,10 +57,10 @@ resource "google_dataproc_metastore_federation" "default" {
5757
version = "3.1.2"
5858
5959
backend_metastores {
60-
rank = "1"
61-
name = google_dataproc_metastore_service.default.id
62-
metastore_type = "DATAPROC_METASTORE"
63-
}
60+
rank = "1"
61+
name = google_dataproc_metastore_service.default.id
62+
metastore_type = "DATAPROC_METASTORE"
63+
}
6464
}
6565
6666
resource "google_dataproc_metastore_service" "default" {
@@ -78,6 +78,71 @@ resource "google_dataproc_metastore_service" "default" {
7878
`, context)
7979
}
8080

81+
func TestAccDataprocMetastoreFederation_dataprocMetastoreFederationBigqueryExample(t *testing.T) {
82+
t.Parallel()
83+
84+
context := map[string]interface{}{
85+
"random_suffix": randString(t, 10),
86+
}
87+
88+
vcrTest(t, resource.TestCase{
89+
PreCheck: func() { testAccPreCheck(t) },
90+
Providers: testAccProvidersOiCS,
91+
CheckDestroy: testAccCheckDataprocMetastoreFederationDestroyProducer(t),
92+
Steps: []resource.TestStep{
93+
{
94+
Config: testAccDataprocMetastoreFederation_dataprocMetastoreFederationBigqueryExample(context),
95+
},
96+
{
97+
ResourceName: "google_dataproc_metastore_federation.default",
98+
ImportState: true,
99+
ImportStateVerify: true,
100+
ImportStateVerifyIgnore: []string{"federation_id", "location"},
101+
},
102+
},
103+
})
104+
}
105+
106+
func testAccDataprocMetastoreFederation_dataprocMetastoreFederationBigqueryExample(context map[string]interface{}) string {
107+
return Nprintf(`
108+
resource "google_dataproc_metastore_federation" "default" {
109+
provider = google-beta
110+
location = "us-central1"
111+
federation_id = "tf-test-fed-2%{random_suffix}"
112+
version = "3.1.2"
113+
114+
backend_metastores {
115+
rank = "2"
116+
name = data.google_project.project.id
117+
metastore_type = "BIGQUERY"
118+
}
119+
120+
backend_metastores {
121+
rank = "1"
122+
name = google_dataproc_metastore_service.default.id
123+
metastore_type = "DATAPROC_METASTORE"
124+
}
125+
}
126+
127+
resource "google_dataproc_metastore_service" "default" {
128+
provider = google-beta
129+
service_id = "tf-test-fed-2%{random_suffix}"
130+
location = "us-central1"
131+
tier = "DEVELOPER"
132+
133+
134+
hive_metastore_config {
135+
version = "3.1.2"
136+
endpoint_protocol = "GRPC"
137+
}
138+
}
139+
140+
data "google_project" "project" {
141+
provider = google-beta
142+
}
143+
`, context)
144+
}
145+
81146
func testAccCheckDataprocMetastoreFederationDestroyProducer(t *testing.T) func(s *terraform.State) error {
82147
return func(s *terraform.State) error {
83148
for name, rs := range s.RootModule().Resources {

‎website/docs/r/dataproc_metastore_federation.html.markdown‎

Lines changed: 51 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -42,10 +42,10 @@ resource "google_dataproc_metastore_federation" "default" {
4242
version = "3.1.2"
4343
4444
backend_metastores {
45-
rank = "1"
46-
name = google_dataproc_metastore_service.default.id
47-
metastore_type = "DATAPROC_METASTORE"
48-
}
45+
rank = "1"
46+
name = google_dataproc_metastore_service.default.id
47+
metastore_type = "DATAPROC_METASTORE"
48+
}
4949
}
5050
5151
resource "google_dataproc_metastore_service" "default" {
@@ -61,6 +61,51 @@ resource "google_dataproc_metastore_service" "default" {
6161
}
6262
}
6363
```
64+
<div class = "oics-button" style="float: right; margin: 0 0 -15px">
65+
<a href="https://console.cloud.google.com/cloudshell/open?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2Fterraform-google-modules%2Fdocs-examples.git&cloudshell_working_dir=dataproc_metastore_federation_bigquery&cloudshell_image=gcr.io%2Fgraphite-cloud-shell-images%2Fterraform%3Alatest&open_in_editor=main.tf&cloudshell_print=.%2Fmotd&cloudshell_tutorial=.%2Ftutorial.md" target="_blank">
66+
<img alt="Open in Cloud Shell" src="//gstatic.com/cloudssh/images/open-btn.svg" style="max-height: 44px; margin: 32px auto; max-width: 100%;">
67+
</a>
68+
</div>
69+
## Example Usage - Dataproc Metastore Federation Bigquery
70+
71+
72+
```hcl
73+
resource "google_dataproc_metastore_federation" "default" {
74+
provider = google-beta
75+
location = "us-central1"
76+
federation_id = "fed-2"
77+
version = "3.1.2"
78+
79+
backend_metastores {
80+
rank = "2"
81+
name = data.google_project.project.id
82+
metastore_type = "BIGQUERY"
83+
}
84+
85+
backend_metastores {
86+
rank = "1"
87+
name = google_dataproc_metastore_service.default.id
88+
metastore_type = "DATAPROC_METASTORE"
89+
}
90+
}
91+
92+
resource "google_dataproc_metastore_service" "default" {
93+
provider = google-beta
94+
service_id = "fed-2"
95+
location = "us-central1"
96+
tier = "DEVELOPER"
97+
98+
99+
hive_metastore_config {
100+
version = "3.1.2"
101+
endpoint_protocol = "GRPC"
102+
}
103+
}
104+
105+
data "google_project" "project" {
106+
provider = google-beta
107+
}
108+
```
64109

65110
## Argument Reference
66111

@@ -89,12 +134,12 @@ The following arguments are supported:
89134

90135
* `name` -
91136
(Required)
92-
The relative resource name of the metastore that is being federated.
137+
The relative resource name of the metastore that is being federated. The formats of the relative resource names for the currently supported metastores are listed below: Dataplex: projects/{projectId}/locations/{location}/lakes/{lake_id} BigQuery: projects/{projectId} Dataproc Metastore: projects/{projectId}/locations/{location}/services/{serviceId}
93138

94139
* `metastore_type` -
95140
(Required)
96141
The type of the backend metastore.
97-
Possible values are `METASTORE_TYPE_UNSPECIFIED` and `DATAPROC_METASTORE`.
142+
Possible values are `METASTORE_TYPE_UNSPECIFIED`, `DATAPROC_METASTORE`, and `BIGQUERY`.
98143

99144
- - -
100145

0 commit comments

Comments
 (0)