Skip to content

Commit ec8dce5

Browse files
XianJiChenXianJiChen
authored
Terraform support for Dataplex Profile Publishing (#15355)
1 parent c59fe19 commit ec8dce5

File tree

5 files changed

+303
-1
lines changed

5 files changed

+303
-1
lines changed

‎mmv1/products/dataplex/Datascan.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,14 @@ examples:
7171
datascan_name: 'dataprofile-full'
7272
test_env_vars:
7373
project_name: 'PROJECT_NAME'
74+
exclude_test: true
75+
- name: 'dataplex_datascan_full_profile_test'
76+
primary_resource_id: 'full_profile_test'
77+
vars:
78+
datascan_name: 'dataprofile-full-test'
79+
test_env_vars:
80+
project_name: 'PROJECT_NAME'
81+
exclude_docs: true
7482
- name: 'dataplex_datascan_basic_quality'
7583
primary_resource_id: 'basic_quality'
7684
vars:
@@ -583,6 +591,10 @@ properties:
583591
For instance, if 'x' is of nested field type, listing 'x' is supported but 'x.y.z' is not supported. Here 'y' and 'y.z' are nested fields of 'x'.
584592
item_type:
585593
type: String
594+
- name: 'catalogPublishingEnabled'
595+
type: Boolean
596+
description: |
597+
If set, the latest DataScan job result will be published to Dataplex Catalog.
586598
- name: 'dataDiscoverySpec'
587599
type: NestedObject
588600
description: |

‎mmv1/templates/terraform/examples/dataplex_datascan_full_profile.tf.tmpl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ resource "google_dataplex_datascan" "{{$.PrimaryResourceId}}" {
3333
results_table = "//bigquery.googleapis.com/projects/{{index $.TestEnvVars "project_name"}}/datasets/{{index $.Vars "dataset_name"}}/tables/profile_export"
3434
}
3535
}
36+
catalog_publishing_enabled = true
3637
}
3738

3839
project = "{{index $.TestEnvVars "project_name"}}"

‎mmv1/templates/terraform/examples/dataplex_datascan_full_profile_test.tf.tmpl‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
resource "google_bigquery_dataset" "tf_test_dataset" {
2+
dataset_id = "tf_test_dataset_id_%{random_suffix}"
3+
default_table_expiration_ms = 3600000
4+
delete_contents_on_destroy = true
5+
}
6+
7+
resource "google_bigquery_table" "tf_test_table" {
8+
dataset_id = google_bigquery_dataset.tf_test_dataset.dataset_id
9+
table_id = "tf_test_table_id_%{random_suffix}"
10+
deletion_protection = false
11+
schema = <<EOF
12+
[
13+
{
14+
"name": "word",
15+
"type": "STRING",
16+
"mode": "REQUIRED"
17+
},
18+
{
19+
"name": "word_count",
20+
"type": "INTEGER",
21+
"mode": "REQUIRED"
22+
},
23+
{
24+
"name": "corpus",
25+
"type": "STRING",
26+
"mode": "REQUIRED"
27+
},
28+
{
29+
"name": "corpus_date",
30+
"type": "INTEGER",
31+
"mode": "REQUIRED"
32+
}
33+
]
34+
EOF
35+
}
36+
37+
resource "google_dataplex_datascan" "{{$.PrimaryResourceId}}" {
38+
location = "us-central1"
39+
display_name = "Full Datascan Quality Publishing"
40+
data_scan_id = "{{index $.Vars "datascan_name"}}"
41+
description = "Example resource - Full Datascan Quality with Publishing enabled"
42+
labels = {
43+
author = "billing"
44+
}
45+
46+
data {
47+
resource = "//bigquery.googleapis.com/projects/{{index $.TestEnvVars "project_name"}}/datasets/${google_bigquery_dataset.tf_test_dataset.dataset_id}/tables/${google_bigquery_table.tf_test_table.table_id}"
48+
}
49+
50+
execution_spec {
51+
trigger {
52+
schedule {
53+
cron = "TZ=America/New_York 1 1 * * *"
54+
}
55+
}
56+
}
57+
58+
data_profile_spec {
59+
sampling_percent = 80
60+
row_filter = "word_count > 10"
61+
include_fields {
62+
field_names = ["word_count"]
63+
}
64+
exclude_fields {
65+
field_names = ["property_type"]
66+
}
67+
post_scan_actions {
68+
bigquery_export {
69+
results_table = "//bigquery.googleapis.com/projects/{{index $.TestEnvVars "project_name"}}/datasets/${google_bigquery_dataset.tf_test_dataset.dataset_id}/tables/profile_export_%{random_suffix}"
70+
}
71+
}
72+
catalog_publishing_enabled = true
73+
}
74+
75+
project = "{{index $.TestEnvVars "project_name"}}"
76+
77+
depends_on = [
78+
google_bigquery_dataset.tf_test_dataset
79+
]
80+
}

‎mmv1/templates/terraform/examples/dataplex_datascan_full_quality_test.tf.tmpl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@ resource "google_dataplex_datascan" "{{$.PrimaryResourceId}}" {
172172
rules {
173173
dimension = "VALIDITY"
174174
sql_assertion {
175-
sql_statement = "select * from {{index $.TestEnvVars "project_name"}}.${google_bigquery_dataset.tf_test_dataset.dataset_id}.${google_bigquery_table.tf_test_table.table_id} where address is null"
175+
sql_statement = "select * from $${data()} where address is null"
176176
}
177177
}
178178
}

‎mmv1/third_party/terraform/services/dataplex/resource_dataplex_datascan_test.go‎

Lines changed: 209 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -230,3 +230,212 @@ resource "google_dataplex_datascan" "full_quality" {
230230
}
231231
`, context)
232232
}
233+
234+
func TestAccDataplexDatascan_dataplexDatascanFullProfile_update(t *testing.T) {
235+
t.Parallel()
236+
237+
context := map[string]interface{}{
238+
"project_name": envvar.GetTestProjectFromEnv(),
239+
"random_suffix": acctest.RandString(t, 10),
240+
}
241+
242+
acctest.VcrTest(t, resource.TestCase{
243+
PreCheck: func() { acctest.AccTestPreCheck(t) },
244+
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
245+
CheckDestroy: testAccCheckDataplexDatascanDestroyProducer(t),
246+
Steps: []resource.TestStep{
247+
{
248+
Config: testAccDataplexDatascan_dataplexDatascanFullProfile_full(context),
249+
},
250+
{
251+
ResourceName: "google_dataplex_datascan.full_profile",
252+
ImportState: true,
253+
ImportStateVerify: true,
254+
ImportStateVerifyIgnore: []string{"data_scan_id", "labels", "location", "terraform_labels"},
255+
},
256+
{
257+
Config: testAccDataplexDatascan_dataplexDatascanFullProfile_update(context),
258+
ConfigPlanChecks: resource.ConfigPlanChecks{
259+
PreApply: []plancheck.PlanCheck{
260+
plancheck.ExpectResourceAction("google_dataplex_datascan.full_profile", plancheck.ResourceActionUpdate),
261+
},
262+
},
263+
},
264+
{
265+
ResourceName: "google_dataplex_datascan.full_profile",
266+
ImportState: true,
267+
ImportStateVerify: true,
268+
ImportStateVerifyIgnore: []string{"data_scan_id", "labels", "location", "terraform_labels"},
269+
},
270+
},
271+
})
272+
}
273+
274+
func testAccDataplexDatascan_dataplexDatascanFullProfile_full(context map[string]interface{}) string {
275+
return acctest.Nprintf(`
276+
resource "google_bigquery_dataset" "tf_test_dataset" {
277+
dataset_id = "tf_test_dataset_id_%{random_suffix}"
278+
default_table_expiration_ms = 3600000
279+
delete_contents_on_destroy = true
280+
}
281+
282+
resource "google_bigquery_table" "tf_test_table" {
283+
dataset_id = google_bigquery_dataset.tf_test_dataset.dataset_id
284+
table_id = "tf_test_table_id_%{random_suffix}"
285+
deletion_protection = false
286+
schema = <<EOF
287+
[
288+
{
289+
"name": "word",
290+
"type": "STRING",
291+
"mode": "REQUIRED"
292+
},
293+
{
294+
"name": "word_count",
295+
"type": "INTEGER",
296+
"mode": "REQUIRED"
297+
},
298+
{
299+
"name": "corpus",
300+
"type": "STRING",
301+
"mode": "REQUIRED"
302+
},
303+
{
304+
"name": "corpus_date",
305+
"type": "INTEGER",
306+
"mode": "REQUIRED"
307+
}
308+
]
309+
EOF
310+
}
311+
312+
resource "google_dataplex_datascan" "full_profile" {
313+
location = "us-central1"
314+
display_name = "Full Datascan Quality Publishing"
315+
data_scan_id = "tf-test-dataprofile-full-test%{random_suffix}"
316+
description = "Example resource - Full Datascan Quality with Publishing enabled"
317+
labels = {
318+
author = "billing"
319+
}
320+
321+
data {
322+
resource = "//bigquery.googleapis.com/projects/%{project_name}/datasets/${google_bigquery_dataset.tf_test_dataset.dataset_id}/tables/${google_bigquery_table.tf_test_table.table_id}"
323+
}
324+
325+
execution_spec {
326+
trigger {
327+
schedule {
328+
cron = "TZ=America/New_York 1 1 * * *"
329+
}
330+
}
331+
}
332+
333+
data_profile_spec {
334+
sampling_percent = 80
335+
row_filter = "word_count > 10"
336+
include_fields {
337+
field_names = ["word_count"]
338+
}
339+
exclude_fields {
340+
field_names = ["property_type"]
341+
}
342+
post_scan_actions {
343+
bigquery_export {
344+
results_table = "//bigquery.googleapis.com/projects/%{project_name}/datasets/${google_bigquery_dataset.tf_test_dataset.dataset_id}/tables/profile_export_%{random_suffix}"
345+
}
346+
}
347+
}
348+
349+
project = "%{project_name}"
350+
351+
depends_on = [
352+
google_bigquery_dataset.tf_test_dataset
353+
]
354+
}
355+
`, context)
356+
}
357+
358+
func testAccDataplexDatascan_dataplexDatascanFullProfile_update(context map[string]interface{}) string {
359+
return acctest.Nprintf(`
360+
resource "google_bigquery_dataset" "tf_test_dataset" {
361+
dataset_id = "tf_test_dataset_id_%{random_suffix}"
362+
default_table_expiration_ms = 3600000
363+
delete_contents_on_destroy = true
364+
}
365+
366+
resource "google_bigquery_table" "tf_test_table" {
367+
dataset_id = google_bigquery_dataset.tf_test_dataset.dataset_id
368+
table_id = "tf_test_table_id_%{random_suffix}"
369+
deletion_protection = false
370+
schema = <<EOF
371+
[
372+
{
373+
"name": "word",
374+
"type": "STRING",
375+
"mode": "REQUIRED"
376+
},
377+
{
378+
"name": "word_count",
379+
"type": "INTEGER",
380+
"mode": "REQUIRED"
381+
},
382+
{
383+
"name": "corpus",
384+
"type": "STRING",
385+
"mode": "REQUIRED"
386+
},
387+
{
388+
"name": "corpus_date",
389+
"type": "INTEGER",
390+
"mode": "REQUIRED"
391+
}
392+
]
393+
EOF
394+
}
395+
396+
resource "google_dataplex_datascan" "full_profile" {
397+
location = "us-central1"
398+
display_name = "Full Datascan Quality Publishing"
399+
data_scan_id = "tf-test-dataprofile-full-test%{random_suffix}"
400+
description = "Example resource - Full Datascan Quality with Publishing enabled"
401+
labels = {
402+
author = "billing"
403+
}
404+
405+
data {
406+
resource = "//bigquery.googleapis.com/projects/%{project_name}/datasets/${google_bigquery_dataset.tf_test_dataset.dataset_id}/tables/${google_bigquery_table.tf_test_table.table_id}"
407+
}
408+
409+
execution_spec {
410+
trigger {
411+
schedule {
412+
cron = "TZ=America/New_York 1 1 * * *"
413+
}
414+
}
415+
}
416+
417+
data_profile_spec {
418+
sampling_percent = 80
419+
row_filter = "word_count > 10"
420+
include_fields {
421+
field_names = ["word_count"]
422+
}
423+
exclude_fields {
424+
field_names = ["property_type"]
425+
}
426+
post_scan_actions {
427+
bigquery_export {
428+
results_table = "//bigquery.googleapis.com/projects/%{project_name}/datasets/${google_bigquery_dataset.tf_test_dataset.dataset_id}/tables/profile_export_%{random_suffix}"
429+
}
430+
}
431+
catalog_publishing_enabled = true
432+
}
433+
434+
project = "%{project_name}"
435+
436+
depends_on = [
437+
google_bigquery_dataset.tf_test_dataset
438+
]
439+
}
440+
`, context)
441+
}

0 commit comments

Comments
 (0)