Skip to content

Commit 1e50597

Browse files
mllMarek Lipert
mll
and
Marek Lipert
authored
Improve Dataplex Entry support for first party entries. (#15861)
Co-authored-by: Marek Lipert <[email protected]>
1 parent 064ba1a commit 1e50597

File tree

8 files changed

+519
-1
lines changed

8 files changed

+519
-1
lines changed

‎mmv1/products/dataplex/Entry.yaml‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,9 @@ references:
3232

3333
base_url: 'projects/{{project}}/locations/{{location}}/entryGroups/{{entry_group_id}}/entries/{{entry_id}}'
3434
self_link: 'projects/{{project}}/locations/{{location}}/entryGroups/{{entry_group_id}}/entries/{{entry_id}}'
35-
create_url: 'projects/{{project}}/locations/{{location}}/entryGroups/{{entry_group_id}}/entries?entryId={{entry_id}}'
35+
create_url: 'projects/{{project}}/locations/{{location}}/entryGroups/{{entry_group_id}}/entries/{{entry_id}}'
3636
update_verb: 'PATCH'
37+
create_verb: 'PATCH'
3738
update_mask: true
3839
import_format:
3940
- 'projects/{{project}}/locations/{{location}}/entryGroups/{{entry_group_id}}/entries/{{entry_id}}'
@@ -45,6 +46,11 @@ custom_code:
4546
custom_import: templates/terraform/custom_import/dataplex_entry.go.tmpl
4647
pre_read: templates/terraform/pre_read/dataplex_entry.go.tmpl
4748
pre_update: templates/terraform/pre_update/dataplex_entry.go.tmpl
49+
pre_create: templates/terraform/pre_create/dataplex_entry.go.tmpl
50+
pre_delete: templates/terraform/pre_delete/dataplex_entry.go.tmpl
51+
52+
error_retry_predicates:
53+
- 'transport_tpg.IsDataplex1PEntryIngestedError'
4854

4955
timeouts:
5056
insert_minutes: 5
@@ -74,6 +80,30 @@ examples:
7480
entry_type_name: "entry-type-full"
7581
test_env_vars:
7682
project_number: 'PROJECT_NUMBER'
83+
- name: 'dataplex_entry_bigquery_table'
84+
primary_resource_id: 'tf_test_table'
85+
primary_resource_name: 'fmt.Sprintf("tf_test_table%s", context["random_suffix"])'
86+
ignore_read_extra:
87+
- 'aspects'
88+
vars:
89+
table_id: 'table-basic'
90+
dataset_id: 'dataset-basic'
91+
aspect_type_name: "aspect-type"
92+
test_env_vars:
93+
project_number: 'PROJECT_NUMBER'
94+
project_id: 'PROJECT_NAME'
95+
- name: 'dataplex_entry_glossary_term'
96+
primary_resource_id: 'tf_test_glossary_term'
97+
primary_resource_name: 'fmt.Sprintf("tf_test_glossary_term%s", context["random_suffix"])'
98+
ignore_read_extra:
99+
- 'aspects'
100+
vars:
101+
glossary_id: 'glossary-basic'
102+
glossary_term_id: 'glossary-term'
103+
test_env_vars:
104+
project_number: 'PROJECT_NUMBER'
105+
project_id: 'PROJECT_NAME'
106+
77107

78108
parameters:
79109
- name: 'location'

‎mmv1/templates/terraform/examples/dataplex_entry_bigquery_table.tf.tmpl‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
resource "google_dataplex_aspect_type" "aspect-type-full-one" {
2+
aspect_type_id = "{{index $.Vars "aspect_type_name"}}-one"
3+
location = "us-central1"
4+
project = "{{index $.TestEnvVars "project_number"}}"
5+
6+
metadata_template = <<EOF
7+
{
8+
"name": "tf-test-template",
9+
"type": "record",
10+
"recordFields": [
11+
{
12+
"name": "type",
13+
"type": "enum",
14+
"annotations": {
15+
"displayName": "Type",
16+
"description": "Specifies the type of view represented by the entry."
17+
},
18+
"index": 1,
19+
"constraints": {
20+
"required": true
21+
},
22+
"enumValues": [
23+
{
24+
"name": "VIEW",
25+
"index": 1
26+
}
27+
]
28+
}
29+
]
30+
}
31+
EOF
32+
}
33+
34+
resource "google_dataplex_aspect_type" "aspect-type-full-two" {
35+
aspect_type_id = "{{index $.Vars "aspect_type_name"}}-two"
36+
location = "us-central1"
37+
project = "{{index $.TestEnvVars "project_number"}}"
38+
39+
metadata_template = <<EOF
40+
{
41+
"name": "tf-test-template",
42+
"type": "record",
43+
"recordFields": [
44+
{
45+
"name": "story",
46+
"type": "enum",
47+
"annotations": {
48+
"displayName": "Story",
49+
"description": "Specifies the story of an entry."
50+
},
51+
"index": 1,
52+
"constraints": {
53+
"required": true
54+
},
55+
"enumValues": [
56+
{
57+
"name": "SEQUENCE",
58+
"index": 1
59+
}
60+
]
61+
}
62+
]
63+
}
64+
EOF
65+
}
66+
67+
resource "google_bigquery_dataset" "example-dataset" {
68+
dataset_id = "{{index $.Vars "dataset_id"}}"
69+
friendly_name = "Example Dataset"
70+
location = "us-central1"
71+
delete_contents_on_destroy = true
72+
}
73+
74+
75+
resource "google_bigquery_table" "example-table" {
76+
dataset_id = google_bigquery_dataset.example-dataset.dataset_id
77+
table_id = "{{index $.Vars "table_id"}}"
78+
deletion_protection = false
79+
# Define the table schema
80+
schema = jsonencode([
81+
{
82+
name = "event_time"
83+
type = "TIMESTAMP"
84+
mode = "REQUIRED"
85+
},
86+
{
87+
name = "user_id"
88+
type = "STRING"
89+
mode = "NULLABLE"
90+
},
91+
{
92+
name = "event_type"
93+
type = "STRING"
94+
mode = "NULLABLE"
95+
}
96+
])
97+
}
98+
99+
resource "google_dataplex_entry" "{{$.PrimaryResourceId}}" {
100+
entry_group_id = "@bigquery"
101+
project = "{{index $.TestEnvVars "project_number"}}"
102+
location = "us-central1"
103+
entry_id = "bigquery.googleapis.com/projects/{{index $.TestEnvVars "project_id"}}/datasets/${google_bigquery_dataset.example-dataset.dataset_id}/tables/${google_bigquery_table.example-table.table_id}"
104+
entry_type = "projects/655216118709/locations/global/entryTypes/bigquery-table"
105+
fully_qualified_name = "bigquery:{{index $.TestEnvVars "project_id"}}.${google_bigquery_dataset.example-dataset.dataset_id}.${google_bigquery_table.example-table.table_id}"
106+
parent_entry = "projects/{{index $.TestEnvVars "project_number"}}/locations/us-central1/entryGroups/@bigquery/entries/bigquery.googleapis.com/projects/{{index $.TestEnvVars "project_id"}}/datasets/${google_bigquery_dataset.example-dataset.dataset_id}"
107+
108+
aspects {
109+
aspect_key = "{{index $.TestEnvVars "project_number"}}.us-central1.{{index $.Vars "aspect_type_name"}}-one"
110+
aspect {
111+
data = <<EOF
112+
{"type": "VIEW" }
113+
EOF
114+
}
115+
}
116+
117+
aspects {
118+
aspect_key = "{{index $.TestEnvVars "project_number"}}.us-central1.{{index $.Vars "aspect_type_name"}}[email protected]_type"
119+
aspect {
120+
data = <<EOF
121+
{"story": "SEQUENCE" }
122+
EOF
123+
}
124+
}
125+
depends_on = [google_dataplex_aspect_type.aspect-type-full-two, google_dataplex_aspect_type.aspect-type-full-one]
126+
}

‎mmv1/templates/terraform/examples/dataplex_entry_glossary_term.tf.tmpl‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
resource "google_dataplex_glossary" "example-glossary" {
2+
glossary_id = "{{index $.Vars "glossary_id"}}"
3+
location = "us-central1"
4+
}
5+
6+
resource "google_dataplex_glossary_term" "example-glossary-term" {
7+
parent = "projects/{{index $.TestEnvVars "project_id"}}/locations/us-central1/glossaries/${google_dataplex_glossary.example-glossary.glossary_id}"
8+
glossary_id = google_dataplex_glossary.example-glossary.glossary_id
9+
location = "us-central1"
10+
term_id = "{{index $.Vars "glossary_term_id"}}"
11+
}
12+
13+
resource "google_dataplex_entry" "{{$.PrimaryResourceId}}" {
14+
entry_group_id = "@dataplex"
15+
project = "{{index $.TestEnvVars "project_number"}}"
16+
location = "us-central1"
17+
entry_id = "projects/{{index $.TestEnvVars "project_number"}}/locations/us-central1/glossaries/${google_dataplex_glossary.example-glossary.glossary_id}/terms/${google_dataplex_glossary_term.example-glossary-term.term_id}"
18+
entry_type = "projects/655216118709/locations/global/entryTypes/glossary-term"
19+
parent_entry = "projects/{{index $.TestEnvVars "project_number"}}/locations/us-central1/entryGroups/@dataplex/entries/projects/{{index $.TestEnvVars "project_number"}}/locations/us-central1/glossaries/${google_dataplex_glossary.example-glossary.glossary_id}"
20+
21+
aspects {
22+
aspect_key = "655216118709.global.overview"
23+
aspect {
24+
data = <<EOF
25+
{"content": "Term Content" }
26+
EOF
27+
}
28+
}
29+
}

‎mmv1/templates/terraform/pre_create/dataplex_entry.go.tmpl‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
if v, ok := d.GetOkExists("entry_group_id"); ok && strings.HasPrefix(v.(string), "@") {
2+
url, err = transport_tpg.AddQueryParams(url, map[string]string{"allow_missing": "false",
3+
"updateMask": "aspects"})
4+
} else {
5+
url, err = transport_tpg.AddQueryParams(url, map[string]string{"allow_missing": "true"})
6+
}

‎mmv1/templates/terraform/pre_delete/dataplex_entry.go.tmpl‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
if v, ok := d.GetOkExists("entry_group_id"); ok && strings.HasPrefix(v.(string), "@") {
2+
// Ingestion based resources need to be removed from terraform state but cannot be deleted in Dataplex.
3+
d.SetId("")
4+
return nil
5+
}

‎mmv1/templates/terraform/pre_update/dataplex_entry.go.tmpl‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,3 +37,5 @@ if d.HasChange("aspects") {
3737
return err
3838
}
3939
}
40+
41+
url, err = transport_tpg.AddQueryParams(url, map[string]string{"allow_missing": "false"})

0 commit comments

Comments
 (0)