|
| 1 | +# Copyright 2024 Google LLC |
| 2 | +# |
| 3 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +# you may not use this file except in compliance with the License. |
| 5 | +# You may obtain a copy of the License at |
| 6 | +# |
| 7 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +# |
| 9 | +# Unless required by applicable law or agreed to in writing, software |
| 10 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +# See the License for the specific language governing permissions and |
| 13 | +# limitations under the License. |
| 14 | +# [START dataplex_quickstart] |
| 15 | +import time |
| 16 | + |
| 17 | +from google.cloud import dataplex_v1 |
| 18 | +from google.protobuf import struct_pb2 |
| 19 | + |
| 20 | + |
| 21 | +# Method to demonstrate lifecycle of different Dataplex resources and their interactions. |
| 22 | +# Method creates Aspect Type, Entry Type, Entry Group and Entry, retrieves Entry |
| 23 | +# and cleans up created resources. |
| 24 | +def quickstart( |
| 25 | + project_id: str, |
| 26 | + location: str, |
| 27 | + aspect_type_id: str, |
| 28 | + entry_type_id: str, |
| 29 | + entry_group_id: str, |
| 30 | + entry_id: str, |
| 31 | +) -> None: |
| 32 | + # Initialize client that will be used to send requests across threads. This |
| 33 | + # client only needs to be created once, and can be reused for multiple requests. |
| 34 | + # After completing all of your requests, call the "__exit__()" method to safely |
| 35 | + # clean up any remaining background resources. Alternatively, use the client as |
| 36 | + # a context manager. |
| 37 | + with dataplex_v1.CatalogServiceClient() as client: |
| 38 | + # 0) Prepare variables used in following steps |
| 39 | + global_parent = f"projects/{project_id}/locations/global" |
| 40 | + specific_location_parent = f"projects/{project_id}/locations/{location}" |
| 41 | + |
| 42 | + # 1) Create Aspect Type that will be attached to Entry Type |
| 43 | + aspect_field = dataplex_v1.AspectType.MetadataTemplate( |
| 44 | + # The name must follow regex ^(([a-zA-Z]{1})([\\w\\-_]{0,62}))$ |
| 45 | + # That means name must only contain alphanumeric character or dashes or underscores, |
| 46 | + # start with an alphabet, and must be less than 63 characters. |
| 47 | + name="example_field", |
| 48 | + # Metadata Template is recursive structure, |
| 49 | + # primitive types such as "string" or "integer" indicate leaf node, |
| 50 | + # complex types such as "record" or "array" would require nested Metadata Template |
| 51 | + type="string", |
| 52 | + index=1, |
| 53 | + annotations=dataplex_v1.AspectType.MetadataTemplate.Annotations( |
| 54 | + description="example field to be filled during entry creation" |
| 55 | + ), |
| 56 | + constraints=dataplex_v1.AspectType.MetadataTemplate.Constraints( |
| 57 | + # Specifies if field will be required in Aspect Type. |
| 58 | + required=True |
| 59 | + ), |
| 60 | + ) |
| 61 | + aspect_type = dataplex_v1.AspectType( |
| 62 | + description="aspect type for dataplex quickstart", |
| 63 | + metadata_template=dataplex_v1.AspectType.MetadataTemplate( |
| 64 | + name="example_template", |
| 65 | + type="record", |
| 66 | + # Aspect Type fields, that themselves are Metadata Templates. |
| 67 | + record_fields=[aspect_field], |
| 68 | + ), |
| 69 | + ) |
| 70 | + aspect_type_create_operation = client.create_aspect_type( |
| 71 | + # Aspect Type is created in "global" location to highlight, that resources from |
| 72 | + # "global" region can be attached to Entry created in specific location |
| 73 | + parent=global_parent, |
| 74 | + aspect_type=aspect_type, |
| 75 | + aspect_type_id=aspect_type_id, |
| 76 | + ) |
| 77 | + created_aspect_type = aspect_type_create_operation.result(60) |
| 78 | + print(f"Step 1: Created aspect type -> {created_aspect_type.name}") |
| 79 | + |
| 80 | + # 2) Create Entry Type, of which type Entry will be created |
| 81 | + entry_type = dataplex_v1.EntryType( |
| 82 | + description="entry type for dataplex quickstart", |
| 83 | + required_aspects=[ |
| 84 | + dataplex_v1.EntryType.AspectInfo( |
| 85 | + # Aspect Type created in step 1 |
| 86 | + type=f"projects/{project_id}/locations/global/aspectTypes/{aspect_type_id}" |
| 87 | + ) |
| 88 | + ], |
| 89 | + ) |
| 90 | + entry_type_create_operation = client.create_entry_type( |
| 91 | + # Entry Type is created in "global" location to highlight, that resources from |
| 92 | + # "global" region can be attached to Entry created in specific location |
| 93 | + parent=global_parent, |
| 94 | + entry_type=entry_type, |
| 95 | + entry_type_id=entry_type_id, |
| 96 | + ) |
| 97 | + created_entry_type = entry_type_create_operation.result(60) |
| 98 | + print(f"Step 2: Created entry type -> {created_entry_type.name}") |
| 99 | + |
| 100 | + # 3) Create Entry Group in which Entry will be located |
| 101 | + entry_group = dataplex_v1.EntryGroup( |
| 102 | + description="entry group for dataplex quickstart" |
| 103 | + ) |
| 104 | + entry_group_create_operation = client.create_entry_group( |
| 105 | + # Entry Group is created for specific location |
| 106 | + parent=specific_location_parent, |
| 107 | + entry_group=entry_group, |
| 108 | + entry_group_id=entry_group_id, |
| 109 | + ) |
| 110 | + created_entry_group = entry_group_create_operation.result(60) |
| 111 | + print(f"Step 3: Created entry group -> {created_entry_group.name}") |
| 112 | + |
| 113 | + # 4) Create Entry |
| 114 | + # Wait 10 second to allow previously created resources to propagate |
| 115 | + time.sleep(10) |
| 116 | + aspect_key = f"{project_id}.global.{aspect_type_id}" |
| 117 | + entry = dataplex_v1.Entry( |
| 118 | + # Entry is an instance of Entry Type created in step 2 |
| 119 | + entry_type=f"projects/{project_id}/locations/global/entryTypes/{entry_type_id}", |
| 120 | + entry_source=dataplex_v1.EntrySource( |
| 121 | + description="entry for dataplex quickstart" |
| 122 | + ), |
| 123 | + aspects={ |
| 124 | + # Attach Aspect that is an instance of Aspect Type created in step 1 |
| 125 | + aspect_key: dataplex_v1.Aspect( |
| 126 | + aspect_type=f"projects/{project_id}/locations/global/aspectTypes/{aspect_type_id}", |
| 127 | + data=struct_pb2.Struct( |
| 128 | + fields={ |
| 129 | + "example_field": struct_pb2.Value( |
| 130 | + string_value="example value for the field" |
| 131 | + ), |
| 132 | + } |
| 133 | + ), |
| 134 | + ) |
| 135 | + }, |
| 136 | + ) |
| 137 | + created_entry = client.create_entry( |
| 138 | + # Entry is created in specific location, but it is still possible to link it with |
| 139 | + # resources (Aspect Type and Entry Type) from "global" location |
| 140 | + parent=f"projects/{project_id}/locations/{location}/entryGroups/{entry_group_id}", |
| 141 | + entry=entry, |
| 142 | + entry_id=entry_id, |
| 143 | + ) |
| 144 | + print(f"Step 4: Created entry -> {created_entry.name}") |
| 145 | + |
| 146 | + # 5) Retrieve created Entry |
| 147 | + get_entry_request = dataplex_v1.GetEntryRequest( |
| 148 | + name=f"projects/{project_id}/locations/{location}/entryGroups/{entry_group_id}/entries/{entry_id}", |
| 149 | + view=dataplex_v1.EntryView.FULL, |
| 150 | + ) |
| 151 | + retrieved_entry = client.get_entry(request=get_entry_request) |
| 152 | + print(f"Step 5: Retrieved entry -> {retrieved_entry.name}") |
| 153 | + for retrieved_aspect in retrieved_entry.aspects.values(): |
| 154 | + print("Retrieved aspect for entry:") |
| 155 | + print(f" * aspect type -> {retrieved_aspect.aspect_type}") |
| 156 | + print(f" * aspect field value -> {retrieved_aspect.data['example_field']}") |
| 157 | + |
| 158 | + # 6) Use Search capabilities to find Entry |
| 159 | + # Wait 30 second to allow resources to propagate to Search |
| 160 | + print("Step 6: Waiting for resources to propagate to Search...") |
| 161 | + time.sleep(30) |
| 162 | + search_entries_request = dataplex_v1.SearchEntriesRequest( |
| 163 | + name=global_parent, query="name:dataplex-quickstart-entry" |
| 164 | + ) |
| 165 | + results = client.search_entries(search_entries_request) |
| 166 | + search_entries_response = results._response |
| 167 | + entries_from_search = [ |
| 168 | + result.dataplex_entry for result in search_entries_response.results |
| 169 | + ] |
| 170 | + print("Entries found in Search:") |
| 171 | + # Please note in output that Entry Group and Entry Type are also represented as Entries |
| 172 | + for entry_from_search in entries_from_search: |
| 173 | + print(f" * {entry_from_search.name}") |
| 174 | + |
| 175 | + # 7) Clean created resources |
| 176 | + client.delete_entry_group( |
| 177 | + name=f"projects/{project_id}/locations/{location}/entryGroups/{entry_group_id}" |
| 178 | + ) |
| 179 | + client.delete_entry_type( |
| 180 | + name=f"projects/{project_id}/locations/global/entryTypes/{entry_type_id}" |
| 181 | + ) |
| 182 | + client.delete_aspect_type( |
| 183 | + name=f"projects/{project_id}/locations/global/aspectTypes/{aspect_type_id}" |
| 184 | + ) |
| 185 | + print("Step 7: Successfully cleaned up resources") |
| 186 | + |
| 187 | + |
| 188 | +if __name__ == "__main__": |
| 189 | + # TODO(developer): Replace these variables before running the sample. |
| 190 | + project_id = "MY_PROJECT_ID" |
| 191 | + # Available locations: https://cloud.google.com/dataplex/docs/locations |
| 192 | + location = "MY_LOCATION" |
| 193 | + # Variables below can be replaced with custom values or defaults can be kept |
| 194 | + aspect_type_id = "dataplex-quickstart-aspect-type" |
| 195 | + entry_type_id = "dataplex-quickstart-entry-type" |
| 196 | + entry_group_id = "dataplex-quickstart-entry-group" |
| 197 | + entry_id = "dataplex-quickstart-entry" |
| 198 | + |
| 199 | + quickstart( |
| 200 | + project_id, location, aspect_type_id, entry_type_id, entry_group_id, entry_id |
| 201 | + ) |
| 202 | +# [END dataplex_quickstart] |
0 commit comments