|
| 1 | +/* |
| 2 | + * Copyright 2024 Google LLC |
| 3 | + * |
| 4 | + * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | + * you may not use this file except in compliance with the License. |
| 6 | + * You may obtain a copy of the License at |
| 7 | + * |
| 8 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | + * |
| 10 | + * Unless required by applicable law or agreed to in writing, software |
| 11 | + * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | + * See the License for the specific language governing permissions and |
| 14 | + * limitations under the License. |
| 15 | + */ |
| 16 | + |
| 17 | +package dataplex; |
| 18 | + |
| 19 | +// [START dataplex_quickstart] |
| 20 | +import com.google.cloud.dataplex.v1.Aspect; |
| 21 | +import com.google.cloud.dataplex.v1.AspectType; |
| 22 | +import com.google.cloud.dataplex.v1.CatalogServiceClient; |
| 23 | +import com.google.cloud.dataplex.v1.Entry; |
| 24 | +import com.google.cloud.dataplex.v1.EntryGroup; |
| 25 | +import com.google.cloud.dataplex.v1.EntryGroupName; |
| 26 | +import com.google.cloud.dataplex.v1.EntryName; |
| 27 | +import com.google.cloud.dataplex.v1.EntrySource; |
| 28 | +import com.google.cloud.dataplex.v1.EntryType; |
| 29 | +import com.google.cloud.dataplex.v1.EntryView; |
| 30 | +import com.google.cloud.dataplex.v1.GetEntryRequest; |
| 31 | +import com.google.cloud.dataplex.v1.LocationName; |
| 32 | +import com.google.cloud.dataplex.v1.SearchEntriesRequest; |
| 33 | +import com.google.cloud.dataplex.v1.SearchEntriesResult; |
| 34 | +import com.google.protobuf.Struct; |
| 35 | +import com.google.protobuf.Value; |
| 36 | +import java.io.IOException; |
| 37 | +import java.util.List; |
| 38 | +import java.util.Map; |
| 39 | +import java.util.concurrent.ExecutionException; |
| 40 | +import java.util.stream.Collectors; |
| 41 | + |
| 42 | +public class Quickstart { |
| 43 | + |
| 44 | + public static void main(String[] args) { |
| 45 | + // TODO(developer): Replace these variables before running the sample. |
| 46 | + String projectId = "MY_PROJECT_ID"; |
| 47 | + // Available locations: https://cloud.google.com/dataplex/docs/locations |
| 48 | + String location = "MY_LOCATION"; |
| 49 | + // Variables below can be replaced with custom values or defaults can be kept |
| 50 | + String aspectTypeId = "dataplex-quickstart-aspect-type"; |
| 51 | + String entryTypeId = "dataplex-quickstart-entry-type"; |
| 52 | + String entryGroupId = "dataplex-quickstart-entry-group"; |
| 53 | + String entryId = "dataplex-quickstart-entry"; |
| 54 | + |
| 55 | + quickstart(projectId, location, aspectTypeId, entryTypeId, entryGroupId, entryId); |
| 56 | + } |
| 57 | + |
| 58 | + // Method to demonstrate lifecycle of different Dataplex resources and their interactions. |
| 59 | + // Method creates Aspect Type, Entry Type, Entry Group and Entry, retrieves Entry |
| 60 | + // and cleans up created resources. |
| 61 | + public static void quickstart( |
| 62 | + String projectId, |
| 63 | + String location, |
| 64 | + String aspectTypeId, |
| 65 | + String entryTypeId, |
| 66 | + String entryGroupId, |
| 67 | + String entryId) { |
| 68 | + // Initialize client that will be used to send requests. This client only needs to be created |
| 69 | + // once, and can be reused for multiple requests. |
| 70 | + try (CatalogServiceClient client = CatalogServiceClient.create()) { |
| 71 | + // 0) Prepare variables used in following steps |
| 72 | + LocationName globalLocationName = LocationName.of(projectId, "global"); |
| 73 | + LocationName specificLocationName = LocationName.of(projectId, location); |
| 74 | + |
| 75 | + // 1) Create Aspect Type that will be attached to Entry Type |
| 76 | + AspectType.MetadataTemplate aspectField = |
| 77 | + AspectType.MetadataTemplate.newBuilder() |
| 78 | + // The name must follow regex ^(([a-zA-Z]{1})([\\w\\-_]{0,62}))$ |
| 79 | + // That means name must only contain alphanumeric character or dashes or underscores, |
| 80 | + // start with an alphabet, and must be less than 63 characters. |
| 81 | + .setName("example_field") |
| 82 | + // Metadata Template is recursive structure, |
| 83 | + // primitive types such as "string" or "integer" indicate leaf node, |
| 84 | + // complex types such as "record" or "array" would require nested Metadata Template |
| 85 | + .setType("string") |
| 86 | + .setIndex(1) |
| 87 | + .setAnnotations( |
| 88 | + AspectType.MetadataTemplate.Annotations.newBuilder() |
| 89 | + .setDescription("example field to be filled during entry creation") |
| 90 | + .build()) |
| 91 | + .setConstraints( |
| 92 | + AspectType.MetadataTemplate.Constraints.newBuilder() |
| 93 | + // Specifies if field will be required in Aspect Type. |
| 94 | + .setRequired(true) |
| 95 | + .build()) |
| 96 | + .build(); |
| 97 | + AspectType aspectType = |
| 98 | + AspectType.newBuilder() |
| 99 | + .setDescription("aspect type for dataplex quickstart") |
| 100 | + .setMetadataTemplate( |
| 101 | + AspectType.MetadataTemplate.newBuilder() |
| 102 | + .setName("example_template") |
| 103 | + .setType("record") |
| 104 | + // Aspect Type fields, that themselves are Metadata Templates |
| 105 | + .addAllRecordFields(List.of(aspectField)) |
| 106 | + .build()) |
| 107 | + .build(); |
| 108 | + AspectType createdAspectType = |
| 109 | + client |
| 110 | + .createAspectTypeAsync( |
| 111 | + // Aspect Type is created in "global" location to highlight, that resources from |
| 112 | + // "global" region can be attached to Entry created in specific location |
| 113 | + globalLocationName, aspectType, aspectTypeId) |
| 114 | + .get(); |
| 115 | + System.out.println("Step 1: Created aspect type -> " + createdAspectType.getName()); |
| 116 | + |
| 117 | + // 2) Create Entry Type, of which type Entry will be created |
| 118 | + EntryType entryType = |
| 119 | + EntryType.newBuilder() |
| 120 | + .setDescription("entry type for dataplex quickstart") |
| 121 | + .addRequiredAspects( |
| 122 | + EntryType.AspectInfo.newBuilder() |
| 123 | + // Aspect Type created in step 1 |
| 124 | + .setType( |
| 125 | + String.format( |
| 126 | + "projects/%s/locations/global/aspectTypes/%s", |
| 127 | + projectId, aspectTypeId)) |
| 128 | + .build()) |
| 129 | + .build(); |
| 130 | + EntryType createdEntryType = |
| 131 | + client |
| 132 | + // Entry Type is created in "global" location to highlight, that resources from |
| 133 | + // "global" region can be attached to Entry created in specific location |
| 134 | + .createEntryTypeAsync(globalLocationName, entryType, entryTypeId) |
| 135 | + .get(); |
| 136 | + System.out.println("Step 2: Created entry type -> " + createdEntryType.getName()); |
| 137 | + |
| 138 | + // 3) Create Entry Group in which Entry will be located |
| 139 | + EntryGroup entryGroup = |
| 140 | + EntryGroup.newBuilder().setDescription("entry group for dataplex quickstart").build(); |
| 141 | + EntryGroup createdEntryGroup = |
| 142 | + client |
| 143 | + // Entry Group is created for specific location |
| 144 | + .createEntryGroupAsync(specificLocationName, entryGroup, entryGroupId) |
| 145 | + .get(); |
| 146 | + System.out.println("Step 3: Created entry group -> " + createdEntryGroup.getName()); |
| 147 | + |
| 148 | + // 4) Create Entry |
| 149 | + // Wait 10 second to allow previously created resources to propagate |
| 150 | + Thread.sleep(10000); |
| 151 | + String aspectKey = String.format("%s.global.%s", projectId, aspectTypeId); |
| 152 | + Entry entry = |
| 153 | + Entry.newBuilder() |
| 154 | + .setEntryType( |
| 155 | + // Entry is an instance of Entry Type created in step 2 |
| 156 | + String.format( |
| 157 | + "projects/%s/locations/global/entryTypes/%s", projectId, entryTypeId)) |
| 158 | + .setEntrySource( |
| 159 | + EntrySource.newBuilder().setDescription("entry for dataplex quickstart").build()) |
| 160 | + .putAllAspects( |
| 161 | + Map.of( |
| 162 | + // Attach Aspect that is an instance of Aspect Type created in step 1 |
| 163 | + aspectKey, |
| 164 | + Aspect.newBuilder() |
| 165 | + .setAspectType( |
| 166 | + String.format( |
| 167 | + "projects/%s/locations/global/aspectTypes/%s", |
| 168 | + projectId, aspectTypeId)) |
| 169 | + .setData( |
| 170 | + Struct.newBuilder() |
| 171 | + .putFields( |
| 172 | + "example_field", |
| 173 | + Value.newBuilder() |
| 174 | + .setStringValue("example value for the field") |
| 175 | + .build()) |
| 176 | + .build()) |
| 177 | + .build())) |
| 178 | + .build(); |
| 179 | + Entry createdEntry = |
| 180 | + client.createEntry( |
| 181 | + // Entry is created in specific location, but it is still possible to link it with |
| 182 | + // resources (Aspect Type and Entry Type) from "global" location |
| 183 | + EntryGroupName.of(projectId, location, entryGroupId), entry, entryId); |
| 184 | + System.out.println("Step 4: Created entry -> " + createdEntry.getName()); |
| 185 | + |
| 186 | + // 5) Retrieve created Entry |
| 187 | + GetEntryRequest getEntryRequest = |
| 188 | + GetEntryRequest.newBuilder() |
| 189 | + .setName(EntryName.of(projectId, location, entryGroupId, entryId).toString()) |
| 190 | + .setView(EntryView.FULL) |
| 191 | + .build(); |
| 192 | + Entry retrievedEntry = client.getEntry(getEntryRequest); |
| 193 | + System.out.println("Step 5: Retrieved entry -> " + retrievedEntry.getName()); |
| 194 | + retrievedEntry |
| 195 | + .getAspectsMap() |
| 196 | + .values() |
| 197 | + .forEach( |
| 198 | + retrievedAspect -> { |
| 199 | + System.out.println("Retrieved aspect for entry:"); |
| 200 | + System.out.println(" * aspect type -> " + retrievedAspect.getAspectType()); |
| 201 | + System.out.println( |
| 202 | + " * aspect field value -> " |
| 203 | + + retrievedAspect |
| 204 | + .getData() |
| 205 | + .getFieldsMap() |
| 206 | + .get("example_field") |
| 207 | + .getStringValue()); |
| 208 | + }); |
| 209 | + |
| 210 | + // 6) Use Search capabilities to find Entry |
| 211 | + // Wait 30 second to allow resources to propagate to Search |
| 212 | + System.out.println("Step 6: Waiting for resources to propagate to Search..."); |
| 213 | + Thread.sleep(30000); |
| 214 | + SearchEntriesRequest searchEntriesRequest = |
| 215 | + SearchEntriesRequest.newBuilder() |
| 216 | + .setName(globalLocationName.toString()) |
| 217 | + .setQuery("name:dataplex-quickstart-entry") |
| 218 | + .build(); |
| 219 | + CatalogServiceClient.SearchEntriesPagedResponse searchEntriesResponse = |
| 220 | + client.searchEntries(searchEntriesRequest); |
| 221 | + List<Entry> entriesFromSearch = |
| 222 | + searchEntriesResponse.getPage().getResponse().getResultsList().stream() |
| 223 | + .map(SearchEntriesResult::getDataplexEntry) |
| 224 | + .collect(Collectors.toList()); |
| 225 | + System.out.println("Entries found in Search:"); |
| 226 | + // Please note in output that Entry Group and Entry Type are also represented as Entries |
| 227 | + entriesFromSearch.forEach( |
| 228 | + entryFromSearch -> System.out.println(" * " + entryFromSearch.getName())); |
| 229 | + |
| 230 | + // 7) Clean created resources |
| 231 | + client |
| 232 | + .deleteEntryGroupAsync( |
| 233 | + String.format( |
| 234 | + "projects/%s/locations/%s/entryGroups/%s", projectId, location, entryGroupId)) |
| 235 | + .get(); |
| 236 | + client |
| 237 | + .deleteEntryTypeAsync( |
| 238 | + String.format("projects/%s/locations/global/entryTypes/%s", projectId, entryTypeId)) |
| 239 | + .get(); |
| 240 | + client |
| 241 | + .deleteAspectTypeAsync( |
| 242 | + String.format("projects/%s/locations/global/aspectTypes/%s", projectId, aspectTypeId)) |
| 243 | + .get(); |
| 244 | + System.out.println("Step 7: Successfully cleaned up resources"); |
| 245 | + |
| 246 | + } catch (IOException | InterruptedException | ExecutionException e) { |
| 247 | + System.err.println("Error during quickstart execution: " + e); |
| 248 | + } |
| 249 | + } |
| 250 | +} |
| 251 | +// [END dataplex_quickstart] |
0 commit comments