Skip to content

Commit 8cd5a4a

Browse files
gcf-owl-bot[bot]gcf-owl-bot[bot]
authored
feat: [dataplex] A new message DataDocumentationSpec is added representing Data Documentation Spec (#6837)
* feat: A new message `DataDocumentationSpec` is added representing Data Documentation Spec feat: A new message `DataDocumentationResult` is added representing Data Documentation Result feat: A new field `data_documentation_spec` is added for Data Documentation Spec to message `.google.cloud.dataplex.v1.DataScan` feat: A new field `data_documentation_result` is added for Data Documentation Result to message `.google.cloud.dataplex.v1.DataScan` feat: A new field `data_documentation_spec` is added for Data Documentation Spec to message `.google.cloud.dataplex.v1.DataScanJob` feat: A new field `data_documentation_result` is added for Data Documentation Result in `.google.cloud.dataplex.v1.DataScanJob` feat: A new data scan type Data documentation added. docs: A comment for message `DataScan` is changed docs: A comment for field `resource` in message `.google.cloud.dataplex.v1.DataSource` is changed PiperOrigin-RevId: 822921010 Source-Link: googleapis/googleapis@5ad924f Source-Link: googleapis/googleapis-gen@854a6eb Copy-Tag: eyJwIjoicGFja2FnZXMvZ29vZ2xlLWNsb3VkLWRhdGFwbGV4Ly5Pd2xCb3QueWFtbCIsImgiOiI4NTRhNmViMDhmOTA4NjViMTEzMjk1MTk5NjI4OGUxYWI5ZDk2OTkxIn0= * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
1 parent 31da8eb commit 8cd5a4a

26 files changed

+3487
-343
lines changed

‎packages/google-cloud-dataplex/protos/google/cloud/dataplex/v1/catalog.proto‎

Lines changed: 34 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,11 @@ service CatalogService {
246246
}
247247

248248
// Lists Entries within an EntryGroup.
249+
// Caution: The Vertex AI, Bigtable, Spanner, Pub/Sub, Dataform, and Dataproc
250+
// Metastore metadata that is stored in Dataplex Universal Catalog is
251+
// changing. For more information, see [Changes to metadata stored in
252+
// Dataplex Universal
253+
// Catalog](https://cloud.google.com/dataplex/docs/metadata-changes).
249254
rpc ListEntries(ListEntriesRequest) returns (ListEntriesResponse) {
250255
option (google.api.http) = {
251256
get: "/v1/{parent=projects/*/locations/*/entryGroups/*}/entries"
@@ -254,6 +259,11 @@ service CatalogService {
254259
}
255260

256261
// Gets an Entry.
262+
// Caution: The Vertex AI, Bigtable, Spanner, Pub/Sub, Dataform, and Dataproc
263+
// Metastore metadata that is stored in Dataplex Universal Catalog is
264+
// changing. For more information, see [Changes to metadata stored in
265+
// Dataplex Universal
266+
// Catalog](https://cloud.google.com/dataplex/docs/metadata-changes).
257267
rpc GetEntry(GetEntryRequest) returns (Entry) {
258268
option (google.api.http) = {
259269
get: "/v1/{name=projects/*/locations/*/entryGroups/*/entries/**}"
@@ -262,6 +272,11 @@ service CatalogService {
262272
}
263273

264274
// Looks up an entry by name using the permission on the source system.
275+
// Caution: The Vertex AI, Bigtable, Spanner, Pub/Sub, Dataform, and Dataproc
276+
// Metastore metadata that is stored in Dataplex Universal Catalog is
277+
// changing. For more information, see [Changes to metadata stored in
278+
// Dataplex Universal
279+
// Catalog](https://cloud.google.com/dataplex/docs/metadata-changes).
265280
rpc LookupEntry(LookupEntryRequest) returns (Entry) {
266281
option (google.api.http) = {
267282
get: "/v1/{name=projects/*/locations/*}:lookupEntry"
@@ -369,6 +384,15 @@ message AspectType {
369384
[(google.api.field_behavior) = IMMUTABLE];
370385
}
371386

387+
// Classifies the data stored by the aspect.
388+
enum DataClassification {
389+
// Denotes that the aspect contains only metadata.
390+
DATA_CLASSIFICATION_UNSPECIFIED = 0;
391+
392+
// Metadata and data classification.
393+
METADATA_AND_DATA = 1;
394+
}
395+
372396
// MetadataTemplate definition for an AspectType.
373397
message MetadataTemplate {
374398
// Definition of Enumvalue, to be used for enum fields.
@@ -527,6 +551,12 @@ message AspectType {
527551
// delete requests to ensure it has an up-to-date value before proceeding.
528552
string etag = 8;
529553

554+
// Optional. Immutable. Stores data classification of the aspect.
555+
DataClassification data_classification = 9 [
556+
(google.api.field_behavior) = OPTIONAL,
557+
(google.api.field_behavior) = IMMUTABLE
558+
];
559+
530560
// Immutable. Defines the Authorization for this type.
531561
Authorization authorization = 52 [(google.api.field_behavior) = IMMUTABLE];
532562

@@ -1719,14 +1749,13 @@ message MetadataJob {
17191749

17201750
// A boundary on the scope of impact that the metadata import job can have.
17211751
message ImportJobScope {
1722-
// Required. The entry group that is in scope for the import job,
1723-
// specified as a relative resource name in the format
1752+
// Required. The entry groups that are in scope for the import job,
1753+
// specified as relative resource names in the format
17241754
// `projects/{project_number_or_id}/locations/{location_id}/entryGroups/{entry_group_id}`.
1725-
// Only entries and aspects that belong to the specified entry group are
1755+
// Only entries and aspects that belong to the specified entry groups are
17261756
// affected by the job.
17271757
//
1728-
// Must contain exactly one element. The entry group and the job
1729-
// must be in the same location.
1758+
// The entry groups and the job must be in the same location.
17301759
repeated string entry_groups = 1 [
17311760
(google.api.field_behavior) = REQUIRED,
17321761
(google.api.resource_reference) = {
@@ -2066,8 +2095,6 @@ enum EntryView {
20662095
// aspects exceeds 100, the first 100 will be returned.
20672096
CUSTOM = 3;
20682097

2069-
// Returns all aspects. If the number of aspects exceeds 100, the first
2070-
// 100 will be returned.
20712098
ALL = 4;
20722099
}
20732100

‎packages/google-cloud-dataplex/protos/google/cloud/dataplex/v1/cmek.proto‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,9 @@ message EncryptionConfig {
189189
// Output only. Details of the failure if anything related to Cmek db fails.
190190
FailureDetails failure_details = 7
191191
[(google.api.field_behavior) = OUTPUT_ONLY];
192+
193+
// Optional. Represent the state of CMEK opt-in for metastore.
194+
bool enable_metastore_encryption = 8 [(google.api.field_behavior) = OPTIONAL];
192195
}
193196

194197
// Create EncryptionConfig Request

‎packages/google-cloud-dataplex/protos/google/cloud/dataplex/v1/data_documentation.proto‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
// Copyright 2025 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
syntax = "proto3";
16+
17+
package google.cloud.dataplex.v1;
18+
19+
import "google/api/field_behavior.proto";
20+
21+
option go_package = "cloud.google.com/go/dataplex/apiv1/dataplexpb;dataplexpb";
22+
option java_multiple_files = true;
23+
option java_outer_classname = "DataDocumentationProto";
24+
option java_package = "com.google.cloud.dataplex.v1";
25+
26+
// DataDocumentation scan related spec.
27+
message DataDocumentationSpec {}
28+
29+
// The output of a DataDocumentation scan.
30+
message DataDocumentationResult {
31+
// Generated metadata about the table.
32+
message TableResult {
33+
// Output only. The service-qualified full resource name of the cloud
34+
// resource. Ex:
35+
// //bigquery.googleapis.com/projects/PROJECT_ID/datasets/DATASET_ID/tables/TABLE_ID
36+
string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
37+
38+
// Output only. Generated description of the table.
39+
string overview = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
40+
41+
// Output only. Schema of the table with generated metadata of the columns
42+
// in the schema.
43+
Schema schema = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
44+
45+
// Output only. Sample SQL queries for the table.
46+
repeated Query queries = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
47+
}
48+
49+
// A sample SQL query in data documentation.
50+
message Query {
51+
// Output only. The SQL query string which can be executed.
52+
string sql = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
53+
54+
// Output only. The description for the query.
55+
string description = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
56+
}
57+
58+
// Schema of the table with generated metadata of columns.
59+
message Schema {
60+
// Output only. The list of columns.
61+
repeated Field fields = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
62+
}
63+
64+
// Column of a table with generated metadata and nested fields.
65+
message Field {
66+
// Output only. The name of the column.
67+
string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
68+
69+
// Output only. Generated description for columns and fields.
70+
string description = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
71+
72+
// Output only. Nested fields.
73+
repeated Field fields = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
74+
}
75+
76+
// The result of the data documentation scan.
77+
oneof result {
78+
// Output only. Table result for insights.
79+
TableResult table_result = 8 [(google.api.field_behavior) = OUTPUT_ONLY];
80+
}
81+
}

‎packages/google-cloud-dataplex/protos/google/cloud/dataplex/v1/data_quality.proto‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,35 @@ message DataQualityResult {
159159
[(google.api.field_behavior) = OUTPUT_ONLY];
160160
}
161161

162+
// The assets generated by Anomaly Detection Data Scan.
163+
message AnomalyDetectionGeneratedAssets {
164+
// Output only. The result table for anomaly detection.
165+
// Format:
166+
// PROJECT_ID.DATASET_ID.TABLE_ID
167+
// If the result table is set at AnomalyDetectionAssets, the result table
168+
// here would be the same as the one set in the
169+
// AnomalyDetectionAssets.result_table.
170+
string result_table = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
171+
172+
// Output only. The intermediate table for data anomaly detection.
173+
// Format:
174+
// PROJECT_ID.DATASET_ID.TABLE_ID
175+
string data_intermediate_table = 2
176+
[(google.api.field_behavior) = OUTPUT_ONLY];
177+
178+
// Output only. The intermediate table for freshness anomaly detection.
179+
// Format:
180+
// PROJECT_ID.DATASET_ID.TABLE_ID
181+
string freshness_intermediate_table = 3
182+
[(google.api.field_behavior) = OUTPUT_ONLY];
183+
184+
// Output only. The intermediate table for volume anomaly detection.
185+
// Format:
186+
// PROJECT_ID.DATASET_ID.TABLE_ID
187+
string volume_intermediate_table = 4
188+
[(google.api.field_behavior) = OUTPUT_ONLY];
189+
}
190+
162191
// Output only. Overall data quality result -- `true` if all rules passed.
163192
bool passed = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
164193

@@ -199,6 +228,10 @@ message DataQualityResult {
199228
// Catalog metadata.
200229
DataScanCatalogPublishingStatus catalog_publishing_status = 11
201230
[(google.api.field_behavior) = OUTPUT_ONLY];
231+
232+
// Output only. The generated assets for anomaly detection.
233+
AnomalyDetectionGeneratedAssets anomaly_detection_generated_assets = 12
234+
[(google.api.field_behavior) = OUTPUT_ONLY];
202235
}
203236

204237
// DataQualityRuleResult provides a more detailed, per-rule view of the results.

‎packages/google-cloud-dataplex/protos/google/cloud/dataplex/v1/datascans.proto‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import "google/api/field_behavior.proto";
2222
import "google/api/field_info.proto";
2323
import "google/api/resource.proto";
2424
import "google/cloud/dataplex/v1/data_discovery.proto";
25+
import "google/cloud/dataplex/v1/data_documentation.proto";
2526
import "google/cloud/dataplex/v1/data_profile.proto";
2627
import "google/cloud/dataplex/v1/data_quality.proto";
2728
import "google/cloud/dataplex/v1/processing.proto";
@@ -425,6 +426,10 @@ message GenerateDataQualityRulesResponse {
425426
// * Data discovery: scans data in Cloud Storage buckets to extract and then
426427
// catalog metadata. For more information, see [Discover and catalog Cloud
427428
// Storage data](https://cloud.google.com/bigquery/docs/automatic-discovery).
429+
// * Data documentation: analyzes the table details and generates insights
430+
// including descriptions and sample SQL queries for the table. For more
431+
// information, see [Generate data insights in
432+
// BigQuery](https://cloud.google.com/bigquery/docs/data-insights).
428433
message DataScan {
429434
option (google.api.resource) = {
430435
type: "dataplex.googleapis.com/DataScan"
@@ -536,6 +541,9 @@ message DataScan {
536541

537542
// Settings for a data discovery scan.
538543
DataDiscoverySpec data_discovery_spec = 102;
544+
545+
// Settings for a data documentation scan.
546+
DataDocumentationSpec data_documentation_spec = 103;
539547
}
540548

541549
// The result of the data scan.
@@ -551,6 +559,10 @@ message DataScan {
551559
// Output only. The result of a data discovery scan.
552560
DataDiscoveryResult data_discovery_result = 202
553561
[(google.api.field_behavior) = OUTPUT_ONLY];
562+
563+
// Output only. The result of a data documentation scan.
564+
DataDocumentationResult data_documentation_result = 203
565+
[(google.api.field_behavior) = OUTPUT_ONLY];
554566
}
555567
}
556568

@@ -635,6 +647,10 @@ message DataScanJob {
635647
// Output only. Settings for a data discovery scan.
636648
DataDiscoverySpec data_discovery_spec = 102
637649
[(google.api.field_behavior) = OUTPUT_ONLY];
650+
651+
// Output only. Settings for a data documentation scan.
652+
DataDocumentationSpec data_documentation_spec = 103
653+
[(google.api.field_behavior) = OUTPUT_ONLY];
638654
}
639655

640656
// The result of the data scan.
@@ -650,6 +666,10 @@ message DataScanJob {
650666
// Output only. The result of a data discovery scan.
651667
DataDiscoveryResult data_discovery_result = 202
652668
[(google.api.field_behavior) = OUTPUT_ONLY];
669+
670+
// Output only. The result of a data documentation scan.
671+
DataDocumentationResult data_documentation_result = 203
672+
[(google.api.field_behavior) = OUTPUT_ONLY];
653673
}
654674
}
655675

@@ -666,4 +686,7 @@ enum DataScanType {
666686

667687
// Data discovery scan.
668688
DATA_DISCOVERY = 3;
689+
690+
// Data documentation scan.
691+
DATA_DOCUMENTATION = 4;
669692
}

‎packages/google-cloud-dataplex/protos/google/cloud/dataplex/v1/processing.proto‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,8 +78,8 @@ message DataSource {
7878
// Storage bucket for DataDiscoveryScan Format:
7979
// //storage.googleapis.com/projects/PROJECT_ID/buckets/BUCKET_ID
8080
// or
81-
// BigQuery table of type "TABLE" for DataProfileScan/DataQualityScan
82-
// Format:
81+
// BigQuery table of type "TABLE" for
82+
// DataProfileScan/DataQualityScan/DataDocumentationScan Format:
8383
// //bigquery.googleapis.com/projects/PROJECT_ID/datasets/DATASET_ID/tables/TABLE_ID
8484
string resource = 101 [(google.api.field_behavior) = IMMUTABLE];
8585
}

0 commit comments

Comments
 (0)