Skip to content

Commit fb4e83d

Browse files
modular-magicianmodular-magician
authored
Add discoveryengine_datastore layout_parsing_config nested fields. (#14425) (#23478)
[upstream:746b9d476f12af10519818f3fa402f3af1c58425] Signed-off-by: Modular Magician <[email protected]>
1 parent 7fbfadb commit fb4e83d

File tree

5 files changed

+422
-4
lines changed

5 files changed

+422
-4
lines changed

‎.changelog/14425.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
```release-note:enhancement
2+
discoveryengine: Added `enable_table_annotation`, `enable_image_annotation`, `structured_content_types`, `exclude_html_elements`, `exclude_html_classes` and `exclude_html_ids` to `layout_parsing_config` structure of `google_discovery_engine_data_store` resource
3+
```

‎google/services/discoveryengine/resource_discovery_engine_data_store.go

Lines changed: 296 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -184,7 +184,50 @@ config will be applied to all file types for Document parsing.`,
184184
Description: `Configurations applied to layout parser.`,
185185
MaxItems: 1,
186186
Elem: &schema.Resource{
187-
Schema: map[string]*schema.Schema{},
187+
Schema: map[string]*schema.Schema{
188+
"enable_image_annotation": {
189+
Type: schema.TypeBool,
190+
Optional: true,
191+
Description: `If true, the LLM based annotation is added to the image during parsing.`,
192+
},
193+
"enable_table_annotation": {
194+
Type: schema.TypeBool,
195+
Optional: true,
196+
Description: `If true, the LLM based annotation is added to the table during parsing.`,
197+
},
198+
"exclude_html_classes": {
199+
Type: schema.TypeList,
200+
Optional: true,
201+
Description: `List of HTML classes to exclude from the parsed content.`,
202+
Elem: &schema.Schema{
203+
Type: schema.TypeString,
204+
},
205+
},
206+
"exclude_html_elements": {
207+
Type: schema.TypeList,
208+
Optional: true,
209+
Description: `List of HTML elements to exclude from the parsed content.`,
210+
Elem: &schema.Schema{
211+
Type: schema.TypeString,
212+
},
213+
},
214+
"exclude_html_ids": {
215+
Type: schema.TypeList,
216+
Optional: true,
217+
Description: `List of HTML ids to exclude from the parsed content.`,
218+
Elem: &schema.Schema{
219+
Type: schema.TypeString,
220+
},
221+
},
222+
"structured_content_types": {
223+
Type: schema.TypeList,
224+
Optional: true,
225+
Description: `Contains the required structure types to extract from the document. Supported values: 'shareholder-structure'.`,
226+
Elem: &schema.Schema{
227+
Type: schema.TypeString,
228+
},
229+
},
230+
},
188231
},
189232
ExactlyOneOf: []string{},
190233
},
@@ -236,7 +279,50 @@ config will be applied to all file types for Document parsing.`,
236279
Description: `Configurations applied to layout parser.`,
237280
MaxItems: 1,
238281
Elem: &schema.Resource{
239-
Schema: map[string]*schema.Schema{},
282+
Schema: map[string]*schema.Schema{
283+
"enable_image_annotation": {
284+
Type: schema.TypeBool,
285+
Optional: true,
286+
Description: `If true, the LLM based annotation is added to the image during parsing.`,
287+
},
288+
"enable_table_annotation": {
289+
Type: schema.TypeBool,
290+
Optional: true,
291+
Description: `If true, the LLM based annotation is added to the table during parsing.`,
292+
},
293+
"exclude_html_classes": {
294+
Type: schema.TypeList,
295+
Optional: true,
296+
Description: `List of HTML classes to exclude from the parsed content.`,
297+
Elem: &schema.Schema{
298+
Type: schema.TypeString,
299+
},
300+
},
301+
"exclude_html_elements": {
302+
Type: schema.TypeList,
303+
Optional: true,
304+
Description: `List of HTML elements to exclude from the parsed content.`,
305+
Elem: &schema.Schema{
306+
Type: schema.TypeString,
307+
},
308+
},
309+
"exclude_html_ids": {
310+
Type: schema.TypeList,
311+
Optional: true,
312+
Description: `List of HTML ids to exclude from the parsed content.`,
313+
Elem: &schema.Schema{
314+
Type: schema.TypeString,
315+
},
316+
},
317+
"structured_content_types": {
318+
Type: schema.TypeList,
319+
Optional: true,
320+
Description: `Contains the required structure types to extract from the document. Supported values: 'shareholder-structure'.`,
321+
Elem: &schema.Schema{
322+
Type: schema.TypeString,
323+
},
324+
},
325+
},
240326
},
241327
ExactlyOneOf: []string{},
242328
},
@@ -828,9 +914,45 @@ func flattenDiscoveryEngineDataStoreDocumentProcessingConfigDefaultParsingConfig
828914
if v == nil {
829915
return nil
830916
}
917+
original := v.(map[string]interface{})
831918
transformed := make(map[string]interface{})
919+
transformed["enable_table_annotation"] =
920+
flattenDiscoveryEngineDataStoreDocumentProcessingConfigDefaultParsingConfigLayoutParsingConfigEnableTableAnnotation(original["enableTableAnnotation"], d, config)
921+
transformed["enable_image_annotation"] =
922+
flattenDiscoveryEngineDataStoreDocumentProcessingConfigDefaultParsingConfigLayoutParsingConfigEnableImageAnnotation(original["enableImageAnnotation"], d, config)
923+
transformed["structured_content_types"] =
924+
flattenDiscoveryEngineDataStoreDocumentProcessingConfigDefaultParsingConfigLayoutParsingConfigStructuredContentTypes(original["structuredContentTypes"], d, config)
925+
transformed["exclude_html_elements"] =
926+
flattenDiscoveryEngineDataStoreDocumentProcessingConfigDefaultParsingConfigLayoutParsingConfigExcludeHtmlElements(original["excludeHtmlElements"], d, config)
927+
transformed["exclude_html_classes"] =
928+
flattenDiscoveryEngineDataStoreDocumentProcessingConfigDefaultParsingConfigLayoutParsingConfigExcludeHtmlClasses(original["excludeHtmlClasses"], d, config)
929+
transformed["exclude_html_ids"] =
930+
flattenDiscoveryEngineDataStoreDocumentProcessingConfigDefaultParsingConfigLayoutParsingConfigExcludeHtmlIds(original["excludeHtmlIds"], d, config)
832931
return []interface{}{transformed}
833932
}
933+
func flattenDiscoveryEngineDataStoreDocumentProcessingConfigDefaultParsingConfigLayoutParsingConfigEnableTableAnnotation(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
934+
return v
935+
}
936+
937+
func flattenDiscoveryEngineDataStoreDocumentProcessingConfigDefaultParsingConfigLayoutParsingConfigEnableImageAnnotation(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
938+
return v
939+
}
940+
941+
func flattenDiscoveryEngineDataStoreDocumentProcessingConfigDefaultParsingConfigLayoutParsingConfigStructuredContentTypes(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
942+
return v
943+
}
944+
945+
func flattenDiscoveryEngineDataStoreDocumentProcessingConfigDefaultParsingConfigLayoutParsingConfigExcludeHtmlElements(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
946+
return v
947+
}
948+
949+
func flattenDiscoveryEngineDataStoreDocumentProcessingConfigDefaultParsingConfigLayoutParsingConfigExcludeHtmlClasses(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
950+
return v
951+
}
952+
953+
func flattenDiscoveryEngineDataStoreDocumentProcessingConfigDefaultParsingConfigLayoutParsingConfigExcludeHtmlIds(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
954+
return v
955+
}
834956

835957
func flattenDiscoveryEngineDataStoreDocumentProcessingConfigParsingConfigOverrides(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
836958
if v == nil {
@@ -878,9 +1000,45 @@ func flattenDiscoveryEngineDataStoreDocumentProcessingConfigParsingConfigOverrid
8781000
if v == nil {
8791001
return nil
8801002
}
1003+
original := v.(map[string]interface{})
8811004
transformed := make(map[string]interface{})
1005+
transformed["enable_table_annotation"] =
1006+
flattenDiscoveryEngineDataStoreDocumentProcessingConfigParsingConfigOverridesLayoutParsingConfigEnableTableAnnotation(original["enableTableAnnotation"], d, config)
1007+
transformed["enable_image_annotation"] =
1008+
flattenDiscoveryEngineDataStoreDocumentProcessingConfigParsingConfigOverridesLayoutParsingConfigEnableImageAnnotation(original["enableImageAnnotation"], d, config)
1009+
transformed["structured_content_types"] =
1010+
flattenDiscoveryEngineDataStoreDocumentProcessingConfigParsingConfigOverridesLayoutParsingConfigStructuredContentTypes(original["structuredContentTypes"], d, config)
1011+
transformed["exclude_html_elements"] =
1012+
flattenDiscoveryEngineDataStoreDocumentProcessingConfigParsingConfigOverridesLayoutParsingConfigExcludeHtmlElements(original["excludeHtmlElements"], d, config)
1013+
transformed["exclude_html_classes"] =
1014+
flattenDiscoveryEngineDataStoreDocumentProcessingConfigParsingConfigOverridesLayoutParsingConfigExcludeHtmlClasses(original["excludeHtmlClasses"], d, config)
1015+
transformed["exclude_html_ids"] =
1016+
flattenDiscoveryEngineDataStoreDocumentProcessingConfigParsingConfigOverridesLayoutParsingConfigExcludeHtmlIds(original["excludeHtmlIds"], d, config)
8821017
return []interface{}{transformed}
8831018
}
1019+
func flattenDiscoveryEngineDataStoreDocumentProcessingConfigParsingConfigOverridesLayoutParsingConfigEnableTableAnnotation(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
1020+
return v
1021+
}
1022+
1023+
func flattenDiscoveryEngineDataStoreDocumentProcessingConfigParsingConfigOverridesLayoutParsingConfigEnableImageAnnotation(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
1024+
return v
1025+
}
1026+
1027+
func flattenDiscoveryEngineDataStoreDocumentProcessingConfigParsingConfigOverridesLayoutParsingConfigStructuredContentTypes(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
1028+
return v
1029+
}
1030+
1031+
func flattenDiscoveryEngineDataStoreDocumentProcessingConfigParsingConfigOverridesLayoutParsingConfigExcludeHtmlElements(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
1032+
return v
1033+
}
1034+
1035+
func flattenDiscoveryEngineDataStoreDocumentProcessingConfigParsingConfigOverridesLayoutParsingConfigExcludeHtmlClasses(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
1036+
return v
1037+
}
1038+
1039+
func flattenDiscoveryEngineDataStoreDocumentProcessingConfigParsingConfigOverridesLayoutParsingConfigExcludeHtmlIds(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
1040+
return v
1041+
}
8841042

8851043
func flattenDiscoveryEngineDataStoreCreateTime(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
8861044
return v
@@ -1123,11 +1281,79 @@ func expandDiscoveryEngineDataStoreDocumentProcessingConfigDefaultParsingConfigL
11231281
transformed := make(map[string]interface{})
11241282
return transformed, nil
11251283
}
1284+
raw := l[0]
1285+
original := raw.(map[string]interface{})
11261286
transformed := make(map[string]interface{})
11271287

1288+
transformedEnableTableAnnotation, err := expandDiscoveryEngineDataStoreDocumentProcessingConfigDefaultParsingConfigLayoutParsingConfigEnableTableAnnotation(original["enable_table_annotation"], d, config)
1289+
if err != nil {
1290+
return nil, err
1291+
} else if val := reflect.ValueOf(transformedEnableTableAnnotation); val.IsValid() && !tpgresource.IsEmptyValue(val) {
1292+
transformed["enableTableAnnotation"] = transformedEnableTableAnnotation
1293+
}
1294+
1295+
transformedEnableImageAnnotation, err := expandDiscoveryEngineDataStoreDocumentProcessingConfigDefaultParsingConfigLayoutParsingConfigEnableImageAnnotation(original["enable_image_annotation"], d, config)
1296+
if err != nil {
1297+
return nil, err
1298+
} else if val := reflect.ValueOf(transformedEnableImageAnnotation); val.IsValid() && !tpgresource.IsEmptyValue(val) {
1299+
transformed["enableImageAnnotation"] = transformedEnableImageAnnotation
1300+
}
1301+
1302+
transformedStructuredContentTypes, err := expandDiscoveryEngineDataStoreDocumentProcessingConfigDefaultParsingConfigLayoutParsingConfigStructuredContentTypes(original["structured_content_types"], d, config)
1303+
if err != nil {
1304+
return nil, err
1305+
} else if val := reflect.ValueOf(transformedStructuredContentTypes); val.IsValid() && !tpgresource.IsEmptyValue(val) {
1306+
transformed["structuredContentTypes"] = transformedStructuredContentTypes
1307+
}
1308+
1309+
transformedExcludeHtmlElements, err := expandDiscoveryEngineDataStoreDocumentProcessingConfigDefaultParsingConfigLayoutParsingConfigExcludeHtmlElements(original["exclude_html_elements"], d, config)
1310+
if err != nil {
1311+
return nil, err
1312+
} else if val := reflect.ValueOf(transformedExcludeHtmlElements); val.IsValid() && !tpgresource.IsEmptyValue(val) {
1313+
transformed["excludeHtmlElements"] = transformedExcludeHtmlElements
1314+
}
1315+
1316+
transformedExcludeHtmlClasses, err := expandDiscoveryEngineDataStoreDocumentProcessingConfigDefaultParsingConfigLayoutParsingConfigExcludeHtmlClasses(original["exclude_html_classes"], d, config)
1317+
if err != nil {
1318+
return nil, err
1319+
} else if val := reflect.ValueOf(transformedExcludeHtmlClasses); val.IsValid() && !tpgresource.IsEmptyValue(val) {
1320+
transformed["excludeHtmlClasses"] = transformedExcludeHtmlClasses
1321+
}
1322+
1323+
transformedExcludeHtmlIds, err := expandDiscoveryEngineDataStoreDocumentProcessingConfigDefaultParsingConfigLayoutParsingConfigExcludeHtmlIds(original["exclude_html_ids"], d, config)
1324+
if err != nil {
1325+
return nil, err
1326+
} else if val := reflect.ValueOf(transformedExcludeHtmlIds); val.IsValid() && !tpgresource.IsEmptyValue(val) {
1327+
transformed["excludeHtmlIds"] = transformedExcludeHtmlIds
1328+
}
1329+
11281330
return transformed, nil
11291331
}
11301332

1333+
func expandDiscoveryEngineDataStoreDocumentProcessingConfigDefaultParsingConfigLayoutParsingConfigEnableTableAnnotation(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
1334+
return v, nil
1335+
}
1336+
1337+
func expandDiscoveryEngineDataStoreDocumentProcessingConfigDefaultParsingConfigLayoutParsingConfigEnableImageAnnotation(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
1338+
return v, nil
1339+
}
1340+
1341+
func expandDiscoveryEngineDataStoreDocumentProcessingConfigDefaultParsingConfigLayoutParsingConfigStructuredContentTypes(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
1342+
return v, nil
1343+
}
1344+
1345+
func expandDiscoveryEngineDataStoreDocumentProcessingConfigDefaultParsingConfigLayoutParsingConfigExcludeHtmlElements(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
1346+
return v, nil
1347+
}
1348+
1349+
func expandDiscoveryEngineDataStoreDocumentProcessingConfigDefaultParsingConfigLayoutParsingConfigExcludeHtmlClasses(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
1350+
return v, nil
1351+
}
1352+
1353+
func expandDiscoveryEngineDataStoreDocumentProcessingConfigDefaultParsingConfigLayoutParsingConfigExcludeHtmlIds(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
1354+
return v, nil
1355+
}
1356+
11311357
func expandDiscoveryEngineDataStoreDocumentProcessingConfigParsingConfigOverrides(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (map[string]interface{}, error) {
11321358
if v == nil {
11331359
return map[string]interface{}{}, nil
@@ -1215,7 +1441,75 @@ func expandDiscoveryEngineDataStoreDocumentProcessingConfigParsingConfigOverride
12151441
transformed := make(map[string]interface{})
12161442
return transformed, nil
12171443
}
1444+
raw := l[0]
1445+
original := raw.(map[string]interface{})
12181446
transformed := make(map[string]interface{})
12191447

1448+
transformedEnableTableAnnotation, err := expandDiscoveryEngineDataStoreDocumentProcessingConfigParsingConfigOverridesLayoutParsingConfigEnableTableAnnotation(original["enable_table_annotation"], d, config)
1449+
if err != nil {
1450+
return nil, err
1451+
} else if val := reflect.ValueOf(transformedEnableTableAnnotation); val.IsValid() && !tpgresource.IsEmptyValue(val) {
1452+
transformed["enableTableAnnotation"] = transformedEnableTableAnnotation
1453+
}
1454+
1455+
transformedEnableImageAnnotation, err := expandDiscoveryEngineDataStoreDocumentProcessingConfigParsingConfigOverridesLayoutParsingConfigEnableImageAnnotation(original["enable_image_annotation"], d, config)
1456+
if err != nil {
1457+
return nil, err
1458+
} else if val := reflect.ValueOf(transformedEnableImageAnnotation); val.IsValid() && !tpgresource.IsEmptyValue(val) {
1459+
transformed["enableImageAnnotation"] = transformedEnableImageAnnotation
1460+
}
1461+
1462+
transformedStructuredContentTypes, err := expandDiscoveryEngineDataStoreDocumentProcessingConfigParsingConfigOverridesLayoutParsingConfigStructuredContentTypes(original["structured_content_types"], d, config)
1463+
if err != nil {
1464+
return nil, err
1465+
} else if val := reflect.ValueOf(transformedStructuredContentTypes); val.IsValid() && !tpgresource.IsEmptyValue(val) {
1466+
transformed["structuredContentTypes"] = transformedStructuredContentTypes
1467+
}
1468+
1469+
transformedExcludeHtmlElements, err := expandDiscoveryEngineDataStoreDocumentProcessingConfigParsingConfigOverridesLayoutParsingConfigExcludeHtmlElements(original["exclude_html_elements"], d, config)
1470+
if err != nil {
1471+
return nil, err
1472+
} else if val := reflect.ValueOf(transformedExcludeHtmlElements); val.IsValid() && !tpgresource.IsEmptyValue(val) {
1473+
transformed["excludeHtmlElements"] = transformedExcludeHtmlElements
1474+
}
1475+
1476+
transformedExcludeHtmlClasses, err := expandDiscoveryEngineDataStoreDocumentProcessingConfigParsingConfigOverridesLayoutParsingConfigExcludeHtmlClasses(original["exclude_html_classes"], d, config)
1477+
if err != nil {
1478+
return nil, err
1479+
} else if val := reflect.ValueOf(transformedExcludeHtmlClasses); val.IsValid() && !tpgresource.IsEmptyValue(val) {
1480+
transformed["excludeHtmlClasses"] = transformedExcludeHtmlClasses
1481+
}
1482+
1483+
transformedExcludeHtmlIds, err := expandDiscoveryEngineDataStoreDocumentProcessingConfigParsingConfigOverridesLayoutParsingConfigExcludeHtmlIds(original["exclude_html_ids"], d, config)
1484+
if err != nil {
1485+
return nil, err
1486+
} else if val := reflect.ValueOf(transformedExcludeHtmlIds); val.IsValid() && !tpgresource.IsEmptyValue(val) {
1487+
transformed["excludeHtmlIds"] = transformedExcludeHtmlIds
1488+
}
1489+
12201490
return transformed, nil
12211491
}
1492+
1493+
func expandDiscoveryEngineDataStoreDocumentProcessingConfigParsingConfigOverridesLayoutParsingConfigEnableTableAnnotation(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
1494+
return v, nil
1495+
}
1496+
1497+
func expandDiscoveryEngineDataStoreDocumentProcessingConfigParsingConfigOverridesLayoutParsingConfigEnableImageAnnotation(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
1498+
return v, nil
1499+
}
1500+
1501+
func expandDiscoveryEngineDataStoreDocumentProcessingConfigParsingConfigOverridesLayoutParsingConfigStructuredContentTypes(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
1502+
return v, nil
1503+
}
1504+
1505+
func expandDiscoveryEngineDataStoreDocumentProcessingConfigParsingConfigOverridesLayoutParsingConfigExcludeHtmlElements(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
1506+
return v, nil
1507+
}
1508+
1509+
func expandDiscoveryEngineDataStoreDocumentProcessingConfigParsingConfigOverridesLayoutParsingConfigExcludeHtmlClasses(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
1510+
return v, nil
1511+
}
1512+
1513+
func expandDiscoveryEngineDataStoreDocumentProcessingConfigParsingConfigOverridesLayoutParsingConfigExcludeHtmlIds(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
1514+
return v, nil
1515+
}

0 commit comments

Comments
 (0)