fix(hashicorp/terraform-provider-google#23718): removing unnecessary … (#14659) (#10438)

[upstream:63cf48ea739c3db50912823c11a4ffdb13bb7002]

Signed-off-by: Modular Magician <[email protected]>

Author: modular-magician

.changelog/14659.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+bigquery: fixed a crash in `google_bigquery_table` when configured as an external table with `parquet_options`
+```

google-beta/services/bigquery/resource_bigquery_table.go

@@ -1368,6 +1368,12 @@ func ResourceBigQueryTable() *schema.Resource {
 				Description: `Whether Terraform will prevent implicitly added columns in schema from showing diff.`,
 			},
 
+			"generated_schema_columns": {
+				Type:        schema.TypeString,
+				Computed:    true,
+				Description: `(Output-only) A list of autogenerated schema fields.`,
+			},
+
 			// TableConstraints: [Optional] Defines the primary key and foreign keys.
 			"table_constraints": {
 				Type:        schema.TypeList,
@@ -1618,21 +1624,21 @@ func ResourceBigQueryTable() *schema.Resource {
 }
 
 // filterLiveSchemaByConfig compares a live schema from the BQ API with a schema from
-// the Terraform config. It returns a new schema containing only the fields
-// that are defined in the config, effectively removing any columns that were
-// auto-generated by the service (e.g., hive partitioning keys).
-//
+// the Terraform config. It returns two values:
+// 1. A new *bigquery.TableSchema containing a filtered list of fields that are defined in the config,
+// effectively removing any columns that were auto-generated by the service (e.g., hive partitioning keys).
+// 2. A slice of *bigquery.TableFieldSchema for the fields that were auto-generated by the service.
 // Parameters:
 //   - liveSchema: The schema returned from a BigQuery API Read/Get call. This may contain extra columns.
 //   - configSchema: The schema built from the user's Terraform configuration (`d.Get("schema")`). This is the source of truth.
-//
-// Returns:
-//
-//	A new *bigquery.TableSchema containing a filtered list of fields.
-func filterLiveSchemaByConfig(liveSchema *bigquery.TableSchema, configSchema *bigquery.TableSchema) *bigquery.TableSchema {
-	if liveSchema == nil || configSchema == nil {
-		// If either schema is nil, there's nothing to compare, so return an empty schema.
-		return &bigquery.TableSchema{Fields: []*bigquery.TableFieldSchema{}}
+func filterLiveSchemaByConfig(liveSchema *bigquery.TableSchema, configSchema *bigquery.TableSchema) (*bigquery.TableSchema, []*bigquery.TableFieldSchema) {
+	if liveSchema == nil {
+		// If live schema is nil, there's nothing to filter or collect.
+		return &bigquery.TableSchema{Fields: []*bigquery.TableFieldSchema{}}, nil
+	}
+	if configSchema == nil || len(configSchema.Fields) == 0 {
+		// If config schema is nil or empty, all live fields are considered auto-generated.
+		return &bigquery.TableSchema{Fields: []*bigquery.TableFieldSchema{}}, liveSchema.Fields
 	}
 
 	// 1. Create a lookup map of all column names defined in the configuration.
@@ -1645,19 +1651,21 @@ func filterLiveSchemaByConfig(liveSchema *bigquery.TableSchema, configSchema *bi
 	// 2. Iterate through the fields in the live schema and keep only the ones
 	//    that exist in our configuration map.
 	var filteredFields []*bigquery.TableFieldSchema
+	var autogeneratedFields []*bigquery.TableFieldSchema
 	for _, liveField := range liveSchema.Fields {
 		// If the live field's name is present in the map of configured fields...
 		if _, ok := configFieldsMap[liveField.Name]; ok {
 			// ...then it's a field we care about. Add it to our filtered list.
 			filteredFields = append(filteredFields, liveField)
 		} else {
-			log.Printf("[DEBUG] auto-generated column `%s` dropped during Table read.", liveField.Name)
+			log.Printf("[DEBUG] auto-generated column `%s` collected during Table read.", liveField.Name)
+			autogeneratedFields = append(autogeneratedFields, liveField)
 		}
 	}
 
 	return &bigquery.TableSchema{
 		Fields: filteredFields,
-	}
+	}, autogeneratedFields
 }
 
 func resourceTable(d *schema.ResourceData, meta interface{}) (*bigquery.Table, error) {
@@ -2049,15 +2057,36 @@ func resourceBigQueryTableRead(d *schema.ResourceData, meta interface{}) error {
 	}
 
 	if res.Schema != nil {
-		table, err := resourceTable(d, meta)
-		if err != nil {
-			return err
+		var configSchema *bigquery.TableSchema
+		if v, ok := d.GetOk("schema"); ok {
+			_, viewPresent := d.GetOk("view")
+			_, materializedViewPresent := d.GetOk("materialized_view")
+			managePolicyTags := !viewPresent && !materializedViewPresent
+			configSchema, err = expandSchema(v, managePolicyTags)
+			if err != nil {
+				return err
+			}
 		}
 
 		schemaFiltered := res.Schema
 		ignore, ok := d.Get("ignore_auto_generated_schema").(bool)
 		if ok && ignore {
-			schemaFiltered = filterLiveSchemaByConfig(res.Schema, table.Schema)
+			var autogeneratedFields []*bigquery.TableFieldSchema
+			schemaFiltered, autogeneratedFields = filterLiveSchemaByConfig(res.Schema, configSchema)
+			if len(autogeneratedFields) > 0 {
+				autogeneratedFieldsJson, err := json.Marshal(autogeneratedFields)
+				if err != nil {
+					return fmt.Errorf("error marshalling autogenerated schema fields: %w", err)
+				}
+				if err := d.Set("generated_schema_columns", string(autogeneratedFieldsJson)); err != nil {
+					return fmt.Errorf("error setting generated_schema_columns: %w", err)
+				}
+			} else {
+				d.Set("generated_schema_columns", "")
+			}
+		} else {
+			// If not ignoring, ensure the field is cleared
+			d.Set("generated_schema_columns", "")
 		}
 		schema, err := flattenSchema(schemaFiltered)
 		if err != nil {
@@ -2144,6 +2173,25 @@ type TableReference struct {
 	tableID   string
 }
 
+func addAutoGenSchemaFields(d *schema.ResourceData, table *bigquery.Table) error {
+	// When ignore_auto_generated_schema is true, we must include the autogenerated fields
+	// in the update payload to avoid the API thinking we're trying to delete them.
+	if ignore, enabled := d.Get("ignore_auto_generated_schema").(bool); enabled && ignore {
+		// Only proceed if the table has a schema to begin with.
+		if table.Schema != nil {
+			if autogenStr, ok := d.Get("generated_schema_columns").(string); ok && autogenStr != "" {
+				var autogenFields []*bigquery.TableFieldSchema
+				if err := json.Unmarshal([]byte(autogenStr), &autogenFields); err != nil {
+					return fmt.Errorf("failed to unmarshal autogenerated schema fields: %w", err)
+				}
+				table.Schema.Fields = append(table.Schema.Fields, autogenFields...)
+				log.Printf("[DEBUG] Appended %d autogenerated fields to schema for update", len(autogenFields))
+			}
+		}
+	}
+	return nil
+}
+
 func resourceBigQueryTableUpdate(d *schema.ResourceData, meta interface{}) error {
 	// If only client-side fields were modified, short-circuit the Update function to avoid sending an update API request.
 	clientSideFields := map[string]bool{"deletion_protection": true, "ignore_schema_changes": true, "ignore_auto_generated_schema": true, "table_metadata_view": true}
@@ -2169,6 +2217,10 @@ func resourceBigQueryTableUpdate(d *schema.ResourceData, meta interface{}) error
 		return err
 	}
 
+	if err := addAutoGenSchemaFields(d, table); err != nil {
+		return err
+	}
+
 	if table.ExternalDataConfiguration != nil && table.ExternalDataConfiguration.Schema != nil {
 		log.Printf("[INFO] Removing ExternalDataConfiguration.Schema when updating BigQuery table %s", d.Id())
 		table.ExternalDataConfiguration.Schema = nil