Fixing hive partitioning diff (#14483) (#10366)

[upstream:43ca35a159e375c2668ddf5b45122fdeab748f89]

Signed-off-by: Modular Magician <[email protected]>

Author: modular-magician

.changelog/14483.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+bigquery: added `ignore_auto_generated_schema` virtual field to `google_bigquery_table` resource to ignore server-added columns in the `schema` field
+```

google-beta/services/bigquery/resource_bigquery_table.go

@@ -1361,6 +1361,13 @@ func ResourceBigQueryTable() *schema.Resource {
 				Description: `Whether Terraform will be prevented from destroying the instance. When the field is set to true or unset in Terraform state, a terraform apply or terraform destroy that would delete the table will fail. When the field is set to false, deleting the table is allowed.`,
 			},
 
+			"ignore_auto_generated_schema": {
+				Type:        schema.TypeBool,
+				Optional:    true,
+				Default:     false,
+				Description: `Whether Terraform will prevent implicitly added columns in schema from showing diff.`,
+			},
+
 			// TableConstraints: [Optional] Defines the primary key and foreign keys.
 			"table_constraints": {
 				Type:        schema.TypeList,
@@ -1610,6 +1617,49 @@ func ResourceBigQueryTable() *schema.Resource {
 	}
 }
 
+// filterLiveSchemaByConfig compares a live schema from the BQ API with a schema from
+// the Terraform config. It returns a new schema containing only the fields
+// that are defined in the config, effectively removing any columns that were
+// auto-generated by the service (e.g., hive partitioning keys).
+//
+// Parameters:
+//   - liveSchema: The schema returned from a BigQuery API Read/Get call. This may contain extra columns.
+//   - configSchema: The schema built from the user's Terraform configuration (`d.Get("schema")`). This is the source of truth.
+//
+// Returns:
+//
+//	A new *bigquery.TableSchema containing a filtered list of fields.
+func filterLiveSchemaByConfig(liveSchema *bigquery.TableSchema, configSchema *bigquery.TableSchema) *bigquery.TableSchema {
+	if liveSchema == nil || configSchema == nil {
+		// If either schema is nil, there's nothing to compare, so return an empty schema.
+		return &bigquery.TableSchema{Fields: []*bigquery.TableFieldSchema{}}
+	}
+
+	// 1. Create a lookup map of all column names defined in the configuration.
+	//    This provides fast O(1) average time complexity for lookups.
+	configFieldsMap := make(map[string]bool)
+	for _, field := range configSchema.Fields {
+		configFieldsMap[field.Name] = true
+	}
+
+	// 2. Iterate through the fields in the live schema and keep only the ones
+	//    that exist in our configuration map.
+	var filteredFields []*bigquery.TableFieldSchema
+	for _, liveField := range liveSchema.Fields {
+		// If the live field's name is present in the map of configured fields...
+		if _, ok := configFieldsMap[liveField.Name]; ok {
+			// ...then it's a field we care about. Add it to our filtered list.
+			filteredFields = append(filteredFields, liveField)
+		} else {
+			log.Printf("[DEBUG] auto-generated column `%s` dropped during Table read.", liveField.Name)
+		}
+	}
+
+	return &bigquery.TableSchema{
+		Fields: filteredFields,
+	}
+}
+
 func resourceTable(d *schema.ResourceData, meta interface{}) (*bigquery.Table, error) {
 	config := meta.(*transport_tpg.Config)
 
@@ -1999,7 +2049,17 @@ func resourceBigQueryTableRead(d *schema.ResourceData, meta interface{}) error {
 	}
 
 	if res.Schema != nil {
-		schema, err := flattenSchema(res.Schema)
+		table, err := resourceTable(d, meta)
+		if err != nil {
+			return err
+		}
+
+		schemaFiltered := res.Schema
+		ignore, ok := d.Get("ignore_auto_generated_schema").(bool)
+		if ok && ignore {
+			schemaFiltered = filterLiveSchemaByConfig(res.Schema, table.Schema)
+		}
+		schema, err := flattenSchema(schemaFiltered)
 		if err != nil {
 			return err
 		}
@@ -2086,7 +2146,7 @@ type TableReference struct {
 
 func resourceBigQueryTableUpdate(d *schema.ResourceData, meta interface{}) error {
 	// If only client-side fields were modified, short-circuit the Update function to avoid sending an update API request.
-	clientSideFields := map[string]bool{"deletion_protection": true, "ignore_schema_changes": true, "table_metadata_view": true}
+	clientSideFields := map[string]bool{"deletion_protection": true, "ignore_schema_changes": true, "ignore_auto_generated_schema": true, "table_metadata_view": true}
 	clientSideOnly := true
 	for field := range ResourceBigQueryTable().Schema {
 		if d.HasChange(field) && !clientSideFields[field] {
@@ -2134,8 +2194,11 @@ func resourceBigQueryTableUpdate(d *schema.ResourceData, meta interface{}) error
 		tableID:   tableID,
 	}
 
-	if err = resourceBigQueryTableColumnDrop(config, userAgent, table, tableReference, tableMetadataView); err != nil {
-		return err
+	// If we are supposed to ignore server generated schema columns, we don't need to drop them
+	if !d.Get("ignore_auto_generated_schema").(bool) {
+		if err = resourceBigQueryTableColumnDrop(config, userAgent, table, tableReference, tableMetadataView); err != nil {
+			return err
+		}
 	}
 
 	if _, err = config.NewBigQueryClient(userAgent).Tables.Update(project, datasetID, tableID, table).Do(); err != nil {