Add volatile default requires full table rewrite hazard (#258)

* Add hazard for new NOT NULL columns requiring backfill

Adding a new NOT NULL column usually needs a backfill.
This change adds a hazard and shows a suggested online migration path
(add nullable -> backfill -> set NOT NULL).

* Tweak

---------

Co-authored-by: catatsuy <[email protected]>

Author: bplunkett-stripe

internal/migration_acceptance_tests/column_cases_test.go

@@ -1207,6 +1207,9 @@ var columnAcceptanceTestCases = []acceptanceTestCase{
             );
 			`,
 		},
+		expectedHazardTypes: []diff.MigrationHazardType{
+			diff.MigrationHazardTypeAcquiresAccessExclusiveLock,
+		},
 	},
 	{
 		name: "Drop generated column",
@@ -1270,6 +1273,9 @@ var columnAcceptanceTestCases = []acceptanceTestCase{
             );
 			`,
 		},
+		expectedHazardTypes: []diff.MigrationHazardType{
+			diff.MigrationHazardTypeAcquiresAccessExclusiveLock,
+		},
 	},
 	{
 		name: "Generated column with index",
@@ -1296,6 +1302,7 @@ var columnAcceptanceTestCases = []acceptanceTestCase{
 			`,
 		},
 		expectedHazardTypes: []diff.MigrationHazardType{
+			diff.MigrationHazardTypeAcquiresAccessExclusiveLock,
 			diff.MigrationHazardTypeIndexBuild,
 		},
 	},

internal/migration_acceptance_tests/procedure_cases_test.go

@@ -95,7 +95,9 @@ var procedureAcceptanceTestCases = []acceptanceTestCase{
             $$;
 			`,
 		},
-		expectedHazardTypes: []diff.MigrationHazardType{diff.MigrationHazardTypeHasUntrackableDependencies},
+		expectedHazardTypes: []diff.MigrationHazardType{
+			diff.MigrationHazardTypeHasUntrackableDependencies,
+		},
 	},
 	{
 		name: "Drop procedure and its dependencies",

internal/migration_acceptance_tests/table_cases_test.go

@@ -569,6 +569,42 @@ var tableAcceptanceTestCases = []acceptanceTestCase{
 			diff.MigrationHazardTypeImpactsDatabasePerformance,
 		},
 	},
+	{
+		name: "Add NOT NULL column without default",
+		oldSchemaDDL: []string{
+			`
+            CREATE TABLE t(
+                id INT
+            );
+			`,
+		},
+		newSchemaDDL: []string{
+			`
+            CREATE TABLE t(
+                id INT,
+                city_name VARCHAR(255) NOT NULL
+            );
+			`,
+		},
+	},
+	{
+		name: "Add NOT NULL column with constant default avoids backfill hazard",
+		oldSchemaDDL: []string{
+			`
+            CREATE TABLE t(
+                id INT
+            );
+			`,
+		},
+		newSchemaDDL: []string{
+			`
+            CREATE TABLE t(
+                id INT,
+                city_name VARCHAR(255) NOT NULL DEFAULT ''
+            );
+			`,
+		},
+	},
 }
 
 func TestTableTestCases(t *testing.T) {

internal/queries/queries.sql

@@ -90,6 +90,7 @@ WITH identity_col_seq AS (
 SELECT
     a.attname::TEXT AS column_name,
     a.attnotnull AS is_not_null,
+    a.atthasmissing AS has_missing_val_optimization,
     a.attlen AS column_size,
     a.attidentity::TEXT AS identity_type,
     identity_col_seq.seqstart AS start_value,

internal/queries/queries.sql.go

@@ -275,6 +275,8 @@ type (
 		// Only populated if IsGenerated is true.
 		GenerationExpression string
 		IsNullable           bool
+		// HasMissingValOptimization refers to the 'attmissingval' optimization for adding columns with a default.
+		HasMissingValOptimization bool
 		// Size is the number of bytes required to store the value.
 		// It is used for data-packing purposes
 		Size     int
@@ -981,10 +983,11 @@ func (s *schemaFetcher) buildTable(
 		}
 
 		columns = append(columns, Column{
-			Name:       column.ColumnName,
-			Type:       column.ColumnType,
-			Collation:  collation,
-			IsNullable: !column.IsNotNull,
+			Name:                      column.ColumnName,
+			Type:                      column.ColumnType,
+			Collation:                 collation,
+			IsNullable:                !column.IsNotNull,
+			HasMissingValOptimization: column.HasMissingValOptimization,
 			// If the column has a default value, this will be a SQL string representing that value.
 			// Examples:
 			//   ''::text

internal/schema/schema.go

@@ -228,8 +228,11 @@ var (
 			CREATE VIEW schema_filtered_1.foo_view AS
 				SELECT id, author
 				FROM schema_2.foo;
+
+			-- Add a column with a default to test HasMissingValOptimization
+			ALTER TABLE schema_2.foo ADD COLUMN added_col TEXT DEFAULT 'some_default';
 		`},
-			expectedHash: "ff9ed400558572aa",
+			expectedHash: "386c0eb7ee3f4874",
 			expectedSchema: Schema{
 				NamedSchemas: []NamedSchema{
 					{Name: "public"},
@@ -271,6 +274,7 @@ var (
 							{Name: "content", Type: "text", Default: "''::text", Size: -1, Collation: defaultCollation},
 							{Name: "created_at", Type: "timestamp without time zone", Default: "CURRENT_TIMESTAMP", Size: 8},
 							{Name: "version", Type: "integer", Default: "0", Size: 4},
+							{Name: "added_col", Type: "text", Default: "'some_default'::text", IsNullable: true, Size: -1, Collation: defaultCollation, HasMissingValOptimization: true},
 						},
 						CheckConstraints: []CheckConstraint{
 							{Name: "author_content_check", Expression: "((length(content) > 0) AND (length(author) > 0))", KeyColumns: []string{"author", "content"}},
@@ -571,7 +575,7 @@ var (
 			ALTER TABLE foo_fk_1 ADD CONSTRAINT foo_fk_1_fk FOREIGN KEY (author, content) REFERENCES foo_1 (author, content)
 				NOT VALID;
 		`},
-			expectedHash: "9647ef46a878d426",
+			expectedHash: "1f2c44c4589a8d6a",
 			expectedSchema: Schema{
 				NamedSchemas: []NamedSchema{
 					{Name: "public"},
@@ -606,7 +610,7 @@ var (
 							{Name: "content", Type: "text", Default: "''::text", Size: -1, Collation: defaultCollation},
 							{Name: "genre", Type: "character varying(256)", Size: -1, Collation: defaultCollation},
 							{Name: "created_at", Type: "timestamp without time zone", Default: "CURRENT_TIMESTAMP", Size: 8},
-							{Name: "version", Type: "integer", IsNullable: false, Size: 4},
+							{Name: "version", Type: "integer", Size: 4},
 						},
 						CheckConstraints: nil,
 						ReplicaIdentity:  ReplicaIdentityNothing,

internal/schema/schema_test.go

@@ -66,6 +66,10 @@ var (
 		Type:    MigrationHazardTypeExtensionVersionUpgrade,
 		Message: "This extension's version is being upgraded. Be sure the newer version is backwards compatible with your use case.",
 	}
+	migrationHazardNewColumnFullTableRewrite = MigrationHazard{
+		Type:    MigrationHazardTypeAcquiresAccessExclusiveLock,
+		Message: "Adding a new column with a volatile default value will result in a full table rewrite.",
+	}
 )
 
 type oldAndNew[S any] struct {
@@ -1204,11 +1208,28 @@ func (csg *columnSQLVertexGenerator) Add(column schema.Column) ([]Statement, err
 	if err != nil {
 		return nil, fmt.Errorf("building column definition: %w", err)
 	}
-	return []Statement{{
+	stmt := Statement{
 		DDL:         fmt.Sprintf("%s ADD COLUMN %s", alterTablePrefix(csg.tableName), columnDef),
 		Timeout:     statementTimeoutDefault,
 		LockTimeout: lockTimeoutDefault,
-	}}, nil
+	}
+	if newColumnRequiresFullTableRewrite(column) {
+		stmt.Hazards = append(stmt.Hazards, migrationHazardNewColumnFullTableRewrite)
+	}
+	return []Statement{stmt}, nil
+}
+
+func newColumnRequiresFullTableRewrite(column schema.Column) bool {
+	// Generated columns require computing the expression for every existing row, causing a full
+	// table rewrite.
+	if column.IsGenerated {
+		return true
+	}
+	// Columns with defaults use PostgreSQL's "fast default" optimization (PostgreSQL 11+) for
+	// constant defaults, which avoids a full table rewrite. We can't reliably detect volatile
+	// defaults (e.g., now()) from schema comparison, but they're rare. Identity columns also
+	// don't require a full table rewrite.
+	return false
 }
 
 func (csg *columnSQLVertexGenerator) Delete(column schema.Column) ([]Statement, error) {