diff --git a/.gitignore b/.gitignore index 550b268..b249fe8 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,6 @@ vendor/ # Intellij .idea/ + +# Claude settings +.claude/settings.local.json \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..56d3909 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,210 @@ +# pg-schema-diff + +A declarative schema migration tool for PostgreSQL that computes the difference between two database schemas and generates minimal, optimized SQL to migrate from one to the other with zero-downtime where possible. + +## Project Overview + +**Problem Solved**: Developers declare their desired database schema in DDL files, and pg-schema-diff automatically generates safe, optimized migration SQL that minimizes downtime and locks. + +**Key Features**: +- Computes diffs between schemas (DDL files, databases, or directories) +- Generates SQL using native Postgres online operations (concurrent index builds, online constraint validation) +- Provides hazard warnings for dangerous operations +- Validates migration plans against temporary databases before execution + +## Directory Structure + +``` +cmd/pg-schema-diff/ # CLI entry point (Cobra-based) +├── plan_cmd.go # 'plan' subcommand - generates migration SQL +├── apply_cmd.go # 'apply' subcommand - applies migrations +├── flags.go # Flag parsing and DB connection handling + +pkg/ # Public API packages +├── diff/ # Core diffing and plan generation (main library interface) +├── tempdb/ # Temporary database factory for plan validation +├── log/ # Logging interface +├── schema/ # Public schema API wrapper +├── sqldb/ # Database queryable interface + +internal/ # Internal implementation +├── schema/ # Complete schema representation types (schema.go is 46KB) +├── queries/ # SQL queries via sqlc for schema introspection +├── migration_acceptance_tests/ # Comprehensive test suite (24 test files) +├── pgengine/ # Postgres engine management for tests +├── pgdump/ # pg_dump integration +├── graph/ # Dependency graph for statement ordering +``` + +## Key Packages + +### pkg/diff/ - Core Diffing Engine +- `plan_generator.go`: Orchestrates plan generation and validation +- `sql_generator.go`: Generates SQL statements for all object types (2,700+ lines) +- `sql_graph.go`: Dependency graph for correct statement ordering +- `schema_source.go`: Schema sources (DDL files, database, directories) + +### internal/schema/ - Schema Representation +Core types in `schema.go`: +- `Schema`: Top-level container for all database objects +- `Table`: Tables with columns, constraints, policies, triggers +- `Index`: Index definitions including partial indexes and expressions +- `Column`, `ForeignKeyConstraint`, `CheckConstraint`, `View`, `Function`, etc. + +### internal/queries/ - Database Queries +Uses **sqlc** for type-safe SQL queries. To modify: +1. Edit `queries.sql` +2. Run `make sqlc` to regenerate `queries.sql.go` + +## Development Commands + +```bash +# Run all tests (requires Docker or local Postgres) +go test -v -race ./... -timeout 30m + +# Run specific acceptance tests +go test -v ./internal/migration_acceptance_tests/... -run TestIndexAcceptance + +# Lint +make lint + +# Fix lint issues +make lint_fix + +# Regenerate sqlc code +make sqlc + +# Tidy dependencies +make go_mod_tidy +``` + +## Testing + +### Acceptance Tests +Located in `internal/migration_acceptance_tests/`. Each test file covers specific features: +- `index_cases_test.go`: Index operations +- `table_cases_test.go`: Table operations +- `column_cases_test.go`: Column operations +- `check_constraint_cases_test.go`, `foreign_key_constraint_cases_test.go`: Constraints +- `view_cases_test.go`, `function_cases_test.go`, `trigger_cases_test.go`, etc. + +Test case structure: +```go +acceptanceTestCase{ + name: "test name", + oldSchemaDDL: []string{"CREATE TABLE ..."}, + newSchemaDDL: []string{"CREATE TABLE ... (modified)"}, + expectedHazardTypes: []diff.MigrationHazardType{...}, + expectedPlanDDL: []string{"ALTER TABLE ..."}, // optional: assert exact DDL + expectEmptyPlan: false, // optional: assert no changes + planOpts: []diff.PlanOpt{...}, // optional: custom plan options +} +``` + +### Running Tests with Docker +```bash +docker build -f build/Dockerfile.test --build-arg PG_MAJOR=15 -t pg-schema-diff-test . +docker run pg-schema-diff-test +``` + +## Key Concepts + +### Migration Hazards +Operations are flagged with hazard types: +- `MigrationHazardTypeAcquiresAccessExclusiveLock`: Full table lock +- `MigrationHazardTypeDeletesData`: Potential data loss +- `MigrationHazardTypeIndexBuild`: Performance impact during build +- `MigrationHazardTypeIndexDropped`: Query performance may degrade +- `MigrationHazardTypeCorrectness`: Potential correctness issues + +### Plan and Statements +```go +type Plan struct { + Statements []Statement + CurrentSchemaHash string // For validation before applying +} + +type Statement struct { + DDL string // SQL to execute + Timeout time.Duration // statement_timeout + LockTimeout time.Duration // lock_timeout + Hazards []MigrationHazard +} +``` + +### Online Migration Techniques +- **Concurrent Index Building**: `CREATE INDEX CONCURRENTLY` +- **Online Index Replacement**: Rename old, build new concurrently, drop old +- **Online NOT NULL**: Uses check constraints temporarily +- **Online Constraint Validation**: Add as `NOT VALID`, validate separately + +## CLI Usage + +```bash +# Generate migration plan (from database to DDL files) +pg-schema-diff plan \ + --from-dsn "postgres://user:pass@localhost:5432/mydb" \ + --to-dir ./schema + +# Generate plan between two databases +pg-schema-diff plan \ + --from-dsn "postgres://..." \ + --to-dsn "postgres://..." + +# Apply migration (requires hazard approval) +pg-schema-diff apply \ + --from-dsn "postgres://user:pass@localhost:5432/mydb" \ + --to-dir ./schema \ + --allow-hazards INDEX_BUILD,ACQUIRES_ACCESS_EXCLUSIVE_LOCK + +# Output formats: sql (default), json, pretty +pg-schema-diff plan --from-dsn "..." --to-dir ./schema --output-format json +``` + +## Library Usage + +```go +import ( + "github.com/stripe/pg-schema-diff/pkg/diff" + "github.com/stripe/pg-schema-diff/pkg/tempdb" +) + +// Create temp database factory for plan validation +tempDbFactory, _ := tempdb.NewOnInstanceFactory(ctx, func(ctx context.Context, dbName string) (*sql.DB, error) { + return sql.Open("postgres", fmt.Sprintf(".../%s", dbName)) +}) + +// Define schema sources +currentSchema := diff.DBSchemaSource(db) // db is *sql.DB or sqldb.Queryable +targetSchema, _ := diff.DirSchemaSource([]string{"./schema"}) // returns (SchemaSource, error) + +// Generate plan +plan, _ := diff.Generate(ctx, currentSchema, targetSchema, + diff.WithTempDbFactory(tempDbFactory), +) + +// Apply statements (set timeouts before each statement) +for _, stmt := range plan.Statements { + conn.ExecContext(ctx, fmt.Sprintf("SET SESSION statement_timeout = %d", stmt.Timeout.Milliseconds())) + conn.ExecContext(ctx, fmt.Sprintf("SET SESSION lock_timeout = %d", stmt.LockTimeout.Milliseconds())) + conn.ExecContext(ctx, stmt.ToSQL()) +} +``` + +## Code Patterns + +### Adding New Schema Object Support +1. Add type to `internal/schema/schema.go` +2. Add query to `internal/queries/queries.sql`, run `make sqlc` +3. Update schema fetching logic, schema structs, and tests in `internal/schema` +4. Add diffing logic in `pkg/diff/diff.go` +5. Add SQL generation logic in `pkg/diff/x_sql_generator.go` +6. Add acceptance tests in `internal/migration_acceptance_tests/` + +### Error Handling +Use `fmt.Errorf` with `%w` for error wrapping. Functions return `error` as last return value. + +### Testing Conventions +- Use `testify/assert` and `testify/require` +- Acceptance tests use shared Postgres via `pgengine` +- Test cases are typically table-driven diff --git a/internal/migration_acceptance_tests/materialized_view_cases_test.go b/internal/migration_acceptance_tests/materialized_view_cases_test.go index 06651ef..942e927 100644 --- a/internal/migration_acceptance_tests/materialized_view_cases_test.go +++ b/internal/migration_acceptance_tests/materialized_view_cases_test.go @@ -518,8 +518,8 @@ var materializedViewAcceptanceTestCases = []acceptanceTestCase{ ); CREATE MATERIALIZED VIEW foobar_view WITH (toast_tuple_target = 2048) AS - SELECT id, foo, bar - FROM foobar + SELECT id, foo, bar + FROM foobar WHERE buzz = true; `, }, @@ -533,12 +533,82 @@ var materializedViewAcceptanceTestCases = []acceptanceTestCase{ ); CREATE MATERIALIZED VIEW foobar_view AS - SELECT id, foo, bar - FROM foobar + SELECT id, foo, bar + FROM foobar WHERE buzz = true; `, }, }, + { + name: "Add materialized view with index", + oldSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255) + ); + `, + }, + newSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255) + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT id, foo FROM foobar; + + CREATE INDEX foobar_view_foo_idx ON foobar_view(foo); + `, + }, + // No hazards - index build hazard stripped when materialized view is new + }, + { + name: "Drop materialized view with index", + oldSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255) + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT id, foo FROM foobar; + + CREATE INDEX foobar_view_foo_idx ON foobar_view(foo); + `, + }, + newSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255) + ); + `, + }, + // No IndexDropped hazard - handled by DROP MATERIALIZED VIEW cascade + }, + { + name: "Drop materialized view with index and underlying table", + oldSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255) + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT id, foo FROM foobar; + + CREATE INDEX foobar_view_foo_idx ON foobar_view(foo); + `, + }, + newSchemaDDL: nil, + expectedHazardTypes: []diff.MigrationHazardType{ + diff.MigrationHazardTypeDeletesData, + }, + }, } func TestMaterializedViewTestCases(t *testing.T) { diff --git a/internal/migration_acceptance_tests/materialized_view_index_cases_test.go b/internal/migration_acceptance_tests/materialized_view_index_cases_test.go new file mode 100644 index 0000000..ab8b245 --- /dev/null +++ b/internal/migration_acceptance_tests/materialized_view_index_cases_test.go @@ -0,0 +1,139 @@ +package migration_acceptance_tests + +import ( + "testing" + + "github.com/stripe/pg-schema-diff/pkg/diff" +) + +var materializedViewIndexAcceptanceTestCases = []acceptanceTestCase{ + { + name: "Add index to existing materialized view", + oldSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255) + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT id, foo FROM foobar; + `, + }, + newSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255) + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT id, foo FROM foobar; + + CREATE INDEX foobar_view_foo_idx ON foobar_view(foo); + `, + }, + expectedHazardTypes: []diff.MigrationHazardType{ + diff.MigrationHazardTypeIndexBuild, + }, + }, + { + name: "Drop index from materialized view", + oldSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255) + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT id, foo FROM foobar; + + CREATE INDEX foobar_view_foo_idx ON foobar_view(foo); + `, + }, + newSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255) + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT id, foo FROM foobar; + `, + }, + expectedHazardTypes: []diff.MigrationHazardType{ + diff.MigrationHazardTypeIndexDropped, + }, + }, + { + name: "Add unique index to materialized view", + oldSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255) + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT id, foo FROM foobar; + `, + }, + newSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255) + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT id, foo FROM foobar; + + CREATE UNIQUE INDEX foobar_view_id_idx ON foobar_view(id); + `, + }, + expectedHazardTypes: []diff.MigrationHazardType{ + diff.MigrationHazardTypeIndexBuild, + }, + }, + { + name: "Change index columns on materialized view", + oldSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255), + bar VARCHAR(255) + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT id, foo, bar FROM foobar; + + CREATE INDEX foobar_view_idx ON foobar_view(foo); + `, + }, + newSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255), + bar VARCHAR(255) + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT id, foo, bar FROM foobar; + + CREATE INDEX foobar_view_idx ON foobar_view(bar); + `, + }, + expectedHazardTypes: []diff.MigrationHazardType{ + diff.MigrationHazardTypeIndexDropped, + diff.MigrationHazardTypeIndexBuild, + }, + }, +} + +func TestMaterializedViewIndexTestCases(t *testing.T) { + runTestCases(t, materializedViewIndexAcceptanceTestCases) +} diff --git a/internal/queries/queries.sql b/internal/queries/queries.sql index 3f78827..9316e07 100644 --- a/internal/queries/queries.sql +++ b/internal/queries/queries.sql @@ -141,6 +141,7 @@ SELECT c.relname::TEXT AS index_name, table_c.relname::TEXT AS table_name, table_namespace.nspname::TEXT AS table_schema_name, + table_c.relkind::TEXT AS owning_table_relkind, pg_catalog.pg_get_indexdef(c.oid)::TEXT AS def_stmt, COALESCE(con.conname, '')::TEXT AS constraint_name, COALESCE(con.contype, '')::TEXT AS constraint_type, diff --git a/internal/queries/queries.sql.go b/internal/queries/queries.sql.go index d1ac4a3..09ff092 100644 --- a/internal/queries/queries.sql.go +++ b/internal/queries/queries.sql.go @@ -465,6 +465,7 @@ SELECT c.relname::TEXT AS index_name, table_c.relname::TEXT AS table_name, table_namespace.nspname::TEXT AS table_schema_name, + table_c.relkind::TEXT AS owning_table_relkind, pg_catalog.pg_get_indexdef(c.oid)::TEXT AS def_stmt, COALESCE(con.conname, '')::TEXT AS constraint_name, COALESCE(con.contype, '')::TEXT AS constraint_type, @@ -526,6 +527,7 @@ type GetIndexesRow struct { IndexName string TableName string TableSchemaName string + OwningTableRelkind string DefStmt string ConstraintName string ConstraintType string @@ -553,6 +555,7 @@ func (q *Queries) GetIndexes(ctx context.Context) ([]GetIndexesRow, error) { &i.IndexName, &i.TableName, &i.TableSchemaName, + &i.OwningTableRelkind, &i.DefStmt, &i.ConstraintName, &i.ConstraintType, diff --git a/internal/schema/schema.go b/internal/schema/schema.go index 1a6a948..3fc16e9 100644 --- a/internal/schema/schema.go +++ b/internal/schema/schema.go @@ -315,6 +315,7 @@ func (i GetIndexDefStatement) ToCreateIndexConcurrently() (string, error) { type ( IndexConstraintType string + RelKind string // IndexConstraint informally represents a constraint that is always 1:1 with an index, i.e., // primary and unique constraints. It's easiest to just treat these like a property of the index rather than @@ -329,11 +330,14 @@ type ( Index struct { // Name is the name of the index. We don't store the schema because the schema is just the schema of the table. // Referencing the name is an anti-pattern because it is not qualified. Use should use GetSchemaQualifiedName instead. - Name string - OwningTable SchemaQualifiedName - Columns []string - IsInvalid bool - IsUnique bool + Name string + // OwningRelName refers to the owning table or materialized view. + OwningRelName SchemaQualifiedName + // OwningRelKind is the relkind of the owning relation. + OwningRelKind RelKind + Columns []string + IsInvalid bool + IsUnique bool Constraint *IndexConstraint @@ -346,6 +350,10 @@ type ( const ( PkIndexConstraintType IndexConstraintType = "p" + + RelKindOrdinaryTable RelKind = "r" + RelKindPartitionedTable RelKind = "p" + RelKindMaterializedView RelKind = "m" ) func (i Index) GetName() string { @@ -354,7 +362,7 @@ func (i Index) GetName() string { func (i Index) GetSchemaQualifiedName() SchemaQualifiedName { return SchemaQualifiedName{ - SchemaName: i.OwningTable.SchemaName, + SchemaName: i.OwningRelName.SchemaName, EscapedName: EscapeIdentifier(i.Name), } } @@ -1136,10 +1144,11 @@ func (s *schemaFetcher) buildIndex(rawIndex queries.GetIndexesRow) Index { } return Index{ - OwningTable: SchemaQualifiedName{ + OwningRelName: SchemaQualifiedName{ SchemaName: rawIndex.TableSchemaName, EscapedName: EscapeIdentifier(rawIndex.TableName), }, + OwningRelKind: RelKind(rawIndex.OwningTableRelkind), Name: rawIndex.IndexName, Columns: rawIndex.ColumnNames, GetIndexDefStmt: GetIndexDefStatement(rawIndex.DefStmt), diff --git a/internal/schema/schema_test.go b/internal/schema/schema_test.go index 943eadb..7f98e3b 100644 --- a/internal/schema/schema_test.go +++ b/internal/schema/schema_test.go @@ -36,6 +36,9 @@ var ( } testCases = []*testCase{ + // Exclude materialized views from the test for now because Postgres 14-15 fully qualify column names while Postgres + // 16+ simplify fully qualified names to unqualified. The easiest option will be to create a version specific test + // that is skipped for Postgres 14-15. { name: "Simple schema (validate all schema objects and schema name filters)", opts: []GetSchemaOpt{ @@ -187,7 +190,7 @@ var ( ON DELETE CASCADE NOT VALID; - CREATE VIEW schema_2.foo_view + CREATE VIEW schema_2.foo_view WITH (security_barrier=true) AS SELECT foo.id, foo.author FROM schema_2.foo @@ -232,7 +235,7 @@ var ( -- Add a column with a default to test HasMissingValOptimization ALTER TABLE schema_2.foo ADD COLUMN added_col TEXT DEFAULT 'some_default'; `}, - expectedHash: "386c0eb7ee3f4874", + expectedHash: "fdff644bbabb9fc", expectedSchema: Schema{ NamedSchemas: []NamedSchema{ {Name: "public"}, @@ -402,7 +405,8 @@ var ( }, Indexes: []Index{ { - OwningTable: SchemaQualifiedName{SchemaName: "schema_2", EscapedName: "\"foo\""}, + OwningRelName: SchemaQualifiedName{SchemaName: "schema_2", EscapedName: "\"foo\""}, + OwningRelKind: RelKindOrdinaryTable, Name: "foo_pkey", Columns: []string{"id", "version"}, IsUnique: true, @@ -410,30 +414,34 @@ var ( GetIndexDefStmt: "CREATE UNIQUE INDEX foo_pkey ON schema_2.foo USING btree (id, version)", }, { - OwningTable: SchemaQualifiedName{SchemaName: "schema_2", EscapedName: "\"foo\""}, + OwningRelName: SchemaQualifiedName{SchemaName: "schema_2", EscapedName: "\"foo\""}, + OwningRelKind: RelKindOrdinaryTable, Name: "some_idx", Columns: []string{"created_at", "author"}, GetIndexDefStmt: "CREATE INDEX some_idx ON schema_2.foo USING btree (created_at DESC, author)", }, { - OwningTable: SchemaQualifiedName{SchemaName: "schema_2", EscapedName: "\"foo\""}, + OwningRelName: SchemaQualifiedName{SchemaName: "schema_2", EscapedName: "\"foo\""}, + OwningRelKind: RelKindOrdinaryTable, Name: "some_unique_idx", Columns: []string{"content"}, IsUnique: true, GetIndexDefStmt: "CREATE UNIQUE INDEX some_unique_idx ON schema_2.foo USING btree (content)", }, { - OwningTable: SchemaQualifiedName{SchemaName: "schema_2", EscapedName: "\"foo\""}, + OwningRelName: SchemaQualifiedName{SchemaName: "schema_2", EscapedName: "\"foo\""}, + OwningRelKind: RelKindOrdinaryTable, Name: "some_gin_idx", Columns: []string{"author"}, GetIndexDefStmt: "CREATE INDEX some_gin_idx ON schema_2.foo USING gin (author schema_1.gin_trgm_ops)", }, { Name: "some_idx", - OwningTable: SchemaQualifiedName{ + OwningRelName: SchemaQualifiedName{ SchemaName: "schema_1", EscapedName: "\"foo_fk\"", }, + OwningRelKind: RelKindOrdinaryTable, Columns: []string{ "id", "version", @@ -575,7 +583,7 @@ var ( ALTER TABLE foo_fk_1 ADD CONSTRAINT foo_fk_1_fk FOREIGN KEY (author, content) REFERENCES foo_1 (author, content) NOT VALID; `}, - expectedHash: "1f2c44c4589a8d6a", + expectedHash: "301808413c59ab76", expectedSchema: Schema{ NamedSchemas: []NamedSchema{ {Name: "public"}, @@ -716,97 +724,113 @@ var ( }, Indexes: []Index{ { - OwningTable: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo\""}, - Name: "foo_pkey", Columns: []string{"author", "id"}, IsUnique: true, + OwningRelName: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo\""}, + OwningRelKind: RelKindPartitionedTable, + Name: "foo_pkey", Columns: []string{"author", "id"}, IsUnique: true, Constraint: &IndexConstraint{Type: PkIndexConstraintType, EscapedConstraintName: "\"foo_pkey\"", ConstraintDef: "PRIMARY KEY (author, id)", IsLocal: true}, GetIndexDefStmt: "CREATE UNIQUE INDEX foo_pkey ON ONLY public.foo USING btree (author, id)", }, { - OwningTable: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo\""}, - Name: "some_partitioned_idx", Columns: []string{"author"}, + OwningRelName: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo\""}, + OwningRelKind: RelKindPartitionedTable, + Name: "some_partitioned_idx", Columns: []string{"author"}, GetIndexDefStmt: "CREATE INDEX some_partitioned_idx ON ONLY public.foo USING hash (author)", }, { - OwningTable: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo\""}, - Name: "some_unique_partitioned_idx", Columns: []string{"author", "created_at"}, IsUnique: true, + OwningRelName: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo\""}, + OwningRelKind: RelKindPartitionedTable, + Name: "some_unique_partitioned_idx", Columns: []string{"author", "created_at"}, IsUnique: true, GetIndexDefStmt: "CREATE UNIQUE INDEX some_unique_partitioned_idx ON ONLY public.foo USING btree (author, created_at DESC)", }, { - OwningTable: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo\""}, - Name: "some_invalid_idx", Columns: []string{"author", "genre"}, IsInvalid: true, IsUnique: false, + OwningRelName: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo\""}, + OwningRelKind: RelKindPartitionedTable, + Name: "some_invalid_idx", Columns: []string{"author", "genre"}, IsInvalid: true, IsUnique: false, GetIndexDefStmt: "CREATE INDEX some_invalid_idx ON ONLY public.foo USING btree (author, genre)", }, // foo_1 indexes { - OwningTable: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_1\""}, - Name: "foo_1_author_idx", Columns: []string{"author"}, + OwningRelName: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_1\""}, + OwningRelKind: RelKindOrdinaryTable, + Name: "foo_1_author_idx", Columns: []string{"author"}, ParentIdx: &SchemaQualifiedName{SchemaName: "public", EscapedName: "\"some_partitioned_idx\""}, GetIndexDefStmt: "CREATE INDEX foo_1_author_idx ON public.foo_1 USING hash (author)", }, { - OwningTable: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_1\""}, - Name: "foo_1_author_created_at_idx", Columns: []string{"author", "created_at"}, IsUnique: true, + OwningRelName: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_1\""}, + OwningRelKind: RelKindOrdinaryTable, + Name: "foo_1_author_created_at_idx", Columns: []string{"author", "created_at"}, IsUnique: true, ParentIdx: &SchemaQualifiedName{SchemaName: "public", EscapedName: "\"some_unique_partitioned_idx\""}, GetIndexDefStmt: "CREATE UNIQUE INDEX foo_1_author_created_at_idx ON public.foo_1 USING btree (author, created_at DESC)", }, { - OwningTable: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_1\""}, - Name: "foo_1_local_idx", Columns: []string{"author", "content"}, IsUnique: true, + OwningRelName: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_1\""}, + OwningRelKind: RelKindOrdinaryTable, + Name: "foo_1_local_idx", Columns: []string{"author", "content"}, IsUnique: true, GetIndexDefStmt: "CREATE UNIQUE INDEX foo_1_local_idx ON public.foo_1 USING btree (author, content)", }, { - OwningTable: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_1\""}, - Name: "foo_1_pkey", Columns: []string{"author", "id"}, IsUnique: true, + OwningRelName: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_1\""}, + OwningRelKind: RelKindOrdinaryTable, + Name: "foo_1_pkey", Columns: []string{"author", "id"}, IsUnique: true, ParentIdx: &SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_pkey\""}, Constraint: &IndexConstraint{Type: PkIndexConstraintType, EscapedConstraintName: "\"foo_1_pkey\"", ConstraintDef: "PRIMARY KEY (author, id)"}, GetIndexDefStmt: "CREATE UNIQUE INDEX foo_1_pkey ON public.foo_1 USING btree (author, id)", }, // foo_2 indexes { - OwningTable: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_2\""}, - Name: "foo_2_author_idx", Columns: []string{"author"}, + OwningRelName: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_2\""}, + OwningRelKind: RelKindOrdinaryTable, + Name: "foo_2_author_idx", Columns: []string{"author"}, ParentIdx: &SchemaQualifiedName{SchemaName: "public", EscapedName: "\"some_partitioned_idx\""}, GetIndexDefStmt: "CREATE INDEX foo_2_author_idx ON public.foo_2 USING hash (author)", }, { - OwningTable: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_2\""}, - Name: "foo_2_author_created_at_idx", Columns: []string{"author", "created_at"}, IsUnique: true, + OwningRelName: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_2\""}, + OwningRelKind: RelKindOrdinaryTable, + Name: "foo_2_author_created_at_idx", Columns: []string{"author", "created_at"}, IsUnique: true, ParentIdx: &SchemaQualifiedName{SchemaName: "public", EscapedName: "\"some_unique_partitioned_idx\""}, GetIndexDefStmt: "CREATE UNIQUE INDEX foo_2_author_created_at_idx ON public.foo_2 USING btree (author, created_at DESC)", }, { - OwningTable: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_2\""}, - Name: "foo_2_local_idx", Columns: []string{"author", "id"}, IsUnique: true, + OwningRelName: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_2\""}, + OwningRelKind: RelKindOrdinaryTable, + Name: "foo_2_local_idx", Columns: []string{"author", "id"}, IsUnique: true, GetIndexDefStmt: "CREATE UNIQUE INDEX foo_2_local_idx ON public.foo_2 USING btree (author DESC, id)", }, { - OwningTable: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_2\""}, - Name: "foo_2_pkey", Columns: []string{"author", "id"}, IsUnique: true, + OwningRelName: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_2\""}, + OwningRelKind: RelKindOrdinaryTable, + Name: "foo_2_pkey", Columns: []string{"author", "id"}, IsUnique: true, ParentIdx: &SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_pkey\""}, Constraint: &IndexConstraint{Type: PkIndexConstraintType, EscapedConstraintName: "\"foo_2_pkey\"", ConstraintDef: "PRIMARY KEY (author, id)"}, GetIndexDefStmt: "CREATE UNIQUE INDEX foo_2_pkey ON public.foo_2 USING btree (author, id)", }, // foo_3 indexes { - OwningTable: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_3\""}, - Name: "foo_3_author_idx", Columns: []string{"author"}, + OwningRelName: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_3\""}, + OwningRelKind: RelKindOrdinaryTable, + Name: "foo_3_author_idx", Columns: []string{"author"}, ParentIdx: &SchemaQualifiedName{SchemaName: "public", EscapedName: "\"some_partitioned_idx\""}, GetIndexDefStmt: "CREATE INDEX foo_3_author_idx ON public.foo_3 USING hash (author)", }, { - OwningTable: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_3\""}, - Name: "foo_3_author_created_at_idx", Columns: []string{"author", "created_at"}, IsUnique: true, + OwningRelName: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_3\""}, + OwningRelKind: RelKindOrdinaryTable, + Name: "foo_3_author_created_at_idx", Columns: []string{"author", "created_at"}, IsUnique: true, ParentIdx: &SchemaQualifiedName{SchemaName: "public", EscapedName: "\"some_unique_partitioned_idx\""}, GetIndexDefStmt: "CREATE UNIQUE INDEX foo_3_author_created_at_idx ON public.foo_3 USING btree (author, created_at DESC)", }, { - OwningTable: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_3\""}, - Name: "foo_3_local_idx", Columns: []string{"author", "created_at"}, IsUnique: true, + OwningRelName: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_3\""}, + OwningRelKind: RelKindOrdinaryTable, + Name: "foo_3_local_idx", Columns: []string{"author", "created_at"}, IsUnique: true, GetIndexDefStmt: "CREATE UNIQUE INDEX foo_3_local_idx ON public.foo_3 USING btree (author, created_at)", }, { - OwningTable: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_3\""}, - Name: "foo_3_pkey", Columns: []string{"author", "id"}, IsUnique: true, + OwningRelName: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_3\""}, + OwningRelKind: RelKindOrdinaryTable, + Name: "foo_3_pkey", Columns: []string{"author", "id"}, IsUnique: true, ParentIdx: &SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_pkey\""}, Constraint: &IndexConstraint{Type: PkIndexConstraintType, EscapedConstraintName: "\"foo_3_pkey\"", ConstraintDef: "PRIMARY KEY (author, id)"}, GetIndexDefStmt: "CREATE UNIQUE INDEX foo_3_pkey ON public.foo_3 USING btree (author, id)", @@ -908,8 +932,9 @@ var ( }, Indexes: []Index{ { - OwningTable: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_1\""}, - Name: "foo_1_pkey", Columns: []string{"author", "id"}, IsUnique: true, + OwningRelName: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo_1\""}, + OwningRelKind: RelKindOrdinaryTable, + Name: "foo_1_pkey", Columns: []string{"author", "id"}, IsUnique: true, Constraint: &IndexConstraint{Type: PkIndexConstraintType, EscapedConstraintName: "\"foo_1_pkey\"", ConstraintDef: "PRIMARY KEY (author, id)", IsLocal: true}, GetIndexDefStmt: "CREATE UNIQUE INDEX foo_1_pkey ON public.foo_1 USING btree (author, id)", }, @@ -1024,8 +1049,9 @@ var ( }, Indexes: []Index{ { - OwningTable: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo\""}, - Name: "foo_value_idx", Columns: []string{"renamed_value"}, + OwningRelName: SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foo\""}, + OwningRelKind: RelKindOrdinaryTable, + Name: "foo_value_idx", Columns: []string{"renamed_value"}, GetIndexDefStmt: "CREATE INDEX foo_value_idx ON public.foo USING btree (renamed_value)", }, }, diff --git a/pkg/diff/schema_migration_plan_test.go b/pkg/diff/schema_migration_plan_test.go index 2c3311f..6d8544b 100644 --- a/pkg/diff/schema_migration_plan_test.go +++ b/pkg/diff/schema_migration_plan_test.go @@ -54,8 +54,8 @@ var ( }, Indexes: []schema.Index{ { - OwningTable: schema.SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foobar\""}, - Name: "some_idx", Columns: []string{"foo", "bar"}, + OwningRelName: schema.SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foobar\""}, + Name: "some_idx", Columns: []string{"foo", "bar"}, GetIndexDefStmt: "CREATE INDEX some_idx ON public.foobar USING btree (foo, bar)", IsUnique: true, IsInvalid: true, }, @@ -77,8 +77,8 @@ var ( Indexes: []schema.Index{ { - OwningTable: schema.SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foobar\""}, - Name: "some_idx", Columns: []string{"foo", "bar"}, + OwningRelName: schema.SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foobar\""}, + Name: "some_idx", Columns: []string{"foo", "bar"}, GetIndexDefStmt: "CREATE INDEX some_idx ON public.foobar USING btree (foo, bar)", IsUnique: true, }, @@ -135,15 +135,15 @@ var ( Indexes: []schema.Index{ // foobar indexes { - OwningTable: schema.SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foobar\""}, - Name: "some_idx", Columns: []string{"foo, bar"}, + OwningRelName: schema.SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foobar\""}, + Name: "some_idx", Columns: []string{"foo, bar"}, GetIndexDefStmt: "CREATE INDEX some_idx ON ONLY public.foobar USING btree (foo, bar)", IsInvalid: true, }, // foobar_1 indexes { - OwningTable: schema.SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foobar_1\""}, - Name: "foobar_1_some_idx", Columns: []string{"foo", "bar"}, + OwningRelName: schema.SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foobar_1\""}, + Name: "foobar_1_some_idx", Columns: []string{"foo", "bar"}, GetIndexDefStmt: "CREATE INDEX foobar_1_some_idx ON public.foobar_1 USING btree (foo, bar)", IsInvalid: true, }, @@ -178,14 +178,14 @@ var ( Indexes: []schema.Index{ // foobar indexes { - OwningTable: schema.SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foobar\""}, - Name: "some_idx", Columns: []string{"foo, bar"}, + OwningRelName: schema.SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foobar\""}, + Name: "some_idx", Columns: []string{"foo, bar"}, GetIndexDefStmt: "CREATE INDEX some_idx ON ONLY public.foobar USING btree (foo, bar)", }, // foobar_1 indexes { - OwningTable: schema.SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foobar_1\""}, - Name: "foobar_1_some_idx", Columns: []string{"foo", "bar"}, + OwningRelName: schema.SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foobar_1\""}, + Name: "foobar_1_some_idx", Columns: []string{"foo", "bar"}, ParentIdx: &schema.SchemaQualifiedName{SchemaName: "public", EscapedName: "\"some_idx\""}, GetIndexDefStmt: "CREATE INDEX foobar_1_some_idx ON public.foobar_1 USING btree (foo, bar)", }, @@ -282,7 +282,7 @@ var ( Indexes: []schema.Index{ // foobar indexes { - OwningTable: schema.SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foobar\""}, + OwningRelName: schema.SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foobar\""}, // This index points to its child, which is wrong, but induces a loop Name: "some_idx", Columns: []string{"foo", "bar"}, ParentIdx: &schema.SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foobar_1_some_idx\""}, @@ -290,8 +290,8 @@ var ( }, // foobar_1 indexes { - OwningTable: schema.SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foobar_1\""}, - Name: "foobar_1_some_idx", Columns: []string{"foo", "bar"}, + OwningRelName: schema.SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foobar_1\""}, + Name: "foobar_1_some_idx", Columns: []string{"foo", "bar"}, ParentIdx: &schema.SchemaQualifiedName{SchemaName: "public", EscapedName: "\"some_idx\""}, GetIndexDefStmt: "CREATE INDEX foobar_1_some_idx ON public.foobar_1 USING btree (foo, bar)", }, @@ -326,7 +326,7 @@ var ( Indexes: []schema.Index{ // foobar indexes { - OwningTable: schema.SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foobar\""}, + OwningRelName: schema.SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foobar\""}, // This index points to its child, which is wrong, but induces a loop Name: "some_idx", Columns: []string{"foo", "bar"}, ParentIdx: &schema.SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foobar_1_some_idx\""}, @@ -334,8 +334,8 @@ var ( }, // foobar_1 indexes { - OwningTable: schema.SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foobar_1\""}, - Name: "foobar_1_some_idx", Columns: []string{"foo", "bar"}, + OwningRelName: schema.SchemaQualifiedName{SchemaName: "public", EscapedName: "\"foobar_1\""}, + Name: "foobar_1_some_idx", Columns: []string{"foo", "bar"}, ParentIdx: &schema.SchemaQualifiedName{SchemaName: "public", EscapedName: "\"some_idx\""}, GetIndexDefStmt: "CREATE INDEX foobar_1_some_idx ON public.foobar_1 USING btree (foo, bar)", }, diff --git a/pkg/diff/sql_generator.go b/pkg/diff/sql_generator.go index bb38a38..e28d948 100644 --- a/pkg/diff/sql_generator.go +++ b/pkg/diff/sql_generator.go @@ -459,7 +459,7 @@ func buildIndexDiff(deps indexDiffConfig, old, new schema.Index) (diff indexDiff updatedOld := old - if _, isOnNewTable := deps.addedTablesByName[new.OwningTable.GetName()]; isOnNewTable { + if _, isOnNewTable := deps.addedTablesByName[new.OwningRelName.GetName()]; isOnNewTable { // If the table is new, then it must be re-created (this occurs if the base table has been // re-created). In other words, an index must be re-created if the owning table is re-created return indexDiff{}, true, nil @@ -543,6 +543,10 @@ func (s schemaSQLGenerator) Alter(diff schemaDiff) ([]Statement, error) { addedTablesByName := buildSchemaObjByNameMap(diff.tableDiffs.adds) functionsInNewSchemaByName := buildSchemaObjByNameMap(diff.new.Functions) + materializedViewsInNewSchemaByName := buildSchemaObjByNameMap(diff.new.MaterializedViews) + deletedMaterializedViewsByName := buildSchemaObjByNameMap(diff.materializedViewDiffs.deletes) + addedMaterializedViewsByName := buildSchemaObjByNameMap(diff.materializedViewDiffs.adds) + namedSchemaStatements, err := diff.namedSchemaDiffs.resolveToSQLGroupedByEffect(&namedSchemaSQLGenerator{}) if err != nil { return nil, fmt.Errorf("resolving named schema sql statements: %w", err) @@ -586,9 +590,14 @@ func (s schemaSQLGenerator) Alter(diff schemaDiff) ([]Statement, error) { partialGraph = concatPartialGraphs(partialGraph, renameConflictingIndexesPartialGraph) indexGenerator := legacyToNewSqlVertexGenerator[schema.Index, indexDiff](&indexSQLVertexGenerator{ - deletedTablesByName: deletedTablesByName, - addedTablesByName: addedTablesByName, - tablesInNewSchemaByName: tablesInNewSchemaByName, + deletedTablesByName: deletedTablesByName, + addedTablesByName: addedTablesByName, + tablesInNewSchemaByName: tablesInNewSchemaByName, + + deletedMaterializedViewsByName: deletedMaterializedViewsByName, + addedMaterializedViewsByName: addedMaterializedViewsByName, + materializedViewsInNewSchemaByName: materializedViewsInNewSchemaByName, + indexesInNewSchemaByName: buildSchemaObjByNameMap(diff.new.Indexes), renameSQLVertexGenerator: renameConflictingIndexesGenerator, @@ -692,7 +701,7 @@ func (s schemaSQLGenerator) Alter(diff schemaDiff) ([]Statement, error) { func buildIndexesByTableNameMap(indexes []schema.Index) map[string][]schema.Index { var output = make(map[string][]schema.Index) for _, idx := range indexes { - output[idx.OwningTable.GetName()] = append(output[idx.OwningTable.GetName()], idx) + output[idx.OwningRelName.GetName()] = append(output[idx.OwningRelName.GetName()], idx) } return output } @@ -1590,6 +1599,16 @@ type indexSQLVertexGenerator struct { // tablesInNewSchemaByName is a map of table name to tables (and partitions) in the new schema. // These tables are not necessarily new. This is used to identify if the table is partitioned tablesInNewSchemaByName map[string]schema.Table + + // deletedMaterializedViewsByName is a map of materialized view name to the deleted materialized views + deletedMaterializedViewsByName map[string]schema.MaterializedView + // addedMaterializedViewsByName is a map of materialized view name to the new materialized views + // This is used to identify if hazards are necessary + addedMaterializedViewsByName map[string]schema.MaterializedView + // materializedViewsInNewSchemaByName is a map of materialized view name to materialized views in the new schema. + // These materialized views are not necessarily new. + materializedViewsInNewSchemaByName map[string]schema.MaterializedView + // indexesInNewSchemaByName is a map of index name to the index // This is used to identify the parent index is a primary key indexesInNewSchemaByName map[string]schema.Index @@ -1602,13 +1621,51 @@ type indexSQLVertexGenerator struct { planOptions *planOptions } +// wasOwningRelationDeleted returns true if the owning table or materialized view was deleted. +// If the owning relation was deleted, the index will be automatically dropped. +func (isg *indexSQLVertexGenerator) wasOwningRelationDeleted(index schema.Index) bool { + switch index.OwningRelKind { + case schema.RelKindMaterializedView: + _, deleted := isg.deletedMaterializedViewsByName[index.OwningRelName.GetName()] + return deleted + default: + _, deleted := isg.deletedTablesByName[index.OwningRelName.GetName()] + return deleted + } +} + +// wasOwningRelationAdded returns true if the owning table or materialized view is newly added. +// Used to determine if index hazards should be stripped (new relations have no data). +func (isg *indexSQLVertexGenerator) wasOwningRelationAdded(index schema.Index) bool { + switch index.OwningRelKind { + case schema.RelKindMaterializedView: + _, added := isg.addedMaterializedViewsByName[index.OwningRelName.GetName()] + return added + default: + _, added := isg.addedTablesByName[index.OwningRelName.GetName()] + return added + } +} + +// isOwningRelationInNewSchema returns true if the owning table or materialized view exists in the new schema. +func (isg *indexSQLVertexGenerator) isOwningRelationInNewSchema(index schema.Index) bool { + switch index.OwningRelKind { + case schema.RelKindMaterializedView: + _, exists := isg.materializedViewsInNewSchemaByName[index.OwningRelName.GetName()] + return exists + default: + _, exists := isg.tablesInNewSchemaByName[index.OwningRelName.GetName()] + return exists + } +} + func (isg *indexSQLVertexGenerator) Add(index schema.Index) ([]Statement, error) { stmts, err := isg.addIdxStmtsWithHazards(index) if err != nil { return stmts, err } - if _, isNewTable := isg.addedTablesByName[index.OwningTable.GetName()]; isNewTable { + if isg.wasOwningRelationAdded(index) { stmts = stripMigrationHazards(stmts...) } return stmts, nil @@ -1632,7 +1689,7 @@ func (isg *indexSQLVertexGenerator) addIdxStmtsWithHazards(index schema.Index) ( //table. We can then concurrently build all of the partitioned indexes and attach them. // Without "ONLY", all the partitioned indexes will be automatically built return []Statement{{ - DDL: fmt.Sprintf("ALTER TABLE ONLY %s ADD CONSTRAINT %s %s", index.OwningTable.GetFQEscapedName(), index.Constraint.EscapedConstraintName, index.Constraint.ConstraintDef), + DDL: fmt.Sprintf("ALTER TABLE ONLY %s ADD CONSTRAINT %s %s", index.OwningRelName.GetFQEscapedName(), index.Constraint.EscapedConstraintName, index.Constraint.ConstraintDef), Timeout: statementTimeoutDefault, LockTimeout: lockTimeoutDefault, }}, nil @@ -1654,7 +1711,7 @@ func (isg *indexSQLVertexGenerator) addIdxStmtsWithHazards(index schema.Index) ( stmts = append(stmts, addConstraintStmt) } - if index.ParentIdx != nil && isg.attachPartitionSQLVertexGenerator.isPartitionAlreadyAttachedBeforeIndexBuilds(index.OwningTable) { + if index.ParentIdx != nil && isg.attachPartitionSQLVertexGenerator.isPartitionAlreadyAttachedBeforeIndexBuilds(index.OwningRelName) { // Only attach the index if the index is built after the table is partitioned. If the partition // hasn't already been attached, the index/constraint will be automatically attached when the table partition is // attached @@ -1697,9 +1754,8 @@ func (isg *indexSQLVertexGenerator) buildCreateIndexStatement(index schema.Index } func (isg *indexSQLVertexGenerator) Delete(index schema.Index) ([]Statement, error) { - _, tableWasDeleted := isg.deletedTablesByName[index.OwningTable.GetName()] - // An index will be dropped if its owning table is dropped. - if tableWasDeleted { + // An index will be dropped if its owning relation (table or materialized view) is dropped. + if isg.wasOwningRelationDeleted(index) { return nil, nil } @@ -1728,7 +1784,7 @@ func (isg *indexSQLVertexGenerator) Delete(index schema.Index) ([]Statement, err // the constraint without dropping the index return []Statement{ { - DDL: dropConstraintDDL(index.OwningTable, escapedConstraintName), + DDL: dropConstraintDDL(index.OwningRelName, escapedConstraintName), Timeout: statementTimeoutDefault, LockTimeout: lockTimeoutDefault, @@ -1801,14 +1857,22 @@ func (isg *indexSQLVertexGenerator) Alter(diff indexDiff) ([]Statement, error) { } func (isg *indexSQLVertexGenerator) isIndexOnPartitionedTable(index schema.Index) (bool, error) { + // Materialized views cannot be partitioned + if index.OwningRelKind == schema.RelKindMaterializedView { + return false, nil + } return isIndexOnPartitionedTable(isg.tablesInNewSchemaByName, index) } // Returns true if the table the index belongs too is partitioned. If the table is a partition of a -// partitioned table, this will always return false +// partitioned table, this will always return false. Materialized views cannot be partitioned. func isIndexOnPartitionedTable(tablesInNewSchemaByName map[string]schema.Table, index schema.Index) (bool, error) { - if owningTable, ok := tablesInNewSchemaByName[index.OwningTable.GetName()]; !ok { - return false, fmt.Errorf("could not find table in new schema with name %s", index.OwningTable.GetName()) + // Materialized views cannot be partitioned + if index.OwningRelKind == schema.RelKindMaterializedView { + return false, nil + } + if owningTable, ok := tablesInNewSchemaByName[index.OwningRelName.GetName()]; !ok { + return false, fmt.Errorf("could not find table in new schema with name %s", index.OwningRelName.GetName()) } else { return owningTable.IsPartitioned(), nil } @@ -1821,7 +1885,7 @@ func (isg *indexSQLVertexGenerator) addIndexConstraint(index schema.Index) (Stat } return Statement{ DDL: fmt.Sprintf("%s %s USING INDEX %s", - addConstraintPrefix(index.OwningTable, index.Constraint.EscapedConstraintName), + addConstraintPrefix(index.OwningRelName, index.Constraint.EscapedConstraintName), sqlConstraintType, index.GetSchemaQualifiedName().EscapedName), Timeout: statementTimeoutDefault, @@ -1858,11 +1922,19 @@ func buildIndexVertexId(name schema.SchemaQualifiedName, diffType diffType) sqlV func (isg *indexSQLVertexGenerator) GetAddAlterDependencies(index, _ schema.Index) ([]dependency, error) { dependencies := []dependency{ - mustRun(isg.GetSQLVertexId(index, diffTypeAddAlter)).after(buildTableVertexId(index.OwningTable, diffTypeAddAlter)), // To allow for online changes to indexes, rename the older version of the index (if it exists) before the new version is added mustRun(isg.GetSQLVertexId(index, diffTypeAddAlter)).after(buildRenameConflictingIndexVertexId(index.GetSchemaQualifiedName(), diffTypeAddAlter)), } + // Add dependency on owning relation based on relation kind + if index.OwningRelKind == schema.RelKindMaterializedView { + dependencies = append(dependencies, + mustRun(isg.GetSQLVertexId(index, diffTypeAddAlter)).after(buildMaterializedViewVertexId(index.OwningRelName, diffTypeAddAlter))) + } else { + dependencies = append(dependencies, + mustRun(isg.GetSQLVertexId(index, diffTypeAddAlter)).after(buildTableVertexId(index.OwningRelName, diffTypeAddAlter))) + } + if index.ParentIdx != nil { // Partitions of indexes must be created after the parent index is created dependencies = append(dependencies, @@ -1874,25 +1946,45 @@ func (isg *indexSQLVertexGenerator) GetAddAlterDependencies(index, _ schema.Inde func (isg *indexSQLVertexGenerator) GetDeleteDependencies(index schema.Index) ([]dependency, error) { dependencies := []dependency{ - mustRun(isg.GetSQLVertexId(index, diffTypeDelete)).after(buildTableVertexId(index.OwningTable, diffTypeDelete)), // Drop the index after it has been potentially renamed mustRun(isg.GetSQLVertexId(index, diffTypeDelete)).after(buildRenameConflictingIndexVertexId(index.GetSchemaQualifiedName(), diffTypeAddAlter)), } + // Add dependency on owning relation based on relation kind + if index.OwningRelKind == schema.RelKindMaterializedView { + dependencies = append(dependencies, + mustRun(isg.GetSQLVertexId(index, diffTypeDelete)).after(buildMaterializedViewVertexId(index.OwningRelName, diffTypeDelete))) + } else { + dependencies = append(dependencies, + mustRun(isg.GetSQLVertexId(index, diffTypeDelete)).after(buildTableVertexId(index.OwningRelName, diffTypeDelete))) + } + if index.ParentIdx != nil { // Since dropping the parent index will cause the partition of the index to drop, the parent drop should come // before dependencies = append(dependencies, mustRun(isg.GetSQLVertexId(index, diffTypeDelete)).after(buildIndexVertexId(*index.ParentIdx, diffTypeDelete))) } - dependencies = append(dependencies, isg.addDepsOnTableAddAlterIfNecessary(index)...) + dependencies = append(dependencies, isg.addDepsOnRelationAddAlterIfNecessary(index)...) return dependencies, nil } -func (isg *indexSQLVertexGenerator) addDepsOnTableAddAlterIfNecessary(index schema.Index) []dependency { +func (isg *indexSQLVertexGenerator) addDepsOnRelationAddAlterIfNecessary(index schema.Index) []dependency { + // Materialized views don't have column-level dependencies or partitioning + if index.OwningRelKind == schema.RelKindMaterializedView { + if !isg.isOwningRelationInNewSchema(index) { + // If the MV is deleted, we don't need dependencies + return nil + } + // For MVs, we just need the index drop to come before MV alterations + return []dependency{ + mustRun(isg.GetSQLVertexId(index, diffTypeDelete)).before(buildMaterializedViewVertexId(index.OwningRelName, diffTypeAddAlter)), + } + } + // This could be cleaner if start sorting columns separately in the graph - parentTable, ok := isg.tablesInNewSchemaByName[index.OwningTable.GetName()] + parentTable, ok := isg.tablesInNewSchemaByName[index.OwningRelName.GetName()] if !ok { // If the parent table is deleted, we don't need to worry about making the index statement come // before any alters @@ -1901,7 +1993,7 @@ func (isg *indexSQLVertexGenerator) addDepsOnTableAddAlterIfNecessary(index sche // These dependencies will force the index deletion statement to come before the table AddAlter addAlterColumnDeps := []dependency{ - mustRun(isg.GetSQLVertexId(index, diffTypeDelete)).before(buildTableVertexId(index.OwningTable, diffTypeAddAlter)), + mustRun(isg.GetSQLVertexId(index, diffTypeDelete)).before(buildTableVertexId(index.OwningRelName, diffTypeAddAlter)), } if parentTable.ParentTable != nil { // If the table is partitioned, columns modifications occur on the base table not the children. Thus, we