diff --git a/README.md b/README.md index 27c9a27..346ecdc 100644 --- a/README.md +++ b/README.md @@ -171,12 +171,11 @@ for _, stmt := range plan.Statements { ``` # Supported Postgres versions -Supported: 14, 15, 16, 17 +Supported: 14, 15, 16, 17 Unsupported: <= 13 are not supported. Use at your own risk. # Unsupported migrations An abridged list of unsupported migrations: -- Materialized Views (Planned); Views **are** supported - Privileges (Planned) - Types (Only enums are currently supported) - Renaming. The diffing library relies on names to identify the old and new versions of a table, index, etc. If you rename diff --git a/internal/migration_acceptance_tests/materialized_view_cases_test.go b/internal/migration_acceptance_tests/materialized_view_cases_test.go new file mode 100644 index 0000000..06651ef --- /dev/null +++ b/internal/migration_acceptance_tests/materialized_view_cases_test.go @@ -0,0 +1,546 @@ +package migration_acceptance_tests + +import ( + "testing" + + "github.com/stripe/pg-schema-diff/pkg/diff" +) + +var materializedViewAcceptanceTestCases = []acceptanceTestCase{ + { + name: "No-op", + oldSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255) NOT NULL, + bar VARCHAR(255) UNIQUE, + buzz BOOLEAN DEFAULT true + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT id, foo, bar + FROM foobar + WHERE buzz = true; + + CREATE MATERIALIZED VIEW foobar_count AS + SELECT COUNT(*) as total_count + FROM foobar; + `, + }, + newSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255) NOT NULL, + bar VARCHAR(255) UNIQUE, + buzz BOOLEAN DEFAULT true + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT id, foo, bar + FROM foobar + WHERE buzz = true; + + CREATE MATERIALIZED VIEW foobar_count AS + SELECT COUNT(*) as total_count + FROM foobar; + `, + }, + expectEmptyPlan: true, + }, + { + name: "Add materialized view", + oldSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255) NOT NULL, + bar DECIMAL(10,2), + buzz INT + ); + `, + }, + newSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255) NOT NULL, + bar DECIMAL(10,2), + buzz INT + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT id, foo, bar + FROM foobar + WHERE bar > 100.00; + `, + }, + }, + { + name: "Add recursive materialized view", + oldSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255) NOT NULL, + parent_id INT REFERENCES foobar(id) + ); + `, + }, + newSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255) NOT NULL, + parent_id INT REFERENCES foobar(id) + ); + + CREATE MATERIALIZED VIEW foobar_hierarchy AS + WITH RECURSIVE hierarchy AS ( + SELECT id, foo, parent_id, 0 as level + FROM foobar + WHERE parent_id IS NULL + UNION ALL + SELECT f.id, f.foo, f.parent_id, h.level + 1 + FROM foobar f + JOIN hierarchy h ON f.parent_id = h.id + ) + SELECT * FROM hierarchy; + `, + }, + }, + { + name: "Add materialized view with dependent table", + oldSchemaDDL: nil, + newSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo INT NOT NULL, + bar DECIMAL(10,2), + buzz VARCHAR(50) DEFAULT 'pending' + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT id, foo, bar + FROM foobar + WHERE buzz = 'pending'; + `, + }, + }, + { + name: "Add materialized view with dependent column", + oldSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255) NOT NULL, + bar VARCHAR(100) + ); + `, + }, + newSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255) NOT NULL, + bar VARCHAR(100), + buzz DECIMAL(10,2) + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT id, foo, bar, buzz + FROM foobar; + `, + }, + }, + { + name: "Drop materialized view", + oldSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255), + bar INT, + buzz VARCHAR(100) + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT id, foo, bar + FROM foobar + WHERE bar < 10; + `, + }, + newSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255), + bar INT, + buzz VARCHAR(100) + ); + `, + }, + }, + { + name: "Drop materialized view and underlying table", + oldSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255), + bar TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT id, foo, bar + FROM foobar + WHERE bar > CURRENT_DATE - INTERVAL '7 days'; + `, + }, + newSchemaDDL: nil, + expectedHazardTypes: []diff.MigrationHazardType{ + diff.MigrationHazardTypeDeletesData, + }, + }, + { + name: "Drop materialized view and underlying column", + oldSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255), + bar INT, + buzz VARCHAR(255), + fizz TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT id, foo, bar, buzz + FROM foobar; + `, + }, + newSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255), + bar INT, + fizz TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + `, + }, + expectedHazardTypes: []diff.MigrationHazardType{ + diff.MigrationHazardTypeDeletesData, + }, + }, + { + name: "Recreate materialized view due to table recreation (unpartitioned to partitioned)", + oldSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo INT, + bar DECIMAL(10,2), + buzz DATE + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT DATE_TRUNC('month', buzz) as month, SUM(bar) as total + FROM foobar + GROUP BY DATE_TRUNC('month', buzz); + `, + }, + newSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT, + foo INT, + bar DECIMAL(10,2), + buzz DATE, + PRIMARY KEY (id, buzz) + ) PARTITION BY RANGE (buzz); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT DATE_TRUNC('month', buzz) as month, SUM(bar) as total + FROM foobar + GROUP BY DATE_TRUNC('month', buzz); + `, + }, + expectedHazardTypes: []diff.MigrationHazardType{ + diff.MigrationHazardTypeDeletesData, + }, + }, + { + name: "Recreate materialized view due dependent table changing", + oldSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255), + bar DECIMAL(10,2) + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT foo, AVG(bar) as avg_bar + FROM foobar + GROUP BY foo; + `, + }, + newSchemaDDL: []string{ + ` + CREATE TABLE bar( + id INT PRIMARY KEY, + foo VARCHAR(255), + bar DECIMAL(10,2) + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT foo, AVG(bar) as avg_bar + FROM bar + GROUP BY foo; + `, + }, + expectedHazardTypes: []diff.MigrationHazardType{ + diff.MigrationHazardTypeDeletesData, + }, + }, + { + name: "Recreate materialized view due to dependent column changing", + oldSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + bar TIMESTAMP, + foo VARCHAR(255), + old_buzz VARCHAR(50) + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT id, foo, old_buzz as buzz + FROM foobar + WHERE old_buzz = 'active'; + `, + }, + newSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + bar TIMESTAMP, + foo VARCHAR(255), + new_buzz VARCHAR(50) + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT id, foo, new_buzz as buzz + FROM foobar + WHERE new_buzz = 'active'; + `, + }, + expectedHazardTypes: []diff.MigrationHazardType{ + diff.MigrationHazardTypeDeletesData, + }, + }, + { + name: "alter - add column and change condition", + oldSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + bar TIMESTAMP, + foo VARCHAR(255) + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT id, foo + FROM foobar + `, + }, + newSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + bar TIMESTAMP, + foo VARCHAR(255) + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT id, foo, bar + FROM foobar + WHERE bar < CURRENT_TIMESTAMP + `, + }, + }, + { + name: "alter - removing select column", + oldSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255), + bar VARCHAR(255), + buzz VARCHAR(255), + fizz BOOLEAN DEFAULT true + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT id, foo, bar + FROM foobar + WHERE fizz = true; + `, + }, + newSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255), + bar VARCHAR(255), + buzz VARCHAR(255), + fizz BOOLEAN DEFAULT true + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT foo, bar + FROM foobar + WHERE fizz = true; + `, + }, + }, + { + name: "alter - add fill factor option", + oldSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255), + bar INT, + buzz TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT foo, bar, buzz + FROM foobar + WHERE buzz > CURRENT_DATE - INTERVAL '1 day'; + `, + }, + newSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255), + bar INT, + buzz TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + + CREATE MATERIALIZED VIEW foobar_view WITH (fillfactor = 70) AS + SELECT foo, bar, buzz + FROM foobar + WHERE buzz > CURRENT_DATE - INTERVAL '1 day'; + `, + }, + }, + { + name: "alter - change autovacuum option", + oldSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo INT NOT NULL, + bar VARCHAR(50) DEFAULT 'pending', + buzz DECIMAL(10,2) CHECK (buzz > 0) + ); + + CREATE MATERIALIZED VIEW foobar_view WITH (autovacuum_enabled = true) AS + SELECT id, foo, buzz + FROM foobar + WHERE bar = 'pending'; + `, + }, + newSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo INT NOT NULL, + bar VARCHAR(50) DEFAULT 'pending', + buzz DECIMAL(10,2) CHECK (buzz > 0) + ); + + CREATE MATERIALIZED VIEW foobar_view WITH (autovacuum_enabled = false) AS + SELECT id, foo, buzz + FROM foobar + WHERE bar = 'pending'; + `, + }, + }, + { + name: "alter - add log_autovacuum_min_duration option", + oldSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255) NOT NULL, + bar DECIMAL(10,2) CHECK (bar > 0), + buzz BOOLEAN DEFAULT true + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT id, foo, bar + FROM foobar + WHERE buzz = true; + `, + }, + newSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255) NOT NULL, + bar DECIMAL(10,2) CHECK (bar > 0), + buzz BOOLEAN DEFAULT true + ); + + CREATE MATERIALIZED VIEW foobar_view WITH (log_autovacuum_min_duration = 1000) AS + SELECT id, foo, bar + FROM foobar + WHERE buzz = true; + `, + }, + }, + { + name: "alter - remove toast_tuple_target option", + oldSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255), + bar INT CHECK (bar >= 0), + buzz BOOLEAN DEFAULT true + ); + + CREATE MATERIALIZED VIEW foobar_view WITH (toast_tuple_target = 2048) AS + SELECT id, foo, bar + FROM foobar + WHERE buzz = true; + `, + }, + newSchemaDDL: []string{ + ` + CREATE TABLE foobar( + id INT PRIMARY KEY, + foo VARCHAR(255), + bar INT CHECK (bar >= 0), + buzz BOOLEAN DEFAULT true + ); + + CREATE MATERIALIZED VIEW foobar_view AS + SELECT id, foo, bar + FROM foobar + WHERE buzz = true; + `, + }, + }, +} + +func TestMaterializedViewTestCases(t *testing.T) { + runTestCases(t, materializedViewAcceptanceTestCases) +} diff --git a/internal/queries/queries.sql b/internal/queries/queries.sql index 3f54404..5889b65 100644 --- a/internal/queries/queries.sql +++ b/internal/queries/queries.sql @@ -539,3 +539,75 @@ WHERE AND depend.objid = c.oid AND depend.deptype = 'e' ); + +-- name: GetMaterializedViews :many +SELECT + n.nspname::TEXT AS schema_name, + c.relname::TEXT AS view_name, + c.reloptions::TEXT [] AS rel_options, + COALESCE(ts.spcname, '')::TEXT AS tablespace_name, + (SELECT + ARRAY_AGG(DISTINCT JSONB_BUILD_OBJECT( + 'schema', dep_ns.nspname, + 'name', dep_c.relname, + 'columns', ( + SELECT + ARRAY_AGG( + a.attname::TEXT + ORDER BY a.attnum + ) + FROM pg_catalog.pg_attribute AS a + WHERE + a.attrelid = dep_c.oid + AND a.attnum > 0 + AND NOT a.attisdropped + AND a.attnum IN ( + -- Get only columns that the materialized view depends + -- on. + SELECT DISTINCT d3.refobjsubid + FROM pg_catalog.pg_depend AS d3 + WHERE + d3.refobjid = dep_c.oid + AND d3.refobjsubid > 0 + AND d3.classid = 'pg_rewrite'::REGCLASS + AND EXISTS ( + SELECT 1 + FROM pg_catalog.pg_rewrite AS rw + WHERE + rw.oid = d3.objid + AND rw.ev_class = c.oid + ) + ) + ) + )) + FROM pg_catalog.pg_depend AS d + INNER JOIN pg_catalog.pg_rewrite AS r ON d.objid = r.oid + INNER JOIN pg_catalog.pg_depend AS d2 ON r.oid = d2.objid + INNER JOIN + pg_catalog.pg_class AS dep_c + ON d2.refobjid = dep_c.oid AND dep_c.relkind IN ('r', 'p') + INNER JOIN + pg_catalog.pg_namespace AS dep_ns + ON dep_c.relnamespace = dep_ns.oid + -- Cast to text because pgv4/pq does not support unmarshalling JSON + -- arrays into []json.RawMessage. + -- Instead, they must be unmarshalled as string arrays. + -- https://github.com/lib/pq/pull/466 + WHERE d.refobjid = c.oid)::TEXT [] AS table_dependencies, + PG_GET_VIEWDEF(c.oid, true) AS view_definition +FROM pg_catalog.pg_class AS c +INNER JOIN pg_catalog.pg_namespace AS n ON c.relnamespace = n.oid +LEFT JOIN pg_catalog.pg_tablespace AS ts ON c.reltablespace = ts.oid +WHERE + c.relkind = 'm' + AND n.nspname NOT IN ('pg_catalog', 'information_schema') + AND n.nspname !~ '^pg_toast' + AND n.nspname !~ '^pg_temp' + AND NOT EXISTS ( + SELECT depend.objid + FROM pg_catalog.pg_depend AS depend + WHERE + depend.classid = 'pg_class'::REGCLASS + AND depend.objid = c.oid + AND depend.deptype = 'e' + ); diff --git a/internal/queries/queries.sql.go b/internal/queries/queries.sql.go index d0cad3a..cdbb8ac 100644 --- a/internal/queries/queries.sql.go +++ b/internal/queries/queries.sql.go @@ -560,6 +560,118 @@ func (q *Queries) GetIndexes(ctx context.Context) ([]GetIndexesRow, error) { return items, nil } +const getMaterializedViews = `-- name: GetMaterializedViews :many +SELECT + n.nspname::TEXT AS schema_name, + c.relname::TEXT AS view_name, + c.reloptions::TEXT [] AS rel_options, + COALESCE(ts.spcname, '')::TEXT AS tablespace_name, + (SELECT + ARRAY_AGG(DISTINCT JSONB_BUILD_OBJECT( + 'schema', dep_ns.nspname, + 'name', dep_c.relname, + 'columns', ( + SELECT + ARRAY_AGG( + a.attname::TEXT + ORDER BY a.attnum + ) + FROM pg_catalog.pg_attribute AS a + WHERE + a.attrelid = dep_c.oid + AND a.attnum > 0 + AND NOT a.attisdropped + AND a.attnum IN ( + -- Get only columns that the materialized view depends + -- on. + SELECT DISTINCT d3.refobjsubid + FROM pg_catalog.pg_depend AS d3 + WHERE + d3.refobjid = dep_c.oid + AND d3.refobjsubid > 0 + AND d3.classid = 'pg_rewrite'::REGCLASS + AND EXISTS ( + SELECT 1 + FROM pg_catalog.pg_rewrite AS rw + WHERE + rw.oid = d3.objid + AND rw.ev_class = c.oid + ) + ) + ) + )) + FROM pg_catalog.pg_depend AS d + INNER JOIN pg_catalog.pg_rewrite AS r ON d.objid = r.oid + INNER JOIN pg_catalog.pg_depend AS d2 ON r.oid = d2.objid + INNER JOIN + pg_catalog.pg_class AS dep_c + ON d2.refobjid = dep_c.oid AND dep_c.relkind IN ('r', 'p') + INNER JOIN + pg_catalog.pg_namespace AS dep_ns + ON dep_c.relnamespace = dep_ns.oid + -- Cast to text because pgv4/pq does not support unmarshalling JSON + -- arrays into []json.RawMessage. + -- Instead, they must be unmarshalled as string arrays. + -- https://github.com/lib/pq/pull/466 + WHERE d.refobjid = c.oid)::TEXT [] AS table_dependencies, + PG_GET_VIEWDEF(c.oid, true) AS view_definition +FROM pg_catalog.pg_class AS c +INNER JOIN pg_catalog.pg_namespace AS n ON c.relnamespace = n.oid +LEFT JOIN pg_catalog.pg_tablespace AS ts ON c.reltablespace = ts.oid +WHERE + c.relkind = 'm' + AND n.nspname NOT IN ('pg_catalog', 'information_schema') + AND n.nspname !~ '^pg_toast' + AND n.nspname !~ '^pg_temp' + AND NOT EXISTS ( + SELECT depend.objid + FROM pg_catalog.pg_depend AS depend + WHERE + depend.classid = 'pg_class'::REGCLASS + AND depend.objid = c.oid + AND depend.deptype = 'e' + ) +` + +type GetMaterializedViewsRow struct { + SchemaName string + ViewName string + RelOptions []string + TablespaceName string + TableDependencies []string + ViewDefinition string +} + +func (q *Queries) GetMaterializedViews(ctx context.Context) ([]GetMaterializedViewsRow, error) { + rows, err := q.db.QueryContext(ctx, getMaterializedViews) + if err != nil { + return nil, err + } + defer rows.Close() + var items []GetMaterializedViewsRow + for rows.Next() { + var i GetMaterializedViewsRow + if err := rows.Scan( + &i.SchemaName, + &i.ViewName, + pq.Array(&i.RelOptions), + &i.TablespaceName, + pq.Array(&i.TableDependencies), + &i.ViewDefinition, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getPolicies = `-- name: GetPolicies :many WITH roles AS ( SELECT diff --git a/internal/schema/schema.go b/internal/schema/schema.go index 9222431..3f8b294 100644 --- a/internal/schema/schema.go +++ b/internal/schema/schema.go @@ -62,6 +62,7 @@ type Schema struct { Procedures []Procedure Triggers []Trigger Views []View + MaterializedViews []MaterializedView } // Normalize normalizes the schema (alphabetically sorts tables and columns in tables). @@ -97,6 +98,12 @@ func (s Schema) Normalize() Schema { } s.Views = normViews + var normMaterializedViews []MaterializedView + for _, mv := range sortSchemaObjectsByName(s.MaterializedViews) { + normMaterializedViews = append(normMaterializedViews, normalizeMaterializedView(mv)) + } + s.MaterializedViews = normMaterializedViews + return s } @@ -137,6 +144,16 @@ func normalizeView(v View) View { return v } +func normalizeMaterializedView(mv MaterializedView) MaterializedView { + var normTableDeps []TableDependency + for _, d := range sortSchemaObjectsByName(mv.TableDependencies) { + d.Columns = sortByKey(d.Columns, func(s string) string { return s }) + normTableDeps = append(normTableDeps, d) + } + mv.TableDependencies = normTableDeps + return mv +} + // sortSchemaObjectsByName returns a (copied) sorted list of schema objects. func sortSchemaObjectsByName[S Object](vals []S) []S { return sortByKey(vals, func(v S) string { @@ -475,6 +492,19 @@ type View struct { TableDependencies []TableDependency } +type MaterializedView struct { + SchemaQualifiedName + // ViewDefinition is the select query that defines the materialized view. It is derived from pg_get_viewdef. + ViewDefinition string + // Options represents key value map of materialized view options, i.e., pg_class.reloptions. + Options map[string]string + // Tablespace is the tablespace where the materialized view is stored. Empty string means default tablespace. + Tablespace string + + // TableDependencies is a list of tables the materialized view depends on. + TableDependencies []TableDependency +} + type ( GetSchemaOpt func(*getSchemaOptions) ) @@ -689,6 +719,13 @@ func (s *schemaFetcher) getSchema(ctx context.Context) (Schema, error) { return Schema{}, fmt.Errorf("starting views future: %w", err) } + materializedViewsFuture, err := concurrent.SubmitFuture(ctx, goroutineRunner, func() ([]MaterializedView, error) { + return s.fetchMaterializedViews(ctx) + }) + if err != nil { + return Schema{}, fmt.Errorf("starting materialized views future: %w", err) + } + schemas, err := namedSchemasFuture.Get(ctx) if err != nil { return Schema{}, fmt.Errorf("getting named schemas: %w", err) @@ -744,6 +781,11 @@ func (s *schemaFetcher) getSchema(ctx context.Context) (Schema, error) { return Schema{}, fmt.Errorf("getting views: %w", err) } + materializedViews, err := materializedViewsFuture.Get(ctx) + if err != nil { + return Schema{}, fmt.Errorf("getting materialized views: %w", err) + } + return Schema{ NamedSchemas: schemas, Extensions: extensions, @@ -756,6 +798,7 @@ func (s *schemaFetcher) getSchema(ctx context.Context) (Schema, error) { Procedures: procedures, Triggers: triggers, Views: views, + MaterializedViews: materializedViews, }, nil } @@ -1384,6 +1427,45 @@ func (s *schemaFetcher) fetchViews(ctx context.Context) ([]View, error) { return views, nil } +func (s *schemaFetcher) fetchMaterializedViews(ctx context.Context) ([]MaterializedView, error) { + rawMaterializedViews, err := s.q.GetMaterializedViews(ctx) + if err != nil { + return nil, fmt.Errorf("GetMaterializedViews: %w", err) + } + + var materializedViews []MaterializedView + for _, mv := range rawMaterializedViews { + options, err := relOptionsToMap(mv.RelOptions) + if err != nil { + return nil, fmt.Errorf("materialized view (%q): %w", mv.ViewName, err) + } + + tableDependencies, err := parseJSONTableDependencies(mv.TableDependencies) + if err != nil { + return nil, fmt.Errorf("parsing schema qualified names JSON: %w", err) + } + + materializedViews = append(materializedViews, MaterializedView{ + SchemaQualifiedName: buildNameFromUnescaped(mv.ViewName, mv.SchemaName), + ViewDefinition: mv.ViewDefinition, + Options: options, + Tablespace: mv.TablespaceName, + + TableDependencies: tableDependencies, + }) + } + + materializedViews = filterSliceByName( + materializedViews, + func(materializedView MaterializedView) SchemaQualifiedName { + return materializedView.SchemaQualifiedName + }, + s.nameFilter, + ) + + return materializedViews, nil +} + // parseViewJSONTableDependencies takes an slice of JSON values with schema, // `schema: string; table: string, columns: []string` and unmarshals them into a go struct. func parseJSONTableDependencies(vals []string) ([]TableDependency, error) { diff --git a/internal/schema/schema_test.go b/internal/schema/schema_test.go index 15fd279..d1dcaa3 100644 --- a/internal/schema/schema_test.go +++ b/internal/schema/schema_test.go @@ -229,7 +229,7 @@ var ( SELECT id, author FROM schema_2.foo; `}, - expectedHash: "7963a034855b09c4", + expectedHash: "f0fb3f95f68ba482", expectedSchema: Schema{ NamedSchemas: []NamedSchema{ {Name: "public"}, @@ -571,7 +571,7 @@ var ( ALTER TABLE foo_fk_1 ADD CONSTRAINT foo_fk_1_fk FOREIGN KEY (author, content) REFERENCES foo_1 (author, content) NOT VALID; `}, - expectedHash: "f9c9df4070681684", + expectedHash: "bcad7c978a081c30", expectedSchema: Schema{ NamedSchemas: []NamedSchema{ {Name: "public"}, @@ -1135,7 +1135,7 @@ var ( CREATE TYPE pg_temp.color AS ENUM ('red', 'green', 'blue'); `}, // Assert empty schema hash, since we want to validate specifically that this hash is deterministic - expectedHash: "564ae7220a4cd0ca", + expectedHash: "9c413c6ad2f4a042", expectedSchema: Schema{ NamedSchemas: []NamedSchema{ {Name: "public"}, diff --git a/pkg/diff/materialized_view_sql_generator.go b/pkg/diff/materialized_view_sql_generator.go new file mode 100644 index 0000000..5c1c21a --- /dev/null +++ b/pkg/diff/materialized_view_sql_generator.go @@ -0,0 +1,159 @@ +package diff + +import ( + "errors" + "fmt" + "sort" + "strings" + + "github.com/google/go-cmp/cmp" + "github.com/stripe/pg-schema-diff/internal/schema" + "github.com/stripe/pg-schema-diff/internal/util" +) + +type materializedViewDiff struct { + oldAndNew[schema.MaterializedView] +} + +func buildMaterializedViewDiff( + deletedTablesByName map[string]schema.Table, + tableDiffsByName map[string]tableDiff, + old, new schema.MaterializedView) (materializedViewDiff, bool, error) { + // Assuming the materialized view's outputted columns do not change, there are few situations where the materialized view + // needs to be totally recreated (delete then re-add): + //- One of its dependent columns is deleted then added. As in, a column that it depends on in the old and new is recreated. + //- Same as above but for the table itself. + //- "Outputted" columns of the materialized view change (remove or type altered) + // + // It does not need to be recreated in the following situations: + // - The recreated column/table is only just becoming a dependency: In this case, it can rely on being altered. + // --> A column "foobar" is added to the table, and "foobar" is being added to the materialized view. + // - The recreated column/table is no longer a dependency: In this case, it can rely on being altered. + // --> A column "foobar" is removed to the table, and "foobar" is being removed from the materialized view. + // + // For now, we will go with the simplest behavior and always recreate the materialized view if a dependent column/table, + // and that column/table is deleted/recreated. In part, this is because we cannot depend on individual column + // changes...all added and removes columns are combined into the same SQL vertex. + // - See https://github.com/stripe/pg-schema-diff/issues/135#issuecomment-2357382217 for details. + // - For some table X, it is currently not possible to create a SQL statement outside the table sql genreator + // that comes before a column Y's delete statement but after a column Z's add statement. + for _, t := range old.TableDependencies { + if _, ok := deletedTablesByName[t.GetName()]; ok { + // Recreate if a dependent table was deleted (or recreated). + return materializedViewDiff{}, true, nil + } + // It's possible a dependent column was deleted (or recreated). + td, ok := tableDiffsByName[t.GetName()] + if !ok { + return materializedViewDiff{}, false, fmt.Errorf("processing materialized view table dependencies: expected a table diff to exist for %q. have=\n%s", t.GetName(), util.Keys(tableDiffsByName)) + } + deletedColumnsByName := buildSchemaObjByNameMap(td.columnsDiff.deletes) + for _, c := range t.Columns { + if _, ok := deletedColumnsByName[c]; ok { + // Recreate if a dependent column was deleted (or recreated). + return materializedViewDiff{}, true, nil + } + } + } + + // Recreate if the materialized view SQL generator cannot alter the materialized view. + d := materializedViewDiff{oldAndNew: oldAndNew[schema.MaterializedView]{old: old, new: new}} + if _, err := newMaterializedViewSQLVertexGenerator().Alter(d); err != nil { + if errors.Is(err, ErrNotImplemented) { + // The SQL generator cannot alter the materialized view, so add and delete it. + return materializedViewDiff{}, true, nil + } + return materializedViewDiff{}, false, fmt.Errorf("generating materialized view alter statements: %w", err) + } + return d, false, nil +} + +type materializedViewSQLGenerator struct { +} + +func newMaterializedViewSQLVertexGenerator() sqlVertexGenerator[schema.MaterializedView, materializedViewDiff] { + return &materializedViewSQLGenerator{} +} + +func (mvsg *materializedViewSQLGenerator) Add(mv schema.MaterializedView) (partialSQLGraph, error) { + materializedViewSb := strings.Builder{} + materializedViewSb.WriteString(fmt.Sprintf("CREATE MATERIALIZED VIEW %s", mv.GetFQEscapedName())) + if len(mv.Options) > 0 { + var kvs []string + for k, v := range mv.Options { + kvs = append(kvs, fmt.Sprintf("%s=%s", k, v)) + } + // Sort kvs so the generated DDL is deterministic. This is unnecessarily verbose because the slices + // package is not yet available. + // // TODO(https://github.com/stripe/pg-schema-diff/issues/227) - Remove this + sort.Strings(kvs) + materializedViewSb.WriteString(fmt.Sprintf(" WITH (%s)", strings.Join(kvs, ", "))) + } + if len(mv.Tablespace) > 0 { + materializedViewSb.WriteString(fmt.Sprintf(" TABLESPACE %s", schema.EscapeIdentifier(mv.Tablespace))) + } + materializedViewSb.WriteString(" AS\n") + materializedViewSb.WriteString(mv.ViewDefinition) + + addVertexId := buildMaterializedViewVertexId(mv.SchemaQualifiedName, diffTypeAddAlter) + + var deps []dependency + + // Run after re-create (if recreated). + deps = append(deps, mustRun(addVertexId).after(buildMaterializedViewVertexId(mv.SchemaQualifiedName, diffTypeDelete))) + + // Run after any dependent tables are added/altered. + for _, t := range mv.TableDependencies { + deps = append(deps, mustRun(addVertexId).after(buildTableVertexId(t.SchemaQualifiedName, diffTypeDelete))) + deps = append(deps, mustRun(addVertexId).after(buildTableVertexId(t.SchemaQualifiedName, diffTypeAddAlter))) + } + + return partialSQLGraph{ + vertices: []sqlVertex{{ + id: addVertexId, + priority: sqlPrioritySooner, + statements: []Statement{{ + DDL: materializedViewSb.String(), + Timeout: statementTimeoutDefault, + LockTimeout: lockTimeoutDefault, + }}, + }}, + dependencies: deps, + }, nil +} + +func (mvsg *materializedViewSQLGenerator) Delete(mv schema.MaterializedView) (partialSQLGraph, error) { + deleteVertexId := buildMaterializedViewVertexId(mv.SchemaQualifiedName, diffTypeDelete) + + // Run before any dependent tables are deleted or added/altered. + var deps []dependency + for _, t := range mv.TableDependencies { + deps = append(deps, mustRun(deleteVertexId).before(buildTableVertexId(t.SchemaQualifiedName, diffTypeDelete))) + deps = append(deps, mustRun(deleteVertexId).before(buildTableVertexId(t.SchemaQualifiedName, diffTypeAddAlter))) + } + + return partialSQLGraph{ + vertices: []sqlVertex{{ + id: deleteVertexId, + priority: sqlPriorityLater, + statements: []Statement{{ + DDL: fmt.Sprintf("DROP MATERIALIZED VIEW %s", mv.GetFQEscapedName()), + Timeout: statementTimeoutDefault, + LockTimeout: lockTimeoutDefault, + }}, + }}, + dependencies: deps, + }, nil +} + +func (mvsg *materializedViewSQLGenerator) Alter(mvd materializedViewDiff) (partialSQLGraph, error) { + // In the initial MVP, we will not support altering. + if !cmp.Equal(mvd.old, mvd.new) { + return partialSQLGraph{}, ErrNotImplemented + } + return partialSQLGraph{}, nil +} + +func buildMaterializedViewVertexId(n schema.SchemaQualifiedName, d diffType) sqlVertexId { + return buildSchemaObjVertexId("materialized_view", n.GetFQEscapedName(), d) +} diff --git a/pkg/diff/sql_generator.go b/pkg/diff/sql_generator.go index 35ce7e0..54d8ef1 100644 --- a/pkg/diff/sql_generator.go +++ b/pkg/diff/sql_generator.go @@ -141,6 +141,7 @@ type schemaDiff struct { proceduresDiffs listDiff[schema.Procedure, procedureDiff] triggerDiffs listDiff[schema.Trigger, triggerDiff] viewDiff listDiff[schema.View, viewDiff] + materializedViewDiffs listDiff[schema.MaterializedView, materializedViewDiff] } // The procedure for DIFFING schemas and GENERATING/RESOLVING the SQL required to migrate the old schema to the new schema is @@ -318,6 +319,13 @@ func buildSchemaDiff(old, new schema.Schema) (schemaDiff, bool, error) { return schemaDiff{}, false, fmt.Errorf("diffing views: %w", err) } + materializedViewDiffs, err := diffLists(old.MaterializedViews, new.MaterializedViews, func(old, new schema.MaterializedView, _, _ int) (diff materializedViewDiff, requiresRecreation bool, error error) { + return buildMaterializedViewDiff(deletedTablesByName, tableDiffsByName, old, new) + }) + if err != nil { + return schemaDiff{}, false, fmt.Errorf("diffing materialized views: %w", err) + } + return schemaDiff{ oldAndNew: oldAndNew[schema.Schema]{ old: old, @@ -334,6 +342,7 @@ func buildSchemaDiff(old, new schema.Schema) (schemaDiff, bool, error) { proceduresDiffs: procedureDiffs, triggerDiffs: triggerDiffs, viewDiff: viewDiffs, + materializedViewDiffs: materializedViewDiffs, }, false, nil } @@ -633,6 +642,13 @@ func (s schemaSQLGenerator) Alter(diff schemaDiff) ([]Statement, error) { } partialGraph = concatPartialGraphs(partialGraph, viewPartialGraph) + materializedViewGenerator := newMaterializedViewSQLVertexGenerator() + materializedViewPartialGraph, err := generatePartialGraph(materializedViewGenerator, diff.materializedViewDiffs) + if err != nil { + return nil, fmt.Errorf("resolving materialized view diff: %w", err) + } + partialGraph = concatPartialGraphs(partialGraph, materializedViewPartialGraph) + sqlGraph, err := graphFromPartials(partialGraph) if err != nil { return nil, fmt.Errorf("converting to graph: %w", err)