Merge pull request rails#41856 from MSNexploder/virtual-columns

Add support for generated columns in PostgreSQL (Redux)

Author: matthewd

activerecord/CHANGELOG.md

@@ -1,3 +1,18 @@
+*   Add support for generated columns in PostgreSQL adapter
+
+    Generated columns are supported since version 12.0 of PostgreSQL. This adds
+    support of those to the PostgreSQL adapter.
+
+    ```ruby
+    create_table :users do |t|
+      t.string :name
+      t.virtual :name_upcased, type: :string, as: 'upper(name)', stored: true
+    end
+    ```
+
+    *Michał Begejowicz*
+
+
 ## Rails 7.0.0.alpha2 (September 15, 2021) ##
 
 *   No changes.

activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb

@@ -469,7 +469,7 @@ def default_insert_value(column)
         end
 
         def build_fixture_sql(fixtures, table_name)
-          columns = schema_cache.columns_hash(table_name)
+          columns = schema_cache.columns_hash(table_name).reject { |_, column| supports_virtual_columns? && column.virtual? }
 
           values_list = fixtures.map do |fixture|
             fixture = fixture.stringify_keys

activerecord/lib/active_record/connection_adapters/postgresql/column.rb

@@ -1,20 +1,32 @@
 # frozen_string_literal: true
 
+require "active_support/core_ext/object/blank"
+
 module ActiveRecord
   module ConnectionAdapters
     module PostgreSQL
       class Column < ConnectionAdapters::Column # :nodoc:
         delegate :oid, :fmod, to: :sql_type_metadata
 
-        def initialize(*, serial: nil, **)
+        def initialize(*, serial: nil, generated: nil, **)
           super
           @serial = serial
+          @generated = generated
         end
 
         def serial?
           @serial
         end
 
+        def virtual?
+          # We assume every generated column is virtual, no matter the concrete type
+          @generated.present?
+        end
+
+        def has_default?
+          super && !virtual?
+        end
+
         def array
           sql_type_metadata.sql_type.end_with?("[]")
         end

activerecord/lib/active_record/connection_adapters/postgresql/schema_creation.rb

@@ -61,6 +61,19 @@ def add_column_options!(sql, options)
             if options[:collation]
               sql << " COLLATE \"#{options[:collation]}\""
             end
+
+            if as = options[:as]
+              sql << " GENERATED ALWAYS AS (#{as})"
+
+              if options[:stored]
+                sql << " STORED"
+              else
+                raise ArgumentError, <<~MSG
+                  PostgreSQL currently does not support VIRTUAL (not persisted) generated columns.
+                  Specify 'stored: true' option for '#{options[:column].name}'
+                MSG
+              end
+            end
             super
           end
 

activerecord/lib/active_record/connection_adapters/postgresql/schema_definitions.rb

@@ -191,6 +191,15 @@ def initialize(*, **)
           @unlogged = ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.create_unlogged_tables
         end
 
+        def new_column_definition(name, type, **options) # :nodoc:
+          case type
+          when :virtual
+            type = options[:type]
+          end
+
+          super
+        end
+
         private
           def aliased_types(name, fallback)
             fallback

activerecord/lib/active_record/connection_adapters/postgresql/schema_dumper.rb

@@ -19,6 +19,13 @@ def extensions(stream)
           def prepare_column_options(column)
             spec = super
             spec[:array] = "true" if column.array?
+
+            if @connection.supports_virtual_columns? && column.virtual?
+              spec[:as] = extract_expression_for_virtual_column(column)
+              spec[:stored] = true
+              spec = { type: schema_type(column).inspect }.merge!(spec)
+            end
+
             spec
           end
 
@@ -43,6 +50,10 @@ def schema_type(column)
           def schema_expression(column)
             super unless column.serial?
           end
+
+          def extract_expression_for_virtual_column(column)
+            column.default_function.inspect
+          end
       end
     end
   end

activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb

@@ -654,7 +654,7 @@ def create_alter_table(name)
           end
 
           def new_column_from_field(table_name, field)
-            column_name, type, default, notnull, oid, fmod, collation, comment = field
+            column_name, type, default, notnull, oid, fmod, collation, comment, attgenerated = field
             type_metadata = fetch_type_metadata(column_name, type, oid.to_i, fmod.to_i)
             default_value = extract_value_from_default(default)
             default_function = extract_default_function(default_value, default)
@@ -671,7 +671,8 @@ def new_column_from_field(table_name, field)
               default_function,
               collation: collation,
               comment: comment.presence,
-              serial: serial
+              serial: serial,
+              generated: attgenerated
             )
           end
 

activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb

@@ -181,7 +181,7 @@ def supports_index_sort_order?
       end
 
       def supports_partitioned_indexes?
-        database_version >= 110_000
+        database_version >= 110_000 # >= 11.0
       end
 
       def supports_partial_index?
@@ -233,12 +233,16 @@ def supports_insert_returning?
       end
 
       def supports_insert_on_conflict?
-        database_version >= 90500
+        database_version >= 90500 # >= 9.5
       end
       alias supports_insert_on_duplicate_skip? supports_insert_on_conflict?
       alias supports_insert_on_duplicate_update? supports_insert_on_conflict?
       alias supports_insert_conflict_target? supports_insert_on_conflict?
 
+      def supports_virtual_columns?
+        database_version >= 120_000 # >= 12.0
+      end
+
       def index_algorithms
         { concurrently: "CONCURRENTLY" }
       end
@@ -388,7 +392,7 @@ def supports_foreign_tables?
       end
 
       def supports_pgcrypto_uuid?
-        database_version >= 90400
+        database_version >= 90400 # >= 9.4
       end
 
       def supports_optimizer_hints?
@@ -489,7 +493,7 @@ def build_insert_sql(insert) # :nodoc:
       end
 
       def check_version # :nodoc:
-        if database_version < 90300
+        if database_version < 90300 # < 9.3
           raise "Your version of PostgreSQL (#{database_version}) is too old. Active Record supports PostgreSQL >= 9.3."
         end
       end
@@ -874,7 +878,8 @@ def column_definitions(table_name)
           query(<<~SQL, "SCHEMA")
               SELECT a.attname, format_type(a.atttypid, a.atttypmod),
                      pg_get_expr(d.adbin, d.adrelid), a.attnotnull, a.atttypid, a.atttypmod,
-                     c.collname, col_description(a.attrelid, a.attnum) AS comment
+                     c.collname, col_description(a.attrelid, a.attnum) AS comment,
+                     #{supports_virtual_columns? ? 'attgenerated' : quote('')} as attgenerated
                 FROM pg_attribute a
                 LEFT JOIN pg_attrdef d ON a.attrelid = d.adrelid AND a.attnum = d.adnum
                 LEFT JOIN pg_type t ON a.atttypid = t.oid

activerecord/test/cases/adapters/postgresql/virtual_column_test.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "cases/helper"
+require "support/schema_dumping_helper"
+
+if ActiveRecord::Base.connection.supports_virtual_columns?
+  class PostgresqlVirtualColumnTest < ActiveRecord::PostgreSQLTestCase
+    include SchemaDumpingHelper
+
+    self.use_transactional_tests = false
+
+    class VirtualColumn < ActiveRecord::Base
+    end
+
+    def setup
+      @connection = ActiveRecord::Base.connection
+      @connection.create_table :virtual_columns, force: true do |t|
+        t.string  :name
+        t.virtual :upper_name,  type: :string,  as: "UPPER(name)", stored: true
+        t.virtual :name_length, type: :integer, as: "LENGTH(name)", stored: true
+        t.virtual :name_octet_length, type: :integer, as: "OCTET_LENGTH(name)", stored: true
+      end
+      VirtualColumn.create(name: "Rails")
+    end
+
+    def teardown
+      @connection.drop_table :virtual_columns, if_exists: true
+      VirtualColumn.reset_column_information
+    end
+
+    def test_virtual_column
+      column = VirtualColumn.columns_hash["upper_name"]
+      assert_predicate column, :virtual?
+      assert_equal "RAILS", VirtualColumn.take.upper_name
+    end
+
+    def test_stored_column
+      column = VirtualColumn.columns_hash["name_length"]
+      assert_predicate column, :virtual?
+      assert_equal 5, VirtualColumn.take.name_length
+    end
+
+    def test_change_table
+      @connection.change_table :virtual_columns do |t|
+        t.virtual :lower_name, type: :string, as: "LOWER(name)", stored: true
+      end
+      VirtualColumn.reset_column_information
+      column = VirtualColumn.columns_hash["lower_name"]
+      assert_predicate column, :virtual?
+      assert_equal "rails", VirtualColumn.take.lower_name
+    end
+
+    def test_non_persisted_column
+      message = <<~MSG
+        PostgreSQL currently does not support VIRTUAL (not persisted) generated columns.
+        Specify 'stored: true' option for 'invalid_definition'
+      MSG
+
+      assert_raise ArgumentError, message do
+        @connection.change_table :virtual_columns do |t|
+          t.virtual :invalid_definition, type: :string, as: "LOWER(name)"
+        end
+      end
+    end
+
+    def test_schema_dumping
+      output = dump_table_schema("virtual_columns")
+      assert_match(/t\.virtual\s+"upper_name",\s+type: :string,\s+as: "upper\(\(name\)::text\)", stored: true$/i, output)
+      assert_match(/t\.virtual\s+"name_length",\s+type: :integer,\s+as: "length\(\(name\)::text\)", stored: true$/i, output)
+      assert_match(/t\.virtual\s+"name_octet_length",\s+type: :integer,\s+as: "octet_length\(\(name\)::text\)", stored: true$/i, output)
+    end
+
+    def test_build_fixture_sql
+      ActiveRecord::FixtureSet.create_fixtures(FIXTURES_ROOT, :virtual_columns)
+    end
+  end
+end

activerecord/test/fixtures/virtual_columns.yml

@@ -0,0 +1,5 @@
+one:
+  name: hello
+
+two:
+  name: world