Automatically create additional databases when specified

- when a decomposed database, such as the ci database, is specified it
  will be created when on the same node as the primary database
- external databases will not have decomposed databases created
  automatically

Closes https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6638

Changelog: added

Signed-off-by: Balasankar "Balu" C <[email protected]>

Author: balasankarc

files/gitlab-cookbooks/gitlab/libraries/rails_migration_helper.rb

@@ -41,6 +41,11 @@ def connection_digest
       db_socket
     ).collect { |attribute| attributes_node[attribute] }
 
+    # To trigger a db:migrate run when new databases are introduced, so that
+    # those schema is populated in them.
+    database_names = attributes_node['databases']&.select { |_, details| details['enable'] }&.keys
+    connection_attributes << database_names if database_names
+
     Digest::MD5.hexdigest(Marshal.dump(connection_attributes))
   end
 end

files/gitlab-cookbooks/gitlab/resources/database_objects.rb

@@ -13,11 +13,18 @@
   pg_host = node['postgresql']['unix_socket_directory']
   pg_port = node['postgresql']['port']
   database_name = node['gitlab']['gitlab_rails']['db_database']
+  database_host = node['gitlab']['gitlab_rails']['db_host']
   gitlab_sql_user = node['postgresql']['sql_user']
   gitlab_sql_user_password = node['postgresql']['sql_user_password']
   sql_replication_user = node['postgresql']['sql_replication_user']
   sql_replication_password = node['postgresql']['sql_replication_password']
 
+  # Geo Database will be handled separately in gitlab-ee::geo-postgresql
+  # recipe.
+  ignored_databases = %w[geo]
+  # We only create databases that are configured on the same DB host as the main database
+  databases = node['gitlab']['gitlab_rails']['databases'].select { |db, details| details['enable'] && details['db_host'] == database_host && !ignored_databases.include?(db) }
+
   postgresql_user gitlab_sql_user do
     password "md5#{gitlab_sql_user_password}" unless gitlab_sql_user_password.nil?
     action :create
@@ -29,23 +36,27 @@
     action :create
   end
 
-  postgresql_database database_name do
-    database_port pg_port
-    database_socket pg_host
-    owner gitlab_sql_user
-    user postgresql_username
-    helper new_resource.pg_helper
+  databases.each do |_, settings|
+    database_name = settings['db_database']
 
-    only_if { rails_enabled }
-  end
+    postgresql_database database_name do
+      database_port pg_port
+      database_socket pg_host
+      owner gitlab_sql_user
+      user postgresql_username
+      helper new_resource.pg_helper
 
-  postgresql_extension 'pg_trgm' do
-    database database_name
-    action :enable
-  end
+      only_if { rails_enabled }
+    end
+
+    postgresql_extension 'pg_trgm' do
+      database database_name
+      action :enable
+    end
 
-  postgresql_extension 'btree_gist' do
-    database database_name
-    action :enable
+    postgresql_extension 'btree_gist' do
+      database database_name
+      action :enable
+    end
   end
 end

spec/chef/cookbooks/postgresql/resources/database_objects_spec.rb

@@ -0,0 +1,86 @@
+require 'chef_helper'
+
+RSpec.describe 'database_objects' do
+  let(:chef_run) { ChefSpec::SoloRunner.new(step_into: %w(database_objects postgresql_extension)).converge('gitlab::config', 'test_postgresql::postgresql_database_objects') }
+
+  before do
+    allow(Gitlab).to receive(:[]).and_call_original
+  end
+
+  describe 'create' do
+    context 'by default' do
+      it 'creates main database' do
+        expect(chef_run).to create_postgresql_database('gitlabhq_production')
+      end
+    end
+
+    context 'when additional databases are specified' do
+      context 'on same host as that of main database' do
+        before do
+          stub_gitlab_rb(
+            gitlab_rails: {
+              databases: {
+                ci: {
+                  enable: true,
+                  db_database: 'gitlabhq_production_ci'
+                }
+              }
+            }
+          )
+        end
+
+        it 'creates specified databases in addition to main database' do
+          %w[gitlabhq_production_ci gitlabhq_production].each do |db|
+            expect(chef_run).to create_postgresql_database(db)
+          end
+        end
+      end
+
+      context 'on a different host than that of main database' do
+        before do
+          stub_gitlab_rb(
+            gitlab_rails: {
+              databases: {
+                ci: {
+                  enable: true,
+                  db_database: 'gitlabhq_production_ci',
+                  db_host: 'different.db.host'
+                }
+              }
+            }
+          )
+        end
+
+        it 'creates only main database and not CI database' do
+          expect(chef_run).to create_postgresql_database('gitlabhq_production')
+          expect(chef_run).not_to create_postgresql_database('gitlabhq_production_ci')
+        end
+      end
+    end
+
+    context 'when Geo database is specified' do
+      before do
+        stub_gitlab_rb(
+          gitlab_rails: {
+            databases: {
+              ci: {
+                enable: true,
+                db_database: 'gitlabhq_production_ci'
+              }
+            }
+          },
+          geo_secondary_role: {
+            enable: true
+          }
+        )
+      end
+
+      it 'creates specified databases in addition to main database except geo' do
+        expect(chef_run).to create_postgresql_database('gitlabhq_production')
+        expect(chef_run).to create_postgresql_database('gitlabhq_production_ci')
+
+        expect(chef_run).not_to create_postgresql_database('gitlabhq_geo_production')
+      end
+    end
+  end
+end

spec/chef/fixtures/cookbooks/test_postgresql/recipes/postgresql_database_objects.rb

@@ -0,0 +1,4 @@
+database_objects 'postgresql' do
+  pg_helper PgHelper.new(node)
+  account_helper AccountHelper.new(node)
+end