Ensure post upgrade steps are run after geo_pg_upgrade

* Running these steps before geo_pg_upgrade is called breaks the upgrade
  of the geo-postgresql database

Changelog: fixed

Author: ibaum

files/gitlab-cookbooks/gitlab-ee/recipes/geo-postgresql.rb

@@ -29,6 +29,8 @@
 
 geo_pg_helper = GeoPgHelper.new(node)
 
+database_version = geo_pg_helper.database_version || geo_pg_helper.version
+
 directory node['gitlab']['geo_postgresql']['dir'] do
   owner postgresql_username
   mode '0755'
@@ -107,7 +109,8 @@
   options({
     log_directory: logging_settings[:log_directory],
     log_user: logging_settings[:runit_owner],
-    log_group: logging_settings[:runit_group]
+    log_group: logging_settings[:runit_group],
+    database_version: database_version.major
   }.merge(params))
   log_options logging_settings[:options]
 end

files/gitlab-cookbooks/gitlab-ee/templates/default/sv-geo-postgresql-run.erb

@@ -1,4 +1,7 @@
 #!/bin/sh
 exec 2>&1
 <%= render('mount_point_check.erb') %>
-exec chpst -P -U <%= node['postgresql']['username'] %>:<%= node['postgresql']['group'] %> -u <%= node['postgresql']['username'] %>:<%= node['postgresql']['group'] %> /opt/gitlab/embedded/bin/postgres -D <%= File.join(node['gitlab']['geo_postgresql']['dir'], 'data') %>
+exec chpst -P -U <%= node['postgresql']['username'] %>:<%= node['postgresql']['group'] %> \
+   -u <%= node['postgresql']['username'] %>:<%= node['postgresql']['group'] %> \
+   /opt/gitlab/embedded/postgresql/<%= @options[:database_version] %>/bin/postgres \
+   -D <%= File.join(node['gitlab']['geo_postgresql']['dir'], 'data') %>

files/gitlab-cookbooks/postgresql/recipes/bin.rb

@@ -20,12 +20,11 @@
 geo_pg_helper = GeoPgHelper.new(node)
 omnibus_helper = OmnibusHelper.new(node)
 postgresql_install_dir = File.join(node['package']['install-dir'], 'embedded/postgresql')
-
-if Services.enabled?('postgresql')
-  running_db_version = pg_helper.database_version
-elsif Services.enabled?('geo_postgresql')
-  running_db_version = geo_pg_helper.database_version
-end
+running_db_version = if Services.enabled?('geo_postgresql') && !Services.enabled?('postgresql')
+                       geo_pg_helper.database_version
+                     else
+                       pg_helper.database_version
+                     end
 
 db_version = node['postgresql']['version'] || running_db_version
 db_path = Dir.glob("#{postgresql_install_dir}/#{db_version}*").min if db_version

files/gitlab-cookbooks/postgresql/recipes/standalone.rb

@@ -15,7 +15,7 @@
   options({
     log_directory: logging_settings[:log_directory],
     log_user: logging_settings[:runit_owner],
-    log_group: logging_settings[:runit_group],
+    log_group: logging_settings[:runit_group]
   }.merge(params))
   log_options logging_settings[:options]
 end

files/gitlab-ctl-commands/lib/gitlab_ctl/pg_upgrade.rb

@@ -16,6 +16,7 @@ module GitlabCtl
   class PgUpgrade
     include GitlabCtl::Util
     attr_accessor :base_path, :data_path, :tmp_dir, :timeout, :target_version, :initial_version, :psql_command, :port
+
     attr_writer :data_dir, :tmp_data_dir
 
     def initialize(base_path, data_path, target_version, tmp_dir = nil, timeout = nil, psql_command = nil, port = nil)
@@ -26,7 +27,7 @@ def initialize(base_path, data_path, target_version, tmp_dir = nil, timeout = ni
       @target_version = target_version
       @initial_version = fetch_running_version
       @port = port || public_node_attributes['postgresql']['port']
-      @psql_command ||= "gitlab-psql"
+      @psql_command = psql_command || "gitlab-psql"
     end
 
     def data_dir
@@ -136,6 +137,10 @@ def upgrade_artifact_exists?(path)
       !Dir.empty?(path)
     end
 
+    def log(message)
+      $stderr.puts message
+    end
+
     def run_pg_upgrade
       unless GitlabCtl::Util.progress_message('Upgrading the data') do
         begin
@@ -147,17 +152,17 @@ def run_pg_upgrade
             "-B #{target_version_path}/bin "
           )
         rescue GitlabCtl::Errors::ExecutionError => e
-          $stderr.puts "Error upgrading the data to version #{target_version}"
-          $stderr.puts "STDOUT: #{e.stdout}"
-          $stderr.puts "STDERR: #{e.stderr}"
+          log "Error upgrading the data to version #{target_version}"
+          log "STDOUT: #{e.stdout}"
+          log "STDERR: #{e.stderr}"
           false
         rescue Mixlib::ShellOut::CommandTimeout
-          $stderr.puts
-          $stderr.puts "Timed out during the database upgrade.".color(:red)
-          $stderr.puts "To run with more time, remove the temporary directory #{tmp_data_dir}.#{target_version.major},".color(:red)
-          $stderr.puts "then re-run your previous command, adding the --timeout option.".color(:red)
-          $stderr.puts "See the docs for more information: https://docs.gitlab.com/omnibus/settings/database.html#upgrade-packaged-postgresql-server".color(:red)
-          $stderr.puts "Or run gitlab-ctl pg-upgrade --help for usage".color(:red)
+          log
+          log "Timed out during the database upgrade.".color(:red)
+          log "To run with more time, remove the temporary directory #{tmp_data_dir}.#{target_version.major},".color(:red)
+          log "then re-run your previous command, adding the --timeout option.".color(:red)
+          log "See the docs for more information: https://docs.gitlab.com/omnibus/settings/database.html#upgrade-packaged-postgresql-server".color(:red)
+          log "Or run gitlab-ctl pg-upgrade --help for usage".color(:red)
           false
         end
       end

files/gitlab-ctl-commands/pg-upgrade.rb

@@ -366,15 +366,16 @@ def patroni_leader_upgrade
 def patroni_replica_upgrade
   stop_database
   create_links(@db_worker.target_version)
-  common_post_upgrade(false)
+  common_post_upgrade(!@geo_pg_enabled)
+  cleanup_data_dir
   geo_pg_upgrade
 end
 
 def patroni_standby_leader_upgrade
   stop_database
   create_links(@db_worker.target_version)
   remove_patroni_cluster_state
-  common_post_upgrade(false)
+  common_post_upgrade(!@geo_pg_enabled)
   geo_pg_upgrade
 end
 
@@ -469,7 +470,6 @@ def geo_secondary_upgrade(tmp_dir, timeout)
   end
 
   geo_pg_upgrade
-  common_post_upgrade
 end
 
 def geo_pg_upgrade
@@ -493,6 +493,7 @@ def geo_pg_upgrade
   rescue GitlabCtl::Errors::ExecutionError
     die "Error running pg_upgrade on secondary, please check logs"
   end
+  common_post_upgrade
 end
 
 def get_locale_encoding