Merge branch '8285-support-external-consul-2' into 'master'

Support external consul binary

Closes #8332

See merge request https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/7278

Merged-by: Balasankar 'Balu' C <[email protected]>
Reviewed-by: Balasankar 'Balu' C <[email protected]>
Co-authored-by: Clemens Beck <[email protected]>
Co-authored-by: Hossein Pursultani <[email protected]>

Author: 3 people

config/software/consul.rb

@@ -16,6 +16,8 @@
 
 name 'consul'
 
+# This version should be keep in sync with consul versions in
+# consul_download.rb and consul_helper.rb.
 version = Gitlab::Version.new('consul', 'v1.16.3')
 default_version version.print(false)
 

deps.yml

@@ -79,6 +79,13 @@ dependencies:
         - filename: config/software/consul.rb
           pattern: "version = Gitlab::Version.new\\('consul', 'v(\\S+)'\\)"
           range: '< 1.17.0'
+        - filename: files/gitlab-ctl-commands-ee/lib/consul_download.rb
+          pattern: "DEFAULT_VERSION = '(\\S+)'"
+          range: '< 1.17.0'
+        - filename: files/gitlab-cookbooks/consul/libraries/consul_helper.rb
+          pattern: "SUPPORTED_MINOR = '(\\S+)'"
+          tag_filter:
+            output_as: '$1.$2'
       # go-crond
       https://github.com/webdevops/go-crond.git:
         replace_in_files:

files/gitlab-config-template/gitlab.rb.template

@@ -3234,6 +3234,7 @@ external_url 'GENERATED_EXTERNAL_URL'
 # Consul (EE only)
 ################################################################################
 # consul['enable'] = false
+# consul['binary_path'] = '/opt/gitlab/embedded/bin/consul'
 # consul['dir'] = '/var/opt/gitlab/consul'
 # consul['username'] = 'gitlab-consul'
 # consul['group'] = 'gitlab-consul'

files/gitlab-cookbooks/consul/attributes/default.rb

@@ -1,4 +1,5 @@
 default['consul']['enable'] = false
+default['consul']['binary_path'] = '/opt/gitlab/embedded/bin/consul'
 default['consul']['dir'] = '/var/opt/gitlab/consul'
 default['consul']['username'] = 'gitlab-consul'
 default['consul']['group'] = 'gitlab-consul'

files/gitlab-cookbooks/consul/libraries/consul_helper.rb

@@ -9,6 +9,10 @@ class ConsulHelper
   # provide an `enable_service_#{service_name}` and `disable_service_#{service_name}` recipe
   SERVICES = %w(postgresql).freeze
 
+  # This version should be keep in sync with consul versions in
+  # software/consul.rb and consul_download.rb.
+  SUPPORTED_MINOR = '1.16'.freeze
+
   def initialize(node)
     @node = node
     @default_configuration = {
@@ -176,7 +180,7 @@ def disabled_services
   def installed_version
     return unless OmnibusHelper.new(@node).service_up?('consul')
 
-    command = '/opt/gitlab/embedded/bin/consul version'
+    command = "#{@node['consul']['binary_path']} version"
     command_output = VersionHelper.version(command)
     raise "Execution of the command `#{command}` failed" unless command_output
 
@@ -195,6 +199,16 @@ def running_version
     info[:Config][:Version] unless info.empty?
   end
 
+  def installed_is_supported?
+    installed = installed_version
+    return true if installed.nil?
+
+    major_installed, minor_installed = installed.split('.')[0..1]
+    major_supported, minor_supported = SUPPORTED_MINOR.split('.')
+
+    major_installed == major_supported && minor_installed == minor_supported
+  end
+
   private
 
   def verify_incoming?

files/gitlab-cookbooks/consul/recipes/enable_daemon.rb

@@ -20,6 +20,7 @@
 
 runit_service 'consul' do
   options({
+            binary_path: node['consul']['binary_path'],
             config_dir: node['consul']['config_dir'],
             custom_config_dir: node['consul']['custom_config_dir'],
             config_file: node['consul']['config_file'],
@@ -57,3 +58,17 @@
   end
   only_if { consul_helper.running_version != consul_helper.installed_version }
 end
+
+ruby_block 'warn consul version mismatch' do
+  block do
+    message = <<~MESSAGE
+      The version of the installed consul service is different than what is expected.
+      Please run `gitlab-ctl consul-download --force` and restart consul to start
+      the new version:
+
+      https://docs.gitlab.com/ee/administration/consul.html#restart-consul
+    MESSAGE
+    LoggingHelper.warning(message)
+  end
+  not_if { consul_helper.installed_is_supported? }
+end

files/gitlab-cookbooks/consul/templates/default/sv-consul-run.erb

@@ -9,7 +9,9 @@ cd <%= @options[:dir] %>
 exec chpst -P -e <%= @options[:env_dir] %> \
 -U <%= @options[:user] %>:<%= @options[:groupname] %> \
 -u <%= @options[:user] %>:<%= @options[:groupname] %> \
-/opt/gitlab/embedded/bin/consul agent -config-file <%= @options[:config_file] %> \
+<%= @options[:binary_path] %> \
+  agent \
+  -config-file <%= @options[:config_file] %> \
   -config-dir <%= @options[:config_dir] %> \
   <% unless @options[:custom_config_dir].nil? -%>
   -config-dir <%= @options[:custom_config_dir] %> \

files/gitlab-ctl-commands-ee/consul.rb

@@ -15,8 +15,13 @@
 #
 
 require "#{base_path}/embedded/service/omnibus-ctl-ee/lib/consul"
+require "#{base_path}/embedded/service/omnibus-ctl-ee/lib/consul_download"
 
 add_command_under_category('consul', 'consul', 'Interact with the gitlab-consul cluster', 2) do
   consul = ConsulHandler.new(ARGV, $stdin.gets)
   consul.execute
 end
+
+add_command_under_category('consul-download', 'consul', 'Download consul for the gitlab-consul cluster', 2) do
+  ConsulDownloadCommand.new(ARGV).run
+end

files/gitlab-ctl-commands-ee/lib/consul.rb

@@ -1,4 +1,10 @@
 require 'mixlib/shellout'
+# For testing purposes, if the first path cannot be found load the second
+begin
+  require_relative '../../omnibus-ctl/lib/gitlab_ctl'
+rescue LoadError
+  require_relative '../../gitlab-ctl-commands/lib/gitlab_ctl'
+end
 
 class ConsulHandler
   WatcherError = Class.new(StandardError)
@@ -17,7 +23,7 @@ def execute
 
   class << self
     def run_consul(cmd)
-      command = Mixlib::ShellOut.new("/opt/gitlab/embedded/bin/consul #{cmd}")
+      command = Mixlib::ShellOut.new("#{consul_binary} #{cmd}")
       command.run_command
       begin
         command.error!
@@ -28,6 +34,12 @@ def run_consul(cmd)
       end
       command.stdout
     end
+
+    private
+
+    def consul_binary
+      GitlabCtl::Util.get_node_attributes.dig('consul', 'binary_path')
+    end
   end
 
   class Kv

files/gitlab-ctl-commands-ee/lib/consul_download.rb

@@ -0,0 +1,84 @@
+require 'net/http'
+require 'zip'
+
+class ConsulDownloadCommand
+  # This version should be keep in sync with consul versions in
+  # software/consul.rb and consul_helper.rb.
+  DEFAULT_VERSION = '1.16.3'.freeze
+
+  def initialize(args)
+    @args = args
+    @options = {
+      version: DEFAULT_VERSION,
+      architecture: default_architecture,
+      output: '/usr/local/bin/consul',
+      force: false,
+    }
+
+    parse_options!
+  end
+
+  def run
+    if !@options[:force] && File.exist?(@options[:output])
+      warn("Consul binary already exists! Use --force to overwrite #{@options[:output]}.")
+      return
+    end
+
+    response = Net::HTTP.get_response(download_url)
+
+    if response.code != '200'
+      warn("Error downloading consul from #{download_url}: #{resp.code}")
+      return
+    end
+
+    Zip::File.open_buffer(StringIO.new(response.body)) do |zipio|
+      zipio.each do |entry|
+        next unless entry.file? && entry.name == 'consul'
+
+        File.write(@options[:output], entry.get_input_stream.read)
+        File.chmod(0755, @options[:output])
+        puts "Successfully downloaded Consul #{@options[:version]} to #{@options[:output]}."
+        puts "Start using the external Consul by setting consul['binary_path'] = '#{@options[:output]}' in your `gitlab.rb` and run `gitlab-ctl reconfigure`"
+      end
+    end
+  end
+
+  private
+
+  def parse_options!
+    opts_parser = OptionParser.new do |opts|
+      opts.banner = 'Usage: gitlab-ctl consul download [options]'
+
+      opts.on('-v', '--version=VERSION', 'Version of the consul binary to be downloaded') do |v|
+        @options[:version] = v
+      end
+
+      opts.on('-a', '--arch=ARCHITECTURE', 'Architecture of the consul binary to be downloaded') do |v|
+        @options[:architecture] = v
+      end
+
+      opts.on('-o', '--output=PATH', 'Path to write the consul binary to') do |v|
+        @options[:output] = v
+      end
+
+      opts.on('-f', '--force', 'Overwrite existing consul binary') do |v|
+        @options[:force] = v
+      end
+
+      opts.on_tail('-h', '--help', 'Show this message') do
+        puts opts
+        exit
+      end
+    end
+
+    opts_parser.parse!(@args.dup)
+  end
+
+  def download_url
+    URI.parse("https://releases.hashicorp.com/consul/#{@options[:version]}/consul_#{@options[:version]}_linux_#{@options[:architecture]}.zip")
+  end
+
+  def default_architecture
+    GitlabCtl::Util.get_node_attributes.dig('kernel', 'machine') || 'amd64'
+  end
+end