Merge pull request #770 from grtjn/703-verified-restart

Fixed #703: restart using rest api, iterate hosts to verify

Author: RobertSzkutak

deploy/lib/Help.rb

@@ -213,11 +213,14 @@ def self.restart
 
       General options:
         -v, [--verbose]  # Verbose output
+        --no-verify      # Skip verifying the restart
+        --legacy         # Don't use Management REST api
 
       Restart the MarkLogic process in the given environment on each host in the
       specified group. If no group is specified, restart the MarkLogic process
       on each host in the group to which the target host belongs. Use 'cluster'
-      to restart all hosts within the cluster to which the target belongs.
+      to restart all hosts within the cluster to which the target belongs. Restart
+      is verified by default, Management REST api is used if available.
     DOC
   end
 

deploy/lib/MLClient.rb

@@ -88,13 +88,15 @@ def build_request_params(url, verb)
     }
   end
 
-  def go(url, verb, headers = {}, params = nil, body = nil, xcc = false)
+  def go(url, verb, headers = nil, params = nil, body = nil, xcc = false)
     logger.debug(%Q{[#{verb.upcase}]\t#{url}})
     password_prompt
     request_params = build_request_params(url, verb)
     # configure headers
-    headers.each do |k, v|
-      request_params[:request][k] = v
+    if headers
+      headers.each do |k, v|
+        request_params[:request][k] = v
+      end
     end
 
     raise ExitException.new("Don't combine params and body. One or the other please") if (params && body)

deploy/lib/server_config.rb

@@ -533,76 +533,141 @@ def execute_query(query, properties = {})
     return r
   end
 
-  def restart_basic
-    @ml_username = @properties['ml.bootstrap-user'] || @properties['ml.user']
-    if @ml_username == @properties['ml.bootstrap-user']
-      @ml_password = @properties['ml.bootstrap-password']
+  def restart_group(group = nil, legacy = false)
+    logger.debug "group: #{group}"
+    logger.debug "legacy: #{legacy}"
+
+    if ! group
+      # Note:
+      # Restarting partial cluster is unsafe when working with multiple groups.
+      # Therefor restart entire cluster by default..
+      group = "cluster"
+    end
+
+    if group == "cluster"
+      logger.info "Restarting MarkLogic Server cluster of #{@hostname}"
     else
-      @ml_password = @properties['ml.password']
+      logger.info "Restarting MarkLogic Server group #{group}"
     end
 
-    group = nil
-    ARGV.each do |arg|
-      # Exclude any argument passed from command line.
-      if ! arg.match("^-")
-        group = arg
+    if @server_version > 7 && !legacy
+      # MarkLogic 8+, make use of Management REST api and return details of all involved hosts
+
+      if group == "cluster"
+        r = go(%Q{http://#{@properties["ml.server"]}:#{@properties["ml.bootstrap-port"]}/manage/v2?format=json}, "post", {
+          'Content-Type' => 'application/json'
+        }, nil, %Q{
+          { "operation": "restart-local-cluster" }
+        })
+      else
+        r = go(%Q{http://#{@properties["ml.server"]}:#{@properties["ml.bootstrap-port"]}/manage/v2/groups/#{group}?format=json}, "post", {
+          'Content-Type' => 'application/json'
+        }, nil, %Q{
+          { "operation": "restart-group" }
+        })
       end
-    end
 
-    if group && group == "cluster"
-      logger.info "Restarting MarkLogic Server cluster of #{@hostname}"
-    elsif group
-      logger.info "Restarting MarkLogic Server group #{group}"
+      raise ExitException.new(r.body) unless r.code.to_i == 202
+
+      return JSON.parse(r.body)['restart']['last-startup']
     else
-      # restarting partial cluster unsafe when working with multiple groups
-      #logger.info "Restarting MarkLogic Server group of #{@hostname}"
-      logger.info "Restarting MarkLogic Server cluster of #{@hostname}"
-      group = "cluster"
+      # MarkLogic 7- fallback, restart as before, and only verify restart of bootstrap host
+
+      old_timestamp = go(%Q{http://#{@properties["ml.server"]}:8001/admin/v1/timestamp}, "get").body
+
+      setup = File.read ServerConfig.expand_path("#{@@path}/lib/xquery/setup.xqy")
+      r = execute_query %Q{#{setup} setup:do-restart("#{group}")}
+      logger.debug "code: #{r.code.to_i}"
+
+      r.body = parse_body(r.body)
+      logger.info r.body
+
+      return [{
+        'host-id' => @properties["ml.server"],
+        'value' => old_timestamp
+      }]
     end
-    logger.debug "this: #{self}"
-    setup = File.read ServerConfig.expand_path("#{@@path}/lib/xquery/setup.xqy")
-    r = execute_query %Q{#{setup} setup:do-restart("#{group}")}
-    logger.debug "code: #{r.code.to_i}"
+  end
 
-    r.body = parse_body(r.body)
-    logger.info r.body
-    return true
+  def get_host_names
+    r = go(%Q{http://#{@properties["ml.server"]}:8002/manage/v2/hosts?format=json}, "get")
+
+    raise ExitException.new(r.body) unless r.code.to_i == 200
+
+    names = { @properties["ml.server"] => @properties["ml.server"] } # ml7 fallback
+
+    JSON.parse(r.body)['host-default-list']['list-items']['list-item'].each do |host|
+      names[host['idref']] = host['nameref']
+    end
+
+    return names
   end
 
-  # implemented verified restart
   def restart
-    verify = find_arg(['--verify'])
-    if verify==='false'
-      restart_basic
+    # Default to verified restart
+    verify = find_arg(['--no-verify']) == nil  && find_arg(['--verify']) != 'false'
+    # Default to using Management Rest api (if available)
+    legacy = find_arg(['--legacy']) != nil
+
+    logger.debug "verify: #{verify}"
+    logger.debug "legacy: #{legacy}"
+
+    group = next_arg("^[^-]")
+
+    @ml_username = @properties['ml.bootstrap-user'] || @properties['ml.user']
+    if @ml_username == @properties['ml.bootstrap-user']
+      @ml_password = @properties['ml.bootstrap-password']
     else
-      # defaults to verified restart
-      old_timestamp = go(%Q{http://#{@properties["ml.server"]}:8001/admin/v1/timestamp}, "get").body
-      restart_basic
-      retry_count = 0
-      retry_max = @properties["ml.verify_retry_max"].to_i
-      retry_interval = @properties["ml.verify_retry_interval"].to_i
-      new_timestamp = old_timestamp
-      while retry_count < retry_max do
-        begin
-          new_timestamp = go(%Q{http://#{@properties["ml.server"]}:8001/admin/v1/timestamp}, "get").body
-        rescue
-          logger.info 'Verifying restart ...'
-          logger.debug 'Retry attempt ' + retry_count.to_s + ' failed'
+      @ml_password = @properties['ml.password']
+    end
+
+    if ! verify
+      restart_group(group, legacy)
+    else
+      host_names = get_host_names()
+
+      old_timestamps = restart_group(group, legacy)
+
+      # Iterate until all hosts have restarted (or max is reached)
+      old_timestamps.each do |host|
+        host_name = host_names[host['host-id']]
+        old_timestamp = host['value']
+
+        print "Verifying restart for #{host_name}"
+
+        # Initialize vars for repeated check
+        retry_count = 0
+        retry_max = @properties["ml.verify_retry_max"].to_i
+        retry_interval = [@properties["ml.verify_retry_interval"].to_i, 10].max # 10 sec sleep at least
+        new_timestamp = old_timestamp
+
+        while retry_count < retry_max do
+          begin
+            new_timestamp = go(%Q{http://#{host_name}:8001/admin/v1/timestamp}, "get").body
+          rescue
+            logger.debug 'Retry attempt ' + retry_count.to_s + ' failed'
+          end
+          
+          if new_timestamp != old_timestamp
+            # Indicates that restart is confirmed successful
+            break
+          end
+
+          # Retry..
+          print ".."
+          sleep retry_interval
+          retry_count += 1
         end
-        if new_timestamp != old_timestamp
-          # indicates that restart is confirmed successful
-          break
+
+        if retry_max < 1
+          puts ": SKIPPED"
+        elsif new_timestamp == old_timestamp
+          puts ": FAILED"
+        else
+          puts ": OK"
         end
-        logger.debug "Verifying restart..."
-        sleep retry_interval
-        retry_count += 1
-      end
-      if new_timestamp == old_timestamp
-        logger.warn "Could not verify restart"
-      else
-        logger.info 'Verified restart.'
-        logger.debug "Verified restart new #{new_timestamp} old #{old_timestamp}"
       end
+
     end
   end
 

deploy/lib/util.rb

@@ -36,6 +36,22 @@ def find_arg(args = [])
   nil
 end
 
+def next_arg(match)
+  ARGV.each do |arg|
+    # Exclude any argument passed from command line.
+    if arg.match(match)
+
+      # Remove group from arguments list
+      index = ARGV.index(arg)
+      ARGV.slice!(index)
+
+      # Bail out on first valid arg
+      return arg
+    end
+  end
+  nil
+end
+
 def load_prop_from_args(props)
   ARGV.each do |a|
     if a.match(/(^--)(ml\..*)(=)(.*)/)