Fixed #703: restart using rest api, iterate hosts to verify

Author: grtjn

deploy/lib/MLClient.rb

@@ -88,13 +88,15 @@ def build_request_params(url, verb)
     }
   end
 
-  def go(url, verb, headers = {}, params = nil, body = nil, xcc = false)
+  def go(url, verb, headers = nil, params = nil, body = nil, xcc = false)
     logger.debug(%Q{[#{verb.upcase}]\t#{url}})
     password_prompt
     request_params = build_request_params(url, verb)
     # configure headers
-    headers.each do |k, v|
-      request_params[:request][k] = v
+    if headers
+      headers.each do |k, v|
+        request_params[:request][k] = v
+      end
     end
 
     raise ExitException.new("Don't combine params and body. One or the other please") if (params && body)

deploy/lib/server_config.rb

@@ -533,76 +533,106 @@ def execute_query(query, properties = {})
     return r
   end
 
-  def restart_basic
-    @ml_username = @properties['ml.bootstrap-user'] || @properties['ml.user']
-    if @ml_username == @properties['ml.bootstrap-user']
-      @ml_password = @properties['ml.bootstrap-password']
-    else
-      @ml_password = @properties['ml.password']
-    end
-
-    group = nil
-    ARGV.each do |arg|
-      # Exclude any argument passed from command line.
-      if ! arg.match("^-")
-        group = arg
-      end
+  def restart_group(group = nil)
+    if ! group
+      # Note:
+      # Restarting partial cluster is unsafe when working with multiple groups.
+      # Therefor restart entire cluster by default..
+      group = "cluster"
     end
 
-    if group && group == "cluster"
+    if group == "cluster"
       logger.info "Restarting MarkLogic Server cluster of #{@hostname}"
-    elsif group
-      logger.info "Restarting MarkLogic Server group #{group}"
+      r = go(%Q{http://#{@properties["ml.server"]}:#{@properties["ml.bootstrap-port"]}/manage/v2?format=json}, "post", {
+        'Content-Type' => 'application/json'
+      }, nil, %Q{
+        { "operation": "restart-local-cluster" }
+      })
     else
-      # restarting partial cluster unsafe when working with multiple groups
-      #logger.info "Restarting MarkLogic Server group of #{@hostname}"
-      logger.info "Restarting MarkLogic Server cluster of #{@hostname}"
-      group = "cluster"
+      logger.info "Restarting MarkLogic Server group #{group}"
+      r = go(%Q{http://#{@properties["ml.server"]}:#{@properties["ml.bootstrap-port"]}/manage/v2/groups/#{group}?format=json}, "post", {
+        'Content-Type' => 'application/json'
+      }, nil, %Q{
+        { "operation": "restart-group" }
+      })
     end
-    logger.debug "this: #{self}"
-    setup = File.read ServerConfig.expand_path("#{@@path}/lib/xquery/setup.xqy")
-    r = execute_query %Q{#{setup} setup:do-restart("#{group}")}
-    logger.debug "code: #{r.code.to_i}"
 
-    r.body = parse_body(r.body)
-    logger.info r.body
-    return true
+    raise ExitException.new(r.body) unless r.code.to_i == 202
+
+    return JSON.parse(r.body)['restart']['last-startup']
+  end
+
+  def get_host_names
+    r = go(%Q{http://#{@properties["ml.server"]}:8002/manage/v2/hosts?format=json}, "get")
+
+    raise ExitException.new(r.body) unless r.code.to_i == 200
+
+    names = {}
+    JSON.parse(r.body)['host-default-list']['list-items']['list-item'].each do |host|
+      names[host['idref']] = host['nameref']
+    end
+
+    return names
   end
 
-  # implemented verified restart
   def restart
-    verify = find_arg(['--verify'])
-    if verify==='false'
-      restart_basic
+    # Default to verified restart
+    verify = find_arg(['--verify']) != 'false'
+
+    group = next_arg("^[^-]")
+
+    @ml_username = @properties['ml.bootstrap-user'] || @properties['ml.user']
+    if @ml_username == @properties['ml.bootstrap-user']
+      @ml_password = @properties['ml.bootstrap-password']
+    else
+      @ml_password = @properties['ml.password']
+    end
+
+    if ! verify
+      restart_group(group)
     else
-      # defaults to verified restart
-      old_timestamp = go(%Q{http://#{@properties["ml.server"]}:8001/admin/v1/timestamp}, "get").body
-      restart_basic
-      retry_count = 0
-      retry_max = @properties["ml.verify_retry_max"].to_i
-      retry_interval = @properties["ml.verify_retry_interval"].to_i
-      new_timestamp = old_timestamp
-      while retry_count < retry_max do
-        begin
-          new_timestamp = go(%Q{http://#{@properties["ml.server"]}:8001/admin/v1/timestamp}, "get").body
-        rescue
-          logger.info 'Verifying restart ...'
-          logger.debug 'Retry attempt ' + retry_count.to_s + ' failed'
+      old_timestamps = restart_group(group)
+
+      host_names = get_host_names()
+
+      # Iterate until all hosts have restarted (or max is reached)
+      old_timestamps.each do |host|
+        host_name = host_names[host['host-id']]
+        old_timestamp = host['value']
+
+        print "Verifying restart for #{host_name}"
+
+        # Initialize vars for repeated check
+        retry_count = 0
+        retry_max = @properties["ml.verify_retry_max"].to_i
+        retry_interval = [@properties["ml.verify_retry_interval"].to_i, 10].max # 10 sec sleep at least
+        new_timestamp = old_timestamp
+
+        while retry_count < retry_max do
+          begin
+            new_timestamp = go(%Q{http://#{host_name}:8001/admin/v1/timestamp}, "get").body
+          rescue
+            logger.debug 'Retry attempt ' + retry_count.to_s + ' failed'
+          end
+          
+          if new_timestamp != old_timestamp
+            # Indicates that restart is confirmed successful
+            break
+          end
+
+          # Retry..
+          print ".."
+          sleep retry_interval
+          retry_count += 1
         end
-        if new_timestamp != old_timestamp
-          # indicates that restart is confirmed successful
-          break
+
+        if new_timestamp == old_timestamp
+          puts ": FAILED"
+        else
+          puts ": OK"
         end
-        logger.debug "Verifying restart..."
-        sleep retry_interval
-        retry_count += 1
-      end
-      if new_timestamp == old_timestamp
-        logger.warn "Could not verify restart"
-      else
-        logger.info 'Verified restart.'
-        logger.debug "Verified restart new #{new_timestamp} old #{old_timestamp}"
       end
+
     end
   end
 

deploy/lib/util.rb

@@ -36,6 +36,22 @@ def find_arg(args = [])
   nil
 end
 
+def next_arg(match)
+  ARGV.each do |arg|
+    # Exclude any argument passed from command line.
+    if arg.match(match)
+
+      # Remove group from arguments list
+      index = ARGV.index(arg)
+      ARGV.slice!(index)
+
+      # Bail out on first valid arg
+      return arg
+    end
+  end
+  nil
+end
+
 def load_prop_from_args(props)
   ARGV.each do |a|
     if a.match(/(^--)(ml\..*)(=)(.*)/)