Merge branch 'dev' into Issue-665

Author: dmcassel

deploy/default.properties

@@ -265,3 +265,8 @@ http.retry-delay=15
 # Application configuration files
 #
 application-conf-file=src/app/config/config.xqy
+
+# Verified restart config
+#
+verify_retry_max=5
+verify_retry_interval=10

deploy/lib/server_config.rb

@@ -533,7 +533,7 @@ def execute_query(query, properties = {})
     return r
   end
 
-  def restart
+  def restart_basic
     @ml_username = @properties['ml.bootstrap-user'] || @properties['ml.user']
     if @ml_username == @properties['ml.bootstrap-user']
       @ml_password = @properties['ml.bootstrap-password']
@@ -569,6 +569,43 @@ def restart
     return true
   end
 
+  # implemented verified restart
+  def restart
+    verify = find_arg(['--verify'])
+    if verify==='false'
+      restart_basic
+    else
+      # defaults to verified restart
+      old_timestamp = go(%Q{http://#{@properties["ml.server"]}:8001/admin/v1/timestamp}, "get").body
+      restart_basic
+      retry_count = 0
+      retry_max = @properties["ml.verify_retry_max"].to_i
+      retry_interval = @properties["ml.verify_retry_interval"].to_i
+      new_timestamp = old_timestamp
+      while retry_count < retry_max do
+        begin
+          new_timestamp = go(%Q{http://#{@properties["ml.server"]}:8001/admin/v1/timestamp}, "get").body
+        rescue
+          logger.info 'Verifying restart ...'
+          logger.debug 'Retry attempt ' + retry_count.to_s + ' failed'
+        end
+        if new_timestamp != old_timestamp
+          # indicates that restart is confirmed successful
+          break
+        end
+        logger.debug "Verifying restart..."
+        sleep retry_interval
+        retry_count += 1
+      end
+      if new_timestamp == old_timestamp
+        logger.warn "Could not verify restart"
+      else
+        logger.info 'Verified restart.'
+        logger.debug "Verified restart new #{new_timestamp} old #{old_timestamp}"
+      end
+    end
+  end
+
   def merge
     what = ARGV.shift
     raise HelpException.new("merge", "Missing WHAT") unless what

deploy/lib/xquery/setup.xqy

@@ -72,6 +72,11 @@ declare variable $replicating-map-file := "/roxy/status/cleanup/replicating-map.
 declare variable $replicating-map-file-internal := "/roxy/status/cleanup/replicating-map-internal.xml";
 declare variable $replicating-map := map:map();
 
+(: Several functions take an optional invalid-values parameter. Use this as the
+ : default value when it's not provided.
+ :)
+declare variable $default-invalid-values := "reject";
+
 declare variable $group-settings :=
   <settings>
     <setting>list-cache-size</setting>
@@ -1170,9 +1175,9 @@ declare function setup:do-restart($group-name as xs:string?) as item()*
         "Restarting hosts to make configuration changes take effect"),
 
       if ($group-id) then
-        fn:concat("Group ", $group-name, " restarted")
+        fn:concat("Invoked group ", $group-name, " restart")
       else
-        fn:concat("Cluster restarted")
+        fn:concat("Invoked cluster restart")
     )
   }
   catch ($ex)
@@ -2258,7 +2263,7 @@ declare function setup:validated-range-element-indexes(
             $index-config/db:localname/fn:string(.),
             fn:string($index-config/db:collation[../db:scalar-type = 'string']),
             ($index-config/db:range-value-positions/xs:boolean(.), false())[1],
-            ($index-config/db:invalid-values, "reject")[1]
+            ($index-config/db:invalid-values, $default-invalid-values)[1]
           )
         else
           xdmp:apply(
@@ -2320,7 +2325,7 @@ declare function setup:validated-range-element-attribute-indexes(
             $index-config/db:localname/fn:string(.),
             fn:string($index-config/db:collation[../db:scalar-type = 'string']),
             ($index-config/db:range-value-positions/xs:boolean(.), false())[1],
-            ($index-config/db:invalid-values, "reject")[1]
+            ($index-config/db:invalid-values, $default-invalid-values)[1]
           )
         else
           xdmp:apply(
@@ -2449,7 +2454,7 @@ declare function setup:add-range-path-indexes(
                $index/db:path-expression,
                $index/db:collation,
                $index/db:range-value-positions,
-               $index/db:invalid-values
+               ($index/db:invalid-values, $default-invalid-values)[1]
              )
          )"
       )
@@ -2478,16 +2483,18 @@ declare function setup:validate-range-path-indexes(
       declare namespace db="http://marklogic.com/xdmp/database";
       declare variable $database external;
       declare variable $x external;
+      declare variable $default-invalid-values external;
 
       admin:database-range-path-index(
        $database,
        $x/db:scalar-type,
        $x/db:path-expression,
        fn:string($x/db:collation[../db:scalar-type = "string"]),
        $x/db:range-value-positions,
-       $x/db:invalid-values)',
+       ($x/db:invalid-values, $default-invalid-values)[1])',
       (xs:QName("database"), $database,
-       xs:QName("x"), $expected))
+       xs:QName("x"), $expected,
+       xs:QName("default-invalid-values"), $default-invalid-values))
   return
     if ($existing[fn:deep-equal(., $expected)]) then ()
     else
@@ -2689,7 +2696,7 @@ declare function setup:add-range-field-indexes-helper(
             $index/db:field-name,
             ($index/db:collation/fn:string(), "")[1], (: ML6 requires xs:string; later requires xs:string? :)
             $index/db:range-value-positions,
-            $index/db:invalid-values
+            ($index/db:invalid-values, $default-invalid-values)[1]
           )
         else
           admin:database-range-field-index(
@@ -2738,7 +2745,7 @@ declare function setup:add-geospatial-element-indexes(
           $index/db:coordinate-system,
           $index/db:range-value-positions,
           ($index/db:point-format, "point")[1],
-          ($index/db:invalid-values, "ignore")[1]
+          ($index/db:invalid-values, $default-invalid-values)[1]
         )
       else
         admin:database-geospatial-element-index(
@@ -2792,7 +2799,7 @@ declare function setup:add-geospatial-element-attribute-pair-indexes(
           $index/db:longitude-localname,
           $index/db:coordinate-system,
           $index/db:range-value-positions,
-          ($index/db:invalid-values, "ignore")[1]
+          ($index/db:invalid-values, $default-invalid-values)[1]
         )
       else
         admin:database-geospatial-element-attribute-pair-index(
@@ -2849,7 +2856,7 @@ declare function setup:add-geospatial-element-pair-indexes(
           $index/db:longitude-localname,
           $index/db:coordinate-system,
           $index/db:range-value-positions,
-          ($index/db:invalid-values, "ignore")[1]
+          ($index/db:invalid-values, $default-invalid-values)[1]
         )
       else
         admin:database-geospatial-element-pair-index(
@@ -2905,7 +2912,7 @@ declare function setup:add-geospatial-element-child-indexes(
           $index/db:coordinate-system,
           $index/db:range-value-positions,
           ($index/db:point-format, "point")[1],
-          ($index/db:invalid-values, "ignore")[1]
+          ($index/db:invalid-values, $default-invalid-values)[1]
         )
       else
         admin:database-geospatial-element-child-index(
@@ -4742,6 +4749,9 @@ declare function setup:validate-external-security(
 declare function setup:create-roles(
   $import-config as element(configuration))
 {
+  (: get the existing role names from the default security DB :)
+  let $existing-role-names := setup:get-existing-role-names()
+
   (: Create all missing roles :)
   for $role in $import-config/sec:roles/sec:role
   let $role-name as xs:string := $role/sec:role-name
@@ -4754,7 +4764,7 @@ declare function setup:create-roles(
     </options>
   return
     (: if the role exists, then don't create it :)
-    if (setup:get-roles(())/sec:role[sec:role-name = $role-name]) then ()
+    if ($existing-role-names[. = $role-name]) then ()
     else
     (
       xdmp:eval(
@@ -4911,6 +4921,9 @@ declare function setup:create-roles(
 declare function setup:validate-roles(
   $import-config as element(configuration))
 {
+  (: get the existing role names from the default security DB :)
+  let $existing-roles := setup:get-roles(())
+
   for $role in $import-config/sec:roles/sec:role
   let $role-name as xs:string := $role/sec:role-name
   let $description as xs:string? := $role/sec:description
@@ -4921,7 +4934,7 @@ declare function setup:validate-roles(
   let $privileges as element(sec:privilege)* := $role/sec:privileges/sec:privilege
   let $amps as element(sec:amp)* := $role/sec:amps/*
   let $external-names as xs:string* := $role/sec:external-names/sec:external-name
-  let $match := setup:get-roles(())/sec:role[sec:role-name = $role-name]
+  let $match := $existing-roles/sec:role[sec:role-name = $role-name]
   return
     if ($match) then
       if ($match/sec:role-name != $role-name or
@@ -4964,6 +4977,10 @@ declare function setup:associate-users-with-roles($import-config as element(conf
 
 declare function setup:create-users($import-config as element(configuration))
 {
+  (: get the existing user names from the default security DB :)
+  let $existing-user-names := setup:get-existing-user-names()
+
+  (: Create all missing users :)
   for $user in $import-config/sec:users/sec:user
   let $user-name as xs:string := $user/sec:user-name
   let $description as xs:string? := $user/sec:description
@@ -4978,7 +4995,7 @@ declare function setup:create-users($import-config as element(configuration))
       <isolation>different-transaction</isolation>
     </options>
   return
-    if (setup:get-users(())/sec:user[sec:user-name = $user-name]) then
+    if ($existing-user-names[. = $user-name]) then
     (
       xdmp:eval(
         'import module namespace sec="http://marklogic.com/xdmp/security" at "/MarkLogic/security.xqy";
@@ -5088,6 +5105,9 @@ declare function setup:create-users($import-config as element(configuration))
 
 declare function setup:validate-users($import-config as element(configuration))
 {
+  (: get the existing users from the default security DB :)
+  let $existing-users := setup:get-users(())
+
   for $user in $import-config/sec:users/sec:user
   let $user-name as xs:string := $user/sec:user-name
   let $description as xs:string? := $user/sec:description
@@ -5096,7 +5116,7 @@ declare function setup:validate-users($import-config as element(configuration))
   let $permissions as element(sec:permission)* := $user/sec:permissions/*
   let $collections as xs:string* := $user/sec:collections/*
   let $external-names as xs:string* := $user/sec:external-names/sec:external-name
-  let $match := setup:get-users(())/sec:user[sec:user-name = $user-name]
+  let $match := $existing-users/sec:user[sec:user-name = $user-name]
   return
     if ($match) then
       if ($match/sec:description != $description or
@@ -5451,6 +5471,19 @@ declare function setup:get-privilege-by-name($name as xs:string) as element(sec:
     </options>)
 };
 
+(: Gets the user names from the default security database :)
+declare function setup:get-existing-user-names() as element(sec:user-name)* {
+  let $user-names :=
+    xdmp:eval(
+      'import module namespace sec="http://marklogic.com/xdmp/security" at "/MarkLogic/security.xqy";
+       /sec:user',
+       (),
+       <options xmlns="xdmp:eval">
+        <database>{$default-security}</database>
+       </options>)/sec:user-name
+  return $user-names
+};
+
 declare function setup:get-users-by-name($names as xs:string*) as element(sec:users)? {
   let $ids :=
     for $name in $names
@@ -5530,6 +5563,19 @@ declare function setup:get-user-id($user-name as xs:string) as xs:unsignedLong?
      </options>)
 };
 
+(: Gets the role names from the default security database :)
+declare function setup:get-existing-role-names() as element(sec:role-name)* {
+  let $role-names :=
+    xdmp:eval(
+      'import module namespace sec="http://marklogic.com/xdmp/security" at "/MarkLogic/security.xqy";
+        /sec:role',
+      (),
+      <options xmlns="xdmp:eval">
+        <database>{$default-security}</database>
+      </options>)/sec:role-name
+  return $role-names
+};
+
 declare function setup:get-roles-by-name($roles as xs:string*) as element(sec:roles)? {
   let $ids :=
     for $role in $roles

src/app/config/config.xqy

@@ -68,7 +68,7 @@ declare variable $c:ROXY-ROUTES :=
  :
  : ***********************************************
  :)
-declare variable $c:CTRL-EXT := ("@ml.controller-ext", $def:CTRL-EXT)[1];
+declare variable $c:CTRL-EXT := ("@ml.controller-ext", $def:CTRL-EXT)[fn:not(. eq "@" || "ml.controller-ext")][1];
 
 (:
  : ***********************************************

version.txt

@@ -1 +1 @@
-1.7.5
+April-2017-dev