Fixed #809, #810, #811: pass through, and preserve comments, indent config, and fix conflicting attrs

Author: grtjn

deploy/lib/server_config.rb

@@ -62,7 +62,6 @@ def initialize(options)
 
     @properties = options[:properties]
     @environment = @properties["environment"]
-    @config_file = @properties["ml.config.file"]
 
     @properties["ml.server"] = @properties["ml.#{@environment}-server"] unless @properties["ml.server"]
     if (@properties["ml.server"] == nil)
@@ -771,15 +770,23 @@ def properties_map
 
   def config
     setup = File.read ServerConfig.expand_path("#{@@path}/lib/xquery/setup.xqy")
-    r = execute_query %Q{
+    query = %Q{
       #{setup}
       try {
-        setup:rewrite-config(#{get_config}, #{properties_map})
+        xdmp:quote(
+          setup:rewrite-config(#{get_config}, #{properties_map}, (), fn:true()),
+          <options xmlns="xdmp:quote">
+            <indent>yes</indent>
+            <indent-untyped>yes</indent-untyped>
+          </options>
+        )
       } catch($ex) {
         xdmp:log($ex),
         fn:concat($ex/err:format-string/text(), '&#10;See MarkLogic Server error log for more details.')
       }
     }
+    logger.debug query
+    r = execute_query query
     logger.debug "code: #{r.code.to_i}"
 
     r.body = parse_body(r.body)
@@ -2605,8 +2612,14 @@ def execute_query_8(query, properties = {})
       end
     end
 
+    #logger.debug headers
+    #logger.debug params
+
     r = go "#{@protocol}://#{@hostname}:#{@qconsole_port}/v1/eval", "post", headers, params
 
+    #logger.debug r.code
+    #logger.debug r.body
+
     raise ExitException.new(JSON.pretty_generate(JSON.parse(r.body))) if r.body.match(/\{"error"/)
 
     r
@@ -3028,7 +3041,7 @@ def build_config(config_files)
     config.gsub!("{", "{{")
     config.gsub!("}", "}}")
 
-      configs << config
+      configs << "<file>" + config + "</file>/node()"
     end
 
     %Q{(#{configs.join(", ")})}

deploy/lib/xquery/setup.xqy

@@ -549,17 +549,44 @@ declare function setup:suppress-comments($nodes) {
     default return $node
 };
 
+declare function setup:unique-attributes($attrs) {
+  let $result := map:map()
+  let $_ :=
+    for $attr in $attrs
+    let $attr-name := fn:name($attr)
+    let $existing-attr := map:get($result, $attr-name)
+    return
+      if (fn:exists($existing-attr) and fn:string($existing-attr) != fn:string($attr)) then
+        fn:error(
+          xs:QName("ATTR-CONFLICT"),
+          fn:concat("The config-files contain multiple attributes ", $attr-name, " with different values: " || fn:string($existing-attr) || " vs " || fn:string($attr))
+        )
+      else
+        map:put($result, name($attr), $attr)
+  return map:keys($result) ! map:get($result, .)
+};
+
 (: for backwards-compatibility :)
-declare function setup:rewrite-config($import-configs as element(configuration)+, $properties as map:map) as element(configuration)
+declare function setup:rewrite-config($import-configs as node()+, $properties as map:map) as element(configuration)
 {
   setup:rewrite-config($import-configs, $properties, ())
 };
 
-declare function setup:rewrite-config($import-configs as element(configuration)+, $properties as map:map, $silent as xs:boolean?) as element(configuration)
+declare function setup:rewrite-config($import-configs as node()+, $properties as map:map, $silent as xs:boolean?) as element(configuration)
+{
+  setup:rewrite-config($import-configs, $properties, $silent, ())
+};
+
+declare function setup:rewrite-config($import-configs as node()+, $properties as map:map, $silent as xs:boolean?, $keep-comments as xs:boolean?) as element(configuration)
 {
   let $config :=
     element { fn:node-name($import-configs[1]) } {
-      $import-configs/@*,
+      setup:unique-attributes($import-configs/@*),
+
+      (: capture comments before gr:groups, and its older counterparts :)
+      $import-configs/(
+        gr:groups, gr:http-servers, gr:xdbc-servers, gr:odbc-servers, gr:task-server
+      )/preceding-sibling::node(),
 
       <groups xmlns="http://marklogic.com/xdmp/group">{
         $import-configs/gr:groups/@*,
@@ -578,9 +605,10 @@ declare function setup:rewrite-config($import-configs as element(configuration)+
         where fn:exists($servers | $databases | $group-config)
         return
           <group>
-            { $group-config/@* }
+            { setup:unique-attributes($group-config/@*) }
             <group-name>{$group}</group-name>
             {
+              $group-config/(node() except (gr:group-name, gr:http-servers, gr:xdbc-servers, gr:odbc-servers, gr:task-server)),
               if ($http-servers) then
                 <http-servers>{$http-servers}</http-servers>
               else (),
@@ -592,13 +620,20 @@ declare function setup:rewrite-config($import-configs as element(configuration)+
               else (),
               if ($task-server) then
                 $task-server
-              else (),
-              $group-config/(node() except gr:group-name)
+              else ()
             }
           </group>
       }</groups>,
 
-      $import-configs/(node() except (gr:groups, gr:http-servers, gr:xdbc-servers, gr:odbc-servers, gr:task-server))
+      (: capture anything following gr:groups, and its older counterparts :)
+      $import-configs/(
+        gr:groups, gr:http-servers, gr:xdbc-servers, gr:odbc-servers, gr:task-server
+      )/following-sibling::node(),
+
+      (: other fragments with configuration as root :)
+      $import-configs[fn:empty((
+        gr:groups, gr:http-servers, gr:xdbc-servers, gr:odbc-servers, gr:task-server
+      ))]/node()
     }
 
   (: Check config on group consistency! :)
@@ -611,10 +646,13 @@ declare function setup:rewrite-config($import-configs as element(configuration)+
       return
         fn:error(
           xs:QName("NO_HOSTS_IN_GROUP"),
-          fn:concat("No hosts assigned to group ", $group, ", needed for app servers and forests!"))
+          fn:concat("No hosts assigned to group ", $group, ", needed for app servers and forests!")
+        )
 
   (: all good :)
-  return setup:process-conditionals(setup:suppress-comments($config), $properties)
+  return setup:process-conditionals(
+    if ($keep-comments) then $config else setup:suppress-comments($config)
+  )
 };
 
 
@@ -663,7 +701,7 @@ declare private function setup:parse-options( $options as xs:string ) as map:map
   return $optionsMap
 };
 
-declare function setup:do-setup($import-config as element(configuration)+, $options as xs:string, $properties as map:map) as item()*
+declare function setup:do-setup($import-config as node()+, $options as xs:string, $properties as map:map) as item()*
 {
   let $optionsMap := setup:parse-options( $options )
   let $do-internals := map:contains( $optionsMap, "internals" )
@@ -730,7 +768,7 @@ declare function setup:do-setup($import-config as element(configuration)+, $opti
   }
 };
 
-declare function setup:do-wipe($import-config as element(configuration)+, $options as xs:string, $properties as map:map) as item()*
+declare function setup:do-wipe($import-config as node()+, $options as xs:string, $properties as map:map) as item()*
 {
   let $options := if(fn:empty($options) or $options eq "") then ("all") else fn:tokenize($options, ",")
 
@@ -1161,7 +1199,7 @@ declare function setup:do-wipe($import-config as element(configuration)+, $optio
   Attempt to remove replicas that are to be decommissioned due to scaling out of the cluster.
   Replicas are only removed after their new replacements have gone to sync replication.
 :)
-declare function setup:do-clean-replicas($import-config as element(configuration)+, $options as xs:string, $properties as map:map) as item()*
+declare function setup:do-clean-replicas($import-config as node()+, $options as xs:string, $properties as map:map) as item()*
 {
   let $optionsMap := setup:parse-options( $options )
   let $do-internals := map:contains( $optionsMap, "internals" )