fixing traces inside of trace appserver

Author: paxtonhare

marklogic-data-hub/src/main/java/com/marklogic/hub/scaffold/Scaffolding.java

@@ -144,6 +144,8 @@ public List<String> updateLegacyFlows(String fromVersion, String entityName) thr
         Path harmonizeDir = entityDir.resolve("harmonize");
 
 
+        updateLegacyEntity(entityName);
+
         List<String> updatedFlows = new ArrayList<>();
         File[] inputFlows = inputDir.toFile().listFiles((pathname) -> pathname.isDirectory() && !pathname.getName().equals("REST"));
         if (inputFlows != null) {
@@ -166,6 +168,20 @@ public List<String> updateLegacyFlows(String fromVersion, String entityName) thr
         return updatedFlows;
     }
 
+    public void updateLegacyEntity(String entityName) {
+        Path entityDir = entitiesDir.resolve(entityName);
+
+        File[] entityFiles = entityDir.toFile().listFiles((dir, name) -> name.matches("[^.]+\\.entity\\.json"));
+        if (entityFiles.length == 0) {
+            try {
+                String fileContents = getFileContent("scaffolding/Entity.json", entityName);
+                writeToFile(fileContents, entityDir.resolve(entityName + ".entity.json").toFile());
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        }
+    }
+
     public boolean updateLegacyFlow(String fromVersion, String entityName, String flowName, FlowType flowType) throws IOException {
         boolean updated = false;
 

marklogic-data-hub/src/main/resources/ml-modules/root/com.marklogic.hub/lib/trace-lib.xqy

@@ -396,6 +396,8 @@ declare function trace:_walk_json($nodes as node()* ,$o)
               map:put($o, $name, $unquoted)
           else if ($name = "error") then
             map:put($o, $name, $n)
+          else if ($name = ("collectorPlugin", "contentPlugin", "headersPlugin", "triplesPlugin", "writerPlugin")) then
+            ()
           else
             let $_ := trace:_walk_json($n/node(), $oo)
             return
@@ -452,19 +454,48 @@ declare function trace:_walk_json($nodes as node()* ,$o)
         $n
 };
 
-declare function trace:trace-to-json($trace)
+declare function trace:trace-to-json-legacy($trace)
 {
   let $o := json:object()
-  let $walk-me :=
-    let $n := $trace/node()
+  let $_ :=
+    for $n in $trace/node()
+    let $name := fn:string(fn:node-name($n))
+    where fn:not($name = ("collectorPlugin", "contentPlugin", "headersPlugin", "triplesPlugin", "writerPlugin"))
     return
-      if ($n instance of object-node()) then
-        $n/node()
-      else
-        $n
-  let $_ := trace:_walk_json($walk-me, $o)
-  return
-    $o
+      map:put($o, $name, $n/data())
+  let $steps := json:array()
+  let $_build_steps :=
+    for $n in $trace/node()
+    let $name := fn:string(fn:node-name($n))
+    where $name = ("collectorPlugin", "contentPlugin", "headersPlugin", "triplesPlugin", "writerPlugin")
+    return
+      let $step := json:object()
+      let $_ := map:put($step, "label", fn:replace($name, "Plugin", ""))
+      let $_ :=
+        trace:_walk_json($n/node(), $step)
+      return
+        json:array-push($steps, $step)
+  let $_ := map:put($o, "steps", $steps)
+  return $o
+};
+
+
+declare function trace:trace-to-json($trace)
+{
+  if (fn:exists($trace/steps)) then
+    let $o := json:object()
+    let $walk-me :=
+      let $n := $trace/node()
+      return
+        if ($n instance of object-node()) then
+          $n/node()
+        else
+          $n
+    let $_ := trace:_walk_json($walk-me, $o)
+    return
+      $o
+  else
+    trace:trace-to-json-legacy($trace)
 };
 
 declare function trace:trace-to-json-slim($trace)

marklogic-data-hub/src/main/resources/scaffolding/Entity.json

@@ -0,0 +1,14 @@
+{
+  "info" : {
+    "title" : "placeholder",
+    "version" : "0.0.1"
+  },
+  "definitions" : {
+    "placeholder" : {
+      "required" : [ ],
+      "rangeIndex" : [ ],
+      "wordLexicon" : [ ],
+      "properties" : { }
+    }
+  }
+}

marklogic-data-hub/src/test/resources/tracing-test/traces/legacy-json-error-trace.json

@@ -0,0 +1 @@
+{"trace":{"format":"xml", "identifier":"/34324.json", "flowType":"input", "contentPlugin":{"duration":"PT0.021033S", "pluginModuleUri":"/entities/Employee/input/load-acme-tech/content/content.sjs", "input":{"rawContent":{"id":"34324", "firstName":"Robert", "lastName":"Smith", "dateOfBirth":"01/01/1981", "hireDate":"12/19/2005", "salaryHistory":[{"effectiveDate":"12/21/2005", "salary":59832}, {"effectiveDate":"05/14/2009", "salary":60832}]}}, "error":{"code":"error", "name":"boom", "xqueryVersion":"javascript", "message":"error", "formatString":"error (boom): fn.error(xs.QName('boom')); -- ", "retryable":"false", "expr":"fn.error(xs.QName('boom'));", "data":[], "stacks":[{"uri":"/entities/Employee/input/load-acme-tech/content/content.sjs", "line":"11", "column":"5", "operation":"createContent()", "xqueryVersion":"javascript", "variables":{}}, {"uri":"/entities/Employee/input/load-acme-tech/content/content.sjs", "line":"698", "column":"6", "operation":"flow:call-plugin-function(<plugin dest=\"content\" type=\"javascript\" module=\"/entities/Employee...\" .../>, \"/34324.json\", document{object-node{\"id\":text{\"34324\"}, \"firstName\":text{\"Robert\"}, \"lastName\":text{\"Smith\"}, ...}}, (), (), \"input\", fn:true(), map:map(<map:map xmlns:xs=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" .../>))", "xqueryVersion":"javascript", "variables":{}}, {"uri":"/com.marklogic.hub/lib/flow-lib.xqy", "line":"698", "column":"6", "operation":"flow:run-plugin(<plugin dest=\"content\" type=\"javascript\" module=\"/entities/Employee...\" .../>, <data-format xmlns=\"http://marklogic.com/data-hub\">application/json</data-format>, \"/34324.json\", document{object-node{\"id\":text{\"34324\"}, \"firstName\":text{\"Robert\"}, \"lastName\":text{\"Smith\"}, ...}}, (), (), <type xmlns=\"http://marklogic.com/data-hub\">input</type>, fn:true(), map:map(<map:map xmlns:xs=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" .../>))", "xqueryVersion":"1.0-ml", "variables":{"plugin":"<plugin dest=\"content\" type=\"javascript\" module=\"/entities/Employee...\" .../>", "data-format":"\"application/json\"", "identifier":"\"/34324.json\"", "content":"document{object-node{\"id\":text{\"34324\"}, \"firstName\":text{\"Robert\"}, \"lastName\":text{\"Smith\"}, ...}}", "headers":"()", "triples":"()", "flow-type":"\"input\"", "simple":"fn:true()", "options":"map:map(<map:map xmlns:xs=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" .../>)", "module-uri":"\"/entities/Employee/input/load-acme-tech/content/content.sjs\"", "destination":"\"content\"", "trace-input":"json:object(<json:object xmlns:xs=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" .../>)", "before":"xs:dayTimeDuration(\"PT0.215026S\")"}}, {"uri":"/com.marklogic.hub/lib/flow-lib.xqy", "line":"513", "column":"6", "operation":"flow:run-plugins(<flow xmlns=\"http://marklogic.com/data-hub\"><name>load-acme-tech</name>...</flow>, \"/34324.json\", document{object-node{\"id\":text{\"34324\"}, \"firstName\":text{\"Robert\"}, \"lastName\":text{\"Smith\"}, ...}}, map:map(<map:map xmlns:xs=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" .../>))", "xqueryVersion":"1.0-ml", "variables":{"flow":"<flow xmlns=\"http://marklogic.com/data-hub\"><name>load-acme-tech</name>...</flow>", "identifier":"\"/34324.json\"", "content":"map:map(<map:map xmlns:xs=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" .../>)", "options":"map:map(<map:map xmlns:xs=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" .../>)", "data-format":"<data-format xmlns=\"http://marklogic.com/data-hub\">application/json</data-format>", "flow-type":"<type xmlns=\"http://marklogic.com/data-hub\">input</type>", "flow-complexity":"<complexity xmlns=\"http://marklogic.com/data-hub\">simple</complexity>", "_":"()", "plugin":"<plugin dest=\"content\" type=\"javascript\" module=\"/entities/Employee...\" .../>", "destination":"attribute{fn:QName(\"\",\"dest\")}{\"content\"}"}}, {"uri":"/com.marklogic.hub/mlcp-flow-transform.xqy", "line":"44", "column":"23", "operation":"function() as item()*()", "xqueryVersion":"1.0-ml", "variables":{"context":"map:map(<map:map xmlns:xs=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" .../>)", "params":"map:map(<map:map xmlns:xs=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" .../>)", "flow":"<flow xmlns=\"http://marklogic.com/data-hub\"><name>load-acme-tech</name>...</flow>", "_":"()"}}, {"uri":"/com.marklogic.hub/lib/perflog-lib.xqy", "line":"33", "column":"9", "operation":"perf:log(\"mlcp-flow-transform(/34324.json)\", function() as item()*)", "xqueryVersion":"1.0-ml", "variables":{"what":"map:map(<map:map xmlns:xs=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" .../>)", "func":"map:map(<map:map xmlns:xs=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" .../>)"}}, {"uri":"/com.marklogic.hub/mlcp-flow-transform.xqy", "line":"25", "column":"4", "operation":"xdmp:function(fn:QName(\"http://marklogic.com/data-hub/mlcp-flow-transform\",\"transform\"), \"/com.marklogic.hub/mlcp-flow-transform.xqy\")(map:map(<map:map xmlns:xs=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" .../>), map:map(<map:map xmlns:xs=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" .../>))", "xqueryVersion":"1.0-ml", "variables":{"content":"map:map(<map:map xmlns:xs=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" .../>)", "context":"map:map(<map:map xmlns:xs=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" .../>)", "uri":"\"/34324.json\""}}, {"uri":"/MarkLogic/hadoop.xqy", "line":"374", "column":"5", "operation":"transform-insert-item(\"/com.marklogic.hub/mlcp-flow-transform.xqy\", xs:untypedAtomic(\"http://marklogic.com/data-hub/mlcp-flow-transform\"), \"transform\", xs:untypedAtomic(\"entity=Employee,flow=load-acme-tech,flowType=input,jobId=3c490f6...\"), \"/34324.json\", \"{&#10;  &quot;id&quot;: &quot;34324&quot;,&#10;  &quot;firstName...\", json:object(<json:object xmlns:xs=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" .../>), (), ())", "xqueryVersion":"1.0-ml", "variables":{"transform-module":"\"/com.marklogic.hub/mlcp-flow-transform.xqy\"", "transform-namespace":"\"http://marklogic.com/data-hub/mlcp-flow-transform\"", "transform-function":"\"transform\"", "transform-param":"\"entity=Employee,flow=load-acme-tech,flowType=input,jobId=3c490f6...\"", "uri":"\"/34324.json\"", "value":"\"{&#10;  &quot;id&quot;: &quot;34324&quot;,&#10;  &quot;firstName...\"", "options-map":"json:object(<json:object xmlns:xs=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" .../>)", "modules":"()", "modules-root":"()", "value-type":"xs:untypedAtomic(\"xs:string\")", "namespace":"()", "language":"()", "roles-execute":"xs:untypedAtomic(\"\")", "roles-insert":"xs:untypedAtomic(\"\")", "roles-read":"xs:untypedAtomic(\"rest-reader\")", "roles-update":"xs:untypedAtomic(\"rest-writer\")", "roles-node-update":"xs:untypedAtomic(\"\")", "collections":"xs:untypedAtomic(\"Employee,load-acme-tech,input\")", "temporal-collection":"()", "quality":"xs:int(\"0\")", "xml-repair-level":"()", "permissions":"(<sec:permission xmlns:sec=\"http://marklogic.com/xdmp/security\"><sec:capability>...</sec:capability>...</sec:permission>, <sec:permission xmlns:sec=\"http://marklogic.com/xdmp/security\"><sec:capability>...</sec:capability>...</sec:permission>)", "context":"map:map(<map:map xmlns:xs=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" .../>)", "dummy":"()", "newVal":"document{object-node{\"id\":text{\"34324\"}, \"firstName\":text{\"Robert\"}, \"lastName\":text{\"Smith\"}, ...}}", "content":"map:map(<map:map xmlns:xs=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" .../>)"}}, {"uri":"/MarkLogic/hadoop.xqy", "line":"256", "column":"8", "operation":"hadoop:transform-insert-batch(\"/com.marklogic.hub/mlcp-flow-transform.xqy\", \"transform\", json:object(<json:object xmlns:xs=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" .../>), \"/34324.json\", \"{&#10;  &quot;id&quot;: &quot;34324&quot;,&#10;  &quot;firstName...\", json:object(<json:object xmlns:xs=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" .../>))", "xqueryVersion":"1.0-ml", "variables":{"transform-module":"\"/com.marklogic.hub/mlcp-flow-transform.xqy\"", "transform-function":"\"transform\"", "transform-option":"json:object(<json:object xmlns:xs=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" .../>)", "uris":"\"/34324.json\"", "values":"\"{&#10;  &quot;id&quot;: &quot;34324&quot;,&#10;  &quot;firstName...\"", "insert-options":"json:object(<json:object xmlns:xs=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" .../>)", "modules-name":"()", "modules":"()", "modules-root":"()", "transform-namespace":"xs:untypedAtomic(\"http://marklogic.com/data-hub/mlcp-flow-transform\")", "transform-param":"xs:untypedAtomic(\"entity=Employee,flow=load-acme-tech,flowType=input,jobId=3c490f6...\")", "uri":"\"/34324.json\"", "i":"1"}}, {"uri":"/eval", "line":"7", "column":"0", "xqueryVersion":"1.0-ml", "variables":{}}]}}, "hasError":true, "created":"2017-09-12T09:15:01.928285-07:00", "jobId":"3c490f69-288c-41fd-bc95-dec01437ab1a", "traceId":"9091223776754160124"}}

marklogic-data-hub/src/test/resources/tracing-test/traces/legacy-json-trace.json

@@ -0,0 +1 @@
+{"trace":{"headersPlugin":{"duration":"PT0.001007S", "output":{}, "pluginModuleUri":"/entities/Employee/input/load-acme-tech/headers/headers.sjs", "input":{"content":{"id":"32920", "firstName":"Rosanne", "lastName":"Henckle", "dateOfBirth":"05/19/1979", "hireDate":"12/19/2005", "salaryHistory":[{"effectiveDate":"12/23/2005", "salary":63439}, {"effectiveDate":"01/14/2010", "salary":66300}]}}}, "format":"json", "identifier":"/32920.json", "flowType":"input", "contentPlugin":{"duration":"PT0.000632S", "output":{"id":"32920", "firstName":"Rosanne", "lastName":"Henckle", "dateOfBirth":"05/19/1979", "hireDate":"12/19/2005", "salaryHistory":[{"effectiveDate":"12/23/2005", "salary":63439}, {"effectiveDate":"01/14/2010", "salary":66300}]}, "pluginModuleUri":"/entities/Employee/input/load-acme-tech/content/content.sjs", "input":{"rawContent":{"id":"32920", "firstName":"Rosanne", "lastName":"Henckle", "dateOfBirth":"05/19/1979", "hireDate":"12/19/2005", "salaryHistory":[{"effectiveDate":"12/23/2005", "salary":63439}, {"effectiveDate":"01/14/2010", "salary":66300}]}}}, "writerPlugin":{"duration":"PT0S", "output":null, "pluginModuleUri":null, "input":{"triples":[], "headers":{}, "content":{"id":"32920", "firstName":"Rosanne", "lastName":"Henckle", "dateOfBirth":"05/19/1979", "hireDate":"12/19/2005", "salaryHistory":[{"effectiveDate":"12/23/2005", "salary":63439}, {"effectiveDate":"01/14/2010", "salary":66300}]}}}, "hasError":false, "created":"2017-09-12T08:27:01.343158-07:00", "triplesPlugin":{"duration":"PT0.000851S", "output":[], "pluginModuleUri":"/entities/Employee/input/load-acme-tech/triples/triples.sjs", "input":{"content":{"id":"32920", "firstName":"Rosanne", "lastName":"Henckle", "dateOfBirth":"05/19/1979", "hireDate":"12/19/2005", "salaryHistory":[{"effectiveDate":"12/23/2005", "salary":63439}, {"effectiveDate":"01/14/2010", "salary":66300}]}, "headers":{}}}, "jobId":"da0c156f-e3c0-4e3d-80fb-004f61b9365a", "traceId":"9054050403310939961"}}