Merge pull request #59 from paxtonhare/5_spec_for_plugins

more changes to the plugin signature

Author: dmcassel

data-hub/src/main/java/com/marklogic/hub/Mlcp.java

@@ -30,23 +30,23 @@ public class Mlcp {
     private static final Logger LOGGER = LoggerFactory.getLogger(Mlcp.class);
 
     private List<MlcpSource> sources = new ArrayList<>();
-    
+
     private String mlcpPath;
-    
+
     private String host;
-    
+
     private String port;
-    
+
     private String user;
-    
+
     private String password;
-    
+
     public Mlcp(String mlcpHome, String host, String port, String user, String password) {
         this.host = host;
         this.port = port;
         this.user = user;
         this.password = password;
-        
+
         // set the mlcp executable path based on OS
         this.mlcpPath = mlcpHome;
         String osName = System.getProperty("os.name");
@@ -57,19 +57,19 @@ public Mlcp(String mlcpHome, String host, String port, String user, String passw
             mlcpPath += "/bin/mlcp.sh";
         }
     }
-    
+
     public void addSourceDirectory(String directoryPath, SourceOptions options) {
         MlcpSource source = new MlcpSource(directoryPath, options);
         sources.add(source);
     }
-    
+
     public void loadContent() {
         for (MlcpSource source : sources) {
             Thread inputThread = null;
             Thread errorThread = null;
             try {
                 List<String> arguments = new ArrayList<>();
-                
+
                 arguments.add(mlcpPath);
                 arguments.add("import");
                 arguments.add("-mode");
@@ -82,20 +82,20 @@ public void loadContent() {
                 arguments.add(user);
                 arguments.add("-password");
                 arguments.add(password);
-                
+
                 // add arguments related to the source
                 List<String> sourceArguments = source.getMlcpArguments();
                 arguments.addAll(sourceArguments);
-                
+
                 ProcessBuilder pb = new ProcessBuilder(arguments.toArray(new String[0]));
                 Process process = pb.start();
-                
+
                 inputThread = IOUtil.createInputStreamSink(process.getInputStream(), LOGGER, LogLevel.DEBUG);
                 errorThread = IOUtil.createInputStreamSink(process.getErrorStream(), LOGGER, LogLevel.ERROR);
-                
+
                 inputThread.start();
                 errorThread.start();
-                
+
                 process.waitFor();
             }
             catch (Exception e) {
@@ -111,24 +111,24 @@ public void loadContent() {
             }
         }
     }
-    
+
     private static class MlcpSource {
         private String sourcePath;
         private SourceOptions sourceOptions;
-        
+
         public MlcpSource(String sourcePath, SourceOptions sourceOptions) {
             this.sourcePath = sourcePath;
             this.sourceOptions = sourceOptions;
         }
-        
+
         public String getSourcePath() {
             return sourcePath;
         }
-        
+
         public List<String> getMlcpArguments() throws IOException {
             File file = new File(sourcePath);
             String canonicalPath = file.getCanonicalPath();
-            
+
             List<String> arguments = new ArrayList<>();
             arguments.add("-input_file_path");
             arguments.add(canonicalPath);
@@ -139,71 +139,70 @@ public List<String> getMlcpArguments() throws IOException {
             else {
                 arguments.add(sourceOptions.getInputFileType());
             }
-            
+
             if (sourceOptions.getInputFilePattern() != null) {
                 arguments.add("-input_file_pattern");
                 arguments.add(sourceOptions.getInputFilePattern());
             }
-            
+
             // by default, cut the source directory path to make URIs shorter
-            String uriReplace = "/" + canonicalPath + ",''";
+            String uriReplace = canonicalPath + ",''";
             uriReplace = uriReplace.replaceAll("\\\\", "/");
-            
+
             arguments.add("-output_uri_replace");
-            arguments.add(uriReplace);
-            
+            arguments.add("\"" + uriReplace + "\"");
+
             arguments.add("-transform_module");
             arguments.add("/com.marklogic.hub/mlcp-flow-transform.xqy");
             arguments.add("-transform_namespace");
             arguments.add("http://marklogic.com/hub-in-a-box/mlcp-flow-transform");
             arguments.add("-transform_param");
             arguments.add("\"" + sourceOptions.getTransformParams() + "\"");
-            
             return arguments;
         }
     }
-    
+
     public static class SourceOptions {
         private String domainName;
         private String flowName;
         private String flowType;
         private String inputFileType;
         private String inputFilePattern;
-        
+
         public SourceOptions(String domainName, String flowName, String flowType) {
             this.domainName = domainName;
             this.flowName = flowName;
             this.flowType = flowType;
         }
-        
+
         public String getDomainName() {
             return domainName;
         }
-        
+
         public String getFlowName() {
             return flowName;
         }
-        
+
         public String getFlowType() {
             return flowType;
         }
-        
+
         public String getInputFileType() {
             return inputFileType;
         }
-        
+
         public void setInputFileType(String inputFileType) {
             this.inputFileType = inputFileType;
         }
-        
+
         public String getInputFilePattern() {
             return inputFilePattern;
         }
-        
+
         public void setInputFilePattern(String inputFilePattern) {
             this.inputFilePattern = inputFilePattern;
         }
-        
+
         protected String getTransformParams() {
             return String.format("<params><domain-name>%s</domain-name><flow-name>%s</flow-name><flow-type>%s</flow-type></params>", domainName, flowName, flowType);
         }

data-hub/src/main/resources/ml-modules/root/com.marklogic.hub/lib/flow-lib.xqy

@@ -387,10 +387,23 @@ declare function flow:run-flow(
   $flow as element(hub:flow),
   $identifier as xs:string,
   $options as map:map) as empty-sequence()
+{
+  flow:run-flow($flow, $identifier, (), $options)
+};
+
+declare function flow:run-plugins(
+  $flow as element(hub:flow),
+  $identifier as xs:string,
+  $content as node()?,
+  $options as map:map
+) as element(envelope:envelope)
 {
   let $content :=
     map:new((
-      map:entry("identifier", $identifier)
+      map:entry("identifier", $identifier),
+      if ($content) then
+        map:entry("content", $content)
+      else ()
     ))
   let $_ :=
     xdmp:invoke-function(function() {
@@ -413,7 +426,17 @@ declare function flow:run-flow(
       map:entry("isolation", "different-transaction"),
       map:entry("transactionMode", "query")
     )))
-  let $envelope := flow:make-envelope($content)
+  return
+    flow:make-envelope($content)
+};
+
+declare function flow:run-flow(
+  $flow as element(hub:flow),
+  $identifier as xs:string,
+  $content as node()?,
+  $options as map:map) as empty-sequence()
+{
+  let $envelope := flow:run-plugins($flow, $identifier, $content, $options)
   let $_ :=
     for $writer in $flow/hub:writer
     return
@@ -452,9 +475,9 @@ declare function flow:make-envelope($map as map:map)
 declare function flow:run-plugin(
   $plugin as element(hub:plugin),
   $identifier as xs:string,
-  $content as element()?,
-  $headers as element()*,
-  $triples as element(sem:triple)*,
+  $content as node()?,
+  $headers as node()*,
+  $triples as sem:triple*,
   $options as map:map)
 {
   let $module-uri := $plugin/@module
@@ -483,7 +506,7 @@ declare function flow:run-plugin(
 declare function flow:run-writer(
   $writer as element(hub:writer),
   $identifier as xs:string,
-  $envelope as element(),
+  $envelope as element(envelope:envelope),
   $options as map:map)
 {
   let $module-uri as xs:string := $writer/@module

data-hub/src/main/resources/ml-modules/root/com.marklogic.hub/mlcp-flow-transform.xqy

@@ -2,31 +2,26 @@ xquery version "1.0-ml";
 
 module namespace mlcpFlow = "http://marklogic.com/hub-in-a-box/mlcp-flow-transform";
 
-import module namespace flowLib = "http://marklogic.com/hub-in-a-box/flow-lib" at "/com.marklogic.hub/lib/flow-lib.xqy";
+import module namespace flow = "http://marklogic.com/hub-in-a-box/flow-lib"
+  at "/com.marklogic.hub/lib/flow-lib.xqy";
 
 declare function mlcpFlow:transform(
   $content as map:map,
   $context as map:map
 ) as map:map*
 {
   let $uri := map:get($content, "uri")
-  let $_ := xdmp:log('mlcp-flow-transform received: ' || $uri)
-  
+
   let $paramNodes := xdmp:unquote(map:get($context, 'transform_param'))/node()/*
   let $paramMap := map:new()
   let $_ := $paramNodes ! map:put($paramMap, fn:local-name(.), ./string())
-  
-  let $_ := xdmp:log($paramMap)
-  
-  let $flow := flowLib:get-flow(map:get($paramMap, 'domain-name'), map:get($paramMap, 'flow-name'), map:get($paramMap, 'flow-type'))
-  let $_ := xdmp:log('Flow:')
-  let $_ := xdmp:log($flow)
-  
-  let $_ := xdmp:log('Running flow with: ' || $uri)
-  
-  let $flowResult := flowLib:run-flow($flow, $uri, $paramMap)
-  let $_ := xdmp:log('Flow Result:')
-  let $_ := xdmp:log($flowResult)
-  
-  return ()
-};
+
+  let $flow := flow:get-flow(
+    map:get($paramMap, 'domain-name'),
+    map:get($paramMap, 'flow-name'),
+    map:get($paramMap, 'flow-type'))
+
+  let $envelope := flow:run-plugins($flow, $uri, map:get($content, "value"), $paramMap)
+  return
+    $content
+};

data-hub/src/main/resources/ml-modules/root/com.marklogic.hub/plugins/raw.xqy

@@ -32,10 +32,10 @@ declare option xdmp:mapping "false";
  :)
 declare function plugin:create-content(
   $id as xs:string,
-  $content as element()?,
-  $headers as element()*,
+  $content as node()?,
+  $headers as node()*,
   $triples as element()*,
-  $options as map:map) as element()?
+  $options as map:map) as node()?
 {
   fn:doc($id)/*
 };

data-hub/src/main/resources/ml-modules/root/com.marklogic.hub/writers/default.xqy

@@ -21,7 +21,7 @@ declare option xdmp:mapping "false";
 
 declare function plugin:write(
   $id as xs:string,
-  $content as element(),
+  $content as node(),
   $context as map:map)
 {
   xdmp:document-insert("/conformed" || $id, $content)

data-hub/src/test/resources/flow-manager-test/my-test-flow-with-all/content/content.xqy

@@ -17,10 +17,10 @@ declare option xdmp:mapping "false";
  :)
 declare function plugin:create-content(
   $id as xs:string,
-  $content as element()?,
-  $headers as element()*,
+  $content as node()?,
+  $headers as node()*,
   $triples as element()*,
-  $options as map:map) as element()?
+  $options as map:map) as node()?
 {
   let $source := fn:doc($id)
   return

data-hub/src/test/resources/flow-manager-test/my-test-flow-with-all/headers/headers.xqy

@@ -17,10 +17,10 @@ declare option xdmp:mapping "false";
  :)
 declare function plugin:create-headers(
   $id as xs:string,
-  $content as element()?,
-  $headers as element()*,
+  $content as node()?,
+  $headers as node()*,
   $triples as element()*,
-  $options as map:map) as element()*
+  $options as map:map) as node()*
 {
   <testing>123</testing>
 };

data-hub/src/test/resources/flow-manager-test/my-test-flow-with-all/triples/triples.xqy

@@ -17,10 +17,10 @@ declare option xdmp:mapping "false";
  :)
 declare function plugin:create-triples(
   $id as xs:string,
-  $content as element()?,
-  $headers as element()*,
+  $content as node()?,
+  $headers as node()*,
   $triples as element()*,
-  $options as map:map) as element(sem:triple)*
+  $options as map:map) as sem:triple*
 {
   document { sem:triple("this-test", "is", "awesome") }/*
 };

data-hub/src/test/resources/flow-manager-test/my-test-flow-with-header/content/content.xqy

@@ -17,10 +17,10 @@ declare option xdmp:mapping "false";
  :)
 declare function plugin:create-content(
   $id as xs:string,
-  $content as element()?,
-  $headers as element()*,
+  $content as node()?,
+  $headers as node()*,
   $triples as element()*,
-  $options as map:map) as element()?
+  $options as map:map) as node()?
 {
   fn:doc($id)/*
 };