fixing load binaries example

Author: paxtonhare

examples/load-binaries/README.md

@@ -2,9 +2,9 @@
 This example shows how to load binary documents with the Hub Framework.
 
 # TLDR; How do I run it?
-1. Download the latest quick-start jar from the [releases page](https://github.com/marklogic-community/marklogic-data-hub/releases) into this folder.
+1. Download the [latest quick-start war](https://github.com/marklogic-community/marklogic-data-hub/releases/download/v2.0.0-rc.2/quick-start-2.0.0-rc.2.war) into this folder.
 
-1. Run the quick-start jar `java -jar quick-start.jar`
+1. Run the quick-start war `java -jar quick-start-2.0.0-rc.2.war`
 
 1. Open your web browser to [http://localhost:8080](http://localhost:8080).
 
@@ -16,52 +16,31 @@ This example shows how to load binary documents with the Hub Framework.
 
 1. Install the Hub into MarkLogic (if necessary)
 
-1. Load the sample pdf.
-
-  1. Click on the Guides entity on the left.
-  1. Click on the "LoadAsXml" input flow.
-  1. Click on the "Run Flow" button.
-  1. Browse to the input folder.
-  1. Expand the "General Options" section.
-  1. Add ,\\.pdf,'.xml' to the end of "Output URI Replace". It should look something like: /Users/yourname/data-hub/examples/load-binaries,'','\\.pdf','.xml'
-  1. Change "Document Type" to "binary".
-  1. Scroll down and press the "RUN IMPORT" button.
-
-1. At this point you have loaded the sample data. You can browse the data via [QConsole](http://localhost:8000/qconsole) or by searching the REST endpoint on the Staging Http Server [http://localhost:8010/v1/search](http://localhost:8010/v1/search). *Your port may be different if you changed it during setup*
+## Loading the Sample PDF
+1. Click on the **Entities** Tab at the top.
+1. Click on the Guides entity on the left.
+1. Click on the "LoadAsXml" or "LoadAsJson" input flow.
+1. Browse to the input folder.
+1. Expand the "General Options" section.
+1. Add ,\\.pdf,'.xml' to the end of "Output URI Replace". It should look something like:  
+***nix**  
+`/Users/yourname/data-hub/examples/load-binaries/input,'',\\.pdf,'.xml'`  
+**windows**  
+`/c:/Users/yourname/data-hub/examples/load-binaries/input,'',\\.pdf,'.xml'`  
+1. Change "Document Type" to "binary".
+1. Scroll down and press the "RUN IMPORT" button.
+
+## Browsing the sample data
+1. At this point you have loaded the sample data. You can browse the data by clicking on the **Browse** tab at the top.
 
 # Ingesting Binaries
 
 When you ingest a binary via the Quick Start MLCP process you will want to do a few things.
 
 1. Change the file extension to xml or json via "Output URI Replace".
 
-This is because we will be manually storing the binary an using the ingest for creating XML or JSON metadata.
+This is because we will be manually storing the binary, but returning XML or JSON metadata for MLCP to store into Marklogic.
 
 1. Change the document type to binary via the "Document Type" option. This is necessary because MLCP might think this document is an XML or JSON after we changed the file extension above.
 
-1. Store the binary manually and return the metatada. In your content.xqy put this:
-
-```xquery
-declare function plugin:create-content(
-  $id as xs:string,
-  $raw-content as node()?,
-  $options as map:map) as node()?
-{
-  (: name the binary uri with a pdf extension :)
-  let $binary-uri := fn:replace($id, ".xml", ".pdf")
-
-  (: stash the binary uri in the options map for later:)
-  let $_ := map:put($options, 'binary-uri', $binary-uri)
-
-  (: save the incoming binary as a pdf :)
-  return
-    xdmp:document-insert($binary-uri, $raw-content),
-
-  (: extract the contents of the pdf and return them
-   : as the content for the envelope
-   :)
-  xdmp:document-filter($raw-content)
-};
-```
-
 Now you have loaded the binary manually and returned xhtml content to store in the envelope.

examples/load-binaries/plugins/entities/Guides/input/LoadAsJson/LoadAsJson.properties

@@ -0,0 +1,6 @@
+#
+#Mon Sep 18 10:42:06 EDT 2017
+mainModule=main.sjs
+mainCodeFormat=sjs
+codeFormat=sjs
+dataFormat=json

examples/load-binaries/plugins/entities/Guides/input/LoadAsJson/content.sjs

@@ -0,0 +1,34 @@
+/*
+ * Create Content Plugin
+ *
+ * @param id         - the identifier returned by the collector
+ * @param rawContent - the raw content being loaded.
+ * @param options    - an object containing options. Options are sent from Java
+ *
+ * @return - your content
+ */
+function createContent(id, rawContent, options) {
+  // name the binary uri with a pdf extension
+	var binaryUri = fn.replace(id, ".xml", ".pdf");
+
+  // stash the binary uri in the options map for later
+  options.binaryUri = binaryUri;
+
+  // save the incoming binary as a pdf
+  xdmp.eval('declareUpdate(); xdmp.documentInsert(binaryUri, rawContent)', {
+    binaryUri: binaryUri,
+    rawContent: rawContent
+  },{
+    isolation: 'different-transaction',
+    commit: 'auto'
+  });
+
+  // extract the contents of the pdf and return them
+  // as the content for the envelope
+  return xdmp.documentFilter(rawContent);
+
+}
+
+module.exports = {
+  createContent: createContent
+};

examples/load-binaries/plugins/entities/Guides/input/LoadAsJson/headers.sjs

@@ -0,0 +1,19 @@
+/*
+ * Create Headers Plugin
+ *
+ * @param id       - the identifier returned by the collector
+ * @param content  - the output of your content plugin
+ * @param options  - an object containing options. Options are sent from Java
+ *
+ * @return - an object of headers
+ */
+function createHeaders(id, content, options) {
+  return {
+    binaryUri: options.binaryUri
+  };
+}
+
+module.exports = {
+  createHeaders: createHeaders
+};
+

examples/load-binaries/plugins/entities/Guides/input/LoadAsJson/main.sjs

@@ -0,0 +1,39 @@
+// dhf.xqy exposes helper functions to make your life easier
+// See documentation at:
+// https://github.com/marklogic/marklogic-data-hub/wiki/dhf-lib
+const dhf = require('/com.marklogic.hub/dhf.xqy');
+
+const contentPlugin = require('./content.sjs');
+const headersPlugin = require('./headers.sjs');
+const triplesPlugin = require('./triples.sjs');
+
+/*
+ * Plugin Entry point
+ *
+ * @param id          - the identifier returned by the collector
+ * @param rawContent  - the raw content being loaded
+ * @param options     - a map containing options. Options are sent from Java
+ *
+ */
+function main(id, rawContent, options) {
+  var contentContext = dhf.contentContext(rawContent);
+  var content = dhf.run(contentContext, function() {
+    return contentPlugin.createContent(id, rawContent, options);
+  });
+
+  var headerContext = dhf.headersContext(content);
+  var headers = dhf.run(headerContext, function() {
+    return headersPlugin.createHeaders(id, content, options);
+  });
+
+  var tripleContext = dhf.triplesContext(content, headers);
+  var triples = dhf.run(tripleContext, function() {
+    return triplesPlugin.createTriples(id, content, headers, options);
+  });
+
+  return dhf.makeEnvelope(content, headers, triples, options.dataFormat);
+}
+
+module.exports = {
+  main: main
+};

examples/load-binaries/plugins/entities/Guides/input/LoadAsJson/triples.sjs

@@ -0,0 +1,18 @@
+/*
+ * Create Triples Plugin
+ *
+ * @param id       - the identifier returned by the collector
+ * @param content  - the output of your content plugin
+ * @param headers  - the output of your heaaders plugin
+ * @param options  - an object containing options. Options are sent from Java
+ *
+ * @return - an array of triples
+ */
+function createTriples(id, content, headers, options) {
+  return [];
+}
+
+module.exports = {
+  createTriples: createTriples
+};
+

examples/load-binaries/plugins/entities/Guides/input/LoadAsXml/content/content.xqy

@@ -1,37 +1,49 @@
-xquery version "1.0-ml";
-
-module namespace plugin = "http://marklogic.com/data-hub/plugins";
-
-declare namespace envelope = "http://marklogic.com/data-hub/envelope";
-
-declare option xdmp:mapping "false";
-
-(:~
- : Create Content Plugin
- :
- : @param $id          - the identifier returned by the collector
- : @param $raw-content - the raw content being loaded.
- : @param $options     - a map containing options. Options are sent from Java
- :
- : @return - your transformed content
- :)
-declare function plugin:create-content(
-  $id as xs:string,
-  $raw-content as node()?,
-  $options as map:map) as node()?
-{
-  (: name the binary uri with a pdf extension :)
-  let $binary-uri := fn:replace($id, ".xml", ".pdf")
-
-  (: stash the binary uri in the options map for later:)
-  let $_ := map:put($options, 'binary-uri', $binary-uri)
-
-  (: save the incoming binary as a pdf :)
-  return
-    xdmp:document-insert($binary-uri, $raw-content),
-
-  (: extract the contents of the pdf and return them
-   : as the content for the envelope
-   :)
-  xdmp:document-filter($raw-content)
-};
+xquery version "1.0-ml";
+
+module namespace plugin = "http://marklogic.com/data-hub/plugins";
+
+declare option xdmp:mapping "false";
+
+(:~
+ : Create Content Plugin
+ :
+ : @param $id          - the identifier returned by the collector
+ : @param $raw-content - the raw content being loaded.
+ : @param $options     - a map containing options. Options are sent from Java
+ :
+ : @return - your transformed content
+ :)
+declare function plugin:create-content(
+  $id as xs:string,
+  $raw-content as node()?,
+  $options as map:map) as node()?
+{
+  (: name the binary uri with a pdf extension :)
+  let $binary-uri := fn:replace($id, ".xml", ".pdf")
+
+  (: stash the binary uri in the options map for later:)
+  let $_ := map:put($options, 'binary-uri', $binary-uri)
+
+  (: save the incoming binary as a pdf :)
+  let $_ :=
+    xdmp:eval('
+    	declare variable $binary-uri external;
+      declare variable $raw-content external;
+      xdmp:document-insert($binary-uri, $raw-content)
+      ',
+      map:new((
+        map:entry("binary-uri", $binary-uri),
+        map:entry("raw-content", $raw-content)
+      )),
+      map:new((
+        map:entry("isolation", "different-transaction"),
+        map:entry("commit", "auto")
+      ))
+   )
+  return
+  (: 
+   : extract the contents of the pdf and return them
+   : as the content for the envelope
+   :)
+    xdmp:document-filter($raw-content)
+};

examples/load-binaries/plugins/entities/Guides/input/LoadAsXml/headers/headers.xqy

@@ -1,27 +1,25 @@
-xquery version "1.0-ml";
-
-module namespace plugin = "http://marklogic.com/data-hub/plugins";
-
-declare namespace envelope = "http://marklogic.com/data-hub/envelope";
-
-declare option xdmp:mapping "false";
-
-(:~
- : Create Headers Plugin
- :
- : @param $id      - the identifier returned by the collector
- : @param $content - the output of your content plugin
- : @param $options - a map containing options. Options are sent from Java
- :
- : @return - zero or more header nodes
- :)
-declare function plugin:create-headers(
-  $id as xs:string,
-  $content as node()?,
-  $options as map:map) as node()*
-{
-  (: put the binary uri in the header of the envelope :)
-  (
-    <binary-uri>{map:get($options, "binary-uri")}</binary-uri>
-  )
-};
+xquery version "1.0-ml";
+
+module namespace plugin = "http://marklogic.com/data-hub/plugins";
+
+declare option xdmp:mapping "false";
+
+(:~
+ : Create Headers Plugin
+ :
+ : @param $id      - the identifier returned by the collector
+ : @param $content - the output of your content plugin
+ : @param $options - a map containing options. Options are sent from Java
+ :
+ : @return - zero or more header nodes
+ :)
+declare function plugin:create-headers(
+  $id as xs:string,
+  $content as node()?,
+  $options as map:map) as node()*
+{
+  (: put the binary uri in the header of the envelope :)
+  (
+    <binary-uri>{map:get($options, "binary-uri")}</binary-uri>
+  )
+};