MarkLogic Workflow PoC Proven

Completed test scripts
Fixed out of process method for userTask
step and process cleanup verified to be accurate

Author: adamfowleruk

data/examples/bpmn2/021-initiating-attachment.bpmn

@@ -1,26 +1,67 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <bpmn2:definitions xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:bpmn2="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_1" targetNamespace="http://marklogic.com/workflow">
-  <bpmn2:process id="051_domain" name="Default Process" processType="Public">
+  <bpmn2:interface id="Interface_1" implementationRef="MarkLogic" name="MarkLogic">
+    <bpmn2:operation id="_Operation_3" implementationRef="LaunchDomain" name="LaunchDomain"/>
+  </bpmn2:interface>
+  <bpmn2:process id="051_domain" name="Domain Launched" isExecutable="true" processType="Public">
     <bpmn2:startEvent id="StartEvent_1">
       <bpmn2:outgoing>SequenceFlow_1</bpmn2:outgoing>
+      <bpmn2:dataOutput id="DataOutput_1" name="Message_1_Output"/>
+      <bpmn2:dataOutputAssociation id="DataOutputAssociation_1">
+        <bpmn2:sourceRef>DataOutput_1</bpmn2:sourceRef>
+      </bpmn2:dataOutputAssociation>
+      <bpmn2:outputSet id="OutputSet_1" name="Output Set 1">
+        <bpmn2:dataOutputRefs>DataOutput_1</bpmn2:dataOutputRefs>
+      </bpmn2:outputSet>
+      <bpmn2:messageEventDefinition id="MessageEventDefinition_1">
+        <bpmn2:operationRef>_Operation_3</bpmn2:operationRef>
+      </bpmn2:messageEventDefinition>
     </bpmn2:startEvent>
-    <bpmn2:sequenceFlow id="SequenceFlow_1" sourceRef="StartEvent_1" targetRef="EndEvent_1"/>
+    <bpmn2:sequenceFlow id="SequenceFlow_1" sourceRef="StartEvent_1" targetRef="Task_1"/>
     <bpmn2:endEvent id="EndEvent_1">
-      <bpmn2:incoming>SequenceFlow_1</bpmn2:incoming>
+      <bpmn2:incoming>SequenceFlow_2</bpmn2:incoming>
     </bpmn2:endEvent>
+    <bpmn2:task id="Task_1" name="Do Something">
+      <bpmn2:incoming>SequenceFlow_1</bpmn2:incoming>
+      <bpmn2:outgoing>SequenceFlow_2</bpmn2:outgoing>
+    </bpmn2:task>
+    <bpmn2:sequenceFlow id="SequenceFlow_2" sourceRef="Task_1" targetRef="EndEvent_1"/>
+    <bpmn2:textAnnotation id="TextAnnotation_1">
+      <bpmn2:text>Start event includes a Message definition, configured for a MarkLogic LaunchDomain. This is a CPF Domain linked to a URI folder (depth 1) for a particular process data document. (NOT an initiating attachment - see 021-initiating-attachment for that)</bpmn2:text>
+    </bpmn2:textAnnotation>
+    <bpmn2:association id="Association_1" sourceRef="TextAnnotation_1" targetRef="StartEvent_1"/>
   </bpmn2:process>
-  <bpmndi:BPMNDiagram id="BPMNDiagram_1" name="Default Process Diagram">
+  <bpmndi:BPMNDiagram id="BPMNDiagram_1" name="Domain Launched">
     <bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="051_domain">
       <bpmndi:BPMNShape id="BPMNShape_1" bpmnElement="StartEvent_1">
         <dc:Bounds height="36.0" width="36.0" x="100.0" y="100.0"/>
       </bpmndi:BPMNShape>
       <bpmndi:BPMNShape id="BPMNShape_2" bpmnElement="EndEvent_1">
         <dc:Bounds height="36.0" width="36.0" x="500.0" y="100.0"/>
       </bpmndi:BPMNShape>
-      <bpmndi:BPMNEdge id="BPMNEdge_SequenceFlow_1" bpmnElement="SequenceFlow_1" sourceElement="BPMNShape_1" targetElement="BPMNShape_2">
+      <bpmndi:BPMNShape id="BPMNShape_Task_1" bpmnElement="Task_1">
+        <dc:Bounds height="50.0" width="110.0" x="270.0" y="93.0"/>
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="BPMNShape_TextAnnotation_1" bpmnElement="TextAnnotation_1">
+        <dc:Bounds height="100.0" width="291.0" x="160.0" y="180.0"/>
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNEdge id="BPMNEdge_SequenceFlow_1" bpmnElement="SequenceFlow_1" sourceElement="BPMNShape_1" targetElement="BPMNShape_Task_1">
         <di:waypoint xsi:type="dc:Point" x="136.0" y="118.0"/>
+        <di:waypoint xsi:type="dc:Point" x="196.0" y="118.0"/>
+        <di:waypoint xsi:type="dc:Point" x="196.0" y="118.0"/>
+        <di:waypoint xsi:type="dc:Point" x="270.0" y="118.0"/>
+      </bpmndi:BPMNEdge>
+      <bpmndi:BPMNEdge id="BPMNEdge_SequenceFlow_2" bpmnElement="SequenceFlow_2" sourceElement="BPMNShape_Task_1" targetElement="BPMNShape_2">
+        <di:waypoint xsi:type="dc:Point" x="380.0" y="118.0"/>
+        <di:waypoint xsi:type="dc:Point" x="434.0" y="118.0"/>
+        <di:waypoint xsi:type="dc:Point" x="434.0" y="118.0"/>
         <di:waypoint xsi:type="dc:Point" x="500.0" y="118.0"/>
       </bpmndi:BPMNEdge>
+      <bpmndi:BPMNEdge id="BPMNEdge_Association_1" bpmnElement="Association_1" sourceElement="BPMNShape_TextAnnotation_1" targetElement="BPMNShape_1">
+        <di:waypoint xsi:type="dc:Point" x="160.0" y="230.0"/>
+        <di:waypoint xsi:type="dc:Point" x="118.0" y="230.0"/>
+        <di:waypoint xsi:type="dc:Point" x="118.0" y="136.0"/>
+      </bpmndi:BPMNEdge>
     </bpmndi:BPMNPlane>
   </bpmndi:BPMNDiagram>
 </bpmn2:definitions>

data/examples/bpmn2/051_domain.bpmn

@@ -29,12 +29,15 @@ be transitioned. In Workflow an out of CPF action may need to complete a state.
 that a CPF action (genericComplete.xqy) can move the state onwards. This CANNOT be done outside of a CPF action (despite
   what the MarkLogic documentation website says).
 
-In order to do this an element (/prop:properties/wf:currentStep/wf:step-status) is initially set to ENTERED, then when
-the set up action (E.g. add to a queue) completes, this is set to IN PROGRESS. When the out of sequence action (user task,
-  web service call) wants the state to complete it sets this to COMPLETE. CPF then in turn runs the isComplete.xqy
-condition, which returns true, running the genericComplete.xqy action, which calls wfu:complete.
-
-NEVER call wfu:complete or wfu:completeById outside of a CPF action.
+In order to do this :-
+- In your initiating CPF action (E.g. userTask.xqy) set an element (/prop:properties/wf:currentStep/wf:step-status) to ENTERED
+- Then something out of process happens, setting just this property to either IN PROGRESS, then eventually COMPLETE
+- The restart.xqy status change action fires, spots COMPLETE, and forces CPF to the state specified in prop:properties/wf:currentStep/wf:state
+- This also calls wfu:complete, which deletes prop:properties/wf:currentStep
+- CPF then transitions to the new state, which is where you should have an action to run any cleanup for your out of process activity
+- Also, you must call the genericComplete.xqy action, which in turn calls wfu:complete, moving on to the success state specified
+
+NEVER call wfu:complete or wfu:completeById outside of a CPF action - only wfu:finallyComplete (as done in process POST rest api call)
 
 ## Notes on the CPF configuration of custom steps
 

documentation/DEV-NEWACTIVITY.md

@@ -16,6 +16,7 @@ This PoC development will occur in small sprints until a useful PoC system is av
  - SLA timer and routing on failure
  - Email notification
  - Assigning tasks to MarkLogic roles
+ - Assigning Role dynamically via XQuery expression
 - UK-D-I
 
 ## Theoretical requirements
@@ -34,7 +35,7 @@ steps if a duplicate is found. Requires search (by property hash AND not (same u
 ## Sprint 1 - Basic workflow
 
 - DONE BPMN2: generic blank task
-- TEST BPMN2: user task -> aka human step
+- DONE BPMN2: user task -> aka human step
 - DONE BPMN2: exclusive gateway -> Decision point with one outcome, multiple options
 - DONE Evaluation: Support for /xpath/path to process model data
 - DONE Evaluation: Support for fn:not(fn:empty(/xpath/evaluation)) style boolean evaluation for true/false conditions in BPMN2 model
@@ -46,7 +47,7 @@ steps if a duplicate is found. Requires search (by property hash AND not (same u
  - Create import step for this BPMN2 method
  - Create CPF step to represent this
 - TEST Evaluation: Support for (/some/path/one,/some/path/two)[1] style evaluation for set task
-- TEST Tools: Process Data model XSD (for modeler import)
+- DONE Tools: Process Data model XSD (for modeler import)
 - IN PROGRESS Tools: Eclipse BPMN 2 Modeler Palette and Process diagram support, including new diagram creation for MarkLogic
 - DEFERRED UI: Ridiculously basic HTML widget in MLJS for rendering step and choosing action (for ease of testing)
 - TEST Start process using an Alert (content subscription)
@@ -61,10 +62,10 @@ steps if a duplicate is found. Requires search (by property hash AND not (same u
   - DONE GET fetch the current state of a business process
  - TEST processsubscription.xqy
   - TEST PUT create a process subscription (alert) to create a new process instance (creating a content doc creates a process doc with an initiating attachment)
- - TEST processinbox.xqy
- - TEST processqueue.xqy
+ - DONE processinbox.xqy
+ - DONE processqueue.xqy
  - TEST support for roles (processroleinbox.xqy) on user tasks (For BD)
-- TEST Test scripts for automating install, create, get, update, complete via REST API
+- DONE Test scripts for automating install, create, get, update, complete via REST API
 - DONE Bug: Change process model URI folder to include major and minor - else doing process doc update may run new pipeline instead of old one
 - DONE Bug: Multiple wf:status properties on in process process, complete and running
 - BPMN2 specification test process models modified and tested to MarkLogic executable standard

documentation/SPRINTS.md

@@ -3,6 +3,7 @@ xquery version "1.0-ml";
 module namespace m="http://marklogic.com/workflow-actions";
 
 import module namespace wfu="http://marklogic.com/workflow-util" at "/app/models/workflow-util.xqy";
+import module namespace cpf = "http://marklogic.com/cpf" at "/MarkLogic/cpf/cpf.xqy";
 
 declare namespace prop = "http://marklogic.com/xdmp/property";
 declare namespace wf="http://marklogic.com/workflow";
@@ -60,16 +61,18 @@ declare function complete-generic($processId as xs:string) as empty-sequence() {
   let $_ := xdmp:log("In wfa:complete-generic: processId: "||$processId)
   let $props := wfu:getProperties($processId)
   let $_ := xdmp:log($props)
-  let $next := $props/wf:currentStep/wf:state/text()
+  (:
+  let $next := $props/prop:properties/wf:currentStep/wf:state/text()  (: WHERE SHOULD THIS BE FROM INSTEAD ? :)
   let $_ := xdmp:log($next)
-  let $transition := $props/wf:currentStep/wf:state-transition/text()
-  let $_ := xdmp:log($transition)
-  let $startTime := xs:dateTime($props/wf:currentStep/wf:startTime)
-  let $_ := xdmp:log($startTime)
+  :)
+  let $transition := $props/cpf:state/text()
+  let $_ := xdmp:log("wfa:complete-generic: transition: " || $transition)
+(:)  let $startTime := xs:dateTime($props/prop:properties/wf:currentStep/wf:startTime):)
+  (:let $_ := xdmp:log($startTime):)
   (:let $update := xdmp:document-remove-properties(wfu:getProcessUri($processId),xs:QName("wf:currentStep")) :)
   (:let $update := xdmp:node-delete($props/wf:currentStep)
   let $_ := xdmp:log($update) :)
   return
     (: wfu:completeById($processId,$transition,xs:anyURI($next),$startTime) :)
-    wfu:inProgress($processId,$transition)
+    wfu:finallyComplete($processId,$transition)
 };

modules/app/models/workflow-actions.xqy

@@ -435,7 +435,17 @@ declare function m:bpmn2-to-cpf($pname as xs:string, $doc as element(b2:definiti
   return
     p:create($pname,$pname,
       p:action("/MarkLogic/cpf/actions/success-action.xqy",(),()),
-      $failureAction,(),
+      $failureAction,
+      (
+        p:status-transition("updated","Restart process on external action",() (: success :), $failureState,500,
+          p:action("/workflowengine/actions/restart.xqy",
+            "Check for restarting process.",() (: options :)
+          )
+          ,
+          () (: rules :)
+        )
+      ) (: status transitions :)
+      ,
       (
 
           (: create entry CPF action :)
@@ -587,10 +597,13 @@ declare function m:bpmn2-to-cpf($pname as xs:string, $doc as element(b2:definiti
                       {
                         if (fn:not(fn:empty($role))) then <wf:role>{$role}</wf:role> else ()
                       }
+                      <wf:state>{xs:anyURI("http://marklogic.com/states/"||$pname||"/"||xs:string($state/@id)||"__complete")}</wf:state>
                     </p:options>
                   ) (: p action :)
                   ,"Apply set up user task action on entry"
                 ) (: p:execute :)
+
+                (:
                 ,
                 p:execute(
                   p:condition("/workflowengine/conditions/isComplete.xqy","Check if complete",())
@@ -603,10 +616,37 @@ declare function m:bpmn2-to-cpf($pname as xs:string, $doc as element(b2:definiti
                   ),
                   "Apply default complete action"
                 ) (: p execute :)
+                :)
 
 
               ) (: execute set :)
             ) (: state transition :)
+            ,
+            (:
+            p:state-transition(xs:anyURI("http://marklogic.com/states/"||$pname||"/"||xs:string($state/@id)||"__inprogress"),
+              "",(),
+              $failureState,(),
+              () (: empty default action :)
+              ,
+              (
+
+
+              ) (: execute set :)
+            )
+            ,
+            :)
+            p:state-transition(xs:anyURI("http://marklogic.com/states/"||$pname||"/"||xs:string($state/@id)||"__complete"),
+              "",xs:anyURI("http://marklogic.com/states/"||$pname||"/"||xs:string($sf/@targetRef)),
+              $failureState,(),
+              p:action("/workflowengine/actions/genericComplete.xqy",
+                "User completion occurred: "||xs:string($state/@name),() (: options :)
+              ) (: empty default action :)
+              ,
+              (
+
+
+              ) (: execute set :)
+            )
           ) (: state transition set :)
 
 

modules/app/models/workflow-import.xqy

@@ -151,7 +151,11 @@ declare function m:complete($processUri as xs:string,$transition as node(),$stat
   let $_ := xdmp:log($startTime)
   let $_ := xdmp:log("  transition")
   let $_ := xdmp:log($transition)
-  let $_ := xdmp:node-replace(xdmp:document-properties($processUri)/prop:properties/wf:status,<wf:status>COMPLETE</wf:status>)
+
+  (: clean up BPMN2 activity step properties :)
+  let $cs := xdmp:document-properties($processUri)/prop:properties/wf:currentStep
+  let $_ := if (fn:not(fn:empty($cs))) then xdmp:node-delete($cs) else ()
+
   let $_ := m:metric($processUri,$transition/p:state/text(),$startTime,fn:current-dateTime(),fn:true())
   (: Add audit event :)
   let $_ := m:audit($processUri,$transition/p:state/text(),"ProcessEngine","Completed step",())
@@ -171,13 +175,28 @@ declare function m:completeById($processId as xs:string,$transition as xs:string
 };
 
 (:
- : Can be called from out of sequence (non CPF) modules - forces a CPF re-evaluation, then complete, in conjunction with the genericComplete.xqy action
+ : Can be called from out of sequence (non CPF) modules - forces a CPF re-evaluation, then complete, in conjunction with the restart.xqy action
  :)
 declare function m:finallyComplete($processId as xs:string,$transition as xs:string) as empty-sequence() {
   let $processUri := m:getProcessUri($processId)
-  let $_ := m:audit($processUri,$transition/p:state/text(),"ProcessEngine","Marking as Complete",())
+  let $props := xdmp:document-properties($processUri)/prop:properties
+  let $_ := m:audit($processUri,$transition,"ProcessEngine","Marking as Complete",())
   return
-    xdmp:node-replace(xdmp:document-properties($processUri)/prop:properties/wf:currentStep/wf:step-status,<wf:step-status>COMPLETE</wf:step-status>)
+    (
+    xdmp:node-replace($props/wf:currentStep/wf:step-status,<wf:step-status>COMPLETE</wf:step-status>)
+    (:,
+    xdmp:node-replace($props/cpf:status,<cpf:processing-status>active</cpf:processing-status>)
+    :)
+    (:
+    ),
+    xdmp:node-replace($props/cpf:status,<cpf:state>{$props/wf:currentStep/wf:state/text()}</cpf:state>)
+    :)
+    (:
+    ,
+    (: god awful hack to test CPF updated status change handling :)
+    xdmp:node-insert-after(fn:doc($processUri)/wf:process/wf:data,<tag>You're it</tag>)
+    :)
+    )
 };
 
 (:
@@ -186,7 +205,7 @@ declare function m:finallyComplete($processId as xs:string,$transition as xs:str
  :)
 declare function m:inProgress($processId as xs:string,$transition as xs:string) as empty-sequence() {
   let $processUri := m:getProcessUri($processId)
-  let $_ := m:audit($processUri,$transition/p:state/text(),"ProcessEngine","In Progress",())
+  let $_ := m:audit($processUri,$transition,"ProcessEngine","In Progress",())
   return
     xdmp:node-replace(xdmp:document-properties($processUri)/prop:properties/wf:currentStep/wf:step-status,<wf:step-status>IN PROGRESS</wf:step-status>)
 };

modules/app/models/workflow-util.xqy

@@ -37,6 +37,8 @@ try {
       )
     )
     ,
+    xdmp:node-replace(xdmp:document-properties($cpf:document-uri)/prop:properties/wf:status,<wf:status>COMPLETE</wf:status>)
+    ,
     wfu:complete( $cpf:document-uri, $cpf:transition, (), $st )
   )
 } catch ($e) {

modules/workflowengine/actions/endEvent.xqy

@@ -11,12 +11,14 @@ declare variable $cpf:transition as node() external;
 declare variable $cpf:options as element() external;
 
 try {
+  let $st := fn:current-dateTime()
+  let $_ := xdmp:log("MarkLogic Workflow generic complete CPF action called for: "||$cpf:document-uri)
   let $props := xdmp:document-properties($cpf:document-uri)/prop:properties
   let $_ := xdmp:log($props)
-  let $next := $cpf:options/wf:state/text()
-  let $_ := xdmp:log($next)
-  let $startTime := xs:dateTime($props/wf:currentStep/wf:startTime)
-  return wfu:complete( $cpf:document-uri, $cpf:transition, $next, $startTime )
+
+  (: Allow state transition to happen :)
+
+  return wfu:complete( $cpf:document-uri, $cpf:transition, (), $st )
 } catch ($e) {
   wfu:failure( $cpf:document-uri, $cpf:transition, $e, () )
 }

modules/workflowengine/actions/genericComplete.xqy

@@ -0,0 +1,51 @@
+xquery version "1.0-ml";
+
+import module namespace cpf = "http://marklogic.com/cpf" at "/MarkLogic/cpf/cpf.xqy";
+import module namespace wfu="http://marklogic.com/workflow-util" at "/app/models/workflow-util.xqy";
+
+declare namespace wf="http://marklogic.com/workflow";
+declare namespace prop = "http://marklogic.com/xdmp/property";
+
+declare variable $cpf:document-uri as xs:string external;
+declare variable $cpf:transition as node() external;
+declare variable $cpf:options as element() external;
+
+(:
+ : Check for all property-update and document-update eventualities that mean we should restart the process.
+ :)
+
+
+try {
+  let $_ := xdmp:log("MarkLogic Workflow restart CPF action called for: "||$cpf:document-uri)
+
+  (: 1. Check for external task completion status (step-status = "COMPLETE") :)
+  let $_ := xdmp:log("restart condition check for: " || $cpf:document-uri)
+  let $ready := xdmp:document-properties($cpf:document-uri)/prop:properties/wf:currentStep/wf:step-status
+  let $_ := xdmp:log($ready)
+  let $result := "COMPLETE" eq $ready
+  let $_ := xdmp:log($result)
+
+  return
+    if ($result) then
+
+      let $props := xdmp:document-properties($cpf:document-uri)/prop:properties
+      let $_ := xdmp:log($props)
+      let $next := xs:string($props/wf:currentStep/wf:state)
+      let $_ := xdmp:log("Next state: "||$next)
+      let $startTime := xs:dateTime($props/wf:currentStep/wf:startTime)
+      return wfu:complete( $cpf:document-uri, $cpf:transition, xs:anyURI($next), $startTime )
+    else (
+      (: From set-updated-action.xqy in CPF:)
+
+      if (cpf:check-transition($cpf:document-uri,$cpf:transition)) then
+        (
+          cpf:document-set-last-updated( $cpf:document-uri, fn:current-dateTime() )
+          ,
+          cpf:success( $cpf:document-uri, $cpf:transition, () )
+        )
+      else ()
+
+    ) (: do what CPF normally does... :)
+} catch ($e) {
+  wfu:failure( $cpf:document-uri, $cpf:transition, $e, () )
+}