#447 New tasks for system plugins, along with mlBundle support

Author: rjrudin

examples/dependency-project/README.md

@@ -27,12 +27,12 @@ will be included when the application is deployed:
 
 ```
 Found mlBundle configuration, will extract all of its dependencies to build/mlBundle
-Extracting file: /Users/rrudin/.m2/repository/com/marklogic/example-dependency/1.0.0/example-dependency-1.0.0.jar
     [unzip] Expanding: /Users/rrudin/.m2/repository/com/marklogic/example-dependency/1.0.0/example-dependency-1.0.0.jar into /Users/rrudin/dev/workspace/ml-gradle/examples/dependency-project/ml-gradle-client-project/build/mlBundle
 Finished extracting mlBundle dependencies
-Module paths: [/Users/rrudin/dev/workspace/ml-gradle/examples/dependency-project/ml-gradle-client-project/build/mlBundle/example-dependency/ml-modules, /Users/rrudin/dev/workspace/ml-gradle/examples/dependency-project/ml-gradle-client-project/src/main/ml-modules]
-Data paths: [/Users/rrudin/dev/workspace/ml-gradle/examples/dependency-project/ml-gradle-client-project/build/mlBundle/example-dependency/ml-data, /Users/rrudin/dev/workspace/ml-gradle/examples/dependency-project/ml-gradle-client-project/src/main/ml-data]
-:mlPrepareBundles (Thread[Task worker for ':' Thread 4,5,main]) completed. Took 0.165 secs.
+Module paths including mlBundle paths: [/Users/rrudin/dev/workspace/ml-gradle/examples/dependency-project/ml-gradle-client-project/build/mlBundle/example-dependency/ml-modules, /Users/rrudin/dev/workspace/ml-gradle/examples/dependency-project/ml-gradle-client-project/src/main/ml-modules]
+Data paths including mlBundle paths: [/Users/rrudin/dev/workspace/ml-gradle/examples/dependency-project/ml-gradle-client-project/build/mlBundle/example-dependency/ml-data, /Users/rrudin/dev/workspace/ml-gradle/examples/dependency-project/ml-gradle-client-project/src/main/ml-data]
+Plugin paths include mlBundle paths: [/Users/rrudin/dev/workspace/ml-gradle/examples/dependency-project/ml-gradle-client-project/build/mlBundle/example-dependency/ml-plugins, /Users/rrudin/dev/workspace/ml-gradle/examples/dependency-project/ml-gradle-client-project/src/main/ml-plugins]
+:mlPrepareBundles (Thread[Task worker for ':',5,main]) completed. Took 0.059 secs.
 ```
 
 And you'll see logging like this that indicates that the example-dependency modules were loaded:
@@ -58,10 +58,37 @@ Shutting down ExecutorService
 Writing 2 documents to MarkLogic; port: 8030
 ```
 
+And also some logging like this that indicates that a system plugin was installed:
+
+```
+Writing plugin zip file to URI: /com.marklogic/plugins/varianceplugin.zip
+Writing content for /com.marklogic/plugins/varianceplugin.zip
+Installing plugin with scope 'native' from URI: /com.marklogic/plugins/varianceplugin.zip
+Installed plugin with scope 'native', result: 1
+Finished executing command [com.marklogic.appdeployer.command.plugins.InstallPluginsCommand]
+```
+
 You can then use qconsole to verify that the following documents were inserted:
 
 - In ml-gradle-client-modules: /example.sjs
-- In ml-gradle-client-content: /example/data1.json and /example/data2.json
+- In ml-gradle-client-content: /example/data1.json and /example/data2.json (in addition to the data files including 
+in this project: /testdata/test1.json, /testdata/test2.json, and /testdata/test3.json)
+- In Extensions: /native/scope.xml, /native/varianceplugin/libvarianceplugin.dylib, and /native/varianceplugin/manifest.xml 
+
+Note that ml-gradle-client-content also contains the plugin zip file at /com.marklogic/plugins/varianceplugin.zip . If 
+you don't want that to remain, you can use a custom Gradle task to delete it after the deployment, or use the 
+mlPluginDatabaseName property to store the zip file in a different database.
+
+You can also verify that the plugin was installed correctly by running the following task:
+
+    gradle testPlugin
+    
+Which should return:
+
+```
+> Task :testPlugin
+66.6666666666667
+```
 
 See [Loading data](https://github.com/marklogic-community/ml-app-deployer/wiki/Loading-data) for more
 information on configuring how data is loaded during a deployment.

examples/dependency-project/ml-gradle-client-project/build.gradle

@@ -21,3 +21,8 @@ repositories {
 dependencies {
 	mlBundle "com.marklogic:example-dependency:1.0.0"
 }
+
+task testPlugin(type: com.marklogic.gradle.task.ServerEvalTask) {
+	description = "Run this to verify that the plugin from the example-dependency bundle was installed correctly; should get 66.67 as a result"
+	javascript = "cts.aggregate('native/varianceplugin', 'variance', cts.elementReference(xs.QName('amount')))"
+}

examples/dependency-project/ml-gradle-client-project/src/main/ml-config/databases/content-database.json

@@ -0,0 +1,13 @@
+{
+	"database-name": "%%DATABASE%%",
+	"range-element-index": [
+		{
+			"collation": "",
+			"invalid-values": "reject",
+			"localname": "amount",
+			"namespace-uri": "",
+			"range-value-positions": false,
+			"scalar-type": "double"
+		}
+	]
+}

examples/dependency-project/ml-gradle-client-project/src/main/ml-data/testdata/test1.json

@@ -0,0 +1,3 @@
+{
+	"amount": 10
+}

examples/dependency-project/ml-gradle-client-project/src/main/ml-data/testdata/test2.json

@@ -0,0 +1,3 @@
+{
+	"amount": 20
+}

examples/dependency-project/ml-gradle-client-project/src/main/ml-data/testdata/test3.json

@@ -0,0 +1,3 @@
+{
+	"amount": 30
+}

examples/dependency-project/provider-project/build.gradle

@@ -16,6 +16,9 @@ task bundleJar(type: Jar) {
 	from("src/main/ml-data") {
 		into("example-dependency/ml-data")
 	}
+	from("src/main/ml-plugins") {
+		into("example-dependency/ml-plugins")
+	}
 	destinationDir file("build/libs")
 	baseName "example-dependency"
 }

examples/dependency-project/provider-project/src/main/ml-plugins/variance/.gitignore

@@ -0,0 +1,4 @@
+*.dylib
+*.o
+*.zip
+manifest.xml

examples/dependency-project/provider-project/src/main/ml-plugins/variance/Makefile

@@ -0,0 +1,108 @@
+PLUGIN_NAME = varianceplugin
+PLUGIN_VERSION = 0.1
+PLUGIN_PROVIDER = MarkLogic
+PLUGIN_DESCRIPTION = variance native plugin
+
+PLUGIN_SRCS = \
+	VariancePlugin.cpp
+
+MANIFEST = manifest.xml
+
+ZIP_NAME = $(PLUGIN_NAME).zip
+PLUGIN_OBJS = ${PLUGIN_SRCS:.cpp=.o}
+
+all:: $(PLUGIN_NAME)
+
+ifneq "$(WINDIR)" ""
+
+-include defs.win32
+
+CPPFLAGS = \
+	-nologo \
+	-W3 \
+	-GR \
+	-GS- \
+	-EHsc \
+	-wd4996 \
+	-Zi -MTd -bigobj \
+	-O2 -Oy- \
+	-D_WIN32_WINNT=0x501 \
+	-D_SECURE_SCL=0
+
+FULL_LIB_NAME = $(PLUGIN_NAME).dll
+
+DLL_LIBS="$(ProgramW6432)/MarkLogic/MarkLogic.lib"
+INCLUDES=-I "$(ProgramW6432)/MarkLogic/include"
+
+$(FULL_LIB_NAME): $(PLUGIN_OBJS)
+	$(LINK) $(PLUGIN_OBJS) $(DLL_LIBS) \
+		-nologo -incremental:no -dll -out:$@ -largeaddressaware
+
+%.o: %.cpp
+	$(CPP) $(CPPFLAGS) $(INCLUDES) -Fo$@ -I. -c $<
+
+vsinstalls: makevcinclude.sh findvcinstalls.cmd
+	rm -rf $@ vsinstalls
+	./findvcinstalls.cmd bash ./makevcinclude.sh > $@
+
+clean::
+	rm -rf $(FULL_LIB_NAME:.dll=.lib)
+	rm -rf $(FULL_LIB_NAME).manifest
+	rm -rf $(FULL_LIB_NAME:.dll=.exp)
+	rm -rf $(FULL_LIB_NAME:.dll=.pdb)
+	rm -rf ${PLUGIN_SRCS:.cpp=.lib}
+	rm -rf ${PLUGIN_SRCS:.cpp=.exp}
+	rm -rf ${PLUGIN_SRCS:.cpp=.pdb}
+	rm -rf *.pdb
+	rm -rf vsinstalls
+else
+
+ifeq "$(shell uname -s)" "Linux"
+MARKLOGIC = /opt/MarkLogic
+FULL_LIB_NAME = lib$(PLUGIN_NAME).so
+CPP=g++
+LIB_FLAGS = -m64 -fPIC -shared
+endif
+
+ifeq "$(shell uname -s)" "Darwin"
+MARKLOGIC := $(shell cd ~; pwd)/Library/MarkLogic
+FULL_LIB_NAME = lib$(PLUGIN_NAME).dylib
+CPP=g++
+LIB_FLAGS = -m64 -dynamiclib -Wl,-undefined,dynamic_lookup
+endif
+
+$(FULL_LIB_NAME): $(PLUGIN_OBJS)
+	$(CPP) $(LIB_FLAGS) -o $(FULL_LIB_NAME) $(PLUGIN_OBJS)
+
+EXE_EXTRA = -ldl
+EXE_FLAGS = -rdynamic
+
+%.o: %.cpp
+	$(CPP) -m64 -fPIC -g -O2 -Woverloaded-virtual -I. -I$(MARKLOGIC)/include -o $@ -c $<
+endif
+
+clean::
+	rm -rf $(ZIP_NAME)
+	rm -rf $(FULL_LIB_NAME)
+	rm -rf $(PLUGIN_OBJS)
+	rm -rf $(MANIFEST)
+
+$(MANIFEST):
+	echo '<?xml version="1.0" encoding="UTF-8"?>' > $(MANIFEST)
+	echo '<plugin xmlns="http://marklogic.com/extension/plugin">' >> $(MANIFEST)
+	echo '  <name>$(PLUGIN_NAME)</name>' >> $(MANIFEST)
+	echo '  <id>$(PLUGIN_NAME)</id>' >> $(MANIFEST)
+	echo '  <version>$(PLUGIN_VERSION)</version>' >> $(MANIFEST)
+	echo '  <provider-name>$(PLUGIN_PROVIDER)</provider-name>' >> $(MANIFEST)
+	echo '  <description>$(PLUGIN_DESCRIPTION)</description>' >> $(MANIFEST)
+	echo '  <native>' >> $(MANIFEST)
+	echo '    <path>$(FULL_LIB_NAME)</path>' >> $(MANIFEST)
+	echo '  </native>' >> $(MANIFEST)
+	echo '</plugin>' >> $(MANIFEST)
+
+$(ZIP_NAME): $(FULL_LIB_NAME) $(MANIFEST)
+	zip $(ZIP_NAME) $(FULL_LIB_NAME) $(MANIFEST)
+
+$(PLUGIN_NAME): $(ZIP_NAME)
+
+.PHONY: $(PLUGIN_NAME)

examples/dependency-project/provider-project/src/main/ml-plugins/variance/VariancePlugin.cpp

@@ -0,0 +1,128 @@
+/*
+ * Copyright © 2017 MarkLogic Corporation
+ */
+
+#include <MarkLogic.h>
+
+#include <iomanip>
+#include <sstream>
+#include <string>
+#include <vector>
+#include <stdio.h>
+#include <math.h>
+
+#ifdef _MSC_VER
+#define PLUGIN_DLL __declspec(dllexport)
+#else // !_MSC_VER
+#define PLUGIN_DLL
+#endif
+
+using namespace marklogic;
+using namespace std;
+
+////////////////////////////////////////////////////////////////////////////////
+
+class Variance : public AggregateUDF
+{
+public:
+  struct Value {
+    double sum, sum_sq, count;
+    Value() : sum(0), sum_sq(0), count(0) {}
+  } value;
+
+public:
+  AggregateUDF* clone() const { return new Variance(*this); }
+  void close() { delete this; }
+
+  void start(Sequence&, Reporter&) {}
+  void finish(OutputSequence& os, Reporter& reporter);
+
+  void map(TupleIterator& values, Reporter& reporter);
+  void reduce(const AggregateUDF* _o, Reporter& reporter);
+
+  void encode(Encoder& e, Reporter& reporter);
+  void decode(Decoder& d, Reporter& reporter);
+
+protected:
+  virtual double getResult();
+};
+
+double Variance::
+getResult()
+{
+  double mean = (value.sum / value.count);
+  double variance = (value.sum_sq / value.count) - (mean * mean);
+
+  // this can be a really small negative number due to floating point
+  // rounding errors so set to zero if less than zero
+  if (variance < 0) {
+    variance = 0;
+  }
+  return variance;
+}
+
+void Variance::
+finish(OutputSequence& os, Reporter& reporter)
+{
+  os.writeValue(getResult());
+}
+
+void Variance::
+map(TupleIterator& values, Reporter& reporter)
+{
+  for(; !values.done(); values.next()) {
+    if(!values.null(0)) {
+      double v; values.value(0,v);
+      value.sum += v * (double)values.frequency();
+      value.sum_sq += (v * v) * (double)values.frequency();
+      value.count += (double)values.frequency();
+    }
+  }
+}
+
+void Variance::
+reduce(const AggregateUDF* _o, Reporter& reporter)
+{
+  const Variance *o = (const Variance*)_o;
+  value.sum += o->value.sum;
+  value.sum_sq += o->value.sum_sq;
+  value.count += o->value.count;
+}
+
+void Variance::
+encode(Encoder& e, Reporter& reporter)
+{
+  e.encode(&value,sizeof(Value));
+}
+
+void Variance::
+decode(Decoder& d, Reporter& reporter)
+{
+  d.decode(&value,sizeof(Value));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+class Stddev : public Variance
+{
+public:
+  AggregateUDF* clone() const { return new Stddev(*this); }
+
+protected:
+  virtual double getResult();
+};
+
+double Stddev::
+getResult()
+{
+  return sqrt(Variance::getResult());
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+extern "C" PLUGIN_DLL void
+marklogicPlugin(Registry& r)
+{
+  r.version();
+  r.registerAggregate<Variance>("variance");
+  r.registerAggregate<Stddev>("stddev");
+}