fixing docker image

fixed #416

Author: paxtonhare

.travis/Dockerfile-ml

@@ -1,13 +1,25 @@
 FROM centos:centos7
 
 ENV LANG C.UTF-8
-
+ENV JAVA_VERSION 8u131
+ENV BUILD_VERSION b11
+ENV SUM d54c1d3a095b4ff2b6607d096fa80163
 RUN yum -y install glibc.i686 \
     gdb.x86_64 redhat-lsb.x86_64 vim \
     bzip2 \
     unzip \
-    xz-utils \
-    java-1.8.0-openjdk-devel 2>&1 > /dev/null
+    wget \
+    xz-utils 2>&1 > /dev/null
+
+# Downloading Java
+RUN wget --no-cookies --no-check-certificate --header "Cookie: oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/$JAVA_VERSION-$BUILD_VERSION/$SUM/jdk-$JAVA_VERSION-linux-x64.rpm" -O /tmp/jdk-8-linux-x64.rpm
+RUN yum -y install /tmp/jdk-8-linux-x64.rpm
+
+RUN alternatives --install /usr/bin/java jar /usr/java/latest/bin/java 200000
+RUN alternatives --install /usr/bin/javaws javaws /usr/java/latest/bin/javaws 200000
+RUN alternatives --install /usr/bin/javac javac /usr/java/latest/bin/javac 200000
+
+ENV JAVA_HOME /usr/java/latest
 
 # Install MarkLogic
 COPY ./MarkLogic.rpm /tmp/MarkLogic.rpm

examples/spring-batch/build.gradle

@@ -1,9 +1,3 @@
-buildscript {
-  repositories {
-    maven {url 'http://developer.marklogic.com/maven2/'}
-  }
-}
-
 plugins {
     id 'java'
     id 'eclipse'
@@ -20,9 +14,11 @@ repositories {
 
 dependencies {
     compile 'com.marklogic:marklogic-data-hub:1.1.2'
-    compile 'com.marklogic:marklogic-spring-batch-core:0.6.0'
-    compile 'com.marklogic:ml-javaclient-util:2.14.0'
-    testCompile 'com.marklogic:marklogic-spring-batch-test:0.6.0'
+    compile "com.marklogic:marklogic-spring-batch-core:0.7.4"
+    compile 'com.marklogic:ml-javaclient-util:4.0.alpha4'
+
+    testCompile "com.marklogic:marklogic-spring-batch-test:0.7.4"
+    runtime "com.marklogic:marklogic-spring-batch-core:0.7.2"
 }
 
 distributions {
@@ -31,4 +27,4 @@ distributions {
     }
 }
 
-mainClassName = "com.marklogic.spring.batch.hub.HubJobRunner"
+mainClassName = "com.marklogic.spring.batch.Main"

examples/spring-batch/gradle.properties

@@ -8,31 +8,65 @@
 #
 # ....
 mlHost=localhost
-mlAppName=data-hub
 
+# Your MarkLogic Username and Password
 mlUsername=admin
 mlPassword=admin
 
+# If specified, the manage username/password combo is used with the ML Management REST API for managing application
+# resources; this user must have the manage-admin and rest-admin roles.
+#
+# If these are not set, then mlUsername/mlPassword is used for managing application resources.
+# mlManageUsername=
+# mlManagePassword=
+
+# If specified, these values can override where the DHF thinks
+# MarkLogic default ports are at. You would only use this if you
+# have changed the ports on which MarkLogic listens
+#
+# mlAppServicesPort=8000
+# mlAdminPort=8001
+# mlManagePort=8002
+
+# If specified, the admin username/password combo is used with the ML Management REST API for creating users and roles. This
+# user must have the manage-admin or admin role. A good practice is to use your admin account here to create app-specific
+# users and roles, which can then be used as mlManageUsername/mlManagePassword and mlUsername/mlPassword.
+#
+# These properties are also used for connecting to the admin application on port 8001 - e.g. for initializing ML and for
+# waiting for ML to restart.
+#
+# If these properties are not set, then mlUsername/mlPassword will be used.
+# mlAdminUsername=
+# mlAdminPassword=
+
 mlStagingAppserverName=data-hub-STAGING
 mlStagingPort=8010
 mlStagingDbName=data-hub-STAGING
 mlStagingForestsPerHost=4
+mlStagingAuth=digest
 
 mlFinalAppserverName=data-hub-FINAL
 mlFinalPort=8011
 mlFinalDbName=data-hub-FINAL
 mlFinalForestsPerHost=4
+mlFinalAuth=digest
 
 mlTraceAppserverName=data-hub-TRACING
 mlTracePort=8012
 mlTraceDbName=data-hub-TRACING
 mlTraceForestsPerHost=1
+mlTraceAuth=digest
 
 mlJobAppserverName=data-hub-JOBS
 mlJobPort=8013
 mlJobDbName=data-hub-JOBS
 mlJobForestsPerHost=1
+mlJobAuth=digest
 
 mlModulesDbName=data-hub-MODULES
 mlTriggersDbName=data-hub-TRIGGERS
 mlSchemasDbName=data-hub-SCHEMAS
+
+# The name of the Role to create for Hub Access
+mlHubUserRole=data-hub-ROLE
+mlHubUserName=data-hub-user

examples/spring-batch/src/main/java/example/LoadAndRunFlow.java

@@ -5,18 +5,16 @@
 import com.marklogic.client.document.GenericDocumentManager;
 import com.marklogic.client.document.ServerTransform;
 import com.marklogic.client.helper.DatabaseClientProvider;
-import com.marklogic.hub.HubConfig;
-import com.marklogic.spring.batch.hub.AbstractMarkLogicBatchConfig;
-import com.marklogic.spring.batch.hub.FlowConfig;
-import com.marklogic.spring.batch.hub.StagingConfig;
 import com.marklogic.spring.batch.item.processor.ResourceToDocumentWriteOperationItemProcessor;
 import com.marklogic.spring.batch.item.reader.EnhancedResourcesItemReader;
 import org.springframework.batch.core.Job;
 import org.springframework.batch.core.Step;
+import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
+import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
 import org.springframework.batch.core.configuration.annotation.JobScope;
+import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
 import org.springframework.batch.item.ItemProcessor;
 import org.springframework.batch.item.ItemWriter;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.EnvironmentAware;
 import org.springframework.context.annotation.Bean;
@@ -26,30 +24,28 @@
 
 import java.util.*;
 
-@Import({ StagingConfig.class, FlowConfig.class })
-public class LoadAndRunFlow extends AbstractMarkLogicBatchConfig implements EnvironmentAware {
+@EnableBatchProcessing
+public class LoadAndRunFlow implements EnvironmentAware {
 
     private Environment env;
 
     private final String JOB_NAME = "yourJob";
 
     @Bean
-    public Job job(Step step) {
+    public Job job(JobBuilderFactory jobBuilderFactory, Step step) {
         return jobBuilderFactory.get(JOB_NAME).start(step).build();
     }
 
-    // This provider gives us the connection info for talking to MarkLogic
-    @Autowired
-    private DatabaseClientProvider databaseClientProvider;
-
     @Bean
     @JobScope
     public Step step(
-        @Value("#{jobParameters['input_file_path']}") String inputFilePath,
-        @Value("#{jobParameters['input_file_pattern']}") String inputFilePattern,
-        @Value("#{jobParameters['entity_name']}") String entityName,
-        @Value("#{jobParameters['flow_name']}") String flowName) {
-
+            StepBuilderFactory stepBuilderFactory,
+            DatabaseClientProvider databaseClientProvider,
+            @Value("#{jobParameters['input_file_path']}") String inputFilePath,
+            @Value("#{jobParameters['input_file_pattern']}") String inputFilePattern,
+            @Value("#{jobParameters['entity_name']}") String entityName,
+            @Value("#{jobParameters['flow_name']}") String flowName,
+            @Value("#{jobParameters['output_collections']}") String[] collections) {
 
         GenericDocumentManager docMgr = databaseClientProvider.getDatabaseClient().newDocumentManager();
 
@@ -71,7 +67,7 @@ public void write(List<? extends DocumentWriteOperation> items) throws Exception
         };
 
         return stepBuilderFactory.get("step1")
-            .<Resource, DocumentWriteOperation>chunk(getChunkSize())
+            .<Resource, DocumentWriteOperation>chunk(10)
             .reader(new EnhancedResourcesItemReader(inputFilePath, inputFilePattern))
             .processor(processor)
             .writer(writer)

marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/DeployHubDatabaseCommand.java

@@ -115,9 +115,9 @@ protected JsonNode mergeDatabaseFiles(AppConfig appConfig) {
         File userDatabaseDir = hubConfig.getUserDatabaseDir().toFile();
         files.add(new File(databaseDir, this.databaseFilename));
 
-        File otherDatabaseFile = new File(userDatabaseDir, this.databaseFilename);
-        if (otherDatabaseFile != null && otherDatabaseFile.exists()) {
-            files.add(otherDatabaseFile);
+        File userDatabaseFile = new File(userDatabaseDir, this.databaseFilename);
+        if (userDatabaseFile != null && userDatabaseFile.exists()) {
+            files.add(userDatabaseFile);
         }
         if (logger.isInfoEnabled()) {
             logger.info("Merging JSON files at locations: " + files);

marklogic-data-hub/src/main/java/com/marklogic/hub/flow/BatchCompleteListener.java

@@ -1,31 +1,6 @@
 {
   "database-name": "%%mlFinalDbName%%",
-  "range-element-index": [
-    {
-      "scalar-type": "dateTime",
-      "namespace-uri": "http://marklogic.com/xdmp/dls",
-      "localname": "created",
-      "collation": "",
-      "range-value-positions": false,
-      "invalid-values": "reject"
-    },
-    {
-      "scalar-type": "dateTime",
-      "namespace-uri": "http://marklogic.com/xdmp/dls",
-      "localname": "replaced",
-      "collation": "",
-      "range-value-positions": false,
-      "invalid-values": "reject"
-    },
-    {
-      "scalar-type": "unsignedLong",
-      "namespace-uri": "http://marklogic.com/xdmp/dls",
-      "localname": "version-id",
-      "collation": "",
-      "range-value-positions": false,
-      "invalid-values": "reject"
-    }
-  ],
+  "range-element-index": [],
   "schema-database": "%%SCHEMAS_DATABASE%%",
   "triggers-database": "%%TRIGGERS_DATABASE%%",
   "triple-index": true,

marklogic-data-hub/src/main/resources/ml-config/databases/final-database.json

@@ -1,31 +1,6 @@
 {
   "database-name": "%%mlStagingDbName%%",
-  "range-element-index": [
-    {
-      "scalar-type": "dateTime",
-      "namespace-uri": "http://marklogic.com/xdmp/dls",
-      "localname": "created",
-      "collation": "",
-      "range-value-positions": false,
-      "invalid-values": "reject"
-    },
-    {
-      "scalar-type": "dateTime",
-      "namespace-uri": "http://marklogic.com/xdmp/dls",
-      "localname": "replaced",
-      "collation": "",
-      "range-value-positions": false,
-      "invalid-values": "reject"
-    },
-    {
-      "scalar-type": "unsignedLong",
-      "namespace-uri": "http://marklogic.com/xdmp/dls",
-      "localname": "version-id",
-      "collation": "",
-      "range-value-positions": false,
-      "invalid-values": "reject"
-    }
-  ],
+  "range-element-index": [],
   "schema-database": "%%SCHEMAS_DATABASE%%",
   "triggers-database": "%%TRIGGERS_DATABASE%%",
   "triple-index": true,

marklogic-data-hub/src/main/resources/ml-config/databases/staging-database.json

@@ -1,31 +1,6 @@
 {
   "database-name": "%%mlTraceDbName%%",
-  "range-element-index": [
-    {
-      "scalar-type": "dateTime",
-      "namespace-uri": "http://marklogic.com/xdmp/dls",
-      "localname": "created",
-      "collation": "",
-      "range-value-positions": false,
-      "invalid-values": "reject"
-    },
-    {
-      "scalar-type": "dateTime",
-      "namespace-uri": "http://marklogic.com/xdmp/dls",
-      "localname": "replaced",
-      "collation": "",
-      "range-value-positions": false,
-      "invalid-values": "reject"
-    },
-    {
-      "scalar-type": "unsignedLong",
-      "namespace-uri": "http://marklogic.com/xdmp/dls",
-      "localname": "version-id",
-      "collation": "",
-      "range-value-positions": false,
-      "invalid-values": "reject"
-    }
-  ],
+  "range-element-index": [],
   "range-path-index": [
     {
       "scalar-type": "string",

marklogic-data-hub/src/main/resources/ml-config/databases/trace-database.json

@@ -7,12 +7,12 @@ import com.marklogic.gradle.exception.HubNotInstalledException
 import com.marklogic.hub.FlowManager
 import com.marklogic.hub.flow.Flow
 import com.marklogic.hub.flow.FlowRunner
-import com.marklogic.hub.flow.FlowStatusListener
 import com.marklogic.hub.flow.FlowType
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.TaskAction
 
 class RunFlowTask extends HubTask {
+
     @Input
     public String entityName
 
@@ -28,6 +28,9 @@ class RunFlowTask extends HubTask {
     @Input
     public Integer threadCount
 
+    @Input
+    public Boolean showOptions
+
     @TaskAction
     void runFlow() {
         if (entityName == null) {
@@ -54,6 +57,10 @@ class RunFlowTask extends HubTask {
             threadCount = project.hasProperty("threadCount") ?
                 Integer.parseInt(project.property("threadCount")) : 4
         }
+        if (showOptions == null) {
+            showOptions = project.hasProperty("showOptions") ?
+                Boolean.parseBoolean(project.property("showOptions")) : false
+        }
 
         if (!isHubInstalled()) {
             throw new HubNotInstalledException()
@@ -66,9 +73,24 @@ class RunFlowTask extends HubTask {
             throw new FlowNotFoundException(entityName, flowName);
         }
 
+        Map<String, Object> options = new HashMap<>()
+        project.ext.properties.each { key, value ->
+            if (key.toString().startsWith("dfh.")) {
+                options.put(key, value)
+            }
+        }
         println("Running Flow: [" + entityName + ":" + flowName + "] with batch size: " + batchSize)
+
+        if (showOptions) {
+            println("   and options:")
+            options.each { key, value ->
+                println("      " + key + " = " + value)
+            }
+        }
+
         FlowRunner flowRunner = fm.newFlowRunner()
             .withFlow(flow)
+            .withOptions(options)
             .withBatchSize(batchSize)
             .withThreadCount(threadCount)
         flowRunner.run()