Merge remote-tracking branch 'origin/master' into 72_define_search_options

# Conflicts:
#	quick-start/src/main/java/com/marklogic/hub/factory/DomainModelFactory.java
#	quick-start/src/main/java/com/marklogic/hub/service/DomainManagerService.java
#	quick-start/src/main/java/com/marklogic/hub/service/SyncStatusService.java
#	quick-start/src/main/java/com/marklogic/hub/web/controller/api/DataHubServerApiController.java

Author: paxtonhare

.gitignore

@@ -1,6 +1,7 @@
 .DS_Store
 .gradle/
 .settings/
+.metadata/
 data-hub/bin/
 quick-start/bin/
 build/

README.md

@@ -11,6 +11,8 @@ This project allows you to deploy a skeleton Data Hub into MarkLogic. With some
 - create a data-hub/gradle-local.properties file and point it to your MarkLogic server _(see data-hub/gradle-local.sample)_
 - run ```./gradlew mlDeploy```
 
+  - currently, use quickstart to deploy. just run ./gradlew build -x test to build, then use quickstart
+
 ## Running Tests
 - run ```./gradlew test```
 

data-hub/src/main/java/com/marklogic/hub/DataHub.java

@@ -45,12 +45,13 @@
 import com.marklogic.appdeployer.impl.SimpleAppDeployer;
 import com.marklogic.client.DatabaseClient;
 import com.marklogic.client.DatabaseClientFactory;
-import com.marklogic.client.modulesloader.impl.DefaultModulesLoader;
 import com.marklogic.client.modulesloader.impl.PropertiesModuleManager;
+import com.marklogic.client.modulesloader.impl.XccAssetLoader;
 import com.marklogic.hub.commands.DeployHubDatabaseCommand;
 import com.marklogic.hub.commands.DeployModulesDatabaseCommand;
 import com.marklogic.hub.commands.DeployRestApiCommand;
 import com.marklogic.hub.commands.LoadModulesCommand;
+import com.marklogic.hub.util.HubFileFilter;
 import com.marklogic.hub.util.HubModulesLoader;
 import com.marklogic.mgmt.ManageClient;
 import com.marklogic.mgmt.ManageConfig;
@@ -63,9 +64,9 @@
 public class DataHub {
 
     static final private Logger LOGGER = LoggerFactory.getLogger(DataHub.class);
-    static final private String STAGING_NAME = "data-hub-in-a-box-STAGING";
-    static final private String FINAL_NAME = "data-hub-in-a-box-FINAL";
-    static final private String MODULES_DB_NAME = "data-hub-in-a-box-modules";
+    static final public String STAGING_NAME = "data-hub-in-a-box-STAGING";
+    static final public String FINAL_NAME = "data-hub-in-a-box-FINAL";
+    static final public String MODULES_DB_NAME = "data-hub-in-a-box-modules";
     private ManageConfig config;
     private ManageClient client;
     public static String HUB_NAME = "data-hub-in-a-box";
@@ -76,7 +77,7 @@ public class DataHub {
     private String username;
     private String password;
 
-    private File assetInstallTimeFile;
+    private File assetInstallTimeFile = new File("./assetInstallTime.properties");
 
     private final static int DEFAULT_STAGING_REST_PORT = 8010;
     private final static int DEFAULT_FINAL_REST_PORT = 8011;
@@ -186,6 +187,10 @@ private AppConfig getAppConfig() throws IOException {
      * @throws IOException
      */
     public void install() throws IOException {
+        // clean up any lingering cache for deployed modules
+        PropertiesModuleManager moduleManager = new PropertiesModuleManager(this.assetInstallTimeFile);
+        moduleManager.deletePropertiesFile();
+
         AdminManager manager = new AdminManager();
         AppConfig config = getAppConfig();
         SimpleAppDeployer deployer = new SimpleAppDeployer(client, manager);
@@ -215,39 +220,37 @@ public Set<File> installUserModules(String pathToUserModules) throws IOException
         DatabaseClient finalClient = DatabaseClientFactory.newClient(host, finalRestPort, username, password,
                 config.getRestAuthentication(), config.getRestSslContext(), config.getRestSslHostnameVerifier());
 
-        PropertiesModuleManager modulesManager = new PropertiesModuleManager(assetInstallTimeFile);
 
         Set<File> loadedFiles = new HashSet<File>();
 
-        DefaultModulesLoader modulesLoader = new DefaultModulesLoader(config.newXccAssetLoader());
-        HubModulesLoader hubModulesLoader = new HubModulesLoader(config.newXccAssetLoader());
-        if (assetInstallTimeFile != null) {
-            modulesLoader.setModulesManager(modulesManager);
-            hubModulesLoader.setModulesManager(modulesManager);
-        }
-        Path startPath = Paths.get(pathToUserModules, "domains");
+        XccAssetLoader assetLoader = config.newXccAssetLoader();
+        assetLoader.setFileFilter(new HubFileFilter());
+
+        PropertiesModuleManager moduleManager = new PropertiesModuleManager(this.assetInstallTimeFile);
+        HubModulesLoader hubModulesLoader = new HubModulesLoader(assetLoader, moduleManager);
+        File baseDir = Paths.get(pathToUserModules).normalize().toAbsolutePath().toFile();
+        loadedFiles.addAll(hubModulesLoader.loadModules(baseDir, new AssetModulesFinder(), stagingClient));
+        Path startPath = Paths.get(pathToUserModules, "entities");
+
         Files.walkFileTree(startPath, new SimpleFileVisitor<Path>() {
             @Override
             public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
                 throws IOException
             {
-                boolean isRest = dir.endsWith("REST") || dir.endsWith("rest");
+                boolean isRest = dir.endsWith("REST");
+
                 String dirStr = dir.toString();
                 boolean isInputDir = dirStr.matches(".*/input/.*");
-                boolean isConformanceDir = dirStr.matches(".*conformance/.*");
+                boolean isConformanceDir = dirStr.matches(".*/conformance/.*");
                 if (isRest) {
                     if (isInputDir) {
-                        loadedFiles.addAll(modulesLoader.loadModules(dir.normalize().toAbsolutePath().toFile(), new AllButAssetsModulesFinder(), stagingClient));
+                        loadedFiles.addAll(hubModulesLoader.loadModules(dir.normalize().toAbsolutePath().toFile(), new AllButAssetsModulesFinder(), stagingClient));
                     }
                     else if (isConformanceDir) {
-                        loadedFiles.addAll(modulesLoader.loadModules(dir.normalize().toAbsolutePath().toFile(), new AllButAssetsModulesFinder(), finalClient));
+                        loadedFiles.addAll(hubModulesLoader.loadModules(dir.normalize().toAbsolutePath().toFile(), new AllButAssetsModulesFinder(), finalClient));
                     }
                     return FileVisitResult.SKIP_SUBTREE;
                 }
-                else if (isInputDir || isConformanceDir) {
-                    loadedFiles.addAll(hubModulesLoader.loadModules(dir.normalize().toAbsolutePath().toFile(), new AssetModulesFinder(), stagingClient));
-                    return FileVisitResult.SKIP_SUBTREE;
-                }
                 else {
                     return FileVisitResult.CONTINUE;
                 }
@@ -302,5 +305,9 @@ public void uninstall() throws IOException {
         SimpleAppDeployer deployer = new SimpleAppDeployer(client, manager);
         deployer.setCommands(getCommands(config));
         deployer.undeploy(config);
+
+        // clean up any lingering cache for deployed modules
+        PropertiesModuleManager moduleManager = new PropertiesModuleManager(this.assetInstallTimeFile);
+        moduleManager.deletePropertiesFile();
     }
 }

data-hub/src/main/java/com/marklogic/hub/DataHubContentPump.java

@@ -1,6 +1,5 @@
 package com.marklogic.hub;
 
-import java.io.File;
 import java.util.List;
 
 import com.marklogic.contentpump.ContentPump;
@@ -17,10 +16,10 @@ public DataHubContentPump(List<String> arguments) {
     public DataHubContentPump(String[] arguments) {
         this.arguments = arguments;
     }
-    
+
     /**
      * Run the Content Pump.
-     * 
+     *
      * @return true if the content pump executed successfully, false otherwise.
      */
     public boolean execute() {
@@ -34,7 +33,7 @@ public boolean execute() {
             System.err.println(ex.getMessage());
             System.err.println("Try 'mlcp help' for usage.");
         }
-        
+
         return rc == 0;
     }
 }

data-hub/src/main/java/com/marklogic/hub/DomainManager.java renamed to data-hub/src/main/java/com/marklogic/hub/EntityManager.java

@@ -29,24 +29,25 @@
 import com.marklogic.client.extensions.ResourceServices.ServiceResultIterator;
 import com.marklogic.client.io.DOMHandle;
 import com.marklogic.client.util.RequestParameters;
-import com.marklogic.hub.domain.Domain;
-import com.marklogic.hub.domain.DomainImpl;
+import com.marklogic.hub.entity.Entity;
+import com.marklogic.hub.entity.EntityImpl;
 
-public class DomainManager extends ResourceManager {
-    static final public String NAME = "domain";
+public class EntityManager extends ResourceManager {
+    static final public String NAME = "entity";
     private DatabaseClient client;
 
-    public DomainManager(DatabaseClient client) {
+    public EntityManager(DatabaseClient client) {
         super();
         this.client = client;
         this.client.init(NAME, this);
     }
 
     /**
-     * Retrieves a list of domains from the Server
-     * @return a list of domains
+     * Retrieves a list of entities from the Server
+     * 
+     * @return a list of entities
      */
-    public List<Domain> getDomains() {
+    public List<Entity> getEntities() {
         RequestParameters params = new RequestParameters();
         ServiceResultIterator resultItr = this.getServices().get(params);
         if (resultItr == null || ! resultItr.hasNext()) {
@@ -57,42 +58,44 @@ public List<Domain> getDomains() {
         Document parent = res.getContent(handle).get();
         NodeList children = parent.getDocumentElement().getChildNodes();
 
-        ArrayList<Domain> domains = null;
+        ArrayList<Entity> entities = null;
         if (children.getLength() > 0) {
-            domains = new ArrayList<Domain>();
+            entities = new ArrayList<Entity>();
         }
 
         Node node;
         for (int i = 0; i < children.getLength(); i++) {
             node = children.item(i);
             if (node.getNodeType() == Node.ELEMENT_NODE) {
-                domains.add(domainFromXml((Element)children.item(i)));
+                entities.add(entityFromXml((Element) children.item(i)));
             }
         }
-        return domains;
+        return entities;
     }
 
     /**
-     * Retrieve a named domain
-     * @param domainName - the name of the domain to retrieve
-     * @return a domain
+     * Retrieve a named entity
+     * 
+     * @param entityName
+     *            - the name of the entity to retrieve
+     * @return a entity
      */
-    public Domain getDomain(String domainName) {
+    public Entity getEntity(String entityName) {
         RequestParameters params = new RequestParameters();
-        params.add("domain-name", domainName);
+        params.add("entity-name", entityName);
         ServiceResultIterator resultItr = this.getServices().get(params);
         if (resultItr == null || ! resultItr.hasNext()) {
             return null;
         }
         ServiceResult res = resultItr.next();
         DOMHandle handle = new DOMHandle();
         Document parent = res.getContent(handle).get();
-        return new DomainImpl(parent.getDocumentElement());
+        return new EntityImpl(parent.getDocumentElement());
     }
 
 
-    private Domain domainFromXml(Element doc) {
-        Domain d = new DomainImpl(doc);
+    private Entity entityFromXml(Element doc) {
+        Entity d = new EntityImpl(doc);
         return d;
     }
 }

data-hub/src/main/java/com/marklogic/hub/FlowManager.java

@@ -24,6 +24,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.batch.core.Job;
+import org.springframework.batch.core.JobExecution;
 import org.springframework.batch.core.JobExecutionListener;
 import org.springframework.batch.core.JobParameters;
 import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
@@ -116,12 +117,14 @@ protected void initializeDefaultSpringBatchComponents() {
 
     /**
      * Retrieves a list of flows installed on the MarkLogic server
-     * @param domainName - the domain from which to fetch the flows
-     * @return - a list of flows for the given domain
+     * 
+     * @param entityName
+     *            - the entity from which to fetch the flows
+     * @return - a list of flows for the given entity
      */
-    public List<Flow> getFlows(String domainName) {
+    public List<Flow> getFlows(String entityName) {
         RequestParameters params = new RequestParameters();
-        params.add("domain-name", domainName);
+        params.add("entity-name", entityName);
         ServiceResultIterator resultItr = this.getServices().get(params);
         if (resultItr == null || ! resultItr.hasNext()) {
             return null;
@@ -147,18 +150,21 @@ public List<Flow> getFlows(String domainName) {
     }
 
     /**
-     * Retrieves a named flow from a given domain
-     * @param domainName - the domain that the flow belongs to
-     * @param flowName - the name of the flow to get
+     * Retrieves a named flow from a given entity
+     * 
+     * @param entityName
+     *            - the entity that the flow belongs to
+     * @param flowName
+     *            - the name of the flow to get
      * @return the flow
      */
-    public Flow getFlow(String domainName, String flowName) {
-        return getFlow(domainName, flowName, null);
+    public Flow getFlow(String entityName, String flowName) {
+        return getFlow(entityName, flowName, null);
     }
 
-    public Flow getFlow(String domainName, String flowName, FlowType flowType) {
+    public Flow getFlow(String entityName, String flowName, FlowType flowType) {
         RequestParameters params = new RequestParameters();
-        params.add("domain-name", domainName);
+        params.add("entity-name", entityName);
         params.add("flow-name", flowName);
         if (flowType != null) {
             params.add("flow-type", flowType.toString());
@@ -181,8 +187,8 @@ public void uninstallFlow(String flowName) {
 
     }
 
-    public void runFlow(Flow flow, int batchSize) {
-        runFlow(flow, batchSize, null);
+    public JobExecution runFlow(Flow flow, int batchSize) {
+        return runFlow(flow, batchSize, null);
     }
 
     // might want to add Job tracking support
@@ -193,8 +199,9 @@ public void runFlow(Flow flow, int batchSize) {
      * @param flow - the flow to run
      * @param batchSize - the size to use for batching transactions
      * @param listener - the JobExecutionListener to receive status updates about the job
+     * @return 
      */
-    public void runFlow(Flow flow, int batchSize, JobExecutionListener listener) {
+    public JobExecution runFlow(Flow flow, int batchSize, JobExecutionListener listener) {
         Collector c = flow.getCollector();
         if (c instanceof ServerCollector) {
             ((ServerCollector)c).setClient(client);
@@ -206,15 +213,16 @@ public void runFlow(Flow flow, int batchSize, JobExecutionListener listener) {
                 .<String, String> chunk(batchSize)
                 .reader(reader).writer(writer).build();
 
-        String jobName = flow.getDomainName() + ":" + flow.getType() + ":" + flow.getName() + "-" + System.currentTimeMillis();
+        String jobName = flow.getEntityName() + ":" + flow.getType() + ":"
+                + flow.getName() + "-" + System.currentTimeMillis();
         SimpleJobBuilder builder = jobBuilderFactory.get(jobName).start(step);
         if (listener != null) {
             builder = builder.listener(listener);
         }
         Job job = builder.build();
 
         try {
-            jobLauncher.run(job, new JobParameters());
+            return jobLauncher.run(job, new JobParameters());
         }
         catch (Exception e) {
             throw new RuntimeException(e);
@@ -224,12 +232,16 @@ public void runFlow(Flow flow, int batchSize, JobExecutionListener listener) {
     public void runInputFlow(Flow flow, HubConfig config) {
         try {
             Mlcp mlcp = new Mlcp(config.getHost(), config.getStagingPort(), config.getAdminUsername(), config.getAdminPassword());
-            SourceOptions sourceOptions = new SourceOptions(flow.getDomainName(), flow.getName(), FlowType.INPUT.toString());
+            SourceOptions sourceOptions = new SourceOptions(
+                    flow.getEntityName(), flow.getName(),
+                    FlowType.INPUT.toString());
             mlcp.addSourceDirectory(config.getModulesPath(), sourceOptions);
             mlcp.loadContent();
         }
         catch (IOException e) {
-            LOGGER.error("Error encountered while trying to run flow:  " + flow.getDomainName() + " > " + flow.getName(), e);
+            LOGGER.error(
+                    "Error encountered while trying to run flow:  "
+                            + flow.getEntityName() + " > " + flow.getName(), e);
         }
     }
 
@@ -286,7 +298,7 @@ public void run(Flow flow, String identifier, Transaction transaction) {
             this.getServices().post(params, handle, transaction);
         }
     }
-	public void testFlow(Flow flow) {
-
+	public JobExecution testFlow(Flow flow) {
+	    return null;
 	}
 }