Skip to content

Commit 5f4599b

Browse files
paxtonharepaxtonhare
committed
Merge pull request #90 from paxtonhare/72_define_search_options
72 define search options
2 parents 237fa42 + bb348be commit 5f4599b

File tree

45 files changed

+1285
-746
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+1285
-746
lines changed

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
.DS_Store
22
.gradle/
33
.settings/
4+
.metadata/
45
data-hub/bin/
56
quick-start/bin/
67
build/
@@ -9,6 +10,6 @@ releases/
910
.project
1011
gradle-local.properties
1112
/quick-start/environment.properties
12-
/quick-start/assetInstallTime.json
13+
/quick-start/assetInstallTime.properties
1314
/quick-start/input
1415
/quick-start/plugins

‎.metadata/.plugins/org.eclipse.ui.workbench/workingsets.xml‎

Lines changed: 0 additions & 4 deletions
This file was deleted.

‎data-hub/src/main/java/com/marklogic/hub/DataHub.java‎

Lines changed: 68 additions & 101 deletions
Original file line numberDiff line numberDiff line change
@@ -17,60 +17,56 @@
1717

1818
import java.io.File;
1919
import java.io.IOException;
20+
import java.nio.file.FileVisitResult;
21+
import java.nio.file.Files;
22+
import java.nio.file.Path;
23+
import java.nio.file.Paths;
24+
import java.nio.file.SimpleFileVisitor;
25+
import java.nio.file.attribute.BasicFileAttributes;
2026
import java.util.ArrayList;
21-
import java.util.Date;
22-
import java.util.HashMap;
27+
import java.util.HashSet;
2328
import java.util.List;
24-
import java.util.Map;
2529
import java.util.Set;
2630

27-
import org.apache.commons.io.FileUtils;
2831
import org.slf4j.Logger;
2932
import org.slf4j.LoggerFactory;
3033
import org.springframework.core.io.ClassPathResource;
31-
import org.springframework.core.io.Resource;
3234
import org.springframework.web.client.ResourceAccessException;
3335

34-
import com.fasterxml.jackson.core.JsonProcessingException;
35-
import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
36-
import com.fasterxml.jackson.databind.ObjectMapper;
37-
import com.fasterxml.jackson.databind.node.ObjectNode;
38-
import com.google.gson.Gson;
3936
import com.marklogic.appdeployer.AppConfig;
4037
import com.marklogic.appdeployer.ConfigDir;
4138
import com.marklogic.appdeployer.command.Command;
42-
import com.marklogic.appdeployer.command.appservers.UpdateRestApiServersCommand;
43-
//import com.marklogic.appdeployer.command.databases.DeployDatabaseCommand;
4439
import com.marklogic.appdeployer.command.databases.DeploySchemasDatabaseCommand;
4540
import com.marklogic.appdeployer.command.databases.DeployTriggersDatabaseCommand;
41+
import com.marklogic.appdeployer.command.modules.AllButAssetsModulesFinder;
4642
import com.marklogic.appdeployer.command.modules.AssetModulesFinder;
47-
import com.marklogic.appdeployer.command.restapis.DeployRestApiServersCommand;
4843
import com.marklogic.appdeployer.command.security.DeployRolesCommand;
4944
import com.marklogic.appdeployer.command.security.DeployUsersCommand;
5045
import com.marklogic.appdeployer.impl.SimpleAppDeployer;
51-
import com.marklogic.client.modulesloader.Modules;
52-
import com.marklogic.client.modulesloader.ModulesFinder;
53-
import com.marklogic.client.modulesloader.ModulesManager;
46+
import com.marklogic.client.DatabaseClient;
47+
import com.marklogic.client.DatabaseClientFactory;
48+
import com.marklogic.client.modulesloader.impl.PropertiesModuleManager;
49+
import com.marklogic.client.modulesloader.impl.XccAssetLoader;
5450
import com.marklogic.hub.commands.DeployHubDatabaseCommand;
5551
import com.marklogic.hub.commands.DeployModulesDatabaseCommand;
5652
import com.marklogic.hub.commands.DeployRestApiCommand;
5753
import com.marklogic.hub.commands.LoadModulesCommand;
58-
import com.marklogic.hub.util.GsonUtil;
54+
import com.marklogic.hub.util.HubFileFilter;
55+
import com.marklogic.hub.util.HubModulesLoader;
5956
import com.marklogic.mgmt.ManageClient;
6057
import com.marklogic.mgmt.ManageConfig;
6158
import com.marklogic.mgmt.admin.AdminConfig;
6259
import com.marklogic.mgmt.admin.AdminManager;
6360
import com.marklogic.mgmt.appservers.ServerManager;
6461
import com.marklogic.mgmt.databases.DatabaseManager;
65-
import com.marklogic.mgmt.restapis.RestApiManager;
6662
import com.marklogic.rest.util.Fragment;
6763

6864
public class DataHub {
6965

7066
static final private Logger LOGGER = LoggerFactory.getLogger(DataHub.class);
71-
static final private String STAGING_NAME = "data-hub-in-a-box-STAGING";
72-
static final private String FINAL_NAME = "data-hub-in-a-box-FINAL";
73-
static final private String MODULES_DB_NAME = "data-hub-in-a-box-modules";
67+
static final public String STAGING_NAME = "data-hub-in-a-box-STAGING";
68+
static final public String FINAL_NAME = "data-hub-in-a-box-FINAL";
69+
static final public String MODULES_DB_NAME = "data-hub-in-a-box-modules";
7470
private ManageConfig config;
7571
private ManageClient client;
7672
public static String HUB_NAME = "data-hub-in-a-box";
@@ -81,7 +77,7 @@ public class DataHub {
8177
private String username;
8278
private String password;
8379

84-
private File assetInstallTimeFile;
80+
private File assetInstallTimeFile = new File("./assetInstallTime.properties");
8581

8682
private final static int DEFAULT_STAGING_REST_PORT = 8010;
8783
private final static int DEFAULT_FINAL_REST_PORT = 8011;
@@ -191,6 +187,10 @@ private AppConfig getAppConfig() throws IOException {
191187
* @throws IOException
192188
*/
193189
public void install() throws IOException {
190+
// clean up any lingering cache for deployed modules
191+
PropertiesModuleManager moduleManager = new PropertiesModuleManager(this.assetInstallTimeFile);
192+
moduleManager.deletePropertiesFile();
193+
194194
AdminManager manager = new AdminManager();
195195
AppConfig config = getAppConfig();
196196
SimpleAppDeployer deployer = new SimpleAppDeployer(client, manager);
@@ -207,53 +207,56 @@ public void install() throws IOException {
207207
* with its install time
208208
* @throws IOException
209209
*/
210-
public Map<File, Date> installUserModules(String pathToUserModules) throws IOException {
210+
public Set<File> installUserModules(String pathToUserModules) throws IOException {
211211
AppConfig config = new AppConfig();
212212
config.setHost(host);
213-
config.setRestPort(stagingRestPort);
213+
config.setRestPort(finalRestPort);
214214
config.setName(HUB_NAME);
215215
config.setRestAdminUsername(username);
216216
config.setRestAdminPassword(password);
217217

218-
RestAssetLoader loader = new RestAssetLoader(config.newDatabaseClient());
219-
DataHubModuleManager modulesManager = new DataHubModuleManager();
220-
if (assetInstallTimeFile != null) {
221-
loader.setModulesManager(modulesManager);
222-
}
223-
224-
ModulesFinder finder = new AssetModulesFinder();
225-
Modules modules = finder.findModules(new File(pathToUserModules));
226-
227-
List<Resource> dirs = modules.getAssetDirectories();
228-
if (dirs == null || dirs.isEmpty()) {
229-
return new HashMap<>();
230-
}
231-
232-
String[] paths = new String[dirs.size()];
233-
for (int i = 0; i < dirs.size(); i++) {
234-
paths[i] = dirs.get(i).getFile().getAbsolutePath();
235-
}
236-
Set<File> loadedFiles = loader.loadAssetsViaREST(paths);
237-
238-
if (assetInstallTimeFile != null) {
239-
Gson gson = GsonUtil.createGson();
240-
241-
String json = gson.toJson(modulesManager.getLastInstallInfo());
242-
try {
243-
FileUtils.write(assetInstallTimeFile, json);
244-
} catch (IOException e) {
245-
LOGGER.error("Cannot write asset install info.", e);
218+
DatabaseClient stagingClient = DatabaseClientFactory.newClient(host, stagingRestPort, username, password,
219+
config.getRestAuthentication(), config.getRestSslContext(), config.getRestSslHostnameVerifier());
220+
DatabaseClient finalClient = DatabaseClientFactory.newClient(host, finalRestPort, username, password,
221+
config.getRestAuthentication(), config.getRestSslContext(), config.getRestSslHostnameVerifier());
222+
223+
224+
Set<File> loadedFiles = new HashSet<File>();
225+
226+
XccAssetLoader assetLoader = config.newXccAssetLoader();
227+
assetLoader.setFileFilter(new HubFileFilter());
228+
229+
PropertiesModuleManager moduleManager = new PropertiesModuleManager(this.assetInstallTimeFile);
230+
HubModulesLoader hubModulesLoader = new HubModulesLoader(assetLoader, moduleManager);
231+
File baseDir = Paths.get(pathToUserModules).normalize().toAbsolutePath().toFile();
232+
loadedFiles.addAll(hubModulesLoader.loadModules(baseDir, new AssetModulesFinder(), stagingClient));
233+
Path startPath = Paths.get(pathToUserModules, "entities");
234+
235+
Files.walkFileTree(startPath, new SimpleFileVisitor<Path>() {
236+
@Override
237+
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
238+
throws IOException
239+
{
240+
boolean isRest = dir.endsWith("REST");
241+
242+
String dirStr = dir.toString();
243+
boolean isInputDir = dirStr.matches(".*/input/.*");
244+
boolean isConformanceDir = dirStr.matches(".*/conformance/.*");
245+
if (isRest) {
246+
if (isInputDir) {
247+
loadedFiles.addAll(hubModulesLoader.loadModules(dir.normalize().toAbsolutePath().toFile(), new AllButAssetsModulesFinder(), stagingClient));
248+
}
249+
else if (isConformanceDir) {
250+
loadedFiles.addAll(hubModulesLoader.loadModules(dir.normalize().toAbsolutePath().toFile(), new AllButAssetsModulesFinder(), finalClient));
251+
}
252+
return FileVisitResult.SKIP_SUBTREE;
253+
}
254+
else {
255+
return FileVisitResult.CONTINUE;
256+
}
246257
}
247-
248-
return modulesManager.getLastInstallInfo();
249-
}
250-
else {
251-
Map<File, Date> fileMap = new HashMap<>();
252-
for (File file : loadedFiles) {
253-
fileMap.put(file, file.exists() ? new Date(file.lastModified()) : null);
254-
}
255-
return fileMap;
256-
}
258+
});
259+
return loadedFiles;
257260
}
258261

259262
private List<Command> getCommands(AppConfig config) {
@@ -302,45 +305,9 @@ public void uninstall() throws IOException {
302305
SimpleAppDeployer deployer = new SimpleAppDeployer(client, manager);
303306
deployer.setCommands(getCommands(config));
304307
deployer.undeploy(config);
305-
}
306-
307-
private class DataHubModuleManager implements ModulesManager {
308-
private Map<File, Date> lastInstallInfo = new HashMap<>();
309-
310-
public Map<File, Date> getLastInstallInfo() {
311-
return lastInstallInfo;
312-
}
313-
314-
@Override
315-
public void initialize() {
316-
LOGGER.debug("initializing DataHubModuleManager");
317-
}
318-
319-
@Override
320-
public boolean hasFileBeenModifiedSinceLastInstalled(File file) {
321-
Date lastInstallDate = null;
322-
try {
323-
lastInstallDate = lastInstallInfo.get(new File(file.getCanonicalPath()));
324-
} catch (IOException e) {
325-
LOGGER.warn("Cannot get canonical path of {}. Using absolute path instead.", file);
326-
lastInstallDate = lastInstallInfo.get(new File(file.getAbsolutePath()));
327-
}
328-
329-
// a file has been modified if it has not been previously installed (new file)
330-
// or when its modified time is after the last install time
331-
return lastInstallDate == null || file.lastModified() > lastInstallDate.getTime();
332-
}
333-
334-
@Override
335-
public void saveLastInstalledTimestamp(File file, Date date) {
336-
try {
337-
LOGGER.trace("saving last timestamp of " + file.getCanonicalPath() + ": " + date);
338-
lastInstallInfo.put(new File(file.getCanonicalPath()), date);
339-
} catch (IOException e) {
340-
LOGGER.warn("Cannot store canonical path of {}. Storing absolute path instead.", file);
341-
lastInstallInfo.put(new File(file.getAbsolutePath()), date);
342-
}
343-
}
344308

309+
// clean up any lingering cache for deployed modules
310+
PropertiesModuleManager moduleManager = new PropertiesModuleManager(this.assetInstallTimeFile);
311+
moduleManager.deletePropertiesFile();
345312
}
346313
}

‎data-hub/src/main/java/com/marklogic/hub/DataHubContentPump.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
package com.marklogic.hub;
22

3-
import java.io.File;
43
import java.util.List;
54

65
import com.marklogic.contentpump.ContentPump;
@@ -17,10 +16,10 @@ public DataHubContentPump(List<String> arguments) {
1716
public DataHubContentPump(String[] arguments) {
1817
this.arguments = arguments;
1918
}
20-
19+
2120
/**
2221
* Run the Content Pump.
23-
*
22+
*
2423
* @return true if the content pump executed successfully, false otherwise.
2524
*/
2625
public boolean execute() {
@@ -34,7 +33,7 @@ public boolean execute() {
3433
System.err.println(ex.getMessage());
3534
System.err.println("Try 'mlcp help' for usage.");
3635
}
37-
36+
3837
return rc == 0;
3938
}
4039
}

‎data-hub/src/main/java/com/marklogic/hub/RestAssetLoader.java‎

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,7 @@
3131
import org.apache.commons.io.FilenameUtils;
3232

3333
import com.marklogic.client.DatabaseClient;
34-
import com.marklogic.client.admin.ExtensionLibrariesManager;
35-
import com.marklogic.client.admin.ExtensionLibraryDescriptor;
34+
import com.marklogic.client.Transaction;
3635
import com.marklogic.client.helper.LoggingObject;
3736
import com.marklogic.client.io.FileHandle;
3837
import com.marklogic.client.io.Format;
@@ -56,6 +55,7 @@ public class RestAssetLoader extends LoggingObject implements FileVisitor<Path>
5655
private Path currentAssetPath;
5756
private Path currentRootPath;
5857
private Set<File> filesLoaded;
58+
private Transaction currentTransaction = null;
5959

6060
private ModulesManager modulesManager;
6161

@@ -67,18 +67,25 @@ public RestAssetLoader(DatabaseClient client) {
6767
* For walking one or many paths and loading modules in each of them.
6868
*/
6969
public Set<File> loadAssetsViaREST(String... paths) {
70+
currentTransaction = client.openTransaction();
7071
filesLoaded = new HashSet<>();
71-
for (String path : paths) {
72-
if (logger.isDebugEnabled()) {
73-
logger.debug(format("Loading assets from path: %s", path));
74-
}
75-
this.currentAssetPath = Paths.get(path);
76-
this.currentRootPath = this.currentAssetPath;
77-
try {
78-
Files.walkFileTree(this.currentAssetPath, this);
79-
} catch (IOException ie) {
80-
throw new RuntimeException(format("Error while walking assets file tree: %s", ie.getMessage()), ie);
72+
try {
73+
for (String path : paths) {
74+
if (logger.isDebugEnabled()) {
75+
logger.debug(format("Loading assets from path: %s", path));
76+
}
77+
this.currentAssetPath = Paths.get(path);
78+
this.currentRootPath = this.currentAssetPath;
79+
try {
80+
Files.walkFileTree(this.currentAssetPath, this);
81+
} catch (IOException ie) {
82+
throw new RuntimeException(format("Error while walking assets file tree: %s", ie.getMessage()), ie);
83+
}
8184
}
85+
currentTransaction.commit();
86+
}
87+
catch(Exception e) {
88+
currentTransaction.rollback();
8289
}
8390
return filesLoaded;
8491
}
@@ -137,12 +144,6 @@ protected void loadFile(String uri, File f) {
137144
return;
138145
}
139146

140-
ExtensionLibrariesManager libsMgr = client
141-
.newServerConfigManager().newExtensionLibrariesManager();
142-
143-
ExtensionLibraryDescriptor moduleDescriptor = new ExtensionLibraryDescriptor();
144-
moduleDescriptor.setPath(uri);
145-
146147
FileHandle handle = new FileHandle(f);
147148

148149
String ext = FilenameUtils.getExtension(f.getName());
@@ -157,7 +158,7 @@ protected void loadFile(String uri, File f) {
157158
handle.setFormat(Format.TEXT);
158159
}
159160

160-
libsMgr.write(moduleDescriptor, handle);
161+
client.newDocumentManager().write(uri, handle, currentTransaction);
161162

162163
if (modulesManager != null) {
163164
modulesManager.saveLastInstalledTimestamp(f, new Date());

‎data-hub/src/main/java/com/marklogic/hub/commands/JarExtensionMetadataProvider.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,6 @@ public ExtensionMetadataAndParams provideExtensionMetadataAndParams(Resource r)
6161
}
6262
}
6363
} catch (Exception e) {
64-
logger.warn("Unable to build metadata from resource file: " + r.getURL().toString()
65-
+ "; cause: " + e.getMessage(), e);
6664
setDefaults(m, r);
6765
}
6866
} else {

0 commit comments

Comments
 (0)