Skip to content

Commit 4beb3e5

Browse files
paxtonharepaxtonhare
committed
Merge pull request #49 from maeisabelle/Quick-Start-SynchingAndUI
Quick start synching and ui
2 parents 094b062 + 0fcc5e0 commit 4beb3e5

File tree

20 files changed

+640
-285
lines changed

20 files changed

+640
-285
lines changed

‎data-hub/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ dependencies {
2525
compile 'com.marklogic:ml-javaclient-util:2.6'
2626
compile 'com.marklogic:ml-app-deployer:2.0'
2727
compile 'commons-io:commons-io:2.4'
28+
compile 'com.google.code.gson:gson:2.6.1'
2829
testCompile 'junit:junit:4.12'
2930
testCompile 'xmlunit:xmlunit:1.3'
3031
testCompile 'org.hamcrest:hamcrest-junit:2.0.0.0'

‎data-hub/src/main/java/com/marklogic/hub/DataHub.java‎

Lines changed: 86 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,16 +18,20 @@
1818
import java.io.File;
1919
import java.io.IOException;
2020
import java.util.ArrayList;
21-
import java.util.HashSet;
21+
import java.util.Date;
22+
import java.util.HashMap;
2223
import java.util.List;
24+
import java.util.Map;
2325
import java.util.Set;
2426

27+
import org.apache.commons.io.FileUtils;
2528
import org.slf4j.Logger;
2629
import org.slf4j.LoggerFactory;
2730
import org.springframework.core.io.ClassPathResource;
2831
import org.springframework.core.io.Resource;
2932
import org.springframework.web.client.ResourceAccessException;
3033

34+
import com.google.gson.Gson;
3135
import com.marklogic.appdeployer.AppConfig;
3236
import com.marklogic.appdeployer.ConfigDir;
3337
import com.marklogic.appdeployer.command.Command;
@@ -42,7 +46,10 @@
4246
import com.marklogic.appdeployer.impl.SimpleAppDeployer;
4347
import com.marklogic.client.modulesloader.Modules;
4448
import com.marklogic.client.modulesloader.ModulesFinder;
49+
import com.marklogic.client.modulesloader.ModulesManager;
50+
import com.marklogic.client.modulesloader.impl.DefaultModulesLoader;
4551
import com.marklogic.hub.commands.LoadModulesCommand;
52+
import com.marklogic.hub.util.GsonUtil;
4653
import com.marklogic.mgmt.ManageClient;
4754
import com.marklogic.mgmt.ManageConfig;
4855
import com.marklogic.mgmt.admin.AdminConfig;
@@ -51,7 +58,7 @@
5158

5259
public class DataHub {
5360

54-
static final private Logger logger = LoggerFactory.getLogger(DataHub.class);
61+
static final private Logger LOGGER = LoggerFactory.getLogger(DataHub.class);
5562

5663
private ManageConfig config;
5764
private ManageClient client;
@@ -62,6 +69,8 @@ public class DataHub {
6269
private String username;
6370
private String password;
6471

72+
private File assetInstallTimeFile;
73+
6574
private final static int DEFAULT_REST_PORT = 8010;
6675

6776
public DataHub(HubConfig config) {
@@ -80,7 +89,11 @@ public DataHub(String host, int restPort, String username, String password) {
8089
this.username = username;
8190
this.password = password;
8291
}
83-
92+
93+
public void setAssetInstallTimeFile(File assetInstallTimeFile) {
94+
this.assetInstallTimeFile = assetInstallTimeFile;
95+
}
96+
8497
/**
8598
* Determines if the data hub is installed in MarkLogic
8699
* @return true if installed, false otherwise
@@ -142,11 +155,13 @@ public void install() throws IOException {
142155
/**
143156
* Installs User Provided modules into the Data Hub
144157
*
145-
* @param pathToUserModules - the absolute path to the user's modules folder
146-
* @return the set of files that was loaded into MarkLogic
158+
* @param pathToUserModules
159+
* - the absolute path to the user's modules folder
160+
* @return the canonical/absolute path of files that was loaded, together
161+
* with its install time
147162
* @throws IOException
148163
*/
149-
public Set<File> installUserModules(String pathToUserModules) throws IOException {
164+
public Map<File, Date> installUserModules(String pathToUserModules) throws IOException {
150165
AppConfig config = new AppConfig();
151166
config.setHost(host);
152167
config.setRestPort(restPort);
@@ -155,20 +170,44 @@ public Set<File> installUserModules(String pathToUserModules) throws IOException
155170
config.setRestAdminPassword(password);
156171

157172
RestAssetLoader loader = new RestAssetLoader(config.newDatabaseClient());
173+
DataHubModuleManager modulesManager = new DataHubModuleManager();
174+
if (assetInstallTimeFile != null) {
175+
loader.setModulesManager(modulesManager);
176+
}
158177

159178
ModulesFinder finder = new AssetModulesFinder();
160179
Modules modules = finder.findModules(new File(pathToUserModules));
161180

162181
List<Resource> dirs = modules.getAssetDirectories();
163182
if (dirs == null || dirs.isEmpty()) {
164-
return new HashSet<File>();
183+
return new HashMap<>();
165184
}
166185

167186
String[] paths = new String[dirs.size()];
168187
for (int i = 0; i < dirs.size(); i++) {
169188
paths[i] = dirs.get(i).getFile().getAbsolutePath();
170189
}
171-
return loader.loadAssetsViaREST(paths);
190+
Set<File> loadedFiles = loader.loadAssetsViaREST(paths);
191+
192+
if (assetInstallTimeFile != null) {
193+
Gson gson = GsonUtil.createGson();
194+
195+
String json = gson.toJson(modulesManager.getLastInstallInfo());
196+
try {
197+
FileUtils.write(assetInstallTimeFile, json);
198+
} catch (IOException e) {
199+
LOGGER.error("Cannot write asset install info.", e);
200+
}
201+
202+
return modulesManager.getLastInstallInfo();
203+
}
204+
else {
205+
Map<File, Date> fileMap = new HashMap<>();
206+
for (File file : loadedFiles) {
207+
fileMap.put(file, file.exists() ? new Date(file.lastModified()) : null);
208+
}
209+
return fileMap;
210+
}
172211
}
173212

174213
private List<Command> getCommands(AppConfig config) {
@@ -214,5 +253,44 @@ public void uninstall() throws IOException {
214253
deployer.setCommands(getCommands(config));
215254
deployer.undeploy(config);
216255
}
256+
257+
private class DataHubModuleManager implements ModulesManager {
258+
private Map<File, Date> lastInstallInfo = new HashMap<>();
259+
260+
public Map<File, Date> getLastInstallInfo() {
261+
return lastInstallInfo;
262+
}
263+
264+
@Override
265+
public void initialize() {
266+
LOGGER.debug("initializing DataHubModuleManager");
267+
}
268+
269+
@Override
270+
public boolean hasFileBeenModifiedSinceLastInstalled(File file) {
271+
Date lastInstallDate = null;
272+
try {
273+
lastInstallDate = lastInstallInfo.get(new File(file.getCanonicalPath()));
274+
} catch (IOException e) {
275+
LOGGER.warn("Cannot get canonical path of {}. Using absolute path instead.", file);
276+
lastInstallDate = lastInstallInfo.get(new File(file.getAbsolutePath()));
277+
}
278+
279+
// a file has been modified if it has not been previously installed (new file)
280+
// or when its modified time is after the last install time
281+
return lastInstallDate == null || file.lastModified() > lastInstallDate.getTime();
282+
}
217283

284+
@Override
285+
public void saveLastInstalledTimestamp(File file, Date date) {
286+
try {
287+
LOGGER.trace("saving last timestamp of " + file.getCanonicalPath() + ": " + date);
288+
lastInstallInfo.put(new File(file.getCanonicalPath()), date);
289+
} catch (IOException e) {
290+
LOGGER.warn("Cannot store canonical path of {}. Storing absolute path instead.", file);
291+
lastInstallInfo.put(new File(file.getAbsolutePath()), date);
292+
}
293+
}
294+
295+
}
218296
}

‎data-hub/src/main/java/com/marklogic/hub/util/GsonUtil.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package com.marklogic.hub.util;
2+
3+
import java.lang.reflect.Type;
4+
import java.text.SimpleDateFormat;
5+
import java.util.Date;
6+
7+
import com.google.gson.Gson;
8+
import com.google.gson.GsonBuilder;
9+
import com.google.gson.JsonElement;
10+
import com.google.gson.JsonPrimitive;
11+
import com.google.gson.JsonSerializationContext;
12+
import com.google.gson.JsonSerializer;
13+
14+
public class GsonUtil {
15+
16+
public static Gson createGson() {
17+
GsonBuilder builder = new GsonBuilder();
18+
builder.setDateFormat("yyyy-MM-dd hh:mm:ss");
19+
builder.registerTypeAdapter(Date.class, new JsonSerializer<Date>() {
20+
@Override
21+
public JsonElement serialize(Date time, Type type, JsonSerializationContext context) {
22+
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
23+
JsonPrimitive json = new JsonPrimitive(sdf.format(time));
24+
25+
return json;
26+
}
27+
});
28+
builder.setPrettyPrinting();
29+
30+
return builder.create();
31+
}
32+
}

‎quick-start/src/main/java/com/marklogic/hub/config/EnvironmentConfiguration.java‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -116,17 +116,30 @@ public String getUserPluginDir() {
116116
}
117117

118118
public String getMlcpHomeDir() {
119-
String value = this.properties.getProperty("mlcpHome");
119+
String value = this.properties.getProperty("mlcpHomeDir");
120120
if (value != null) {
121121
return value;
122122
}
123-
value = this.environment.getProperty("mlcpHome");
123+
value = this.environment.getProperty("mlcpHomeDir");
124124
if (value != null) {
125-
this.properties.setProperty("mlcpHome", value);
125+
this.properties.setProperty("mlcpHomeDir", value);
126126
return value;
127127
}
128128
return "./mlcp";
129129
}
130+
131+
public String getAssetInstallTimeFilePath() {
132+
String value = this.properties.getProperty("assetInstallTimeFile");
133+
if (value != null) {
134+
return value;
135+
}
136+
value = this.environment.getProperty("assetInstallTimeFile");
137+
if (value != null) {
138+
this.properties.setProperty("assetInstallTimeFile", value);
139+
return value;
140+
}
141+
return "./assetInstallTime.json";
142+
}
130143

131144
public void setMLHost(String mlHost) {
132145
this.properties.setProperty("mlHost", mlHost);
@@ -148,8 +161,12 @@ public void setUserPluginDir(String userPluginDir) {
148161
this.properties.setProperty("userPluginDir", userPluginDir);
149162
}
150163

151-
public void setMlcpHome(String mlcpHomeDir) {
152-
this.properties.setProperty("mlcpHome", mlcpHomeDir);
164+
public void setMlcpHomeDir(String mlcpHomeDir) {
165+
this.properties.setProperty("mlcpHomeDir", mlcpHomeDir);
166+
}
167+
168+
public void setAssetInstallTimeFilePath(String assetInstallTimeFilePath) {
169+
this.properties.setProperty("assetInstallTimeFile", assetInstallTimeFilePath);
153170
}
154171

155172
public void loadConfigurationFromFile() {

‎quick-start/src/main/java/com/marklogic/hub/model/RunFlowModel.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
public class RunFlowModel {
44
private String domainName;
55
private String flowName;
6+
private String inputPath;
67

78
public String getDomainName() {
89
return domainName;
@@ -12,4 +13,7 @@ public String getFlowName() {
1213
return flowName;
1314
}
1415

16+
public String getInputPath() {
17+
return inputPath;
18+
}
1519
}

‎quick-start/src/main/java/com/marklogic/hub/service/DataHubService.java‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
package com.marklogic.hub.service;
22

33
import java.io.File;
4-
import java.util.Set;
4+
import java.util.Date;
5+
import java.util.Map;
56

67
import org.slf4j.Logger;
78
import org.slf4j.LoggerFactory;
@@ -30,7 +31,7 @@ public void install() throws DataHubException {
3031
}
3132
}
3233

33-
public Set<File> installUserModules() throws DataHubException {
34+
public Map<File, Date> installUserModules() throws DataHubException {
3435
DataHub dataHub = getDataHub();
3536
try {
3637
return dataHub.installUserModules(environmentConfiguration.getUserPluginDir());
@@ -47,8 +48,11 @@ private DataHub getDataHub() throws DataHubException {
4748
,environmentConfiguration.getMLRestPort()
4849
,environmentConfiguration.getMLUsername()
4950
});
50-
return new DataHub(environmentConfiguration.getMLHost(), Integer.parseInt(environmentConfiguration.getMLRestPort()), environmentConfiguration.getMLUsername(),
51-
environmentConfiguration.getMLPassword());
51+
DataHub dataHub = new DataHub(environmentConfiguration.getMLHost(), Integer.parseInt(environmentConfiguration.getMLRestPort()), environmentConfiguration.getMLUsername(),
52+
environmentConfiguration.getMLPassword());
53+
dataHub.setAssetInstallTimeFile(new File(environmentConfiguration.getAssetInstallTimeFilePath()));
54+
55+
return dataHub;
5256
} catch(Throwable e) {
5357
throw new DataHubException(e.getMessage(), e);
5458
}
@@ -83,5 +87,7 @@ public void uninstall() throws DataHubException {
8387
throw new DataHubException(e.getMessage(), e);
8488
}
8589
}
90+
91+
8692

8793
}

‎quick-start/src/main/java/com/marklogic/hub/service/DomainManagerService.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import org.slf4j.Logger;
99
import org.slf4j.LoggerFactory;
1010
import org.springframework.beans.factory.annotation.Autowired;
11+
import org.springframework.context.annotation.Scope;
1112
import org.springframework.stereotype.Service;
1213

1314
import com.marklogic.client.DatabaseClient;
@@ -19,16 +20,22 @@
1920
import com.marklogic.hub.exception.DomainManagerException;
2021
import com.marklogic.hub.factory.DomainModelFactory;
2122
import com.marklogic.hub.model.DomainModel;
23+
import com.marklogic.hub.model.FlowModel;
24+
import com.marklogic.hub.model.FlowType;
2225
import com.marklogic.hub.util.FileUtil;
2326

2427
@Service
28+
@Scope("session")
2529
public class DomainManagerService {
2630

2731
private static final Logger LOGGER = LoggerFactory
2832
.getLogger(DomainManagerService.class);
2933

3034
@Autowired
3135
private EnvironmentConfiguration environmentConfiguration;
36+
37+
@Autowired
38+
private SyncStatusService syncStatusService;
3239

3340
public DomainManager getDomainManager() {
3441

@@ -57,8 +64,28 @@ public List<DomainModel> getDomains() {
5764
domains.add(domainModelFactory.createDomain(domainName, domainsPath
5865
+ File.separator + domainName));
5966
}
67+
68+
// update the sync status of the domains and flows
69+
// TODO: if we improve DomainModelFactory and FlowModelFactory implementation,
70+
// we may be able to set the status correctly during model creation.
71+
updateSyncStatus(domains);
72+
6073
return domains;
6174
}
75+
76+
protected void updateSyncStatus(List<DomainModel> domains) {
77+
for (DomainModel domainModel : domains) {
78+
domainModel.setSynched(syncStatusService.isDomainSynched(domainModel.getDomainName()));
79+
80+
for (FlowModel flowModel : domainModel.getInputFlows()) {
81+
flowModel.setSynched(syncStatusService.isFlowSynched(domainModel.getDomainName(), FlowType.INPUT, flowModel.getFlowName()));
82+
}
83+
84+
for (FlowModel flowModel : domainModel.getConformFlows()) {
85+
flowModel.setSynched(syncStatusService.isFlowSynched(domainModel.getDomainName(), FlowType.CONFORM, flowModel.getFlowName()));
86+
}
87+
}
88+
}
6289

6390
private List<Domain> getDomainsInServer() {
6491
List<Domain> domainsInServer = new ArrayList<>();

0 commit comments

Comments
 (0)