Skip to content

Commit 0829932

Browse files
maeisabelle08maeisabelle08
committed
Added the ability to save and load previous settings per flow
1 parent 1816847 commit 0829932

File tree

7 files changed

+180
-118
lines changed

7 files changed

+180
-118
lines changed

‎marklogic-data-hub/src/main/java/com/marklogic/hub/Mlcp.java‎

Lines changed: 24 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,19 @@
3030
import com.marklogic.client.io.Format;
3131

3232
public class Mlcp {
33-
private static final Logger LOGGER = LoggerFactory.getLogger(Mlcp.class);
34-
35-
private final static String DEFAULT_HADOOP_HOME_DIR = "./hadoop/";
33+
34+
public static final String DOCUMENT_TYPE_KEY = "-document_type";
35+
public static final String INPUT_FILE_PATH_KEY = "-input_file_path";
36+
public static final String INPUT_FILE_TYPE_KEY = "-input_file_type";
37+
public static final String OUTPUT_URI_REPLACE_KEY = "-output_uri_replace";
38+
public static final String MODE_KEY = "-mode";
39+
public static final String HOST_KEY = "-host";
40+
public static final String PORT_KEY = "-port";
41+
public static final String USERNAME_KEY = "-username";
42+
public static final String PASSWORD_KEY = "-password";
43+
44+
private static final Logger LOGGER = LoggerFactory.getLogger(Mlcp.class);
45+
private static final String DEFAULT_HADOOP_HOME_DIR = "./hadoop/";
3646

3747
private List<MlcpSource> sources = new ArrayList<>();
3848

@@ -82,8 +92,7 @@ protected void setHadoopHomeDir() throws IOException {
8292
}
8393

8494
public static class MlcpSource {
85-
private static final String DOCUMENT_TYPE_KEY = "-document_type";
86-
private String sourcePath;
95+
private String sourcePath;
8796
private SourceOptions sourceOptions;
8897

8998
public MlcpSource(String sourcePath, SourceOptions sourceOptions) {
@@ -101,11 +110,14 @@ public List<String> getMlcpArguments() throws IOException, JSONException {
101110

102111
List<String> arguments = new ArrayList<>();
103112

104-
arguments.add("-input_file_path");
113+
arguments.add(INPUT_FILE_PATH_KEY);
105114
arguments.add(canonicalPath);
106115

107-
arguments.add("-output_uri_replace");
116+
arguments.add(OUTPUT_URI_REPLACE_KEY);
108117
arguments.add("\""+canonicalPath+",''\"");
118+
119+
arguments.add(INPUT_FILE_TYPE_KEY);
120+
arguments.add(sourceOptions.getInputFileType());
109121

110122
addOtherArguments(arguments, sourceOptions.getOtherOptions());
111123

@@ -193,15 +205,15 @@ public List<String> getMlcpOptions(MlcpSource source) throws IOException, JSONEx
193205
List<String> mlcpOptions = new ArrayList<>();
194206

195207
mlcpOptions.add("import");
196-
mlcpOptions.add("-mode");
208+
mlcpOptions.add(MODE_KEY);
197209
mlcpOptions.add("local");
198-
mlcpOptions.add("-host");
210+
mlcpOptions.add(HOST_KEY);
199211
mlcpOptions.add(host);
200-
mlcpOptions.add("-port");
212+
mlcpOptions.add(PORT_KEY);
201213
mlcpOptions.add(Integer.toString(port));
202-
mlcpOptions.add("-username");
214+
mlcpOptions.add(USERNAME_KEY);
203215
mlcpOptions.add(user);
204-
mlcpOptions.add("-password");
216+
mlcpOptions.add(PASSWORD_KEY);
205217
mlcpOptions.add(password);
206218

207219
List<String> sourceArguments = source.getMlcpArguments();

‎quick-start/src/main/java/com/marklogic/hub/config/EnvironmentConfiguration.java‎

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
package com.marklogic.hub.config;
22

3+
import java.io.BufferedWriter;
34
import java.io.File;
45
import java.io.FileInputStream;
56
import java.io.FileNotFoundException;
67
import java.io.FileOutputStream;
8+
import java.io.FileWriter;
79
import java.io.IOException;
810
import java.io.InputStream;
911
import java.io.OutputStream;
@@ -15,7 +17,8 @@
1517
import org.springframework.core.env.Environment;
1618
import org.springframework.stereotype.Component;
1719

18-
import com.marklogic.hub.DataHub;
20+
import com.google.common.base.Charsets;
21+
import com.google.common.io.Files;
1922
import com.marklogic.hub.HubConfig;
2023

2124
/***
@@ -31,7 +34,6 @@ public class EnvironmentConfiguration {
3134
.getLogger(EnvironmentConfiguration.class);
3235

3336
private static final String ENVIRONMENT_PROPERTIES_FILENAME = "environment.properties";
34-
private static final String FLOW_PROPERTIES_FILENAME = "flow.properties";
3537
private static final String DEFAULT_SUFFIX = ".default";
3638
private static final String SERVER_PORT = "server.port";
3739
private static final String ML_HOST = "mlHost";
@@ -43,12 +45,12 @@ public class EnvironmentConfiguration {
4345
private static final String ML_AUTH = "mlAuth";
4446
private static final String USER_PLUGIN_DIR = "userPluginDir";
4547
private static final String ASSET_INSTALL_TIME_FILE = "assetInstallTimeFile";
48+
private static final String MLCP_OPTIONS_DIR = "mlcp-options";
4649

4750
@Autowired
4851
private Environment environment;
4952

5053
private Properties environmentProperties = new Properties();
51-
private Properties flowProperties = new Properties();
5254

5355
public String getServerPort() {
5456
return this.environment.getProperty(SERVER_PORT);
@@ -205,7 +207,6 @@ public void setAssetInstallTimeFilePath(String assetInstallTimeFilePath) {
205207

206208
public void loadConfigurationFromFiles() {
207209
loadConfigurationFromFile(environmentProperties, ENVIRONMENT_PROPERTIES_FILENAME);
208-
loadConfigurationFromFile(environmentProperties, FLOW_PROPERTIES_FILENAME);
209210
}
210211

211212
public void loadConfigurationFromFile(Properties configProperties, String fileName) {
@@ -245,13 +246,25 @@ private void saveConfigurationToFile(Properties configProperties, String fileNam
245246
}
246247
}
247248

248-
public void saveOrUpdateFlowInputPath(String entityName, String flowName, String inputPath) {
249-
this.flowProperties.setProperty(entityName + "-" + flowName, inputPath);
250-
saveConfigurationToFile(flowProperties, FLOW_PROPERTIES_FILENAME);
249+
public void saveOrUpdateFlowMlcpOptionsToFile(String entityName, String flowName, String mlcpOptionsFileContent) throws IOException {
250+
String filePath = getMlcpOptionsFilePath(entityName, flowName);
251+
FileWriter fw = new FileWriter(filePath);
252+
BufferedWriter bw = new BufferedWriter(fw);
253+
bw.write(mlcpOptionsFileContent);
254+
bw.close();
251255
}
252256

253-
public String getFlowInputPath(String entityName, String flowName) {
254-
return this.flowProperties.getProperty(entityName + "-" + flowName);
257+
private String getMlcpOptionsFilePath(String entityName, String flowName) {
258+
return "." + File.separator + MLCP_OPTIONS_DIR + File.separator + entityName + "-" + flowName + ".txt";
259+
}
260+
261+
public String getFlowMlcpOptionsFromFile(String entityName, String flowName) throws IOException {
262+
String filePath = getMlcpOptionsFilePath(entityName, flowName);
263+
File file = new File(filePath);
264+
if(file.exists()) {
265+
return Files.toString(file, Charsets.UTF_8);
266+
}
267+
return null;
255268
}
256269

257270
public HubConfig getHubConfig() {

‎quick-start/src/main/java/com/marklogic/hub/service/FlowManagerService.java‎

Lines changed: 47 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,12 @@
22

33
import java.io.File;
44
import java.io.IOException;
5+
import java.util.HashMap;
56
import java.util.List;
7+
import java.util.Map;
68

79
import org.codehaus.jettison.json.JSONException;
10+
import org.codehaus.jettison.json.JSONObject;
811
import org.slf4j.Logger;
912
import org.slf4j.LoggerFactory;
1013
import org.springframework.batch.core.JobExecution;
@@ -19,9 +22,9 @@
1922
import com.marklogic.client.io.Format;
2023
import com.marklogic.hub.FlowManager;
2124
import com.marklogic.hub.Mlcp;
22-
import com.marklogic.hub.PluginFormat;
2325
import com.marklogic.hub.Mlcp.MlcpSource;
2426
import com.marklogic.hub.Mlcp.SourceOptions;
27+
import com.marklogic.hub.PluginFormat;
2528
import com.marklogic.hub.config.EnvironmentConfiguration;
2629
import com.marklogic.hub.exception.FlowManagerException;
2730
import com.marklogic.hub.factory.FlowModelFactory;
@@ -38,6 +41,8 @@ public class FlowManagerService {
3841

3942
private static final String NEW_LINE = "\n";
4043

44+
private static final String DASH = "-";
45+
4146
@Autowired
4247
private EnvironmentConfiguration environmentConfiguration;
4348

@@ -138,10 +143,50 @@ private Mlcp createMlcpInstance(FlowOptionsModel flowOptionsModel, SourceOptions
138143
return mlcp;
139144
}
140145

141-
public String buildMlcpConfigContent(FlowOptionsModel flowOptionsModel) throws NumberFormatException, IOException, JSONException {
146+
public String buildMlcpConfigContent(FlowOptionsModel flowOptionsModel) throws IOException, JSONException {
142147
SourceOptions sourceOptions = createSourceOptionsInstance(flowOptionsModel);
143148
Mlcp mlcp = createMlcpInstance(flowOptionsModel,sourceOptions);
144149
List<String> mlcpOptions = mlcp.getMlcpOptions(new MlcpSource(flowOptionsModel.getInputPath(), sourceOptions));
145150
return StringUtils.collectionToDelimitedString(mlcpOptions, NEW_LINE);
146151
}
152+
153+
public void populateMlcpOptions(FlowOptionsModel flowOptionsModel, String optionsFileContent) {
154+
//put options to map to have option key-value pairs
155+
String[] options = StringUtils.delimitedListToStringArray(optionsFileContent, NEW_LINE);
156+
Map<String,String> optionsMap = new HashMap<String,String>();
157+
String key = null;
158+
for (String option : options) {
159+
if(option.startsWith(DASH)) {
160+
key = option;
161+
} else {
162+
optionsMap.put(key, option);
163+
}
164+
}
165+
166+
//populate the flowOptionsModel
167+
LOGGER.debug("Options for flow with entity name "+ flowOptionsModel.getEntityName() +
168+
" and flow name "+ flowOptionsModel.getFlowName());
169+
//start with the required options
170+
String inputPath = optionsMap.get(Mlcp.INPUT_FILE_PATH_KEY);
171+
String inputFileType = optionsMap.get(Mlcp.INPUT_FILE_TYPE_KEY);
172+
LOGGER.debug("{" + Mlcp.INPUT_FILE_PATH_KEY + ": "+ inputPath + "," + Mlcp.INPUT_FILE_TYPE_KEY + ":" + inputFileType + "}");
173+
flowOptionsModel.setInputPath(inputPath);
174+
flowOptionsModel.setInputFileType(inputFileType);
175+
176+
//then the other options
177+
//by removing the unnecessary options from the map
178+
optionsMap.remove(Mlcp.INPUT_FILE_PATH_KEY);
179+
optionsMap.remove(Mlcp.INPUT_FILE_TYPE_KEY);
180+
optionsMap.remove(null);
181+
optionsMap.remove(Mlcp.MODE_KEY);
182+
optionsMap.remove(Mlcp.HOST_KEY);
183+
optionsMap.remove(Mlcp.PORT_KEY);
184+
optionsMap.remove(Mlcp.USERNAME_KEY);
185+
optionsMap.remove(Mlcp.PASSWORD_KEY);
186+
//then converting the map to a json string
187+
String otherOptions = new JSONObject(optionsMap).toString();
188+
LOGGER.debug("Other options = "+ otherOptions);
189+
flowOptionsModel.setOtherOptions(otherOptions);
190+
191+
}
147192
}

‎quick-start/src/main/java/com/marklogic/hub/web/controller/api/FlowApiController.java‎

Lines changed: 20 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,6 @@
3131
import org.springframework.web.bind.annotation.ResponseBody;
3232
import org.springframework.web.bind.annotation.RestController;
3333

34-
import com.marklogic.hub.Mlcp;
35-
import com.marklogic.hub.Mlcp.MlcpSource;
36-
import com.marklogic.hub.Mlcp.SourceOptions;
3734
import com.marklogic.hub.config.EnvironmentConfiguration;
3835
import com.marklogic.hub.flow.Flow;
3936
import com.marklogic.hub.flow.FlowType;
@@ -200,9 +197,9 @@ public void afterJob(JobExecution jobExecution) {
200197
}
201198

202199
@RequestMapping(value="/run/input", method = RequestMethod.POST)
203-
public BigInteger runInputFlow(@RequestBody FlowOptionsModel flowOptionsModel) {
200+
public BigInteger runInputFlow(@RequestBody FlowOptionsModel flowOptionsModel) throws IOException, JSONException {
204201

205-
saveInputPath(flowOptionsModel);
202+
saveMlcpOptionsToFile(flowOptionsModel);
206203

207204
CancellableTask task = new CancellableTask() {
208205

@@ -231,21 +228,31 @@ public void run(BasicFuture<?> resultFuture) {
231228
return taskManagerService.addTask(task);
232229
}
233230

234-
@RequestMapping(value = "/input-path", method = RequestMethod.GET, produces = { MediaType.TEXT_PLAIN_VALUE })
231+
@RequestMapping(value = "/options", method = RequestMethod.GET, produces = { MediaType.APPLICATION_JSON_UTF8_VALUE })
235232
@ResponseBody
236-
public String getPreviousInputPath(HttpServletRequest request) {
233+
public FlowOptionsModel getPreviousLoadOptions(HttpServletRequest request) throws IOException {
237234
String entityName = request.getParameter("entityName");
238235
String flowName = request.getParameter("flowName");
239-
return getPreviousInputPath(entityName,flowName);
236+
return loadMlcpOptionsFromFile(entityName,flowName);
240237
}
241238

242-
private String getPreviousInputPath(String entityName, String flowName) {
243-
String value = environmentConfiguration.getFlowInputPath(entityName, flowName);
244-
return value == null ? "." : value;
239+
private FlowOptionsModel loadMlcpOptionsFromFile(String entityName, String flowName) throws IOException {
240+
FlowOptionsModel flowOptionsModel = new FlowOptionsModel();
241+
flowOptionsModel.setEntityName(entityName);
242+
flowOptionsModel.setFlowName(flowName);
243+
flowOptionsModel.setInputPath(".");
244+
flowOptionsModel.setInputFileType("documents");
245+
String optionsFileContent = environmentConfiguration.getFlowMlcpOptionsFromFile(entityName, flowName);
246+
if(optionsFileContent != null) {
247+
flowManagerService.populateMlcpOptions(flowOptionsModel, optionsFileContent);
248+
}
249+
return flowOptionsModel;
245250
}
246251

247-
private void saveInputPath(FlowOptionsModel runFlow) {
248-
environmentConfiguration.saveOrUpdateFlowInputPath(runFlow.getEntityName(), runFlow.getFlowName(), runFlow.getInputPath());
252+
private void saveMlcpOptionsToFile(FlowOptionsModel flowOptionsModel) throws IOException, JSONException {
253+
String mlcpOptionsFileContent = flowManagerService.buildMlcpConfigContent(flowOptionsModel);
254+
environmentConfiguration.saveOrUpdateFlowMlcpOptionsToFile(flowOptionsModel.getEntityName(),
255+
flowOptionsModel.getFlowName(), mlcpOptionsFileContent);
249256
}
250257

251258
@RequestMapping(value = "/runInParallel", method = RequestMethod.POST)

‎quick-start/src/main/resources/static/app/services/dataHubService.js‎

Lines changed: 5 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
displayMessage: displayMessage,
3232
searchPath: searchPath,
3333
showApiDoc: showApiDoc,
34-
getPreviousInputPath: getPreviousInputPath,
34+
getPreviousOptions: getPreviousOptions,
3535
getJsonFile: getJsonFile,
3636
downloadMlcpOptionsFile: downloadMlcpOptionsFile
3737
});
@@ -156,14 +156,7 @@
156156
return $http.post('api/flows/run', data);
157157
}
158158

159-
function runInputFlow(entityName, flowName, form) {
160-
var data = {
161-
entityName: entityName,
162-
flowName: flowName,
163-
inputPath: form.inputPath,
164-
dataFormat: form.dataFormat,
165-
otherOptions: form.otherOptions
166-
};
159+
function runInputFlow(data) {
167160
return $http.post('api/flows/run/input', data);
168161
}
169162

@@ -201,12 +194,12 @@
201194
$window.open('#/api-doc', '_blank');
202195
}
203196

204-
function getPreviousInputPath(entityName, flowName) {
197+
function getPreviousOptions(entityName, flowName) {
205198
var params = {
206199
entityName: entityName,
207200
flowName: flowName
208201
};
209-
return $http.get('api/flows/input-path', {
202+
return $http.get('api/flows/options', {
210203
'params': params
211204
});
212205
}
@@ -215,14 +208,7 @@
215208
return $http.get(filePath);
216209
}
217210

218-
function downloadMlcpOptionsFile(entityName, flowName, form) {
219-
var data = {
220-
entityName: entityName,
221-
flowName: flowName,
222-
inputPath: form.inputPath,
223-
dataFormat: form.dataFormat,
224-
otherOptions: form.otherOptions
225-
};
211+
function downloadMlcpOptionsFile(data) {
226212
return $http.post('api/flows/options/download', data);
227213
}
228214

0 commit comments

Comments
 (0)