Skip to content

Commit 6512ef7

Browse files
maeisabelle08maeisabelle08
committed
Added the download config feature
1 parent 4cbea99 commit 6512ef7

File tree

7 files changed

+141
-107
lines changed

7 files changed

+141
-107
lines changed

‎marklogic-data-hub/src/main/java/com/marklogic/hub/Mlcp.java‎

Lines changed: 37 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -61,23 +61,7 @@ public void addSourceDirectory(String directoryPath, SourceOptions options) {
6161
public void loadContent() throws IOException, JSONException {
6262
for (MlcpSource source : sources) {
6363
try {
64-
List<String> arguments = new ArrayList<>();
65-
66-
arguments.add("import");
67-
arguments.add("-mode");
68-
arguments.add("local");
69-
arguments.add("-host");
70-
arguments.add(host);
71-
arguments.add("-port");
72-
arguments.add(Integer.toString(port));
73-
arguments.add("-username");
74-
arguments.add(user);
75-
arguments.add("-password");
76-
arguments.add(password);
77-
78-
// add arguments related to the source
79-
List<String> sourceArguments = source.getMlcpArguments();
80-
arguments.addAll(sourceArguments);
64+
List<String> arguments = getMlcpOptions(source);
8165

8266
LOGGER.info(arguments.toString());
8367
DataHubContentPump contentPump = new DataHubContentPump(arguments);
@@ -97,8 +81,9 @@ protected void setHadoopHomeDir() throws IOException {
9781
System.setProperty("hadoop.home.dir", new File(home).getCanonicalPath());
9882
}
9983

100-
private static class MlcpSource {
101-
private String sourcePath;
84+
public static class MlcpSource {
85+
private static final String DOCUMENT_TYPE_KEY = "-document_type";
86+
private String sourcePath;
10287
private SourceOptions sourceOptions;
10388

10489
public MlcpSource(String sourcePath, SourceOptions sourceOptions) {
@@ -110,42 +95,26 @@ public String getSourcePath() {
11095
return sourcePath;
11196
}
11297

113-
public List<String> getMlcpArguments() throws IOException, JSONException {
98+
public List<String> getMlcpArguments() throws IOException, JSONException {
11499
File file = new File(sourcePath);
115100
String canonicalPath = file.getCanonicalPath();
116101

117102
List<String> arguments = new ArrayList<>();
118-
arguments.add("-generate_uri");
119-
arguments.add("true");
120-
103+
121104
arguments.add("-input_file_path");
122105
arguments.add(canonicalPath);
123-
// arguments.add("-input_file_type");
124-
// if (sourceOptions.getInputFileType() == null) {
125-
// arguments.add("documents");
126-
// } else {
127-
// arguments.add(sourceOptions.getInputFileType());
128-
// }
129-
130-
// String collections = this.getOutputCollections();
131-
// arguments.add("-output_collections");
132-
// arguments.add("\"" + collections + "\"");
133-
//
134-
// if (sourceOptions.getInputCompressed()) {
135-
// arguments.add("-input_compressed");
136-
// }
137-
138-
// by default, cut the source directory path to make URIs shorter
139-
// String uriReplace = canonicalPath + ",''";
140-
// uriReplace = uriReplace.replaceAll("\\\\", "/");
141-
//
142-
// arguments.add("-output_uri_replace");
143-
// arguments.add("\"" + uriReplace + "\"");
144-
145-
arguments.add("-document_type");
146-
arguments.add(sourceOptions.getDataFormat());
147106

107+
arguments.add("-output_uri_replace");
108+
arguments.add("\""+canonicalPath+",''\"");
109+
148110
addOtherArguments(arguments, sourceOptions.getOtherOptions());
111+
112+
//add document type only if it does not exist in the list
113+
if(!arguments.contains(DOCUMENT_TYPE_KEY)) {
114+
arguments.add(DOCUMENT_TYPE_KEY);
115+
arguments.add(sourceOptions.getDataFormat());
116+
}
117+
149118
return arguments;
150119
}
151120

@@ -219,4 +188,25 @@ public void setOtherOptions(String otherOptions) {
219188
this.otherOptions = otherOptions;
220189
}
221190
}
191+
192+
public List<String> getMlcpOptions(MlcpSource source) throws IOException, JSONException {
193+
List<String> mlcpOptions = new ArrayList<>();
194+
195+
mlcpOptions.add("import");
196+
mlcpOptions.add("-mode");
197+
mlcpOptions.add("local");
198+
mlcpOptions.add("-host");
199+
mlcpOptions.add(host);
200+
mlcpOptions.add("-port");
201+
mlcpOptions.add(Integer.toString(port));
202+
mlcpOptions.add("-username");
203+
mlcpOptions.add(user);
204+
mlcpOptions.add("-password");
205+
mlcpOptions.add(password);
206+
207+
List<String> sourceArguments = source.getMlcpArguments();
208+
mlcpOptions.addAll(sourceArguments);
209+
210+
return mlcpOptions;
211+
}
222212
}

‎quick-start/src/main/java/com/marklogic/hub/model/FlowOptionsModel.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
package com.marklogic.hub.model;
22

3-
public class RunFlowModel {
3+
public class FlowOptionsModel {
44
private String entityName;
55
private String flowName;
66
private String inputPath;

‎quick-start/src/main/java/com/marklogic/hub/model/RunFlowModel.java‎

Lines changed: 0 additions & 34 deletions
This file was deleted.

‎quick-start/src/main/java/com/marklogic/hub/web/controller/api/FlowApiController.java‎

Lines changed: 73 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
package com.marklogic.hub.web.controller.api;
22

3+
import java.io.ByteArrayInputStream;
34
import java.io.IOException;
5+
import java.io.InputStream;
46
import java.math.BigInteger;
7+
import java.nio.charset.StandardCharsets;
58
import java.util.ArrayList;
69
import java.util.List;
710

@@ -16,7 +19,11 @@
1619
import org.springframework.batch.core.JobExecution;
1720
import org.springframework.batch.core.JobExecutionListener;
1821
import org.springframework.beans.factory.annotation.Autowired;
22+
import org.springframework.core.io.InputStreamResource;
23+
import org.springframework.http.HttpHeaders;
1924
import org.springframework.http.MediaType;
25+
import org.springframework.http.ResponseEntity;
26+
import org.springframework.util.StringUtils;
2027
import org.springframework.validation.BindingResult;
2128
import org.springframework.web.bind.annotation.RequestBody;
2229
import org.springframework.web.bind.annotation.RequestMapping;
@@ -25,13 +32,14 @@
2532
import org.springframework.web.bind.annotation.RestController;
2633

2734
import com.marklogic.hub.Mlcp;
35+
import com.marklogic.hub.Mlcp.MlcpSource;
2836
import com.marklogic.hub.Mlcp.SourceOptions;
2937
import com.marklogic.hub.config.EnvironmentConfiguration;
3038
import com.marklogic.hub.flow.Flow;
3139
import com.marklogic.hub.flow.FlowType;
3240
import com.marklogic.hub.model.EntityModel;
3341
import com.marklogic.hub.model.FlowModel;
34-
import com.marklogic.hub.model.RunFlowModel;
42+
import com.marklogic.hub.model.FlowOptionsModel;
3543
import com.marklogic.hub.service.CancellableTask;
3644
import com.marklogic.hub.service.FlowManagerService;
3745
import com.marklogic.hub.service.TaskManagerService;
@@ -45,6 +53,10 @@ public class FlowApiController extends BaseController {
4553

4654
private static final Logger LOGGER = LoggerFactory
4755
.getLogger(FlowApiController.class);
56+
57+
private static final String MLCP_OPTIONS_FILENAME = "mlcpOptions.txt";
58+
59+
private static final String NEW_LINE = "\n";
4860

4961
@Autowired
5062
private EnvironmentConfiguration environmentConfiguration;
@@ -137,7 +149,7 @@ public void run(BasicFuture<?> resultFuture) {
137149
}
138150

139151
@RequestMapping(value = "/run", method = RequestMethod.POST)
140-
public BigInteger runFlow(@RequestBody RunFlowModel runFlow) {
152+
public BigInteger runFlow(@RequestBody FlowOptionsModel runFlow) {
141153
CancellableTask task = new CancellableTask() {
142154

143155
private JobExecution jobExecution;
@@ -190,9 +202,9 @@ public void afterJob(JobExecution jobExecution) {
190202
}
191203

192204
@RequestMapping(value="/run/input", method = RequestMethod.POST)
193-
public BigInteger runInputFlow(@RequestBody RunFlowModel runFlow) {
205+
public BigInteger runInputFlow(@RequestBody FlowOptionsModel flowOptionsModel) {
194206

195-
saveInputPath(runFlow);
207+
saveInputPath(flowOptionsModel);
196208

197209
CancellableTask task = new CancellableTask() {
198210

@@ -204,31 +216,16 @@ public void cancel(BasicFuture<?> resultFuture) {
204216
@Override
205217
public void run(BasicFuture<?> resultFuture) {
206218
try {
207-
Flow flow = flowManagerService.getFlow(runFlow.getEntityName(), runFlow.getFlowName());
208-
209-
Mlcp mlcp = new Mlcp(
210-
environmentConfiguration.getMLHost()
211-
,Integer.parseInt(environmentConfiguration.getMLStagingRestPort())
212-
,environmentConfiguration.getMLUsername()
213-
,environmentConfiguration.getMLPassword()
214-
);
215-
216-
SourceOptions sourceOptions = new SourceOptions(
217-
runFlow.getEntityName(), runFlow.getFlowName(),
218-
FlowType.INPUT.toString(),
219-
flow.getDataFormat());
220-
221-
sourceOptions.setInputFileType(runFlow.getInputFileType());
222-
sourceOptions.setOtherOptions(runFlow.getOtherOptions());
223-
mlcp.addSourceDirectory(runFlow.getInputPath(), sourceOptions);
219+
SourceOptions sourceOptions = createSourceOptionsInstance(flowOptionsModel);
220+
Mlcp mlcp = createMlcpInstance(flowOptionsModel,sourceOptions);
224221
mlcp.loadContent();
225222

226223
resultFuture.completed(null);
227224
}
228225

229226
catch (IOException | JSONException e) {
230227
LOGGER.error("Error encountered while trying to run flow: "
231-
+ runFlow.getEntityName() + " > " + runFlow.getFlowName(),
228+
+ flowOptionsModel.getEntityName() + " > " + flowOptionsModel.getFlowName(),
232229
e);
233230
resultFuture.failed(e);
234231
}
@@ -238,6 +235,31 @@ public void run(BasicFuture<?> resultFuture) {
238235
return taskManagerService.addTask(task);
239236
}
240237

238+
protected Mlcp createMlcpInstance(FlowOptionsModel flowOptionsModel, SourceOptions sourceOptions) throws NumberFormatException, IOException {
239+
Mlcp mlcp = new Mlcp(
240+
environmentConfiguration.getMLHost()
241+
,Integer.parseInt(environmentConfiguration.getMLStagingRestPort())
242+
,environmentConfiguration.getMLUsername()
243+
,environmentConfiguration.getMLPassword()
244+
);
245+
mlcp.addSourceDirectory(flowOptionsModel.getInputPath(), sourceOptions);
246+
return mlcp;
247+
}
248+
249+
protected SourceOptions createSourceOptionsInstance(FlowOptionsModel flowOptionsModel) throws NumberFormatException, IOException {
250+
Flow flow = flowManagerService.getFlow(flowOptionsModel.getEntityName(), flowOptionsModel.getFlowName());
251+
252+
SourceOptions sourceOptions = new SourceOptions(
253+
flowOptionsModel.getEntityName(), flowOptionsModel.getFlowName(),
254+
FlowType.INPUT.toString(),
255+
flow.getDataFormat());
256+
257+
sourceOptions.setInputFileType(flowOptionsModel.getInputFileType());
258+
sourceOptions.setOtherOptions(flowOptionsModel.getOtherOptions());
259+
260+
return sourceOptions;
261+
}
262+
241263
@RequestMapping(value = "/input-path", method = RequestMethod.GET, produces = { MediaType.TEXT_PLAIN_VALUE })
242264
@ResponseBody
243265
public String getPreviousInputPath(HttpServletRequest request) {
@@ -251,7 +273,7 @@ private String getPreviousInputPath(String entityName, String flowName) {
251273
return value == null ? "." : value;
252274
}
253275

254-
private void saveInputPath(RunFlowModel runFlow) {
276+
private void saveInputPath(FlowOptionsModel runFlow) {
255277
environmentConfiguration.saveOrUpdateFlowInputPath(runFlow.getEntityName(), runFlow.getFlowName(), runFlow.getInputPath());
256278
}
257279

@@ -267,5 +289,33 @@ public void runFlowsInParallel(HttpServletRequest request) {
267289
flowManagerService.runFlowsInParallel(flows.toArray(new Flow[flows
268290
.size()]));
269291
}
292+
293+
@RequestMapping(value = "/options/download", method = RequestMethod.POST, consumes = { MediaType.APPLICATION_JSON_UTF8_VALUE }, produces = { MediaType.TEXT_PLAIN_VALUE })
294+
public ResponseEntity<InputStreamResource> downloadMlcpConfig(@RequestBody FlowOptionsModel flowOptionsModel) throws IOException, NumberFormatException, JSONException {
295+
String mlcpConfigContent = buildMlcpConfigContent(flowOptionsModel);
296+
byte[] contentBytes = mlcpConfigContent.getBytes(StandardCharsets.UTF_8);
297+
InputStream inputStream = new ByteArrayInputStream(contentBytes);
298+
HttpHeaders headers = new HttpHeaders();
299+
addRemoveCachingInHeaders(headers);
300+
headers.add("content-disposition", "attachment; filename=" + MLCP_OPTIONS_FILENAME);
301+
return ResponseEntity
302+
.ok()
303+
.contentLength(contentBytes.length)
304+
.contentType(MediaType.TEXT_PLAIN)
305+
.headers(headers)
306+
.body(new InputStreamResource(inputStream));
307+
}
270308

309+
private String buildMlcpConfigContent(FlowOptionsModel flowOptionsModel) throws NumberFormatException, IOException, JSONException {
310+
SourceOptions sourceOptions = createSourceOptionsInstance(flowOptionsModel);
311+
Mlcp mlcp = createMlcpInstance(flowOptionsModel,sourceOptions);
312+
List<String> mlcpOptions = mlcp.getMlcpOptions(new MlcpSource(flowOptionsModel.getInputPath(), sourceOptions));
313+
return StringUtils.collectionToDelimitedString(mlcpOptions, NEW_LINE);
314+
}
315+
316+
private void addRemoveCachingInHeaders(HttpHeaders headers) {
317+
headers.add("Cache-Control", "no-cache, no-store, must-revalidate");
318+
headers.add("Pragma", "no-cache");
319+
headers.add("Expires", "0");
320+
}
271321
}

‎quick-start/src/main/resources/static/app/services/dataHubService.js‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,8 @@
3232
searchPath: searchPath,
3333
showApiDoc: showApiDoc,
3434
getPreviousInputPath: getPreviousInputPath,
35-
getJsonFile: getJsonFile
35+
getJsonFile: getJsonFile,
36+
downloadMlcpOptionsFile: downloadMlcpOptionsFile
3637
});
3738

3839
function login(loginForm) {
@@ -194,7 +195,6 @@
194195
$rootScope.notificationBar.messageType = messageType;
195196
$rootScope.notificationBar.message = message;
196197
$rootScope.notificationBar.show = true;
197-
console.log(message);
198198
}
199199

200200
function showApiDoc() {
@@ -214,6 +214,17 @@
214214
function getJsonFile(filePath) {
215215
return $http.get(filePath);
216216
}
217+
218+
function downloadMlcpOptionsFile(entityName, flowName, form) {
219+
var data = {
220+
entityName: entityName,
221+
flowName: flowName,
222+
inputPath: form.inputPath,
223+
dataFormat: form.dataFormat,
224+
otherOptions: form.otherOptions
225+
};
226+
return $http.post('api/flows/options/download', data);
227+
}
217228

218229
}
219230

0 commit comments

Comments
 (0)