Skip to content

Commit 4cbea99

Browse files
maeisabelle08maeisabelle08
committed
Changes for the Load Data feature:
1. Added new mlcp options which are rearranged into groups (displayed as accordions) 2. Added a section for the mlcp command that changes based on the set mlcp options TODO: Download of config and saving/loading of previously set options
1 parent fea3b91 commit 4cbea99

File tree

9 files changed

+484
-126
lines changed

9 files changed

+484
-126
lines changed

‎marklogic-data-hub/src/main/java/com/marklogic/hub/FlowManager.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
import javax.xml.stream.XMLStreamException;
2323

24+
import org.codehaus.jettison.json.JSONException;
2425
import org.slf4j.Logger;
2526
import org.slf4j.LoggerFactory;
2627
import org.springframework.batch.core.Job;
@@ -239,7 +240,7 @@ public void runInputFlow(Flow flow, HubConfig config) {
239240
mlcp.addSourceDirectory(config.getModulesPath(), sourceOptions);
240241
mlcp.loadContent();
241242
}
242-
catch (IOException e) {
243+
catch (IOException | JSONException e) {
243244
LOGGER.error(
244245
"Error encountered while trying to run flow: "
245246
+ flow.getEntityName() + " > " + flow.getName(), e);

‎marklogic-data-hub/src/main/java/com/marklogic/hub/Mlcp.java‎

Lines changed: 51 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,12 @@
1818
import java.io.File;
1919
import java.io.IOException;
2020
import java.util.ArrayList;
21+
import java.util.Iterator;
2122
import java.util.List;
2223

24+
import org.codehaus.jettison.json.JSONArray;
25+
import org.codehaus.jettison.json.JSONException;
26+
import org.codehaus.jettison.json.JSONObject;
2327
import org.slf4j.Logger;
2428
import org.slf4j.LoggerFactory;
2529

@@ -54,7 +58,7 @@ public void addSourceDirectory(String directoryPath, SourceOptions options) {
5458
sources.add(source);
5559
}
5660

57-
public void loadContent() throws IOException {
61+
public void loadContent() throws IOException, JSONException {
5862
for (MlcpSource source : sources) {
5963
try {
6064
List<String> arguments = new ArrayList<>();
@@ -106,7 +110,7 @@ public String getSourcePath() {
106110
return sourcePath;
107111
}
108112

109-
public List<String> getMlcpArguments() throws IOException {
113+
public List<String> getMlcpArguments() throws IOException, JSONException {
110114
File file = new File(sourcePath);
111115
String canonicalPath = file.getCanonicalPath();
112116

@@ -116,58 +120,51 @@ public List<String> getMlcpArguments() throws IOException {
116120

117121
arguments.add("-input_file_path");
118122
arguments.add(canonicalPath);
119-
arguments.add("-input_file_type");
120-
if (sourceOptions.getInputFileType() == null) {
121-
arguments.add("documents");
122-
} else {
123-
arguments.add(sourceOptions.getInputFileType());
124-
}
125-
126-
if (sourceOptions.getInputFilePattern() != null) {
127-
arguments.add("-input_file_pattern");
128-
arguments.add(sourceOptions.getInputFilePattern());
129-
}
130-
131-
String collections = this.getOutputCollections();
132-
arguments.add("-output_collections");
133-
arguments.add("\"" + collections + "\"");
134-
135-
if (sourceOptions.getInputCompressed()) {
136-
arguments.add("-input_compressed");
137-
}
123+
// arguments.add("-input_file_type");
124+
// if (sourceOptions.getInputFileType() == null) {
125+
// arguments.add("documents");
126+
// } else {
127+
// arguments.add(sourceOptions.getInputFileType());
128+
// }
129+
130+
// String collections = this.getOutputCollections();
131+
// arguments.add("-output_collections");
132+
// arguments.add("\"" + collections + "\"");
133+
//
134+
// if (sourceOptions.getInputCompressed()) {
135+
// arguments.add("-input_compressed");
136+
// }
138137

139138
// by default, cut the source directory path to make URIs shorter
140-
String uriReplace = canonicalPath + ",''";
141-
uriReplace = uriReplace.replaceAll("\\\\", "/");
142-
143-
arguments.add("-output_uri_replace");
144-
arguments.add("\"" + uriReplace + "\"");
139+
// String uriReplace = canonicalPath + ",''";
140+
// uriReplace = uriReplace.replaceAll("\\\\", "/");
141+
//
142+
// arguments.add("-output_uri_replace");
143+
// arguments.add("\"" + uriReplace + "\"");
145144

146145
arguments.add("-document_type");
147146
arguments.add(sourceOptions.getDataFormat());
148-
149-
arguments.add("-transform_module");
150-
arguments.add("/com.marklogic.hub/mlcp-flow-transform.xqy");
151-
arguments.add("-transform_namespace");
152-
arguments.add("http://marklogic.com/data-hub/mlcp-flow-transform");
153-
arguments.add("-transform_param");
154-
arguments.add("\"" + sourceOptions.getTransformParams() + "\"");
147+
148+
addOtherArguments(arguments, sourceOptions.getOtherOptions());
155149
return arguments;
156150
}
157151

158-
private String getOutputCollections() {
159-
StringBuilder collectionsBuilder = new StringBuilder();
160-
collectionsBuilder.append(sourceOptions.getEntityName());
161-
collectionsBuilder.append(",");
162-
collectionsBuilder.append(sourceOptions.getFlowName());
163-
collectionsBuilder.append(",");
164-
collectionsBuilder.append(sourceOptions.getFlowType());
165-
if (sourceOptions.getCollection() != null) {
166-
collectionsBuilder.append(",");
167-
collectionsBuilder.append(sourceOptions.getCollection());
168-
}
169-
return collectionsBuilder.toString();
170-
}
152+
private void addOtherArguments(List<String> arguments,
153+
String otherOptions) throws JSONException {
154+
JSONArray jsonArray = new JSONArray(otherOptions);
155+
for (int i = 0; i < jsonArray.length(); i++) {
156+
JSONObject jsonObject = jsonArray.getJSONObject(i);
157+
@SuppressWarnings("rawtypes")
158+
Iterator keysIterator = jsonObject.keys();
159+
while(keysIterator.hasNext()) {
160+
String key = (String)keysIterator.next();
161+
arguments.add(key);
162+
arguments.add(jsonObject.getString(key));
163+
}
164+
165+
}
166+
167+
}
171168
}
172169

173170
public static class SourceOptions {
@@ -176,9 +173,7 @@ public static class SourceOptions {
176173
private String flowType;
177174
private String dataFormat = "json";
178175
private String inputFileType;
179-
private String inputFilePattern;
180-
private String collection;
181-
private boolean inputCompressed = false;
176+
private String otherOptions;
182177

183178
public SourceOptions(String entityName, String flowName, String flowType, Format dataFormat) {
184179
this.entityName = entityName;
@@ -215,35 +210,13 @@ public String getInputFileType() {
215210
public void setInputFileType(String inputFileType) {
216211
this.inputFileType = inputFileType;
217212
}
213+
214+
public String getOtherOptions() {
215+
return otherOptions;
216+
}
218217

219-
public String getInputFilePattern() {
220-
return inputFilePattern;
221-
}
222-
223-
public void setInputFilePattern(String inputFilePattern) {
224-
this.inputFilePattern = inputFilePattern;
225-
}
226-
227-
public String getCollection() {
228-
return collection;
229-
}
230-
231-
public void setCollection(String collection) {
232-
this.collection = collection;
233-
}
234-
235-
public void setInputCompressed(boolean inputCompressed) {
236-
this.inputCompressed = inputCompressed;
237-
}
238-
239-
public boolean getInputCompressed() {
240-
return this.inputCompressed;
241-
}
242-
243-
protected String getTransformParams() {
244-
return String.format(
245-
"<params><entity-name>%s</entity-name><flow-name>%s</flow-name><flow-type>%s</flow-type></params>",
246-
entityName, flowName, flowType);
247-
}
218+
public void setOtherOptions(String otherOptions) {
219+
this.otherOptions = otherOptions;
220+
}
248221
}
249222
}

‎quick-start/src/main/java/com/marklogic/hub/model/FlowOptionsModel.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
package com.marklogic.hub.model;
2+
3+
public class RunFlowModel {
4+
private String entityName;
5+
private String flowName;
6+
private String inputPath;
7+
private String inputFileType;
8+
private String otherOptions;
9+
10+
public String getEntityName() {
11+
return entityName;
12+
}
13+
14+
public String getFlowName() {
15+
return flowName;
16+
}
17+
18+
public String getInputPath() {
19+
return inputPath;
20+
}
21+
22+
public String getOtherOptions() {
23+
return otherOptions;
24+
}
25+
26+
public void setOtherOptions(String otherOptions) {
27+
this.otherOptions = otherOptions;
28+
}
29+
30+
public void setEntityName(String entityName) {
31+
this.entityName = entityName;
32+
}
33+
34+
public void setFlowName(String flowName) {
35+
this.flowName = flowName;
36+
}
37+
38+
public void setInputPath(String inputPath) {
39+
this.inputPath = inputPath;
40+
}
41+
42+
public String getInputFileType() {
43+
return inputFileType;
44+
}
45+
46+
public void setInputFileType(String inputFileType) {
47+
this.inputFileType = inputFileType;
48+
}
49+
50+
51+
}

‎quick-start/src/main/java/com/marklogic/hub/web/controller/api/FlowApiController.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import javax.servlet.http.HttpSession;
1010

1111
import org.apache.http.concurrent.BasicFuture;
12+
import org.codehaus.jettison.json.JSONException;
1213
import org.slf4j.Logger;
1314
import org.slf4j.LoggerFactory;
1415
import org.springframework.batch.core.ExitStatus;
@@ -217,16 +218,15 @@ public void run(BasicFuture<?> resultFuture) {
217218
FlowType.INPUT.toString(),
218219
flow.getDataFormat());
219220

220-
sourceOptions.setInputFileType(runFlow.getDataFormat());
221-
sourceOptions.setCollection(runFlow.getCollection());
222-
sourceOptions.setInputCompressed(runFlow.getInputCompressed());
221+
sourceOptions.setInputFileType(runFlow.getInputFileType());
222+
sourceOptions.setOtherOptions(runFlow.getOtherOptions());
223223
mlcp.addSourceDirectory(runFlow.getInputPath(), sourceOptions);
224224
mlcp.loadContent();
225225

226226
resultFuture.completed(null);
227227
}
228228

229-
catch (IOException e) {
229+
catch (IOException | JSONException e) {
230230
LOGGER.error("Error encountered while trying to run flow: "
231231
+ runFlow.getEntityName() + " > " + runFlow.getFlowName(),
232232
e);

‎quick-start/src/main/resources/static/app/services/dataHubService.js‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@
3131
displayMessage: displayMessage,
3232
searchPath: searchPath,
3333
showApiDoc: showApiDoc,
34-
getPreviousInputPath : getPreviousInputPath
34+
getPreviousInputPath: getPreviousInputPath,
35+
getJsonFile: getJsonFile
3536
});
3637

3738
function login(loginForm) {
@@ -154,14 +155,13 @@
154155
return $http.post('api/flows/run', data);
155156
}
156157

157-
function runInputFlow(entityName, flowName, mlcpOptions) {
158+
function runInputFlow(entityName, flowName, form) {
158159
var data = {
159160
entityName: entityName,
160161
flowName: flowName,
161-
inputPath: mlcpOptions.inputPath,
162-
collection: mlcpOptions.collection,
163-
dataFormat: mlcpOptions.dataFormat,
164-
inputCompressed: mlcpOptions.inputCompressed
162+
inputPath: form.inputPath,
163+
dataFormat: form.dataFormat,
164+
otherOptions: form.otherOptions
165165
};
166166
return $http.post('api/flows/run/input', data);
167167
}
@@ -194,6 +194,7 @@
194194
$rootScope.notificationBar.messageType = messageType;
195195
$rootScope.notificationBar.message = message;
196196
$rootScope.notificationBar.show = true;
197+
console.log(message);
197198
}
198199

199200
function showApiDoc() {
@@ -209,6 +210,10 @@
209210
'params': params
210211
});
211212
}
213+
214+
function getJsonFile(filePath) {
215+
return $http.get(filePath);
216+
}
212217

213218
}
214219

0 commit comments

Comments
 (0)