Skip to content

Commit 9cb7013

Browse files
paxtonharepaxtonhare
committed
fixed searching
fixed entity def generating search options
1 parent 675fb31 commit 9cb7013

File tree

59 files changed

+655
-511
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

59 files changed

+655
-511
lines changed

‎.travis/startml.sh‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ export MARKLOGIC_MLCMD_PID_FILE=/var/run/mlcmd.pid
1010
export MARKLOGIC_UMASK=022
1111

1212
export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/MarkLogic/mlcmd/bin
13-
export LD_PRELOAD=/opt/MarkLogic/lib/libjemalloc.so.1
1413
export LD_LIBRARY_PATH=/opt/MarkLogic/lib:/data/Lib
1514

1615
echo "STARTING MARKLOGIC"

‎examples/spring-batch/build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ repositories {
1515
dependencies {
1616
compile 'com.marklogic:marklogic-data-hub:2.0.0-alpha.2'
1717
compile 'com.marklogic:marklogic-spring-batch-core:0.6.0'
18-
compile 'com.marklogic:ml-javaclient-util:2.12.1'
18+
compile 'com.marklogic:ml-javaclient-util:4.0.alpha2'
1919
testCompile 'com.marklogic:marklogic-spring-batch-test:0.6.0'
2020
}
2121

‎marklogic-data-hub/build.gradle‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@ sourceCompatibility = 1.8
1919
targetCompatibility = 1.8
2020

2121
dependencies {
22-
compile 'com.marklogic:marklogic-client-api:4.0-SNAPSHOT'
23-
compile 'com.marklogic:ml-javaclient-util:2.13.0'
22+
compile 'com.marklogic:marklogic-client-api:4.0.1'
23+
compile 'com.marklogic:ml-javaclient-util:4.0.alpha2'
2424
compile 'com.marklogic:ml-app-deployer:2.6.0'
2525
compile 'commons-io:commons-io:2.4'
2626
testCompile 'junit:junit:4.12'
@@ -84,6 +84,7 @@ processResources {
8484

8585
if (!(
8686
gradle.startParameter.taskNames*.toLowerCase().contains("bootrun") ||
87+
gradle.startParameter.taskNames*.toLowerCase().contains("test") ||
8788
gradle.startParameter.taskNames*.toLowerCase().contains("publishplugins") ||
8889
gradle.startParameter.taskNames*.toLowerCase().contains("bintrayUpload")
8990
)

‎marklogic-data-hub/src/main/java/com/marklogic/hub/DataHub.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
import com.marklogic.appdeployer.command.Command;
2222
import com.marklogic.appdeployer.command.CommandMapBuilder;
2323
import com.marklogic.appdeployer.command.appservers.DeployOtherServersCommand;
24-
import com.marklogic.appdeployer.command.mimetypes.DeployMimetypesCommand;
2524
import com.marklogic.appdeployer.impl.SimpleAppDeployer;
2625
import com.marklogic.client.DatabaseClient;
2726
import com.marklogic.client.FailedRequestException;

‎marklogic-data-hub/src/main/java/com/marklogic/hub/HubConfig.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@ public class HubConfig {
4646
public static final String OLD_HUB_CONFIG_DIR = "marklogic-config";
4747
public static final String HUB_CONFIG_DIR = "hub-internal-config";
4848
public static final String USER_CONFIG_DIR = "user-config";
49-
public static final String SEARCH_OPTIONS_FILE = "all-search-options.xml";
49+
public static final String ENTITY_CONFIG_DIR = "entity-config";
50+
public static final String ENTITY_SEARCH_OPTIONS_FILE = "entity-options.xml";
5051

5152
public static final String DEFAULT_HOST = "localhost";
5253

@@ -356,6 +357,11 @@ public Path getUserDatabaseDir() {
356357
return Paths.get(this.projectDir, USER_CONFIG_DIR, "databases");
357358
}
358359

360+
public Path getEntityDatabaseDir() {
361+
return Paths.get(this.projectDir, ENTITY_CONFIG_DIR, "databases");
362+
}
363+
364+
359365
public Path getUserServersDir() {
360366
return Paths.get(this.projectDir, USER_CONFIG_DIR, "servers");
361367
}
@@ -508,6 +514,8 @@ public void updateAppConfig(AppConfig config) {
508514
ConfigDir configDir = new ConfigDir(getUserConfigDir().toFile());
509515
config.setConfigDir(configDir);
510516

517+
config.setSchemasPath(getUserConfigDir().resolve("schemas").toString());
518+
511519
Map<String, String> customTokens = config.getCustomTokens();
512520

513521
customTokens.put("%%mlStagingAppserverName%%", stagingHttpName);

‎marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/DeployHubDatabaseCommand.java‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -113,12 +113,19 @@ protected JsonNode mergeDatabaseFiles(AppConfig appConfig) {
113113
List<File> files = new ArrayList<>();
114114
File databaseDir = hubConfig.getHubDatabaseDir().toFile();
115115
File userDatabaseDir = hubConfig.getUserDatabaseDir().toFile();
116+
File entityDatabaseDir = hubConfig.getEntityDatabaseDir().toFile();
116117
files.add(new File(databaseDir, this.databaseFilename));
117118

118-
File otherDatabaseFile = new File(userDatabaseDir, this.databaseFilename);
119-
if (otherDatabaseFile != null && otherDatabaseFile.exists()) {
120-
files.add(otherDatabaseFile);
119+
File userDatabaseFile = new File(userDatabaseDir, this.databaseFilename);
120+
if (userDatabaseFile != null && userDatabaseFile.exists()) {
121+
files.add(userDatabaseFile);
121122
}
123+
124+
File entityDatabaseFile = new File(entityDatabaseDir, this.databaseFilename);
125+
if (entityDatabaseFile != null && entityDatabaseFile.exists()) {
126+
files.add(entityDatabaseFile);
127+
}
128+
122129
if (logger.isInfoEnabled()) {
123130
logger.info("Merging JSON files at locations: " + files);
124131
}

‎marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/DeployHubServersCommand.java‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -66,12 +66,7 @@ protected void deleteResource(final ResourceManager mgr, CommandContext context,
6666
final String payload = getPayload(f, context);
6767
try {
6868
if (isRestartAfterDelete()) {
69-
context.getAdminManager().invokeActionRequiringRestart(new ActionRequiringRestart() {
70-
@Override
71-
public boolean execute() {
72-
return mgr.delete(payload).isDeleted();
73-
}
74-
});
69+
context.getAdminManager().invokeActionRequiringRestart(() -> mgr.delete(payload).isDeleted());
7570
} else {
7671
mgr.delete(payload);
7772
}

‎marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/LoadHubModulesCommand.java‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ private Content prepContent(String uri, InputStream inputStream, AppConfig confi
9393

9494
private void initializeActiveSession(CommandContext context) {
9595
AppConfig config = context.getAppConfig();
96-
XccAssetLoader xccAssetLoader = context.getAppConfig().newXccAssetLoader();
96+
XccAssetLoader xccAssetLoader = config.newXccAssetLoader();
9797

9898
this.modulesLoader = new DefaultModulesLoader(xccAssetLoader);
9999
this.threadPoolTaskExecutor = new ThreadPoolTaskExecutor();
@@ -211,6 +211,30 @@ public void execute(CommandContext context) {
211211
duration = (endTime - startTime);
212212
logger.info("Job Rest Options took: " + (duration / 1000000000) + " seconds");
213213

214+
logger.info("Loading Default Search Options to Staging");
215+
// switch to job db to do this:
216+
this.modulesLoader.setDatabaseClient(hubConfig.newStagingClient());
217+
startTime = System.nanoTime();
218+
resources = findResources("classpath*:/ml-modules/options", "/**/default.xml");
219+
for (Resource r : resources) {
220+
this.modulesLoader.installQueryOptions(r);
221+
}
222+
endTime = System.nanoTime();
223+
duration = (endTime - startTime);
224+
logger.info("Default Search Options took: " + (duration / 1000000000) + " seconds");
225+
226+
logger.info("Loading Default Search Options to Final");
227+
// switch to job db to do this:
228+
this.modulesLoader.setDatabaseClient(hubConfig.newFinalClient());
229+
startTime = System.nanoTime();
230+
resources = findResources("classpath*:/ml-modules/options", "/**/default.xml");
231+
for (Resource r : resources) {
232+
this.modulesLoader.installQueryOptions(r);
233+
}
234+
endTime = System.nanoTime();
235+
duration = (endTime - startTime);
236+
logger.info("Default Search Options took: " + (duration / 1000000000) + " seconds");
237+
214238
waitForTaskExecutorToFinish();
215239
logger.info("Finished Loading Modules");
216240
}

‎marklogic-data-hub/src/main/java/com/marklogic/hub/flow/impl/FlowRunnerImpl.java‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,8 @@ public JobTicket run() {
154154

155155
DataMovementManager dataMovementManager = srcClient.newDataMovementManager();
156156

157+
double batchCount = Math.ceil(uris.size() / batchSize);
158+
157159
QueryBatcher queryBatcher = dataMovementManager.newQueryBatcher(uris.iterator())
158160
.withBatchSize(batchSize)
159161
.withThreadCount(threadCount)
@@ -164,6 +166,13 @@ public JobTicket run() {
164166
failedEvents.addAndGet(response.errorCount);
165167
successfulEvents.addAndGet(response.totalCount - response.errorCount);
166168
successfulBatches.addAndGet(1);
169+
170+
int percentComplete = (int) (((double)successfulBatches.get() / batchCount) * 100.0);
171+
172+
flowStatusListeners.forEach((FlowStatusListener listener) -> {
173+
listener.onStatusChange(jobId, percentComplete, "");
174+
});
175+
167176
count.addAndGet(1);
168177
if (flowItemCompleteListeners.size() > 0) {
169178
response.completedItems.forEach((String item) -> {
@@ -199,6 +208,10 @@ public JobTicket run() {
199208
runningThread = new Thread(() -> {
200209
queryBatcher.awaitCompletion();
201210

211+
flowStatusListeners.forEach((FlowStatusListener listener) -> {
212+
listener.onStatusChange(jobTicket.getJobId(), 100, "");
213+
});
214+
202215
flowFinishedListeners.forEach((FlowFinishedListener::onFlowFinished));
203216

204217
dataMovementManager.stopJob(queryBatcher);
@@ -259,10 +272,12 @@ public RunFlowResponse run(String jobId, String[] items, Map<String, Object> opt
259272
if (resultItr == null || ! resultItr.hasNext()) {
260273
resp = new RunFlowResponse();
261274
}
262-
ResourceServices.ServiceResult res = resultItr.next();
263-
StringHandle handle = new StringHandle();
264-
ObjectMapper objectMapper = new ObjectMapper();
265-
resp = objectMapper.readValue(res.getContent(handle).get(), RunFlowResponse.class);
275+
else {
276+
ResourceServices.ServiceResult res = resultItr.next();
277+
StringHandle handle = new StringHandle();
278+
ObjectMapper objectMapper = new ObjectMapper();
279+
resp = objectMapper.readValue(res.getContent(handle).get(), RunFlowResponse.class);
280+
}
266281

267282
}
268283
catch(Exception e) {

‎marklogic-data-hub/src/main/resources/ml-modules/options/default.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
<options xmlns="http://marklogic.com/appservices/search">
2+
<constraint name="Collection">
3+
<collection/>
4+
</constraint>
5+
</options>

0 commit comments

Comments
 (0)