Skip to content
This repository was archived by the owner on Sep 16, 2024. It is now read-only.

Commit a660981

Browse files
rjrudinrjrudin
committed
#339 Support for loading data
1 parent ac698fa commit a660981

File tree

15 files changed

+421
-47
lines changed

15 files changed

+421
-47
lines changed

‎src/main/java/com/marklogic/appdeployer/AppConfig.java‎

Lines changed: 30 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import com.marklogic.appdeployer.command.forests.ForestNamingStrategy;
44
import com.marklogic.appdeployer.command.forests.ReplicaBuilderStrategy;
5+
import com.marklogic.appdeployer.util.MapPropertiesSource;
56
import com.marklogic.client.DatabaseClient;
67
import com.marklogic.client.DatabaseClientFactory;
78
import com.marklogic.client.DatabaseClientFactory.SSLHostnameVerifier;
@@ -11,7 +12,10 @@
1112
import com.marklogic.client.ext.SecurityContextType;
1213
import com.marklogic.client.ext.modulesloader.impl.PropertiesModuleManager;
1314
import com.marklogic.client.ext.modulesloader.ssl.SimpleX509TrustManager;
15+
import com.marklogic.client.ext.tokenreplacer.DefaultTokenReplacer;
1416
import com.marklogic.client.ext.tokenreplacer.PropertiesSource;
17+
import com.marklogic.client.ext.tokenreplacer.RoxyTokenReplacer;
18+
import com.marklogic.client.ext.tokenreplacer.TokenReplacer;
1519

1620
import javax.net.ssl.SSLContext;
1721
import javax.net.ssl.X509TrustManager;
@@ -201,8 +205,6 @@ public class AppConfig {
201205
private boolean replaceTokensInModules = true;
202206
// Whether or not to prefix each module token with "@ml."
203207
private boolean useRoxyTokenPrefix = false;
204-
// Additional PropertiesSources instance to use for replacing module tokens
205-
private List<PropertiesSource> moduleTokensPropertiesSources = new ArrayList<>();
206208

207209
private Pattern moduleFilenamesIncludePattern;
208210

@@ -234,13 +236,17 @@ public class AppConfig {
234236

235237
private File projectDir;
236238

239+
private DataConfig dataConfig;
240+
237241
public AppConfig() {
238242
this(null);
239243
}
240244

241245
public AppConfig(File projectDir) {
242246
this.projectDir = projectDir;
243247

248+
dataConfig = new DataConfig(projectDir);
249+
244250
modulePaths = new ArrayList<>();
245251
String path = projectDir != null ? new File(projectDir, DEFAULT_MODULES_PATH).getAbsolutePath() : DEFAULT_MODULES_PATH;
246252
modulePaths.add(path);
@@ -284,6 +290,20 @@ public void populateCustomTokens(PropertiesSource propertiesSource, String prefi
284290
}
285291
}
286292

293+
/**
294+
* Builds a TokenReplacer based on the customTokens map held by this class.
295+
*
296+
* @return
297+
*/
298+
public TokenReplacer buildTokenReplacer() {
299+
DefaultTokenReplacer r = isUseRoxyTokenPrefix() ? new RoxyTokenReplacer() : new DefaultTokenReplacer();
300+
final Map<String, String> customTokens = getCustomTokens();
301+
if (customTokens != null) {
302+
r.addPropertiesSource(new MapPropertiesSource(customTokens));
303+
}
304+
return r;
305+
}
306+
287307
public void setSimpleSslConfig() {
288308
setRestSslContext(SimpleX509TrustManager.newSSLContext());
289309
setRestSslHostnameVerifier(DatabaseClientFactory.SSLHostnameVerifier.ANY);
@@ -697,14 +717,6 @@ public void setUseRoxyTokenPrefix(boolean useRoxyTokenPrefix) {
697717
this.useRoxyTokenPrefix = useRoxyTokenPrefix;
698718
}
699719

700-
public List<PropertiesSource> getModuleTokensPropertiesSources() {
701-
return moduleTokensPropertiesSources;
702-
}
703-
704-
public void setModuleTokensPropertiesSources(List<PropertiesSource> moduleTokensPropertiesSources) {
705-
this.moduleTokensPropertiesSources = moduleTokensPropertiesSources;
706-
}
707-
708720
public boolean isStaticCheckAssets() {
709721
return staticCheckAssets;
710722
}
@@ -1305,4 +1317,12 @@ public boolean isAddHostNameTokens() {
13051317
public void setAddHostNameTokens(boolean addHostNameTokens) {
13061318
this.addHostNameTokens = addHostNameTokens;
13071319
}
1320+
1321+
public DataConfig getDataConfig() {
1322+
return dataConfig;
1323+
}
1324+
1325+
public void setDataConfig(DataConfig dataConfig) {
1326+
this.dataConfig = dataConfig;
1327+
}
13081328
}

‎src/main/java/com/marklogic/appdeployer/DataConfig.java‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
package com.marklogic.appdeployer;
2+
3+
import java.io.File;
4+
import java.io.FileFilter;
5+
import java.util.ArrayList;
6+
import java.util.List;
7+
8+
public class DataConfig {
9+
10+
public final static String DEFAULT_DATA_PATH = "src/main/ml-data";
11+
12+
private List<String> dataPaths;
13+
private boolean dataLoadingEnabled = true;
14+
private String databaseName;
15+
private Integer batchSize;
16+
private boolean replaceTokensInData = true;
17+
private FileFilter fileFilter;
18+
private boolean logUris = true;
19+
20+
private String[] collections;
21+
22+
// Comma-delimited list of role,capability,role,capability
23+
private String permissions;
24+
25+
private File projectDir;
26+
27+
public DataConfig(File projectDir) {
28+
this.projectDir = projectDir;
29+
30+
dataPaths = new ArrayList<>();
31+
String path = projectDir != null ? new File(projectDir, DEFAULT_DATA_PATH).getAbsolutePath() : DEFAULT_DATA_PATH;
32+
dataPaths.add(path);
33+
}
34+
35+
public List<String> getDataPaths() {
36+
return dataPaths;
37+
}
38+
39+
public void setDataPaths(List<String> dataPaths) {
40+
this.dataPaths = dataPaths;
41+
}
42+
43+
public boolean isDataLoadingEnabled() {
44+
return dataLoadingEnabled;
45+
}
46+
47+
public void setDataLoadingEnabled(boolean dataLoadingEnabled) {
48+
this.dataLoadingEnabled = dataLoadingEnabled;
49+
}
50+
51+
public String getDatabaseName() {
52+
return databaseName;
53+
}
54+
55+
public void setDatabaseName(String databaseName) {
56+
this.databaseName = databaseName;
57+
}
58+
59+
public Integer getBatchSize() {
60+
return batchSize;
61+
}
62+
63+
public void setBatchSize(Integer batchSize) {
64+
this.batchSize = batchSize;
65+
}
66+
67+
public boolean isReplaceTokensInData() {
68+
return replaceTokensInData;
69+
}
70+
71+
public void setReplaceTokensInData(boolean replaceTokensInData) {
72+
this.replaceTokensInData = replaceTokensInData;
73+
}
74+
75+
public String getPermissions() {
76+
return permissions;
77+
}
78+
79+
public void setPermissions(String permissions) {
80+
this.permissions = permissions;
81+
}
82+
83+
public FileFilter getFileFilter() {
84+
return fileFilter;
85+
}
86+
87+
public void setFileFilter(FileFilter fileFilter) {
88+
this.fileFilter = fileFilter;
89+
}
90+
91+
public String[] getCollections() {
92+
return collections;
93+
}
94+
95+
public void setCollections(String[] collections) {
96+
this.collections = collections;
97+
}
98+
99+
public boolean isLogUris() {
100+
return logUris;
101+
}
102+
103+
public void setLogUris(boolean logUris) {
104+
this.logUris = logUris;
105+
}
106+
107+
public File getProjectDir() {
108+
return projectDir;
109+
}
110+
}

‎src/main/java/com/marklogic/appdeployer/DefaultAppConfigFactory.java‎

Lines changed: 59 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,21 @@ public DefaultAppConfigFactory(PropertySource propertySource) {
2525

2626
private Map<String, BiConsumer<AppConfig, String>> propertyConsumerMap;
2727

28+
@Override
29+
public AppConfig newAppConfig() {
30+
final AppConfig appConfig = new AppConfig(this.projectDir);
31+
for (String propertyName : propertyConsumerMap.keySet()) {
32+
String value = getProperty(propertyName);
33+
if (value != null) {
34+
propertyConsumerMap.get(propertyName).accept(appConfig, value);
35+
}
36+
}
37+
38+
setDefaultsForDatabasesWithForestsOnOneHost(appConfig);
39+
40+
return appConfig;
41+
}
42+
2843
/**
2944
* Registers all of the property handlers.
3045
*/
@@ -673,21 +688,55 @@ public void initialize() {
673688
logger.info("Update mimetype when properties are equal (defaults to false to avoid unnecessary ML restarts): " + prop);
674689
config.setUpdateMimetypeWhenPropertiesAreEqual(Boolean.parseBoolean(prop));
675690
});
691+
692+
registerDataLoadingProperties();
676693
}
677694

678-
@Override
679-
public AppConfig newAppConfig() {
680-
final AppConfig appConfig = new AppConfig(this.projectDir);
681-
for (String propertyName : propertyConsumerMap.keySet()) {
682-
String value = getProperty(propertyName);
683-
if (value != null) {
684-
propertyConsumerMap.get(propertyName).accept(appConfig, value);
695+
protected void registerDataLoadingProperties() {
696+
propertyConsumerMap.put("mlDataBatchSize", (config, prop) -> {
697+
logger.info("Batch size for loading data: " + prop);
698+
config.getDataConfig().setBatchSize(Integer.parseInt(prop));
699+
});
700+
701+
propertyConsumerMap.put("mlDataCollections", (config, prop) -> {
702+
logger.info("Collections that data will be loaded into: " + prop);
703+
config.getDataConfig().setCollections(prop.split(","));
704+
});
705+
706+
propertyConsumerMap.put("mlDataDatabaseName", (config, prop) -> {
707+
logger.info("Database that data will be loaded into: " + prop);
708+
config.getDataConfig().setDatabaseName(prop);
709+
});
710+
711+
propertyConsumerMap.put("mlDataPaths", (config, prop) -> {
712+
logger.info("Paths that data will be loaded from: " + prop);
713+
List<String> paths = new ArrayList<>();
714+
for (String s : prop.split(",")) {
715+
String path = this.projectDir != null ? new File(projectDir, s).getAbsolutePath() : s;
716+
paths.add(path);
685717
}
686-
}
718+
config.getDataConfig().setDataPaths(paths);
719+
});
687720

688-
setDefaultsForDatabasesWithForestsOnOneHost(appConfig);
721+
propertyConsumerMap.put("mlDataLoadingEnabled", (config, prop) -> {
722+
logger.info("Whether data loading is enabled: " + prop);
723+
config.getDataConfig().setDataLoadingEnabled(Boolean.parseBoolean(prop));
724+
});
689725

690-
return appConfig;
726+
propertyConsumerMap.put("mlDataLogUris", (config, prop) -> {
727+
logger.info("Log URIs when loading data: " + prop);
728+
config.getDataConfig().setLogUris(Boolean.parseBoolean(prop));
729+
});
730+
731+
propertyConsumerMap.put("mlDataPermissions", (config, prop) -> {
732+
logger.info("Permissions to be applied to loaded data: " + prop);
733+
config.getDataConfig().setPermissions(prop);
734+
});
735+
736+
propertyConsumerMap.put("mlDataReplaceTokens", (config, prop) -> {
737+
logger.info("Whether tokens will be replaced when loading data: " + prop);
738+
config.getDataConfig().setReplaceTokensInData(Boolean.parseBoolean(prop));
739+
});
691740
}
692741

693742
protected ConfigDir buildConfigDir(String path) {

‎src/main/java/com/marklogic/appdeployer/command/SortOrderConstants.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,8 @@ public abstract class SortOrderConstants {
7575

7676
public static Integer DEPLOY_FOREST_REPLICAS = 1200;
7777

78+
public static Integer LOAD_DATA = 1300;
79+
7880
public static Integer DELETE_MIMETYPES = 8500;
7981

8082
public static Integer UNASSIGN_HOSTS_FROM_GROUPS = 8590;

0 commit comments

Comments
 (0)