Skip to content
This repository was archived by the owner on Oct 5, 2021. It is now read-only.

Commit 2d161e2

Browse files
Alexander PatrikalakisAlexander Patrikalakis
Alexander Patrikalakis
authored and
Alexander Patrikalakis
committed
fixed local titan ids
1 parent 91efb55 commit 2d161e2

File tree

4 files changed

+44
-64
lines changed

4 files changed

+44
-64
lines changed

‎pom.xml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@
3333
<opencsv.version>2.4</opencsv.version>
3434
<metrics3.version>3.0.1</metrics3.version>
3535
<commons.logging.version>1.1.1</commons.logging.version>
36-
<hamcrest.version>1.3</hamcrest.version>
3736
<hadoop.version>2.2.0</hadoop.version>
3837
<mockito.version>1.9.5</mockito.version>
3938
<lombok.version>1.16.16</lombok.version>
@@ -329,7 +328,7 @@
329328
<dynamodb-control-plane-rate>10000</dynamodb-control-plane-rate>
330329
<dynamodb-unlimited-iops>true</dynamodb-unlimited-iops>
331330
<properties-file>src/test/resources/dynamodb-local.properties</properties-file>
332-
<dynamo.endpoint>http://localhost:${dynamodb-local.port}</dynamo.endpoint>
331+
<dynamo.endpoint>${dynamodb-local.endpoint}</dynamo.endpoint>
333332
</systemPropertyVariables>
334333
</configuration>
335334
</execution>

‎src/main/java/com/amazon/janusgraph/diskstorage/dynamodb/Client.java

Lines changed: 27 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import com.amazonaws.auth.AWSCredentialsProvider;
3737
import com.amazonaws.auth.AWSStaticCredentialsProvider;
3838
import com.google.common.annotations.VisibleForTesting;
39+
import com.google.common.base.Preconditions;
3940
import com.google.common.base.Strings;
4041
import com.google.common.util.concurrent.RateLimiter;
4142
import com.google.common.util.concurrent.RateLimiterCreator;
@@ -68,24 +69,24 @@ public class Client {
6869
private final String prefix;
6970

7071
public Client(org.janusgraph.diskstorage.configuration.Configuration config) {
71-
String credentialsClassName = config.get(Constants.DYNAMODB_CREDENTIALS_CLASS_NAME);
72-
Class<?> clazz;
72+
final String credentialsClassName = config.get(Constants.DYNAMODB_CREDENTIALS_CLASS_NAME);
73+
final Class<?> clazz;
7374
try {
7475
clazz = Class.forName(credentialsClassName);
7576
} catch (ClassNotFoundException e) {
7677
throw new IllegalArgumentException(VALIDATE_CREDENTIALS_CLASS_NAME, e);
7778
}
7879

79-
String[] credentialsConstructorArgsValues = config.get(Constants.DYNAMODB_CREDENTIALS_CONSTRUCTOR_ARGS);
80-
final List<String> filteredArgList = new ArrayList<String>();
80+
final String[] credentialsConstructorArgsValues = config.get(Constants.DYNAMODB_CREDENTIALS_CONSTRUCTOR_ARGS);
81+
final List<String> filteredArgList = new ArrayList<>();
8182
for(Object obj : credentialsConstructorArgsValues) {
8283
final String str = obj.toString();
8384
if(!str.isEmpty()) {
8485
filteredArgList.add(str);
8586
}
8687
}
8788

88-
AWSCredentialsProvider credentialsProvider;
89+
final AWSCredentialsProvider credentialsProvider;
8990
if (AWSCredentials.class.isAssignableFrom(clazz)) {
9091
AWSCredentials credentials = createCredentials(clazz, filteredArgList.toArray(new String[filteredArgList.size()]));
9192
credentialsProvider = new AWSStaticCredentialsProvider(credentials);
@@ -97,24 +98,23 @@ public Client(org.janusgraph.diskstorage.configuration.Configuration config) {
9798
//begin adaptation of constructor at
9899
//https://github.com/buka/titan/blob/master/src/main/java/com/thinkaurelius/titan/diskstorage/dynamodb/DynamoDBClient.java#L77
99100
ClientConfiguration clientConfig = new ClientConfiguration();
100-
clientConfig.withConnectionTimeout(config.get(Constants.DYNAMODB_CLIENT_CONN_TIMEOUT)) //
101-
.withConnectionTTL(config.get(Constants.DYNAMODB_CLIENT_CONN_TTL)) //
102-
.withMaxConnections(config.get(Constants.DYNAMODB_CLIENT_MAX_CONN)) //
103-
.withMaxErrorRetry(config.get(Constants.DYNAMODB_CLIENT_MAX_ERROR_RETRY)) //
104-
.withGzip(config.get(Constants.DYNAMODB_CLIENT_USE_GZIP)) //
105-
.withReaper(config.get(Constants.DYNAMODB_CLIENT_USE_REAPER)) //
106-
.withUserAgentSuffix(config.get(Constants.DYNAMODB_CLIENT_USER_AGENT)) //
107-
.withSocketTimeout(config.get(Constants.DYNAMODB_CLIENT_SOCKET_TIMEOUT)) //
108-
.withSocketBufferSizeHints( //
109-
config.get(Constants.DYNAMODB_CLIENT_SOCKET_BUFFER_SEND_HINT), //
110-
config.get(Constants.DYNAMODB_CLIENT_SOCKET_BUFFER_RECV_HINT)) //
111-
.withProxyDomain(config.get(Constants.DYNAMODB_CLIENT_PROXY_DOMAIN)) //
112-
.withProxyWorkstation(config.get(Constants.DYNAMODB_CLIENT_PROXY_WORKSTATION)) //
113-
.withProxyHost(config.get(Constants.DYNAMODB_CLIENT_PROXY_HOST)) //
114-
.withProxyPort(config.get(Constants.DYNAMODB_CLIENT_PROXY_PORT)) //
115-
.withProxyUsername(config.get(Constants.DYNAMODB_CLIENT_PROXY_USERNAME)) //
116-
.withProxyPassword(config.get(Constants.DYNAMODB_CLIENT_PROXY_PASSWORD)); //
117-
101+
clientConfig.withConnectionTimeout(config.get(Constants.DYNAMODB_CLIENT_CONN_TIMEOUT))
102+
.withConnectionTTL(config.get(Constants.DYNAMODB_CLIENT_CONN_TTL))
103+
.withMaxConnections(config.get(Constants.DYNAMODB_CLIENT_MAX_CONN))
104+
.withMaxErrorRetry(config.get(Constants.DYNAMODB_CLIENT_MAX_ERROR_RETRY))
105+
.withGzip(config.get(Constants.DYNAMODB_CLIENT_USE_GZIP))
106+
.withReaper(config.get(Constants.DYNAMODB_CLIENT_USE_REAPER))
107+
.withUserAgentSuffix(config.get(Constants.DYNAMODB_CLIENT_USER_AGENT))
108+
.withSocketTimeout(config.get(Constants.DYNAMODB_CLIENT_SOCKET_TIMEOUT))
109+
.withSocketBufferSizeHints(
110+
config.get(Constants.DYNAMODB_CLIENT_SOCKET_BUFFER_SEND_HINT),
111+
config.get(Constants.DYNAMODB_CLIENT_SOCKET_BUFFER_RECV_HINT))
112+
.withProxyDomain(config.get(Constants.DYNAMODB_CLIENT_PROXY_DOMAIN))
113+
.withProxyWorkstation(config.get(Constants.DYNAMODB_CLIENT_PROXY_WORKSTATION))
114+
.withProxyHost(config.get(Constants.DYNAMODB_CLIENT_PROXY_HOST))
115+
.withProxyPort(config.get(Constants.DYNAMODB_CLIENT_PROXY_PORT))
116+
.withProxyUsername(config.get(Constants.DYNAMODB_CLIENT_PROXY_USERNAME))
117+
.withProxyPassword(config.get(Constants.DYNAMODB_CLIENT_PROXY_PASSWORD));
118118
forceConsistentRead = config.get(Constants.DYNAMODB_FORCE_CONSISTENT_READ);
119119
//end adaptation of constructor at
120120
//https://github.com/buka/titan/blob/master/src/main/java/com/thinkaurelius/titan/diskstorage/dynamodb/DynamoDBClient.java#L77
@@ -139,11 +139,9 @@ public Client(org.janusgraph.diskstorage.configuration.Configuration config) {
139139
final Map<String, RateLimiter> readRateLimit = new HashMap<>();
140140
final Map<String, RateLimiter> writeRateLimit = new HashMap<>();
141141

142-
Set<String> storeNames = new HashSet<String>(Constants.REQUIRED_BACKEND_STORES);
142+
final Set<String> storeNames = new HashSet<>(config.get(Constants.DYNAMODB_USE_TITAN_ID_STORE) ? Constants.ALTERNATE_BACKEND_STORES : Constants.REQUIRED_BACKEND_STORES);
143143
storeNames.addAll(config.getContainedNamespaces(Constants.DYNAMODB_STORES_NAMESPACE));
144-
for(String storeName : storeNames) {
145-
setupStore(config, prefix, readRateLimit, writeRateLimit, storeName);
146-
}
144+
storeNames.forEach(storeName -> setupStore(config, prefix, readRateLimit, writeRateLimit, storeName));
147145

148146
endpoint = JanusGraphConfigUtil.getNullableConfigValue(config, Constants.DYNAMODB_CLIENT_ENDPOINT);
149147
signingRegion = JanusGraphConfigUtil.getNullableConfigValue(config, Constants.DYNAMODB_CLIENT_SIGNING_REGION);
@@ -201,10 +199,12 @@ public boolean enableParallelScan() {
201199
}
202200

203201
public long readCapacity(String tableName) {
202+
Preconditions.checkNotNull(tableName, "table name may not be null when looking up read capacity");
204203
return capacityRead.get(tableName);
205204
}
206205

207206
public long writeCapacity(String tableName) {
207+
Preconditions.checkNotNull(tableName, "table name may not be null when looking up write capacity");
208208
return capacityWrite.get(tableName);
209209
}
210210

@@ -216,27 +216,6 @@ public int scanLimit(String tableName) {
216216
return scanLimit.get(tableName);
217217
}
218218

219-
public static final AWSCredentialsProvider createAWSCredentialsProvider(String credentialsClassName,
220-
String[] credentialsConstructorArgsValues) {
221-
Class<?> clazz;
222-
try {
223-
clazz = Class.forName(credentialsClassName);
224-
} catch (ClassNotFoundException e) {
225-
throw new IllegalArgumentException(VALIDATE_CREDENTIALS_CLASS_NAME, e);
226-
}
227-
AWSCredentialsProvider credentialsProvider;
228-
if (AWSCredentials.class.isAssignableFrom(clazz)) {
229-
AWSCredentials credentials = createCredentials(clazz, credentialsConstructorArgsValues);
230-
credentialsProvider = new AWSStaticCredentialsProvider(credentials);
231-
} else if (AWSCredentialsProvider.class.isAssignableFrom(clazz)) {
232-
credentialsProvider = createCredentialsProvider(clazz, credentialsConstructorArgsValues);
233-
} else {
234-
throw new IllegalArgumentException(VALIDATE_CREDENTIALS_CLASS_NAME);
235-
}
236-
237-
return credentialsProvider;
238-
}
239-
240219
private static final AWSCredentialsProvider createCredentialsProvider(Class<?> clazz, String[] credentialsProviderConstructorArgs) {
241220
return (AWSCredentialsProvider) createInstance(clazz, credentialsProviderConstructorArgs);
242221
}

‎src/main/java/com/amazon/janusgraph/diskstorage/dynamodb/Constants.java

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
* Constants for the DynamoDB backend.
3434
*
3535
* @author Matthew Sowders
36+
* @author Alexander Patrikalakis
3637
*
3738
*/
3839
public class Constants {
@@ -47,10 +48,16 @@ public class Constants {
4748
public static final String JANUSGRAPH_USER_AGENT = "dynamodb-janusgraph010-storage-backend_1.0.0";
4849

4950
public static final List<String> REQUIRED_BACKEND_STORES = Arrays.asList(Backend.EDGESTORE_NAME, //
50-
Backend.INDEXSTORE_NAME, //
51-
Backend.ID_STORE_NAME, //
52-
Backend.SYSTEM_TX_LOG_NAME, //
53-
Backend.SYSTEM_MGMT_LOG_NAME, //
51+
Backend.INDEXSTORE_NAME,
52+
Backend.ID_STORE_NAME,
53+
Backend.SYSTEM_TX_LOG_NAME,
54+
Backend.SYSTEM_MGMT_LOG_NAME,
55+
GraphDatabaseConfiguration.SYSTEM_PROPERTIES_STORE_NAME);
56+
public static final List<String> ALTERNATE_BACKEND_STORES = Arrays.asList(Backend.EDGESTORE_NAME, //
57+
"titan_ids",
58+
Backend.ID_STORE_NAME,
59+
Backend.SYSTEM_TX_LOG_NAME,
60+
Backend.SYSTEM_MGMT_LOG_NAME,
5461
GraphDatabaseConfiguration.SYSTEM_PROPERTIES_STORE_NAME);
5562

5663
public static final ConfigNamespace DYNAMODB_CONFIGURATION_NAMESPACE =
@@ -85,8 +92,8 @@ public class Constants {
8592
new ConfigOption<>(DYNAMODB_CONFIGURATION_NAMESPACE, "enable-parallel-scans",
8693
"This feature enables scans to run in parallel, which should decrease the total blocking time "
8794
+ "spent when iterating over large sets of vertices. "
88-
+ "WARNING: while this feature is enabled Titan's OLAP libraries are NOT supported."
89-
+ "The Fulgora implementations of OLAP rely on consistent scan orders across multiple scans, "
95+
+ "WARNING: while this feature is enabled JanusGraph's OLAP libraries are NOT supported."
96+
+ "The JanusGraph-Hadoop implementations of OLAP rely on consistent scan orders across multiple scans, "
9097
+ "which cannot be guaranteed when scans are run in parallel",
9198
LOCAL, false);
9299
public static final ConfigOption<String> STORES_DATA_MODEL =
@@ -126,11 +133,11 @@ public class Constants {
126133
public static final ConfigOption<Long> STORES_INITIAL_CAPACITY_READ =
127134
new ConfigOption<>(Constants.DYNAMODB_STORES_NAMESPACE, "initial-capacity-read",
128135
"Define the initial read capacity for a given dynamodb table.",
129-
LOCAL, 4L);
136+
LOCAL, 4L);
130137
public static final ConfigOption<Long> STORES_INITIAL_CAPACITY_WRITE =
131138
new ConfigOption<>(Constants.DYNAMODB_STORES_NAMESPACE, "initial-capacity-write",
132139
"Define the initial write capacity for a given dynamodb table.",
133-
LOCAL, 4L);
140+
LOCAL, 4L);
134141
public static final ConfigOption<Double> STORES_READ_RATE_LIMIT =
135142
new ConfigOption<>(Constants.DYNAMODB_STORES_NAMESPACE, "read-rate",
136143
"The max number of reads per second.",

‎src/test/java/com/amazon/janusgraph/TestGraphUtil.java

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -57,11 +57,6 @@ public enum TestGraphUtil {
5757
private final boolean unlimitedIops;
5858
private final int provisionedReadAndWriteTps;
5959
private final File propertiesFile;
60-
private static final List<String> ALTERNATE_REQUIRED_STORES = new ArrayList<>(Constants.REQUIRED_BACKEND_STORES);
61-
static {
62-
ALTERNATE_REQUIRED_STORES.remove(Backend.ID_STORE_NAME);
63-
ALTERNATE_REQUIRED_STORES.add("titan_ids");
64-
}
6560

6661
TestGraphUtil() {
6762
dynamoDBPartitions = Integer.valueOf(System.getProperty("dynamodb-partitions", String.valueOf(1)));
@@ -118,7 +113,7 @@ public Configuration createTestGraphConfig(final BackendDataModel backendDataMod
118113
final String storesNsPrefix = "storage.dynamodb.stores.";
119114
final List<String> storeList;
120115
if (properties.getBoolean("storage.dynamodb.use-titan-ids", true)) {
121-
storeList = ALTERNATE_REQUIRED_STORES;
116+
storeList = Constants.ALTERNATE_BACKEND_STORES;
122117
} else {
123118
storeList = Constants.REQUIRED_BACKEND_STORES;
124119
}

0 commit comments

Comments
 (0)