diff --git a/pom.xml b/pom.xml
index f5bb33ec..7b474241 100644
--- a/pom.xml
+++ b/pom.xml
@@ -20,6 +20,8 @@
11
11
11
+ 2.48.0
+ 20240303
@@ -136,6 +138,20 @@
+
+
+ org.json
+ json
+ ${json.version}
+
+
+
+ com.google.cloud
+ google-cloud-secretmanager
+ ${google-cloud-secretmanager.version}
+
+
+
org.junit.jupiter
@@ -228,6 +244,12 @@
+
+
+ com.google.common
+ shaded.com.google.common
+
+
@@ -278,17 +300,17 @@
COMPLEXITY
COVEREDRATIO
- 0.33
+ 0.20
INSTRUCTION
COVEREDRATIO
- 45%
+ 35%
LINE
MISSEDCOUNT
- 1400
+ 1500
diff --git a/src/main/java/com/datastax/cdm/properties/KnownProperties.java b/src/main/java/com/datastax/cdm/properties/KnownProperties.java
index fc8c10be..74e44008 100644
--- a/src/main/java/com/datastax/cdm/properties/KnownProperties.java
+++ b/src/main/java/com/datastax/cdm/properties/KnownProperties.java
@@ -36,24 +36,30 @@ public enum PropertyType {
// ==========================================================================
// Common connection parameters
// ==========================================================================
+ public static final String CONNECT_GCP_SECRET_PROJECT_ID = "spark.cdm.connect.gcp.secret.project.id";
+
public static final String CONNECT_ORIGIN_HOST = "spark.cdm.connect.origin.host";
public static final String CONNECT_ORIGIN_PORT = "spark.cdm.connect.origin.port";
public static final String CONNECT_ORIGIN_SCB = "spark.cdm.connect.origin.scb";
public static final String CONNECT_ORIGIN_USERNAME = "spark.cdm.connect.origin.username";
public static final String CONNECT_ORIGIN_PASSWORD = "spark.cdm.connect.origin.password";
+ public static final String CONNECT_ORIGIN_GCP_SECRET_NAME = "spark.cdm.connect.origin.gcp.secret.name";
public static final String CONNECT_TARGET_HOST = "spark.cdm.connect.target.host";
public static final String CONNECT_TARGET_PORT = "spark.cdm.connect.target.port";
public static final String CONNECT_TARGET_SCB = "spark.cdm.connect.target.scb";
public static final String CONNECT_TARGET_USERNAME = "spark.cdm.connect.target.username";
public static final String CONNECT_TARGET_PASSWORD = "spark.cdm.connect.target.password";
+ public static final String CONNECT_TARGET_GCP_SECRET_NAME = "spark.cdm.connect.target.gcp.secret.name";
static {
+ types.put(CONNECT_GCP_SECRET_PROJECT_ID, PropertyType.STRING);
types.put(CONNECT_ORIGIN_HOST, PropertyType.STRING);
defaults.put(CONNECT_ORIGIN_HOST, "localhost");
types.put(CONNECT_ORIGIN_PORT, PropertyType.NUMBER);
defaults.put(CONNECT_ORIGIN_PORT, "9042");
types.put(CONNECT_ORIGIN_SCB, PropertyType.STRING);
+ types.put(CONNECT_ORIGIN_GCP_SECRET_NAME, PropertyType.STRING);
types.put(CONNECT_ORIGIN_USERNAME, PropertyType.STRING);
defaults.put(CONNECT_ORIGIN_USERNAME, "cassandra");
types.put(CONNECT_ORIGIN_PASSWORD, PropertyType.STRING);
@@ -64,6 +70,7 @@ public enum PropertyType {
types.put(CONNECT_TARGET_PORT, PropertyType.NUMBER);
defaults.put(CONNECT_TARGET_PORT, "9042");
types.put(CONNECT_TARGET_SCB, PropertyType.STRING);
+ types.put(CONNECT_TARGET_GCP_SECRET_NAME, PropertyType.STRING);
types.put(CONNECT_TARGET_USERNAME, PropertyType.STRING);
defaults.put(CONNECT_TARGET_USERNAME, "cassandra");
types.put(CONNECT_TARGET_PASSWORD, PropertyType.STRING);
diff --git a/src/main/scala/com/datastax/cdm/job/ConnectionFetcher.scala b/src/main/scala/com/datastax/cdm/job/ConnectionFetcher.scala
index 9a17b100..e73160a4 100644
--- a/src/main/scala/com/datastax/cdm/job/ConnectionFetcher.scala
+++ b/src/main/scala/com/datastax/cdm/job/ConnectionFetcher.scala
@@ -17,21 +17,36 @@ package com.datastax.cdm.job
import com.datastax.cdm.properties.{KnownProperties, PropertyHelper}
import com.datastax.spark.connector.cql.CassandraConnector
+import com.google.cloud.secretmanager.v1.{AccessSecretVersionRequest, SecretManagerServiceClient, SecretPayload}
import org.apache.spark.{SparkConf, SparkContext}
+import org.json.JSONObject
import org.slf4j.{Logger, LoggerFactory}
// TODO: CDM-31 - add localDC configuration support
class ConnectionFetcher(sparkContext: SparkContext, propertyHelper: PropertyHelper) {
val logger: Logger = LoggerFactory.getLogger(this.getClass.getName)
- def getConnectionDetails(side: String): ConnectionDetails = {
+ def getConnectionDetails(side: String): ConnectionDetails = {
+ val (username, password) = if ("ORIGIN".equals(side.toUpperCase)) {
+ getCredentials(
+ KnownProperties.CONNECT_ORIGIN_USERNAME,
+ KnownProperties.CONNECT_ORIGIN_PASSWORD,
+ KnownProperties.CONNECT_ORIGIN_GCP_SECRET_NAME
+ )
+ } else {
+ getCredentials(
+ KnownProperties.CONNECT_TARGET_USERNAME,
+ KnownProperties.CONNECT_TARGET_PASSWORD,
+ KnownProperties.CONNECT_TARGET_GCP_SECRET_NAME
+ )
+ }
if ("ORIGIN".equals(side.toUpperCase)) {
ConnectionDetails(
propertyHelper.getAsString(KnownProperties.CONNECT_ORIGIN_SCB),
propertyHelper.getAsString(KnownProperties.CONNECT_ORIGIN_HOST),
propertyHelper.getAsString(KnownProperties.CONNECT_ORIGIN_PORT),
- propertyHelper.getAsString(KnownProperties.CONNECT_ORIGIN_USERNAME),
- propertyHelper.getAsString(KnownProperties.CONNECT_ORIGIN_PASSWORD),
+ username,
+ password,
propertyHelper.getAsString(KnownProperties.ORIGIN_TLS_ENABLED),
propertyHelper.getAsString(KnownProperties.ORIGIN_TLS_TRUSTSTORE_PATH),
propertyHelper.getAsString(KnownProperties.ORIGIN_TLS_TRUSTSTORE_PASSWORD),
@@ -40,14 +55,13 @@ class ConnectionFetcher(sparkContext: SparkContext, propertyHelper: PropertyHelp
propertyHelper.getAsString(KnownProperties.ORIGIN_TLS_KEYSTORE_PASSWORD),
propertyHelper.getAsString(KnownProperties.ORIGIN_TLS_ALGORITHMS)
)
- }
- else {
+ } else {
ConnectionDetails(
propertyHelper.getAsString(KnownProperties.CONNECT_TARGET_SCB),
propertyHelper.getAsString(KnownProperties.CONNECT_TARGET_HOST),
propertyHelper.getAsString(KnownProperties.CONNECT_TARGET_PORT),
- propertyHelper.getAsString(KnownProperties.CONNECT_TARGET_USERNAME),
- propertyHelper.getAsString(KnownProperties.CONNECT_TARGET_PASSWORD),
+ username,
+ password,
propertyHelper.getAsString(KnownProperties.TARGET_TLS_ENABLED),
propertyHelper.getAsString(KnownProperties.TARGET_TLS_TRUSTSTORE_PATH),
propertyHelper.getAsString(KnownProperties.TARGET_TLS_TRUSTSTORE_PASSWORD),
@@ -107,4 +121,40 @@ class ConnectionFetcher(sparkContext: SparkContext, propertyHelper: PropertyHelp
.set("spark.cassandra.connection.port", connectionDetails.port))
}
}
+
+ private def getSecret(projectId: String, secretId: String): (String, String) = {
+ var client: SecretManagerServiceClient = null
+ try {
+ client = SecretManagerServiceClient.create()
+ val secretName = s"projects/$projectId/secrets/$secretId/versions/latest"
+ logger.info("Secret Name is: " + secretName)
+ val request = AccessSecretVersionRequest.newBuilder().setName(secretName).build()
+ val payload: SecretPayload = client.accessSecretVersion(request).getPayload
+ val jsonObject: JSONObject = new JSONObject(payload.getData.toStringUtf8)
+
+ val client_id = jsonObject.get("client_id").toString
+ val secret = jsonObject.get("secret").toString
+ (client_id, secret)
+ } catch {
+ case e: Exception => throw new RuntimeException("Failed to get access secret ", e)
+ } finally {
+ if (client != null) client.close()
+ }
+ }
+
+ private def getCredentials(usernameKey: String, passwordKey: String, secretNameKey: String): (String, String) = {
+ val username = propertyHelper.getAsString(usernameKey)
+ val password = propertyHelper.getAsString(passwordKey)
+
+ if ((username != null && username.nonEmpty) && (password != null && password.nonEmpty)) {
+ logger.info("Using provided username and password...")
+ (username, password)
+ } else {
+ logger.info("Fetching credentials from GSM...")
+ val projectId = propertyHelper.getAsString(KnownProperties.CONNECT_GCP_SECRET_PROJECT_ID)
+ val secretName = propertyHelper.getAsString(secretNameKey)
+ val (username, password) = getSecret(projectId, secretName)
+ (username, password)
+ }
+ }
}
\ No newline at end of file
diff --git a/src/resources/cdm.properties b/src/resources/cdm.properties
index 80bf6b14..a2e9e291 100644
--- a/src/resources/cdm.properties
+++ b/src/resources/cdm.properties
@@ -25,16 +25,19 @@
#
#**********************************************************************************************************
#**********************************************************************************************************
+spark.cdm.connect.gcp.secret.project.id
spark.cdm.connect.origin.host localhost
spark.cdm.connect.origin.port 9042
#spark.cdm.connect.origin.scb file:///aaa/bbb/secure-connect-enterprise.zip
+spark.cdm.connect.origin.gcp.secret.name
spark.cdm.connect.origin.username cassandra
spark.cdm.connect.origin.password cassandra
spark.cdm.connect.target.host localhost
spark.cdm.connect.target.port 9042
#spark.cdm.connect.target.scb file:///aaa/bbb/secure-connect-enterprise.zip
+spark.cdm.connect.target.gcp.secret.name
spark.cdm.connect.target.username cassandra
spark.cdm.connect.target.password cassandra