diff --git a/pom.xml b/pom.xml index f5bb33ec..7b474241 100644 --- a/pom.xml +++ b/pom.xml @@ -20,6 +20,8 @@ 11 11 11 + 2.48.0 + 20240303 @@ -136,6 +138,20 @@ + + + org.json + json + ${json.version} + + + + com.google.cloud + google-cloud-secretmanager + ${google-cloud-secretmanager.version} + + + org.junit.jupiter @@ -228,6 +244,12 @@ + + + com.google.common + shaded.com.google.common + + @@ -278,17 +300,17 @@ COMPLEXITY COVEREDRATIO - 0.33 + 0.20 INSTRUCTION COVEREDRATIO - 45% + 35% LINE MISSEDCOUNT - 1400 + 1500 diff --git a/src/main/java/com/datastax/cdm/properties/KnownProperties.java b/src/main/java/com/datastax/cdm/properties/KnownProperties.java index fc8c10be..74e44008 100644 --- a/src/main/java/com/datastax/cdm/properties/KnownProperties.java +++ b/src/main/java/com/datastax/cdm/properties/KnownProperties.java @@ -36,24 +36,30 @@ public enum PropertyType { // ========================================================================== // Common connection parameters // ========================================================================== + public static final String CONNECT_GCP_SECRET_PROJECT_ID = "spark.cdm.connect.gcp.secret.project.id"; + public static final String CONNECT_ORIGIN_HOST = "spark.cdm.connect.origin.host"; public static final String CONNECT_ORIGIN_PORT = "spark.cdm.connect.origin.port"; public static final String CONNECT_ORIGIN_SCB = "spark.cdm.connect.origin.scb"; public static final String CONNECT_ORIGIN_USERNAME = "spark.cdm.connect.origin.username"; public static final String CONNECT_ORIGIN_PASSWORD = "spark.cdm.connect.origin.password"; + public static final String CONNECT_ORIGIN_GCP_SECRET_NAME = "spark.cdm.connect.origin.gcp.secret.name"; public static final String CONNECT_TARGET_HOST = "spark.cdm.connect.target.host"; public static final String CONNECT_TARGET_PORT = "spark.cdm.connect.target.port"; public static final String CONNECT_TARGET_SCB = "spark.cdm.connect.target.scb"; public static final String CONNECT_TARGET_USERNAME = "spark.cdm.connect.target.username"; public static final String CONNECT_TARGET_PASSWORD = "spark.cdm.connect.target.password"; + public static final String CONNECT_TARGET_GCP_SECRET_NAME = "spark.cdm.connect.target.gcp.secret.name"; static { + types.put(CONNECT_GCP_SECRET_PROJECT_ID, PropertyType.STRING); types.put(CONNECT_ORIGIN_HOST, PropertyType.STRING); defaults.put(CONNECT_ORIGIN_HOST, "localhost"); types.put(CONNECT_ORIGIN_PORT, PropertyType.NUMBER); defaults.put(CONNECT_ORIGIN_PORT, "9042"); types.put(CONNECT_ORIGIN_SCB, PropertyType.STRING); + types.put(CONNECT_ORIGIN_GCP_SECRET_NAME, PropertyType.STRING); types.put(CONNECT_ORIGIN_USERNAME, PropertyType.STRING); defaults.put(CONNECT_ORIGIN_USERNAME, "cassandra"); types.put(CONNECT_ORIGIN_PASSWORD, PropertyType.STRING); @@ -64,6 +70,7 @@ public enum PropertyType { types.put(CONNECT_TARGET_PORT, PropertyType.NUMBER); defaults.put(CONNECT_TARGET_PORT, "9042"); types.put(CONNECT_TARGET_SCB, PropertyType.STRING); + types.put(CONNECT_TARGET_GCP_SECRET_NAME, PropertyType.STRING); types.put(CONNECT_TARGET_USERNAME, PropertyType.STRING); defaults.put(CONNECT_TARGET_USERNAME, "cassandra"); types.put(CONNECT_TARGET_PASSWORD, PropertyType.STRING); diff --git a/src/main/scala/com/datastax/cdm/job/ConnectionFetcher.scala b/src/main/scala/com/datastax/cdm/job/ConnectionFetcher.scala index 9a17b100..e73160a4 100644 --- a/src/main/scala/com/datastax/cdm/job/ConnectionFetcher.scala +++ b/src/main/scala/com/datastax/cdm/job/ConnectionFetcher.scala @@ -17,21 +17,36 @@ package com.datastax.cdm.job import com.datastax.cdm.properties.{KnownProperties, PropertyHelper} import com.datastax.spark.connector.cql.CassandraConnector +import com.google.cloud.secretmanager.v1.{AccessSecretVersionRequest, SecretManagerServiceClient, SecretPayload} import org.apache.spark.{SparkConf, SparkContext} +import org.json.JSONObject import org.slf4j.{Logger, LoggerFactory} // TODO: CDM-31 - add localDC configuration support class ConnectionFetcher(sparkContext: SparkContext, propertyHelper: PropertyHelper) { val logger: Logger = LoggerFactory.getLogger(this.getClass.getName) - def getConnectionDetails(side: String): ConnectionDetails = { + def getConnectionDetails(side: String): ConnectionDetails = { + val (username, password) = if ("ORIGIN".equals(side.toUpperCase)) { + getCredentials( + KnownProperties.CONNECT_ORIGIN_USERNAME, + KnownProperties.CONNECT_ORIGIN_PASSWORD, + KnownProperties.CONNECT_ORIGIN_GCP_SECRET_NAME + ) + } else { + getCredentials( + KnownProperties.CONNECT_TARGET_USERNAME, + KnownProperties.CONNECT_TARGET_PASSWORD, + KnownProperties.CONNECT_TARGET_GCP_SECRET_NAME + ) + } if ("ORIGIN".equals(side.toUpperCase)) { ConnectionDetails( propertyHelper.getAsString(KnownProperties.CONNECT_ORIGIN_SCB), propertyHelper.getAsString(KnownProperties.CONNECT_ORIGIN_HOST), propertyHelper.getAsString(KnownProperties.CONNECT_ORIGIN_PORT), - propertyHelper.getAsString(KnownProperties.CONNECT_ORIGIN_USERNAME), - propertyHelper.getAsString(KnownProperties.CONNECT_ORIGIN_PASSWORD), + username, + password, propertyHelper.getAsString(KnownProperties.ORIGIN_TLS_ENABLED), propertyHelper.getAsString(KnownProperties.ORIGIN_TLS_TRUSTSTORE_PATH), propertyHelper.getAsString(KnownProperties.ORIGIN_TLS_TRUSTSTORE_PASSWORD), @@ -40,14 +55,13 @@ class ConnectionFetcher(sparkContext: SparkContext, propertyHelper: PropertyHelp propertyHelper.getAsString(KnownProperties.ORIGIN_TLS_KEYSTORE_PASSWORD), propertyHelper.getAsString(KnownProperties.ORIGIN_TLS_ALGORITHMS) ) - } - else { + } else { ConnectionDetails( propertyHelper.getAsString(KnownProperties.CONNECT_TARGET_SCB), propertyHelper.getAsString(KnownProperties.CONNECT_TARGET_HOST), propertyHelper.getAsString(KnownProperties.CONNECT_TARGET_PORT), - propertyHelper.getAsString(KnownProperties.CONNECT_TARGET_USERNAME), - propertyHelper.getAsString(KnownProperties.CONNECT_TARGET_PASSWORD), + username, + password, propertyHelper.getAsString(KnownProperties.TARGET_TLS_ENABLED), propertyHelper.getAsString(KnownProperties.TARGET_TLS_TRUSTSTORE_PATH), propertyHelper.getAsString(KnownProperties.TARGET_TLS_TRUSTSTORE_PASSWORD), @@ -107,4 +121,40 @@ class ConnectionFetcher(sparkContext: SparkContext, propertyHelper: PropertyHelp .set("spark.cassandra.connection.port", connectionDetails.port)) } } + + private def getSecret(projectId: String, secretId: String): (String, String) = { + var client: SecretManagerServiceClient = null + try { + client = SecretManagerServiceClient.create() + val secretName = s"projects/$projectId/secrets/$secretId/versions/latest" + logger.info("Secret Name is: " + secretName) + val request = AccessSecretVersionRequest.newBuilder().setName(secretName).build() + val payload: SecretPayload = client.accessSecretVersion(request).getPayload + val jsonObject: JSONObject = new JSONObject(payload.getData.toStringUtf8) + + val client_id = jsonObject.get("client_id").toString + val secret = jsonObject.get("secret").toString + (client_id, secret) + } catch { + case e: Exception => throw new RuntimeException("Failed to get access secret ", e) + } finally { + if (client != null) client.close() + } + } + + private def getCredentials(usernameKey: String, passwordKey: String, secretNameKey: String): (String, String) = { + val username = propertyHelper.getAsString(usernameKey) + val password = propertyHelper.getAsString(passwordKey) + + if ((username != null && username.nonEmpty) && (password != null && password.nonEmpty)) { + logger.info("Using provided username and password...") + (username, password) + } else { + logger.info("Fetching credentials from GSM...") + val projectId = propertyHelper.getAsString(KnownProperties.CONNECT_GCP_SECRET_PROJECT_ID) + val secretName = propertyHelper.getAsString(secretNameKey) + val (username, password) = getSecret(projectId, secretName) + (username, password) + } + } } \ No newline at end of file diff --git a/src/resources/cdm.properties b/src/resources/cdm.properties index 80bf6b14..a2e9e291 100644 --- a/src/resources/cdm.properties +++ b/src/resources/cdm.properties @@ -25,16 +25,19 @@ # #********************************************************************************************************** #********************************************************************************************************** +spark.cdm.connect.gcp.secret.project.id spark.cdm.connect.origin.host localhost spark.cdm.connect.origin.port 9042 #spark.cdm.connect.origin.scb file:///aaa/bbb/secure-connect-enterprise.zip +spark.cdm.connect.origin.gcp.secret.name spark.cdm.connect.origin.username cassandra spark.cdm.connect.origin.password cassandra spark.cdm.connect.target.host localhost spark.cdm.connect.target.port 9042 #spark.cdm.connect.target.scb file:///aaa/bbb/secure-connect-enterprise.zip +spark.cdm.connect.target.gcp.secret.name spark.cdm.connect.target.username cassandra spark.cdm.connect.target.password cassandra