Enable listing serverless databases in the explorer (#2407)

abrooksv · web-flow · commit f2b012c3b4b2 · 2021-02-04T15:52:08.000-08:00
diff --git a/.changes/next-release/feature-cea2195d-89dd-47a7-9492-f42057a28a6c.json b/.changes/next-release/feature-cea2195d-89dd-47a7-9492-f42057a28a6c.json
@@ -0,0 +1,4 @@
+{
+  "type" : "feature",
+  "description" : "RDS serverless databases are now visible in the RDS node in the explorer"
+}
diff --git a/jetbrains-core/src/software/aws/toolkits/jetbrains/core/utils/CollectionUtils.kt b/jetbrains-core/src/software/aws/toolkits/jetbrains/core/utils/CollectionUtils.kt
@@ -7,3 +7,8 @@ package software.aws.toolkits.jetbrains.core.utils
  * Replace with [kotlin.collections.buildList] when experimental is removed
  */
 inline fun <E> buildList(builderAction: MutableList<E>.() -> Unit): List<E> = ArrayList<E>().apply(builderAction)
+
+/*
+ * Replace with [kotlin.collections.buildMap] when experimental is removed
+ */
+inline fun <T, U> buildMap(builderAction: MutableMap<T, U>.() -> Unit): Map<T, U> = mutableMapOf<T, U>().apply(builderAction)
diff --git a/jetbrains-ultimate/src-201/software/aws/toolkits/jetbrains/services/rds/resources/RdsResources.kt b/jetbrains-ultimate/src-201/software/aws/toolkits/jetbrains/services/rds/resources/RdsResources.kt
@@ -4,18 +4,19 @@
 package software.aws.toolkits.jetbrains.services.rds.resources
 
 import software.amazon.awssdk.services.rds.RdsClient
-import software.amazon.awssdk.services.rds.model.DBInstance
 import software.amazon.awssdk.services.rds.model.Filter
 import software.aws.toolkits.jetbrains.core.ClientBackedCachedResource
 import software.aws.toolkits.jetbrains.core.Resource
 import software.aws.toolkits.jetbrains.services.rds.AuroraMySql
+import software.aws.toolkits.jetbrains.services.rds.Endpoint
+import software.aws.toolkits.jetbrains.services.rds.RdsDatabase
 import software.aws.toolkits.jetbrains.services.rds.RdsEngine
 
 // FIX_WHEN_MIN_IS_202 remove this one and merge the 202+ one into RdsResources
 // Filters are also just a string
 private const val ENGINE_FILTER = "engine"
 
-val LIST_SUPPORTED_INSTANCES: Resource.Cached<List<DBInstance>> = ClientBackedCachedResource(RdsClient::class, "rds.list_supported_instances") {
+val LIST_SUPPORTED_INSTANCES: Resource.Cached<List<RdsDatabase>> = ClientBackedCachedResource(RdsClient::class, "rds.list_supported_instances") {
     describeDBInstancesPaginator {
         it.filters(
             Filter.builder()
@@ -28,5 +29,48 @@ val LIST_SUPPORTED_INSTANCES: Resource.Cached<List<DBInstance>> = ClientBackedCa
                         .flatMap { e -> e.engines }
                 ).build()
         )
-    }.dbInstances().toList()
+    }.dbInstances()
+        .map {
+            RdsDatabase(
+                // if it has a cluster (aurora), use that as the identifier so it will de-dupe
+                identifier = it.dbClusterIdentifier() ?: it.dbInstanceIdentifier(),
+                arn = it.dbInstanceArn(),
+                engine = it.engine(),
+                iamDatabaseAuthenticationEnabled = it.iamDatabaseAuthenticationEnabled(),
+                masterUsername = it.masterUsername(),
+                endpoint = Endpoint(
+                    host = it.endpoint().address(),
+                    port = it.endpoint().port()
+                )
+            )
+        }.toList()
+}
+
+val LIST_SUPPORTED_CLUSTERS: Resource.Cached<List<RdsDatabase>> = ClientBackedCachedResource(RdsClient::class, "rds.list_supported_cluster") {
+    describeDBClustersPaginator {
+        it.filters(
+            Filter.builder()
+                .name(ENGINE_FILTER)
+                .values(
+                    RdsEngine
+                        .values()
+                        // Filter out AuroraMySql because it is only supported on 202+
+                        .filterNot { e -> e == AuroraMySql }
+                        .flatMap { e -> e.engines }
+                ).build()
+        )
+    }.dbClusters()
+        .map {
+            RdsDatabase(
+                identifier = it.dbClusterIdentifier(),
+                arn = it.dbClusterArn(),
+                engine = it.engine(),
+                iamDatabaseAuthenticationEnabled = it.iamDatabaseAuthenticationEnabled(),
+                masterUsername = it.masterUsername(),
+                endpoint = Endpoint(
+                    host = it.endpoint(),
+                    port = it.port()
+                )
+            )
+        }.toList()
 }
diff --git a/jetbrains-ultimate/src-202+/software/aws/toolkits/jetbrains/services/rds/resources/RdsResources.kt b/jetbrains-ultimate/src-202+/software/aws/toolkits/jetbrains/services/rds/resources/RdsResources.kt
@@ -4,17 +4,18 @@
 package software.aws.toolkits.jetbrains.services.rds.resources
 
 import software.amazon.awssdk.services.rds.RdsClient
-import software.amazon.awssdk.services.rds.model.DBInstance
 import software.amazon.awssdk.services.rds.model.Filter
 import software.aws.toolkits.jetbrains.core.ClientBackedCachedResource
 import software.aws.toolkits.jetbrains.core.Resource
+import software.aws.toolkits.jetbrains.services.rds.Endpoint
+import software.aws.toolkits.jetbrains.services.rds.RdsDatabase
 import software.aws.toolkits.jetbrains.services.rds.RdsEngine
 
 // FIX_WHEN_MIN_IS_202 make this one the default in RdsResources.kt
 // Filters are also just a string
 private const val ENGINE_FILTER = "engine"
 
-val LIST_SUPPORTED_INSTANCES: Resource.Cached<List<DBInstance>> = ClientBackedCachedResource(RdsClient::class, "rds.list_supported_instances") {
+val LIST_SUPPORTED_INSTANCES: Resource.Cached<List<RdsDatabase>> = ClientBackedCachedResource(RdsClient::class, "rds.list_supported_instances") {
     describeDBInstancesPaginator {
         it.filters(
             Filter.builder()
@@ -25,5 +26,46 @@ val LIST_SUPPORTED_INSTANCES: Resource.Cached<List<DBInstance>> = ClientBackedCa
                         .flatMap { e -> e.engines }
                 ).build()
         )
-    }.dbInstances().toList()
+    }.dbInstances()
+        .map {
+            RdsDatabase(
+                // if it has a cluster (aurora), use that as the identifier so it will de-dupe
+                identifier = it.dbClusterIdentifier() ?: it.dbInstanceIdentifier(),
+                arn = it.dbInstanceArn(),
+                engine = it.engine(),
+                iamDatabaseAuthenticationEnabled = it.iamDatabaseAuthenticationEnabled(),
+                masterUsername = it.masterUsername(),
+                endpoint = Endpoint(
+                    host = it.endpoint().address(),
+                    port = it.endpoint().port()
+                )
+            )
+        }.toList()
+}
+
+val LIST_SUPPORTED_CLUSTERS: Resource.Cached<List<RdsDatabase>> = ClientBackedCachedResource(RdsClient::class, "rds.list_supported_cluster") {
+    describeDBClustersPaginator {
+        it.filters(
+            Filter.builder()
+                .name(ENGINE_FILTER)
+                .values(
+                    RdsEngine
+                        .values()
+                        .flatMap { e -> e.engines }
+                ).build()
+        )
+    }.dbClusters()
+        .map {
+            RdsDatabase(
+                identifier = it.dbClusterIdentifier(),
+                arn = it.dbClusterArn(),
+                engine = it.engine(),
+                iamDatabaseAuthenticationEnabled = it.iamDatabaseAuthenticationEnabled(),
+                masterUsername = it.masterUsername(),
+                endpoint = Endpoint(
+                    host = it.endpoint(),
+                    port = it.port()
+                )
+            )
+        }.toList()
 }
diff --git a/jetbrains-ultimate/src/software/aws/toolkits/jetbrains/datagrip/DatabaseSecret.kt b/jetbrains-ultimate/src/software/aws/toolkits/jetbrains/datagrip/DatabaseSecret.kt
@@ -44,14 +44,14 @@ object DatabaseSecret {
         // If it is a resource node, validate that it is the same resource
         when (node) {
             is RdsNode -> {
-                if (node.dbInstance.engine() != dbSecret.engine) return ValidationInfo(
+                if (node.database.engine != dbSecret.engine) return ValidationInfo(
                     message(
                         "datagrip.secretsmanager.validation.different_engine",
                         secretName,
                         dbSecret.engine.toString()
                     )
                 )
-                if (node.dbInstance.endpoint().address() != dbSecret.host) return ValidationInfo(
+                if (node.database.endpoint.host != dbSecret.host) return ValidationInfo(
                     message("datagrip.secretsmanager.validation.different_address", secretName, dbSecret.host.toString())
                 )
             }
diff --git a/jetbrains-ultimate/src/software/aws/toolkits/jetbrains/datagrip/actions/AddSecretsManagerConnection.kt b/jetbrains-ultimate/src/software/aws/toolkits/jetbrains/datagrip/actions/AddSecretsManagerConnection.kt
@@ -60,7 +60,7 @@ class AddSecretsManagerConnection : SingleExplorerNodeAction<AwsExplorerNode<*>>
 
     private fun recordTelemetry(selected: AwsExplorerNode<*>, result: Result, engine: String? = null) {
         val dbEngine = engine ?: if (selected is RdsNode) {
-            selected.dbInstance.engine()
+            selected.database.engine
         } else {
             null
         }
diff --git a/jetbrains-ultimate/src/software/aws/toolkits/jetbrains/services/rds/RdsDatasourceConfiguration.kt b/jetbrains-ultimate/src/software/aws/toolkits/jetbrains/services/rds/RdsDatasourceConfiguration.kt
@@ -3,11 +3,9 @@
 
 package software.aws.toolkits.jetbrains.services.rds
 
-import software.amazon.awssdk.services.rds.model.DBInstance
-
 data class RdsDatasourceConfiguration(
     val regionId: String,
     val credentialId: String,
-    val dbInstance: DBInstance,
+    val database: RdsDatabase,
     val username: String
 )
diff --git a/jetbrains-ultimate/src/software/aws/toolkits/jetbrains/services/rds/RdsExplorerNodes.kt b/jetbrains-ultimate/src/software/aws/toolkits/jetbrains/services/rds/RdsExplorerNodes.kt
@@ -5,28 +5,33 @@ package software.aws.toolkits.jetbrains.services.rds
 
 import com.intellij.openapi.project.Project
 import software.amazon.awssdk.services.rds.RdsClient
-import software.amazon.awssdk.services.rds.model.DBInstance
 import software.aws.toolkits.jetbrains.core.explorer.nodes.AwsExplorerNode
 import software.aws.toolkits.jetbrains.core.explorer.nodes.AwsExplorerResourceNode
 import software.aws.toolkits.jetbrains.core.explorer.nodes.AwsExplorerServiceNode
 import software.aws.toolkits.jetbrains.core.explorer.nodes.AwsExplorerServiceRootNode
 import software.aws.toolkits.jetbrains.core.getResourceNow
+import software.aws.toolkits.jetbrains.core.utils.buildMap
+import software.aws.toolkits.jetbrains.services.rds.resources.LIST_SUPPORTED_CLUSTERS
 import software.aws.toolkits.jetbrains.services.rds.resources.LIST_SUPPORTED_INSTANCES
 
 class RdsExplorerParentNode(project: Project, service: AwsExplorerServiceNode) : AwsExplorerServiceRootNode(project, service) {
-    override fun getChildrenInternal(): List<AwsExplorerNode<*>> = nodeProject.getResourceNow(LIST_SUPPORTED_INSTANCES).map {
+    override fun getChildrenInternal(): List<AwsExplorerNode<*>> = buildMap<String, RdsDatabase> {
+        // De-dupe by db identifier
+        nodeProject.getResourceNow(LIST_SUPPORTED_CLUSTERS).forEach { putIfAbsent(it.identifier, it) }
+        nodeProject.getResourceNow(LIST_SUPPORTED_INSTANCES).forEach { putIfAbsent(it.identifier, it) }
+    }.values.map {
         RdsNode(nodeProject, it)
     }
 }
 
-class RdsNode(project: Project, val dbInstance: DBInstance, private val rdsEngine: RdsEngine = dbInstance.rdsEngine()) : AwsExplorerResourceNode<String>(
+class RdsNode(project: Project, val database: RdsDatabase, private val rdsEngine: RdsEngine = database.rdsEngine()) : AwsExplorerResourceNode<String>(
     project,
     RdsClient.SERVICE_NAME,
-    dbInstance.dbInstanceArn(),
+    database.arn,
     rdsEngine.icon
 ) {
-    override fun displayName(): String = dbInstance.dbInstanceIdentifier()
-    override fun resourceArn(): String = dbInstance.dbInstanceArn()
+    override fun displayName(): String = database.identifier
+    override fun resourceArn(): String = database.arn
     override fun resourceType(): String = "instance"
     override fun statusText(): String? = rdsEngine.additionalInfo
 }
diff --git a/jetbrains-ultimate/src/software/aws/toolkits/jetbrains/services/rds/RdsResources.kt b/jetbrains-ultimate/src/software/aws/toolkits/jetbrains/services/rds/RdsResources.kt
@@ -3,21 +3,29 @@
 
 package software.aws.toolkits.jetbrains.services.rds
 
-import software.amazon.awssdk.services.rds.model.DBInstance
-
 // These are the member engine in DBInstance, but it is a string
 const val mysqlEngineType = "mysql"
 const val postgresEngineType = "postgres"
 
-fun DBInstance.rdsEngine(): RdsEngine = RdsEngine.fromEngine(engine())
-
 const val jdbcMysql = "mysql"
 const val jdbcMysqlAurora = "mysql:aurora"
 const val jdbcPostgres = "postgresql"
 
 object RdsResources {
-    private val RDS_REGION_REGEX =
-        """.*\.(.+).rds\.""".toRegex()
+    private val RDS_REGION_REGEX = """.*\.(.+).rds\.""".toRegex()
 
     fun extractRegionFromUrl(url: String?): String? = url?.let { RDS_REGION_REGEX.find(url)?.groupValues?.get(1) }
 }
+
+data class RdsDatabase(
+    val identifier: String,
+    val arn: String,
+    val engine: String,
+    val iamDatabaseAuthenticationEnabled: Boolean,
+    val endpoint: Endpoint,
+    val masterUsername: String
+) {
+    fun rdsEngine() = RdsEngine.fromEngine(engine)
+}
+
+data class Endpoint(val host: String, val port: Int)
diff --git a/jetbrains-ultimate/src/software/aws/toolkits/jetbrains/services/rds/actions/CreateIamDataSourceAction.kt b/jetbrains-ultimate/src/software/aws/toolkits/jetbrains/services/rds/actions/CreateIamDataSourceAction.kt
@@ -25,7 +25,6 @@ import software.aws.toolkits.jetbrains.services.rds.RdsNode
 import software.aws.toolkits.jetbrains.services.rds.auth.IamAuth
 import software.aws.toolkits.jetbrains.services.rds.auth.RDS_SIGNING_HOST_PROPERTY
 import software.aws.toolkits.jetbrains.services.rds.auth.RDS_SIGNING_PORT_PROPERTY
-import software.aws.toolkits.jetbrains.services.rds.rdsEngine
 import software.aws.toolkits.jetbrains.services.sts.StsResources
 import software.aws.toolkits.jetbrains.utils.actions.OpenBrowserAction
 import software.aws.toolkits.jetbrains.utils.notifyError
@@ -62,18 +61,18 @@ class CreateIamDataSourceAction : SingleExplorerNodeAction<RdsNode>(message("rds
                 selected.nodeProject,
                 result,
                 DatabaseCredentials.IAM,
-                selected.dbInstance.engine()
+                selected.database.engine
             )
         }.queue()
     }
 
     internal fun checkPrerequisites(node: RdsNode): Boolean {
         // Assert IAM auth enabled
-        if (!node.dbInstance.iamDatabaseAuthenticationEnabled()) {
+        if (!node.database.iamDatabaseAuthenticationEnabled) {
             notifyError(
                 project = node.nodeProject,
                 title = message("aws.notification.title"),
-                content = message("rds.validation.no_iam_auth", node.dbInstance.dbInstanceIdentifier()),
+                content = message("rds.validation.no_iam_auth", node.database.identifier),
                 action = OpenBrowserAction(message("rds.validation.setup_guide"), null, HelpIds.RDS_SETUP_IAM_AUTH.url)
             )
             return false
@@ -87,13 +86,13 @@ class CreateIamDataSourceAction : SingleExplorerNodeAction<RdsNode>(message("rds
             node.nodeProject.getResourceNow(StsResources.USER).substringAfter(':')
         } catch (e: Exception) {
             LOG.warn(e) { "Getting username from STS failed, falling back to master username" }
-            node.dbInstance.masterUsername()
+            node.database.masterUsername
         }
         registry.createRdsDatasource(
             RdsDatasourceConfiguration(
                 regionId = node.nodeProject.activeRegion().id,
                 credentialId = node.nodeProject.activeCredentialProvider().id,
-                dbInstance = node.dbInstance,
+                database = node.database,
                 username = username
             )
         )
@@ -105,9 +104,9 @@ class CreateIamDataSourceAction : SingleExplorerNodeAction<RdsNode>(message("rds
 }
 
 fun DataSourceRegistry.createRdsDatasource(config: RdsDatasourceConfiguration) {
-    val engine = config.dbInstance.rdsEngine()
-    val port = config.dbInstance.endpoint().port()
-    val host = config.dbInstance.endpoint().address()
+    val engine = config.database.rdsEngine()
+    val port = config.database.endpoint.port
+    val host = config.database.endpoint.host
     val endpoint = "$host:$port"
 
     builder
diff --git a/jetbrains-ultimate/tst-202+/software/aws/toolkits/jetbrains/rds/actions/CreateConfigurationActionTest202.kt b/jetbrains-ultimate/tst-202+/software/aws/toolkits/jetbrains/rds/actions/CreateConfigurationActionTest202.kt
diff --git a/jetbrains-ultimate/tst/software/aws/toolkits/jetbrains/datagrip/DatabaseSecretTest.kt b/jetbrains-ultimate/tst/software/aws/toolkits/jetbrains/datagrip/DatabaseSecretTest.kt
diff --git a/jetbrains-ultimate/tst/software/aws/toolkits/jetbrains/services/rds/RdsExplorerNodeTest.kt b/jetbrains-ultimate/tst/software/aws/toolkits/jetbrains/services/rds/RdsExplorerNodeTest.kt
diff --git a/jetbrains-ultimate/tst/software/aws/toolkits/jetbrains/services/rds/actions/CreateConfigurationActionTest.kt b/jetbrains-ultimate/tst/software/aws/toolkits/jetbrains/services/rds/actions/CreateConfigurationActionTest.kt

-Original file line number
+Diff line change
@@ @@ -0,0 +1,4 @@ @@
 +{
 +  "type" : "feature",
 +  "description" : "RDS serverless databases are now visible in the RDS node in the explorer"
 +}
Original file line number	Diff line number	Diff line change
`@@ -44,14 +44,14 @@ object DatabaseSecret {`
`44`	`44`	`// If it is a resource node, validate that it is the same resource`
`45`	`45`	`when (node) {`
`46`	`46`	`is RdsNode -> {`
`47`		`- if (node.dbInstance.engine() != dbSecret.engine) return ValidationInfo(`
	`47`	`+ if (node.database.engine != dbSecret.engine) return ValidationInfo(`
`48`	`48`	`message(`
`49`	`49`	`"datagrip.secretsmanager.validation.different_engine",`
`50`	`50`	`secretName,`
`51`	`51`	`dbSecret.engine.toString()`
`52`	`52`	`)`
`53`	`53`	`)`
`54`		`- if (node.dbInstance.endpoint().address() != dbSecret.host) return ValidationInfo(`
	`54`	`+ if (node.database.endpoint.host != dbSecret.host) return ValidationInfo(`
`55`	`55`	`message("datagrip.secretsmanager.validation.different_address", secretName, dbSecret.host.toString())`
`56`	`56`	`)`
`57`	`57`	`}`
Original file line number	Diff line number	Diff line change
`@@ -60,7 +60,7 @@ class AddSecretsManagerConnection : SingleExplorerNodeAction<AwsExplorerNode<*>>`
`60`	`60`
`61`	`61`	`private fun recordTelemetry(selected: AwsExplorerNode<*>, result: Result, engine: String? = null) {`
`62`	`62`	`val dbEngine = engine ?: if (selected is RdsNode) {`
`63`		`- selected.dbInstance.engine()`
	`63`	`+ selected.database.engine`
`64`	`64`	`} else {`
`65`	`65`	`null`
`66`	`66`	`}`