feat: add support for detecting custom metadata in sys props and env vars (#604)

Author: ianbotsf

.changes/16c733de-c490-404e-a5f9-4e1ba917d020.json

@@ -0,0 +1,8 @@
+{
+    "id": "16c733de-c490-404e-a5f9-4e1ba917d020",
+    "type": "feature",
+    "description": "Add support for detecting custom metadata in system properties (starting with `aws.customMetadata.`) and environment variables (starting with `AWS_CUSTOM_METADATA_`)",
+    "issues": [
+        "awslabs/aws-sdk-kotlin#575"
+    ]
+}

aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/AwsSdkSettingTest.kt

@@ -53,7 +53,9 @@ class AwsSdkSettingTest {
 
     private fun mockPlatform(env: Map<String, String>, jvmProps: Map<String, String>): PlatformEnvironProvider {
         return object : PlatformEnvironProvider {
+            override fun getAllEnvVars(): Map<String, String> = env
             override fun getenv(key: String): String? = env[key]
+            override fun getAllProperties(): Map<String, String> = jvmProps
             override fun getProperty(key: String): String? = jvmProps[key]
         }
     }

aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/region/EnvironmentRegionProviderTest.kt

@@ -5,6 +5,7 @@
 
 package aws.sdk.kotlin.runtime.region
 
+import aws.smithy.kotlin.runtime.util.EnvironmentProvider
 import kotlinx.coroutines.ExperimentalCoroutinesApi
 import kotlinx.coroutines.test.runTest
 import kotlin.test.Test
@@ -13,11 +14,15 @@ import kotlin.test.assertNull
 
 @OptIn(ExperimentalCoroutinesApi::class)
 class EnvironmentRegionProviderTest {
+    fun Map<String, String>.asEnvironmentProvider() = object : EnvironmentProvider {
+        override fun getAllEnvVars(): Map<String, String> = this@asEnvironmentProvider
+        override fun getenv(key: String): String? = this@asEnvironmentProvider[key]
+    }
 
     @Test
     fun noRegion() = runTest {
         val environ = mapOf<String, String>()
-        val provider = EnvironmentRegionProvider { environ[it] }
+        val provider = EnvironmentRegionProvider(environ.asEnvironmentProvider())
         assertNull(provider.getRegion())
     }
 
@@ -27,7 +32,7 @@ class EnvironmentRegionProviderTest {
             "AWS_REGION" to "us-east-1"
         )
 
-        val provider = EnvironmentRegionProvider { environ[it] }
+        val provider = EnvironmentRegionProvider(environ.asEnvironmentProvider())
 
         assertEquals("us-east-1", provider.getRegion())
     }

aws-runtime/aws-config/jvm/test/aws/sdk/kotlin/runtime/auth/credentials/DefaultChainCredentialsProviderTest.kt

@@ -46,7 +46,9 @@ class DefaultChainCredentialsProviderTest {
         private val fs: Filesystem
     ) : PlatformProvider, Filesystem by fs {
         override fun osInfo(): OperatingSystem = OperatingSystem(OsFamily.Linux, "test")
+        override fun getAllProperties(): Map<String, String> = mapOf()
         override fun getProperty(key: String): String? = null
+        override fun getAllEnvVars(): Map<String, String> = env
         override fun getenv(key: String): String? = env[key]
     }
 

aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/AwsUserAgentMetadata.kt

@@ -6,9 +6,7 @@
 package aws.sdk.kotlin.runtime.http
 
 import aws.sdk.kotlin.runtime.InternalSdkApi
-import aws.sdk.kotlin.runtime.http.operation.ConfigMetadata
 import aws.sdk.kotlin.runtime.http.operation.CustomUserAgentMetadata
-import aws.sdk.kotlin.runtime.http.operation.FeatureMetadata
 import aws.smithy.kotlin.runtime.util.*
 import kotlin.jvm.JvmInline
 
@@ -69,20 +67,23 @@ public data class AwsUserAgentMetadata(
                 ua.add("md/internal")
             }
 
+            // FIXME user agent strings are now too long so commenting out several metadata below.
+            // Re-enable once user agent strings can be longer.
+
             ua.add("$sdkMetadata")
-            ua.add("$apiMetadata")
+            // ua.add("$apiMetadata")
             ua.add("$osMetadata")
             ua.add("$languageMetadata")
-            execEnvMetadata?.let { ua.add("$it") }
+            // execEnvMetadata?.let { ua.add("$it") }
 
-            val features = customMetadata?.typedExtras?.filterIsInstance<FeatureMetadata>()
-            features?.forEach { ua.add("$it") }
+            // val features = customMetadata?.typedExtras?.filterIsInstance<FeatureMetadata>()
+            // features?.forEach { ua.add("$it") }
 
-            val config = customMetadata?.typedExtras?.filterIsInstance<ConfigMetadata>()
-            config?.forEach { ua.add("$it") }
+            // val config = customMetadata?.typedExtras?.filterIsInstance<ConfigMetadata>()
+            // config?.forEach { ua.add("$it") }
 
             frameworkMetadata?.let { ua.add("$it") }
-            appId?.let { ua.add("app/$it") }
+            // appId?.let { ua.add("app/$it") }
 
             customMetadata?.extras?.let {
                 val wrapper = AdditionalMetadata(it)
@@ -107,6 +108,8 @@ internal fun loadAwsUserAgentMetadataFromEnvironment(platform: PlatformProvider,
     val appId = platform.getProperty(AWS_APP_ID_PROP) ?: platform.getenv(AWS_APP_ID_ENV)
 
     val frameworkMetadata = FrameworkMetadata.fromEnvironment(platform)
+    val customMetadata = CustomUserAgentMetadata.fromEnvironment(platform)
+
     return AwsUserAgentMetadata(
         sdkMeta,
         apiMeta,
@@ -115,6 +118,7 @@ internal fun loadAwsUserAgentMetadataFromEnvironment(platform: PlatformProvider,
         detectExecEnv(platform),
         frameworkMetadata = frameworkMetadata,
         appId = appId,
+        customMetadata = customMetadata,
     )
 }
 
@@ -183,10 +187,12 @@ public data class LanguageMetadata(
 ) {
     override fun toString(): String = buildString {
         append("lang/kotlin/$version")
+        /* FIXME re-enable once user agent strings can be longer
         if (extras.isNotEmpty()) {
             val wrapper = AdditionalMetadata(extras)
             append(" $wrapper")
         }
+         */
     }
 }
 

aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/operation/CustomUserAgentMetadata.kt

@@ -8,6 +8,10 @@ package aws.sdk.kotlin.runtime.http.operation
 import aws.sdk.kotlin.runtime.InternalSdkApi
 import aws.smithy.kotlin.runtime.client.ExecutionContext
 import aws.smithy.kotlin.runtime.util.AttributeKey
+import aws.smithy.kotlin.runtime.util.PlatformEnvironProvider
+
+private const val CUSTOM_METADATA_ENV_PREFIX = "AWS_CUSTOM_METADATA_"
+private const val CUSTOM_METADATA_PROP_PREFIX = "aws.customMetadata."
 
 /**
  * Operation context element for adding additional metadata to the `User-Agent` header string.
@@ -20,6 +24,17 @@ public class CustomUserAgentMetadata {
 
     internal companion object {
         public val ContextKey: AttributeKey<CustomUserAgentMetadata> = AttributeKey("CustomUserAgentMetadata")
+
+        internal fun fromEnvironment(provider: PlatformEnvironProvider): CustomUserAgentMetadata {
+            fun Map<String, String>.findAndStripKeyPrefix(prefix: String) = this
+                .filterKeys { it.startsWith(prefix) }
+                .mapKeys { (key, _) -> key.substring(prefix.length) }
+
+            val envVarMap = provider.getAllEnvVars().findAndStripKeyPrefix(CUSTOM_METADATA_ENV_PREFIX)
+            val propMap = provider.getAllProperties().findAndStripKeyPrefix(CUSTOM_METADATA_PROP_PREFIX)
+            val allProps = envVarMap + propMap
+            return CustomUserAgentMetadata().apply { allProps.forEach { (key, value) -> add(key, value) } }
+        }
     }
 
     /**

aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/AwsUserAgentMetadataTest.kt

@@ -8,7 +8,7 @@ package aws.sdk.kotlin.runtime.http
 import aws.sdk.kotlin.runtime.http.operation.CustomUserAgentMetadata
 import aws.sdk.kotlin.runtime.testing.TestPlatformProvider
 import aws.smithy.kotlin.runtime.util.OsFamily
-import io.kotest.matchers.string.shouldContain
+import io.kotest.matchers.string.shouldNotContain
 import kotlin.test.Test
 import kotlin.test.assertEquals
 
@@ -31,7 +31,15 @@ class AwsUserAgentMetadataTest {
             add("foo", "bar")
         }
         val ua = AwsUserAgentMetadata(sdkMeta, apiMeta, osMetadata, langMeta, customMetadata = custom)
-        val expected = "aws-sdk-kotlin/1.2.3 api/test-service/1.2.3 os/linux/ubuntu-20.04 lang/kotlin/1.4.31 md/jvmVersion/1.11 md/foo/bar"
+        // FIXME re-enable once user agent strings can be longer
+        val expected = listOf(
+            "aws-sdk-kotlin/1.2.3",
+            // "api/test-service/1.2.3",
+            "os/linux/ubuntu-20.04",
+            "lang/kotlin/1.4.31",
+            // "md/jvmVersion/1.11",
+            "md/foo/bar",
+        ).joinToString(separator = " ")
         assertEquals(expected, ua.xAmzUserAgent)
     }
 
@@ -82,7 +90,10 @@ class AwsUserAgentMetadataTest {
         )
         testEnvironments.forEach { test ->
             val actual = loadAwsUserAgentMetadataFromEnvironment(test.provider, ApiMetadata("Test Service", "1.2.3"))
-            actual.xAmzUserAgent.shouldContain(test.expected)
+
+            // FIXME re-enable once user agent strings can be longer
+            // actual.xAmzUserAgent.shouldContain(test.expected)
+            actual.xAmzUserAgent.shouldNotContain(test.expected)
         }
     }
 }

aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/middleware/UserAgentTest.kt

@@ -60,7 +60,9 @@ class UserAgentTest {
         assertTrue(request.headers.contains(USER_AGENT))
         assertTrue(request.headers.contains(X_AMZ_USER_AGENT))
         assertEquals("aws-sdk-kotlin/1.2.3", request.headers[X_AMZ_USER_AGENT])
-        assertTrue(request.headers[USER_AGENT]!!.startsWith("aws-sdk-kotlin/1.2.3 api/test-service/1.2.3"))
+        // FIXME re-enable once user agent strings can be longer
+        // assertTrue(request.headers[USER_AGENT]!!.startsWith("aws-sdk-kotlin/1.2.3 api/test-service/1.2.3"))
+        assertTrue(request.headers[USER_AGENT]!!.startsWith("aws-sdk-kotlin/1.2.3"))
     }
 
     @Test

aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/operation/CustomUserAgentMetadataTest.kt

@@ -10,6 +10,7 @@ import aws.sdk.kotlin.runtime.http.loadAwsUserAgentMetadataFromEnvironment
 import aws.sdk.kotlin.runtime.testing.TestPlatformProvider
 import io.kotest.matchers.string.shouldContain
 import kotlin.test.Test
+import kotlin.test.assertEquals
 
 class CustomUserAgentMetadataTest {
     @Test
@@ -30,15 +31,45 @@ class CustomUserAgentMetadataTest {
 
         val actual = metadata.copy(customMetadata = customMetadata).xAmzUserAgent
 
+        // FIXME re-enable once user agent strings can be longer
         listOf(
             "md/foo/bar",
             "md/truthy",
             "md/falsey/false",
-            "cfg/retry-mode/standard",
-            "ft/s3-transfer/1.2.3",
-            "ft/waiter"
+            // "cfg/retry-mode/standard",
+            // "ft/s3-transfer/1.2.3",
+            // "ft/waiter",
         ).forEach { partial ->
             actual.shouldContain(partial)
         }
     }
+
+    @Test
+    fun testFromEnvironment() {
+        val props = mapOf(
+            "irrelevantProp" to "shouldBeIgnored",
+            "aws.customMetadata" to "shouldBeIgnored",
+            "aws.customMetadata.foo" to "bar",
+            "aws.customMetadata.baz" to "qux",
+            "aws.customMetadata.priority" to "props",
+        )
+        val envVars = mapOf(
+            "IRRELEVANT_PROP" to "shouldBeIgnored",
+            "AWS_CUSTOM_METADATA" to "shouldBeIgnored",
+            "AWS_CUSTOM_METADATA_oof" to "rab",
+            "AWS_CUSTOM_METADATA_zab" to "xuq",
+            "AWS_CUSTOM_METADATA_priority" to "envVars",
+        )
+        val provider = TestPlatformProvider(env = envVars, props = props)
+        val metadata = CustomUserAgentMetadata.fromEnvironment(provider)
+
+        val expected = mapOf(
+            "foo" to "bar",
+            "baz" to "qux",
+            "oof" to "rab",
+            "zab" to "xuq",
+            "priority" to "props", // System properties take precedence over env vars
+        )
+        assertEquals(expected, metadata.extras)
+    }
 }

aws-runtime/testing/common/src/aws/sdk/kotlin/runtime/testing/TestPlatformProvider.kt

@@ -35,6 +35,8 @@ public class TestPlatformProvider(
         }
 
     override fun osInfo(): OperatingSystem = os
+    override fun getAllProperties(): Map<String, String> = props
     override fun getProperty(key: String): String? = props[key]
+    override fun getAllEnvVars(): Map<String, String> = env
     override fun getenv(key: String): String? = env[key]
 }