feat: allow setting endpoint url through env and shared config (#1039)

Author: lucix-aws

.changes/bac6a11b-f4bd-4aae-ac8c-dc27b753c3c8.json

@@ -0,0 +1,5 @@
+{
+    "id": "bac6a11b-f4bd-4aae-ac8c-dc27b753c3c8",
+    "type": "feature",
+    "description": "Allow endpoint URL configuration via env and shared config."
+}

aws-runtime/aws-config/api/aws-config.api

@@ -188,6 +188,7 @@ public final class aws/sdk/kotlin/runtime/auth/credentials/internal/sso/SsoClien
 public final class aws/sdk/kotlin/runtime/auth/credentials/internal/sso/SsoClient$Companion : aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory {
 	public fun builder ()Laws/sdk/kotlin/runtime/auth/credentials/internal/sso/SsoClient$Builder;
 	public synthetic fun builder ()Laws/smithy/kotlin/runtime/client/SdkClient$Builder;
+	public synthetic fun finalizeConfig (Laws/smithy/kotlin/runtime/client/SdkClient$Builder;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
 }
 
 public final class aws/sdk/kotlin/runtime/auth/credentials/internal/sso/SsoClient$Config : aws/sdk/kotlin/runtime/client/AwsSdkClientConfig, aws/smithy/kotlin/runtime/auth/awscredentials/CredentialsProviderConfig, aws/smithy/kotlin/runtime/client/RetryClientConfig, aws/smithy/kotlin/runtime/client/RetryStrategyClientConfig, aws/smithy/kotlin/runtime/client/SdkClientConfig, aws/smithy/kotlin/runtime/http/auth/HttpAuthConfig, aws/smithy/kotlin/runtime/http/config/HttpClientConfig, aws/smithy/kotlin/runtime/http/config/HttpEngineConfig, aws/smithy/kotlin/runtime/telemetry/TelemetryConfig {
@@ -521,6 +522,7 @@ public final class aws/sdk/kotlin/runtime/auth/credentials/internal/ssooidc/SsoO
 public final class aws/sdk/kotlin/runtime/auth/credentials/internal/ssooidc/SsoOidcClient$Companion : aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory {
 	public fun builder ()Laws/sdk/kotlin/runtime/auth/credentials/internal/ssooidc/SsoOidcClient$Builder;
 	public synthetic fun builder ()Laws/smithy/kotlin/runtime/client/SdkClient$Builder;
+	public synthetic fun finalizeConfig (Laws/smithy/kotlin/runtime/client/SdkClient$Builder;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
 }
 
 public final class aws/sdk/kotlin/runtime/auth/credentials/internal/ssooidc/SsoOidcClient$Config : aws/sdk/kotlin/runtime/client/AwsSdkClientConfig, aws/smithy/kotlin/runtime/auth/awscredentials/CredentialsProviderConfig, aws/smithy/kotlin/runtime/client/RetryClientConfig, aws/smithy/kotlin/runtime/client/RetryStrategyClientConfig, aws/smithy/kotlin/runtime/client/SdkClientConfig, aws/smithy/kotlin/runtime/http/auth/HttpAuthConfig, aws/smithy/kotlin/runtime/http/config/HttpClientConfig, aws/smithy/kotlin/runtime/http/config/HttpEngineConfig, aws/smithy/kotlin/runtime/telemetry/TelemetryConfig {
@@ -1043,6 +1045,7 @@ public final class aws/sdk/kotlin/runtime/auth/credentials/internal/sts/StsClien
 public final class aws/sdk/kotlin/runtime/auth/credentials/internal/sts/StsClient$Companion : aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory {
 	public fun builder ()Laws/sdk/kotlin/runtime/auth/credentials/internal/sts/StsClient$Builder;
 	public synthetic fun builder ()Laws/smithy/kotlin/runtime/client/SdkClient$Builder;
+	public synthetic fun finalizeConfig (Laws/smithy/kotlin/runtime/client/SdkClient$Builder;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
 }
 
 public final class aws/sdk/kotlin/runtime/auth/credentials/internal/sts/StsClient$Config : aws/sdk/kotlin/runtime/client/AwsSdkClientConfig, aws/smithy/kotlin/runtime/auth/awscredentials/CredentialsProviderConfig, aws/smithy/kotlin/runtime/client/RetryClientConfig, aws/smithy/kotlin/runtime/client/RetryStrategyClientConfig, aws/smithy/kotlin/runtime/client/SdkClientConfig, aws/smithy/kotlin/runtime/http/auth/HttpAuthConfig, aws/smithy/kotlin/runtime/http/config/HttpClientConfig, aws/smithy/kotlin/runtime/http/config/HttpEngineConfig, aws/smithy/kotlin/runtime/telemetry/TelemetryConfig {
@@ -1659,9 +1662,15 @@ public abstract class aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory
 	public fun invoke (Lkotlin/jvm/functions/Function1;)Laws/smithy/kotlin/runtime/client/SdkClient;
 }
 
+public final class aws/sdk/kotlin/runtime/config/AwsSdkSettingKt {
+}
+
 public final class aws/sdk/kotlin/runtime/config/endpoints/EndpointsKt {
 }
 
+public final class aws/sdk/kotlin/runtime/config/endpoints/ResolveEndpointUrlKt {
+}
+
 public final class aws/sdk/kotlin/runtime/config/imds/EC2MetadataError : aws/sdk/kotlin/runtime/AwsServiceException {
 	public fun <init> (ILjava/lang/String;)V
 	public final fun getStatusCode ()I
@@ -1746,6 +1755,10 @@ public final class aws/sdk/kotlin/runtime/config/profile/AwsConfigParseException
 public final class aws/sdk/kotlin/runtime/config/profile/AwsProfileKt {
 }
 
+public final class aws/sdk/kotlin/runtime/config/profile/AwsSharedConfigKt {
+	public static final fun resolveEndpointUrl (Laws/sdk/kotlin/runtime/config/profile/AwsSharedConfig;Ljava/lang/String;)Laws/smithy/kotlin/runtime/net/Url;
+}
+
 public final class aws/sdk/kotlin/runtime/config/retries/ResolveRetryStrategyKt {
 }
 

aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt

@@ -8,7 +8,7 @@ package aws.sdk.kotlin.runtime.config
 import aws.sdk.kotlin.runtime.client.AwsSdkClientConfig
 import aws.sdk.kotlin.runtime.config.endpoints.resolveUseDualStack
 import aws.sdk.kotlin.runtime.config.endpoints.resolveUseFips
-import aws.sdk.kotlin.runtime.config.profile.AwsProfile
+import aws.sdk.kotlin.runtime.config.profile.AwsSharedConfig
 import aws.sdk.kotlin.runtime.config.profile.loadAwsSharedConfig
 import aws.sdk.kotlin.runtime.config.retries.resolveRetryStrategy
 import aws.sdk.kotlin.runtime.region.resolveRegion
@@ -57,27 +57,28 @@ public abstract class AbstractAwsSdkClientFactory<
         val tracer = telemetryProvider.tracerProvider.getOrCreateTracer("AwsSdkClientFactory")
 
         tracer.withSpan("fromEnvironment") {
-            val profile = asyncLazy { loadAwsSharedConfig(PlatformProvider.System).activeProfile }
+            val sharedConfig = asyncLazy { loadAwsSharedConfig(PlatformProvider.System) }
+            val profile = asyncLazy { sharedConfig.get().activeProfile }
 
             // As a DslBuilderProperty, the value of retryStrategy cannot be checked for nullability because it may have
             // been set using a DSL. Thus, set the resolved strategy _first_ to ensure it's used as the fallback.
             if (config is RetryStrategyClientConfig.Builder) {
                 config.retryStrategy = resolveRetryStrategy(profile = profile)
             }
 
-            if (block != null) config.apply(block)
+            block?.let(config::apply)
 
             config.logMode = config.logMode ?: ClientSettings.LogMode.resolve(platform = PlatformProvider.System)
             config.region = config.region ?: resolveRegion(profile = profile)
             config.useFips = config.useFips ?: resolveUseFips(profile = profile)
             config.useDualStack = config.useDualStack ?: resolveUseDualStack(profile = profile)
-            finalizeConfig(builder, profile)
+            finalizeConfig(builder, sharedConfig)
         }
         return builder.build()
     }
 
     /**
      * Inject any client-specific config.
      */
-    protected open suspend fun finalizeConfig(builder: TClientBuilder, profile: LazyAsyncValue<AwsProfile>) { }
+    protected open suspend fun finalizeConfig(builder: TClientBuilder, sharedConfig: LazyAsyncValue<AwsSharedConfig>) { }
 }

aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt

@@ -8,6 +8,8 @@ package aws.sdk.kotlin.runtime.config
 import aws.sdk.kotlin.runtime.InternalSdkApi
 import aws.smithy.kotlin.runtime.client.config.RetryMode
 import aws.smithy.kotlin.runtime.config.*
+import aws.smithy.kotlin.runtime.net.Url
+import aws.smithy.kotlin.runtime.util.PlatformProvider
 
 // NOTE: The JVM property names MUST match the ones defined in the Java SDK for any setting added.
 // see: https://github.com/aws/aws-sdk-java-v2/blob/master/core/sdk-core/src/main/java/software/amazon/awssdk/core/SdkSystemSetting.java
@@ -152,4 +154,29 @@ public object AwsSdkSetting {
      */
     public val AwsUseDualStackEndpoint: EnvironmentSetting<Boolean> =
         boolEnvSetting("aws.useDualstackEndpoint", "AWS_USE_DUALSTACK_ENDPOINT")
+
+    /**
+     * The globally-configured endpoint URL that applies to all services.
+     */
+    public val AwsEndpointUrl: EnvironmentSetting<Url> =
+        EnvironmentSetting(Url::parse)("aws.endpointUrl", "AWS_ENDPOINT_URL")
+
+    /**
+     * Whether to ignore configured endpoint URLs.
+     */
+    public val AwsIgnoreEndpointUrls: EnvironmentSetting<Boolean> =
+        boolEnvSetting("aws.ignoreConfiguredEndpointUrls", "AWS_IGNORE_CONFIGURED_ENDPOINT_URLS")
+}
+
+/**
+ * Resolves an endpoint url for a given service.
+ */
+@InternalSdkApi
+public fun AwsSdkSetting.resolveEndpointUrl(
+    provider: PlatformProvider,
+    sysPropSuffix: String,
+    envSuffix: String,
+): Url? {
+    val serviceSetting = EnvironmentSetting(Url::parse)("aws.endpointUrl$sysPropSuffix", "AWS_ENDPOINT_URL_$envSuffix")
+    return serviceSetting.resolve(provider) ?: AwsEndpointUrl.resolve(provider)
 }

aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/endpoints/ResolveEndpointUrl.kt

@@ -0,0 +1,56 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package aws.sdk.kotlin.runtime.config.endpoints
+
+import aws.sdk.kotlin.runtime.InternalSdkApi
+import aws.sdk.kotlin.runtime.config.AwsSdkSetting
+import aws.sdk.kotlin.runtime.config.profile.*
+import aws.sdk.kotlin.runtime.config.resolveEndpointUrl
+import aws.smithy.kotlin.runtime.config.resolve
+import aws.smithy.kotlin.runtime.net.Url
+import aws.smithy.kotlin.runtime.util.LazyAsyncValue
+import aws.smithy.kotlin.runtime.util.PlatformProvider
+
+/**
+ * Attempts to find the configured endpoint URL for a specific service.
+ *
+ * We look at the following sources in-order:
+ *
+ * 1. The value provided by a service-specific environment setting:
+ *    1. The environment variable `AWS_ENDPOINT_URL_${SNAKE_CASE_SERVICE_ID}`.
+ *    2. The JVM system property `aws.endpointUrl${JavaSDKClientPrefix}`.
+ * 2. The value provided by the global endpoint environment setting:
+ *    1. The environment variable `AWS_ENDPOINT_URL`.
+ *    2. The JVM system property `aws.endpointUrl`.
+ * 3. The value provided by a service-specific parameter from a services definition section in the shared configuration
+ *    file (`${snake_case_service_id}.endpoint_url`).
+ * 4. The value provided by the global parameter from a profile in the shared configuration file (`endpoint_url`).
+ *
+ * Endpoint URL settings can be disabled globally through either the environment or shared config, in which case the
+ * above list of sources is ignored.
+ */
+@InternalSdkApi
+public suspend fun resolveEndpointUrl(
+    sharedConfig: LazyAsyncValue<AwsSharedConfig>,
+    sysPropSuffix: String,
+    envSuffix: String,
+    sharedConfigKey: String,
+    provider: PlatformProvider = PlatformProvider.System,
+): Url? {
+    if (resolveIgnoreEndpointUrls(provider, sharedConfig)) {
+        return null // the "disable" directive overrides ALL sources, regardless of where it comes from itself
+    }
+
+    return AwsSdkSetting.resolveEndpointUrl(provider, sysPropSuffix, envSuffix)
+        ?: sharedConfig.get().resolveEndpointUrl(sharedConfigKey)
+}
+
+private suspend fun resolveIgnoreEndpointUrls(
+    provider: PlatformProvider,
+    sharedConfig: LazyAsyncValue<AwsSharedConfig>,
+): Boolean =
+    AwsSdkSetting.AwsIgnoreEndpointUrls.resolve(provider)
+        ?: sharedConfig.get().activeProfile.ignoreEndpointUrls
+        ?: false

aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt

@@ -8,6 +8,7 @@ package aws.sdk.kotlin.runtime.config.profile
 import aws.sdk.kotlin.runtime.ConfigurationException
 import aws.sdk.kotlin.runtime.InternalSdkApi
 import aws.smithy.kotlin.runtime.client.config.RetryMode
+import aws.smithy.kotlin.runtime.net.Url
 
 /**
  * Represents an AWS config profile.
@@ -100,17 +101,46 @@ public val AwsProfile.useFips: Boolean?
 public val AwsProfile.useDualStack: Boolean?
     get() = getBooleanOrNull("use_dualstack_endpoint")
 
+/**
+ * The default endpoint URL that applies to all services.
+ */
+@InternalSdkApi
+public val AwsProfile.endpointUrl: Url?
+    get() = getUrlOrNull("endpoint_url")
+
+/**
+ * Whether to ignore configured endpoint URLs.
+ */
+@InternalSdkApi
+public val AwsProfile.ignoreEndpointUrls: Boolean?
+    get() = getBooleanOrNull("ignore_configured_endpoint_urls")
+
+/**
+ * The name of the services config section used by this profile.
+ */
+@InternalSdkApi
+public val AwsProfile.servicesSection: String?
+    get() = getOrNull("services")
+
 /**
  * Parse a config value as a boolean, ignoring case.
  */
 @InternalSdkApi
 public fun AwsProfile.getBooleanOrNull(key: String, subKey: String? = null): Boolean? =
     getOrNull(key, subKey)?.let {
         it.lowercase().toBooleanStrictOrNull() ?: throw ConfigurationException(
-            buildString {
-                append("Failed to parse config property $key")
-                subKey?.let { append(".$it") }
-                append(" as a boolean")
-            },
+            "Failed to parse config property ${buildKeyString(key, subKey)} as a boolean",
         )
     }
+
+internal fun AwsProfile.getUrlOrNull(key: String, subKey: String? = null): Url? =
+    getOrNull(key, subKey)?.let {
+        try {
+            Url.parse(it)
+        } catch (e: Exception) {
+            throw ConfigurationException("Failed to parse config property ${buildKeyString(key, subKey)} as a URL", e)
+        }
+    }
+
+private fun buildKeyString(key: String, subKey: String? = null): String =
+    listOfNotNull(key, subKey).joinToString(".")

aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsSharedConfig.kt

@@ -5,7 +5,9 @@
 
 package aws.sdk.kotlin.runtime.config.profile
 
+import aws.sdk.kotlin.runtime.ConfigurationException
 import aws.sdk.kotlin.runtime.InternalSdkApi
+import aws.smithy.kotlin.runtime.net.Url
 
 /**
  * Represents shared configuration (profiles, SSO sessions, credentials, etc)
@@ -28,9 +30,30 @@ public class AwsSharedConfig internal constructor(
     public val ssoSessions: Map<String, SsoSessionConfig>
         get() = sections[ConfigSectionType.SSO_SESSION] ?: emptyMap()
 
+    /**
+     * Map of section name to ServiceConfig
+     */
+    public val services: Map<String, ServicesConfig>
+        get() = sections[ConfigSectionType.SERVICES] ?: emptyMap()
+
     /**
      * Resolve the active profile or the default profile if none is defined
      */
     public val activeProfile: AwsProfile
         get() = profiles[source.profile] ?: AwsProfile(source.profile, emptyMap())
 }
+
+/**
+ * Get the configured endpoint URL for a specific service, falling back to the global default.
+ * @param serviceKey The config key for the service, generally this is sdkId in snake_case form.
+ */
+public fun AwsSharedConfig.resolveEndpointUrl(serviceKey: String): Url? =
+    resolveServiceEndpointUrl(serviceKey) ?: activeProfile.endpointUrl
+
+private fun AwsSharedConfig.resolveServiceEndpointUrl(serviceKey: String): Url? =
+    activeProfile.servicesSection?.let { sectionName ->
+        val section = services[sectionName]
+            ?: throw ConfigurationException("shared config points to nonexistent services section '$sectionName'")
+
+        section.getUrlOrNull(serviceKey, "endpoint_url")
+    }

aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/ServicesConfig.kt

@@ -0,0 +1,11 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package aws.sdk.kotlin.runtime.config.profile
+
+import aws.sdk.kotlin.runtime.InternalSdkApi
+
+@InternalSdkApi
+public typealias ServicesConfig = ConfigSection