Add support for service-specific endpoint overrides without code changes. (#5562)

* Made protected API changes in support of service-specific endpoint overrides.

Class-Level Changes:
* DefaultServiceEndpointBuilder - Deprecated in favor of the new ClientEndpointProvider and AwsClientEndpointProvider
* ClientEndpointProvider - An interface for resolving the client-level endpoint. This endpoint may be overridden by the request-level EndpointProvider. This joins the existing fleet of endpoint-related providers, like region providers, dualstack providers, and FIPS providers.
* AwsClientEndpointProvider - An implementation of ClientEndpointProvider that checks the client configuration, environment and service metadata to determine the client endpoint.

Field Changes:
* Deprecated SdkClientOption.ENDPOINT and SdkClientOption.ENDPOINT_OVERRIDDEN in favor of a new SdkClientOption.CLIENT_ENDPOINT_PROVIDER. The new property allows updating both values in a single field. It was a necessary improvement so that these properties can stay in sync, reliably.
* Deprecated SdkExecutionAttribute.CLIENT_ENDPOINT and SdkClientOption.ENDPOINT_OVERRIDDEN in favor of a new SdkInternalExecutionAttribute.CLIENT_ENDPOINT_PROVIDER. The new property exists for much the same reasons as the new SdkClientOption, but also provides an opportunity to hide these should-be-internal-to-the-SDK attributes.

* Updates to the way SDK clients are built, to support the new client endpoint providers.

1. Make clients set the SdkClientOption.CLIENT_ENDPOINT_PROVIDER when they are created.
2. Update the default AWS and SDK client builders to default the SdkClientOption.CLIENT_ENDPOINT_PROVIDERs so that old clients continue to work with the new core. Old clients versions won't support setting the endpoint with the new environment variable/system property/profile settings.

This commit also includes EndpointSharedConfigTest which tests to make sure that the new ways of overriding the endpoint work for clients.

* Moved non-client users of DefaultServiceEndpointBuilder over to AwsClientEndpointProvider.

These are the users that remain after the last commit:
1. RdsPresignInterceptors in RDS, Neptune and DocDB. It's unfortunate that these are pretty much copy+pasted classes, but that seems like a future problem to figure out.
2. S3Utilities
3. S3Presigner
4. PollyPresigner

* Miscellaneous changes required to move the SDK over from the old SdkClientOptions and SdkExecutionAttributes to the new SdkClientOptions and SdkInternalExecutionAttribute.

* Codegen test file changes.

* Changelog entry.

* Fix checkstyle issue.

* Update .changes/next-release/feature-AWSSDKforJavav2-eea40a4.json

Co-authored-by: David Ho <[email protected]>

* WIP

* Updated environment variables and profile properties to match the other AWS SDKs.

* Addressed comments.

* Updated test to use new profile property and environment variables.

---------

Co-authored-by: David Ho <[email protected]>

Author: millems

.changes/next-release/feature-AWSSDKforJavav2-eea40a4.json

@@ -0,0 +1,6 @@
+{
+    "type": "feature",
+    "category": "AWS SDK for Java v2",
+    "contributor": "",
+    "description": "Add support for specifying endpoint overrides using environment variables, system properties or profile files. More information about this feature is available here: https://docs.aws.amazon.com/sdkref/latest/guide/feature-ss-endpoints.html"
+}

codegen/src/main/java/software/amazon/awssdk/codegen/internal/Utils.java

@@ -16,6 +16,7 @@
 package software.amazon.awssdk.codegen.internal;
 
 import static java.util.stream.Collectors.toList;
+import static software.amazon.awssdk.utils.StringUtils.lowerCase;
 
 import java.io.Closeable;
 import java.io.File;
@@ -133,7 +134,7 @@ public static String removeLeading(String str, String toRemove) {
         if (str == null) {
             return null;
         }
-        if (str.startsWith(toRemove)) {
+        if (lowerCase(str).startsWith(lowerCase(toRemove))) {
             return str.substring(toRemove.length());
         }
         return str;
@@ -143,7 +144,7 @@ public static String removeTrailing(String str, String toRemove) {
         if (str == null) {
             return null;
         }
-        if (str.endsWith(toRemove)) {
+        if (lowerCase(str).endsWith(lowerCase(toRemove))) {
             return str.substring(0, str.length() - toRemove.length());
         }
         return str;

codegen/src/main/java/software/amazon/awssdk/codegen/naming/DefaultNamingStrategy.java

@@ -114,19 +114,42 @@ private static boolean isJavaKeyword(String word) {
 
     @Override
     public String getServiceName() {
-        String baseName = Stream.of(serviceModel.getMetadata().getServiceId())
-                                .filter(Objects::nonNull)
-                                .filter(s -> !s.trim().isEmpty())
-                                .findFirst()
-                                .orElseThrow(() -> new IllegalStateException("ServiceId is missing in the c2j model."));
-
+        String baseName = serviceId();
         baseName = pascalCase(baseName);
+        baseName = removeRedundantPrefixesAndSuffixes(baseName);
+        return baseName;
+    }
 
-        // Special cases
-        baseName = Utils.removeLeading(baseName, "Amazon");
-        baseName = Utils.removeLeading(baseName, "Aws");
-        baseName = Utils.removeTrailing(baseName, "Service");
+    @Override
+    public String getServiceNameForEnvironmentVariables() {
+        String baseName = serviceId();
+        baseName = baseName.replace(' ', '_');
+        baseName = StringUtils.upperCase(baseName);
+        return baseName;
+    }
+
+    @Override
+    public String getServiceNameForSystemProperties() {
+        return getServiceName();
+    }
+
+    @Override
+    public String getServiceNameForProfileFile() {
+        return StringUtils.lowerCase(getServiceNameForEnvironmentVariables());
+    }
+
+    private String serviceId() {
+        return Stream.of(serviceModel.getMetadata().getServiceId())
+                     .filter(Objects::nonNull)
+                     .filter(s -> !s.trim().isEmpty())
+                     .findFirst()
+                     .orElseThrow(() -> new IllegalStateException("ServiceId is missing in the c2j model."));
+    }
 
+    private static String removeRedundantPrefixesAndSuffixes(String baseName) {
+        baseName = Utils.removeLeading(baseName, "amazon");
+        baseName = Utils.removeLeading(baseName, "aws");
+        baseName = Utils.removeTrailing(baseName, "service");
         return baseName;
     }
 

codegen/src/main/java/software/amazon/awssdk/codegen/naming/NamingStrategy.java

@@ -29,6 +29,21 @@ public interface NamingStrategy {
      */
     String getServiceName();
 
+    /**
+     * Retrieve the service name that should be used for environment variables.
+     */
+    String getServiceNameForEnvironmentVariables();
+
+    /**
+     * Retrieve the service name that should be used for system properties.
+     */
+    String getServiceNameForSystemProperties();
+
+    /**
+     * Retrieve the service name that should be used for profile properties.
+     */
+    String getServiceNameForProfileFile();
+
     /**
      * Retrieve the client package name that should be used based on the service name.
      */

codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/BaseClientBuilderClass.java

@@ -43,6 +43,7 @@
 import software.amazon.awssdk.auth.token.signer.aws.BearerTokenSigner;
 import software.amazon.awssdk.awscore.client.builder.AwsDefaultClientBuilder;
 import software.amazon.awssdk.awscore.client.config.AwsClientOption;
+import software.amazon.awssdk.awscore.endpoint.AwsClientEndpointProvider;
 import software.amazon.awssdk.codegen.internal.Utils;
 import software.amazon.awssdk.codegen.model.intermediate.IntermediateModel;
 import software.amazon.awssdk.codegen.model.intermediate.OperationModel;
@@ -72,6 +73,7 @@
 import software.amazon.awssdk.identity.spi.IdentityProvider;
 import software.amazon.awssdk.identity.spi.IdentityProviders;
 import software.amazon.awssdk.identity.spi.TokenIdentity;
+import software.amazon.awssdk.regions.ServiceMetadataAdvancedOption;
 import software.amazon.awssdk.utils.AttributeMap;
 import software.amazon.awssdk.utils.CollectionUtils;
 import software.amazon.awssdk.utils.StringUtils;
@@ -456,6 +458,27 @@ private MethodSpec finalizeServiceConfigurationMethod() {
             builder.addStatement("builder.option($T.$L, resolveAccountIdEndpointMode(config))",
                             AwsClientOption.class, model.getNamingStrategy().getEnumValueName("accountIdEndpointMode"));
         }
+
+        String serviceNameForEnvVar = model.getNamingStrategy().getServiceNameForEnvironmentVariables();
+        String serviceNameForSystemProperty = model.getNamingStrategy().getServiceNameForSystemProperties();
+        String serviceNameForProfileFile = model.getNamingStrategy().getServiceNameForProfileFile();
+
+        builder.addCode("builder.lazyOptionIfAbsent($T.CLIENT_ENDPOINT_PROVIDER, c ->", SdkClientOption.class)
+               .addCode("  $T.builder()", AwsClientEndpointProvider.class)
+               .addCode("    .serviceEndpointOverrideEnvironmentVariable($S)", "AWS_ENDPOINT_URL_" + serviceNameForEnvVar)
+               .addCode("    .serviceEndpointOverrideSystemProperty($S)", "aws.endpointUrl" + serviceNameForSystemProperty)
+               .addCode("    .serviceProfileProperty($S)", serviceNameForProfileFile)
+               .addCode("    .serviceEndpointPrefix(serviceEndpointPrefix())")
+               .addCode("    .defaultProtocol($S)", "https")
+               .addCode("    .region(c.get($T.AWS_REGION))", AwsClientOption.class)
+               .addCode("    .profileFile(c.get($T.PROFILE_FILE_SUPPLIER))", SdkClientOption.class)
+               .addCode("    .profileName(c.get($T.PROFILE_NAME))", SdkClientOption.class)
+               .addCode("    .putAdvancedOption($T.DEFAULT_S3_US_EAST_1_REGIONAL_ENDPOINT,", ServiceMetadataAdvancedOption.class)
+               .addCode("        c.get($T.DEFAULT_S3_US_EAST_1_REGIONAL_ENDPOINT))", ServiceMetadataAdvancedOption.class)
+               .addCode("    .dualstackEnabled(c.get($T.DUALSTACK_ENDPOINT_ENABLED))", AwsClientOption.class)
+               .addCode("    .fipsEnabled(c.get($T.FIPS_ENDPOINT_ENABLED))", AwsClientOption.class)
+               .addCode("    .build());");
+
         builder.addStatement("return builder.build()");
         return builder.build();
     }

codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/AsyncClientClass.java

@@ -254,8 +254,8 @@ private MethodSpec constructor(TypeSpec.Builder classBuilder) {
                                                                "AsyncEndpointDiscoveryCacheLoader"));
 
             if (model.getCustomizationConfig().allowEndpointOverrideForEndpointDiscoveryRequiredOperations()) {
-                builder.beginControlFlow("if (clientConfiguration.option(SdkClientOption.ENDPOINT_OVERRIDDEN) == "
-                                        + "Boolean.TRUE)");
+                builder.beginControlFlow("if (clientConfiguration.option(SdkClientOption.CLIENT_ENDPOINT_PROVIDER)"
+                                         + ".isEndpointOverridden())");
                 builder.addStatement("log.warn($S)",
                                      "Endpoint discovery is enabled for this client, and an endpoint override was also "
                                      + "specified. This will disable endpoint discovery for methods that require it, instead "
@@ -401,7 +401,8 @@ protected MethodSpec.Builder operationBody(MethodSpec.Builder builder, Operation
             builder.addStatement("boolean endpointDiscoveryEnabled = "
                                  + "clientConfiguration.option(SdkClientOption.ENDPOINT_DISCOVERY_ENABLED)");
             builder.addStatement("boolean endpointOverridden = "
-                                 + "clientConfiguration.option(SdkClientOption.ENDPOINT_OVERRIDDEN) == Boolean.TRUE");
+                                 + "clientConfiguration.option(SdkClientOption.CLIENT_ENDPOINT_PROVIDER)"
+                                 + ".isEndpointOverridden()");
 
             if (opModel.getEndpointDiscovery().isRequired()) {
                 if (!model.getCustomizationConfig().allowEndpointOverrideForEndpointDiscoveryRequiredOperations()) {
@@ -443,7 +444,8 @@ protected MethodSpec.Builder operationBody(MethodSpec.Builder builder, Operation
             builder.addCode("endpointFuture = identityFuture.thenCompose(credentials -> {")
                    .addCode("    $1T endpointDiscoveryRequest = $1T.builder()", EndpointDiscoveryRequest.class)
                    .addCode("        .required($L)", opModel.getInputShape().getEndpointDiscovery().isRequired())
-                   .addCode("        .defaultEndpoint(clientConfiguration.option($T.ENDPOINT))", SdkClientOption.class)
+                   .addCode("        .defaultEndpoint(clientConfiguration.option($T.CLIENT_ENDPOINT_PROVIDER).clientEndpoint())",
+                            SdkClientOption.class)
                    .addCode("        .overrideConfiguration($N.overrideConfiguration().orElse(null))",
                             opModel.getInput().getVariableName())
                    .addCode("        .build();")

codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/SyncClientClass.java

@@ -219,8 +219,8 @@ private MethodSpec constructor() {
                                                                "EndpointDiscoveryCacheLoader"));
 
             if (model.getCustomizationConfig().allowEndpointOverrideForEndpointDiscoveryRequiredOperations()) {
-                builder.beginControlFlow("if (clientConfiguration.option(SdkClientOption.ENDPOINT_OVERRIDDEN) == "
-                                         + "Boolean.TRUE)");
+                builder.beginControlFlow("if (clientConfiguration.option(SdkClientOption.CLIENT_ENDPOINT_PROVIDER)"
+                                         + ".isEndpointOverridden())");
                 builder.addStatement("log.warn(() -> $S)",
                                      "Endpoint discovery is enabled for this client, and an endpoint override was also "
                                      + "specified. This will disable endpoint discovery for methods that require it, instead "
@@ -265,7 +265,8 @@ private MethodSpec traditionalMethod(OperationModel opModel) {
             method.addStatement("boolean endpointDiscoveryEnabled = "
                                 + "clientConfiguration.option(SdkClientOption.ENDPOINT_DISCOVERY_ENABLED)");
             method.addStatement("boolean endpointOverridden = "
-                                + "clientConfiguration.option(SdkClientOption.ENDPOINT_OVERRIDDEN) == Boolean.TRUE");
+                                + "clientConfiguration.option(SdkClientOption.CLIENT_ENDPOINT_PROVIDER)"
+                                + ".isEndpointOverridden()");
 
             if (opModel.getEndpointDiscovery().isRequired()) {
                 if (!model.getCustomizationConfig().allowEndpointOverrideForEndpointDiscoveryRequiredOperations()) {
@@ -309,7 +310,8 @@ private MethodSpec traditionalMethod(OperationModel opModel) {
 
             method.addCode("$1T endpointDiscoveryRequest = $1T.builder()", EndpointDiscoveryRequest.class)
                   .addCode("    .required($L)", opModel.getInputShape().getEndpointDiscovery().isRequired())
-                  .addCode("    .defaultEndpoint(clientConfiguration.option($T.ENDPOINT))", SdkClientOption.class)
+                  .addCode("    .defaultEndpoint(clientConfiguration.option($T.CLIENT_ENDPOINT_PROVIDER).clientEndpoint())",
+                           SdkClientOption.class)
                   .addCode("    .overrideConfiguration($N.overrideConfiguration().orElse(null))",
                            opModel.getInput().getVariableName())
                   .addCode("    .build();");

codegen/src/main/java/software/amazon/awssdk/codegen/poet/model/ServiceClientConfigurationUtils.java

@@ -33,6 +33,7 @@
 import software.amazon.awssdk.codegen.model.intermediate.IntermediateModel;
 import software.amazon.awssdk.codegen.poet.auth.scheme.AuthSchemeSpecUtils;
 import software.amazon.awssdk.codegen.poet.rules.EndpointRulesSpecUtils;
+import software.amazon.awssdk.core.ClientEndpointProvider;
 import software.amazon.awssdk.core.client.config.ClientOption;
 import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
 import software.amazon.awssdk.core.client.config.SdkClientConfiguration;
@@ -161,21 +162,21 @@ private Field endpointOverrideField() {
     private CodeBlock endpointOverrideConfigSetter() {
         return CodeBlock.builder()
                         .beginControlFlow("if (endpointOverride != null)")
-                        .addStatement("config.option($T.ENDPOINT, endpointOverride)", SdkClientOption.class)
-                        .addStatement("config.option($T.ENDPOINT_OVERRIDDEN, true)", SdkClientOption.class)
+                        .addStatement("config.option($T.CLIENT_ENDPOINT_PROVIDER, $T.forEndpointOverride(endpointOverride))",
+                                      SdkClientOption.class, ClientEndpointProvider.class)
                         .nextControlFlow("else")
-                        .addStatement("config.option($T.ENDPOINT, null)", SdkClientOption.class)
-                        .addStatement("config.option($T.ENDPOINT_OVERRIDDEN, false)", SdkClientOption.class)
+                        .addStatement("config.option($T.CLIENT_ENDPOINT_PROVIDER, null)", SdkClientOption.class)
                         .endControlFlow()
                         .addStatement("return this")
                         .build();
     }
 
     private CodeBlock endpointOverrideConfigGetter() {
         return CodeBlock.builder()
-                        .beginControlFlow("if (Boolean.TRUE.equals(config.option($T.ENDPOINT_OVERRIDDEN)))",
-                                          SdkClientOption.class)
-                        .addStatement("return config.option($T.ENDPOINT)", SdkClientOption.class)
+                        .addStatement("$T clientEndpoint = config.option($T.CLIENT_ENDPOINT_PROVIDER)",
+                                      ClientEndpointProvider.class, SdkClientOption.class)
+                        .beginControlFlow("if (clientEndpoint != null && clientEndpoint.isEndpointOverridden())")
+                        .addStatement("return clientEndpoint.clientEndpoint()")
                         .endControlFlow()
                         .addStatement("return null")
                         .build();

codegen/src/main/java/software/amazon/awssdk/codegen/poet/rules/RequestEndpointInterceptorSpec.java

@@ -26,7 +26,6 @@
 import software.amazon.awssdk.core.interceptor.Context;
 import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
 import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
-import software.amazon.awssdk.core.interceptor.SdkExecutionAttribute;
 import software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute;
 import software.amazon.awssdk.endpoints.Endpoint;
 import software.amazon.awssdk.http.SdkHttpRequest;
@@ -71,14 +70,14 @@ private MethodSpec modifyHttpRequestMethod() {
          .addStatement("return context.httpRequest()")
          .endControlFlow().build();
 
-        b.addStatement("$1T endpoint = ($1T) executionAttributes.getAttribute($2T.RESOLVED_ENDPOINT)",
+        b.addStatement("$T endpoint = executionAttributes.getAttribute($T.RESOLVED_ENDPOINT)",
                        Endpoint.class,
                        SdkInternalExecutionAttribute.class);
         b.addStatement("return $T.setUri(context.httpRequest(),"
-                       + "executionAttributes.getAttribute($T.CLIENT_ENDPOINT),"
+                       + "executionAttributes.getAttribute($T.CLIENT_ENDPOINT_PROVIDER).clientEndpoint(),"
                        + "endpoint.url())",
                        endpointRulesSpecUtils.rulesRuntimeClassName("AwsEndpointProviderUtils"),
-                       SdkExecutionAttribute.class);
+                       SdkInternalExecutionAttribute.class);
         return b.build();
     }
 

codegen/src/main/resources/software/amazon/awssdk/codegen/rules/AwsEndpointProviderUtils.java.resource

@@ -44,7 +44,8 @@ public final class AwsEndpointProviderUtils {
     public static String endpointBuiltIn(ExecutionAttributes executionAttributes) {
         if (endpointIsOverridden(executionAttributes)) {
             return invokeSafely(() -> {
-                URI endpointOverride = executionAttributes.getAttribute(SdkExecutionAttribute.CLIENT_ENDPOINT);
+                URI endpointOverride = executionAttributes.getAttribute(SdkInternalExecutionAttribute.CLIENT_ENDPOINT_PROVIDER)
+                                                          .clientEndpoint();
                 return new URI(endpointOverride.getScheme(), null, endpointOverride.getHost(), endpointOverride.getPort(),
                         endpointOverride.getPath(), null, endpointOverride.getFragment()).toString();
             });
@@ -53,11 +54,10 @@ public final class AwsEndpointProviderUtils {
     }
 
     /**
-     * True if the the {@link SdkExecutionAttribute#ENDPOINT_OVERRIDDEN} attribute is present and its value is
-     * {@code true}, {@code false} otherwise.
+     * Read {@link SdkExecutionAttribute#CLIENT_ENDPOINT_PROVIDER}'s isEndpointOverridden attribute.
      */
     public static boolean endpointIsOverridden(ExecutionAttributes attrs) {
-        return attrs.getOptionalAttribute(SdkExecutionAttribute.ENDPOINT_OVERRIDDEN).orElse(false);
+        return attrs.getAttribute(SdkInternalExecutionAttribute.CLIENT_ENDPOINT_PROVIDER).isEndpointOverridden();
     }
 
     /**