Add support for Endpoints 2.0 (#3502)

This release updates internal logic for constructing API endpoints.

We have added rule-based endpoint providers. Additionally there are new
interfaces for endpoint providers and parameters at the service client level.

The SDK provides service-specific `EndpointProvider` interfaces (e.g.
`S3EndpointProvider`), that have a single method `resolveEndpoint`.
`resolveEndpoint` accepts a set of parameters (also service specific) and
computes an endpoint based on those parameters. All clients have a default
implementation, but users are free to implement these to customize their
endpoint logic.

Author: dagnir

.brazil.json

@@ -42,6 +42,9 @@
             "artifactType": "JAR",
             "includes": ["target/aws-sdk-java-third-party-jackson-dataformat-cbor-*.jar"]
         },
+        "ruleset-testing-core": {
+            "packageName": "AwsJavaSdk-Test-RuleSetTestingCore"
+        },
 
         "archetype-app-quickstart": { "skipImport": true },
         "archetype-lambda": { "skipImport": true },
@@ -113,6 +116,10 @@
         "software.amazon.awssdk.crt:aws-crt": { "packageName": "Aws-crt-java", "packageVersion": "1.0.x" },
         "software.amazon.eventstream:eventstream": { "packageName": "AwsFlowJava", "packageVersion": "1.0" },
         "software.amazon.ion:ion-java": { "packageName": "Maven-software-amazon-ion_ion-java", "packageVersion": "1.x" },
-        "software.amazon:flow": { "packageName": "AwsFlowJava", "packageVersion": "1.0" }
+        "software.amazon:flow": { "packageName": "AwsFlowJava", "packageVersion": "1.0" },
+        "org.junit.jupiter:junit-jupiter": { "packageName": "JUnit5", "packageVersion": "5.x" },
+        "org.mockito:mockito-core": { "packageName": "Mockito", "packageVersion": "3.x" },
+        "io.reactivex.rxjava3:rxjava": { "packageName": "RxJava", ",packageVersion": "3.x" },
+        "org.assertj:assertj-core": { "packageName": "AssertJ", ",packageVersion": "3.22.x" }
     }
 }

.changes/next-release/feature-AWSSDKforJavav2-65cafca.json

@@ -0,0 +1,6 @@
+{
+    "type": "feature",
+    "category": "AWS SDK for Java v2",
+    "contributor": "",
+    "description": "This release updates internal logic for constructing API endpoints. \n\nWe have added rule-based endpoint providers. Additionally there are new interfaces for endpoint providers and parameters at the service client level.\n\nThe SDK provides service-specific `EndpointProvider` interfaces (e.g. `S3EndpointProvider`), that have a single method `resolveEndpoint`. `resolveEndpoint` accepts a set of parameters (also service specific) and computes an endpoint based on those parameters. All clients have a default implementation, but users are free to implement these to customize their endpoint logic.\n\nIt is important to know that `EndpointProvider`s and the endpoint override on the client are not mutually exclusive. In all existing cases, the endpoint override is passed as a parameter to the the provider and the provider *may* modify it. For example, the S3 provider may add the bucket name as a prefix to the endpoint override for virtual bucket addressing.\n\nThere may be some behavior changes related to endpoint resolution as as the the endpoint resolution is now standardised across SDKs so this release also bumps the minor version of the SDK to `2.18`.\n\nNotably, there are some changes in S3: when including and endpoint override, the SDK will use virtual bucket addressing as long as the bucket is a valid DNS label. This is different from the previous behavior where this only happens if the endpoint override begins with 's3'.\n\nIn many situations, the exception thrown for validations like including an endpoint override and enabling DualStack has changed to `SdkClientExcpetion` and error messages have changed as well."
+}

buildspecs/release-javadoc.yml

@@ -18,7 +18,7 @@ phases:
     commands:
     - python ./scripts/doc_crosslinks/generate_cross_link_data.py --apiDefinitionsBasePath ./services/ --apiDefinitionsRelativeFilePath src/main/resources/codegen-resources/service-2.json  --templateFilePath ./scripts/doc_crosslinks/crosslink_redirect.html --outputFilePath ./scripts/crosslink_redirect.html
     - mvn install -P quick -T1C
-    - mvn clean install javadoc:aggregate -B -Ppublic-javadoc -Dcheckstyle.skip -Dspotbugs.skip -DskipTests -Ddoclint=none -pl '!:protocol-tests,!:protocol-tests-core,!:codegen-generated-classes-test,!:sdk-benchmarks,!:s3-benchmarks,!:module-path-tests,!:test-utils,!:http-client-tests,!:tests-coverage-reporting,!:sdk-native-image-test'
+    - mvn clean install javadoc:aggregate -B -Ppublic-javadoc -Dcheckstyle.skip -Dspotbugs.skip -DskipTests -Ddoclint=none -pl '!:protocol-tests,!:protocol-tests-core,!:codegen-generated-classes-test,!:sdk-benchmarks,!:s3-benchmarks,!:module-path-tests,!:test-utils,!:http-client-tests,!:tests-coverage-reporting,!:sdk-native-image-test,!:ruleset-testing-core'
     - RELEASE_VERSION=`mvn -q -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec`
     -
     - aws s3 sync target/site/apidocs/ $DOC_PATH/$RELEASE_VERSION/

buildspecs/release-to-maven.yml

@@ -40,7 +40,7 @@ phases:
             awk 'BEGIN { var=ENVIRON["SDK_SIGNING_GPG_KEYNAME"] } { gsub("\\$SDK_SIGNING_GPG_KEYNAME", var, $0); print }' > \
             $SETTINGS_XML
 
-          mvn clean deploy -B -s $SETTINGS_XML -Ppublishing -DperformRelease -Dspotbugs.skip -DskipTests -Dcheckstyle.skip -Djapicmp.skip -Ddoclint=none -pl !:protocol-tests,!:protocol-tests-core,!:codegen-generated-classes-test,!:sdk-benchmarks,!:module-path-tests,!:tests-coverage-reporting,!:stability-tests,!:sdk-native-image-test,!:auth-tests,!:s3-benchmarks,!:region-testing -DautoReleaseAfterClose=true -DstagingProgressTimeoutMinutes=30
+          mvn clean deploy -B -s $SETTINGS_XML -Ppublishing -DperformRelease -Dspotbugs.skip -DskipTests -Dcheckstyle.skip -Djapicmp.skip -Ddoclint=none -pl !:protocol-tests,!:protocol-tests-core,!:codegen-generated-classes-test,!:sdk-benchmarks,!:module-path-tests,!:tests-coverage-reporting,!:stability-tests,!:sdk-native-image-test,!:auth-tests,!:s3-benchmarks,!:region-testing,!:ruleset-testing-core -DautoReleaseAfterClose=true -DstagingProgressTimeoutMinutes=30
         else
           echo "This version was already released."
         fi

codegen-maven-plugin/pom.xml

@@ -57,6 +57,16 @@
             <groupId>software.amazon.awssdk</groupId>
             <version>${awsjavasdk.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter</artifactId>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <scope>compile</scope>
+        </dependency>
         <dependency>
             <artifactId>maven-plugin-api</artifactId>
             <groupId>org.apache.maven</groupId>

codegen-maven-plugin/src/main/java/software/amazon/awssdk/codegen/maven/plugin/GenerationMojo.java

@@ -32,6 +32,8 @@
 import software.amazon.awssdk.codegen.CodeGenerator;
 import software.amazon.awssdk.codegen.internal.Utils;
 import software.amazon.awssdk.codegen.model.config.customization.CustomizationConfig;
+import software.amazon.awssdk.codegen.model.rules.endpoints.EndpointTestSuiteModel;
+import software.amazon.awssdk.codegen.model.service.EndpointRuleSetModel;
 import software.amazon.awssdk.codegen.model.service.Paginators;
 import software.amazon.awssdk.codegen.model.service.ServiceModel;
 import software.amazon.awssdk.codegen.model.service.Waiters;
@@ -47,6 +49,9 @@ public class GenerationMojo extends AbstractMojo {
     private static final String CUSTOMIZATION_CONFIG_FILE = "customization.config";
     private static final String WAITERS_FILE = "waiters-2.json";
     private static final String PAGINATORS_FILE = "paginators-1.json";
+    private static final String ENDPOINT_RULE_SET_FILE = "endpoint-rule-set.json";
+    private static final String ENDPOINT_TESTS_FILE = "endpoint-tests.json";
+
 
     @Parameter(property = "codeGenResources", defaultValue = "${basedir}/src/main/resources/codegen-resources/")
     private File codeGenResources;
@@ -61,11 +66,13 @@ public class GenerationMojo extends AbstractMojo {
     private MavenProject project;
 
     private Path sourcesDirectory;
+    private Path resourcesDirectory;
     private Path testsDirectory;
 
     @Override
     public void execute() throws MojoExecutionException {
         this.sourcesDirectory = Paths.get(outputDirectory).resolve("generated-sources").resolve("sdk");
+        this.resourcesDirectory = Paths.get(outputDirectory).resolve("generated-resources").resolve("sdk-resources");
         this.testsDirectory = Paths.get(outputDirectory).resolve("generated-test-sources").resolve("sdk-tests");
 
         findModelRoots().forEach(p -> {
@@ -76,6 +83,8 @@ public void execute() throws MojoExecutionException {
                                   .serviceModel(loadServiceModel(modelRootPath))
                                   .waitersModel(loadWaiterModel(modelRootPath))
                                   .paginatorsModel(loadPaginatorModel(modelRootPath))
+                                  .endpointRuleSetModel(loadEndpointRuleSetModel(modelRootPath))
+                                  .endpointTestSuiteModel(loadEndpointTestSuiteModel(modelRootPath))
                                   .build());
         });
         project.addCompileSourceRoot(sourcesDirectory.toFile().getAbsolutePath());
@@ -105,6 +114,7 @@ private void generateCode(C2jModels models) {
         CodeGenerator.builder()
                      .models(models)
                      .sourcesDirectory(sourcesDirectory.toFile().getAbsolutePath())
+                     .resourcesDirectory(resourcesDirectory.toFile().getAbsolutePath())
                      .testsDirectory(testsDirectory.toFile().getAbsolutePath())
                      .intermediateModelFileNamePrefix(intermediateModelFileNamePrefix(models))
                      .build()
@@ -134,6 +144,14 @@ private Paginators loadPaginatorModel(Path root) {
         return loadOptionalModel(Paginators.class, root.resolve(PAGINATORS_FILE)).orElse(Paginators.none());
     }
 
+    private EndpointRuleSetModel loadEndpointRuleSetModel(Path root) {
+        return loadRequiredModel(EndpointRuleSetModel.class, root.resolve(ENDPOINT_RULE_SET_FILE));
+    }
+
+    private EndpointTestSuiteModel loadEndpointTestSuiteModel(Path root) {
+        return loadRequiredModel(EndpointTestSuiteModel.class, root.resolve(ENDPOINT_TESTS_FILE));
+    }
+
     /**
      * Load required model from the project resources.
      */

codegen/pom.xml

@@ -62,6 +62,11 @@
             <artifactId>metrics-spi</artifactId>
             <version>${awsjavasdk.version}</version>
         </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>endpoints-spi</artifactId>
+            <version>${awsjavasdk.version}</version>
+        </dependency>
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>regions</artifactId>
@@ -117,6 +122,11 @@
             <artifactId>protocol-core</artifactId>
             <version>${awsjavasdk.version}</version>
         </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>ruleset-testing-core</artifactId>
+            <version>${awsjavasdk.version}</version>
+        </dependency>
 
         <dependency>
             <artifactId>org.eclipse.jdt.core</artifactId>
@@ -158,7 +168,7 @@
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter</artifactId>
-            <scope>test</scope>
+            <scope>compile</scope>
         </dependency>
         <dependency>
             <groupId>org.junit.vintage</groupId>
@@ -178,7 +188,7 @@
         <dependency>
             <groupId>org.mockito</groupId>
             <artifactId>mockito-core</artifactId>
-            <scope>test</scope>
+            <scope>compile</scope>
         </dependency>
     </dependencies>
 </project>

codegen/src/main/java/software/amazon/awssdk/codegen/AddMetadata.java

@@ -58,6 +58,7 @@ public static Metadata constructMetadata(ServiceModel serviceModel,
                 .withRequestTransformPackageName(namingStrategy.getRequestTransformPackageName(serviceName))
                 .withPaginatorsPackageName(namingStrategy.getPaginatorsPackageName(serviceName))
                 .withWaitersPackageName(namingStrategy.getWaitersPackageName(serviceName))
+                .withEndpointRulesPackageName(namingStrategy.getEndpointRulesPackageName(serviceName))
                 .withServiceAbbreviation(serviceMetadata.getServiceAbbreviation())
                 .withServiceFullName(serviceMetadata.getServiceFullName())
                 .withServiceName(serviceName)

codegen/src/main/java/software/amazon/awssdk/codegen/AddOperations.java

@@ -165,6 +165,7 @@ public Map<String, OperationModel> constructOperations() {
             operationModel.setEndpointTrait(op.getEndpoint());
             operationModel.setHttpChecksumRequired(op.isHttpChecksumRequired());
             operationModel.setHttpChecksum(op.getHttpChecksum());
+            operationModel.setStaticContextParams(op.getStaticContextParams());
 
             Input input = op.getInput();
             if (input != null) {

codegen/src/main/java/software/amazon/awssdk/codegen/AddShapes.java

@@ -191,6 +191,7 @@ private MemberModel generateMemberModel(String c2jMemberName, Member c2jMemberDe
         memberModel.setEndpointDiscoveryId(c2jMemberDefinition.isEndpointdiscoveryid());
         memberModel.setXmlAttribute(c2jMemberDefinition.isXmlAttribute());
         memberModel.setUnionEnumTypeName(namingStrategy.getUnionEnumTypeName(memberModel));
+        memberModel.setContextParam(c2jMemberDefinition.getContextParam());
 
 
         // Pass the xmlNameSpace from the member reference