API service calls custom query param validation (#3546)

* Implement RequiredTrait

* Enforce presence of required query params

* Added codegen tests; missing javadoc

* Revised codegen tests

* Added integration tests for missing query parameter

* Added tests for validating nested shapes

* Misc fixes

* Added ability to select traits to validate

* Added codegen tests for verifying marshaller have correct validation enabling method calls

* Handle cases where customization.config has no enabledTraitValidations information

* Address review comments

* Attach RequiredTrait only to operations for enabled services

* Undo misc fixes breaking compatibility

* Enforce RequiredTrait across all shape locations

* Added tests for validating List and Map querystring objects

* Updated CustomizationConfig

* Updated changelog entry

* Verify SdkField is non-null verify checking for traits

* Trimmed changelog entry

* Fixed new integration tests expectations

* Moved new integration tests into separate classes

* Updated expectation of existing integration test

Author: dave-fn

.changes/next-release/bugfix-AWSSDKforJavav2-eeb4182.json

@@ -0,0 +1,6 @@
+{
+    "category": "Amazon S3", 
+    "contributor": "", 
+    "type": "bugfix", 
+    "description": "S3 query param validation\n\nThe SDK will raise SdkClientException when a null value is set for a member marked with the required trait."
+}

codegen/src/main/java/software/amazon/awssdk/codegen/AddShapes.java

@@ -23,6 +23,7 @@
 
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import software.amazon.awssdk.codegen.internal.TypeUtils;
 import software.amazon.awssdk.codegen.model.config.customization.CustomizationConfig;
 import software.amazon.awssdk.codegen.model.intermediate.EnumModel;
@@ -192,6 +193,7 @@ private MemberModel generateMemberModel(String c2jMemberName, Member c2jMemberDe
         memberModel.setXmlAttribute(c2jMemberDefinition.isXmlAttribute());
         memberModel.setUnionEnumTypeName(namingStrategy.getUnionEnumTypeName(memberModel));
         memberModel.setContextParam(c2jMemberDefinition.getContextParam());
+        memberModel.setRequired(isRequiredMember(c2jMemberName, parentShape));
 
 
         // Pass the xmlNameSpace from the member reference
@@ -309,6 +311,12 @@ private boolean isFlattened(Member member, Shape memberShape) {
                || memberShape.isFlattened();
     }
 
+    private boolean isRequiredMember(String memberName, Shape memberShape) {
+        return Optional.ofNullable(memberShape.getRequired())
+                       .map(l -> l.contains(memberName))
+                       .orElse(false);
+    }
+
     /**
      * @param parentShape  Shape containing the member in question.
      * @param allC2jShapes All shapes in the service model.

codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java

@@ -215,6 +215,11 @@ public class CustomizationConfig {
 
     private List<String> interceptors = new ArrayList<>();
 
+    /**
+     * Whether marshallers perform validations against members marked with RequiredTrait.
+     */
+    private boolean requiredTraitValidationEnabled = false;
+
     private CustomizationConfig() {
     }
 
@@ -559,4 +564,12 @@ public List<String> getInterceptors() {
     public void setInterceptors(List<String> interceptors) {
         this.interceptors = interceptors;
     }
+    
+    public boolean isRequiredTraitValidationEnabled() {
+        return requiredTraitValidationEnabled;
+    }
+
+    public void setRequiredTraitValidationEnabled(boolean requiredTraitValidationEnabled) {
+        this.requiredTraitValidationEnabled = requiredTraitValidationEnabled;
+    }
 }

codegen/src/main/java/software/amazon/awssdk/codegen/model/intermediate/MemberModel.java

@@ -59,6 +59,8 @@ public class MemberModel extends DocumentationModel {
 
     private String deprecatedMessage;
 
+    private boolean required;
+
     private ListModel listModel;
 
     private MapModel mapModel;
@@ -316,6 +318,14 @@ public void setDeprecatedMessage(String deprecatedMessage) {
         this.deprecatedMessage = deprecatedMessage;
     }
 
+    public boolean isRequired() {
+        return required;
+    }
+
+    public void setRequired(boolean required) {
+        this.required = required;
+    }
+
     public boolean isEventPayload() {
         return eventPayload;
     }

codegen/src/main/java/software/amazon/awssdk/codegen/poet/model/ShapeModelSpec.java

@@ -45,6 +45,7 @@
 import software.amazon.awssdk.core.traits.LocationTrait;
 import software.amazon.awssdk.core.traits.MapTrait;
 import software.amazon.awssdk.core.traits.PayloadTrait;
+import software.amazon.awssdk.core.traits.RequiredTrait;
 import software.amazon.awssdk.core.traits.TimestampFormatTrait;
 import software.amazon.awssdk.core.traits.XmlAttributeTrait;
 import software.amazon.awssdk.core.traits.XmlAttributesTrait;
@@ -184,6 +185,9 @@ private CodeBlock traits(MemberModel m) {
         if (m.isXmlAttribute()) {
             traits.add(createXmlAttributeTrait());
         }
+        if (customizationConfig.isRequiredTraitValidationEnabled() && m.isRequired()) {
+            traits.add(createRequiredTrait());
+        }
 
         if (!traits.isEmpty()) {
             return CodeBlock.builder()
@@ -268,6 +272,12 @@ private CodeBlock createPayloadTrait() {
                         .build();
     }
 
+    private CodeBlock createRequiredTrait() {
+        return CodeBlock.builder()
+                        .add("$T.create()", ClassName.get(RequiredTrait.class))
+                        .build();
+    }
+
     private CodeBlock createMapTrait(MemberModel m) {
         return CodeBlock.builder()
                         .add("$T.builder()\n"
@@ -306,7 +316,7 @@ private CodeBlock createXmlAttributesTrait(MemberModel model) {
         String uri = xmlNamespace.getUri();
         String prefix = xmlNamespace.getPrefix();
         CodeBlock.Builder codeBlockBuilder = CodeBlock.builder()
-                                         .add("$T.create(", ClassName.get(XmlAttributesTrait.class));
+                                                      .add("$T.create(", ClassName.get(XmlAttributesTrait.class));
 
         String namespacePrefix = "xmlns:" + prefix;
         codeBlockBuilder.add("$T.of($S, $T.builder().attributeGetter((ignore) -> $S).build())",

codegen/src/test/java/software/amazon/awssdk/codegen/AddShapesTest.java

@@ -0,0 +1,91 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.codegen;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.File;
+import java.io.IOException;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.codegen.model.config.customization.CustomizationConfig;
+import software.amazon.awssdk.codegen.model.intermediate.IntermediateModel;
+import software.amazon.awssdk.codegen.model.intermediate.MemberModel;
+import software.amazon.awssdk.codegen.model.intermediate.ShapeModel;
+import software.amazon.awssdk.codegen.model.service.Location;
+import software.amazon.awssdk.codegen.model.service.ServiceModel;
+import software.amazon.awssdk.codegen.poet.model.AwsModelSpecTest;
+import software.amazon.awssdk.codegen.utils.ModelLoaderUtils;
+
+class AddShapesTest {
+
+    private static IntermediateModel intermediateModel;
+
+    @BeforeAll
+    public static void setUp() throws IOException {
+        File serviceModelFile = new File(AwsModelSpecTest.class.getResource("service-2.json").getFile());
+        File customizationConfigFile = new File(AwsModelSpecTest.class
+                                                    .getResource("customization.config")
+                                                    .getFile());
+
+        intermediateModel = new IntermediateModelBuilder(
+            C2jModels.builder()
+                     .serviceModel(ModelLoaderUtils.loadModel(ServiceModel.class, serviceModelFile))
+                     .customizationConfig(ModelLoaderUtils.loadModel(CustomizationConfig.class, customizationConfigFile))
+                     .build())
+            .build();
+    }
+
+    @Test
+    void generateShapeModel_memberRequiredByShape_setsMemberModelAsRequired() {
+        String requestShapeName = "QueryParameterOperationRequest";
+        String queryParamName = "QueryParamOne";
+
+        ShapeModel requestShapeModel = intermediateModel.getShapes().get(requestShapeName);
+        MemberModel requiredMemberModel = requestShapeModel.findMemberModelByC2jName(queryParamName);
+
+        assertThat(requestShapeModel.getRequired()).contains(queryParamName);
+        assertThat(requiredMemberModel.getHttp().getLocation()).isEqualTo(Location.QUERY_STRING);
+        assertThat(requiredMemberModel.isRequired()).isTrue();
+    }
+
+    @Test
+    void generateShapeModel_memberNotRequiredByShape_doesNotSetMemberModelAsRequired() {
+        String requestShapeName = "QueryParameterOperationRequest";
+        String queryParamName = "QueryParamTwo";
+
+        ShapeModel requestShapeModel = intermediateModel.getShapes().get(requestShapeName);
+        MemberModel requiredMemberModel = requestShapeModel.findMemberModelByC2jName(queryParamName);
+
+        assertThat(requestShapeModel.getRequired()).doesNotContain(queryParamName);
+        assertThat(requiredMemberModel.getHttp().getLocation()).isEqualTo(Location.QUERY_STRING);
+        assertThat(requiredMemberModel.isRequired()).isFalse();
+    }
+
+    @Test
+    void generateShapeModel_memberRequiredByNestedShape_setsMemberModelAsRequired() {
+        String requestShapeName = "NestedQueryParameterOperation";
+        String queryParamName = "QueryParamOne";
+
+        ShapeModel requestShapeModel = intermediateModel.getShapes().get(requestShapeName);
+        MemberModel requiredMemberModel = requestShapeModel.findMemberModelByC2jName(queryParamName);
+
+        assertThat(requestShapeModel.getRequired()).contains(queryParamName);
+        assertThat(requiredMemberModel.getHttp().getLocation()).isEqualTo(Location.QUERY_STRING);
+        assertThat(requiredMemberModel.isRequired()).isTrue();
+    }
+
+}

codegen/src/test/java/software/amazon/awssdk/codegen/poet/model/AwsServiceBaseRequestSpecTest.java

@@ -20,36 +20,78 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.Optional;
+import java.util.regex.Pattern;
+import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import software.amazon.awssdk.codegen.C2jModels;
 import software.amazon.awssdk.codegen.IntermediateModelBuilder;
 import software.amazon.awssdk.codegen.model.config.customization.CustomizationConfig;
 import software.amazon.awssdk.codegen.model.intermediate.IntermediateModel;
+import software.amazon.awssdk.codegen.model.intermediate.ShapeModel;
 import software.amazon.awssdk.codegen.model.service.ServiceModel;
+import software.amazon.awssdk.codegen.poet.PoetUtils;
 import software.amazon.awssdk.codegen.utils.ModelLoaderUtils;
 
 public class AwsServiceBaseRequestSpecTest {
+
     private static IntermediateModel intermediateModel;
 
     @BeforeAll
     public static void setUp() throws IOException {
         File serviceModelFile = new File(AwsModelSpecTest.class.getResource("service-2.json").getFile());
         File customizationConfigFile = new File(AwsModelSpecTest.class
-                .getResource("customization.config")
-                .getFile());
+                                                    .getResource("customization.config")
+                                                    .getFile());
 
         intermediateModel = new IntermediateModelBuilder(
-                C2jModels.builder()
-                        .serviceModel(ModelLoaderUtils.loadModel(ServiceModel.class, serviceModelFile))
-                        .customizationConfig(ModelLoaderUtils.loadModel(CustomizationConfig.class, customizationConfigFile))
-                        .build())
-                .build();
+            C2jModels.builder()
+                     .serviceModel(ModelLoaderUtils.loadModel(ServiceModel.class, serviceModelFile))
+                     .customizationConfig(ModelLoaderUtils.loadModel(CustomizationConfig.class, customizationConfigFile))
+                     .build())
+            .build();
     }
 
     @Test
-    public void testGeneration() {
+    void testGeneration() {
         AwsServiceBaseRequestSpec spec = new AwsServiceBaseRequestSpec(intermediateModel);
         assertThat(spec, generatesTo(spec.className().simpleName().toLowerCase() + ".java"));
     }
+
+    @Test
+    void buildJavaFile_memberRequiredByShape_addsTraitToGeneratedCode() {
+        String requestShapeName = "QueryParameterOperationRequest";
+        String queryParamName = "QueryParamOne";
+
+        ShapeModel requestShapeModel = intermediateModel.getShapes().get(requestShapeName);
+        AwsServiceModel spec = new AwsServiceModel(intermediateModel, requestShapeModel);
+        String codeString = PoetUtils.buildJavaFile(spec).toString();
+
+        String uploadIdDeclarationString = findTraitDeclarationString(codeString, queryParamName).get();
+        Assertions.assertThat(uploadIdDeclarationString).contains("RequiredTrait.create()");
+    }
+
+    @Test
+    void buildJavaFile_memberNotRequiredByShape_doesNotAddTraitToGeneratedCode() {
+        String requestShapeName = "QueryParameterOperationRequest";
+        String queryParamName = "QueryParamTwo";
+
+        ShapeModel requestShapeModel = intermediateModel.getShapes().get(requestShapeName);
+        AwsServiceModel spec = new AwsServiceModel(intermediateModel, requestShapeModel);
+        String codeString = PoetUtils.buildJavaFile(spec).toString();
+
+        String uploadIdDeclarationString = findTraitDeclarationString(codeString, queryParamName).get();
+        Assertions.assertThat(uploadIdDeclarationString).doesNotContain("RequiredTrait.create()");
+    }
+
+
+    private static Optional<String> findTraitDeclarationString(String javaFileString, String fieldName) {
+        return Arrays.stream(Pattern.compile("\n\n").split(javaFileString))
+                     .filter(block -> block.contains(fieldName))
+                     .filter(block -> block.contains("traits"))
+                     .findFirst();
+    }
+
 }

codegen/src/test/resources/software/amazon/awssdk/codegen/poet/model/customization.config

@@ -30,5 +30,6 @@
             ]
         }
     },
-    "underscoresInNameBehavior": "ALLOW"
-}
+    "underscoresInNameBehavior": "ALLOW",
+    "requiredTraitValidationEnabled": true
+}