feat: autofill Glacier accountId (#301)

Author: aajtodd

.builder/actions/set_upstream_versions.py

@@ -0,0 +1,78 @@
+import Builder
+import argparse
+import os
+import sys
+
+class SetUpstreamVersions(Builder.Action):
+    """
+    Inspects the upstream dependencies for their actual versions and replaces them in `aws-sdk-kotlin/gradle.properties`.
+
+    This ensures that when `aws-sdk-kotlin` is built with a dependency from a branch that has declared a different version
+    than what is set in `gradle.properties` that it works correctly.
+    """
+
+    def run(self, env):
+        if env.project.name != "aws-sdk-kotlin":
+            # not the root project, probably building as a consumer
+            proj = Builder.Project.find_project("aws-sdk-kotlin")
+        else:
+            proj = env.project
+
+        deps = proj.config.get("upstream")
+
+        discovered_versions = {}
+        for d in deps:
+            dep_version = _get_dependency_version(env, d)
+            if dep_version is not None:
+                discovered_versions[d.name] = dep_version
+
+        print("discovered dependency versions: {}".format(discovered_versions))
+        if "smithy-kotlin" in discovered_versions:
+            _replace_gradle_property(proj, "smithyKotlinVersion", discovered_versions["smithy-kotlin"])
+
+        if "aws-crt-kotlin" in discovered_versions:
+            _replace_gradle_property(proj, "crtKotlinVersion", discovered_versions["aws-crt-kotlin"])
+
+
+def _replace_gradle_property(proj, prop_name, new_value):
+    """
+    Replaces the named property with the value if property name exists in gradle.properties
+    """
+    gradle_props = os.path.join(proj.path, "gradle.properties")
+
+    with open(gradle_props, "r") as f:
+        lines = f.readlines()
+
+    with open(gradle_props, "w") as f:
+        for line in lines:
+            needle = "{}=".format(prop_name)
+            if needle in line:
+                replacement = "{}={}\n".format(prop_name, new_value)
+                f.write(replacement)
+                print("replaced {} with {}".format(line.strip(), replacement.strip()))
+            else:
+                f.write(line)
+
+
+def _get_dependency_version(env, dep): 
+    """
+    Gets the version of the dependency actually used. This may be different than what
+    we have set in our gradle.properties. We need to use the version of the dependency
+    actually downloaded
+    """
+    dep_root = os.path.join(env.deps_dir, dep.name)
+    dep_gradle_props = os.path.join(dep_root, "gradle.properties")
+    if not os.path.exists(dep_gradle_props):
+        dep_proj = Builder.Project.find_project(dep.name)
+        dep_gradle_props = os.path.join(dep_proj.path, "gradle.properties")
+        if not os.path.exists(dep_gradle_props):
+            return None
+
+    with open(dep_gradle_props, "r") as f:
+        lines = f.readlines()
+    version = next(filter(lambda x: "sdkVersion=" in x, lines), None)
+
+    if version is None:
+        return None
+
+    return version.split("=")[1].strip()

.gitignore

@@ -10,6 +10,7 @@ workspace.xml
 build/
 *.iml
 .idea/
+__pycache__/
 local.properties
 # ignore generated files
 services/*/generated-src

builder.json

@@ -5,6 +5,17 @@
     "variables": {
         "gradlew": "{source_dir}/gradlew -p {source_dir}"
     },
+    "build_dir": "target/build",
+    "imports": [
+        "JDK8"
+    ],
+    "env": {
+        "JAVA_HOME": "{java_home}"
+    },
+    "_comment": "prebuild step defined in .builder/actions/set_upstream_versions.py",
+    "pre_build_steps": [
+        "set-upstream-versions"
+    ],
     "build_steps": [
         "{gradlew} assemble --parallel"
     ],
@@ -13,13 +24,6 @@
         "{gradlew} test allTests",
         "{gradlew} testAllProtocols"
     ],
-    "imports": [
-        "JDK8"
-    ],
-    "build_dir": "target/build",
-    "env": {
-        "JAVA_HOME": "{java_home}"
-    },
     "upstream": [
         {
             "name": "smithy-kotlin"

codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/glacier/GlacierAccountIdDefault.kt

@@ -0,0 +1,59 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+
+package aws.sdk.kotlin.codegen.customization.glacier
+
+import aws.sdk.kotlin.codegen.sdkId
+import software.amazon.smithy.kotlin.codegen.KotlinSettings
+import software.amazon.smithy.kotlin.codegen.core.*
+import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration
+import software.amazon.smithy.kotlin.codegen.model.expectShape
+import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerator
+import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware
+import software.amazon.smithy.kotlin.codegen.utils.getOrNull
+import software.amazon.smithy.model.Model
+import software.amazon.smithy.model.shapes.*
+
+/**
+ * Adds a middleware for Glacier to autofill accountId when not set
+ * See: https://github.com/awslabs/aws-sdk-kotlin/issues/246
+ */
+class GlacierAccountIdDefault : KotlinIntegration {
+    override fun enabledForService(model: Model, settings: KotlinSettings): Boolean =
+        model.expectShape<ServiceShape>(settings.service).sdkId.equals("Glacier", ignoreCase = true)
+
+    override fun customizeMiddleware(
+        ctx: ProtocolGenerator.GenerationContext,
+        resolved: List<ProtocolMiddleware>
+    ): List<ProtocolMiddleware> {
+        return resolved + GlacierAccountIdMiddleware()
+    }
+}
+
+private class GlacierAccountIdMiddleware : ProtocolMiddleware {
+    override val name: String
+        get() = "GlacierAccountIdAutoFill"
+
+    override val order: Byte = -100
+
+    override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean {
+        val input = op.input.getOrNull()?.let { ctx.model.expectShape<StructureShape>(it) }
+        return input?.memberNames?.any { it.lowercase() == "accountid" } ?: false
+    }
+
+    override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) {
+        val accountId = ctx.model.expectShape<StructureShape>(op.input.get()).members().first { it.memberName.lowercase() == "accountid" }
+        writer.addImport(RuntimeTypes.Http.Operation.OperationRequest)
+
+        writer.withBlock("execution.initialize.intercept { req, next -> ", "}") {
+            openBlock("if (req.subject.#L.isNullOrEmpty()) {", accountId.defaultName())
+                .write("val updated = req.subject.copy { #L = #S }", accountId.defaultName(), "-")
+                .write("next.call(#T(req.context, updated))", RuntimeTypes.Http.Operation.OperationRequest)
+                .closeAndOpenBlock("} else {")
+                .write("next.call(req)")
+                .closeBlock("}")
+        }
+    }
+}

codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/core/AwsHttpBindingProtocolGenerator.kt

@@ -76,7 +76,6 @@ abstract class AwsHttpBindingProtocolGenerator : HttpBindingProtocolGenerator()
                 "PutAndGetInlineDocumentsInput",
 
                 // Glacier customizations
-                "GlacierAccountId", // smithy-kotlin#163
                 "GlacierChecksums", // smithy-kotlin#164
                 "GlacierMultipartChecksums", // smithy-kotlin#164
             ),

codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/json/AwsJsonProtocolMiddleware.kt

@@ -9,7 +9,7 @@ import aws.sdk.kotlin.codegen.AwsKotlinDependency
 import software.amazon.smithy.kotlin.codegen.core.KotlinWriter
 import software.amazon.smithy.kotlin.codegen.model.buildSymbol
 import software.amazon.smithy.kotlin.codegen.model.namespace
-import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware
+import software.amazon.smithy.kotlin.codegen.rendering.protocol.HttpFeatureMiddleware
 import software.amazon.smithy.model.shapes.ShapeId
 
 /**
@@ -19,20 +19,17 @@ import software.amazon.smithy.model.shapes.ShapeId
 class AwsJsonProtocolMiddleware(
     private val serviceShapeId: ShapeId,
     private val protocolVersion: String
-) : ProtocolMiddleware {
+) : HttpFeatureMiddleware() {
     override val name: String = "AwsJsonProtocol"
+    override val order: Byte = 10
 
-    override fun addImportsAndDependencies(writer: KotlinWriter) {
-        super.addImportsAndDependencies(writer)
+    override fun renderConfigure(writer: KotlinWriter) {
         val awsJsonProtocolSymbol = buildSymbol {
             name = "AwsJsonProtocol"
             namespace(AwsKotlinDependency.AWS_JSON_PROTOCOLS)
         }
 
         writer.addImport(awsJsonProtocolSymbol)
-    }
-
-    override fun renderConfigure(writer: KotlinWriter) {
         writer.write("serviceShapeName = #S", serviceShapeId.name)
         writer.write("version = #S", protocolVersion)
     }

codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/middleware/AwsSignatureVersion4.kt

@@ -13,8 +13,8 @@ import software.amazon.smithy.kotlin.codegen.core.addImport
 import software.amazon.smithy.kotlin.codegen.model.expectShape
 import software.amazon.smithy.kotlin.codegen.model.expectTrait
 import software.amazon.smithy.kotlin.codegen.model.hasTrait
+import software.amazon.smithy.kotlin.codegen.rendering.protocol.HttpFeatureMiddleware
 import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerator
-import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware
 import software.amazon.smithy.model.Model
 import software.amazon.smithy.model.knowledge.ServiceIndex
 import software.amazon.smithy.model.shapes.OperationShape
@@ -26,8 +26,9 @@ import software.amazon.smithy.model.traits.OptionalAuthTrait
  * @param signingServiceName The credential scope service name to sign for
  * See the `name` property of: https://awslabs.github.io/smithy/1.0/spec/aws/aws-auth.html#aws-auth-sigv4-trait
  */
-open class AwsSignatureVersion4(private val signingServiceName: String) : ProtocolMiddleware {
+open class AwsSignatureVersion4(private val signingServiceName: String) : HttpFeatureMiddleware() {
     override val name: String = AwsRuntimeTypes.Auth.AwsSigV4SigningMiddleware.name
+    override val order: Byte = 127
 
     init {
         require(signingServiceName.isNotEmpty()) { "signingServiceName must be specified" }
@@ -38,12 +39,10 @@ open class AwsSignatureVersion4(private val signingServiceName: String) : Protoc
         return hasSigV4AuthScheme(ctx.model, service, op)
     }
 
-    override fun addImportsAndDependencies(writer: KotlinWriter) {
+    override fun renderConfigure(writer: KotlinWriter) {
         writer.addImport(AwsRuntimeTypes.Auth.AwsSigV4SigningMiddleware)
         writer.addImport("DefaultChainCredentialsProvider", AwsKotlinDependency.AWS_AUTH)
-    }
 
-    override fun renderConfigure(writer: KotlinWriter) {
         writer.write("credentialsProvider = config.credentialsProvider ?: DefaultChainCredentialsProvider()")
         writer.write("signingService = #S", signingServiceName)
     }

codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/middleware/EndpointResolverMiddleware.kt

@@ -9,15 +9,15 @@ import aws.sdk.kotlin.codegen.AwsKotlinDependency
 import software.amazon.smithy.kotlin.codegen.core.KotlinWriter
 import software.amazon.smithy.kotlin.codegen.model.buildSymbol
 import software.amazon.smithy.kotlin.codegen.model.namespace
+import software.amazon.smithy.kotlin.codegen.rendering.protocol.HttpFeatureMiddleware
 import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerator
-import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware
 
 /**
  * HTTP client interceptor that resolves service endpoints for a single service
  */
-class EndpointResolverMiddleware(private val ctx: ProtocolGenerator.GenerationContext) : ProtocolMiddleware {
+class EndpointResolverMiddleware(private val ctx: ProtocolGenerator.GenerationContext) : HttpFeatureMiddleware() {
     override val name: String = "ServiceEndpointResolver"
-    override fun addImportsAndDependencies(writer: KotlinWriter) {
+    override fun renderConfigure(writer: KotlinWriter) {
         val resolverFeatureSymbol = buildSymbol {
             name = "ServiceEndpointResolver"
             namespace(AwsKotlinDependency.AWS_HTTP, subpackage = "middleware")
@@ -31,9 +31,7 @@ class EndpointResolverMiddleware(private val ctx: ProtocolGenerator.GenerationCo
 
         writer.addImport(resolverFeatureSymbol)
         writer.addImport(defaultResolverSymbol)
-    }
 
-    override fun renderConfigure(writer: KotlinWriter) {
         writer.write("serviceId = ServiceId")
         writer.write("resolver = config.endpointResolver ?: DefaultEndpointResolver()")
     }

codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/middleware/MutateHeadersMiddleware.kt

@@ -7,7 +7,7 @@ package aws.sdk.kotlin.codegen.protocols.middleware
 
 import software.amazon.smithy.kotlin.codegen.core.KotlinWriter
 import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes
-import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware
+import software.amazon.smithy.kotlin.codegen.rendering.protocol.HttpFeatureMiddleware
 
 /**
  * General purpose middleware that allows mutation of headers
@@ -16,8 +16,9 @@ class MutateHeadersMiddleware(
     val extraHeaders: Map<String, String> = emptyMap(),
     val overrideHeaders: Map<String, String> = emptyMap(),
     val addMissingHeaders: Map<String, String> = emptyMap(),
-) : ProtocolMiddleware {
+) : HttpFeatureMiddleware() {
     override val name: String = "MutateHeaders"
+    override val order: Byte = 10
     override fun renderConfigure(writer: KotlinWriter) {
         writer.addImport(RuntimeTypes.Http.MutateHeadersMiddleware)
         overrideHeaders.forEach {

codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/middleware/UserAgentMiddleware.kt

@@ -9,13 +9,14 @@ import aws.sdk.kotlin.codegen.AwsKotlinDependency
 import software.amazon.smithy.kotlin.codegen.core.KotlinWriter
 import software.amazon.smithy.kotlin.codegen.model.buildSymbol
 import software.amazon.smithy.kotlin.codegen.model.namespace
-import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware
+import software.amazon.smithy.kotlin.codegen.rendering.protocol.HttpFeatureMiddleware
 
 /**
  * Middleware that sets the User-Agent and x-amz-user-agent headers
  */
-class UserAgentMiddleware : ProtocolMiddleware {
+class UserAgentMiddleware : HttpFeatureMiddleware() {
     override val name: String = "UserAgent"
+    override val order: Byte = 20
 
     private val uaSymbol = buildSymbol {
         name = "AwsUserAgentMetadata"