refactor: migrate non AWS specific protocol generation (#1012)

**refactor**: remove unused `KotlinIntegration::onShapeWriterUse` extension function
**refactor**: Replace `KotlinIntegration::renderBindEndpointBuiltins` with a new function `customizeEndpointResolution` which allows for more than just overriding rendering support for builtins. It allows registering custom property renderers and custom function support.
**refactor**: Remove exception base class from `ProtocolGenerator` in favor of declaring a section that can be overwritten. The logic for rendering the exception has been abstracted to `ServiceExceptionBaseClassGenerator` so that it can mostly be re-used when doing so. 
**refactor**: Migrate `mergeSequential` from `aws-sdk-kotlin` runtime to `smithy-kotlin` runtime. There wasn't anything AWS specific about this and it is only used for event streams (currently)
**refactor**: Sigv4 integration
    * Move various bits of codegen into section writers (support for protocol test service client setup)
    * Migrate logic from the AWS endpoint delegator into the new `EndpointCustomization` type
**refactor**: Move support for the `SDK::Endpoint` builtin into a `KotlinIntegration`
**refactor**: Move generation of `mergeServiceDefaults` into `HttpProtocolClientGenerator`. The function was never AWS specific, it just sets default operation execution context attributes.
**refactor**: Always generate an override of the operation `retryPolicy` in `HttpProtocolClientGenerator` since it is always registered as a service client configuration property.
**refactor**: Move the Smithy build transforms from `aws-sdk-kotlin` since they aren't specific to the SDK. Code/tests did not change, just the name of the transform.

Author: aajtodd

.changes/85c1a6d6-9e18-475a-8c8a-766d041530ab.json

@@ -0,0 +1,5 @@
+{
+    "id": "85c1a6d6-9e18-475a-8c8a-766d041530ab",
+    "type": "misc",
+    "description": "Refactor codegen to move non AWS specific support into core"
+}

codegen/smithy-kotlin-codegen-testutils/src/main/kotlin/software/amazon/smithy/kotlin/codegen/test/ModelTestUtils.kt

@@ -122,7 +122,7 @@ fun Model.newTestContext(
     val manifest = MockManifest()
     val provider: SymbolProvider = KotlinCodegenPlugin.createSymbolProvider(model = this, rootNamespace = packageName, serviceName = serviceName, settings = settings)
     val service = this.getShape(ShapeId.from("$packageName#$serviceName")).get().asServiceShape().get()
-    val delegator = KotlinDelegator(settings, this, manifest, provider)
+    val delegator = KotlinDelegator(settings, this, manifest, provider, integrations)
 
     val ctx = ProtocolGenerator.GenerationContext(
         settings,

codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/KotlinDelegator.kt

@@ -111,17 +111,6 @@ class KotlinDelegator(
         writer.addImportReferences(symbol, SymbolReference.ContextOption.DECLARE)
         writer.dependencies.addAll(symbol.dependencies)
         writer.pushState()
-
-        // shape is stored in the property bag when generated, if it's there pull it back out
-        val shape = symbol.getProperty("shape", Shape::class.java)
-        if (shape.isPresent) {
-            // Allow integrations to do things like add onSection callbacks.
-            // these onSection callbacks are removed when popState is called.
-            for (integration in integrations) {
-                integration.onShapeWriterUse(settings, model, symbolProvider, writer, shape.get())
-            }
-        }
-
         block(writer)
         writer.popState()
     }

codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/RuntimeTypes.kt

@@ -171,6 +171,7 @@ object RuntimeTypes {
             val flattenIfPossible = symbol("flattenIfPossible")
             val LazyAsyncValue = symbol("LazyAsyncValue")
             val length = symbol("length")
+            val mergeSequential = symbol("mergeSequential")
             val truthiness = symbol("truthiness")
             val toNumber = symbol("toNumber")
             val type = symbol("type")

codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/integration/KotlinIntegration.kt

@@ -8,14 +8,12 @@ import software.amazon.smithy.codegen.core.SymbolProvider
 import software.amazon.smithy.kotlin.codegen.*
 import software.amazon.smithy.kotlin.codegen.core.CodegenContext
 import software.amazon.smithy.kotlin.codegen.core.KotlinDelegator
-import software.amazon.smithy.kotlin.codegen.core.KotlinWriter
+import software.amazon.smithy.kotlin.codegen.rendering.endpoints.EndpointCustomization
 import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerator
 import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware
 import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigProperty
 import software.amazon.smithy.model.Model
 import software.amazon.smithy.model.shapes.ServiceShape
-import software.amazon.smithy.model.shapes.Shape
-import software.amazon.smithy.rulesengine.language.EndpointRuleSet
 
 /**
  * JVM SPI for customizing Kotlin code generation, registering new protocol
@@ -97,30 +95,6 @@ interface KotlinIntegration {
         symbolProvider: SymbolProvider,
     ): SymbolProvider = symbolProvider
 
-    /**
-     * Called each time a writer is used that defines a shape.
-     *
-     * Any mutations made on the writer (for example, adding
-     * section interceptors) are removed after the callback has completed;
-     * the callback is invoked in between pushing and popping state from
-     * the writer.
-     *
-     * @param settings Settings used to generate.
-     * @param model Model to generate from.
-     * @param symbolProvider Symbol provider used for codegen.
-     * @param writer Writer that will be used.
-     * @param definedShape Shape that is being defined in the writer.
-     */
-    fun onShapeWriterUse(
-        settings: KotlinSettings,
-        model: Model,
-        symbolProvider: SymbolProvider,
-        writer: KotlinWriter,
-        definedShape: Shape,
-    ) {
-        // pass
-    }
-
     /**
      * Write additional files defined by this integration
      * @param ctx The codegen generation context
@@ -154,10 +128,8 @@ interface KotlinIntegration {
     fun authSchemes(ctx: ProtocolGenerator.GenerationContext): List<AuthSchemeHandler> = emptyList()
 
     /**
-     * Render binding of endpoint ruleset builtin parameters
-     * @param ctx The codegen generation context
-     * @param rules The endpoint rules
-     * @param writer The writer to render to
+     * Register an endpoint customization
+     * @param ctx the codegen generation context
      */
-    fun renderBindEndpointBuiltins(ctx: ProtocolGenerator.GenerationContext, rules: EndpointRuleSet, writer: KotlinWriter) {}
+    fun customizeEndpointResolution(ctx: ProtocolGenerator.GenerationContext): EndpointCustomization? = null
 }

codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/ExceptionBaseClassGenerator.kt

@@ -10,26 +10,57 @@ import software.amazon.smithy.codegen.core.Symbol
 import software.amazon.smithy.kotlin.codegen.KotlinSettings
 import software.amazon.smithy.kotlin.codegen.core.*
 import software.amazon.smithy.kotlin.codegen.integration.SectionId
+import software.amazon.smithy.kotlin.codegen.integration.SectionKey
 import software.amazon.smithy.kotlin.codegen.model.buildSymbol
-import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerator
+import software.amazon.smithy.kotlin.codegen.model.namespace
 import software.amazon.smithy.model.knowledge.TopDownIndex
 
 /**
  * Renders the base class that all (modeled) exceptions inherit from.
  * Protocol generators are allowed to override this but they MUST inherit from the base `ServiceException`
  * with the expected constructors.
+ *
  */
 object ExceptionBaseClassGenerator {
 
+    val DefaultServiceExceptionSymbol: Symbol = buildSymbol {
+        name = "ServiceException"
+        namespace(KotlinDependency.CORE)
+    }
+
     /**
      * Defines a section in which code can be added to the body of the base exception type.
      */
-    object ExceptionBaseClassSection : SectionId
+    object ExceptionBaseClassSection : SectionId {
+        val CodegenContext: SectionKey<CodegenContext> = SectionKey("CodegenContext")
+    }
+
+    fun render(ctx: CodegenContext, writer: KotlinWriter) {
+        writer.declareSection(ExceptionBaseClassSection, mapOf(ExceptionBaseClassSection.CodegenContext to ctx)) {
+            ServiceExceptionBaseClassGenerator().render(ctx, writer)
+        }
+    }
+
+    /**
+     * Get the (generated) symbol that constitutes the base class exceptions will inherit from
+     */
+    fun baseExceptionSymbol(settings: KotlinSettings): Symbol = buildSymbol {
+        val serviceName = clientName(settings.sdkId)
+        name = "${serviceName}Exception"
+        namespace = "${settings.pkg.name}.model"
+        definitionFile = "$name.kt"
+    }
+}
 
+/**
+ * Re-usable component for generating the base exception that all generated error shapes will inherit from.
+ * @param parent symbol to inherit from (defaults to [ExceptionBaseClassGenerator.DefaultServiceExceptionSymbol])
+ */
+open class ServiceExceptionBaseClassGenerator(
+    private val parent: Symbol = ExceptionBaseClassGenerator.DefaultServiceExceptionSymbol,
+) {
     fun render(ctx: CodegenContext, writer: KotlinWriter) {
-        val baseException = ctx.protocolGenerator?.exceptionBaseClassSymbol ?: ProtocolGenerator.DefaultServiceExceptionSymbol
-        writer.addImport(baseException)
-        val serviceException = baseExceptionSymbol(ctx.settings).also { checkForCollision(ctx, it) }
+        val serviceException = ExceptionBaseClassGenerator.baseExceptionSymbol(ctx.settings).also { checkForCollision(ctx, it) }
 
         val name = clientName(ctx.settings.sdkId)
         writer.dokka("Base class for all service related exceptions thrown by the $name client")
@@ -38,26 +69,20 @@ object ExceptionBaseClassGenerator {
             "}",
             ctx.settings.api.visibility,
             serviceException,
-            baseException,
+            parent,
         ) {
             write("public constructor() : super()")
             write("public constructor(message: String?) : super(message)")
             write("public constructor(message: String?, cause: Throwable?) : super(message, cause)")
             write("public constructor(cause: Throwable?) : super(cause)")
-
-            writer.declareSection(ExceptionBaseClassSection)
+            renderExtra(ctx, writer)
         }
     }
 
     /**
-     * Get the (generated) symbol that constitutes the base class exceptions will inherit from
+     * Hook for subclasses to render additional overrides or methods on the base exception
      */
-    fun baseExceptionSymbol(settings: KotlinSettings): Symbol = buildSymbol {
-        val serviceName = clientName(settings.sdkId)
-        name = "${serviceName}Exception"
-        namespace = "${settings.pkg.name}.model"
-        definitionFile = "$name.kt"
-    }
+    protected open fun renderExtra(ctx: CodegenContext, writer: KotlinWriter) { }
 
     // Compare generated base exception name with all error type names.  Throw exception if not unique.
     private fun checkForCollision(ctx: CodegenContext, exceptionSymbol: Symbol) {

codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/ServiceClientGenerator.kt

@@ -77,6 +77,7 @@ class ServiceClientGenerator(private val ctx: RenderingContext<ServiceShape>) {
         writer.write("\n\n")
         writer.write("#L const val ServiceId: String = #S", ctx.settings.api.visibility, ctx.settings.sdkId)
         writer.write("#L const val SdkVersion: String = #S", ctx.settings.api.visibility, ctx.settings.pkg.version)
+        writer.write("#L const val ServiceApiVersion: String = #S", ctx.settings.api.visibility, service.version)
         writer.write("\n\n")
 
         writer.putContext("service.name", ctx.settings.sdkId)

codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/auth/BearerTokenAuthSchemeIntegration.kt

@@ -11,9 +11,13 @@ import software.amazon.smithy.kotlin.codegen.KotlinSettings
 import software.amazon.smithy.kotlin.codegen.core.CodegenContext
 import software.amazon.smithy.kotlin.codegen.core.KotlinWriter
 import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes
+import software.amazon.smithy.kotlin.codegen.core.getContextValue
+import software.amazon.smithy.kotlin.codegen.integration.AppendingSectionWriter
 import software.amazon.smithy.kotlin.codegen.integration.AuthSchemeHandler
 import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration
+import software.amazon.smithy.kotlin.codegen.integration.SectionWriterBinding
 import software.amazon.smithy.kotlin.codegen.model.buildSymbol
+import software.amazon.smithy.kotlin.codegen.rendering.protocol.HttpProtocolClientGenerator
 import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerator
 import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigProperty
 import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigPropertyType
@@ -54,6 +58,22 @@ class BearerTokenAuthSchemeIntegration : KotlinIntegration {
 
         return listOf(bearerTokenProviderProp)
     }
+
+    override val sectionWriters: List<SectionWriterBinding>
+        get() = listOf(
+            SectionWriterBinding(HttpProtocolClientGenerator.ClientInitializer, renderClientInitializer),
+        )
+
+    private val renderClientInitializer = AppendingSectionWriter { writer ->
+        val ctx = writer.getContextValue(HttpProtocolClientGenerator.ClientInitializer.GenerationContext)
+        val serviceIndex = ServiceIndex.of(ctx.model)
+        val hasBearerTokenAuth = serviceIndex
+            .getAuthSchemes(ctx.settings.service)
+            .containsKey(HttpBearerAuthTrait.ID)
+        if (hasBearerTokenAuth) {
+            writer.write("managedResources.#T(config.bearerTokenProvider)", RuntimeTypes.Core.IO.addIfManaged)
+        }
+    }
 }
 
 class BearerTokenAuthSchemeHandler : AuthSchemeHandler {