add DSL overloads, paginators, and better builder integration for DDB Mapper ops codegen

hll/build.gradle.kts

@@ -25,8 +25,16 @@ val sdkVersion: String by project
 // capture locally - scope issue with custom KMP plugin
 val libraries = libs
 
+val optinAnnotations = listOf(
+    "aws.smithy.kotlin.runtime.InternalApi",
+    "aws.sdk.kotlin.runtime.InternalSdkApi",
+    "kotlin.RequiresOptIn",
+)
+
 subprojects {
-    if (!needsKmpConfigured) return@subprojects
+    if (!needsKmpConfigured) {
+        return@subprojects
+    }
 
     group = "aws.sdk.kotlin"
     version = sdkVersion
@@ -45,6 +53,10 @@ subprojects {
         sourceSets {
             // dependencies available for all subprojects
 
+            all {
+                optinAnnotations.forEach(languageSettings::optIn)
+            }
+
             named("commonTest") {
                 dependencies {
                     implementation(libraries.kotest.assertions.core)
@@ -60,12 +72,6 @@ subprojects {
         }
     }
 
-    kotlin.sourceSets.all {
-        // Allow subprojects to use internal APIs
-        // See https://kotlinlang.org/docs/reference/opt-in-requirements.html#opting-in-to-using-api
-        listOf("kotlin.RequiresOptIn").forEach { languageSettings.optIn(it) }
-    }
-
     dependencies {
         dokkaPlugin(project(":dokka-aws"))
     }

hll/dynamodb-mapper/dynamodb-mapper-codegen/build.gradle.kts

@@ -24,8 +24,18 @@ dependencies {
     testImplementation(libs.kotlin.test.junit5)
 }
 
+val optinAnnotations = listOf(
+    "aws.smithy.kotlin.runtime.InternalApi",
+    "aws.sdk.kotlin.runtime.InternalSdkApi",
+    "kotlin.RequiresOptIn",
+)
+
 kotlin {
     explicitApi()
+
+    sourceSets.all {
+        optinAnnotations.forEach(languageSettings::optIn)
+    }
 }
 
 tasks.test {

hll/dynamodb-mapper/dynamodb-mapper-codegen/src/main/kotlin/aws/sdk/kotlin/hll/dynamodbmapper/codegen/annotations/rendering/SchemaRenderer.kt

@@ -103,7 +103,7 @@ internal class SchemaRenderer(
 
     private fun renderBuilder() {
         val members = properties.map(Member.Companion::from).toSet()
-        BuilderRenderer(this, classType, members, ctx).render()
+        BuilderRenderer(this, classType, classType, members, ctx).render()
     }
 
     private fun renderItemConverter() {

hll/dynamodb-mapper/dynamodb-mapper-codegen/src/main/kotlin/aws/sdk/kotlin/hll/dynamodbmapper/codegen/model/MapperTypes.kt

@@ -1,10 +1,10 @@
 package aws.sdk.kotlin.hll.dynamodbmapper.codegen.model
 
+import aws.sdk.kotlin.hll.codegen.model.Pkg
 import aws.sdk.kotlin.hll.codegen.model.Type
 import aws.sdk.kotlin.hll.codegen.model.TypeRef
 import aws.sdk.kotlin.hll.codegen.model.TypeVar
 import aws.sdk.kotlin.hll.codegen.model.Types
-import aws.sdk.kotlin.hll.codegen.util.Pkg
 
 /**
  * A container object for various DynamoDbMapper [Type] instances
@@ -17,6 +17,10 @@ internal object MapperTypes {
     // High-level types
     val DynamoDbMapper = TypeRef(Pkg.Hl.Base, "DynamoDbMapper")
 
+    object Annotations {
+        val ManualPagination = TypeRef(Pkg.Hl.Annotations, "ManualPagination")
+    }
+
     object Items {
         fun itemSchema(typeVar: String) = TypeRef(Pkg.Hl.Items, "ItemSchema", listOf(TypeVar(typeVar)))
         fun itemSchemaPartitionKey(objectType: TypeRef, pkType: TypeRef) = TypeRef(Pkg.Hl.Items, "ItemSchema.PartitionKey", listOf(objectType, pkType))

hll/dynamodb-mapper/dynamodb-mapper-codegen/src/main/kotlin/aws/sdk/kotlin/hll/dynamodbmapper/codegen/operations/HighLevelOpsProcessor.kt

@@ -5,9 +5,11 @@
 package aws.sdk.kotlin.hll.dynamodbmapper.codegen.operations
 
 import aws.sdk.kotlin.hll.codegen.core.CodeGeneratorFactory
+import aws.sdk.kotlin.hll.codegen.model.Operation
+import aws.sdk.kotlin.hll.codegen.model.Pkg
 import aws.sdk.kotlin.hll.codegen.rendering.RenderContext
-import aws.sdk.kotlin.hll.codegen.util.Pkg
-import aws.sdk.kotlin.hll.dynamodbmapper.codegen.operations.model.Operation
+import aws.sdk.kotlin.hll.codegen.util.KspLoggerOutputStream
+import aws.sdk.kotlin.hll.codegen.util.asPrintStream
 import aws.sdk.kotlin.hll.dynamodbmapper.codegen.operations.model.toHighLevel
 import aws.sdk.kotlin.hll.dynamodbmapper.codegen.operations.rendering.HighLevelRenderer
 import aws.sdk.kotlin.services.dynamodb.DynamoDbClient
@@ -33,6 +35,9 @@ internal class HighLevelOpsProcessor(environment: SymbolProcessorEnvironment) :
         if (!invoked) {
             invoked = true
 
+            val loggerOutStream = KspLoggerOutputStream(logger)
+            System.setOut(loggerOutStream.asPrintStream())
+
             logger.info("Scanning low-level DDB client for operations and types")
             val operations = getOperations(resolver)
 
@@ -45,11 +50,19 @@ internal class HighLevelOpsProcessor(environment: SymbolProcessorEnvironment) :
         return listOf()
     }
 
-    private fun allow(func: KSFunctionDeclaration) =
-        (opAllowlist?.contains(func.simpleName.getShortName()) ?: true).also {
-            if (!it) logger.warn("${func.simpleName.getShortName()} not in allowlist; skipping codegen")
+    private fun allow(func: KSFunctionDeclaration): Boolean {
+        val name = func.simpleName.getShortName()
+        val allowed = opAllowlist?.contains(name)
+
+        when (allowed) {
+            false -> logger.warn("$name not in allowlist; skipping codegen")
+            true -> logger.info("$name in allowlist; processing...")
+            null -> Unit // There is no allowlist—don't log anything
         }
 
+        return allowed ?: true
+    }
+
     private fun getOperations(resolver: Resolver): List<Operation> = resolver
         .getClassDeclarationByName<DynamoDbClient>()!!
         .getDeclaredFunctions()

hll/dynamodb-mapper/dynamodb-mapper-codegen/src/main/kotlin/aws/sdk/kotlin/hll/dynamodbmapper/codegen/operations/model/ItemSourceKind.kt

@@ -4,9 +4,10 @@
  */
 package aws.sdk.kotlin.hll.dynamodbmapper.codegen.operations.model
 
+import aws.sdk.kotlin.hll.codegen.model.Operation
+import aws.sdk.kotlin.hll.codegen.model.Pkg
 import aws.sdk.kotlin.hll.codegen.model.TypeRef
 import aws.sdk.kotlin.hll.codegen.model.TypeVar
-import aws.sdk.kotlin.hll.codegen.util.Pkg
 
 /**
  * Identifies a type in the `ItemSource<T>` hierarchy

hll/dynamodb-mapper/dynamodb-mapper-codegen/src/main/kotlin/aws/sdk/kotlin/hll/dynamodbmapper/codegen/operations/model/MemberCodegenBehavior.kt

@@ -5,33 +5,21 @@
 package aws.sdk.kotlin.hll.dynamodbmapper.codegen.operations.model
 
 import aws.sdk.kotlin.hll.codegen.model.Member
+import aws.sdk.kotlin.hll.codegen.model.Pkg
 import aws.sdk.kotlin.hll.codegen.model.TypeRef
 import aws.sdk.kotlin.hll.codegen.model.Types
 import aws.sdk.kotlin.hll.codegen.model.nullable
-import aws.sdk.kotlin.hll.codegen.util.Pkg
 import aws.sdk.kotlin.hll.dynamodbmapper.codegen.model.MapperTypes
 
 private val attrMapTypes = setOf(MapperTypes.AttributeMap, MapperTypes.AttributeMap.nullable())
+private val attrMapListTypes = Types.Kotlin.list(MapperTypes.AttributeMap).let { setOf(it, it.nullable()) }
 
 /**
  * Describes a behavior to apply for a given [Member] in a low-level structure when generating code for an equivalent
  * high-level structure. This interface implements no behaviors on its own; it merely gives strongly-typed names to
  * behaviors that will be implemented by calling code.
  */
 internal sealed interface MemberCodegenBehavior {
-    companion object {
-        /**
-         * Identifies a [MemberCodegenBehavior] for the given [Member] by way of various heuristics
-         * @param member The [Member] for which to identify a codegen behavior
-         */
-        fun identifyFor(member: Member) = when {
-            member in unsupportedMembers -> Drop
-            member.type in attrMapTypes -> if (member.name == "key") MapKeys else MapAll
-            member.isTableName -> Hoist
-            else -> PassThrough
-        }
-    }
-
     /**
      * Indicates that a member should be copied as-is from a low-level structure to a high-level equivalent (i.e., no
      * changes to name, type, etc. are required)
@@ -47,11 +35,18 @@ internal sealed interface MemberCodegenBehavior {
 
     /**
      * Indicates that a member is an attribute map which contains _key_ attributes for a data type (as opposed to _all_
-     * attributes) and should be replaced with a generic type (i.e., a `Map<String, AttributeValue>` member
-     * in a low-level structure should be replaced with a generic `T` member in a high-level structure)
+     * attributes) and should be replaced with a generic type (i.e., a `Map<String, AttributeValue>` member in a
+     * low-level structure should be replaced with a generic `T` member in a high-level structure)
      */
     data object MapKeys : MemberCodegenBehavior
 
+    /**
+     * Indicates that a member is a list of attribute maps which may contain attributes for a data type and should be
+     * replaced with a generic list type (i.e., a `List<Map<String, AttributeValue>>` member in a low-level structure
+     * should be replaced with a generic `List<T>` member in a high-level structure)
+     */
+    data object ListMapAll : MemberCodegenBehavior
+
     /**
      * Indicates that a member is unsupported and should not be replicated from a low-level structure to the high-level
      * equivalent (e.g., a deprecated member that has been replaced with new features need not be carried forward)
@@ -74,6 +69,7 @@ internal val Member.codegenBehavior: MemberCodegenBehavior
     get() = when {
         this in unsupportedMembers -> MemberCodegenBehavior.Drop
         type in attrMapTypes -> if (name == "key") MemberCodegenBehavior.MapKeys else MemberCodegenBehavior.MapAll
+        type in attrMapListTypes -> MemberCodegenBehavior.ListMapAll
         isTableName || isIndexName -> MemberCodegenBehavior.Hoist
         else -> MemberCodegenBehavior.PassThrough
     }

hll/dynamodb-mapper/dynamodb-mapper-codegen/src/main/kotlin/aws/sdk/kotlin/hll/dynamodbmapper/codegen/operations/model/ModelAttributes.kt

@@ -4,6 +4,8 @@
  */
 package aws.sdk.kotlin.hll.dynamodbmapper.codegen.operations.model
 
+import aws.sdk.kotlin.hll.codegen.model.Operation
+import aws.sdk.kotlin.hll.codegen.model.Structure
 import aws.smithy.kotlin.runtime.collections.AttributeKey
 
 /**
@@ -19,4 +21,6 @@ internal object ModelAttributes {
      * For a given high-level [Structure], this attribute key identifies the associated low-level [Structure]
      */
     val LowLevelStructure: AttributeKey<Structure> = AttributeKey("aws.sdk.kotlin.ddbmapper#LowLevelStructure")
+
+    val PaginationInfo: AttributeKey<PaginationMembers> = AttributeKey("aws.sdk.kotlin.ddbmapper#PaginationInfo")
 }

hll/dynamodb-mapper/dynamodb-mapper-codegen/src/main/kotlin/aws/sdk/kotlin/hll/dynamodbmapper/codegen/operations/model/Operation.kt

@@ -4,44 +4,9 @@
  */
 package aws.sdk.kotlin.hll.dynamodbmapper.codegen.operations.model
 
-import aws.sdk.kotlin.hll.codegen.util.capitalizeFirstChar
+import aws.sdk.kotlin.hll.codegen.model.Operation
 import aws.sdk.kotlin.hll.codegen.util.plus
-import aws.smithy.kotlin.runtime.collections.Attributes
-import aws.smithy.kotlin.runtime.collections.emptyAttributes
 import aws.smithy.kotlin.runtime.collections.get
-import com.google.devtools.ksp.symbol.KSFunctionDeclaration
-
-/**
- * Describes a service operation (i.e., API method)
- * @param methodName The name of the operation as a code-suitable method name. For example, `getItem` is a suitable
- * method name in Kotlin, whereas `GetItem` is not (improperly cased) nor is `get item` (contains a space).
- * @param request The [Structure] for requests/inputs to this operation
- * @param response The [Structure] for responses/output from this operation
- * @param attributes An [Attributes] collection for associating typed attributes with this operation
- */
-internal data class Operation(
-    val methodName: String,
-    val request: Structure,
-    val response: Structure,
-    val attributes: Attributes = emptyAttributes(),
-) {
-    /**
-     * The capitalized name of this operation's [methodName]. For example, if [methodName] is `getItem` then [name]
-     * would be `GetItem`.
-     */
-    val name = methodName.capitalizeFirstChar // e.g., "GetItem" vs "getItem"
-
-    companion object {
-        /**
-         * Derive an [Operation] from a [KSFunctionDeclaration]
-         */
-        fun from(declaration: KSFunctionDeclaration) = Operation(
-            methodName = declaration.simpleName.getShortName(),
-            request = Structure.from(declaration.parameters.single().type),
-            response = Structure.from(declaration.returnType!!),
-        )
-    }
-}
 
 /**
  * Gets the low-level [Operation] equivalent for this high-level operation

hll/dynamodb-mapper/dynamodb-mapper-codegen/src/main/kotlin/aws/sdk/kotlin/hll/dynamodbmapper/codegen/operations/model/Pagination.kt

@@ -0,0 +1,53 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package aws.sdk.kotlin.hll.dynamodbmapper.codegen.operations.model
+
+import aws.sdk.kotlin.hll.codegen.model.Member
+import aws.sdk.kotlin.hll.codegen.model.ModelParsingPlugin
+import aws.sdk.kotlin.hll.codegen.model.Operation
+import aws.sdk.kotlin.hll.codegen.util.plus
+
+/**
+ * Identifies the [Member] instances of an operation's request and response which control pagination
+ * @param inputToken The field for passing a pagination token into a request
+ * @param outputToken The field for receiving a pagination token from a request
+ * @param limit The field for limiting the number of returned results
+ * @param items The field for getting the low-level items from each page of results
+ */
+internal data class PaginationMembers(
+    val inputToken: Member,
+    val outputToken: Member,
+    val limit: Member,
+    val items: Member,
+) {
+    internal companion object {
+        fun forOperationOrNull(operation: Operation): PaginationMembers? {
+            val inputToken = operation.request.members.find { it.name == "exclusiveStartKey" } ?: return null
+            val outputToken = operation.response.members.find { it.name == "lastEvaluatedKey" } ?: return null
+            val limit = operation.request.members.find { it.name == "limit" } ?: return null
+            val items = operation.response.members.find { it.name == "items" } ?: return null
+
+            return PaginationMembers(inputToken, outputToken, limit, items)
+        }
+    }
+}
+
+/**
+ * Gets the [PaginationMembers] for an operation, if applicable. If the operation does not support pagination, this
+ * property returns `null`.
+ */
+internal val Operation.paginationInfo: PaginationMembers?
+    get() = attributes.getOrNull(ModelAttributes.PaginationInfo)
+
+/**
+ * A codegen plugin that adds DDB-specific pagination info to operations
+ */
+internal class DdbPaginationPlugin : ModelParsingPlugin {
+    override fun postProcessOperation(operation: Operation): Operation {
+        val paginationMembers = PaginationMembers.forOperationOrNull(operation) ?: return operation
+        val newAttributes = operation.attributes + (ModelAttributes.PaginationInfo to paginationMembers)
+        return operation.copy(attributes = newAttributes)
+    }
+}