fix: correctly codegen paginators for items in sparse lists (#1072)

Author: ianbotsf

.changes/e43c0830-b8d0-43d0-a8bb-1e2c93b8ac7c.json

@@ -0,0 +1,5 @@
+{
+    "id": "e43c0830-b8d0-43d0-a8bb-1e2c93b8ac7c",
+    "type": "bugfix",
+    "description": "Correctly codegen paginators for items in sparse lists"
+}

codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/PaginatorGenerator.kt

@@ -16,10 +16,7 @@ import software.amazon.smithy.kotlin.codegen.core.defaultName
 import software.amazon.smithy.kotlin.codegen.core.withBlock
 import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration
 import software.amazon.smithy.kotlin.codegen.lang.KotlinTypes
-import software.amazon.smithy.kotlin.codegen.model.SymbolProperty
-import software.amazon.smithy.kotlin.codegen.model.expectShape
-import software.amazon.smithy.kotlin.codegen.model.hasAllOptionalMembers
-import software.amazon.smithy.kotlin.codegen.model.hasTrait
+import software.amazon.smithy.kotlin.codegen.model.*
 import software.amazon.smithy.kotlin.codegen.model.traits.PaginationTruncationMember
 import software.amazon.smithy.kotlin.codegen.utils.getOrNull
 import software.amazon.smithy.model.Model
@@ -267,6 +264,7 @@ private fun getItemDescriptorOrNull(paginationInfo: PaginationInfo, ctx: Codegen
     val itemLiteral = paginationInfo.itemsMemberPath!!.last()!!.defaultName()
     val itemPathLiteral = paginationInfo.itemsMemberPath.joinToString(separator = "?.") { it.defaultName() }
     val itemMember = ctx.model.expectShape(itemMemberId)
+    val isSparse = itemMember.isSparse
     val (collectionLiteral, targetMember) = when (itemMember) {
         is MapShape ->
             ctx.symbolProvider.toSymbol(itemMember)
@@ -279,7 +277,7 @@ private fun getItemDescriptorOrNull(paginationInfo: PaginationInfo, ctx: Codegen
     }
 
     return ItemDescriptor(
-        collectionLiteral,
+        collectionLiteral + if (isSparse) "?" else "",
         targetMember,
         itemLiteral,
         itemPathLiteral,

codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/PaginatorGeneratorTest.kt

@@ -263,6 +263,143 @@ class PaginatorGeneratorTest {
         actual.shouldContainOnlyOnceWithDiff(expectedImports)
     }
 
+    @Test
+    fun testRenderPaginatorWithSparseItem() {
+        val testModelWithItems = """
+            namespace com.test
+            
+            use aws.protocols#restJson1
+            
+            service Lambda {
+                operations: [ListFunctions]
+            }
+            
+            @paginated(
+                inputToken: "Marker",
+                outputToken: "NextMarker",
+                pageSize: "MaxItems",
+                items: "Functions"
+            )
+            @readonly
+            @http(method: "GET", uri: "/functions", code: 200)
+            operation ListFunctions {
+                input: ListFunctionsRequest,
+                output: ListFunctionsResponse
+            }
+            
+            structure ListFunctionsRequest {
+                @httpQuery("FunctionVersion")
+                FunctionVersion: String,
+                @httpQuery("Marker")
+                Marker: String,
+                @httpQuery("MasterRegion")
+                MasterRegion: String,
+                @httpQuery("MaxItems")
+                MaxItems: Integer
+            }
+            
+            structure ListFunctionsResponse {
+                Functions: FunctionConfigurationList,
+                NextMarker: String
+            }
+            
+            @sparse
+            list FunctionConfigurationList {
+                member: FunctionConfiguration
+            }
+            
+            structure FunctionConfiguration {
+                FunctionName: String
+            }
+        """.toSmithyModel()
+        val testContextWithItems = testModelWithItems.newTestContext("Lambda", "com.test")
+
+        val codegenContextWithItems = object : CodegenContext {
+            override val model: Model = testContextWithItems.generationCtx.model
+            override val symbolProvider: SymbolProvider = testContextWithItems.generationCtx.symbolProvider
+            override val settings: KotlinSettings = testContextWithItems.generationCtx.settings
+            override val protocolGenerator: ProtocolGenerator = testContextWithItems.generator
+            override val integrations: List<KotlinIntegration> = testContextWithItems.generationCtx.integrations
+        }
+
+        val unit = PaginatorGenerator()
+        unit.writeAdditionalFiles(codegenContextWithItems, testContextWithItems.generationCtx.delegator)
+
+        testContextWithItems.generationCtx.delegator.flushWriters()
+        val testManifest = testContextWithItems.generationCtx.delegator.fileManifest as MockManifest
+        val actual = testManifest.expectFileString("src/main/kotlin/com/test/paginators/Paginators.kt")
+
+        val expectedCode = """
+            /**
+             * Paginate over [ListFunctionsResponse] results.
+             *
+             * When this operation is called, a [kotlinx.coroutines.Flow] is created. Flows are lazy (cold) so no service
+             * calls are made until the flow is collected. This also means there is no guarantee that the request is valid
+             * until then. Once you start collecting the flow, the SDK will lazily load response pages by making service
+             * calls until there are no pages left or the flow is cancelled. If there are errors in your request, you will
+             * see the failures only after you start collection.
+             * @param initialRequest A [ListFunctionsRequest] to start pagination
+             * @return A [kotlinx.coroutines.flow.Flow] that can collect [ListFunctionsResponse]
+             */
+            public fun TestClient.listFunctionsPaginated(initialRequest: ListFunctionsRequest = ListFunctionsRequest { }): Flow<ListFunctionsResponse> =
+                flow {
+                    var cursor: kotlin.String? = null
+                    var hasNextPage: Boolean = true
+            
+                    while (hasNextPage) {
+                        val req = initialRequest.copy {
+                            this.marker = cursor
+                        }
+                        val result = [email protected](req)
+                        cursor = result.nextMarker
+                        hasNextPage = cursor?.isNotEmpty() == true
+                        emit(result)
+                    }
+                }
+            
+            /**
+             * Paginate over [ListFunctionsResponse] results.
+             *
+             * When this operation is called, a [kotlinx.coroutines.Flow] is created. Flows are lazy (cold) so no service
+             * calls are made until the flow is collected. This also means there is no guarantee that the request is valid
+             * until then. Once you start collecting the flow, the SDK will lazily load response pages by making service
+             * calls until there are no pages left or the flow is cancelled. If there are errors in your request, you will
+             * see the failures only after you start collection.
+             * @param block A builder block used for DSL-style invocation of the operation
+             * @return A [kotlinx.coroutines.flow.Flow] that can collect [ListFunctionsResponse]
+             */
+            public fun TestClient.listFunctionsPaginated(block: ListFunctionsRequest.Builder.() -> Unit): Flow<ListFunctionsResponse> =
+                listFunctionsPaginated(ListFunctionsRequest.Builder().apply(block).build())
+            
+            /**
+             * This paginator transforms the flow returned by [listFunctionsPaginated]
+             * to access the nested member [FunctionConfiguration]
+             * @return A [kotlinx.coroutines.flow.Flow] that can collect [FunctionConfiguration]
+             */
+            @JvmName("listFunctionsResponseFunctionConfiguration")
+            public fun Flow<ListFunctionsResponse>.functions(): Flow<FunctionConfiguration?> =
+                transform() { response ->
+                    response.functions?.forEach {
+                        emit(it)
+                    }
+                }
+        """.trimIndent()
+
+        actual.shouldContainOnlyOnceWithDiff(expectedCode)
+
+        val expectedImports = """
+            import com.test.model.FunctionConfiguration
+            import com.test.model.ListFunctionsRequest
+            import com.test.model.ListFunctionsResponse
+            import kotlin.jvm.JvmName
+            import kotlinx.coroutines.flow.Flow
+            import kotlinx.coroutines.flow.flow
+            import kotlinx.coroutines.flow.transform
+        """.trimIndent()
+
+        actual.shouldContainOnlyOnceWithDiff(expectedImports)
+    }
+
     @Test
     fun testRenderPaginatorWithTruncationMember() {
         val testModel = """