Skip to content

Commit 5d8d7fd

Browse files
ianbotsfianbotsf
authored
chore: reintegrate E2E smoke tests into the top-level Gradle build; modularize smoke test runners to improve testability (#1262)
1 parent f6557ba commit 5d8d7fd

File tree

6 files changed

+131
-71
lines changed

6 files changed

+131
-71
lines changed

‎codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/RuntimeTypes.kt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,7 @@ object RuntimeTypes {
115115
}
116116

117117
object SmokeTests : RuntimeTypePackage(KotlinDependency.CORE, "smoketests") {
118+
val DefaultPrinter = symbol("DefaultPrinter")
118119
val exitProcess = symbol("exitProcess")
119120
val printExceptionStackTrace = symbol("printExceptionStackTrace")
120121
val SmokeTestsException = symbol("SmokeTestsException")

‎codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/lang/KotlinTypes.kt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ object KotlinTypes {
109109
}
110110

111111
object Text : RuntimeTypePackage(KotlinDependency.KOTLIN_STDLIB, "text") {
112+
val Appendable = stdlibSymbol("Appendable")
112113
val encodeToByteArray = stdlibSymbol("encodeToByteArray")
113114
}
114115

‎codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/smoketests/SmokeTestsRunnerGenerator.kt‎

Lines changed: 49 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import software.amazon.smithy.codegen.core.Symbol
44
import software.amazon.smithy.kotlin.codegen.core.*
55
import software.amazon.smithy.kotlin.codegen.integration.SectionId
66
import software.amazon.smithy.kotlin.codegen.integration.SectionKey
7+
import software.amazon.smithy.kotlin.codegen.lang.KotlinTypes
78
import software.amazon.smithy.kotlin.codegen.model.getTrait
89
import software.amazon.smithy.kotlin.codegen.model.hasTrait
910
import software.amazon.smithy.kotlin.codegen.rendering.ShapeValueGenerator
@@ -17,8 +18,8 @@ import software.amazon.smithy.kotlin.codegen.rendering.util.format
1718
import software.amazon.smithy.kotlin.codegen.utils.dq
1819
import software.amazon.smithy.kotlin.codegen.utils.toCamelCase
1920
import software.amazon.smithy.kotlin.codegen.utils.topDownOperations
20-
import software.amazon.smithy.model.node.*
21-
import software.amazon.smithy.model.shapes.*
21+
import software.amazon.smithy.model.node.Node
22+
import software.amazon.smithy.model.shapes.OperationShape
2223
import software.amazon.smithy.smoketests.traits.SmokeTestCase
2324
import software.amazon.smithy.smoketests.traits.SmokeTestsTrait
2425
import kotlin.jvm.optionals.getOrNull
@@ -61,25 +62,47 @@ class SmokeTestsRunnerGenerator(
6162
) {
6263
internal fun render() {
6364
writer.declareSection(SmokeTestSectionIds.SmokeTestsFile) {
64-
writer.write("private var exitCode = 0")
65+
write("")
66+
67+
withBlock("public suspend fun main() {", "}") {
68+
write("val success = SmokeTestRunner().runAllTests()")
69+
withBlock("if (!success) {", "}") {
70+
write("#T(1)", RuntimeTypes.Core.SmokeTests.exitProcess)
71+
}
72+
}
73+
write("")
74+
renderRunnerClass()
75+
}
76+
}
77+
78+
private fun renderRunnerClass() {
79+
writer.withBlock(
80+
"public class SmokeTestRunner(private val platform: #1T = #1T.System, private val printer: #2T = #3T) {",
81+
"}",
82+
RuntimeTypes.Core.Utils.PlatformProvider,
83+
KotlinTypes.Text.Appendable,
84+
RuntimeTypes.Core.SmokeTests.DefaultPrinter,
85+
) {
6586
renderEnvironmentVariables()
66-
writer.declareSection(SmokeTestSectionIds.AdditionalEnvironmentVariables)
67-
writer.write("")
68-
writer.withBlock("public suspend fun main() {", "}") {
69-
renderFunctionCalls()
70-
write("#T(exitCode)", RuntimeTypes.Core.SmokeTests.exitProcess)
87+
declareSection(SmokeTestSectionIds.AdditionalEnvironmentVariables)
88+
write("")
89+
90+
withBlock("public suspend fun runAllTests(): Boolean =", "") {
91+
withBlock("listOf<suspend () -> Boolean>(", ")") {
92+
renderFunctionReferences()
93+
}
94+
indent()
95+
write(".map { it() }")
96+
write(".all { it }")
97+
dedent()
7198
}
72-
writer.write("")
7399
renderFunctions()
74100
}
75101
}
76102

77103
private fun renderEnvironmentVariables() {
78104
// Skip tags
79-
writer.writeInline(
80-
"private val skipTags = #T.System.getenv(",
81-
RuntimeTypes.Core.Utils.PlatformProvider,
82-
)
105+
writer.writeInline("private val skipTags = platform.getenv(")
83106
writer.declareSection(SmokeTestSectionIds.SkipTags) {
84107
writer.writeInline("#S", SKIP_TAGS)
85108
}
@@ -89,10 +112,7 @@ class SmokeTestsRunnerGenerator(
89112
)
90113

91114
// Service filter
92-
writer.writeInline(
93-
"private val serviceFilter = #T.System.getenv(",
94-
RuntimeTypes.Core.Utils.PlatformProvider,
95-
)
115+
writer.writeInline("private val serviceFilter = platform.getenv(")
96116
writer.declareSection(SmokeTestSectionIds.ServiceFilter) {
97117
writer.writeInline("#S", SERVICE_FILTER)
98118
}
@@ -102,10 +122,10 @@ class SmokeTestsRunnerGenerator(
102122
)
103123
}
104124

105-
private fun renderFunctionCalls() {
125+
private fun renderFunctionReferences() {
106126
operations.forEach { operation ->
107127
operation.getTrait<SmokeTestsTrait>()?.testCases?.forEach { testCase ->
108-
writer.write("${testCase.functionName}()")
128+
writer.write("::${testCase.functionName},")
109129
}
110130
}
111131
}
@@ -120,7 +140,7 @@ class SmokeTestsRunnerGenerator(
120140
}
121141

122142
private fun renderFunction(operation: OperationShape, testCase: SmokeTestCase) {
123-
writer.withBlock("private suspend fun ${testCase.functionName}() {", "}") {
143+
writer.withBlock("private suspend fun ${testCase.functionName}(): Boolean {", "}") {
124144
write("val tags = setOf<String>(${testCase.tags.joinToString(",") { it.dq()} })")
125145
writer.withBlock("if ((serviceFilter.isNotEmpty() && #S !in serviceFilter) || tags.any { it in skipTags }) {", "}", sdkId) {
126146
printTestResult(
@@ -131,10 +151,10 @@ class SmokeTestsRunnerGenerator(
131151
"ok",
132152
"# skip",
133153
)
134-
writer.write("return")
154+
writer.write("return true")
135155
}
136156
write("")
137-
withInlineBlock("try {", "} ") {
157+
withInlineBlock("return try {", "} ") {
138158
renderTestCase(operation, testCase)
139159
}
140160
withBlock("catch (exception: Exception) {", "}") {
@@ -149,6 +169,8 @@ class SmokeTestsRunnerGenerator(
149169
closeAndOpenBlock("}.#T { client ->", RuntimeTypes.Core.IO.use)
150170
renderOperation(operation, testCase)
151171
}
172+
writer.write("")
173+
writer.write("error(#S)", "Unexpectedly completed smoke test operation without throwing exception")
152174
}
153175

154176
private fun renderClientConfig(testCase: SmokeTestCase) {
@@ -212,9 +234,11 @@ class SmokeTestsRunnerGenerator(
212234
)
213235

214236
writer.withBlock("if (!success) {", "}") {
215-
write("#T(exception)", RuntimeTypes.Core.SmokeTests.printExceptionStackTrace)
216-
write("exitCode = 1")
237+
write("printer.appendLine(exception.stackTraceToString().prependIndent(#S))", "# ")
217238
}
239+
240+
writer.write("")
241+
writer.write("success")
218242
}
219243

220244
// Helpers
@@ -241,7 +265,7 @@ class SmokeTestsRunnerGenerator(
241265
val expectation = if (errorExpected) "error expected from service" else "no error expected from service"
242266
val status = statusOverride ?: "\$status"
243267
val testResult = "$status $service $testCase - $expectation $directive"
244-
writer.write("println(#S)", testResult)
268+
writer.write("printer.appendLine(#S)", testResult)
245269
}
246270

247271
/**
@@ -250,18 +274,6 @@ class SmokeTestsRunnerGenerator(
250274
private val SmokeTestCase.functionName: String
251275
get() = this.id.toCamelCase()
252276

253-
/**
254-
* Get the operation parameters for a [SmokeTestCase]
255-
*/
256-
private val SmokeTestCase.operationParameters: Map<StringNode, Node>
257-
get() = this.params.get().members
258-
259-
/**
260-
* Checks if there are operation parameters for a [SmokeTestCase]
261-
*/
262-
private val SmokeTestCase.hasOperationParameters: Boolean
263-
get() = this.params.isPresent
264-
265277
/**
266278
* Check if a [SmokeTestCase] is expecting a specific error
267279
*/

‎codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/smoketests/SmokeTestsRunnerGeneratorTest.kt‎

Lines changed: 65 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -74,10 +74,9 @@ class SmokeTestsRunnerGeneratorTest {
7474
fun variablesTest() {
7575
generatedCode.shouldContainOnlyOnceWithDiff(
7676
"""
77-
private var exitCode = 0
78-
private val skipTags = PlatformProvider.System.getenv("SMOKE_TEST_SKIP_TAGS")?.let { it.split(",").map { it.trim() }.toSet() } ?: emptySet()
79-
private val serviceFilter = PlatformProvider.System.getenv("SMOKE_TEST_SERVICE_IDS")?.let { it.split(",").map { it.trim() }.toSet() }
80-
""".trimIndent(),
77+
private val skipTags = platform.getenv("SMOKE_TEST_SKIP_TAGS")?.let { it.split(",").map { it.trim() }.toSet() } ?: emptySet()
78+
private val serviceFilter = platform.getenv("SMOKE_TEST_SERVICE_IDS")?.let { it.split(",").map { it.trim() }.toSet() }
79+
""".formatForTest(),
8180
)
8281
}
8382

@@ -86,27 +85,50 @@ class SmokeTestsRunnerGeneratorTest {
8685
generatedCode.shouldContainOnlyOnceWithDiff(
8786
"""
8887
public suspend fun main() {
89-
successTest()
90-
invalidMessageErrorTest()
91-
failureTest()
92-
exitProcess(exitCode)
88+
val success = SmokeTestRunner().runAllTests()
89+
if (!success) {
90+
exitProcess(1)
91+
}
9392
}
9493
""".trimIndent(),
9594
)
9695
}
9796

97+
@Test
98+
fun runnerClassTest() {
99+
generatedCode.shouldContainOnlyOnceWithDiff(
100+
"public class SmokeTestRunner(private val platform: PlatformProvider = PlatformProvider.System, private val printer: Appendable = DefaultPrinter) {",
101+
)
102+
}
103+
104+
@Test
105+
fun runAllTestsTest() {
106+
generatedCode.shouldContainOnlyOnceWithDiff(
107+
"""
108+
public suspend fun runAllTests(): Boolean =
109+
listOf<suspend () -> Boolean>(
110+
::successTest,
111+
::invalidMessageErrorTest,
112+
::failureTest,
113+
)
114+
.map { it() }
115+
.all { it }
116+
""".formatForTest(),
117+
)
118+
}
119+
98120
@Test
99121
fun successTest() {
100122
generatedCode.shouldContainOnlyOnceWithDiff(
101123
"""
102-
private suspend fun successTest() {
124+
private suspend fun successTest(): Boolean {
103125
val tags = setOf<String>("success")
104126
if ((serviceFilter.isNotEmpty() && "Test" !in serviceFilter) || tags.any { it in skipTags }) {
105-
println("ok Test SuccessTest - no error expected from service # skip")
106-
return
127+
printer.appendLine("ok Test SuccessTest - no error expected from service # skip")
128+
return true
107129
}
108-
109-
try {
130+
131+
return try {
110132
TestClient {
111133
interceptors.add(SmokeTestsInterceptor())
112134
region = "eu-central-1"
@@ -118,33 +140,36 @@ class SmokeTestsRunnerGeneratorTest {
118140
}
119141
)
120142
}
121-
143+
144+
error("Unexpectedly completed smoke test operation without throwing exception")
145+
122146
} catch (exception: Exception) {
123147
val success: Boolean = exception is SmokeTestsSuccessException
124148
val status: String = if (success) "ok" else "not ok"
125-
println("${'$'}status Test SuccessTest - no error expected from service ")
149+
printer.appendLine("${'$'}status Test SuccessTest - no error expected from service ")
126150
if (!success) {
127-
printExceptionStackTrace(exception)
128-
exitCode = 1
151+
printer.appendLine(exception.stackTraceToString().prependIndent("# "))
129152
}
153+
154+
success
130155
}
131156
}
132-
""".trimIndent(),
157+
""".formatForTest(),
133158
)
134159
}
135160

136161
@Test
137162
fun invalidMessageErrorTest() {
138163
generatedCode.shouldContainOnlyOnceWithDiff(
139164
"""
140-
private suspend fun invalidMessageErrorTest() {
165+
private suspend fun invalidMessageErrorTest(): Boolean {
141166
val tags = setOf<String>()
142167
if ((serviceFilter.isNotEmpty() && "Test" !in serviceFilter) || tags.any { it in skipTags }) {
143-
println("ok Test InvalidMessageErrorTest - error expected from service # skip")
144-
return
168+
printer.appendLine("ok Test InvalidMessageErrorTest - error expected from service # skip")
169+
return true
145170
}
146171
147-
try {
172+
return try {
148173
TestClient {
149174
}.use { client ->
150175
client.testOperation(
@@ -154,32 +179,35 @@ class SmokeTestsRunnerGeneratorTest {
154179
)
155180
}
156181
182+
error("Unexpectedly completed smoke test operation without throwing exception")
183+
157184
} catch (exception: Exception) {
158185
val success: Boolean = exception is InvalidMessageError
159186
val status: String = if (success) "ok" else "not ok"
160-
println("${'$'}status Test InvalidMessageErrorTest - error expected from service ")
187+
printer.appendLine("${'$'}status Test InvalidMessageErrorTest - error expected from service ")
161188
if (!success) {
162-
printExceptionStackTrace(exception)
163-
exitCode = 1
189+
printer.appendLine(exception.stackTraceToString().prependIndent("# "))
164190
}
191+
192+
success
165193
}
166194
}
167-
""".trimIndent(),
195+
""".formatForTest(),
168196
)
169197
}
170198

171199
@Test
172200
fun failureTest() {
173201
generatedCode.shouldContainOnlyOnceWithDiff(
174202
"""
175-
private suspend fun failureTest() {
203+
private suspend fun failureTest(): Boolean {
176204
val tags = setOf<String>()
177205
if ((serviceFilter.isNotEmpty() && "Test" !in serviceFilter) || tags.any { it in skipTags }) {
178-
println("ok Test FailureTest - error expected from service # skip")
179-
return
206+
printer.appendLine("ok Test FailureTest - error expected from service # skip")
207+
return true
180208
}
181209
182-
try {
210+
return try {
183211
TestClient {
184212
interceptors.add(SmokeTestsInterceptor())
185213
}.use { client ->
@@ -190,17 +218,20 @@ class SmokeTestsRunnerGeneratorTest {
190218
)
191219
}
192220
221+
error("Unexpectedly completed smoke test operation without throwing exception")
222+
193223
} catch (exception: Exception) {
194224
val success: Boolean = exception is SmokeTestsFailureException
195225
val status: String = if (success) "ok" else "not ok"
196-
println("${'$'}status Test FailureTest - error expected from service ")
226+
printer.appendLine("${'$'}status Test FailureTest - error expected from service ")
197227
if (!success) {
198-
printExceptionStackTrace(exception)
199-
exitCode = 1
228+
printer.appendLine(exception.stackTraceToString().prependIndent("# "))
200229
}
230+
231+
success
201232
}
202233
}
203-
""".trimIndent(),
234+
""".formatForTest(),
204235
)
205236
}
206237

‎runtime/runtime-core/api/runtime-core.api‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2093,6 +2093,7 @@ public final class aws/smithy/kotlin/runtime/smoketests/SmokeTestsFunctionsJVMKt
20932093
}
20942094

20952095
public final class aws/smithy/kotlin/runtime/smoketests/SmokeTestsFunctionsKt {
2096+
public static final fun getDefaultPrinter ()Ljava/lang/Appendable;
20962097
public static final fun printExceptionStackTrace (Ljava/lang/Exception;)V
20972098
}
20982099

0 commit comments

Comments
 (0)