Skip to content

Commit 44f036a

Browse files
xinsong-cuixinsong-cui
committed
add protocol benchmarks
1 parent 310661b commit 44f036a

File tree

7 files changed

+493
-5
lines changed

7 files changed

+493
-5
lines changed

‎tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/BenchmarkHarness.kt‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,16 @@ private val benchmarks = setOf(
2828
it as ServiceBenchmark<SdkClient>
2929
}
3030

31-
suspend fun main() {
32-
val harness = BenchmarkHarness()
33-
harness.execute()
31+
suspend fun main(args: Array<String>) {
32+
val useProtocolBenchmark = "protocol" in args
33+
34+
if (useProtocolBenchmark) {
35+
val harness = ProtocolBenchmarkHarness()
36+
harness.execute()
37+
} else {
38+
val harness = BenchmarkHarness()
39+
harness.execute()
40+
}
3441
}
3542

3643
class BenchmarkHarness {
@@ -113,7 +120,7 @@ private inline fun forAtLeast(runMode: RunMode, block: () -> Unit) {
113120
}
114121
}
115122

116-
private val RunMode.explanation get() = when (this) {
123+
public val RunMode.explanation get() = when (this) {
117124
is RunMode.Iterations -> "$iterations iterations"
118125
is RunMode.Time -> time.toString()
119126
}

‎tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/Common.kt‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import aws.sdk.kotlin.benchmarks.service.telemetry.MetricAggregator
99
import aws.smithy.kotlin.runtime.ExperimentalApi
1010
import aws.smithy.kotlin.runtime.retries.StandardRetryStrategy
1111
import aws.smithy.kotlin.runtime.util.Uuid
12+
import kotlin.random.Random
1213

1314
object Common {
1415
val metricAggregator = MetricAggregator()
@@ -21,4 +22,16 @@ object Common {
2122
val telemetryProvider = BenchmarkTelemetryProvider(metricAggregator)
2223

2324
fun random(prefix: String = "") = "$prefix${Uuid.random()}"
25+
26+
fun randomStringPayload(scale: Int): String {
27+
return (1..scale)
28+
.map { Random.nextInt(32, 127).toChar() }
29+
.joinToString("")
30+
}
31+
32+
fun randomBytesPayload(scale: Int): ByteArray {
33+
val bytes = ByteArray(scale)
34+
Random.nextBytes(bytes)
35+
return bytes
36+
}
2437
}

‎tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/ProtocolBenchmarkHarness.kt‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
package aws.sdk.kotlin.benchmarks.service
2+
3+
import aws.sdk.kotlin.benchmarks.service.definitions.*
4+
import aws.sdk.kotlin.benchmarks.service.telemetry.MetricSummary
5+
import aws.smithy.kotlin.runtime.client.SdkClient
6+
import aws.smithy.kotlin.runtime.io.use
7+
import kotlin.time.TimeSource
8+
9+
const val DEFAULT_ITERATIONS = 5
10+
const val DEFAULT_WARMUP_ITERATIONS = 1
11+
12+
private val protocolBenchmarks = setOf(
13+
CloudwatchProtocolBenchmark(),
14+
SecretsManagerProtocolBenchmark(),
15+
).map {
16+
@Suppress("UNCHECKED_CAST")
17+
it as ServiceProtocolBenchmark<SdkClient>
18+
}
19+
20+
class ProtocolBenchmarkHarness {
21+
private val summaries = mutableMapOf<String, MutableMap<String, Map<String, MetricSummary>>>()
22+
23+
suspend fun execute() {
24+
protocolBenchmarks.forEach { execute(it) }
25+
println()
26+
printResults()
27+
}
28+
29+
private suspend fun execute(benchmark: ServiceProtocolBenchmark<SdkClient>) {
30+
benchmark.client().use { client ->
31+
println("${client.config.clientName}:")
32+
33+
println(" Setting up...")
34+
benchmark.setup(client)
35+
36+
try {
37+
benchmark.operations.forEach { execute(it, client, benchmark.scales) }
38+
} finally {
39+
benchmark.tearDown(client)
40+
}
41+
}
42+
println()
43+
}
44+
45+
private suspend fun execute(operation: OperationProtocolBenchmark<SdkClient>, client: SdkClient, scales: List<Int>) {
46+
println(" ${operation.name}:")
47+
48+
println(" Setting up...")
49+
operation.setup(client)
50+
51+
try {
52+
if(operation.requireScaling){
53+
for(scale in scales){
54+
println(" Warming up for ${operation.warmupMode.explanation} (scale: ${scale})...")
55+
forAtLeast(operation.warmupMode) { iteration ->
56+
operation.transact(client, scale, iteration)
57+
}
58+
}
59+
}
60+
else{
61+
println(" Warming up for ${operation.warmupMode.explanation}...")
62+
forAtLeast(operation.warmupMode) { iteration ->
63+
operation.transact(client, 0, iteration)
64+
}
65+
}
66+
67+
Common.metricAggregator.clear()
68+
69+
if(operation.requireScaling) {
70+
for (scale in scales) {
71+
println(" Measuring for ${operation.iterationMode.explanation} (scale: ${scale})...")
72+
forAtLeast(operation.iterationMode) { iteration ->
73+
operation.transact(client, scale, iteration)
74+
}
75+
}
76+
}
77+
else{
78+
println(" Measuring for ${operation.iterationMode.explanation}...")
79+
forAtLeast(operation.iterationMode) { iteration ->
80+
operation.transact(client, 0, iteration)
81+
}
82+
}
83+
val summary = Common.metricAggregator.summarizeAndClear()
84+
summaries.getOrPut(client.config.clientName, ::mutableMapOf)[operation.name] = summary
85+
} finally {
86+
println(" Tearing down...")
87+
operation.tearDown(client)
88+
}
89+
}
90+
91+
private fun printResults() {
92+
val table = ResultsTable.from(summaries)
93+
println(table)
94+
}
95+
}
96+
97+
private inline fun forAtLeast(runMode: RunMode, block: (iteration: Int) -> Unit) {
98+
val start = TimeSource.Monotonic.markNow()
99+
100+
when (runMode) {
101+
is RunMode.Time -> {
102+
var cnt = 0
103+
while (start.elapsedNow() < runMode.time) {
104+
block(cnt)
105+
cnt++
106+
}
107+
println(" (completed $cnt iterations)")
108+
}
109+
110+
is RunMode.Iterations -> {
111+
repeat(runMode.iterations) { cnt ->
112+
block(cnt + 1)
113+
}
114+
println(" (took ${start.elapsedNow()})")
115+
}
116+
}
117+
}

‎tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/definitions/CloudwatchProtocolBenchmark.kt‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
package aws.sdk.kotlin.benchmarks.service.definitions
2+
3+
import aws.sdk.kotlin.benchmarks.service.Common
4+
import aws.sdk.kotlin.services.cloudwatch.CloudWatchClient
5+
import aws.sdk.kotlin.services.cloudwatch.model.*
6+
import aws.smithy.kotlin.runtime.ExperimentalApi
7+
import aws.smithy.kotlin.runtime.time.Instant
8+
import aws.smithy.kotlin.runtime.time.fromEpochMilliseconds
9+
import java.util.*
10+
import kotlin.time.Duration.Companion.seconds
11+
import kotlin.random.Random
12+
13+
class CloudwatchProtocolBenchmark : ServiceProtocolBenchmark<CloudWatchClient> {
14+
companion object {
15+
val suiteId = UUID.randomUUID()
16+
val baseTime = System.currentTimeMillis() - 2 * 60 * 60 * 1000
17+
private inline fun Int.padded(): String = String.format("%03d", this)
18+
}
19+
20+
@OptIn(ExperimentalApi::class)
21+
override suspend fun client() = CloudWatchClient.fromEnvironment {
22+
retryStrategy = Common.noRetries
23+
telemetryProvider = Common.telemetryProvider
24+
httpClient {
25+
telemetryProvider = Common.telemetryProvider
26+
connectTimeout = 30.seconds
27+
}
28+
}
29+
30+
override val scales = listOf(16, 64, 256, 1000)
31+
32+
override val operations get() = listOf(
33+
putMetricDataBenchmark,
34+
getMetricDataBenchmark,
35+
listMetricsBenchmark
36+
)
37+
38+
private val putMetricDataBenchmark = object : AbstractOperationProtocolBenchmark<CloudWatchClient>("Put metric data") {
39+
override val requireScaling = true
40+
41+
override suspend fun transact(client: CloudWatchClient, scale: Int, iteration: Int) {
42+
val putMetricDataRequest = PutMetricDataRequest {
43+
namespace = "TestNameSpace"
44+
metricData = (0 until scale).map { metricDatumIndex ->
45+
MetricDatum {
46+
metricName = "TestMetric"
47+
dimensions = listOf(
48+
Dimension {
49+
name = "TestDimension"
50+
value = "$suiteId-${scale}"
51+
}
52+
)
53+
value = Random.nextDouble()
54+
unit = null
55+
timestamp = Instant.fromEpochMilliseconds(2000 * (metricDatumIndex + 1) + baseTime)
56+
}
57+
}.toMutableList()
58+
}
59+
60+
client.putMetricData(putMetricDataRequest)
61+
62+
if ((iteration % 50) == 0) Thread.sleep(2000)
63+
}
64+
}
65+
66+
private val getMetricDataBenchmark = object : AbstractOperationProtocolBenchmark<CloudWatchClient>("Get metric data") {
67+
override val requireScaling = true
68+
69+
override suspend fun transact(client: CloudWatchClient, scale: Int, iteration: Int) {
70+
val getMetricDataRequest = GetMetricDataRequest {
71+
startTime = Instant.fromEpochMilliseconds(baseTime)
72+
endTime = Instant.fromEpochMilliseconds(baseTime + 3600000)
73+
metricDataQueries = listOf(
74+
MetricDataQuery {
75+
id = "m0"
76+
returnData = true
77+
metricStat {
78+
unit = null
79+
stat = "Sum"
80+
metric = Metric {
81+
namespace = "TestNamespace"
82+
metricName = "TestMetric"
83+
dimensions = listOf(
84+
Dimension {
85+
name = "TestDimension"
86+
value = "$suiteId-${scale}"
87+
}
88+
)
89+
}
90+
period = 60
91+
}
92+
}
93+
)
94+
}
95+
96+
client.getMetricData(getMetricDataRequest)
97+
98+
if ((iteration % 50) == 0) Thread.sleep(2000)
99+
}
100+
}
101+
102+
private val listMetricsBenchmark = object : AbstractOperationProtocolBenchmark<CloudWatchClient>("List metrics") {
103+
override val requireScaling = true
104+
105+
override suspend fun transact(client: CloudWatchClient, scale: Int, iteration: Int) {
106+
client.listMetrics (
107+
ListMetricsRequest {
108+
namespace = "TestNamespace"
109+
}
110+
)
111+
112+
if ((iteration % 50) == 0) Thread.sleep(2000)
113+
}
114+
}
115+
}

0 commit comments

Comments
 (0)