Skip to content

Commit 0dbcfe3

Browse files
xinsong-cuixinsong-cui
committed
address pr feedback
1 parent f4cdadd commit 0dbcfe3

File tree

5 files changed

+92
-94
lines changed

5 files changed

+92
-94
lines changed

‎tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/BenchmarkHarness.kt‎

Lines changed: 4 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ import aws.sdk.kotlin.benchmarks.service.telemetry.MetricSummary
99
import aws.smithy.kotlin.runtime.client.SdkClient
1010
import aws.smithy.kotlin.runtime.io.use
1111
import kotlin.time.Duration.Companion.seconds
12-
import kotlin.time.TimeSource
1312

1413
val DEFAULT_WARMUP_TIME = 5.seconds
1514
val DEFAULT_ITERATION_TIME = 15.seconds
@@ -29,6 +28,8 @@ private val benchmarks = setOf(
2928
}
3029

3130
suspend fun main(args: Array<String>) {
31+
// FIXME: Unify service and protocol benchmark interfaces into a common base module
32+
// to support generic parameterized benchmarks and reduce duplication
3233
val useProtocolBenchmark = "protocol" in args
3334

3435
if (useProtocolBenchmark) {
@@ -73,14 +74,14 @@ class BenchmarkHarness {
7374

7475
try {
7576
println(" Warming up for ${operation.warmupMode.explanation}...")
76-
forAtLeast(operation.warmupMode) {
77+
operation.warmupMode.forAtLeast {
7778
operation.transact(client)
7879
}
7980

8081
Common.metricAggregator.clear()
8182

8283
println(" Measuring for ${operation.iterationMode.explanation}...")
83-
forAtLeast(operation.iterationMode) {
84+
operation.iterationMode.forAtLeast {
8485
operation.transact(client)
8586
}
8687

@@ -97,30 +98,3 @@ class BenchmarkHarness {
9798
println(table)
9899
}
99100
}
100-
101-
private inline fun forAtLeast(runMode: RunMode, block: () -> Unit) {
102-
val start = TimeSource.Monotonic.markNow()
103-
104-
when (runMode) {
105-
is RunMode.Time -> {
106-
var cnt = 0
107-
while (start.elapsedNow() < runMode.time) {
108-
block()
109-
cnt++
110-
}
111-
println(" (completed $cnt iterations)")
112-
}
113-
114-
is RunMode.Iterations -> {
115-
repeat(runMode.iterations) {
116-
block()
117-
}
118-
println(" (took ${start.elapsedNow()})")
119-
}
120-
}
121-
}
122-
123-
public val RunMode.explanation get() = when (this) {
124-
is RunMode.Iterations -> "$iterations iterations"
125-
is RunMode.Time -> time.toString()
126-
}

‎tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/ProtocolBenchmarkHarness.kt‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@ import aws.smithy.kotlin.runtime.client.SdkClient
1010
import aws.smithy.kotlin.runtime.io.use
1111
import kotlin.time.TimeSource
1212

13-
const val DEFAULT_ITERATIONS = 5
14-
const val DEFAULT_WARMUP_ITERATIONS = 1
13+
const val DEFAULT_ITERATIONS = 500
14+
const val DEFAULT_WARMUP_ITERATIONS = 100
1515

1616
private val protocolBenchmarks = setOf(
1717
CloudwatchProtocolBenchmark(),
@@ -56,14 +56,14 @@ class ProtocolBenchmarkHarness {
5656
if (operation.requireScaling) {
5757
for (scale in scales) {
5858
println(" Warming up for ${operation.warmupMode.explanation} (scale: $scale)...")
59-
forAtLeast(operation.warmupMode) { iteration ->
60-
operation.transact(client, scale, iteration)
59+
operation.warmupMode.forAtLeast { iteration ->
60+
operation.transact(client, scale, iteration!!)
6161
}
6262
}
6363
} else {
6464
println(" Warming up for ${operation.warmupMode.explanation}...")
65-
forAtLeast(operation.warmupMode) { iteration ->
66-
operation.transact(client, 0, iteration)
65+
operation.warmupMode.forAtLeast { iteration ->
66+
operation.transact(client, 0, iteration!!)
6767
}
6868
}
6969

@@ -72,14 +72,14 @@ class ProtocolBenchmarkHarness {
7272
if (operation.requireScaling) {
7373
for (scale in scales) {
7474
println(" Measuring for ${operation.iterationMode.explanation} (scale: $scale)...")
75-
forAtLeast(operation.iterationMode) { iteration ->
76-
operation.transact(client, scale, iteration)
75+
operation.iterationMode.forAtLeast { iteration ->
76+
operation.transact(client, scale, iteration!!)
7777
}
7878
}
7979
} else {
8080
println(" Measuring for ${operation.iterationMode.explanation}...")
81-
forAtLeast(operation.iterationMode) { iteration ->
82-
operation.transact(client, 0, iteration)
81+
operation.iterationMode.forAtLeast { iteration ->
82+
operation.transact(client, 0, iteration!!)
8383
}
8484
}
8585
val summary = Common.metricAggregator.summarizeAndClear()

‎tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/definitions/CloudwatchProtocolBenchmark.kt‎

Lines changed: 47 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,15 @@ import aws.sdk.kotlin.services.cloudwatch.CloudWatchClient
99
import aws.sdk.kotlin.services.cloudwatch.model.*
1010
import aws.smithy.kotlin.runtime.ExperimentalApi
1111
import aws.smithy.kotlin.runtime.time.Instant
12-
import aws.smithy.kotlin.runtime.time.fromEpochMilliseconds
1312
import java.util.*
1413
import kotlin.random.Random
14+
import kotlin.time.Duration.Companion.hours
1515
import kotlin.time.Duration.Companion.seconds
1616

1717
class CloudwatchProtocolBenchmark : ServiceProtocolBenchmark<CloudWatchClient> {
1818
companion object {
1919
val suiteId = UUID.randomUUID()
20-
val baseTime = System.currentTimeMillis() - 2 * 60 * 60 * 1000
21-
private inline fun Int.padded(): String = String.format("%03d", this)
20+
val baseTime = Instant.now() - 2.hours
2221
}
2322

2423
@OptIn(ExperimentalApi::class)
@@ -43,63 +42,59 @@ class CloudwatchProtocolBenchmark : ServiceProtocolBenchmark<CloudWatchClient> {
4342
override val requireScaling = true
4443

4544
override suspend fun transact(client: CloudWatchClient, scale: Int, iteration: Int) {
46-
val putMetricDataRequest = PutMetricDataRequest {
47-
namespace = "TestNameSpace"
48-
metricData = (0 until scale).map { metricDatumIndex ->
49-
MetricDatum {
50-
metricName = "TestMetric"
51-
dimensions = listOf(
52-
Dimension {
53-
name = "TestDimension"
54-
value = "$suiteId-$scale"
55-
},
56-
)
57-
value = Random.nextDouble()
58-
unit = null
59-
timestamp = Instant.fromEpochMilliseconds(2000 * (metricDatumIndex + 1) + baseTime)
60-
}
61-
}.toMutableList()
62-
}
63-
64-
client.putMetricData(putMetricDataRequest)
65-
66-
if ((iteration % 50) == 0) Thread.sleep(2000)
45+
client.putMetricData (
46+
PutMetricDataRequest {
47+
namespace = "SDK Benchmark Test Data"
48+
metricData = (0 until scale).map { metricDatumIndex ->
49+
MetricDatum {
50+
metricName = "TestMetric"
51+
dimensions = listOf(
52+
Dimension {
53+
name = "TestDimension"
54+
value = "$suiteId-$scale"
55+
},
56+
)
57+
value = Random.nextDouble()
58+
unit = null
59+
timestamp = baseTime + ((metricDatumIndex + 1) * 2).seconds
60+
}
61+
}.toList()
62+
}
63+
)
6764
}
6865
}
6966

7067
private val getMetricDataBenchmark = object : AbstractOperationProtocolBenchmark<CloudWatchClient>("Get metric data") {
7168
override val requireScaling = true
7269

7370
override suspend fun transact(client: CloudWatchClient, scale: Int, iteration: Int) {
74-
val getMetricDataRequest = GetMetricDataRequest {
75-
startTime = Instant.fromEpochMilliseconds(baseTime)
76-
endTime = Instant.fromEpochMilliseconds(baseTime + 3600000)
77-
metricDataQueries = listOf(
78-
MetricDataQuery {
79-
id = "m0"
80-
returnData = true
81-
metricStat {
82-
unit = null
83-
stat = "Sum"
84-
metric = Metric {
85-
namespace = "TestNamespace"
86-
metricName = "TestMetric"
87-
dimensions = listOf(
88-
Dimension {
89-
name = "TestDimension"
90-
value = "$suiteId-$scale"
91-
},
92-
)
71+
client.getMetricData(
72+
GetMetricDataRequest {
73+
startTime = baseTime
74+
endTime = baseTime + 2.hours
75+
metricDataQueries = listOf(
76+
MetricDataQuery {
77+
id = "m0"
78+
returnData = true
79+
metricStat {
80+
unit = null
81+
stat = "Sum"
82+
metric = Metric {
83+
namespace = "TestNamespace"
84+
metricName = "TestMetric"
85+
dimensions = listOf(
86+
Dimension {
87+
name = "TestDimension"
88+
value = "$suiteId-$scale"
89+
},
90+
)
91+
}
92+
period = 60
9393
}
94-
period = 60
95-
}
96-
},
97-
)
98-
}
99-
100-
client.getMetricData(getMetricDataRequest)
101-
102-
if ((iteration % 50) == 0) Thread.sleep(2000)
94+
},
95+
)
96+
}
97+
)
10398
}
10499
}
105100

@@ -112,8 +107,6 @@ class CloudwatchProtocolBenchmark : ServiceProtocolBenchmark<CloudWatchClient> {
112107
namespace = "TestNamespace"
113108
},
114109
)
115-
116-
if ((iteration % 50) == 0) Thread.sleep(2000)
117110
}
118111
}
119112
}

‎tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/definitions/SecretsManagerProtocolBenchmark.kt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@ class SecretsManagerProtocolBenchmark : ServiceProtocolBenchmark<SecretsManagerC
2828
}
2929

3030
override suspend fun setup(client: SecretsManagerClient) {
31+
// FIXME: Add metadata parameters to setup/teardown methods to pass configuration info
32+
// (e.g., configured iteration count). Larger refactoring needed outside this PR scope.
33+
3134
for (iteration in 1..DEFAULT_ITERATIONS) {
3235
val tags = listOf(
3336
Tag {

‎tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/definitions/ServiceBenchmark.kt‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import aws.sdk.kotlin.benchmarks.service.DEFAULT_ITERATION_TIME
88
import aws.sdk.kotlin.benchmarks.service.DEFAULT_WARMUP_TIME
99
import aws.smithy.kotlin.runtime.client.SdkClient
1010
import kotlin.time.Duration
11+
import kotlin.time.TimeSource
1112

1213
/**
1314
* Defines the harness for conducting a benchmark of a service client.
@@ -108,6 +109,33 @@ sealed interface RunMode {
108109
data class Time(val time: Duration) : RunMode
109110
}
110111

112+
val RunMode.explanation get() = when (this) {
113+
is RunMode.Iterations -> "$iterations iterations"
114+
is RunMode.Time -> time.toString()
115+
}
116+
117+
internal inline fun RunMode.forAtLeast(block: (Int?) -> Unit) {
118+
val start = TimeSource.Monotonic.markNow()
119+
120+
when (this) {
121+
is RunMode.Time -> {
122+
var cnt = 0
123+
while (start.elapsedNow() < time) {
124+
block(cnt)
125+
cnt++
126+
}
127+
println(" (completed $cnt iterations)")
128+
}
129+
130+
is RunMode.Iterations -> {
131+
repeat(iterations) { cnt ->
132+
block(cnt + 1)
133+
}
134+
println(" (took ${start.elapsedNow()})")
135+
}
136+
}
137+
}
138+
111139
/**
112140
* An abstract base class for operation benchmarks.
113141
* @param The name of the operation (e.g., `HeadBucket`).

0 commit comments

Comments
 (0)