Skip to content

Commit 5a980e4

Browse files
brdeyobrdeyo
committed
Update Memtier workload benchmark metric names to follow preferred lexical naming standard for Compete.
1 parent 23f46d9 commit 5a980e4

File tree

9 files changed

+738
-696
lines changed

9 files changed

+738
-696
lines changed

‎.pipelines/ado-pipeline-linux.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ resources:
1616

1717
variables:
1818
- name: VcVersion
19-
value: 1.14.1
19+
value: 1.14.18
2020
- name: ROOT
2121
value: $(Build.SourcesDirectory)
2222
- name: CDP_DEFINITION_BUILD_COUNT

‎.pipelines/ado-pipeline-windows.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ pr: none
1212

1313
variables:
1414
- name: VcVersion
15-
value: 1.14.1
15+
value: 1.14.18
1616
- name: ROOT
1717
value: $(Build.SourcesDirectory)
1818
- name: CDP_DEFINITION_BUILD_COUNT

‎.pipelines/azure-pipelines-linux.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ resources:
1616
options: --entrypoint=""
1717

1818
variables:
19-
VcVersion : 1.14.17
19+
VcVersion : 1.14.18
2020
ROOT: $(Build.SourcesDirectory)
2121
CDP_DEFINITION_BUILD_COUNT: $[counter('', 0)] # needed for onebranch.pipeline.version task https://aka.ms/obpipelines/versioning
2222
ENABLE_PRS_DELAYSIGN: 1

‎.pipelines/azure-pipelines.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ pool:
1818
vmImage: windows-latest
1919

2020
variables:
21-
VcVersion : 1.14.17
21+
VcVersion : 1.14.18
2222
ROOT: $(Build.SourcesDirectory)
2323
CDP_DEFINITION_BUILD_COUNT: $[counter('', 0)] # needed for onebranch.pipeline.version task https://aka.ms/obpipelines/versioning
2424
ENABLE_PRS_DELAYSIGN: 1

‎src/VirtualClient/VirtualClient.Actions.UnitTests/Memtier/MemtierBenchmarkClientExecutorTests.cs

Lines changed: 172 additions & 185 deletions
Large diffs are not rendered by default.

‎src/VirtualClient/VirtualClient.Actions.UnitTests/Memtier/MemtierMetricsParserTests.cs

Lines changed: 179 additions & 195 deletions
Large diffs are not rendered by default.

‎src/VirtualClient/VirtualClient.Actions/Memtier/MemtierMetricsParser.cs

Lines changed: 28 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ namespace VirtualClient.Actions
1010
using MathNet.Numerics.Statistics;
1111
using VirtualClient;
1212
using VirtualClient.Contracts;
13-
using VirtualClient.Contracts.Parser;
1413

1514
/// <summary>
1615
/// Parser for Redis Memtier benchmark output.
@@ -68,34 +67,29 @@ public static IList<Metric> Aggregate(IEnumerable<Metric> metrics)
6867
// Create the set of metric aggregates.
6968
//
7069
// e.g.
71-
// GET_Bandwidth -> GET_Bandwidth-Avg
72-
// -> GET_Bandwidth-Min
73-
// -> GET_Bandwidth-Max
74-
// -> GET_Bandwidth-Stddev
75-
// -> GET_Bandwidth-P80
76-
// -> GET_Bandwidth-Total
70+
// GET_Bandwidth -> GET-Bandwidth AVG
71+
// -> GET-Bandwidth MIN
72+
// -> GET-Bandwidth MAX
73+
// -> GET-Bandwidth STDDEV
74+
// -> GET-Bandwidth P80
75+
// -> GET-Bandwidth TOTAL
7776
List<Metric> metricAggregates = new List<Metric>();
7877
foreach (MetricAggregate aggregate in aggregations.Values)
7978
{
80-
metricAggregates.Add(new Metric($"{aggregate.Name}-Avg", aggregate.Average(), aggregate.Unit, aggregate.Relativity, description: aggregate.Description));
81-
metricAggregates.Add(new Metric($"{aggregate.Name}-Min", aggregate.Min(), aggregate.Unit, aggregate.Relativity, description: aggregate.Description));
82-
metricAggregates.Add(new Metric($"{aggregate.Name}-Max", aggregate.Max(), aggregate.Unit, aggregate.Relativity, description: aggregate.Description));
83-
metricAggregates.Add(new Metric($"{aggregate.Name}-Stddev", aggregate.StandardDeviation(), aggregate.Unit, aggregate.Relativity, description: aggregate.Description));
79+
metricAggregates.Add(new Metric($"{aggregate.Name} Avg", aggregate.Average(), aggregate.Unit, aggregate.Relativity, description: aggregate.Description));
80+
metricAggregates.Add(new Metric($"{aggregate.Name} Min", aggregate.Min(), aggregate.Unit, aggregate.Relativity, description: aggregate.Description));
81+
metricAggregates.Add(new Metric($"{aggregate.Name} Max", aggregate.Max(), aggregate.Unit, aggregate.Relativity, description: aggregate.Description));
82+
metricAggregates.Add(new Metric($"{aggregate.Name} Stddev", aggregate.StandardDeviation(), aggregate.Unit, aggregate.Relativity, description: aggregate.Description));
8483

8584
if (MemtierMetricsParser.BandwidthExpression.IsMatch(aggregate.Name))
8685
{
8786
// e.g.
88-
// GET_Bandwidth-P80
89-
// GET_Bandwidth-Total
90-
// GET_Throughput-P80
91-
// GET_Throughput-Total
92-
metricAggregates.Add(new Metric($"{aggregate.Name}-P50", aggregate.Percentile(50), aggregate.Unit, aggregate.Relativity, description: aggregate.Description));
93-
metricAggregates.Add(new Metric($"{aggregate.Name}-P80", aggregate.Percentile(80), aggregate.Unit, aggregate.Relativity, description: aggregate.Description));
94-
metricAggregates.Add(new Metric($"{aggregate.Name}-P90", aggregate.Percentile(90), aggregate.Unit, aggregate.Relativity, description: aggregate.Description));
95-
metricAggregates.Add(new Metric($"{aggregate.Name}-P95", aggregate.Percentile(95), aggregate.Unit, aggregate.Relativity, description: aggregate.Description));
96-
metricAggregates.Add(new Metric($"{aggregate.Name}-P99", aggregate.Percentile(99), aggregate.Unit, aggregate.Relativity, description: aggregate.Description));
97-
metricAggregates.Add(new Metric($"{aggregate.Name}-P99.9", ArrayStatistics.QuantileInplace(aggregate.ToArray(), 0.999d), aggregate.Unit, aggregate.Relativity, description: aggregate.Description));
98-
metricAggregates.Add(new Metric($"{aggregate.Name}-Total", aggregate.Sum(), aggregate.Unit, aggregate.Relativity, description: aggregate.Description));
87+
// GET-Bandwidth P80
88+
// GET-Bandwidth TOTAL
89+
metricAggregates.Add(new Metric($"{aggregate.Name} P20", aggregate.Percentile(20), aggregate.Unit, aggregate.Relativity, description: aggregate.Description));
90+
metricAggregates.Add(new Metric($"{aggregate.Name} P50", aggregate.Percentile(50), aggregate.Unit, aggregate.Relativity, description: aggregate.Description));
91+
metricAggregates.Add(new Metric($"{aggregate.Name} P80", aggregate.Percentile(80), aggregate.Unit, aggregate.Relativity, description: aggregate.Description));
92+
metricAggregates.Add(new Metric($"{aggregate.Name} Total", aggregate.Sum(), aggregate.Unit, aggregate.Relativity, description: aggregate.Description));
9993
}
10094
}
10195

@@ -324,77 +318,77 @@ private static IDictionary<string, MetricAggregate> CreateMetricsBin(string metr
324318
{
325319
"Ops/sec",
326320
new MetricAggregate(
327-
metricNamePrefix == null ? "Throughput" : $"{metricNamePrefix}_Throughput",
321+
metricNamePrefix == null ? "Throughput" : $"{metricNamePrefix}-Throughput",
328322
metricUnit: MetricUnit.RequestsPerSec,
329323
relativity: MetricRelativity.HigherIsBetter,
330324
description: "Total number of requests/operations per second during the period of time.")
331325
},
332326
{
333327
"Hits/sec",
334328
new MetricAggregate(
335-
metricNamePrefix == null ? "Hits/sec" : $"{metricNamePrefix}_Hits/sec",
329+
metricNamePrefix == null ? "Hits/sec" : $"{metricNamePrefix}-Hits/sec",
336330
relativity: MetricRelativity.HigherIsBetter,
337331
description: "Total number of cache hits per second during the period of time.")
338332
},
339333
{
340334
"Misses/sec",
341335
new MetricAggregate(
342-
metricNamePrefix == null ? "Misses/sec" : $"{metricNamePrefix}_Misses/sec",
336+
metricNamePrefix == null ? "Misses/sec" : $"{metricNamePrefix}-Misses/sec",
343337
relativity: MetricRelativity.LowerIsBetter,
344338
description: "Total number of cache misses per second during a period of time. This is an indication of data evictions due to reaching memory limits.")
345339
},
346340
{
347341
"AvgLatency",
348342
new MetricAggregate(
349-
metricNamePrefix == null ? "Latency-Avg" : $"{metricNamePrefix}_Latency-Avg",
343+
metricNamePrefix == null ? "Latency-Avg" : $"{metricNamePrefix}-Latency-Avg",
350344
metricUnit: MetricUnit.Milliseconds,
351345
relativity: MetricRelativity.LowerIsBetter,
352346
description: "Average latency for requests/operations during the period of time.")
353347
},
354348
{
355349
"p50Latency",
356350
new MetricAggregate(
357-
metricNamePrefix == null ? "Latency-P50" : $"{metricNamePrefix}_Latency-P50",
351+
metricNamePrefix == null ? "Latency-P50" : $"{metricNamePrefix}-Latency-P50",
358352
metricUnit: MetricUnit.Milliseconds,
359353
relativity: MetricRelativity.LowerIsBetter,
360354
description: "The latency for 50% of all requests was at or under this value.")
361355
},
362356
{
363357
"p90Latency",
364358
new MetricAggregate(
365-
metricNamePrefix == null ? "Latency-P90" : $"{metricNamePrefix}_Latency-P90",
359+
metricNamePrefix == null ? "Latency-P90" : $"{metricNamePrefix}-Latency-P90",
366360
metricUnit: MetricUnit.Milliseconds,
367361
relativity: MetricRelativity.LowerIsBetter,
368362
description: "The latency for 90% of all requests was at or under this value.")
369363
},
370364
{
371365
"p95Latency",
372366
new MetricAggregate(
373-
metricNamePrefix == null ? "Latency-P95" : $"{metricNamePrefix}_Latency-P95",
367+
metricNamePrefix == null ? "Latency-P95" : $"{metricNamePrefix}-Latency-P95",
374368
metricUnit: MetricUnit.Milliseconds,
375369
relativity: MetricRelativity.LowerIsBetter,
376370
description: "The latency for 95% of all requests was at or under this value.")
377371
},
378372
{
379373
"p99Latency",
380374
new MetricAggregate(
381-
metricNamePrefix == null ? "Latency-P99" : $"{metricNamePrefix}_Latency-P99",
375+
metricNamePrefix == null ? "Latency-P99" : $"{metricNamePrefix}-Latency-P99",
382376
metricUnit: MetricUnit.Milliseconds,
383377
relativity: MetricRelativity.LowerIsBetter,
384378
description: "The latency for 99% of all requests was at or under this value.")
385379
},
386380
{
387381
"p99.9Latency",
388382
new MetricAggregate(
389-
metricNamePrefix == null ? "Latency-P99.9" : $"{metricNamePrefix}_Latency-P99.9",
383+
metricNamePrefix == null ? "Latency-P99.9" : $"{metricNamePrefix}-Latency-P99.9",
390384
metricUnit: MetricUnit.Milliseconds,
391385
relativity: MetricRelativity.LowerIsBetter,
392386
description: "The latency for 99.9% of all requests was at or under this value.")
393387
},
394388
{
395389
"KB/sec",
396390
new MetricAggregate(
397-
metricNamePrefix == null ? "Bandwidth" : $"{metricNamePrefix}_Bandwidth",
391+
metricNamePrefix == null ? "Bandwidth" : $"{metricNamePrefix}-Bandwidth",
398392
metricUnit: MetricUnit.KilobytesPerSecond,
399393
relativity: MetricRelativity.HigherIsBetter,
400394
description: "Total amount of data transferred per second during the period of time.")
@@ -414,7 +408,7 @@ private static void AddAdditionalLatencyPercentileMetrics(string output, List<Me
414408
p80LatencyOnGet = double.Parse(getLatencyP80.Groups[1].Value);
415409

416410
metrics.Add(new Metric(
417-
"GET_Latency-P80",
411+
"GET-Latency-P80",
418412
p80LatencyOnGet,
419413
MetricUnit.Milliseconds,
420414
MetricRelativity.LowerIsBetter,
@@ -427,7 +421,7 @@ private static void AddAdditionalLatencyPercentileMetrics(string output, List<Me
427421
p80LatencyOnSet = double.Parse(setLatencyP80.Groups[1].Value);
428422

429423
metrics.Add(new Metric(
430-
"SET_Latency-P80",
424+
"SET-Latency-P80",
431425
p80LatencyOnSet,
432426
MetricUnit.Milliseconds,
433427
MetricRelativity.LowerIsBetter,

0 commit comments

Comments
 (0)