Improve repeatability & precision of throughput tests on Linux & ARM (#7268)

## Summary of changes

This PR tweaks RPS & k6 configuration variables in order to ensure
better stability of results.
Also it adds x86 suffix for relevant variants.

## Reason for change

Throughput tests had unexpected spikes in Datadog dashboards due to
several factors (warmup not being excluded, incorrect timeseries
queries, too high concurrent VUs traffic). Couple of these issues were
fixed directly in Datadog/benchmarking-platform repository on
dd-trace-dotnet/macro branch, this PR does the last step in order for
fix to apply universally.

Author: ddyurchenko

.gitlab/benchmarks/macrobenchmarks.yml

@@ -67,7 +67,7 @@ update-bp-infra:
     AWS_EPHEMERAL_INFRA_ARTIFACTS_BUCKET_URI: "s3://windows-benchmarking-results/$CI_PROJECT_NAME/$CI_COMMIT_REF_NAME/$CI_JOB_ID"
     AWS_EPHEMERAL_INFRA_REGION: "us-east-1"
 
-.benchmarks:
+.benchmarks-x86:
   stage: benchmarks
   tags: ["runner:apm-k8s-same-cpu"]
   needs: ["check_azure_pipeline"]
@@ -99,35 +99,32 @@ update-bp-infra:
       - platform/artifacts/
     expire_in: 3 months
   variables:
-    K6_OPTIONS_WARMUP_RATE: 14000
+    # k6 configuration, similar to the one in https://grafana.com/docs/k6/latest/using-k6/scenarios/#options
     K6_OPTIONS_WARMUP_DURATION: 3m
     K6_OPTIONS_WARMUP_GRACEFUL_STOP: 10s
-    K6_OPTIONS_WARMUP_PRE_ALLOCATED_VUS: 500
-    K6_OPTIONS_WARMUP_MAX_VUS: 2000
+    K6_OPTIONS_WARMUP_VUS: 2
 
-    K6_OPTIONS_NORMAL_OPERATION_RATE: 13000
-    K6_OPTIONS_NORMAL_OPERATION_DURATION: 7m
+    K6_OPTIONS_NORMAL_OPERATION_RATE: 5000
+    K6_OPTIONS_NORMAL_OPERATION_DURATION: 10m
     K6_OPTIONS_NORMAL_OPERATION_GRACEFUL_STOP: 10s
-    K6_OPTIONS_NORMAL_OPERATION_PRE_ALLOCATED_VUS: 500
-    K6_OPTIONS_NORMAL_OPERATION_MAX_VUS: 2000
+    K6_OPTIONS_NORMAL_OPERATION_PRE_ALLOCATED_VUS: 2
+    K6_OPTIONS_NORMAL_OPERATION_MAX_VUS: 2
 
-    K6_OPTIONS_HIGH_LOAD_RATE: 15000
-    K6_OPTIONS_HIGH_LOAD_DURATION: 3m
-    K6_OPTIONS_HIGH_LOAD_GRACEFUL_STOP: 10s
-    K6_OPTIONS_HIGH_LOAD_PRE_ALLOCATED_VUS: 500
-    K6_OPTIONS_HIGH_LOAD_MAX_VUS: 2000
+    K6_OPTIONS_HIGH_LOAD_DURATION: 5m
+    K6_OPTIONS_HIGH_LOAD_GRACEFUL_STOP: 0s
+    K6_OPTIONS_HIGH_LOAD_VUS: 2
 
-baseline:
-  extends: .benchmarks
+baseline-x86:
+  extends: .benchmarks-x86
   variables:
     NATIVE_PROFILER_PATH: "dd-trace-dotnet/tracer/tracer-home-linux/linux-x64/Datadog.Trace.ClrProfiler.Native.so"
     TRACER_HOME_PATH: "dd-trace-dotnet/tracer/tracer-home-linux"
     COR_ENABLE_PROFILING: 0
     CORECLR_ENABLE_PROFILING: 0
     ENDPOINT: "hello"
-    
-calltarget_ngen:
-  extends: .benchmarks
+
+calltarget_ngen-x86:
+  extends: .benchmarks-x86
   variables:
     NATIVE_PROFILER_PATH: "dd-trace-dotnet/tracer/tracer-home-linux/linux-x64/Datadog.Trace.ClrProfiler.Native.so"
     TRACER_HOME_PATH: "dd-trace-dotnet/tracer/tracer-home-linux"
@@ -137,8 +134,8 @@ calltarget_ngen:
     DD_CLR_ENABLE_NGEN: 1
     ENDPOINT: "hello"
 
-trace_stats:
-  extends: .benchmarks
+trace_stats-x86:
+  extends: .benchmarks-x86
   variables:
     NATIVE_PROFILER_PATH: "dd-trace-dotnet/tracer/tracer-home-linux/linux-x64/Datadog.Trace.ClrProfiler.Native.so"
     TRACER_HOME_PATH: "dd-trace-dotnet/tracer/tracer-home-linux"
@@ -149,8 +146,8 @@ trace_stats:
     DD_TRACE_STATS_COMPUTATION_ENABLED: 1
     ENDPOINT: "hello"
 
-manual_only:
-  extends: .benchmarks
+manual_only-x86:
+  extends: .benchmarks-x86
   variables:
     NATIVE_PROFILER_PATH: "dd-trace-dotnet/tracer/tracer-home-linux/linux-x64/Datadog.Trace.ClrProfiler.Native.so"
     TRACER_HOME_PATH: "dd-trace-dotnet/tracer/tracer-home-linux"
@@ -159,8 +156,8 @@ manual_only:
     DOTNET_BUILD_ARGS: "/p:MANUAL_INSTRUMENTATION=true /p:MANUAL_ONLY_INSTRUMENTATION=true"
     ENDPOINT: "hello"
 
-manual_and_automatic:
-  extends: .benchmarks
+manual_and_automatic-x86:
+  extends: .benchmarks-x86
   variables:
     NATIVE_PROFILER_PATH: "dd-trace-dotnet/tracer/tracer-home-linux/linux-x64/Datadog.Trace.ClrProfiler.Native.so"
     TRACER_HOME_PATH: "dd-trace-dotnet/tracer/tracer-home-linux"
@@ -169,8 +166,8 @@ manual_and_automatic:
     DOTNET_BUILD_ARGS: "/p:MANUAL_INSTRUMENTATION=true"
     ENDPOINT: "hello"
 
-ddtraceenabled_false:
-  extends: .benchmarks
+ddtraceenabled_false-x86:
+  extends: .benchmarks-x86
   variables:
     NATIVE_PROFILER_PATH: "dd-trace-dotnet/tracer/tracer-home-linux/linux-x64/Datadog.Trace.ClrProfiler.Native.so"
     TRACER_HOME_PATH: "dd-trace-dotnet/tracer/tracer-home-linux"
@@ -179,17 +176,17 @@ ddtraceenabled_false:
     DD_TRACE_ENABLED: 0
     ENDPOINT: "hello"
 
-profiler_exceptions_baseline:
-  extends: .benchmarks
+profiler_exceptions_baseline-x86:
+  extends: .benchmarks-x86
   variables:
     NATIVE_PROFILER_PATH: "dd-trace-dotnet/tracer/tracer-home-linux/linux-x64/Datadog.Trace.ClrProfiler.Native.so"
     TRACER_HOME_PATH: "dd-trace-dotnet/tracer/tracer-home-linux"
     COR_ENABLE_PROFILING: 0
     CORECLR_ENABLE_PROFILING: 0
     ENDPOINT: "hello/Exception"
 
-profiler:
-  extends: .benchmarks
+profiler-x86:
+  extends: .benchmarks-x86
   variables:
     NATIVE_PROFILER_PATH: "dd-trace-dotnet/tracer/tracer-home-linux/linux-x64/Datadog.Trace.ClrProfiler.Native.so"
     TRACER_HOME_PATH: "dd-trace-dotnet/tracer/tracer-home-linux"
@@ -199,8 +196,8 @@ profiler:
     COMPlus_EnableDiagnostics: 1
     ENDPOINT: "hello"
 
-profiler_walltime:
-  extends: .benchmarks
+profiler_walltime-x86:
+  extends: .benchmarks-x86
   variables:
     NATIVE_PROFILER_PATH: "dd-trace-dotnet/tracer/tracer-home-linux/linux-x64/Datadog.Trace.ClrProfiler.Native.so"
     TRACER_HOME_PATH: "dd-trace-dotnet/tracer/tracer-home-linux"
@@ -212,8 +209,8 @@ profiler_walltime:
     DD_PROFILING_CPU_ENABLED: 0
     ENDPOINT: "hello"
 
-profiler_exceptions:
-  extends: .benchmarks
+profiler_exceptions-x86:
+  extends: .benchmarks-x86
   variables:
     NATIVE_PROFILER_PATH: "dd-trace-dotnet/tracer/tracer-home-linux/linux-x64/Datadog.Trace.ClrProfiler.Native.so"
     TRACER_HOME_PATH: "dd-trace-dotnet/tracer/tracer-home-linux"
@@ -224,8 +221,8 @@ profiler_exceptions:
     COMPlus_EnableDiagnostics: 1
     ENDPOINT: "hello/Exception"
 
-profiler_cpu:
-  extends: .benchmarks
+profiler_cpu-x86:
+  extends: .benchmarks-x86
   variables:
     NATIVE_PROFILER_PATH: "dd-trace-dotnet/tracer/tracer-home-linux/linux-x64/Datadog.Trace.ClrProfiler.Native.so"
     TRACER_HOME_PATH: "dd-trace-dotnet/tracer/tracer-home-linux"
@@ -237,8 +234,8 @@ profiler_cpu:
     COMPlus_EnableDiagnostics: 1
     ENDPOINT: "hello"
 
-profiler_cpu_timer_create:
-  extends: .benchmarks
+profiler_cpu_timer_create-x86:
+  extends: .benchmarks-x86
   variables:
     NATIVE_PROFILER_PATH: "dd-trace-dotnet/tracer/tracer-home-linux/linux-x64/Datadog.Trace.ClrProfiler.Native.so"
     TRACER_HOME_PATH: "dd-trace-dotnet/tracer/tracer-home-linux"
@@ -274,31 +271,27 @@ profiler_cpu_timer_create:
     - git clone --branch dd-trace-dotnet/macro https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform platform && cd platform
     - ./steps/setup-sut.sh
     - bp-runner bp-runner.arm.yml --debug
-  after_script: !reference [.benchmarks, after_script]
   artifacts:
     name: "artifacts"
     when: always
     paths:
       - platform/artifacts/
     expire_in: 3 months
   variables:
-    K6_OPTIONS_WARMUP_RATE: 14000
+    # k6 configuration, similar to the one in https://grafana.com/docs/k6/latest/using-k6/scenarios/#options
     K6_OPTIONS_WARMUP_DURATION: 3m
     K6_OPTIONS_WARMUP_GRACEFUL_STOP: 10s
-    K6_OPTIONS_WARMUP_PRE_ALLOCATED_VUS: 500
-    K6_OPTIONS_WARMUP_MAX_VUS: 2000
+    K6_OPTIONS_WARMUP_VUS: 2
 
-    K6_OPTIONS_NORMAL_OPERATION_RATE: 13000
-    K6_OPTIONS_NORMAL_OPERATION_DURATION: 7m
+    K6_OPTIONS_NORMAL_OPERATION_RATE: 4000
+    K6_OPTIONS_NORMAL_OPERATION_DURATION: 10m
     K6_OPTIONS_NORMAL_OPERATION_GRACEFUL_STOP: 10s
-    K6_OPTIONS_NORMAL_OPERATION_PRE_ALLOCATED_VUS: 500
-    K6_OPTIONS_NORMAL_OPERATION_MAX_VUS: 2000
+    K6_OPTIONS_NORMAL_OPERATION_PRE_ALLOCATED_VUS: 2
+    K6_OPTIONS_NORMAL_OPERATION_MAX_VUS: 2
 
-    K6_OPTIONS_HIGH_LOAD_RATE: 15000
-    K6_OPTIONS_HIGH_LOAD_DURATION: 3m
-    K6_OPTIONS_HIGH_LOAD_GRACEFUL_STOP: 10s
-    K6_OPTIONS_HIGH_LOAD_PRE_ALLOCATED_VUS: 500
-    K6_OPTIONS_HIGH_LOAD_MAX_VUS: 2000
+    K6_OPTIONS_HIGH_LOAD_DURATION: 5m
+    K6_OPTIONS_HIGH_LOAD_GRACEFUL_STOP: 0s
+    K6_OPTIONS_HIGH_LOAD_VUS: 2
 
 baseline-arm64:
   extends: .benchmarks-arm64
@@ -473,23 +466,20 @@ profiler_cpu_timer_create-arm64:
     # Whether to cleanup ephemeral instances after benchmarks are run
     CLEANUP: "true"
 
-    K6_OPTIONS_WARMUP_RATE: 14000
+    # k6 configuration, similar to the one in https://grafana.com/docs/k6/latest/using-k6/scenarios/#options
     K6_OPTIONS_WARMUP_DURATION: 3m
     K6_OPTIONS_WARMUP_GRACEFUL_STOP: 10s
-    K6_OPTIONS_WARMUP_PRE_ALLOCATED_VUS: 500
-    K6_OPTIONS_WARMUP_MAX_VUS: 2000
+    K6_OPTIONS_WARMUP_VUS: 2
 
     K6_OPTIONS_NORMAL_OPERATION_RATE: 13000
-    K6_OPTIONS_NORMAL_OPERATION_DURATION: 7m
+    K6_OPTIONS_NORMAL_OPERATION_DURATION: 10m
     K6_OPTIONS_NORMAL_OPERATION_GRACEFUL_STOP: 10s
     K6_OPTIONS_NORMAL_OPERATION_PRE_ALLOCATED_VUS: 500
     K6_OPTIONS_NORMAL_OPERATION_MAX_VUS: 2000
 
-    K6_OPTIONS_HIGH_LOAD_RATE: 15000
-    K6_OPTIONS_HIGH_LOAD_DURATION: 3m
-    K6_OPTIONS_HIGH_LOAD_GRACEFUL_STOP: 10s
-    K6_OPTIONS_HIGH_LOAD_PRE_ALLOCATED_VUS: 500
-    K6_OPTIONS_HIGH_LOAD_MAX_VUS: 2000
+    K6_OPTIONS_HIGH_LOAD_DURATION: 5m
+    K6_OPTIONS_HIGH_LOAD_GRACEFUL_STOP: 0s
+    K6_OPTIONS_HIGH_LOAD_VUS: 2
   script:
     - source build-id.txt
     - echo "Building for the following build https://dev.azure.com/datadoghq/dd-trace-dotnet/_build/results?buildId=$buildId&view=results"
@@ -609,3 +599,4 @@ profiler_cpu_timer_create-win:
     DD_PROFILING_CPU_ENABLED: 1
     COMPlus_EnableDiagnostics: 1
     ENDPOINT: "hello"
+