ci: Configurable TBS sampling rate for benchmarks (elastic#17250)

carsonip · web-flow · commit dae557633898 · 2025-06-16T18:33:08.000+01:00
CI benchmarks: make TBS sampling rate configurable, for both standalone and ECH.

Misc fixes.
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
@@ -20,13 +20,18 @@ on:
         required: false
         type: string
         default: "10GB"
+      tailSamplingSampleRate:
+        description: 'Blanket sample rate used by tail-based sampling'
+        required: false
+        type: string
+        default: "0.1"
       profile:
         description: 'System profile for benchmark, e.g. system-profiles/8GBx1zone.tfvars'
         required: false
         type: string
         # defaults set below (TFVARS_SOURCE)
       runOnStable:
-        description: 'Benchmark on latest stable version instead of SNAPSHOT'
+        description: 'Benchmark on latest stable version instead of a build from commit'
         required: false
         type: boolean
         default: false
@@ -69,13 +74,18 @@ on:
         required: false
         type: string
         default: "10GB"
+      tailSamplingSampleRate:
+        description: 'Blanket sample rate used by tail-based sampling'
+        required: false
+        type: string
+        default: "0.1"
       profile:
         description: 'System profile for benchmark, e.g. system-profiles/8GBx1zone.tfvars'
         required: false
         type: string
         # defaults set below (TFVARS_SOURCE)
       runOnStable:
-        description: 'Benchmark on latest stable version instead of SNAPSHOT'
+        description: 'Benchmark on latest stable version instead of a build from commit'
         required: false
         type: boolean
         default: false
@@ -128,6 +138,7 @@ jobs:
       TF_VAR_run_standalone: ${{ inputs.runStandalone || github.event.schedule=='0 0 1 * *' }}
       TF_VAR_apm_server_tail_sampling: ${{ inputs.enableTailSampling || 'false' }} # set the default again otherwise schedules won't work
       TF_VAR_apm_server_tail_sampling_storage_limit: ${{ inputs.tailSamplingStorageLimit || '10GB' }} # set the default again otherwise schedules won't work
+      TF_VAR_apm_server_tail_sampling_sample_rate: ${{ inputs.tailSamplingSampleRate || '0.1' }} # set the default again otherwise schedules won't work
       RUN_STANDALONE: ${{ inputs.runStandalone || github.event.schedule=='0 0 1 * *' }}
       PGO_EXPORT: ${{ inputs.pgoExport || github.event.schedule=='0 0 1 * *' }}
       TFVARS_SOURCE: ${{ inputs.profile || 'system-profiles/8GBx1zone.tfvars' }} # set the default again otherwise schedules won't work
diff --git a/testing/benchmark/main.tf b/testing/benchmark/main.tf
@@ -102,6 +102,7 @@ module "ec_deployment" {
   apm_server_pprof                       = true
   apm_server_tail_sampling               = var.apm_server_tail_sampling
   apm_server_tail_sampling_storage_limit = var.apm_server_tail_sampling_storage_limit
+  apm_server_tail_sampling_sample_rate   = var.apm_server_tail_sampling_sample_rate
 
   elasticsearch_size              = var.elasticsearch_size
   elasticsearch_zone_count        = var.elasticsearch_zone_count
@@ -163,6 +164,7 @@ module "standalone_apm_server" {
 
   apm_server_tail_sampling               = var.apm_server_tail_sampling
   apm_server_tail_sampling_storage_limit = var.apm_server_tail_sampling_storage_limit
+  apm_server_tail_sampling_sample_rate   = var.apm_server_tail_sampling_sample_rate
 
   aws_provisioner_key_name = var.private_key
 
diff --git a/testing/benchmark/system-profiles/16GBx2zone-aws.tfvars b/testing/benchmark/system-profiles/16GBx2zone-aws.tfvars
@@ -5,7 +5,7 @@ user_name = "USER"
 worker_instance_type = "c6i.2xlarge"
 
 # Elastic Cloud
-ess_region = "us-west-2"
+ess_region          = "us-west-2"
 deployment_template = "aws-cpu-optimized-faster-warm"
 # The number of AZs the APM Server should span.
 apm_server_zone_count = 1
diff --git a/testing/benchmark/system-profiles/16GBx2zone-azure.tfvars b/testing/benchmark/system-profiles/16GBx2zone-azure.tfvars
@@ -5,7 +5,7 @@ user_name = "USER"
 worker_instance_type = "c6i.2xlarge"
 
 # Elastic Cloud
-ess_region = "azure-westus2"
+ess_region          = "azure-westus2"
 deployment_template = "azure-cpu-optimized"
 # The number of AZs the APM Server should span.
 apm_server_zone_count = 1
diff --git a/testing/benchmark/system-profiles/16GBx2zone-gcp.tfvars b/testing/benchmark/system-profiles/16GBx2zone-gcp.tfvars
@@ -5,7 +5,7 @@ user_name = "USER"
 worker_instance_type = "c6i.2xlarge"
 
 # Elastic Cloud
-ess_region = "gcp-us-west2"
+ess_region          = "gcp-us-west2"
 deployment_template = "gcp-cpu-optimized"
 # The number of AZs the APM Server should span.
 apm_server_zone_count = 1
diff --git a/testing/benchmark/system-profiles/32GBx2zone-aws.tfvars b/testing/benchmark/system-profiles/32GBx2zone-aws.tfvars
@@ -5,7 +5,7 @@ user_name = "USER"
 worker_instance_type = "c6i.4xlarge"
 
 # Elastic Cloud
-ess_region = "us-west-2"
+ess_region          = "us-west-2"
 deployment_template = "aws-cpu-optimized-faster-warm"
 # The number of AZs the APM Server should span.
 apm_server_zone_count = 1
diff --git a/testing/benchmark/system-profiles/32GBx2zone-azure.tfvars b/testing/benchmark/system-profiles/32GBx2zone-azure.tfvars
@@ -5,7 +5,7 @@ user_name = "USER"
 worker_instance_type = "c6i.4xlarge"
 
 # Elastic Cloud
-ess_region = "azure-westus2"
+ess_region          = "azure-westus2"
 deployment_template = "azure-cpu-optimized"
 # The number of AZs the APM Server should span.
 apm_server_zone_count = 1
diff --git a/testing/benchmark/system-profiles/32GBx2zone-gcp.tfvars b/testing/benchmark/system-profiles/32GBx2zone-gcp.tfvars
@@ -5,7 +5,7 @@ user_name = "USER"
 worker_instance_type = "c6i.4xlarge"
 
 # Elastic Cloud
-ess_region = "gcp-us-west2"
+ess_region          = "gcp-us-west2"
 deployment_template = "gcp-cpu-optimized"
 # The number of AZs the APM Server should span.
 apm_server_zone_count = 1
diff --git a/testing/benchmark/system-profiles/8GBx1zone-aws.tfvars b/testing/benchmark/system-profiles/8GBx1zone-aws.tfvars
@@ -5,7 +5,7 @@ user_name = "USER"
 worker_instance_type = "c6i.2xlarge"
 
 # Elastic Cloud
-ess_region = "us-west-2"
+ess_region          = "us-west-2"
 deployment_template = "aws-cpu-optimized-faster-warm"
 # The number of AZs the APM Server should span.
 apm_server_zone_count = 1
diff --git a/testing/benchmark/system-profiles/8GBx1zone-azure.tfvars b/testing/benchmark/system-profiles/8GBx1zone-azure.tfvars
@@ -5,7 +5,7 @@ user_name = "USER"
 worker_instance_type = "c6i.2xlarge"
 
 # Elastic Cloud
-ess_region = "azure-westus2"
+ess_region          = "azure-westus2"
 deployment_template = "azure-cpu-optimized"
 # The number of AZs the APM Server should span.
 apm_server_zone_count = 1
diff --git a/testing/benchmark/system-profiles/8GBx1zone-gcp.tfvars b/testing/benchmark/system-profiles/8GBx1zone-gcp.tfvars
@@ -5,7 +5,7 @@ user_name = "USER"
 worker_instance_type = "c6i.2xlarge"
 
 # Elastic Cloud
-ess_region = "gcp-us-west2"
+ess_region          = "gcp-us-west2"
 deployment_template = "gcp-cpu-optimized"
 # The number of AZs the APM Server should span.
 apm_server_zone_count = 1
diff --git a/testing/benchmark/terraform.tfvars.example b/testing/benchmark/terraform.tfvars.example
@@ -25,6 +25,7 @@ user_name = "USER"
 # Tail-based sampling settings of the APM Server.
 # apm_server_tail_sampling = false
 # apm_server_tail_sampling_storage_limit = "10GB"
+# apm_server_tail_sampling_sample_rate = 0.1
 
 # The Elasticsearch cluster node size.
 # elasticsearch_size = "1g"
diff --git a/testing/benchmark/variables.tf b/testing/benchmark/variables.tf
@@ -104,6 +104,17 @@ variable "apm_server_tail_sampling_storage_limit" {
   type        = string
 }
 
+variable "apm_server_tail_sampling_sample_rate" {
+  default     = 0.1
+  description = "Sample rate of APM Server tail-based sampling. Defaults to 0.1"
+  type        = number
+
+  validation {
+    condition     = var.apm_server_tail_sampling_sample_rate >= 0 && var.apm_server_tail_sampling_sample_rate <= 1
+    error_message = "The apm_server_tail_sampling_sample_rate value must be in range [0, 1]"
+  }
+}
+
 # Standalone
 
 variable "apm_server_bin_path" {
diff --git a/testing/infra/terraform/modules/ec_deployment/README.md b/testing/infra/terraform/modules/ec_deployment/README.md
@@ -16,7 +16,6 @@ used to configure the module, please refer to the [EC Provider docs](https://reg
 | Name | Version |
 |------|---------|
 | <a name="provider_ec"></a> [ec](#provider\_ec) | 0.5.1 |
-| <a name="provider_external"></a> [external](#provider\_external) | n/a |
 | <a name="provider_local"></a> [local](#provider\_local) | n/a |
 | <a name="provider_null"></a> [null](#provider\_null) | n/a |
 
@@ -37,7 +36,7 @@ used to configure the module, please refer to the [EC Provider docs](https://reg
 | [null_resource.secret_token](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
 | [null_resource.shard_settings](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
 | [ec_stack.deployment_version](https://registry.terraform.io/providers/elastic/ec/0.5.1/docs/data-sources/stack) | data source |
-| [external_external.secret_token](https://registry.terraform.io/providers/hashicorp/external/latest/docs/data-sources/external) | data source |
+| [local_sensitive_file.secret_token](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/sensitive_file) | data source |
 
 ## Inputs
 
@@ -48,6 +47,7 @@ used to configure the module, please refer to the [EC Provider docs](https://reg
 | <a name="input_apm_server_pprof"></a> [apm\_server\_pprof](#input\_apm\_server\_pprof) | Whether or not to enable APM Server's pprof endpoint. Defaults to false | `bool` | `false` | no |
 | <a name="input_apm_server_size"></a> [apm\_server\_size](#input\_apm\_server\_size) | Optional apm server instance size | `string` | `"1g"` | no |
 | <a name="input_apm_server_tail_sampling"></a> [apm\_server\_tail\_sampling](#input\_apm\_server\_tail\_sampling) | Whether or not to enable APM Server tail-based sampling. Defaults to false | `bool` | `false` | no |
+| <a name="input_apm_server_tail_sampling_sample_rate"></a> [apm\_server\_tail\_sampling\_sample\_rate](#input\_apm\_server\_tail\_sampling\_sample\_rate) | Sample rate of APM Server tail-based sampling. Defaults to 0.1 | `number` | `0.1` | no |
 | <a name="input_apm_server_tail_sampling_storage_limit"></a> [apm\_server\_tail\_sampling\_storage\_limit](#input\_apm\_server\_tail\_sampling\_storage\_limit) | Storage size limit of APM Server tail-based sampling. Defaults to 10GB | `string` | `"10GB"` | no |
 | <a name="input_apm_server_zone_count"></a> [apm\_server\_zone\_count](#input\_apm\_server\_zone\_count) | Optional apm server zone count | `number` | `1` | no |
 | <a name="input_custom_apm_integration_pkg_path"></a> [custom\_apm\_integration\_pkg\_path](#input\_custom\_apm\_integration\_pkg\_path) | Path to the zipped custom APM integration package, if empty custom apm integration pkg is not installed | `string` | `""` | no |
@@ -73,6 +73,8 @@ used to configure the module, please refer to the [EC Provider docs](https://reg
 | <a name="output_admin_console_url"></a> [admin\_console\_url](#output\_admin\_console\_url) | n/a |
 | <a name="output_apm_secret_token"></a> [apm\_secret\_token](#output\_apm\_secret\_token) | The APM Secret token |
 | <a name="output_apm_url"></a> [apm\_url](#output\_apm\_url) | The secure APM URL |
+| <a name="output_deployment_id"></a> [deployment\_id](#output\_deployment\_id) | The deployment ID for the created cluster |
+| <a name="output_ech_apm_component_id"></a> [ech\_apm\_component\_id](#output\_ech\_apm\_component\_id) | APM or Integrations Server Cloud component ID |
 | <a name="output_elasticsearch_password"></a> [elasticsearch\_password](#output\_elasticsearch\_password) | The Elasticsearch password |
 | <a name="output_elasticsearch_url"></a> [elasticsearch\_url](#output\_elasticsearch\_url) | The secure Elasticsearch URL |
 | <a name="output_elasticsearch_username"></a> [elasticsearch\_username](#output\_elasticsearch\_username) | The Elasticsearch username |
diff --git a/testing/infra/terraform/modules/ec_deployment/deployment.tf b/testing/infra/terraform/modules/ec_deployment/deployment.tf
@@ -121,6 +121,7 @@ resource "local_file" "enable_features" {
     enable_pprof                = var.apm_server_pprof
     enable_tail_sampling        = var.apm_server_tail_sampling
     tail_sampling_storage_limit = var.apm_server_tail_sampling_storage_limit
+    tail_sampling_sample_rate   = var.apm_server_tail_sampling_sample_rate
   })
   filename = "${path.cwd}/scripts/enable_features.sh"
 }
diff --git a/testing/infra/terraform/modules/ec_deployment/scripts/enable_features.tftpl b/testing/infra/terraform/modules/ec_deployment/scripts/enable_features.tftpl
@@ -21,7 +21,8 @@ echo $${POLICY} | jq '.item' | \
     jq 'select(.inputs[].policy_template == "apmserver").inputs[].vars.expvar_enabled = {type: "bool", value: ${enable_expvar}}' |\
     jq 'select(.inputs[].policy_template == "apmserver").inputs[].vars.pprof_enabled = {type: "bool", value: ${enable_pprof}}' |\
     jq 'select(.inputs[].policy_template == "apmserver").inputs[].vars.tail_sampling_storage_limit = {"value":"${tail_sampling_storage_limit}","type":"text"}' |\
-    jq 'select(.inputs[].policy_template == "apmserver").inputs[].vars.tail_sampling_enabled = {type: "bool", value: ${enable_tail_sampling}}' > policy.json
+    jq 'select(.inputs[].policy_template == "apmserver").inputs[].vars.tail_sampling_enabled = {type: "bool", value: ${enable_tail_sampling}}' |\
+    jq 'select(.inputs[].policy_template == "apmserver").inputs[].vars.tail_sampling_policies = {type: "yaml", value: "- sample_rate: ${tail_sampling_sample_rate}"}' > policy.json
 
 # Update the policy, hide output to prevent secret leakage.
 if ! curl -s -o response.txt -H 'content-type: application/json' -H 'kbn-xsrf: true' -X PUT -k -d@policy.json -u $${KIBANA_AUTH} $${KIBANA_ENDPOINT}; then
diff --git a/testing/infra/terraform/modules/ec_deployment/variables.tf b/testing/infra/terraform/modules/ec_deployment/variables.tf
@@ -134,6 +134,17 @@ variable "apm_server_tail_sampling_storage_limit" {
   type        = string
 }
 
+variable "apm_server_tail_sampling_sample_rate" {
+  default     = 0.1
+  description = "Sample rate of APM Server tail-based sampling. Defaults to 0.1"
+  type        = number
+
+  validation {
+    condition     = var.apm_server_tail_sampling_sample_rate >= 0 && var.apm_server_tail_sampling_sample_rate <= 1
+    error_message = "The apm_server_tail_sampling_sample_rate value must be in range [0, 1]"
+  }
+}
+
 variable "apm_index_shards" {
   default     = 0
   description = "The number of shards to set for APM Indices"
diff --git a/testing/infra/terraform/modules/standalone_apm_server/README.md b/testing/infra/terraform/modules/standalone_apm_server/README.md
@@ -32,8 +32,10 @@ This module can be used to create a standalone APM Server deployment against a s
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_apm_instance_type"></a> [apm\_instance\_type](#input\_apm\_instance\_type) | Optional apm server instance type override | `string` | `""` | no |
+| <a name="input_apm_iops"></a> [apm\_iops](#input\_apm\_iops) | Optional apm server disk IOPS override | `number` | `null` | no |
 | <a name="input_apm_server_bin_path"></a> [apm\_server\_bin\_path](#input\_apm\_server\_bin\_path) | Optionally use the apm-server binary from the specified path instead | `string` | `""` | no |
 | <a name="input_apm_server_tail_sampling"></a> [apm\_server\_tail\_sampling](#input\_apm\_server\_tail\_sampling) | Whether or not to enable APM Server tail-based sampling. Defaults to false | `bool` | `false` | no |
+| <a name="input_apm_server_tail_sampling_sample_rate"></a> [apm\_server\_tail\_sampling\_sample\_rate](#input\_apm\_server\_tail\_sampling\_sample\_rate) | Sample rate of APM Server tail-based sampling. Defaults to 0.1 | `number` | `0.1` | no |
 | <a name="input_apm_server_tail_sampling_storage_limit"></a> [apm\_server\_tail\_sampling\_storage\_limit](#input\_apm\_server\_tail\_sampling\_storage\_limit) | Storage size limit of APM Server tail-based sampling. Defaults to 10GB | `string` | `"10GB"` | no |
 | <a name="input_apm_volume_size"></a> [apm\_volume\_size](#input\_apm\_volume\_size) | Optional apm server volume size in GB override | `number` | `null` | no |
 | <a name="input_apm_volume_type"></a> [apm\_volume\_type](#input\_apm\_volume\_type) | Optional apm server volume type override | `string` | `null` | no |
@@ -44,6 +46,7 @@ This module can be used to create a standalone APM Server deployment against a s
 | <a name="input_elasticsearch_url"></a> [elasticsearch\_url](#input\_elasticsearch\_url) | The secure Elasticsearch URL | `string` | n/a | yes |
 | <a name="input_elasticsearch_username"></a> [elasticsearch\_username](#input\_elasticsearch\_username) | The Elasticsearch username | `string` | n/a | yes |
 | <a name="input_stack_version"></a> [stack\_version](#input\_stack\_version) | Optional stack version | `string` | `"latest"` | no |
+| <a name="input_tags"></a> [tags](#input\_tags) | Optional set of tags to use for all deployments | `map(string)` | `{}` | no |
 | <a name="input_vpc_id"></a> [vpc\_id](#input\_vpc\_id) | VPC ID to provision the EC2 instance | `string` | n/a | yes |
 
 ## Outputs
diff --git a/testing/infra/terraform/modules/standalone_apm_server/apm-server.yml.tftpl b/testing/infra/terraform/modules/standalone_apm_server/apm-server.yml.tftpl
@@ -15,7 +15,7 @@ apm-server:
       enabled: ${apm_server_tail_sampling}
       storage_limit: ${apm_server_tail_sampling_storage_limit}
       policies:
-        - sample_rate: 0.1
+        - sample_rate: ${apm_server_tail_sampling_sample_rate}
 output:
   elasticsearch:
     hosts: [ ${elasticsearch_url} ]
diff --git a/testing/infra/terraform/modules/standalone_apm_server/elastic-agent.yml.tftpl b/testing/infra/terraform/modules/standalone_apm_server/elastic-agent.yml.tftpl
@@ -30,7 +30,7 @@ inputs:
           enabled: ${apm_server_tail_sampling}
           storage_limit: ${apm_server_tail_sampling_storage_limit}
           policies:
-            - sample_rate: 0.1
+            - sample_rate: ${apm_server_tail_sampling_sample_rate}
     logging.level: debug
     logging.to_files: true
     logging.files:
diff --git a/testing/infra/terraform/modules/standalone_apm_server/main.tf b/testing/infra/terraform/modules/standalone_apm_server/main.tf
@@ -231,6 +231,7 @@ resource "aws_instance" "apm" {
       apm_port                               = "${local.apm_port}"
       apm_server_tail_sampling               = "${var.apm_server_tail_sampling}"
       apm_server_tail_sampling_storage_limit = "${var.apm_server_tail_sampling_storage_limit}"
+      apm_server_tail_sampling_sample_rate   = "${var.apm_server_tail_sampling_sample_rate}"
     })
   }
 
diff --git a/testing/infra/terraform/modules/standalone_apm_server/variables.tf b/testing/infra/terraform/modules/standalone_apm_server/variables.tf
@@ -85,6 +85,17 @@ variable "apm_server_tail_sampling_storage_limit" {
   type        = string
 }
 
+variable "apm_server_tail_sampling_sample_rate" {
+  default     = 0.1
+  description = "Sample rate of APM Server tail-based sampling. Defaults to 0.1"
+  type        = number
+
+  validation {
+    condition     = var.apm_server_tail_sampling_sample_rate >= 0 && var.apm_server_tail_sampling_sample_rate <= 1
+    error_message = "The apm_server_tail_sampling_sample_rate value must be in range [0, 1]"
+  }
+}
+
 variable "tags" {
   type        = map(string)
   default     = {}
