Add prometheus soaking and perf tests (#146)

* added environment variables to EC2 AOC deployment

* make env vars only on amazon linux

* remove unneeded changes

* made env vars command modular and added windows env vars

* remove cortex instance endpoint var for backwards compatibility

* ec2 working with windows

* get soaking and performance test working for prometheus_mock

* increase memory for prometheus sample app in soaking/perf tests

* put pull mode soaking compose in defaults

* change var names

* set env vars in ec2 setup amis

Author: Aman Brar

terraform/ec2_setup/amis.tf

@@ -7,6 +7,7 @@ variable "ami_family" {
       otconfig_destination = "/tmp/ot-default.yml"
       download_command_pattern = "wget %s"
       install_command = "sudo rpm -Uvh aws-otel-collector.rpm && sudo chmod 777 /opt/aws/aws-otel-collector/etc/.env && sudo echo 'GODEBUG=madvdontneed=1' >> /opt/aws/aws-otel-collector/etc/.env"
+      set_env_var_command = "sudo chmod 777 /opt/aws/aws-otel-collector/etc/.env && sudo echo 'SAMPLE_APP_HOST=%s' >> /opt/aws/aws-otel-collector/etc/.env && sudo echo 'SAMPLE_APP_PORT=%s' >> /opt/aws/aws-otel-collector/etc/.env"
       start_command = "sudo /opt/aws/aws-otel-collector/bin/aws-otel-collector-ctl -c /tmp/ot-default.yml -a start"
       connection_type = "ssh"
       user_data = ""
@@ -26,6 +27,7 @@ variable "ami_family" {
       otconfig_destination = "C:\\ot-default.yml"
       download_command_pattern = "powershell -command \"Invoke-WebRequest -Uri %s -OutFile C:\\aws-otel-collector.msi\""
       install_command = "msiexec /i C:\\aws-otel-collector.msi"
+      set_env_var_command = "powershell \"[System.Environment]::SetEnvironmentVariable('SAMPLE_APP_HOST', '%s', [System.EnvironmentVariableTarget]::Machine); [System.Environment]::SetEnvironmentVariable('SAMPLE_APP_PORT', '%s', [System.EnvironmentVariableTarget]::Machine)\""
       start_command = "powershell \"& 'C:\\Program Files\\Amazon\\AwsOtelCollector\\aws-otel-collector-ctl.ps1' -ConfigLocation C:\\ot-default.yml -Action start\""
       connection_type = "winrm"
       soaking_cwagent_config = "../templates/cwagent-config/soaking-linux.json.tpl"

terraform/ec2_setup/main.tf

@@ -13,10 +13,20 @@
 # permissions and limitations under the License.
 # -------------------------------------------------------------------------
 
+module "common" {
+  source = "../common"
+
+  aoc_version = var.aoc_version
+}
+
 locals {
   launch_date = formatdate("YYYY-MM-DD", timestamp())
 }
 
+data "aws_ecr_repository" "sample_apps" {
+  name = module.common.sample_app_ecr_repo_name
+}
+
 # launch ec2
 module "ec2_setup" {
   source = "../ec2"
@@ -27,11 +37,11 @@ module "ec2_setup" {
   aoc_version = var.aoc_version
   region = var.region
   testcase = var.testcase
-  sample_app_image = var.soaking_data_emitter_image
+  sample_app_image = var.soaking_sample_app != "" ? "${data.aws_ecr_repository.sample_apps.repository_url}:${var.soaking_sample_app}-latest" : var.soaking_sample_app_image
   skip_validation = true
 
   # soaking test config
-  soaking_compose_file = "../templates/defaults/soaking_docker_compose.tpl"
+  soaking_compose_file = var.sample_app_mode == "push" ? "../templates/defaults/soaking_docker_compose.tpl" : "../templates/defaults/soaking_docker_compose_pull_mode.tpl"
   soaking_data_mode = var.soaking_data_mode
   soaking_data_rate = var.soaking_data_rate
   soaking_data_type = var.soaking_data_type

terraform/ec2_setup/variables.tf

@@ -16,10 +16,18 @@
 variable "testing_ami" {
 }
 
-variable "soaking_data_emitter_image" {
+variable "soaking_sample_app_image" {
   default = "aottestbed/aws-otel-load-generator:v0.1.0"
 }
 
+variable "soaking_sample_app" {
+  default = ""
+}
+
+variable "soaking_compose_file" {
+  default = "../templates/defaults/soaking_docker_compose.tpl"
+}
+
 # data points emitted per second
 variable "soaking_data_rate" {
   default = 1000

terraform/performance/main.tf

@@ -24,6 +24,9 @@ module "ec2_setup" {
   commit_id = var.commit_id
   testcase = var.testcase
   testing_ami = var.testing_ami
+
+  sample_app_mode = var.sample_app_mode
+  soaking_sample_app = var.soaking_sample_app
   soaking_data_rate = var.data_rate
   soaking_data_type = var.data_type
   soaking_data_mode = var.soaking_data_mode

terraform/performance/variables.tf

@@ -62,3 +62,7 @@ variable "collection_period" {
 variable "commit_id" {
   default = "dummy_commit"
 }
+
+variable "soaking_sample_app" {
+  default = ""
+}

terraform/soaking/main.tf

@@ -24,6 +24,8 @@ module "ec2_setup" {
   aoc_version = var.aoc_version
   region = var.region
 
+  sample_app_mode = var.sample_app_mode
+  soaking_sample_app = var.soaking_sample_app
   soaking_data_mode = var.soaking_data_mode
   soaking_data_rate = var.soaking_data_rate
   soaking_data_type = var.soaking_data_type

terraform/soaking/variables.tf

@@ -56,4 +56,8 @@ variable "soaking_data_rate" {
 
 variable "soaking_data_type" {
   default = "otlp"
-}
+}
+
+variable "soaking_sample_app" {
+  default = ""
+}

terraform/templates/defaults/soaking_docker_compose_pull_mode.tpl

@@ -0,0 +1,23 @@
+version: "3.8"
+services:
+  mocked_server:
+    image: ${mocked_server_image}
+    ports:
+      - "80:8080"
+      - "443:443"
+    deploy:
+      resources:
+        limits:
+          memory: 1G
+  ot-metric-emitter:
+    image: ${sample_app_image}
+    command: []
+    ports:
+      - ${sample_app_external_port}:${sample_app_listen_address_port}
+    environment:
+      METRICS_LOAD: ${rate}
+      LISTEN_ADDRESS: ${listen_address}
+    deploy:
+      resources:
+        limits:
+          memory: 16G

terraform/testcases/prometheus_mock/parameters.tfvars

@@ -1,3 +1,9 @@
 validation_config="default-mocked-server-prometheus-validation.yml"
 
 sample_app="prometheus"
+
+soaking_sample_app="prometheus"
+
+soaking_data_type="prometheus"
+
+sample_app_mode="pull"