draft: Parallelize petclinic & insecure-bank benchmarks

.gitlab/benchmarks.yml

@@ -15,7 +15,7 @@
   script:
     - export ARTIFACTS_DIR="$(pwd)/reports" && mkdir -p "${ARTIFACTS_DIR}"
     - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"
-    - git clone --branch dd-trace-java/tracer-benchmarks https://github.com/DataDog/benchmarking-platform.git /platform && cd /platform
+    - git clone --branch dd-trace-java/tracer-benchmarks-parallel https://github.com/DataDog/benchmarking-platform.git /platform && cd /platform
   artifacts:
     name: "reports"
     paths:

benchmark/benchmarks.sh

@@ -33,10 +33,6 @@ if [[ ! -f "${TRACER}" ]]; then
   cd "${SCRIPT_DIR}"
 fi
 
-# Cleanup previous reports
-rm -rf "${REPORTS_DIR}"
-mkdir -p "${REPORTS_DIR}"
-
 if [[ "$#" == '0' ]]; then
   for type in 'startup' 'load' 'dacapo'; do
     run_benchmarks "$type"

benchmark/load/insecure-bank/k6.js

@@ -1,18 +1,71 @@
 import http from 'k6/http';
 import {checkResponse, isOk, isRedirect} from "../../utils/k6.js";
 
-const baseUrl = 'http://localhost:8080';
+const variants = {
+  "no_agent": {
+    "APP_URL": 'http://localhost:8080',
+  },
+  "tracing": {
+    "APP_URL": 'http://localhost:8081',
+  },
+  "profiling": {
+    "APP_URL": 'http://localhost:8082',
+  },
+  "iast": {
+    "APP_URL": 'http://localhost:8083',
+  },
+  "iast_GLOBAL": {
+    "APP_URL": 'http://localhost:8084',
+  },
+  "iast_FULL": {
+    "APP_URL": 'http://localhost:8085',
+  },
+  "iast_INACTIVE": {
+    "APP_URL": 'http://localhost:8086',
+  },
+  "iast_TELEMETRY_OFF": {
+    "APP_URL": 'http://localhost:8087',
+  },
+  "iast_HARDCODED_SECRET_DISABLED": {
+    "APP_URL": 'http://localhost:8088',
+  },
+}
+
+export const options = function (variants) {
+  let scenarios = {};
+  for (const variant of Object.keys(variants)) {
+    scenarios[`load--insecure-bank--${variant}--warmup`] = {
+      executor: 'constant-vus',  // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors
+      vus: 5,
+      duration: '20s',
+      gracefulStop: '2s',
+      env: {
+        "APP_URL": variants[variant]["APP_URL"]
+      }
+    };
+
+    scenarios[`load--insecure-bank--${variant}--high_load`] = {
+      executor: 'constant-vus',
+      vus: 5,
+      startTime: '22s',
+      duration: '15s',
+      gracefulStop: '2s',
+      env: {
+        "APP_URL": variants[variant]["APP_URL"]
+      }
+    };
+  }
 
-export const options = {
-  discardResponseBodies: true,
-  vus: 5,
-  iterations: 40000
-};
+  return {
+    discardResponseBodies: true,
+    scenarios,
+  }
+}(variants);
 
 export default function () {
 
   // login form
-  const loginResponse = http.post(`${baseUrl}/login`, {
+  const loginResponse = http.post(`${__ENV.APP_URL}/login`, {
     username: 'john',
     password: 'test'
   }, {
@@ -21,11 +74,11 @@ export default function () {
   checkResponse(loginResponse, isRedirect);
 
   // dashboard
-  const dashboard = http.get(`${baseUrl}/dashboard`);
+  const dashboard = http.get(`${__ENV.APP_URL}/dashboard`);
   checkResponse(dashboard, isOk);
 
   // logout
-  const logout = http.get(`${baseUrl}/j_spring_security_logout`, {
+  const logout = http.get(`${__ENV.APP_URL}/j_spring_security_logout`, {
     redirects: 0
   });
   checkResponse(logout, isRedirect);

benchmark/load/insecure-bank/start-servers.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -e
+
+start_server() {
+  local VARIANT=$1
+  local JAVA_OPTS=$2
+
+  if [ -n "$CI_JOB_TOKEN" ]; then
+    # Inside BP, so we can assume 24 CPU cores available and set CPU affinity
+    CPU_AFFINITY_APP=$3
+  else
+    CPU_AFFINITY_APP=""
+  fi
+
+  mkdir -p "${LOGS_DIR}/${VARIANT}"
+  ${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${LOGS_DIR}/${VARIANT}/insecure-bank.log &PID=$!
+  echo "${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${LOGS_DIR}/${VARIANT}/insecure-bank.log [PID=$PID]"
+}
+
+start_server "no_agent" "-Dserver.port=8080" "taskset -c 47 " &
+start_server "tracing" "-javaagent:${TRACER} -Dserver.port=8081" "taskset -c 46 " &
+start_server "profiling" "-javaagent:${TRACER} -Ddd.profiling.enabled=true -Dserver.port=8082" "taskset -c 45 " &
+start_server "iast" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Dserver.port=8083" "taskset -c 44 " &
+start_server "iast_GLOBAL" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.context.mode=GLOBAL -Dserver.port=8084" "taskset -c 43 " &
+start_server "iast_FULL" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.detection.mode=FULL -Dserver.port=8085" "taskset -c 42 " &
+start_server "iast_INACTIVE" "-javaagent:${TRACER} -Ddd.iast.enabled=inactive -Dserver.port=8086" "taskset -c 41 " &
+start_server "iast_TELEMETRY_OFF" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.telemetry.verbosity=OFF -Dserver.port=8087" "taskset -c 40 " &
+start_server "iast_HARDCODED_SECRET_DISABLED" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.hardcoded-secret.enabled=false -Dserver.port=8088" "taskset -c 39 " &
+
+wait

benchmark/load/petclinic/k6.js

@@ -1,17 +1,61 @@
 import http from 'k6/http';
 import {checkResponse, isOk} from "../../utils/k6.js";
 
-const baseUrl = 'http://localhost:8080';
+const variants = {
+  "no_agent": {
+    "APP_URL": 'http://localhost:8080',
+  },
+  "tracing": {
+    "APP_URL": 'http://localhost:8081',
+  },
+  "profiling": {
+    "APP_URL": 'http://localhost:8082',
+  },
+  "appsec": {
+    "APP_URL": 'http://localhost:8083',
+  },
+  "iast": {
+    "APP_URL": 'http://localhost:8084',
+  },
+  "code_origins": {
+    "APP_URL": 'http://localhost:8085',
+  }
+}
+
+export const options = function (variants) {
+  const scenarios = {};
+  for (const variant of Object.keys(variants)) {
+    scenarios[`load--petclinic--${variant}--warmup`] = {
+      executor: 'constant-vus',  // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors
+      vus: 5,
+      duration: '20s',
+      gracefulStop: '2s',
+      env: {
+        "APP_URL": variants[variant]["APP_URL"]
+      }
+    };
+
+    scenarios[`load--petclinic--${variant}--high_load`] = {
+      executor: 'constant-vus',
+      vus: 5,
+      startTime: '22s',
+      duration: '15s',
+      gracefulStop: '2s',
+      env: {
+        "APP_URL": variants[variant]["APP_URL"]
+      }
+    };
+  }
 
-export const options = {
-  discardResponseBodies: true,
-  vus: 5,
-  iterations: 80000
-};
+  return {
+    discardResponseBodies: true,
+    scenarios,
+  }
+}(variants);
 
 export default function () {
 
   // find owner
-  const ownersList = http.get(`${baseUrl}/owners?lastName=`);
+  const ownersList = http.get(`${__ENV.APP_URL}/owners?lastName=`);
   checkResponse(ownersList, isOk);
 }

benchmark/load/petclinic/start-servers.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+set -e
+
+start_server() {
+  local VARIANT=$1
+  local JAVA_OPTS=$2
+
+  if [ -n "$CI_JOB_TOKEN" ]; then
+    # Inside BP, so we can assume 24 CPU cores available and set CPU affinity
+    CPU_AFFINITY_APP=$3
+  else
+    CPU_AFFINITY_APP=""
+  fi
+
+  mkdir -p "${LOGS_DIR}/${VARIANT}"
+  ${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${LOGS_DIR}/${VARIANT}/petclinic.log &PID=$!
+  echo "${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${LOGS_DIR}/${VARIANT}/petclinic.log [PID=$!]"
+}
+
+start_server "no_agent" "-Dserver.port=8080" "taskset -c 31-32 " &
+start_server "tracing" "-javaagent:${TRACER} -Dserver.port=8081" "taskset -c 33-34 " &
+start_server "profiling" "-javaagent:${TRACER} -Ddd.profiling.enabled=true -Dserver.port=8082" "taskset -c 35-36 " &
+start_server "appsec" "-javaagent:${TRACER} -Ddd.appsec.enabled=true -Dserver.port=8083" "taskset -c 37-38 " &
+start_server "iast" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Dserver.port=8084" "taskset -c 39-40 " &
+start_server "code_origins" "-javaagent:${TRACER} -Ddd.code.origin.for.spans.enabled=true -Dserver.port=8085" "taskset -c 41-42 " &
+
+wait