Skip to content

Commit e425d41

Browse files
sarahchen6sarahchen6
committed
iteration 1
1 parent 632fb11 commit e425d41

File tree

9 files changed

+321
-18
lines changed

9 files changed

+321
-18
lines changed

‎.gitlab/benchmarks.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,23 +35,23 @@ benchmarks-startup:
3535
script:
3636
- !reference [ .benchmarks, script ]
3737
- ./steps/capture-hardware-software-info.sh
38-
- ./steps/run-benchmarks.sh startup
38+
- BENCHMARK_TYPE=startup bp-runner "${CI_PROJECT_DIR:-.}/.gitlab/benchmarks/bp-runner.yml" --debug -t
3939
- ./steps/analyze-results.sh startup
4040

4141
benchmarks-load:
4242
extends: .benchmarks
4343
script:
4444
- !reference [ .benchmarks, script ]
4545
- ./steps/capture-hardware-software-info.sh
46-
- ./steps/run-benchmarks.sh load
46+
- BENCHMARK_TYPE=load bp-runner "${CI_PROJECT_DIR:-.}/.gitlab/benchmarks/bp-runner.yml" --debug -t
4747
- ./steps/analyze-results.sh load
4848

4949
benchmarks-dacapo:
5050
extends: .benchmarks
5151
script:
5252
- !reference [ .benchmarks, script ]
5353
- ./steps/capture-hardware-software-info.sh
54-
- ./steps/run-benchmarks.sh dacapo
54+
- BENCHMARK_TYPE=dacapo bp-runner "${CI_PROJECT_DIR:-.}/.gitlab/benchmarks/bp-runner.yml" --debug -t
5555
- ./steps/analyze-results.sh dacapo
5656

5757
benchmarks-post-results:

‎.gitlab/benchmarks/bp-runner.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# Usage: BENCHMARK_TYPE=<startup|load|dacapo> bp-runner .gitlab/benchmarks/bp-runner.yml --debug -t
2+
3+
experiments:
4+
- name: run-benchmarks
5+
steps:
6+
- name: benchmarks
7+
cpus: 24-47
8+
run: shell
9+
script: /platform/steps/run-benchmarks.sh $BENCHMARK_TYPE
10+

‎benchmark/Dockerfile‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -78,11 +78,7 @@ ENV JAVA_17_HOME=/usr/lib/jvm/17
7878
ENV JAVA_HOME=${JAVA_8_HOME}
7979
ENV PATH=${PATH}:${JAVA_HOME}/bin
8080

81-
ARG SIRUN_VERSION=0.1.11
82-
RUN wget -O sirun.tar.gz https://github.com/DataDog/sirun/releases/download/v$SIRUN_VERSION/sirun-v$SIRUN_VERSION-x86_64-unknown-linux-musl.tar.gz \
83-
&& tar -xzf sirun.tar.gz \
84-
&& rm sirun.tar.gz \
85-
&& mv sirun /usr/bin/sirun
81+
# Install bp-runner somehow
8682

8783
ARG K6_VERSION=0.45.1
8884
RUN wget -O k6.tar.gz https://github.com/grafana/k6/releases/download/v$K6_VERSION/k6-v$K6_VERSION-linux-amd64.tar.gz \

‎benchmark/README.MD‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ This directory contains different types of benchmarks.
44

55
## Running Benchmarks via Docker
66

7-
Docker allows the execution of benchmarks without needing to install and configure your development environment. For example, package installation and installation of sirun are performed automatically.
7+
Docker allows the execution of benchmarks without needing to install and configure your development environment.
88

99
In order to run benchmarks using Docker, issue the following command from the `benchmark/` folder of this project:
1010

‎benchmark/dacapo/bp-runner.yml‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
# bp-runner configuration for dacapo benchmarks based on sirun's benchmark.json format
2+
#
3+
# Each variant (no_agent, tracing, profiling, appsec, iast, iast_GLOBAL) is defined
4+
# as a separate experiment.
5+
#
6+
# CPU pinning uses cores 24-47 for consistency with CI configuration.
7+
8+
experiments:
9+
# no-agent variant
10+
- name: dacapo-no-agent
11+
steps:
12+
- name: run-dacapo-benchmark
13+
cpus: 24-47
14+
run: shell
15+
script: |
16+
export VARIANT="${NO_AGENT_VARIANT}"
17+
export JAVA_OPTS=""
18+
mkdir -p ${OUTPUT_DIR}/${VARIANT}
19+
20+
java ${JAVA_OPTS} \
21+
-jar ${DACAPO} \
22+
--converge \
23+
--scratch-directory=${OUTPUT_DIR}/${VARIANT}/scratch \
24+
--latency-csv \
25+
${BENCHMARK} &> ${OUTPUT_DIR}/${VARIANT}/dacapo.log
26+
27+
# tracing variant
28+
- name: dacapo-tracing
29+
steps:
30+
- name: run-dacapo-benchmark
31+
cpus: 24-47
32+
run: shell
33+
script: |
34+
export VARIANT="tracing"
35+
export JAVA_OPTS="-javaagent:${TRACER}"
36+
mkdir -p ${OUTPUT_DIR}/${VARIANT}
37+
38+
java ${JAVA_OPTS} \
39+
-jar ${DACAPO} \
40+
--converge \
41+
--scratch-directory=${OUTPUT_DIR}/${VARIANT}/scratch \
42+
--latency-csv \
43+
${BENCHMARK} &> ${OUTPUT_DIR}/${VARIANT}/dacapo.log
44+
45+
# profiling variant
46+
- name: dacapo-profiling
47+
steps:
48+
- name: run-dacapo-benchmark
49+
cpus: 24-47
50+
run: shell
51+
script: |
52+
export VARIANT="profiling"
53+
export JAVA_OPTS="-javaagent:${TRACER} -Ddd.profiling.enabled=true"
54+
mkdir -p ${OUTPUT_DIR}/${VARIANT}
55+
56+
java ${JAVA_OPTS} \
57+
-jar ${DACAPO} \
58+
--converge \
59+
--scratch-directory=${OUTPUT_DIR}/${VARIANT}/scratch \
60+
--latency-csv \
61+
${BENCHMARK} &> ${OUTPUT_DIR}/${VARIANT}/dacapo.log
62+
63+
# appsec variant
64+
- name: dacapo-appsec
65+
steps:
66+
- name: run-dacapo-benchmark
67+
cpus: 24-47
68+
run: shell
69+
script: |
70+
export VARIANT="appsec"
71+
export JAVA_OPTS="-javaagent:${TRACER} -Ddd.appsec.enabled=true -Ddd.iast.enabled=false"
72+
mkdir -p ${OUTPUT_DIR}/${VARIANT}
73+
74+
java ${JAVA_OPTS} \
75+
-jar ${DACAPO} \
76+
--converge \
77+
--scratch-directory=${OUTPUT_DIR}/${VARIANT}/scratch \
78+
--latency-csv \
79+
${BENCHMARK} &> ${OUTPUT_DIR}/${VARIANT}/dacapo.log
80+
81+
# iast variant
82+
- name: dacapo-iast
83+
steps:
84+
- name: run-dacapo-benchmark
85+
cpus: 24-47
86+
run: shell
87+
script: |
88+
export VARIANT="iast"
89+
export JAVA_OPTS="-javaagent:${TRACER} -Ddd.iast.enabled=true"
90+
mkdir -p ${OUTPUT_DIR}/${VARIANT}
91+
92+
java ${JAVA_OPTS} \
93+
-jar ${DACAPO} \
94+
--converge \
95+
--scratch-directory=${OUTPUT_DIR}/${VARIANT}/scratch \
96+
--latency-csv \
97+
${BENCHMARK} &> ${OUTPUT_DIR}/${VARIANT}/dacapo.log
98+
99+
# iast_GLOBAL variant
100+
- name: dacapo-iast-global
101+
steps:
102+
- name: run-dacapo-benchmark
103+
cpus: 24-47
104+
run: shell
105+
script: |
106+
export VARIANT="iast_GLOBAL"
107+
export JAVA_OPTS="-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.context.mode=GLOBAL"
108+
mkdir -p ${OUTPUT_DIR}/${VARIANT}
109+
110+
java ${JAVA_OPTS} \
111+
-jar ${DACAPO} \
112+
--converge \
113+
--scratch-directory=${OUTPUT_DIR}/${VARIANT}/scratch \
114+
--latency-csv \
115+
${BENCHMARK} &> ${OUTPUT_DIR}/${VARIANT}/dacapo.log

‎benchmark/dacapo/run.sh‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,8 @@ run_benchmark() {
1919
export OUTPUT_DIR="${REPORTS_DIR}/dacapo/${type}"
2020
mkdir -p "${OUTPUT_DIR}"
2121

22-
# substitute environment variables in the json file
23-
benchmark=$(mktemp)
24-
# shellcheck disable=SC2046
25-
# shellcheck disable=SC2016
26-
envsubst "$(printf '${%s} ' $(env | cut -d'=' -f1))" <benchmark.json >"${benchmark}"
27-
28-
# run the sirun test
29-
sirun "${benchmark}" &>"${OUTPUT_DIR}/${type}.json"
22+
# run the bp-runner test
23+
bp-runner bp-runner.yml &>"${OUTPUT_DIR}/${type}.json"
3024

3125
message "dacapo benchmark: ${type} finished"
3226
}

‎benchmark/startup/insecure-bank/bp-runner.yml‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
# bp-runner configuration for insecure-bank startup benchmarks based on sirun's benchmark.json format
2+
#
3+
# Each variant (tracing, iast) is defined as a separate experiment.
4+
#
5+
# CPU pinning uses cores 24-47 for consistency with CI configuration.
6+
experiments:
7+
# tracing variant
8+
- name: startup-insecure-bank-tracing
9+
setup:
10+
- name: service-monitor
11+
run: shell
12+
script: ${UTILS_DIR}/run-on-server-ready.sh http://localhost:8080/login 'pkill java'
13+
14+
steps:
15+
- name: run-startup-benchmark
16+
cpus: 24-47
17+
run: shell
18+
script: |
19+
export VARIANT="tracing"
20+
export JAVA_OPTS=""
21+
mkdir -p ${OUTPUT_DIR}/${VARIANT}
22+
23+
for i in $(seq 1 10); do
24+
java -javaagent:${TRACER} \
25+
-Ddd.benchmark.enabled=true \
26+
-Ddd.benchmark.output.dir=${OUTPUT_DIR}/${VARIANT} \
27+
${JAVA_OPTS} \
28+
-jar ${INSECURE_BANK} &> ${OUTPUT_DIR}/${VARIANT}/insecure-bank.log || true
29+
done
30+
31+
# iast variant
32+
- name: startup-insecure-bank-iast
33+
setup:
34+
- name: service-monitor
35+
run: shell
36+
script: ${UTILS_DIR}/run-on-server-ready.sh http://localhost:8080/login 'pkill java'
37+
38+
steps:
39+
- name: run-startup-benchmark
40+
cpus: 24-47
41+
run: shell
42+
script: |
43+
export VARIANT="iast"
44+
export JAVA_OPTS="-Ddd.iast.enabled=true"
45+
mkdir -p ${OUTPUT_DIR}/${VARIANT}
46+
47+
for i in $(seq 1 10); do
48+
java -javaagent:${TRACER} \
49+
-Ddd.benchmark.enabled=true \
50+
-Ddd.benchmark.output.dir=${OUTPUT_DIR}/${VARIANT} \
51+
${JAVA_OPTS} \
52+
-jar ${INSECURE_BANK} &> ${OUTPUT_DIR}/${VARIANT}/insecure-bank.log || true
53+
done

‎benchmark/startup/petclinic/bp-runner.yml‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
# bp-runner configuration for petclinic startup benchmarks based on sirun's benchmark.json format
2+
#
3+
# Each variant (tracing, profiling, appsec, iast) is defined as a separate experiment.
4+
#
5+
# CPU pinning uses cores 24-47 for consistency with CI configuration.
6+
experiments:
7+
# tracing variant
8+
- name: startup-tracing
9+
setup:
10+
- name: service-monitor
11+
run: shell
12+
script: ${UTILS_DIR}/run-on-server-ready.sh http://localhost:8080 'pkill java'
13+
14+
steps:
15+
- name: run-startup-benchmark
16+
cpus: 24-47
17+
run: shell
18+
script: |
19+
export VARIANT="tracing"
20+
export JAVA_OPTS=""
21+
mkdir -p ${OUTPUT_DIR}/${VARIANT}
22+
23+
# Run benchmark iterations (converted from sirun's iterations: 10)
24+
for i in $(seq 1 10); do
25+
java -javaagent:${TRACER} \
26+
-Ddd.benchmark.enabled=true \
27+
-Ddd.benchmark.output.dir=${OUTPUT_DIR}/${VARIANT} \
28+
${JAVA_OPTS} \
29+
-jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log || true
30+
done
31+
32+
# profiling variant
33+
- name: startup-profiling
34+
setup:
35+
- name: service-monitor
36+
run: shell
37+
script: ${UTILS_DIR}/run-on-server-ready.sh http://localhost:8080 'pkill java'
38+
39+
steps:
40+
- name: run-startup-benchmark
41+
cpus: 24-47
42+
run: shell
43+
script: |
44+
export VARIANT="profiling"
45+
export JAVA_OPTS="-Ddd.profiling.enabled=true"
46+
mkdir -p ${OUTPUT_DIR}/${VARIANT}
47+
48+
for i in $(seq 1 10); do
49+
java -javaagent:${TRACER} \
50+
-Ddd.benchmark.enabled=true \
51+
-Ddd.benchmark.output.dir=${OUTPUT_DIR}/${VARIANT} \
52+
${JAVA_OPTS} \
53+
-jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log || true
54+
done
55+
56+
# appsec variant
57+
- name: startup-appsec
58+
setup:
59+
- name: service-monitor
60+
run: shell
61+
script: ${UTILS_DIR}/run-on-server-ready.sh http://localhost:8080 'pkill java'
62+
63+
steps:
64+
- name: run-startup-benchmark
65+
cpus: 24-47
66+
run: shell
67+
script: |
68+
export VARIANT="appsec"
69+
export JAVA_OPTS="-Ddd.appsec.enabled=true"
70+
mkdir -p ${OUTPUT_DIR}/${VARIANT}
71+
72+
for i in $(seq 1 10); do
73+
java -javaagent:${TRACER} \
74+
-Ddd.benchmark.enabled=true \
75+
-Ddd.benchmark.output.dir=${OUTPUT_DIR}/${VARIANT} \
76+
${JAVA_OPTS} \
77+
-jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log || true
78+
done
79+
80+
# iast variant
81+
- name: startup-iast
82+
setup:
83+
- name: service-monitor
84+
run: shell
85+
script: ${UTILS_DIR}/run-on-server-ready.sh http://localhost:8080 'pkill java'
86+
87+
steps:
88+
- name: run-startup-benchmark
89+
cpus: 24-47
90+
run: shell
91+
script: |
92+
export VARIANT="iast"
93+
export JAVA_OPTS="-Ddd.iast.enabled=true"
94+
mkdir -p ${OUTPUT_DIR}/${VARIANT}
95+
96+
for i in $(seq 1 10); do
97+
java -javaagent:${TRACER} \
98+
-Ddd.benchmark.enabled=true \
99+
-Ddd.benchmark.output.dir=${OUTPUT_DIR}/${VARIANT} \
100+
${JAVA_OPTS} \
101+
-jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log || true
102+
done

‎benchmark/startup/run.sh‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,37 @@
22
set -eu
33

44
source "${UTILS_DIR}/update-java-version.sh" 17
5-
"${UTILS_DIR}/run-sirun-benchmarks.sh" "$@"
5+
6+
# from run-sirun-benchmarks.sh, matches dacapo and load benchmark run.sh scripts
7+
function message() {
8+
echo "$(date +"%T"): $1"
9+
}
10+
11+
run_benchmark() {
12+
local type=$1
13+
local app=$2
14+
if [[ -d "${app}" ]] && [[ -f "${app}/benchmark.json" ]]; then
15+
16+
message "${type} benchmark: ${app} started"
17+
cd "${app}"
18+
19+
# create output folder for the test
20+
export OUTPUT_DIR="${REPORTS_DIR}/${type}/${app}"
21+
mkdir -p "${OUTPUT_DIR}"
22+
23+
# run the bp-runner test
24+
bp-runner bp-runner.yml &>"${OUTPUT_DIR}/${app}.json"
25+
26+
message "${type} benchmark: ${app} finished"
27+
28+
cd ..
29+
fi
30+
}
31+
32+
if [ "$#" == '2' ]; then
33+
run_benchmark "$@"
34+
else
35+
for folder in *; do
36+
run_benchmark "$1" "${folder}"
37+
done
38+
fi

0 commit comments

Comments
 (0)