Skip to content

Commit b4571cd

Browse files
sunya-chsunya-ch
authored
Merge pull request #161 from marceloamaral/stress-ng-patch
add more scenarios to stress-ng
2 parents f62ea32 + 9f59a2d commit b4571cd

File tree

1 file changed

+271
-46
lines changed

1 file changed

+271
-46
lines changed

‎model_training/benchmark/stressng.yaml‎

Lines changed: 271 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -11,31 +11,73 @@ spec:
1111
template:
1212
spec:
1313
containers:
14-
- name: stress
15-
image: alexeiled/stress-ng
14+
- name: stress-{{ index .stress 1 }}
15+
image: quay.io/sustainability/stress-ng:0.16.02
1616
imagePullPolicy: IfNotPresent
1717
env:
1818
- name: TIMEOUT
19-
value: 120s
19+
value: "30"
20+
securityContext:
21+
privileged: true
2022
command:
21-
- /stress-ng
22-
{{ if ne (index .stress 0) "0" -}}
23-
- --cpu
24-
- "{{ index .stress 0 }}"
25-
{{ end -}}
26-
{{ if ne (index .stress 1) "0" -}}
27-
- --io
28-
- "{{ index .stress 1 }}"
29-
{{ end -}}
30-
{{ if ne (index .stress 2) "0" -}}
31-
- --vm
32-
- "{{ index .stress 2 }}"
33-
- --vm-bytes
34-
- "{{ index .stress 3 }}"
35-
{{ end -}}
36-
- --timeout
37-
- $(TIMEOUT)
38-
- --metrics-brief
23+
- /bin/sh
24+
- -c
25+
- |
26+
{{ if eq (index .stress 1) "sleep" -}}
27+
sleep $(TIMEOUT)
28+
exit 0
29+
{{ end -}}
30+
31+
# Different processes can be executed, so that to do CPU pinning properly we store the cpu index to be use in a file
32+
# If the files does not exist, we populate the file with the CPU index
33+
lscpu | grep "NUMA node"| tail -n +2 | while read -r line; do
34+
cpus=$(echo $line | awk '{print $4}' | sed 's/,.*//g' | sed 's/-/ /g')
35+
for i in $(seq $cpus); do
36+
echo $i >> cpu-idx
37+
done
38+
done
39+
lscpu | grep "NUMA node"| tail -n +2 | while read -r line; do
40+
cpus=$(echo $line | awk '{print $4}' | sed 's/.*,//g' | sed 's/-/ /g') # the difference here is regex of the ","
41+
for i in $(seq $cpus); do
42+
echo $i >> cpu-idx
43+
done
44+
done
45+
46+
# Store the CPU max freq to revert it back if changed
47+
{{ if ne (index .stress 0) "none" -}}
48+
mount -o remount,rw /sys/devices/system/cpu/cpufreq
49+
cat "/sys/devices/system/cpu/cpufreq/policy0/scaling_max_freq" > max_freq
50+
{{ end -}}
51+
52+
for i in $(seq 1 {{ index .stress 2 }}); do
53+
CPU=$(cat cpu-idx | tail -n 1)
54+
cat cpu-idx | head -n -1 > tmp-cpu && rm -f cpu-idx && mv tmp-cpu cpu-idx
55+
{{ if ne (index .stress 0) "none" -}}
56+
echo {{ index .stress 0 }} | tee /sys/devices/system/cpu/cpufreq/policy${CPU}/scaling_max_freq
57+
{{ end -}}
58+
echo /usr/bin/stress-ng --{{ index .stress 1 }} 1 --taskset ${CPU} {{ if ne (index .stress 3) "none" -}} --{{ index .stress 3 }} "{{ index .stress 4 }}" {{ end -}} --timeout $(TIMEOUT) --aggressive --metrics-brief
59+
/usr/bin/stress-ng --{{ index .stress 1}} 1 --taskset ${CPU} {{ if ne (index .stress 3) "none" -}} --{{ index .stress 3 }} "{{ index .stress 4 }}" {{ end -}} --timeout $(TIMEOUT) --aggressive --metrics-brief &
60+
done
61+
62+
wait
63+
64+
# Revert the CPU freq back to max
65+
{{ if ne (index .stress 0) "none" -}}
66+
FREQ=$(cat max_freq)
67+
echo $FREQ | tee /sys/devices/system/cpu/cpufreq/policy${CPU}/scaling_max_freq
68+
{{ end -}}
69+
echo "finished"
70+
volumeMounts:
71+
- mountPath: /sys/devices/system/cpu/cpufreq
72+
name: system-cpu
73+
propagation: bidirectional
74+
readOnly: false
75+
volumes:
76+
- name: system-cpu
77+
hostPath:
78+
path: /sys/devices/system/cpu/cpufreq
79+
# type: File
80+
type: Directory
3981
restartPolicy: Never
4082
parserKey: stress
4183
repetition: 1
@@ -44,29 +86,212 @@ spec:
4486
iterations:
4587
- name: stress
4688
values:
47-
- "0;0;1;500M"
48-
- "0;0;4;500M"
49-
- "0;0;8;500M"
50-
- "0;0;16;500M"
51-
- "0;0;32;500M"
52-
- "0;0;1;1G"
53-
- "0;0;4;1G"
54-
- "0;0;8;1G"
55-
- "0;0;16;1G"
56-
- "0;0;32;1G"
57-
- "0;0;1;2G"
58-
- "0;0;4;2G"
59-
- "0;0;8;2G"
60-
- "0;0;16;2G"
61-
- "0;0;32;2G"
62-
- "1;0;0;0"
63-
- "4;0;0;0"
64-
- "8;0;0;0"
65-
- "16;0;0;0"
66-
- "32;0;0;0"
67-
- "0;1;0;0"
68-
- "0;4;0;0"
69-
- "0;8;0;0"
70-
- "0;16;0;0"
71-
- "0;32;0;0"
89+
# The baseline scenarios are used to calculate the OS/Backgroud/Idle and activation power.
90+
# These two powers allows us to separate the dynamic power consumption from the user workloads by calculating the delta of scenarios.
91+
# For each scenarios, each workload power = (scenarioPower - OS/Backgroud/IdlePower - activationPower) / numWorkloads
92+
# This will be the best ground truth of the dynamic power to validate the estimated dynamic power consumption later.
93+
#
94+
# The max CPU frequency in baselineMachine is 3600000 and the min is 1200000. We define 3 slopes between the min and max freq as (max-min)/3 = 800000.
95+
# Then we will test with frequencies of 2000000, 2800000 and 3600000.
96+
#
97+
# We first execute the baseline scenarios.
98+
# cpuFrequency;useOrNotHT;mainWorkload;numInstances;extraParam;extraParamValue
99+
- "none;sleep;none;none;none" # capture the OS/backgroud power consumption
100+
- "3600000;cpu;1;none;none" # the incremental power from the previous scenarios is activation + workload power
101+
- "3600000;cpu;2;none;none" # the incremental power from the previous scenarios is only the workload power
102+
#
103+
# Then we execute the all other workloads.
104+
# cpu: is used to stress the CPU
105+
# The baselineMachine has 32 CPUs with 2 hyperthreads
106+
- "2000000;cpu;4;none;none"
107+
- "2000000;cpu;8;none;none"
108+
- "2000000;cpu;15;none;none" # max CPU cores in baselineMachine (15)
109+
- "2000000;cpu;24;none;none"
110+
- "2000000;cpu;32;none;none" # max HT cores in baselineMachine (32)
111+
- "2800000;cpu;4;none;none"
112+
- "2800000;cpu;8;none;none"
113+
- "2800000;cpu;15;none;none" # max CPU cores in baselineMachine (15)
114+
- "2800000;cpu;24;none;none"
115+
- "2800000;cpu;32;none;none" # max HT cores in baselineMachine (32)
116+
- "3600000;cpu;4;none;none"
117+
- "3600000;cpu;8;none;none"
118+
- "3600000;cpu;15;none;none" # max CPU cores in baselineMachine (15)
119+
- "3600000;cpu;24;none;none"
120+
- "3600000;cpu;32;none;none" # max HT cores in baselineMachine (32)
121+
#
122+
# branch: is used to stress branch by branch to 1024 randomly selected locations and hence exercise
123+
# the CPU branch prediction logic
124+
- "2000000;branch;4;none;none"
125+
- "2000000;branch;8;none;none"
126+
- "2000000;branch;15;none;none" # max CPU cores in baselineMachine (15)
127+
- "2000000;branch;24;none;none"
128+
- "2000000;branch;32;none;none" # max HT cores in baselineMachine (32)
129+
- "2800000;branch;4;none;none"
130+
- "2800000;branch;8;none;none"
131+
- "2800000;branch;15;none;none" # max CPU cores in baselineMachine (15)
132+
- "2800000;branch;24;none;none"
133+
- "2800000;branch;32;none;none" # max HT cores in baselineMachine (32)
134+
- "3600000;branch;4;none;none"
135+
- "3600000;branch;8;none;none"
136+
- "3600000;branch;15;none;none" # max CPU cores in baselineMachine (15)
137+
- "3600000;branch;24;none;none"
138+
- "3600000;branch;32;none;none" # max HT cores in baselineMachine (32)
139+
#
140+
# cyclic: is used to stress linux schedulers with cyclic nanosecond sleeps
141+
- "2000000;cyclic;4;none;none"
142+
- "2000000;cyclic;8;none;none"
143+
- "2000000;cyclic;15;none;none" # max CPU cores in baselineMachine (15)
144+
- "2000000;cyclic;24;none;none"
145+
- "2000000;cyclic;32;none;none" # max HT cores in baselineMachine (32)
146+
- "2800000;cyclic;4;none;none"
147+
- "2800000;cyclic;8;none;none"
148+
- "2800000;cyclic;15;none;none" # max CPU cores in baselineMachine (15)
149+
- "2800000;cyclic;24;none;none"
150+
- "2800000;cyclic;32;none;none" # max HT cores in baselineMachine (32)
151+
- "3600000;cyclic;4;none;none"
152+
- "3600000;cyclic;8;none;none"
153+
- "3600000;cyclic;15;none;none" # max CPU cores in baselineMachine (15)
154+
- "3600000;cyclic;24;none;none"
155+
- "3600000;cyclic;32;none;none" # max HT cores in baselineMachine (32)
156+
#
157+
# regs: start N workers exercising CPU generic registers
158+
- "2000000;regs;4;none;none"
159+
- "2000000;regs;8;none;none"
160+
- "2000000;regs;15;none;none" # max CPU cores in baselineMachine (15)
161+
- "2000000;regs;24;none;none"
162+
- "2000000;regs;32;none;none" # max HT cores in baselineMachine (32)
163+
- "2800000;regs;4;none;none"
164+
- "2800000;regs;8;none;none"
165+
- "2800000;regs;15;none;none" # max CPU cores in baselineMachine (15)
166+
- "2800000;regs;24;none;none"
167+
- "2800000;regs;32;none;none" # max HT cores in baselineMachine (32)
168+
- "3600000;regs;4;none;none"
169+
- "3600000;regs;8;none;none"
170+
- "3600000;regs;15;none;none" # max CPU cores in baselineMachine (15)
171+
- "3600000;regs;24;none;none"
172+
- "3600000;regs;32;none;none" # max HT cores in baselineMachine (32)
173+
#
174+
# l1cache: is used to stress CPU level 1 cache with reads and writes
175+
- "2000000;l1cache;4;none;none"
176+
- "2000000;l1cache;8;none;none"
177+
- "2000000;l1cache;15;none;none" # max CPU cores in baselineMachine (15)
178+
- "2000000;l1cache;24;none;none"
179+
- "2000000;l1cache;32;none;none" # max HT cores in baselineMachine (32)
180+
- "2800000;l1cache;4;none;none"
181+
- "2800000;l1cache;8;none;none"
182+
- "2800000;l1cache;15;none;none" # max CPU cores in baselineMachine (15)
183+
- "2800000;l1cache;24;none;none"
184+
- "2800000;l1cache;32;none;none" # max HT cores in baselineMachine (32)
185+
- "3600000;l1cache;4;none;none"
186+
- "3600000;l1cache;8;none;none"
187+
- "3600000;l1cache;15;none;none" # max CPU cores in baselineMachine (15)
188+
- "3600000;l1cache;24;none;none"
189+
- "3600000;l1cache;32;none;none" # max HT cores in baselineMachine (32)
190+
#
191+
# cache: is used to stress the CPU cache with random wide spread memory read and writes to thrash the CPU cache
192+
- "2000000;cache;4;none;none"
193+
- "2000000;cache;8;none;none"
194+
- "2000000;cache;15;none;none" # max CPU cores in baselineMachine (15)
195+
- "2000000;cache;24;none;none"
196+
- "2000000;cache;32;none;none" # max HT cores in baselineMachine (32)
197+
- "2800000;cache;4;none;none"
198+
- "2800000;cache;8;none;none"
199+
- "2800000;cache;15;none;none" # max CPU cores in baselineMachine (15)
200+
- "2800000;cache;24;none;none"
201+
- "2800000;cache;32;none;none" # max HT cores in baselineMachine (32)
202+
- "3600000;cache;4;none;none"
203+
- "3600000;cache;8;none;none"
204+
- "3600000;cache;15;none;none" # max CPU cores in baselineMachine (15)
205+
- "3600000;cache;24;none;none"
206+
- "3600000;cache;32;none;none" # max HT cores in baselineMachine (32)
207+
#
208+
# stream: "Sustainable Memory Bandwidth in High Performance Computers" benchmarking tool by John D. McCalpin
209+
- "2000000;stream;4;none;none"
210+
- "2000000;stream;8;none;none"
211+
- "2000000;stream;15;none;none" # max CPU cores in baselineMachine (15)
212+
- "2000000;stream;24;none;none"
213+
- "2000000;stream;32;none;none" # max HT cores in baselineMachine (32)
214+
- "2800000;stream;4;none;none"
215+
- "2800000;stream;8;none;none"
216+
- "2800000;stream;15;none;none" # max CPU cores in baselineMachine (15)
217+
- "2800000;stream;24;none;none"
218+
- "2800000;stream;32;none;none" # max HT cores in baselineMachine (32)
219+
- "3600000;stream;4;none;none"
220+
- "3600000;stream;8;none;none"
221+
- "3600000;stream;15;none;none" # max CPU cores in baselineMachine (15)
222+
- "3600000;stream;24;none;none"
223+
- "3600000;stream;32;none;none" # max HT cores in baselineMachine (32)
224+
#
225+
# A common recommendation is to use around 80-90% of the available memory for stress testing.
226+
# The baselineMachine has 20Gi free, we make two tests 80%
227+
# --vm-rw: is used to stress the virtual memory subsystem by allocating memory pages and continuously
228+
# writing and reading data to and from them. This simulates a scenario where memory is frequently used
229+
# and modified. This test stress both memory allocation and data access.
230+
- "2000000;vm-rw;4;vm-rw-bytes;16G"
231+
- "2000000;vm-rw;8;vm-rw-bytes;16G"
232+
- "2000000;vm-rw;15;vm-rw-bytes;16G" # max CPU cores in baselineMachine (15)
233+
- "2000000;vm-rw;24;vm-rw-bytes;16G"
234+
- "2000000;vm-rw;32;vm-rw-bytes;16G" # max HT cores in baselineMachine (32)
235+
- "2800000;vm-rw;4;vm-rw-bytes;16G"
236+
- "2800000;vm-rw;8;vm-rw-bytes;16G"
237+
- "2800000;vm-rw;15;vm-rw-bytes;16G" # max CPU cores in baselineMachine (15)
238+
- "2800000;vm-rw;24;vm-rw-bytes;16G"
239+
- "2800000;vm-rw;32;vm-rw-bytes;16G" # max HT cores in baselineMachine (32)
240+
- "3600000;vm-rw;4;vm-rw-bytes;16G"
241+
- "3600000;vm-rw;8;vm-rw-bytes;16G"
242+
- "3600000;vm-rw;15;vm-rw-bytes;16G" # max CPU cores in baselineMachine (15)
243+
- "3600000;vm-rw;24;vm-rw-bytes;16G"
244+
- "3600000;vm-rw;32;vm-rw-bytes;16G" # max HT cores in baselineMachine (32)
245+
#
246+
# --iomix: is used to stress a mix of sequential, random and memory mapped read/write operations as
247+
# well as random copy file read/writes, forced sync'ing and (if run as root) cache dropping.
248+
- "2000000;iomix;4;none;none"
249+
- "2000000;iomix;8;none;none"
250+
- "2000000;iomix;15;none;none" # max CPU cores in baselineMachine (15)
251+
- "2000000;iomix;24;none;none"
252+
- "2000000;iomix;32;none;none" # max HT cores in baselineMachine (32)
253+
- "2800000;iomix;4;none;none"
254+
- "2800000;iomix;8;none;none"
255+
- "2800000;iomix;15;none;none" # max CPU cores in baselineMachine (15)
256+
- "2800000;iomix;24;none;none"
257+
- "2800000;iomix;32;none;none" # max HT cores in baselineMachine (32)
258+
- "3600000;iomix;4;none;none"
259+
- "3600000;iomix;8;none;none"
260+
- "3600000;iomix;15;none;none" # max CPU cores in baselineMachine (15)
261+
- "3600000;iomix;24;none;none"
262+
- "3600000;iomix;32;none;none" # max HT cores in baselineMachine (32)
263+
#
264+
# pipe: is used to stress pipe write operations
265+
- "2000000;pipe;4;none;none"
266+
- "2000000;pipe;8;none;none"
267+
- "2000000;pipe;15;none;none" # max CPU cores in baselineMachine (15)
268+
- "2000000;pipe;24;none;none"
269+
- "2000000;pipe;32;none;none" # max HT cores in baselineMachine (32)
270+
- "2800000;pipe;4;none;none"
271+
- "2800000;pipe;8;none;none"
272+
- "2800000;pipe;15;none;none" # max CPU cores in baselineMachine (15)
273+
- "2800000;pipe;24;none;none"
274+
- "2800000;pipe;32;none;none" # max HT cores in baselineMachine (32)
275+
- "3600000;pipe;4;none;none"
276+
- "3600000;pipe;8;none;none"
277+
- "3600000;pipe;15;none;none" # max CPU cores in baselineMachine (15)
278+
- "3600000;pipe;24;none;none"
279+
- "3600000;pipe;32;none;none" # max HT cores in baselineMachine (32)
280+
#
281+
# sctp: is used to stress the network performing SCTP send/receives
282+
- "2000000;sctp;4;none;none"
283+
- "2000000;sctp;8;none;none"
284+
- "2000000;sctp;15;none;none" # max CPU cores in baselineMachine (15)
285+
- "2000000;sctp;24;none;none"
286+
- "2000000;sctp;32;none;none" # max HT cores in baselineMachine (32)
287+
- "2800000;sctp;4;none;none"
288+
- "2800000;sctp;8;none;none"
289+
- "2800000;sctp;15;none;none" # max CPU cores in baselineMachine (15)
290+
- "2800000;sctp;24;none;none"
291+
- "2800000;sctp;32;none;none" # max HT cores in baselineMachine (32)
292+
- "3600000;sctp;4;none;none"
293+
- "3600000;sctp;8;none;none"
294+
- "3600000;sctp;15;none;none" # max CPU cores in baselineMachine (15)
295+
- "3600000;sctp;24;none;none"
296+
- "3600000;sctp;32;none;none" # max HT cores in baselineMachine (32)
72297
sequential: true

0 commit comments

Comments
 (0)