Merge pull request #84 from converged-computing/redo-osu

osu: fix runs for gpu 128 GKE and CE

Author: vsoch

experiments/google/README.md

@@ -1,6 +1,10 @@
 # Google Cloud Experiments
 
-This directory will hold experiments for Google Cloud.
+This directory will hold experiments for Google Cloud. 
 
 - [gke/cpu](gke/cpu): CPU experiments for Google Kubernetes Engine
 - [gke/gpu](gpu/gpu): GPU experiments for Google Kubernetes Engine
+
+## OSU Benchmarks
+
+Note that for our first experiments, we ran OSU across sizes, however we made the mistake of using flux submit for the GPU runs, which isn't blocking, and meant that osu latency would interfere with bandwidth. To fix this we re-ran GPU for sizes 16 nodes for each of Compute Engine and GKE. This size is what should be used for data analysis.

experiments/google/compute-engine/gpu/size16/README.md

@@ -237,6 +237,7 @@ oras push ghcr.io/converged-computing/metrics-operator-experiments/performance:c
 #### OSU
 
 Write this script to the filesystem `flux-run-combinations.sh`
+Note that the initial study in August 2024 used flux submit, which isn't blocking (and erroneous since they run at the same time). We redid this size in March 2025 with flux run, ensuring it would be blocking.
 
 ```bash
 #/bin/bash
@@ -258,13 +259,13 @@ for i in $hosts; do
   dequeue_from_list $list
   for j in $list; do
     echo "${i} ${j}"
-    flux submit -N 2 -n 2 \
+    flux run -N 2 -n 2 \
       --setattr=user.study_id=$app-2-iter-$iter \
       --requires="hosts:${i},${j}" \
       -o cpu-affinity=per-task \
       -g 1 -o gpu-affinity=per-task \
       singularity exec --nv /opt/containers/metric-osu-gpu_google-gpu.sif /opt/osu-benchmark/build.openmpi/mpi/pt2pt/osu_latency -d cuda H H
-    flux submit -N 2 -n 2 \
+    flux run -N 2 -n 2 \
       --setattr=user.study_id=$app-2-iter-$iter \
       --requires="hosts:${i},${j}" \
       -o cpu-affinity=per-task \
@@ -298,7 +299,12 @@ done
 
 # When they are done:
 ./save.sh $output
+
+# August 2024
 oras push ghcr.io/converged-computing/metrics-operator-experiments/performance:compute-engine-gpu-16-$app $output
+
+# March 2025 with fixed osu latency and osu bw
+oras push ghcr.io/converged-computing/metrics-operator-experiments/performance:compute-engine-gpu-16-$app-fixed $output
 ```
 
 #### Quicksilver

experiments/google/compute-engine/gpu/size16/results/osu-fixed/osu-2-iter-0-1051964997632.out

@@ -0,0 +1,44 @@
+Warning: OMB could not identify the local rank of the process.
+         This can lead to multiple processes using the same GPU.
+         Please use the get_local_rank script in the OMB repo for this.
+Warning: OMB could not identify the local rank of the process.
+         This can lead to multiple processes using the same GPU.
+         Please use the get_local_rank script in the OMB repo for this.
+# OSU MPI-CUDA Latency Test v5.8
+# Send Buffer on HOST (H) and Receive Buffer on HOST (H)
+# Size          Latency (us)
+0                      24.71
+1                      24.37
+2                      24.61
+4                      24.51
+8                      24.43
+16                     24.39
+32                     24.23
+64                     24.40
+128                    24.19
+256                    24.53
+512                    24.81
+1024                   25.39
+2048                   29.46
+4096                   31.57
+8192                   36.56
+16384                  43.71
+32768                  66.71
+65536                 174.45
+131072                186.18
+262144                235.77
+524288                349.58
+1048576               564.08
+2097152               970.02
+4194304              1868.56
+START OF JOBSPEC
+{"resources": [{"type": "node", "count": 2, "with": [{"type": "slot", "count": 1, "with": [{"type": "core", "count": 1}, {"type": "gpu", "count": 1}], "label": "task"}]}], "tasks": [{"command": ["singularity", "exec", "--nv", "/opt/containers/metric-osu-gpu_google-gpu.sif", "/opt/osu-benchmark/build.openmpi/mpi/pt2pt/osu_latency", "-d", "cuda", "H", "H"], "slot": "task", "count": {"per_slot": 1}}], "attributes": {"system": {"duration": 0, "cwd": "/opt/containers", "shell": {"options": {"rlimit": {"cpu": -1, "fsize": -1, "data": -1, "stack": 8388608, "core": 0, "nofile": 1048576, "as": -1, "rss": -1, "nproc": -1}, "cpu-affinity": "per-task", "gpu-affinity": "per-task"}}, "constraints": {"hostlist": ["flux-009,flux-006"]}}, "user": {"study_id": "osu-2-iter-0"}}, "version": 1}
+START OF EVENTLOG
+{"timestamp":1742925379.8451526,"name":"init"}
+{"timestamp":1742925379.8460701,"name":"starting"}
+{"timestamp":1742925379.8658693,"name":"shell.init","context":{"service":"501043911-shell-fUdjYatP","leader-rank":5,"size":2}}
+{"timestamp":1742925379.9225433,"name":"shell.start","context":{"taskmap":{"version":1,"map":[[0,2,1,1]]}}}
+{"timestamp":1742925405.0883598,"name":"shell.task-exit","context":{"localid":0,"rank":0,"state":"Exited","pid":2472,"wait_status":0,"signaled":0,"exitcode":0}}
+{"timestamp":1742925405.1001337,"name":"complete","context":{"status":0}}
+{"timestamp":1742925405.1001668,"name":"done"}
+

experiments/google/compute-engine/gpu/size16/results/osu-fixed/osu-2-iter-0-1480337653760.out

@@ -0,0 +1,43 @@
+Warning: OMB could not identify the local rank of the process.
+         This can lead to multiple processes using the same GPU.
+         Please use the get_local_rank script in the OMB repo for this.
+Warning: OMB could not identify the local rank of the process.
+         This can lead to multiple processes using the same GPU.
+         Please use the get_local_rank script in the OMB repo for this.
+# OSU MPI-CUDA Bandwidth Test v5.8
+# Send Buffer on HOST (H) and Receive Buffer on HOST (H)
+# Size      Bandwidth (MB/s)
+1                       0.18
+2                       0.39
+4                       0.68
+8                       1.48
+16                      3.26
+32                      6.08
+64                     20.84
+128                    40.89
+256                    78.46
+512                   152.48
+1024                  295.84
+2048                  539.71
+4096                  878.73
+8192                 1315.63
+16384                1656.37
+32768                1967.43
+65536                2194.64
+131072               2150.22
+262144               2056.25
+524288               1924.40
+1048576              1853.76
+2097152              1761.99
+4194304              1715.88
+START OF JOBSPEC
+{"resources": [{"type": "node", "count": 2, "with": [{"type": "slot", "count": 1, "with": [{"type": "core", "count": 1}, {"type": "gpu", "count": 1}], "label": "task"}]}], "tasks": [{"command": ["singularity", "exec", "--nv", "/opt/containers/metric-osu-gpu_google-gpu.sif", "/opt/osu-benchmark/build.openmpi/mpi/pt2pt/osu_bw", "-d", "cuda", "H", "H"], "slot": "task", "count": {"per_slot": 1}}], "attributes": {"system": {"duration": 0, "cwd": "/opt/containers", "shell": {"options": {"rlimit": {"cpu": -1, "fsize": -1, "data": -1, "stack": 8388608, "core": 0, "nofile": 1048576, "as": -1, "rss": -1, "nproc": -1}, "cpu-affinity": "per-task", "gpu-affinity": "per-task"}}, "constraints": {"hostlist": ["flux-009,flux-006"]}}, "user": {"study_id": "osu-2-iter-0"}}, "version": 1}
+START OF EVENTLOG
+{"timestamp":1742925405.3776855,"name":"init"}
+{"timestamp":1742925405.3786778,"name":"starting"}
+{"timestamp":1742925405.3981018,"name":"shell.init","context":{"service":"501043911-shell-fftPNRGs","leader-rank":5,"size":2}}
+{"timestamp":1742925405.4020319,"name":"shell.start","context":{"taskmap":{"version":1,"map":[[0,2,1,1]]}}}
+{"timestamp":1742925414.4804258,"name":"shell.task-exit","context":{"localid":0,"rank":0,"state":"Exited","pid":2525,"wait_status":0,"signaled":0,"exitcode":0}}
+{"timestamp":1742925414.4866488,"name":"complete","context":{"status":0}}
+{"timestamp":1742925414.4866798,"name":"done"}
+

experiments/google/compute-engine/gpu/size16/results/osu-fixed/osu-2-iter-1-1637775048704.out

@@ -0,0 +1,44 @@
+Warning: OMB could not identify the local rank of the process.
+         This can lead to multiple processes using the same GPU.
+         Please use the get_local_rank script in the OMB repo for this.
+Warning: OMB could not identify the local rank of the process.
+         This can lead to multiple processes using the same GPU.
+         Please use the get_local_rank script in the OMB repo for this.
+# OSU MPI-CUDA Latency Test v5.8
+# Send Buffer on HOST (H) and Receive Buffer on HOST (H)
+# Size          Latency (us)
+0                      24.53
+1                      24.95
+2                      25.18
+4                      25.36
+8                      24.93
+16                     24.43
+32                     24.97
+64                     24.56
+128                    25.23
+256                    26.02
+512                    25.40
+1024                   26.45
+2048                   29.49
+4096                   31.35
+8192                   36.66
+16384                  45.80
+32768                  67.06
+65536                 181.80
+131072                189.26
+262144                238.14
+524288                356.01
+1048576               562.24
+2097152               965.45
+4194304              1708.20
+START OF JOBSPEC
+{"resources": [{"type": "node", "count": 2, "with": [{"type": "slot", "count": 1, "with": [{"type": "core", "count": 1}, {"type": "gpu", "count": 1}], "label": "task"}]}], "tasks": [{"command": ["singularity", "exec", "--nv", "/opt/containers/metric-osu-gpu_google-gpu.sif", "/opt/osu-benchmark/build.openmpi/mpi/pt2pt/osu_latency", "-d", "cuda", "H", "H"], "slot": "task", "count": {"per_slot": 1}}], "attributes": {"system": {"duration": 0, "cwd": "/opt/containers", "shell": {"options": {"rlimit": {"cpu": -1, "fsize": -1, "data": -1, "stack": 8388608, "core": 0, "nofile": 1048576, "as": -1, "rss": -1, "nproc": -1}, "cpu-affinity": "per-task", "gpu-affinity": "per-task"}}, "constraints": {"hostlist": ["flux-009,flux-002"]}}, "user": {"study_id": "osu-2-iter-1"}}, "version": 1}
+START OF EVENTLOG
+{"timestamp":1742925414.761935,"name":"init"}
+{"timestamp":1742925414.7628424,"name":"starting"}
+{"timestamp":1742925414.7843826,"name":"shell.init","context":{"service":"501043911-shell-fk2FaDkj","leader-rank":1,"size":2}}
+{"timestamp":1742925414.8324976,"name":"shell.start","context":{"taskmap":{"version":1,"map":[[0,2,1,1]]}}}
+{"timestamp":1742925439.7674615,"name":"shell.task-exit","context":{"localid":0,"rank":1,"state":"Exited","pid":2329,"wait_status":0,"signaled":0,"exitcode":0}}
+{"timestamp":1742925439.7992711,"name":"complete","context":{"status":0}}
+{"timestamp":1742925439.7993052,"name":"done"}
+

experiments/google/compute-engine/gpu/size16/results/osu-fixed/osu-2-iter-1-2062540603392.out

@@ -0,0 +1,43 @@
+Warning: OMB could not identify the local rank of the process.
+         This can lead to multiple processes using the same GPU.
+         Please use the get_local_rank script in the OMB repo for this.
+Warning: OMB could not identify the local rank of the process.
+         This can lead to multiple processes using the same GPU.
+         Please use the get_local_rank script in the OMB repo for this.
+# OSU MPI-CUDA Bandwidth Test v5.8
+# Send Buffer on HOST (H) and Receive Buffer on HOST (H)
+# Size      Bandwidth (MB/s)
+1                       0.20
+2                       0.50
+4                       0.92
+8                       1.49
+16                      3.58
+32                      7.05
+64                     20.39
+128                    38.56
+256                    74.51
+512                   142.02
+1024                  271.58
+2048                  507.87
+4096                  829.86
+8192                 1384.97
+16384                1849.12
+32768                2149.54
+65536                2188.41
+131072               2194.09
+262144               1989.21
+524288               1882.12
+1048576              1843.67
+2097152              1829.40
+4194304              1928.82
+START OF JOBSPEC
+{"resources": [{"type": "node", "count": 2, "with": [{"type": "slot", "count": 1, "with": [{"type": "core", "count": 1}, {"type": "gpu", "count": 1}], "label": "task"}]}], "tasks": [{"command": ["singularity", "exec", "--nv", "/opt/containers/metric-osu-gpu_google-gpu.sif", "/opt/osu-benchmark/build.openmpi/mpi/pt2pt/osu_bw", "-d", "cuda", "H", "H"], "slot": "task", "count": {"per_slot": 1}}], "attributes": {"system": {"duration": 0, "cwd": "/opt/containers", "shell": {"options": {"rlimit": {"cpu": -1, "fsize": -1, "data": -1, "stack": 8388608, "core": 0, "nofile": 1048576, "as": -1, "rss": -1, "nproc": -1}, "cpu-affinity": "per-task", "gpu-affinity": "per-task"}}, "constraints": {"hostlist": ["flux-009,flux-002"]}}, "user": {"study_id": "osu-2-iter-1"}}, "version": 1}
+START OF EVENTLOG
+{"timestamp":1742925440.0800998,"name":"init"}
+{"timestamp":1742925440.0810995,"name":"starting"}
+{"timestamp":1742925440.1029232,"name":"shell.init","context":{"service":"501043911-shell-fwBQeiZ5","leader-rank":1,"size":2}}
+{"timestamp":1742925440.1071765,"name":"shell.start","context":{"taskmap":{"version":1,"map":[[0,2,1,1]]}}}
+{"timestamp":1742925448.7218251,"name":"shell.task-exit","context":{"localid":0,"rank":0,"state":"Exited","pid":2244,"wait_status":0,"signaled":0,"exitcode":0}}
+{"timestamp":1742925448.7313514,"name":"complete","context":{"status":0}}
+{"timestamp":1742925448.731385,"name":"done"}
+

experiments/google/compute-engine/gpu/size16/results/osu-fixed/osu-2-iter-10-6579504021504.out

@@ -0,0 +1,44 @@
+Warning: OMB could not identify the local rank of the process.
+         This can lead to multiple processes using the same GPU.
+         Please use the get_local_rank script in the OMB repo for this.
+Warning: OMB could not identify the local rank of the process.
+         This can lead to multiple processes using the same GPU.
+         Please use the get_local_rank script in the OMB repo for this.
+# OSU MPI-CUDA Latency Test v5.8
+# Send Buffer on HOST (H) and Receive Buffer on HOST (H)
+# Size          Latency (us)
+0                      30.16
+1                      30.22
+2                      30.47
+4                      30.38
+8                      30.69
+16                     30.46
+32                     30.60
+64                     30.74
+128                    30.68
+256                    31.60
+512                    31.79
+1024                   32.73
+2048                   36.24
+4096                   38.27
+8192                   43.85
+16384                  54.69
+32768                  80.31
+65536                 211.02
+131072                210.16
+262144                271.94
+524288                367.93
+1048576               591.96
+2097152               975.44
+4194304              1859.91
+START OF JOBSPEC
+{"resources": [{"type": "node", "count": 2, "with": [{"type": "slot", "count": 1, "with": [{"type": "core", "count": 1}, {"type": "gpu", "count": 1}], "label": "task"}]}], "tasks": [{"command": ["singularity", "exec", "--nv", "/opt/containers/metric-osu-gpu_google-gpu.sif", "/opt/osu-benchmark/build.openmpi/mpi/pt2pt/osu_latency", "-d", "cuda", "H", "H"], "slot": "task", "count": {"per_slot": 1}}], "attributes": {"system": {"duration": 0, "cwd": "/opt/containers", "shell": {"options": {"rlimit": {"cpu": -1, "fsize": -1, "data": -1, "stack": 8388608, "core": 0, "nofile": 1048576, "as": -1, "rss": -1, "nproc": -1}, "cpu-affinity": "per-task", "gpu-affinity": "per-task"}}, "constraints": {"hostlist": ["flux-006,flux-007"]}}, "user": {"study_id": "osu-2-iter-10"}}, "version": 1}
+START OF EVENTLOG
+{"timestamp":1742925709.3119636,"name":"init"}
+{"timestamp":1742925709.3130028,"name":"starting"}
+{"timestamp":1742925709.3332787,"name":"shell.init","context":{"service":"501043911-shell-f3yqHEv7h","leader-rank":5,"size":2}}
+{"timestamp":1742925709.3373182,"name":"shell.start","context":{"taskmap":{"version":1,"map":[[0,2,1,1]]}}}
+{"timestamp":1742925730.459857,"name":"shell.task-exit","context":{"localid":0,"rank":0,"state":"Exited","pid":2816,"wait_status":0,"signaled":0,"exitcode":0}}
+{"timestamp":1742925730.4663754,"name":"complete","context":{"status":0}}
+{"timestamp":1742925730.4664104,"name":"done"}
+

experiments/google/compute-engine/gpu/size16/results/osu-fixed/osu-2-iter-10-6939090092032.out

@@ -0,0 +1,43 @@
+Warning: OMB could not identify the local rank of the process.
+         This can lead to multiple processes using the same GPU.
+         Please use the get_local_rank script in the OMB repo for this.
+Warning: OMB could not identify the local rank of the process.
+         This can lead to multiple processes using the same GPU.
+         Please use the get_local_rank script in the OMB repo for this.
+# OSU MPI-CUDA Bandwidth Test v5.8
+# Send Buffer on HOST (H) and Receive Buffer on HOST (H)
+# Size      Bandwidth (MB/s)
+1                       0.17
+2                       0.34
+4                       0.79
+8                       1.05
+16                      3.45
+32                      8.12
+64                     21.08
+128                    43.15
+256                    77.26
+512                   151.05
+1024                  288.65
+2048                  524.34
+4096                  858.74
+8192                 1397.65
+16384                1808.67
+32768                2147.22
+65536                2343.34
+131072               2199.78
+262144               2087.49
+524288               2018.49
+1048576              1963.70
+2097152              1974.59
+4194304              1962.15
+START OF JOBSPEC
+{"resources": [{"type": "node", "count": 2, "with": [{"type": "slot", "count": 1, "with": [{"type": "core", "count": 1}, {"type": "gpu", "count": 1}], "label": "task"}]}], "tasks": [{"command": ["singularity", "exec", "--nv", "/opt/containers/metric-osu-gpu_google-gpu.sif", "/opt/osu-benchmark/build.openmpi/mpi/pt2pt/osu_bw", "-d", "cuda", "H", "H"], "slot": "task", "count": {"per_slot": 1}}], "attributes": {"system": {"duration": 0, "cwd": "/opt/containers", "shell": {"options": {"rlimit": {"cpu": -1, "fsize": -1, "data": -1, "stack": 8388608, "core": 0, "nofile": 1048576, "as": -1, "rss": -1, "nproc": -1}, "cpu-affinity": "per-task", "gpu-affinity": "per-task"}}, "constraints": {"hostlist": ["flux-006,flux-007"]}}, "user": {"study_id": "osu-2-iter-10"}}, "version": 1}
+START OF EVENTLOG
+{"timestamp":1742925730.7448459,"name":"init"}
+{"timestamp":1742925730.7459881,"name":"starting"}
+{"timestamp":1742925730.7653837,"name":"shell.init","context":{"service":"501043911-shell-f49H8dX6P","leader-rank":5,"size":2}}
+{"timestamp":1742925730.7693317,"name":"shell.start","context":{"taskmap":{"version":1,"map":[[0,2,1,1]]}}}
+{"timestamp":1742925739.0274324,"name":"shell.task-exit","context":{"localid":0,"rank":0,"state":"Exited","pid":2856,"wait_status":0,"signaled":0,"exitcode":0}}
+{"timestamp":1742925739.032778,"name":"complete","context":{"status":0}}
+{"timestamp":1742925739.0328097,"name":"done"}
+