Skip to content

Commit a830e9e

Browse files
vancraarvancraar
committed
Add a UFTP script for copying with multiple NG UFTP nodes
1 parent 6103381 commit a830e9e

File tree

2 files changed

+218
-0
lines changed

2 files changed

+218
-0
lines changed

‎third_level_manager/copy_file_from_hawk_to_ng_parallel.sh‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
#!/bin/bash
2+
# execute this script on skx-arch.supermuc.lrz.de
3+
# as described on https://doku.lrz.de/display/PUBLIC/Data+Transfer+Options+on+SuperMUC-NG#DataTransferOptionsonSuperMUCNG-UNICOREFileTransfer(UFTP)
4+
5+
module load uftp-client
6+
7+
8+
# kill all processes from this script if the script is terminated
9+
trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT
10+
11+
ID=${ID:=0}
12+
PATHLRZ=${PATHLRZ:=//hppfs/work/pn36xu/di93yuw/uftp}
13+
PATHHAWK=${PATHHAWK:=/lustre/hpe/ws10/ws10.3/ws/ipvsavcr-discotec/wd_alex/scaled_down_target_to_try_64/}
14+
FILEHAWK=${FILEHAWK:=${PATHHAWK}/dsg_${ID}_step*_0}
15+
HAWKURL=https://gridftp-fr1.hww.hlrs.de:9000/rest/auth/HLRS
16+
USERHAWK=${USERHAWK:=ipvsavcr}
17+
num_hosts=3
18+
19+
TOKEN_TRANSFER_BACKWARD=${FILEHAWK:0:-2}_token.txt
20+
TOKEN_STOP=uftp_transfer_stop.txt
21+
22+
PROCS=${PROCS:=16}
23+
THREADS_PER_PROC=${THREADS_PER_PROC:=8}
24+
STREAMS=${STREAMS:=3}
25+
26+
echo "$FILEHAWK"
27+
echo "$TOKEN_TRANSFER_BACKWARD"
28+
29+
step=0
30+
31+
# loop that is infinite until $TOKEN_STOP is created
32+
while true
33+
do
34+
FILEHAWK=${PATHHAWK}/dsg_${ID}_step${step}_0
35+
TOKEN_TRANSFER_BACKWARD=${FILEHAWK:0:-2}_token.txt
36+
# test if the file is there by trying to copy it (maybe there is a better way?)
37+
# the part after `>` is to ignore the potential (error) output
38+
uftp cp -i ~/.uftp/id_uftp_to_hlrs -u $USERHAWK $HAWKURL:$TOKEN_TRANSFER_BACKWARD /dev/null > /dev/null 2>&1
39+
cp_status=$?
40+
pids=( )
41+
if (exit $cp_status); then
42+
# ./copy_hawk_to_sng.sh
43+
# try to copy the file
44+
FILEHAWK_INSTANCE=$(echo $TOKEN_TRANSFER_BACKWARD)
45+
FILEHAWK_INSTANCE=${FILEHAWK_INSTANCE:0:-10}_0
46+
echo copy $FILEHAWK_INSTANCE from hawk to NG
47+
size=$(uftp ls -i ~/.uftp/id_uftp_to_hlrs -u $USERHAWK $HAWKURL:$FILEHAWK_INSTANCE | awk '{print $2 }')
48+
local_size=$((size/num_hosts))
49+
host=$(hostname)
50+
if [ $host == "login05" ]; then
51+
local_start=0
52+
elif [ $host == "login06" ]; then
53+
local_start=$((local_size))
54+
elif [ $host == "login07" ]; then
55+
local_start=$((local_size*2))
56+
fi
57+
startblock=$local_start
58+
endblock=$((local_start+(local_size/PROCS)))
59+
starttime=`date +%s`
60+
for ((i=1; i<=PROCS; i++)); do
61+
echo "Block: $startblock-$endblock of $size"
62+
uftp cp -B "${startblock}-${endblock}-p" -i ~/.uftp/id_uftp_to_hlrs -u $USERHAWK $HAWKURL:$FILEHAWK_INSTANCE $PATHLRZ/ &
63+
pids+=($!)
64+
startblock=$((endblock))
65+
if [ $i -eq $((PROCS-1)) ]; then
66+
if [ $host == "login07" ]; then
67+
endblock=$((size))
68+
else
69+
endblock=$((local_start+local_size))
70+
fi
71+
else
72+
endblock=$((endblock+local_size/PROCS))
73+
fi
74+
done
75+
# wait for all processes to finish
76+
for pid in "${pids[@]}"; do
77+
wait $pid || exit
78+
done
79+
touch dsg_${ID}_step${step}_0_${host}
80+
if [ $host == "login05" ]; then
81+
while [ ! -f dsg_${ID}_step${step}_0_login05 ] || [ ! -f dsg_${ID}_step${step}_0_login06 ] || [ ! -f dsg_${ID}_step${step}_0_login07 ]; do
82+
sleep 1
83+
done
84+
rm dsg_${ID}_step${step}_0_login05
85+
rm dsg_${ID}_step${step}_0_login06
86+
rm dsg_${ID}_step${step}_0_login07
87+
uftp cp -i ~/.uftp/id_uftp_to_hlrs -u $USERHAWK $HAWKURL:$TOKEN_TRANSFER_BACKWARD $PATHLRZ/
88+
uftp rm --quiet -i ~/.uftp/id_uftp_to_hlrs -u $USERHAWK "$HAWKURL:$TOKEN_TRANSFER_BACKWARD"
89+
fi
90+
91+
endtime=`date +%s`
92+
echo "copied $local_size bites from file $FILEHAWK_INSTANCE in `expr $endtime - $starttime` seconds."
93+
throughput=$( echo "scale=4;($local_size/1024/1024)/(($endtime-$starttime))" | bc )
94+
throughput_bits=$( echo "scale=4;($throughput*8)" | bc )
95+
echo "Local average throughput: $throughput MB/s; $throughput_bits Mbit/s"
96+
if [ $host == "login05" ]; then
97+
throughput=$( echo "scale=4;($size/1024/1024)/(($endtime-$starttime))" | bc )
98+
throughput_bits=$( echo "scale=4;($throughput*8)" | bc )
99+
echo "Approx total average throughput: $throughput MB/s; $throughput_bits Mbit/s"
100+
uftp rm -q -i ~/.uftp/id_uftp_to_hlrs -u $USERHAWK $HAWKURL:$FILEHAWK_INSTANCE
101+
fi
102+
date
103+
step=$((step+1))
104+
fi
105+
if test -f "$PATHLRZ/$TOKEN_STOP"; then
106+
break
107+
fi
108+
sleep 1
109+
done

‎third_level_manager/copy_file_from_ng_to_hawk_parallel.sh‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
#!/bin/bash
2+
# execute this script on skx-arch.supermuc.lrz.de
3+
# as described on https://doku.lrz.de/display/PUBLIC/Data+Transfer+Options+on+SuperMUC-NG#DataTransferOptionsonSuperMUCNG-UNICOREFileTransfer(UFTP)
4+
5+
module load uftp-client
6+
7+
set -e
8+
9+
# kill all processes from this script if the script is terminated
10+
trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT
11+
12+
ID=${ID:=0}
13+
PATHLRZ=${PATHLRZ:=//hppfs/work/pn36xu/di93yuw/uftp}
14+
PATHHAWK=${PATHHAWK:=/lustre/hpe/ws10/ws10.3/ws/ipvsavcr-discotec/wd_alex/scaled_down_target_to_try_64/}
15+
FILELRZ=${FILELRZ:=${PATHLRZ}/dsg_${ID}_step*_0}
16+
HAWKURL=https://gridftp-fr1.hww.hlrs.de:9000/rest/auth/HLRS
17+
USERHAWK=${USERHAWK:=ipvsavcr}
18+
num_hosts=3
19+
20+
21+
TOKEN_TRANSFER_FORWARD=${FILELRZ:0:-2}_token.txt
22+
TOKEN_STOP=uftp_transfer_stop.txt
23+
24+
PROCS=${PROCS:=16}
25+
THREADS_PER_PROC=${THREADS_PER_PROC:=8}
26+
STREAMS=${STREAMS:=3}
27+
28+
echo "$FILELRZ"
29+
echo "$TOKEN_TRANSFER_FORWARD"
30+
31+
step=0
32+
33+
# loop that is infinite until $TOKEN_STOP is created
34+
while true
35+
do
36+
FILELRZ=${PATHLRZ}/dsg_${ID}_step${step}_0
37+
TOKEN_TRANSFER_FORWARD=${FILELRZ:0:-2}_token.txt
38+
39+
if test -f $TOKEN_TRANSFER_FORWARD; then
40+
# ./copy_sng_to_hawk.sh
41+
# try to copy the file
42+
FILELRZ_INSTANCE=$(echo $TOKEN_TRANSFER_FORWARD)
43+
FILELRZ_INSTANCE=${FILELRZ_INSTANCE:0:-10}_0
44+
echo instance "$FILELRZ_INSTANCE"
45+
size=$(stat --printf="%s" $FILELRZ_INSTANCE)
46+
local_size=$((size/num_hosts))
47+
host=$(hostname)
48+
if [ $host == "login05" ]; then
49+
local_start=0
50+
elif [ $host == "login06" ]; then
51+
local_start=$((local_size))
52+
elif [ $host == "login07" ]; then
53+
local_start=$((local_size*2))
54+
fi
55+
startblock=$local_start
56+
endblock=$((local_start+(local_size/PROCS)))
57+
starttime=`date +%s`
58+
pids=( )
59+
for ((i=1; i<=PROCS; i++)); do
60+
echo "Block: $startblock-$endblock of $size"
61+
uftp cp -B "${startblock}-${endblock}-p" -i ~/.uftp/id_uftp_to_hlrs -u $USERHAWK $FILELRZ_INSTANCE $HAWKURL:$PATHHAWK/ &
62+
pids+=($!) # store background pids
63+
startblock=$((endblock))
64+
if [ $i -eq $((PROCS-1)) ]; then
65+
if [ $host == "login07" ]; then
66+
endblock=$((size))
67+
else
68+
endblock=$((local_start+local_size))
69+
fi
70+
else
71+
endblock=$((endblock+local_size/PROCS))
72+
fi
73+
done
74+
# wait for all pids
75+
for pid in "${pids[@]}"; do
76+
wait "$pid" || exit
77+
done
78+
touch "${TOKEN_TRANSFER_FORWARD}_${host}"
79+
if [ $host == "login05" ]; then
80+
while [ ! -f ${TOKEN_TRANSFER_FORWARD}_login05 ] || [ ! -f ${TOKEN_TRANSFER_FORWARD}_login06 ] || [ ! -f ${TOKEN_TRANSFER_FORWARD}_login07 ]; do
81+
sleep 1
82+
done
83+
rm ${TOKEN_TRANSFER_FORWARD}_login05
84+
rm ${TOKEN_TRANSFER_FORWARD}_login06
85+
rm ${TOKEN_TRANSFER_FORWARD}_login07
86+
uftp cp -i ~/.uftp/id_uftp_to_hlrs -u $USERHAWK $TOKEN_TRANSFER_FORWARD $HAWKURL:$PATHHAWK/
87+
rm -f $TOKEN_TRANSFER_FORWARD
88+
fi
89+
90+
endtime=`date +%s`
91+
echo "copied $local_size bites from file "$FILELRZ_INSTANCE" in `expr $endtime - $starttime` seconds."
92+
93+
throughput=$( echo "scale=4;($local_size/1024/1024)/(($endtime-$starttime))" | bc )
94+
throughput_bits=$( echo "scale=4;($throughput*8)" | bc )
95+
echo "Local average throughput: $throughput MB/s; $throughput_bits Mbit/s"
96+
if [ $host == "login05" ]; then
97+
throughput=$( echo "scale=4;($size/1024/1024)/(($endtime-$starttime))" | bc )
98+
throughput_bits=$( echo "scale=4;($throughput*8)" | bc )
99+
echo "Approx total average throughput: $throughput MB/s; $throughput_bits Mbit/s"
100+
rm -f $FILELRZ_INSTANCE
101+
fi
102+
date
103+
step=$((step+1))
104+
fi
105+
if test -f "$PATHLRZ/$TOKEN_STOP"; then
106+
break
107+
fi
108+
sleep 1
109+
done

0 commit comments

Comments
 (0)