pangeo_hpc/instantiate_pangeo_function.sh at master · nicpittman/pangeo_hpc · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
#!/bin/bash -l

# ====================================
# Append the pangeo function to a user's .bashrc
#
#    USAGE: ./instantiate_pangeo_function
#
#    Dougie Squire
#    20/08/2020
# ====================================

APPEND_TO=~/.bashrc

if [[ "$HOSTNAME" == *"gadi"* ]]; then
    SYSTEM=gadi
elif [[ "$HOSTNAME" == *"pearcey"* ]]; then
    SYSTEM=pearcey
elif [[ "$HOSTNAME" == *"zeus"* ]]; then
    SYSTEM=zeus
else
    echo "Cannot determine system" 1>&2
fi

if [[ "$SYSTEM" == "gadi" ]]; then
    PANGEO_FUNCTION='function pangeo {
    WALLTIME=${1:-"02:00:00"}
    NCPUS=${2:-"4"}
    MEM=${3:-"16GB"}
    PROJECT=${4:-"'$PROJECT'"}
    PANGEO_ENV_NAME=${5:-"pangeo"}
    NOTEBOOK_DIR=${6:-"~"}
    PANGEO_RUN_SCRIPT_DIR="'$(pwd)'"

    rm -f jupyter_instructions.txt

    jobid=$(qsub -l walltime=${WALLTIME} -l mem=${MEM} -l ncpus=${NCPUS} -P ${PROJECT} \
    	-v "NOTEBOOK_DIR=${NOTEBOOK_DIR},RUN_SCRIPT_DIR=${PANGEO_RUN_SCRIPT_DIR},PANGEO_ENV_NAME=${PANGEO_ENV_NAME}" \
	${PANGEO_RUN_SCRIPT_DIR}/start_jupyter_Gadi.sh)
    while [ ! -f jupyter_instructions.txt ]; do
        sleep 1
    done
    trap '\'' '\'' INT
    tail -f -n 50 jupyter_instructions.txt

    echo "Closing ${jobid}"
    qdel ${jobid}
    trap - INT
}'
elif [[ "$SYSTEM" == "pearcey" ]]; then
    PANGEO_FUNCTION='function pangeo {
    WALLTIME=${1:-"02:00:00"}
    NCPUS=${2:-"4"}
    MEM=${3:-"6GB"}
    PANGEO_ENV_NAME=${4:-"pangeo"}
    NOTEBOOK_DIR=${5:-"~"}
    PANGEO_RUN_SCRIPT_DIR="'$(pwd)'"

    rm -f jupyter_instructions.txt

    jobid=$(sbatch --time=${WALLTIME} --mem-per-cpu=${MEM} --cpus-per-task=${NCPUS} \
    	--export "NOTEBOOK_DIR=${NOTEBOOK_DIR},RUN_SCRIPT_DIR=${PANGEO_RUN_SCRIPT_DIR},PANGEO_ENV_NAME=${PANGEO_ENV_NAME}" \
	${PANGEO_RUN_SCRIPT_DIR}/start_jupyter_Pearcey.sh | sed '\''s/[^0-9]*//g'\'')
    while [ ! -f jupyter_instructions.txt ]; do
        sleep 1
    done
    trap '\'' '\'' INT
    tail -f -n 50 jupyter_instructions.txt

    echo "Closing ${jobid}"
    scancel ${jobid}
    trap - INT
}'
elif [[ "$SYSTEM" == "zeus" ]]; then
    PANGEO_FUNCTION='function pangeo {
    WALLTIME=${1:-"02:00:00"}
    NCPUS=${2:-"4"}
    MEM=${3:-"4GB"}
    PROJECT=${4:-"'$PAWSEY_PROJECT'"}
    PANGEO_ENV_NAME=${5:-"pangeo"}
    NOTEBOOK_DIR=${6:-"~"}
    PANGEO_RUN_SCRIPT_DIR="'$(pwd)'"

    rm -f jupyter_instructions.txt

    jobid=$(sbatch --time=${WALLTIME} --mem-per-cpu=${MEM} --cpus-per-task=${NCPUS} --account=${PROJECT} \
    	--export "NOTEBOOK_DIR=${NOTEBOOK_DIR},RUN_SCRIPT_DIR=${PANGEO_RUN_SCRIPT_DIR},PANGEO_ENV_NAME=${PANGEO_ENV_NAME}" \
	${PANGEO_RUN_SCRIPT_DIR}/start_jupyter_Zeus.sh | sed '\''s/[^0-9]*//g'\'')
    while [ ! -f jupyter_instructions.txt ]; do
        sleep 1
    done
    trap '\'' '\'' INT
    tail -f -n 50 jupyter_instructions.txt

    echo "Closing ${jobid}"
    scancel ${jobid}
    trap - INT
}'
fi

if grep -Fxq "function pangeo {" ${APPEND_TO}; then
	echo "There is already a pangeo function in ${APPEND_TO}. Please delete this and rerun."
else
	echo "$PANGEO_FUNCTION" >> ${APPEND_TO}
	source  ${APPEND_TO}
	echo "Appended pangeo convenience function to ${APPEND_TO}"
fi