Skip to content

Commit 2190c87

Browse files
hateganhategan
committed
- renamed internal psij functions to have a _psij_ prefix in order to avoid
clashes with other things that might be defined in the environment. - use `StageOutFlags` to express when cleanup happens rather than a boolean flag - fixed equality operator in `JobSpec` - added `__str__`, `__eq__` and `__hash__` methods for staging objects
1 parent 30d0d34 commit 2190c87

File tree

14 files changed

+121
-69
lines changed

14 files changed

+121
-69
lines changed

‎src/psij/executors/batch/cobalt/cobalt.mustache‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ PSIJ_NODEFILE="$COBALT_NODEFILE"
5959
export PSIJ_NODEFILE
6060

6161
{{> stagein}}
62-
update_status ACTIVE
62+
_psij_update_status ACTIVE
6363

6464
{{#psij.launch_command}}{{.}} {{/psij.launch_command}}
6565
_PSIJ_JOB_EC=$?

‎src/psij/executors/batch/common/batch_lib.mustache‎

Lines changed: 48 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
update_status() {
1+
_psij_update_status() {
22
STATUS="$1"
33
44
ADDRS={{psij.us_addrs}}
@@ -7,55 +7,55 @@ update_status() {
77
done
88
}
99

10-
fail() {
11-
[ "{{psij.debug}}" != "0" ] && update_status "LOG Failing: $2"
10+
_psij_fail() {
11+
[ "{{psij.debug}}" != "0" ] && _psij_update_status "LOG Failing: $2"
1212
echo $2
1313
exit $1
1414
}
1515

16-
check_remote() {
16+
_psij_check_remote() {
1717
SCHEME="$1"
1818
HOSTPORT="$2"
1919
2020
if [ "$SCHEME" != "" ] && [ "$SCHEME" != "file" ]; then
21-
fail 121 "$SCHEME staging is not supported"
21+
_psij_fail 121 "$SCHEME staging is not supported"
2222
fi
2323
if [ "$HOSTPORT" != "" ] && [ "$HOSTPORT" != "localhost" ]; then
24-
fail 121 "The host, if specified, must be \"localhost\". Got \"$HOSTPORT\"."
24+
_psij_fail 121 "The host, if specified, must be \"localhost\". Got \"$HOSTPORT\"."
2525
fi
2626
}
2727

28-
do_stagein() {
28+
_psij_do_stagein() {
2929
SOURCE="$1"
3030
TARGET="$2"
3131
MODE="$3"
3232
SCHEME="$6"
3333
HOSTPORT="$7"
3434
35-
check_remote "$SCHEME" "$HOSTPORT" || exit $?
35+
_psij_check_remote "$SCHEME" "$HOSTPORT" || exit $?
3636
37-
do_stage "$SOURCE" "$TARGET" "$MODE" 0
37+
_psij_do_stage "$SOURCE" "$TARGET" "$MODE" 0
3838
}
3939

40-
do_stage() {
40+
_psij_do_stage() {
4141
SOURCE="$1"
4242
TARGET="$2"
4343
MODE="$3"
4444
MISSING_OK="$4"
4545
46-
[ "{{psij.debug}}" != "0" ] && update_status "LOG Stage $SOURCE -> $TARGET, mode: $MODE, missingok: $MISSING_OK"
46+
[ "{{psij.debug}}" != "0" ] && _psij_update_status "LOG Stage $SOURCE -> $TARGET, mode: $MODE, missingok: $MISSING_OK"
4747
4848
if [ ! -e "$SOURCE" ]; then
4949
if [ "$MISSING_OK" == "0" ]; then
50-
[ "{{psij.debug}}" != "0" ] && update_status "LOG Missing source file: $SOURCE"
51-
fail 121 "Missing source file: $SOURCE"
50+
[ "{{psij.debug}}" != "0" ] && _psij_update_status "LOG Missing source file: $SOURCE"
51+
_psij_fail 121 "Missing source file: $SOURCE"
5252
else
53-
[ "{{psij.debug}}" != "0" ] && update_status "LOG Skipping staging of missing file $SOURCE"
53+
[ "{{psij.debug}}" != "0" ] && _psij_update_status "LOG Skipping staging of missing file $SOURCE"
5454
return 0
5555
fi
5656
fi
5757
58-
[ "{{psij.debug}}" != "0" ] && update_status "LOG Staging $SOURCE to $TARGET"
58+
[ "{{psij.debug}}" != "0" ] && _psij_update_status "LOG Staging $SOURCE to $TARGET"
5959
6060
TARGET_DIR=`dirname "$TARGET"`
6161
@@ -64,22 +64,22 @@ do_stage() {
6464
fi
6565
6666
if [ -d "$TARGET" ] && [ ! -d "$SOURCE" ]; then
67-
fail 121 "Target is a directory: $TARGET"
67+
_psij_fail 121 "Target is a directory: $TARGET"
6868
fi
6969
7070
if [ "$MODE" == "1" ]; then
7171
# copy
72-
cp -r -T "$SOURCE" "$TARGET" || fail 121 "Failed to copy \"$SOURCE\" to \"$TARGET\""
72+
cp -r -T "$SOURCE" "$TARGET" || _psij_fail 121 "Failed to copy \"$SOURCE\" to \"$TARGET\""
7373
elif [ "$MODE" == "2" ]; then
7474
# link
7575
{{!we want the same semantics as cp and mv, which is "overwrite if exists"}}
7676
{{!we resolve the source since it may be a path relative to the job dir}}
7777
rm -f "$TARGET"
7878
SOURCE=`readlink -m $SOURCE`
79-
ln -s "$SOURCE" "$TARGET" || fail 121 "Failed to link \"$SOURCE\" to \"$TARGET\""
79+
ln -s "$SOURCE" "$TARGET" || _psij_fail 121 "Failed to link \"$SOURCE\" to \"$TARGET\""
8080
elif [ "$MODE" == "3" ]; then
8181
# move
82-
mv -T -f "$SOURCE" "$TARGET" || fail 121 "Failed to move \"$SOURCE\" to \"$TARGET\""
82+
mv -T -f "$SOURCE" "$TARGET" || _psij_fail 121 "Failed to move \"$SOURCE\" to \"$TARGET\""
8383
fi
8484
}
8585

@@ -88,7 +88,7 @@ _FLAG_ON_SUCCESS=2
8888
_FLAG_ON_ERROR=4
8989
_FLAG_ON_CANCEL=8
9090

91-
do_stageout() {
91+
_psij_do_stageout() {
9292
SOURCE="$1"
9393
TARGET="$2"
9494
MODE="$3"
@@ -97,44 +97,42 @@ do_stageout() {
9797
SCHEME="$6"
9898
HOSTPORT="$7"
9999
100-
check_remote "$SCHEME" "$HOSTPORT"
100+
_psij_check_remote "$SCHEME" "$HOSTPORT"
101101
102-
[ "{{psij.debug}}" != "0" ] && update_status "LOG do_stageout $SOURCE -> $TARGET, mode: $MODE, flags: $FLAGS, failed: $FAILED"
102+
[ "{{psij.debug}}" != "0" ] && _psij_update_status "LOG do_stageout $SOURCE -> $TARGET, mode: $MODE, flags: $FLAGS, failed: $FAILED"
103103
104-
if [ "$FAILED" == "0" ] && [ "$((FLAGS & _FLAG_ON_SUCCESS))" != "0" ]; then
105-
do_stage "$SOURCE" "$TARGET" "$MODE" $((FLAGS & _FLAG_IF_PRESENT))
106-
elif [ "$FAILED" != "0" ] && [ "$((FLAGS & _FLAG_ON_ERROR))" != "0" ]; then
107-
do_stage "$SOURCE" "$TARGET" "$MODE" $((FLAGS & _FLAG_IF_PRESENT))
104+
if [ "$FAILED" == "0" ] && [ "$((FLAGS & _FLAG_ON_SUCCESS))" == "0" ]; then
105+
return 0
108106
fi
107+
if [ "$FAILED" != "0" ] && [ "$((FLAGS & _FLAG_ON_ERROR))" == "0" ]; then
108+
return 0
109+
fi
110+
_psij_do_stage "$SOURCE" "$TARGET" "$MODE" $((FLAGS & _FLAG_IF_PRESENT))
109111
}
110112

111-
do_cleanup() {
113+
_psij_do_cleanup() {
112114
TARGET="$1"
113-
FAILED="$2"
114-
115-
if [ "$FAILED" == "0" ] || [ "{{job.spec.cleanup_on_failure}}" != "0" ]; then
116-
117-
TARGET=`readlink -m "$TARGET"`
118-
DIR=`readlink -m "{{job.spec.directory}}"`
119-
120-
[ "{{psij.debug}}" != "0" ] && update_status "LOG Cleaning up $TARGET"
115+
FLAGS="$2"
116+
FAILED="$3"
121117
122-
case "$TARGET" in
123-
"$DIR"*)
124-
rm -rf "$TARGET"
125-
;;
126-
*)
127-
fail 121 "Cannot clean $TARGET outside of job directory $DIR"
128-
;;
129-
esac
118+
if [ "$FAILED" == "0" ] && [ "$((FLAGS & _FLAG_ON_SUCCESS))" == "0" ]; then
119+
return 0
130120
fi
131-
}
121+
if [ "$FAILED" != "0" ] && [ "$((FLAGS & _FLAG_ON_ERROR))" == "0" ]; then
122+
return 0
123+
fi
124+
125+
TARGET=`readlink -m "$TARGET"`
126+
DIR=`readlink -m "{{job.spec.directory}}"`
132127
133-
stagein() {
134-
update_status STAGE_IN
128+
[ "{{psij.debug}}" != "0" ] && _psij_update_status "LOG Cleaning up $TARGET"
135129
136-
{{#job.spec.stage_in}}
137-
do_stagein "{{source.path}}" "{{target}}" {{mode}} \
138-
"{{{source.scheme}}}" "{{#source.hostname}}{{{.}}}{{#source.port}}:{{{.}}}{{/source.port}}{{/source.hostname}}"
139-
{{/job.spec.stage_in}}
130+
case "$TARGET" in
131+
"$DIR"*)
132+
rm -rf "$TARGET"
133+
;;
134+
*)
135+
_psij_fail 121 "Cannot clean $TARGET outside of job directory $DIR"
136+
;;
137+
esac
140138
}

‎src/psij/executors/batch/common/cleanup.mustache‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
update_status CLEANUP
1+
_psij_update_status CLEANUP
22

33
{{#job.spec.cleanup}}
4-
do_cleanup {{.}} $_PSIJ_JOB_EC
4+
_psij_do_cleanup {{.}} {{job.spec.cleanup_flags}} $_PSIJ_JOB_EC
55
{{/job.spec.cleanup}}

‎src/psij/executors/batch/common/stagein.mustache‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
update_status STAGE_IN
1+
_psij_update_status STAGE_IN
22
{{#job.spec.stage_in}}
3-
do_stagein "{{source.path}}" "{{target}}" {{mode}} \
3+
_psij_do_stagein "{{source.path}}" "{{target}}" {{mode}} \
44
"{{{source.scheme}}}" "{{#source.hostname}}{{{.}}}{{#source.port}}:{{{.}}}{{/source.port}}{{/source.hostname}}"
55
{{/job.spec.stage_in}}

‎src/psij/executors/batch/common/stageout.mustache‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
update_status STAGE_OUT
1+
_psij_update_status STAGE_OUT
22
{{#job.spec.stage_out}}
3-
do_stageout "{{source}}" "{{target.path}}" {{mode}} {{flags}} $_PSIJ_JOB_EC \
3+
_psij_do_stageout "{{source}}" "{{target.path}}" {{mode}} {{flags}} $_PSIJ_JOB_EC \
44
"{{{target.scheme}}}" "{{#target.hostname}}{{{.}}}{{#target.port}}:{{{.}}}{{/target.port}}{{/target.hostname}}"
55
{{/job.spec.stage_out}}

‎src/psij/executors/batch/lsf/lsf.mustache‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ PSIJ_NODEFILE="$LSB_HOSTS"
8383
export PSIJ_NODEFILE
8484

8585
{{> stagein}}
86-
update_status ACTIVE
86+
_psij_update_status ACTIVE
8787

8888
{{#psij.launch_command}}{{.}} {{/psij.launch_command}}
8989
_PSIJ_JOB_EC=$?

‎src/psij/executors/batch/pbs/pbs_classic.mustache‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ cd "{{.}}"
6666
{{/job.spec.directory}}
6767

6868
{{> stagein}}
69-
update_status ACTIVE
69+
_psij_update_status ACTIVE
7070

7171
{{#psij.launch_command}}{{.}} {{/psij.launch_command}}
7272
_PSIJ_JOB_EC=$?

‎src/psij/executors/batch/pbs/pbspro.mustache‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ cd "{{.}}"
6969
{{/job.spec.directory}}
7070

7171
{{> stagein}}
72-
update_status ACTIVE
72+
_psij_update_status ACTIVE
7373

7474
{{#psij.launch_command}}{{.}} {{/psij.launch_command}}
7575
_PSIJ_JOB_EC=$?

‎src/psij/executors/batch/slurm/slurm.mustache‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ fi
115115
export PSIJ_NODEFILE
116116

117117
{{> stagein}}
118-
update_status ACTIVE
118+
_psij_update_status ACTIVE
119119

120120
{{#psij.launch_command}}{{.}} {{/psij.launch_command}}
121121
_PSIJ_JOB_EC=$?

‎src/psij/executors/local/local.mustache‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ cd "{{.}}"
99
{{/job.spec.directory}}
1010

1111
{{> stagein}}
12-
update_status ACTIVE
12+
_psij_update_status ACTIVE
1313

1414
set +e
1515
{{#job.spec.inherit_environment}}env \{{/job.spec.inherit_environment}}{{^job.spec.inherit_environment}}env --ignore-environment \{{/job.spec.inherit_environment}}{{#env}}

0 commit comments

Comments
 (0)