Skip to content

Commit 2d0ecc6

Browse files
dennishendriksendennishendriksen
authored
Merge pull request #275 from pneerincx/master
Updated templates for local and slurm backends and rewrote tests to produce output in a test specific subdir.
2 parents 94d7932 + 196e0a4 commit 2d0ecc6

File tree

329 files changed

+15101
-11619
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

329 files changed

+15101
-11619
lines changed

‎molgenis-compute-core/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<modelVersion>4.0.0</modelVersion>
44
<properties>
55
<molgenis.version>1.2.0</molgenis.version>
6-
<compute.version>16.11.1</compute.version>
6+
<compute.version>17.08.1</compute.version>
77
</properties>
88
<parent>
99
<groupId>org.molgenis</groupId>

‎molgenis-compute-core/src/main/java/org/molgenis/compute/ComputeProperties.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -136,11 +136,9 @@ private void updateWorkflowParameterDefaultsCSV()
136136
else
137137
{
138138
File workflowFile = new File(this.workFlow);
139-
String workflowFilePath = workflowFile.getParent(); // get workflow
140-
// path
139+
String workflowFilePath = workflowFile.getParent(); // get workflow path
141140
String workflowName = workflowFile.getName(); // strip workflow name
142-
workflowName = workflowName.substring(0, workflowName.indexOf('.')); // strip
143-
// extension
141+
workflowName = workflowName.substring(0, workflowName.indexOf('.')); // strip extension
144142

145143
File defaultsFileTest = new File(
146144
workflowFilePath + File.separator + workflowName + "." + Parameters.DEFAULTS_DEFAULT);
@@ -226,6 +224,9 @@ public void parseCommandLine(String[] args)
226224

227225
if (batchOption != null)
228226
{
227+
LOG.error("Batching is currently disabled pending re-implementation of this feature.");
228+
System.exit(1);
229+
229230
String nameValue[] = batchOption.split("=");
230231
if (nameValue.length != 2) LOG.warn("BATCH specification is wrong; usage example sample=2");
231232
else

‎molgenis-compute-core/src/main/resources/templates/localhost/footer.ftl‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,22 @@
1+
<#noparse>#
2+
## Start of footer for backend 'local'.
13
#
2-
## General footer
3-
#
44

5-
# Show that we successfully finished. If the .finished file exists, then this step will be skipped when you resubmit your workflow
6-
touch $ENVIRONMENT_DIR/${taskId}.sh.finished
5+
if [ -d "${MC_tmpFolder:-}" ]; then
6+
echo -n "INFO: Removing MC_tmpFolder ${MC_tmpFolder} ..."
7+
rm -rf "${MC_tmpFolder}"
8+
echo 'done.'
9+
fi
10+
11+
tS=${SECONDS:-0}
12+
tM=$((SECONDS / 60 ))
13+
tH=$((SECONDS / 3600))
14+
echo "On $(date +"%Y-%m-%d %T") ${MC_jobScript} finished successfully after ${tM} minutes." >> molgenis.bookkeeping.log
15+
printf '%s:\t%d seconds\t%d minutes\t%d hours\n' "${MC_jobScript}" "${tS}" "${tM}" "${tH}" >> molgenis.bookkeeping.walltime
16+
17+
mv "${MC_jobScript}".{started,finished}
18+
19+
trap - EXIT
20+
exit 0
721

8-
# Also do bookkeeping
9-
echo "On $(date +"%Y-%m-%d %T"), after $(( ($(date +%s) - $MOLGENIS_START) / 60 )) minutes, task ${taskId}.sh finished successfully" >> $ENVIRONMENT_DIR/molgenis.bookkeeping.log
22+
</#noparse>

‎molgenis-compute-core/src/main/resources/templates/localhost/footer_gaf.ftl‎

Lines changed: 0 additions & 9 deletions
This file was deleted.

‎molgenis-compute-core/src/main/resources/templates/localhost/header.ftl‎

Lines changed: 100 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -1,101 +1,119 @@
1+
#!/bin/bash
12
#
2-
## Header for 'local' backend
3+
# This is job <#if project??>${project}_</#if>${taskId}
34
#
45

5-
#highly recommended to use
6-
#set -e # exit if any subcommand or pipeline returns a non-zero status
7-
#set -u # exit if any uninitialised variable is used
8-
9-
# Set location of *.env and *.log files
10-
ENVIRONMENT_DIR="."
6+
#
7+
## Start of header for backend 'local'.
8+
#
119

12-
# If you detect an error, then exit your script by calling this function
13-
exitWithError(){
14-
errorCode=$1
15-
errorMessage=$2
16-
echo "$errorCode: $errorMessage --- TASK '${taskId}.sh' --- ON $(date +"%Y-%m-%d %T"), AFTER RUNNING $(( ($(date +%s) - $MOLGENIS_START) / 60 )) MINUTES" >> $ENVIRONMENT_DIR/molgenis.error.log
17-
exit $errorCode
18-
}
10+
set -e
11+
set -u
1912

20-
# For bookkeeping how long your task takes
21-
MOLGENIS_START=$(date +%s)
13+
ENVIRONMENT_DIR='.'
2214

23-
# Show that the task has started
24-
touch $ENVIRONMENT_DIR/${taskId}.sh.started
15+
#
16+
# Variables declared in MOLGENIS Compute headers/footers always start with an MC_ prefix.
17+
#
18+
declare MC_jobScript="${taskId}.sh"
19+
declare MC_jobScriptSTDERR="${taskId}.err"
20+
declare MC_jobScriptSTDOUT="${taskId}.out"
21+
declare MC_failedFile="molgenis.pipeline.failed"
2522

2623
<#noparse>
27-
getFile()
28-
{
29-
ARGS=($@)
30-
NUMBER="${#ARGS[@]}";
31-
if [ "$NUMBER" -eq "1" ]
32-
then
33-
myFile=${ARGS[0]}
3424

35-
if test ! -e $myFile;
36-
then
37-
echo "WARNING in getFile/putFile: $myFile is missing" 1>&2
38-
fi
25+
declare MC_singleSeperatorLine=$(head -c 120 /dev/zero | tr '\0' '-')
26+
declare MC_doubleSeperatorLine=$(head -c 120 /dev/zero | tr '\0' '=')
27+
declare MC_tmpFolder='tmpFolder'
28+
declare MC_tmpFile='tmpFile'
29+
declare MC_tmpFolderCreated=0
3930

40-
else
41-
echo "Example usage: getData \"\$TMPDIR/datadir/myfile.txt\""
42-
fi
43-
}
31+
#
32+
##
33+
### Header functions.
34+
##
35+
#
4436

45-
putFile()
46-
{
47-
`getFile $@`
37+
function errorExitAndCleanUp() {
38+
local _signal="${1}"
39+
local _problematicLine="${2}"
40+
local _exitStatus="${3:-$?}"
41+
local _executionHost="$(hostname)"
42+
local _format='INFO: Last 50 lines or less of %s:\n'
43+
local _errorMessage="FATAL: Trapped ${_signal} signal in ${MC_jobScript} running on ${_executionHost}. Exit status code was ${_exitStatus}."
44+
if [ "${_signal}" == 'ERR' ]; then
45+
_errorMessage="FATAL: Trapped ${_signal} signal on line ${_problematicLine} in ${MC_jobScript} running on ${_executionHost}. Exit status code was ${_exitStatus}."
46+
fi
47+
_errorMessage=${4:-"${_errorMessage}"} # Optionally use custom error message as 4th argument.
48+
echo "${_errorMessage}"
49+
echo "${MC_doubleSeperatorLine}" > "${MC_failedFile}"
50+
echo "${_errorMessage}" >> "${MC_failedFile}"
51+
if [ -f "${MC_jobScriptSTDERR}" ]; then
52+
echo "${MC_singleSeperatorLine}" >> "${MC_failedFile}"
53+
printf "${_format}" "${MC_jobScriptSTDERR}" >> "${MC_failedFile}"
54+
echo "${MC_singleSeperatorLine}" >> "${MC_failedFile}"
55+
tail -50 "${MC_jobScriptSTDERR}" >> "${MC_failedFile}"
56+
fi
57+
if [ -f "${MC_jobScriptSTDOUT}" ]; then
58+
echo "${MC_singleSeperatorLine}" >> "${MC_failedFile}"
59+
printf "${_format}" "${MC_jobScriptSTDOUT}" >> "${MC_failedFile}"
60+
echo "${MC_singleSeperatorLine}" >> "${MC_failedFile}"
61+
tail -50 "${MC_jobScriptSTDOUT}" >> "${MC_failedFile}"
62+
fi
63+
echo "${MC_doubleSeperatorLine}" >> "${MC_failedFile}"
4864
}
4965

50-
inputs()
51-
{
52-
for name in $@
53-
do
54-
if test ! -e $name;
55-
then
56-
echo "$name is missing" 1>&2
57-
exit 1;
58-
fi
59-
done
66+
#
67+
# Create tmp dir per script/job.
68+
# To be called with with either a file or folder as first and only argument.
69+
# Defines two globally set variables:
70+
# 1. MC_tmpFolder: a tmp dir for this job/script. When function is called multiple times MC_tmpFolder will always be the same.
71+
# 2. MC_tmpFile: when the first argument was a folder, MC_tmpFile == MC_tmpFolder
72+
# when the first argument was a file, MC_tmpFile will be a path to a tmp file inside MC_tmpFolder.
73+
#
74+
function makeTmpDir {
75+
#
76+
# Compile paths.
77+
#
78+
local _originalPath="${1}"
79+
local _myMD5="$(md5sum ${MC_jobScript} | cut -d ' ' -f 1)"
80+
local _tmpSubFolder="tmp_${MC_jobScript}_${_myMD5}"
81+
local _dir
82+
local _base
83+
if [[ -d "${_originalPath}" ]]; then
84+
_dir="${_originalPath}"
85+
_base=''
86+
else
87+
_base=$(basename "${_originalPath}")
88+
_dir=$(dirname "${_originalPath}")
89+
fi
90+
MC_tmpFolder="${_dir}/${_tmpSubFolder}/"
91+
MC_tmpFile="${MC_tmpFolder}/${_base}"
92+
echo "DEBUG ${MC_jobScript}::makeTmpDir: dir='${_dir}';base='${_base}';MC_tmpFile='${MC_tmpFile}'"
93+
#
94+
# Cleanup the previously created tmpFolder first if this script was resubmitted.
95+
#
96+
if [[ ${MC_tmpFolderCreated} -eq 0 && -d "${MC_tmpFolder}" ]]; then
97+
rm -rf "${MC_tmpFolder}"
98+
fi
99+
#
100+
# (Re-)create tmpFolder.
101+
#
102+
mkdir -p "${MC_tmpFolder}"
103+
MC_tmpFolderCreated=1
60104
}
61105

62-
outputs()
63-
{
64-
for name in $@
65-
do
66-
if test -e $name;
67-
then
68-
echo "skipped"
69-
echo "skipped" 1>&2
70-
exit 0;
71-
else
72-
return;
73-
fi
74-
done
75-
}
106+
trap 'errorExitAndCleanUp HUP NA $?' HUP
107+
trap 'errorExitAndCleanUp INT NA $?' INT
108+
trap 'errorExitAndCleanUp QUIT NA $?' QUIT
109+
trap 'errorExitAndCleanUp TERM NA $?' TERM
110+
trap 'errorExitAndCleanUp EXIT NA $?' EXIT
111+
trap 'errorExitAndCleanUp ERR $LINENO $?' ERR
76112

77-
alloutputsexist()
78-
{
79-
all_exist=true
80-
for name in $@
81-
do
82-
if test ! -e $name;
83-
then
84-
all_exist=false
85-
fi
86-
done
87-
if $all_exist;
88-
then
89-
echo "skipped"
90-
echo "skipped" 1>&2
91-
sleep 30
92-
exit 0;
93-
else
94-
return 0;
95-
fi
96-
}
97-
</#noparse>
113+
touch "${MC_jobScript}.started"
98114

99115
#
100-
## End of header for 'local' backend
116+
## End of header for backend 'local'
101117
#
118+
119+
</#noparse>

‎molgenis-compute-core/src/main/resources/templates/localhost/header_gaf.ftl‎

Lines changed: 0 additions & 43 deletions
This file was deleted.

0 commit comments

Comments
 (0)