fix

Signed-off-by: oliver könig <[email protected]>

Author: ko3n1g

nemo_run/core/execution/templates/ft_launcher_dgxc.j2

@@ -3,6 +3,20 @@
 # Fault tolerance related items
 export FAULT_TOL_CFG_PATH="{{fault_tol_cfg_path}}"
 export FAULT_TOL_FINISHED_FLAG_FILE="{{fault_tol_finished_flag_file}}"
+
+JOB_RESULTS_FILE="{{fault_tol_job_results_file}}"
+
+is_training_finished() {
+    test -f "$(dirname $JOB_RESULTS_FILE)/$(basename $FAULT_TOL_FINISHED_FLAG_FILE)"
+}
+
+if is_training_finished ; then
+    echo "Training is finished";
+    exit 0;
+else
+    rm -f "$FAULT_TOL_FINISHED_FLAG_FILE" "$JOB_RESULTS_FILE"
+fi
+
 {%- endmacro %}
 
 {% macro ft_launcher_teardown() -%}