diff --git a/taskvine/src/bindings/python3/ndcctools/taskvine/display.py b/taskvine/src/bindings/python3/ndcctools/taskvine/display.py index 95da12d924..db174298d1 100644 --- a/taskvine/src/bindings/python3/ndcctools/taskvine/display.py +++ b/taskvine/src/bindings/python3/ndcctools/taskvine/display.py @@ -110,6 +110,7 @@ def generate_manager_table(self, manager_s): "tasks_done", "tasks_waiting", "tasks_running", + "recovery_tasks_submitted", "tasks_exhausted_attempts", "workers_connected", "workers_busy", diff --git a/taskvine/src/manager/taskvine.h b/taskvine/src/manager/taskvine.h index 16152b7102..a28ff2a022 100644 --- a/taskvine/src/manager/taskvine.h +++ b/taskvine/src/manager/taskvine.h @@ -176,6 +176,7 @@ struct vine_stats { int tasks_on_workers; /**< Number of tasks currently dispatched to some worker. */ int tasks_running; /**< Number of tasks currently executing at some worker. */ int tasks_with_results; /**< Number of tasks with retrieved results and waiting to be returned to user. */ + int recovery_tasks_submitted; /**< Total number of recovery tasks submitted since the manager started. */ /* Cumulative stats for tasks: */ int tasks_submitted; /**< Total number of tasks submitted to the manager. */ diff --git a/taskvine/src/manager/vine_manager.c b/taskvine/src/manager/vine_manager.c index ca72d257d9..04d3edeca7 100644 --- a/taskvine/src/manager/vine_manager.c +++ b/taskvine/src/manager/vine_manager.c @@ -2262,6 +2262,7 @@ static struct jx *manager_to_jx(struct vine_manager *q) jx_insert_integer(j, "tasks_on_workers", info.tasks_on_workers); jx_insert_integer(j, "tasks_running", info.tasks_running); jx_insert_integer(j, "tasks_with_results", info.tasks_with_results); + jx_insert_integer(j, "recovery_tasks_submitted", info.recovery_tasks_submitted); jx_insert_integer(j, "tasks_left", q->num_tasks_left); jx_insert_integer(j, "tasks_submitted", info.tasks_submitted); @@ -3519,7 +3520,7 @@ static void vine_manager_consider_recovery_task(struct vine_manager *q, struct v case VINE_TASK_INITIAL: /* The recovery task has never been run, so submit it now. */ vine_submit(q, rt); - notice(D_VINE, "Submitted recovery task %d (%s) to re-create lost temporary file %s.", rt->task_id, rt->command_line, lost_file->cached_name); + debug(D_VINE, "Submitted recovery task %d (%s) to re-create lost temporary file %s.", rt->task_id, rt->command_line, lost_file->cached_name); break; case VINE_TASK_READY: case VINE_TASK_RUNNING: @@ -3533,7 +3534,7 @@ static void vine_manager_consider_recovery_task(struct vine_manager *q, struct v * here. */ vine_task_reset(rt); vine_submit(q, rt); - notice(D_VINE, "Submitted recovery task %d (%s) to re-create lost temporary file %s.", rt->task_id, rt->command_line, lost_file->cached_name); + debug(D_VINE, "Submitted recovery task %d (%s) to re-create lost temporary file %s.", rt->task_id, rt->command_line, lost_file->cached_name); break; } } @@ -4869,6 +4870,7 @@ int vine_submit(struct vine_manager *q, struct vine_task *t) * this distinction is important when many files are lost and the workflow is effectively rerun from scratch. */ if (t->type == VINE_TASK_TYPE_RECOVERY) { vine_task_set_priority(t, t->priority + priority_queue_get_top_priority(q->ready_tasks) + 1); + q->stats->recovery_tasks_submitted++; } if (t->has_fixed_locations) {