Fix log file paths (#185)

daniel-thom · web-flow · commit 6581bcd7ad2f · 2026-03-05T18:52:26.000-07:00
diff --git a/docs/src/core/monitoring/working-with-logs.md b/docs/src/core/monitoring/working-with-logs.md
@@ -128,7 +128,7 @@ Analyzing: output/
 
 Files with detected errors:
 
-  output/job_stdio/job_wf123_j456_r1.e
+  output/job_stdio/job_wf123_j456_r1_a1.e
     Line 42: MemoryError: Unable to allocate 8.00 GiB
     Severity: critical
     Type: Python Memory Error
diff --git a/docs/src/specialized/hpc/debugging-slurm.md b/docs/src/specialized/hpc/debugging-slurm.md
@@ -24,8 +24,8 @@ these additional log paths:
 
 ```json
 {
-  "job_stdout": "torc_output/job_stdio/job_wf1_j456_r1.o",
-  "job_stderr": "torc_output/job_stdio/job_wf1_j456_r1.e",
+  "job_stdout": "torc_output/job_stdio/job_wf1_j456_r1_a1.o",
+  "job_stderr": "torc_output/job_stdio/job_wf1_j456_r1_a1.e",
   "job_runner_log": "torc_output/job_runner_slurm_wf1_sl12345_n0_pid67890.log",
   "slurm_stdout": "torc_output/slurm_output_wf1_sl12345.o",
   "slurm_stderr": "torc_output/slurm_output_wf1_sl12345.e",
diff --git a/examples/json/hyperparameter_sweep_shared_params.json5 b/examples/json/hyperparameter_sweep_shared_params.json5
@@ -5,7 +5,6 @@
 // Compare this to hyperparameter_sweep.json5 - same result, but parameters defined only once!
 {
   name: "hyperparameter_sweep_shared_params",
-  user: "mluser",
   description: "Grid search over learning rate, batch size, and optimizer (using shared parameters)",
 
   // Workflow-level shared parameters - defined once, used by multiple jobs and files
diff --git a/examples/json/resource_monitoring_demo.json5 b/examples/json/resource_monitoring_demo.json5
@@ -54,7 +54,7 @@
     "resource_monitor": {
     "enabled": true,
     "granularity": "time_series",
-    "sample_interval_seconds": 2,
+    "sample_interval_seconds": 5,
     "generate_plots": false
   },
 
diff --git a/examples/kdl/resource_monitoring_demo.kdl b/examples/kdl/resource_monitoring_demo.kdl
@@ -16,7 +16,7 @@ slurm_defaults {
 resource_monitor {
     enabled #true
     granularity "time_series"
-    sample_interval_seconds 2
+    sample_interval_seconds 5
     generate_plots #false
 }
 
diff --git a/examples/yaml/data_pipeline_parameterized.yaml b/examples/yaml/data_pipeline_parameterized.yaml
@@ -3,7 +3,6 @@
 # Processes data from multiple sources through multiple stages
 
 name: data_pipeline
-user: datauser
 description: Multi-stage data processing pipeline for multiple datasets
 
 jobs:
diff --git a/examples/yaml/diamond_workflow.yaml b/examples/yaml/diamond_workflow.yaml
@@ -18,7 +18,6 @@
 #     f6
 
 name: diamond_workflow
-user: test_user
 description: A diamond-shaped workflow demonstrating fan-out and fan-in job dependencies
 
 # Job definitions
diff --git a/examples/yaml/hundred_jobs_parameterized.yaml b/examples/yaml/hundred_jobs_parameterized.yaml
@@ -3,7 +3,6 @@
 # Compare this file to hundred_jobs_workflow.yaml - same result, 90% less code!
 
 name: hundred_jobs_parameterized
-user: testuser
 description: A workflow with 100 independent jobs using parameterization
 
 # Job definitions - ONE job specification expands to 100 jobs
diff --git a/examples/yaml/hyperparameter_sweep.yaml b/examples/yaml/hyperparameter_sweep.yaml
@@ -3,7 +3,6 @@
 # This creates 3 * 3 * 2 = 18 training jobs with different hyperparameter combinations
 
 name: hyperparameter_sweep
-user: mluser
 description: Grid search over learning rate, batch size, and optimizer
 
 # Prepare datasets (one-time setup jobs)
diff --git a/examples/yaml/hyperparameter_sweep_shared_params.yaml b/examples/yaml/hyperparameter_sweep_shared_params.yaml
@@ -5,7 +5,6 @@
 # Compare this to hyperparameter_sweep.yaml - same result, but parameters defined only once!
 
 name: hyperparameter_sweep_shared_params
-user: mluser
 description: Grid search over learning rate, batch size, and optimizer (using shared parameters)
 
 # Workflow-level shared parameters - defined once, used by multiple jobs and files
diff --git a/examples/yaml/resource_monitoring_demo.yaml b/examples/yaml/resource_monitoring_demo.yaml
@@ -10,7 +10,6 @@
 # 5. Check results with resource metrics: torc results list <workflow_id>
 
 name: resource_monitoring_demo
-user: demo_user
 description: Demonstrates resource monitoring with CPU and memory intensive Python jobs
 
 # Default Slurm parameters applied to all schedulers
@@ -23,7 +22,7 @@ slurm_defaults:
 resource_monitor:
   enabled: true
   granularity: "time_series"  # Options: "summary" or "time_series"
-  sample_interval_seconds: 2   # Sample every 2 seconds
+  sample_interval_seconds: 5   # Sample every 5 seconds
   generate_plots: false         # Not yet implemented
 
 # Three independent jobs that run concurrently
diff --git a/examples/yaml/ro_crate_provenance.yaml b/examples/yaml/ro_crate_provenance.yaml
@@ -30,7 +30,6 @@
 #     report.html
 
 name: ro_crate_demo
-user: researcher
 description: Data processing workflow with RO-Crate provenance tracking
 
 # Enable automatic RO-Crate entity generation
diff --git a/examples/yaml/sample_workflow.yaml b/examples/yaml/sample_workflow.yaml
@@ -2,7 +2,6 @@
 # This demonstrates a complete workflow specification in YAML format
 
 name: sample_data_processing_workflow
-user: testuser
 description: A sample workflow that demonstrates data processing with multiple jobs
 
 # Job definitions - these define the computational tasks
diff --git a/examples/yaml/simulation_sweep.yaml b/examples/yaml/simulation_sweep.yaml
@@ -3,7 +3,6 @@
 # Runs simulations across temperature and pressure ranges
 
 name: simulation_sweep
-user: scientist
 description: Molecular dynamics simulations across temperature and pressure ranges
 
 jobs:
diff --git a/examples/yaml/slurm_staged_pipeline.yaml b/examples/yaml/slurm_staged_pipeline.yaml
@@ -1,6 +1,5 @@
 name: "slurm_staged_pipeline"
 description: "Multi-stage Slurm pipeline with automated scheduling and resource monitoring"
-user: "pipeline_user"
 
 # Define three Slurm schedulers for different pipeline stages
 slurm_schedulers:
diff --git a/examples/yaml/workflow_actions_data_pipeline.yaml b/examples/yaml/workflow_actions_data_pipeline.yaml
@@ -1,5 +1,4 @@
 name: "Data Processing Pipeline with Actions"
-user: "data_engineer"
 description: "Demonstrates multiple action types for different job groups"
 
 jobs:
diff --git a/examples/yaml/workflow_actions_ml_training.yaml b/examples/yaml/workflow_actions_ml_training.yaml
@@ -1,5 +1,4 @@
 name: "ML Training with Auto-Scaling"
-user: "ml_researcher"
 description: "Demonstrates on_jobs_ready action for dynamic resource allocation"
 
 jobs:
diff --git a/examples/yaml/workflow_actions_simple.yaml b/examples/yaml/workflow_actions_simple.yaml
@@ -1,5 +1,4 @@
 name: "Simple Workflow with Actions"
-user: "demo"
 description: "Demonstrates scheduler actions for different workflow stages"
 
 jobs:
diff --git a/examples/yaml/workflow_actions_simple_slurm.yaml b/examples/yaml/workflow_actions_simple_slurm.yaml
@@ -1,5 +1,4 @@
 name: "Simple Workflow with Actions"
-user: "demo"
 description: "Demonstrates scheduler actions for different workflow stages"
 
 jobs:
diff --git a/src/server/http_server.rs b/src/server/http_server.rs
@@ -4717,6 +4717,7 @@ where
                 workflow_id,
                 name,
                 command,
+                invocation_script,
                 status,
                 cancel_on_blocking_job_failure,
                 supports_termination,
@@ -4762,7 +4763,7 @@ where
                 workflow_id: row.get("workflow_id"),
                 name: row.get("name"),
                 command: row.get("command"),
-                invocation_script: None,
+                invocation_script: row.get("invocation_script"),
                 status: Some(models::JobStatus::Pending),
                 schedule_compute_nodes: None,
                 cancel_on_blocking_job_failure: Some(row.get("cancel_on_blocking_job_failure")),
@@ -6142,7 +6143,7 @@ where
                     workflow_id: row.get("workflow_id"),
                     name: row.get("name"),
                     command: row.get("command"),
-                    invocation_script: None,
+                    invocation_script: row.get("invocation_script"),
                     status: Some(models::JobStatus::Pending),
                     schedule_compute_nodes: None,
                     cancel_on_blocking_job_failure: Some(row.get("cancel_on_blocking_job_failure")),
diff --git a/src/tui/README.md b/src/tui/README.md
@@ -190,7 +190,7 @@ Press `l` to view logs:
 │   4  Epoch 4/100: loss=0.352                                     │
 │   ...                                                            │
 ├──────────────────────────────────────────────────────────────────┤
-│ Path: output/job_stdio/job_wf1_j2_r1.o                           │
+│ Path: output/job_stdio/job_wf1_j2_r1_a1.o                        │
 │ q: close | /: search | n/N: next/prev | g/G: top/bottom | y: path│
 └──────────────────────────────────────────────────────────────────┘
 ```
diff --git a/tests/scripts/fake_sacct.sh b/tests/scripts/fake_sacct.sh
@@ -13,7 +13,9 @@ fi
 
 # Parse arguments
 JOB_ID=""
-_FORMAT="JobID,JobName%20,state,start,end,Account,Partition%15,QOS"
+_FORMAT=""
+PIPE_SEPARATED=false
+NO_HEADER=false
 
 while [[ $# -gt 0 ]]; do
     case $1 in
@@ -25,12 +27,45 @@ while [[ $# -gt 0 ]]; do
             _FORMAT="${1#--format=}"
             shift
             ;;
+        --format)
+            _FORMAT="$2"
+            shift 2
+            ;;
+        -P)
+            PIPE_SEPARATED=true
+            shift
+            ;;
+        -n)
+            NO_HEADER=true
+            shift
+            ;;
         *)
             shift
             ;;
     esac
 done
 
+# Pipe-separated, no-header mode (used by collect_sacct_stats in async_cli_command.rs)
+# Format: JobName,MaxRSS,MaxVMSize,MaxDiskRead,MaxDiskWrite,AveCPU,NodeList
+if $PIPE_SEPARATED && $NO_HEADER; then
+    # Return fake step records for any wf*_j*_r*_a* step names the runner asks about.
+    # The runner queries by slurm_job_id and filters by step_name in code.
+    # We return a batch step plus generic srun step records with mock accounting data.
+    # Generate a wide range of workflow/job IDs to cover any test scenario.
+    echo "batch|1024K|2048K|100M|50M|00:00:01|node001"
+    for wf in $(seq 1 20); do
+        for j in $(seq 1 20); do
+            echo "wf${wf}_j${j}_r1_a1|512K|1024K|50M|25M|00:00:01|node001"
+        done
+    done
+    exit 0
+fi
+
+# Default format for standard header-based output
+if [ -z "$_FORMAT" ]; then
+    _FORMAT="JobID,JobName%20,state,start,end,Account,Partition%15,QOS"
+fi
+
 # The sacct output has a specific format:
 # Line 1: Header with column names
 # Line 2: Separator line with dashes
diff --git a/tests/test_claim_jobs_based_on_resources.rs b/tests/test_claim_jobs_based_on_resources.rs
@@ -8,7 +8,8 @@ use common::{
     ServerProcess, create_custom_resources_workflow, create_dependency_chain_workflow,
     create_diverse_jobs_workflow, create_gpu_workflow, create_high_cpu_workflow,
     create_high_memory_workflow, create_many_jobs_workflow, create_maximum_resources_workflow,
-    create_minimal_resources_workflow, create_multi_node_workflow, start_server,
+    create_minimal_resources_workflow, create_multi_node_workflow,
+    create_test_resource_requirements, start_server,
 };
 use rstest::rstest;
 
@@ -1786,3 +1787,62 @@ fn test_prepare_jobs_concurrent_allocation(start_server: &ServerProcess) {
         duplicates
     );
 }
+
+/// Test that claim_jobs_based_on_resources returns invocation_script when set on a job
+#[rstest]
+fn test_claim_jobs_based_on_resources_returns_invocation_script(start_server: &ServerProcess) {
+    let config = &start_server.config;
+
+    // Create workflow
+    let workflow = models::WorkflowModel::new(
+        "invocation_script_resource_test".to_string(),
+        "test_user".to_string(),
+    );
+    let created_workflow =
+        default_api::create_workflow(config, workflow).expect("Failed to create workflow");
+    let workflow_id = created_workflow.id.unwrap();
+
+    // Create resource requirements
+    let resource_req = create_test_resource_requirements(
+        config,
+        workflow_id,
+        "test_rr",
+        1,        // num_cpus
+        0,        // num_gpus
+        1,        // num_nodes
+        "1g",     // memory
+        "P0DT1H", // runtime
+    );
+
+    // Create a job with invocation_script set
+    let invocation_script = "#!/bin/bash\nset -e\nexport MY_VAR=test\n".to_string();
+    let mut job = models::JobModel::new(
+        workflow_id,
+        "job_with_invocation_script".to_string(),
+        "echo hello".to_string(),
+    );
+    job.invocation_script = Some(invocation_script.clone());
+    job.resource_requirements_id = Some(resource_req.id.unwrap());
+
+    let _created_job = default_api::create_job(config, job).expect("Failed to create job");
+
+    // Initialize jobs
+    default_api::initialize_jobs(config, workflow_id, None, None, None)
+        .expect("Failed to initialize jobs");
+
+    // Claim the job with sufficient resources
+    let resources = models::ComputeNodesResources::new(1, 1.0, 0, 1);
+    let result =
+        default_api::claim_jobs_based_on_resources(config, workflow_id, &resources, 10, None, None)
+            .expect("claim_jobs_based_on_resources should succeed");
+
+    let returned_jobs = result.jobs.expect("Server must return jobs array");
+    assert_eq!(returned_jobs.len(), 1, "Should return exactly 1 job");
+
+    let returned_job = &returned_jobs[0];
+    assert_eq!(
+        returned_job.invocation_script,
+        Some(invocation_script),
+        "invocation_script should be returned by claim_jobs_based_on_resources"
+    );
+}
diff --git a/tests/test_claim_next_jobs.rs b/tests/test_claim_next_jobs.rs
@@ -694,3 +694,47 @@ fn test_prepare_next_jobs_zero_limit(start_server: &ServerProcess) {
         "Should return empty array when limit is 0"
     );
 }
+
+/// Test that claim_next_jobs returns invocation_script when set on a job
+#[rstest]
+fn test_claim_next_jobs_returns_invocation_script(start_server: &ServerProcess) {
+    let config = &start_server.config;
+
+    // Create workflow
+    let workflow = models::WorkflowModel::new(
+        "invocation_script_test".to_string(),
+        "test_user".to_string(),
+    );
+    let created_workflow =
+        default_api::create_workflow(config, workflow).expect("Failed to create workflow");
+    let workflow_id = created_workflow.id.unwrap();
+
+    // Create a job with invocation_script set
+    let invocation_script = "#!/bin/bash\nset -e\nexport MY_VAR=test\n".to_string();
+    let mut job = models::JobModel::new(
+        workflow_id,
+        "job_with_invocation_script".to_string(),
+        "echo hello".to_string(),
+    );
+    job.invocation_script = Some(invocation_script.clone());
+
+    let _created_job = default_api::create_job(config, job).expect("Failed to create job");
+
+    // Initialize jobs
+    default_api::initialize_jobs(config, workflow_id, None, None, None)
+        .expect("Failed to initialize jobs");
+
+    // Claim the job
+    let result = default_api::claim_next_jobs(config, workflow_id, Some(1), None)
+        .expect("claim_next_jobs should succeed");
+
+    let returned_jobs = result.jobs.expect("Server must return jobs array");
+    assert_eq!(returned_jobs.len(), 1, "Should return exactly 1 job");
+
+    let returned_job = &returned_jobs[0];
+    assert_eq!(
+        returned_job.invocation_script,
+        Some(invocation_script),
+        "invocation_script should be returned by claim_next_jobs"
+    );
+}
diff --git a/tests/test_completion_reversal.rs b/tests/test_completion_reversal.rs
@@ -39,6 +39,7 @@ fn test_completion_reversal_resets_downstream_jobs(start_server: &ServerProcess)
         "echo 'job2 success'".to_string(),
     );
     job2.depends_on_job_ids = Some(vec![job1_id]);
+    job2.cancel_on_blocking_job_failure = Some(false);
     let created_job2 = default_api::create_job(config, job2).expect("Failed to create job2");
     let job2_id = created_job2.id.unwrap();
 
@@ -49,6 +50,7 @@ fn test_completion_reversal_resets_downstream_jobs(start_server: &ServerProcess)
         "echo 'job3 success'".to_string(),
     );
     job3.depends_on_job_ids = Some(vec![job2_id]);
+    job3.cancel_on_blocking_job_failure = Some(false);
     let created_job3 = default_api::create_job(config, job3).expect("Failed to create job3");
     let job3_id = created_job3.id.unwrap();
 
diff --git a/tests/test_log_bundles.rs b/tests/test_log_bundles.rs
diff --git a/tests/test_slurm_commands.rs b/tests/test_slurm_commands.rs
diff --git a/torc-dash/static/js/app-debugging.js b/torc-dash/static/js/app-debugging.js

Original file line number	Diff line number	Diff line change
`@@ -5,7 +5,6 @@`
`5`	`5`	`// Compare this to hyperparameter_sweep.json5 - same result, but parameters defined only once!`
`6`	`6`	`{`
`7`	`7`	`name: "hyperparameter_sweep_shared_params",`
`8`		`- user: "mluser",`
`9`	`8`	`description: "Grid search over learning rate, batch size, and optimizer (using shared parameters)",`
`10`	`9`
`11`	`10`	`// Workflow-level shared parameters - defined once, used by multiple jobs and files`
Original file line number	Diff line number	Diff line change
`@@ -16,7 +16,7 @@ slurm_defaults {`
`16`	`16`	`resource_monitor {`
`17`	`17`	`enabled #true`
`18`	`18`	`granularity "time_series"`
`19`		`- sample_interval_seconds 2`
	`19`	`+ sample_interval_seconds 5`
`20`	`20`	`generate_plots #false`
`21`	`21`	`}`
`22`	`22`