add: update slurm scripts runner

Author: LennartPurucker

tabflow_slurm/setup_slurm_base.py

@@ -641,25 +641,27 @@ def get_jobs_dict(self):
         }
         return {"defaults": default_args, "jobs": jobs}
 
-    def setup_jobs(self):
+    def setup_jobs(self) -> str:
         """Setup the jobs to run by generating the SLURM job JSON file."""
         jobs_dict = self.get_jobs_dict()
         n_jobs = len(jobs_dict["jobs"])
         if n_jobs == 0:
             print("No jobs to run.")
             Path(self.slurm_job_json).unlink(missing_ok=True)
             Path(self.configs).unlink(missing_ok=True)
-            return
+            return "N/A"
 
         with open(self.slurm_job_json, "w") as f:
             json.dump(jobs_dict, f)
 
+        run_command = f"sbatch --array=0-{n_jobs - 1}%100 {self.slurm_base_command} {self.slurm_job_json}"
         print(
             f"##### Setup Jobs for {self._safe_benchmark_name}"
             "\nRun the following command to start the jobs:"
-            f"\nsbatch --array=0-{n_jobs - 1}%100 {self.slurm_base_command} {self.slurm_job_json}"
+            f"\n{run_command}"
             "\n"
         )
+        return run_command
 
     @property
     def models_to_constraints(self) -> dict[str, dict[str, int]]:

tabflow_slurm/submit_template.sh

@@ -40,7 +40,6 @@ PYTHON_PATH=$(jq -r '.defaults.python' "$JSON_FILE")
 RUNSCRIPT=$(jq -r '.defaults.run_script' "$JSON_FILE")
 OPENML_CACHE_DIR=$(jq -r '.defaults.openml_cache_dir' "$JSON_FILE")
 CONFIGS_YAML_FILE=$(jq -r '.defaults.configs_yaml_file' "$JSON_FILE")
-TABREPO_CACHE_DIR=$(jq -r '.defaults.tabrepo_cache_dir' "$JSON_FILE")
 OUTPUT_DIR=$(jq -r '.defaults.output_dir' "$JSON_FILE")
 NUM_CPUS=$(jq -r '.defaults.num_cpus' "$JSON_FILE")
 NUM_GPUS=$(jq -r '.defaults.num_gpus' "$JSON_FILE")
@@ -53,7 +52,6 @@ echo "Python Path: $PYTHON_PATH"
 echo "Run Script: $RUNSCRIPT"
 echo "OpenML Cache Directory: $OPENML_CACHE_DIR"
 echo "Configs YAML File: $CONFIGS_YAML_FILE"
-echo "Tabrepo Cache Directory: $TABREPO_CACHE_DIR"
 echo "Output Directory: $OUTPUT_DIR"
 echo "Number of CPUs: $NUM_CPUS"
 echo "Number of GPUs: $NUM_GPUS"
@@ -89,7 +87,6 @@ for CI in "${CONFIG_ARRAY[@]}"; do
         --config_index $CI \
         --configs_yaml_file $CONFIGS_YAML_FILE \
         --openml_cache_dir $OPENML_CACHE_DIR \
-        --tabrepo_cache_dir $TABREPO_CACHE_DIR \
         --output_dir $OUTPUT_DIR \
         --num_cpus $NUM_CPUS \
         --num_gpus $NUM_GPUS \