Merge pull request #208 from CliMA/ne/fix

Fix `DerechoBackend`

Author: nefrathenrici

experiments/surface_fluxes_perfect_model/utils.jl

@@ -20,7 +20,7 @@ end
 function compare_g_ensemble(eki1, eki2)
     @testset "Compare g_ensemble between two EKP objects" begin
         for (g1, g2) in zip(eki1.g, eki2.g)
-            @test g1.stored_data == g2.stored_data
+            @test g1.stored_data ≈ g2.stored_data rtol = 1e-5
         end
     end
 end

src/pbs.jl

@@ -22,6 +22,7 @@ function generate_pbs_script(
     model_interface::AbstractString,
     module_load_str::AbstractString;
     hpc_kwargs = Dict(),
+    exeflags = "",
 )
     member_log = path_to_model_log(output_dir, iter, member)
 
@@ -46,23 +47,23 @@ function generate_pbs_script(
     member_path = path_to_ensemble_member(output_dir, iter, member)
     julia_filepath = joinpath(member_path, "model_run.jl")
 
-    pbs_script = """
-#!/bin/bash
-#PBS -N run_$(iter)_$member
-#PBS -j oe
-#PBS -A UCIT0011
-#PBS -q $queue
-#PBS -o $member_log
-#PBS -l walltime=$walltime
-#PBS -l select=$num_nodes:ncpus=$cpus_per_node:ngpus=$gpus_per_node:mpiprocs=$ranks_per_node
-
-$module_load_str
-
-export JULIA_MPI_HAS_CUDA=true
-export CLIMACOMMS_DEVICE="$climacomms_device"
-export CLIMACOMMS_CONTEXT="MPI"
-\$MPITRAMPOLINE_MPIEXEC -n $total_ranks -ppn $ranks_per_node $set_gpu_rank julia --project=$experiment_dir $julia_filepath
-"""
+    pbs_script = """\
+    #!/bin/bash
+    #PBS -N run_$(iter)_$member
+    #PBS -j oe
+    #PBS -A UCIT0011
+    #PBS -q $queue
+    #PBS -o $member_log
+    #PBS -l walltime=$walltime
+    #PBS -l select=$num_nodes:ncpus=$cpus_per_node:ngpus=$gpus_per_node:mpiprocs=$ranks_per_node
+
+    $module_load_str
+
+    export JULIA_MPI_HAS_CUDA=true
+    export CLIMACOMMS_DEVICE="$climacomms_device"
+    export CLIMACOMMS_CONTEXT="MPI"
+    \$MPITRAMPOLINE_MPIEXEC -n $total_ranks -ppn $ranks_per_node $set_gpu_rank julia $exeflags --project=$experiment_dir $julia_filepath
+    """
 
     julia_script = """\
     import ClimaCalibrate as CAL
@@ -88,6 +89,7 @@ function pbs_model_run(
     module_load_str;
     hpc_kwargs,
     debug = false,
+    exeflags = "",
 )
     # Type and existence checks
     @assert isdir(output_dir) "Output directory does not exist: $output_dir"
@@ -106,6 +108,7 @@ function pbs_model_run(
         model_interface,
         module_load_str;
         hpc_kwargs,
+        exeflags,
     )
     # TODO: Improve and figure out how to clean up the model_run.jl file
     # julia_filepath does not get cleaned up. `mktemp() do` syntax is ideal 

test/pbs_unit_tests.jl

@@ -55,7 +55,7 @@ module list
 export JULIA_MPI_HAS_CUDA=true
 export CLIMACOMMS_DEVICE="CUDA"
 export CLIMACOMMS_CONTEXT="MPI"
-\$MPITRAMPOLINE_MPIEXEC -n 4 -ppn 2 set_gpu_rank julia --project=exp/dir test/iteration_001/member_001/model_run.jl
+\$MPITRAMPOLINE_MPIEXEC -n 4 -ppn 2 set_gpu_rank julia  --project=exp/dir test/iteration_001/member_001/model_run.jl
 """
 
 for (generated_str, test_str) in