Skip to content

Commit ba3cf37

Browse files
SbozzoloSbozzolo
authored
Merge pull request #2606 from CliMA/gb/remove_existing_folders
Use ClimaUtilities.OutputPathGenerator to organize output directories
2 parents 1c4e63f + b2b6c96 commit ba3cf37

File tree

14 files changed

+414
-287
lines changed

14 files changed

+414
-287
lines changed

‎.buildkite/gpu_pipeline/pipeline.yml‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ steps:
5050
nsys profile --trace=nvtx,mpi,cuda,osrt --output=target_gpu_implicit_baroclinic_wave/report
5151
julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl
5252
--config_file ${GPU_CONFIG_PATH}target_gpu_implicit_baroclinic_wave.yml
53-
artifact_paths: "target_gpu_implicit_baroclinic_wave/*"
53+
artifact_paths: "target_gpu_implicit_baroclinic_wave/output_active/*"
5454
agents:
5555
slurm_gpus: 1
5656
slurm_cpus_per_task: 4
@@ -64,7 +64,7 @@ steps:
6464
nsys profile --trace=nvtx,mpi,cuda,osrt --output=gpu_hs_rhoe_equil_55km_nz63_0M/report
6565
julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl
6666
--config_file ${GPU_CONFIG_PATH}gpu_hs_rhoe_equil_55km_nz63_0M.yml
67-
artifact_paths: "gpu_hs_rhoe_equil_55km_nz63_0M/*"
67+
artifact_paths: "gpu_hs_rhoe_equil_55km_nz63_0M/output_active/*"
6868
agents:
6969
slurm_gpus: 1
7070
slurm_cpus_per_task: 4
@@ -79,7 +79,7 @@ steps:
7979
nsys profile --trace=nvtx,mpi,cuda,osrt --output=gpu_hs_rhoe_equil_55km_nz63_0M_4process/report-%q{PMI_RANK}
8080
julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl
8181
--config_file ${GPU_CONFIG_PATH}gpu_hs_rhoe_equil_55km_nz63_0M_4process.yml
82-
artifact_paths: "gpu_hs_rhoe_equil_55km_nz63_0M_4process/*"
82+
artifact_paths: "gpu_hs_rhoe_equil_55km_nz63_0M_4process/output_active/*"
8383
agents:
8484
slurm_gpus_per_task: 1
8585
slurm_cpus_per_task: 4
@@ -95,7 +95,7 @@ steps:
9595
nsys profile --trace=osrt,nvtx,cuda,mpi,ucx --output=target_gpu_implicit_baroclinic_wave_4process/report-%q{PMI_RANK}
9696
julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl
9797
--config_file ${GPU_CONFIG_PATH}target_gpu_implicit_baroclinic_wave_4process.yml
98-
artifact_paths: "target_gpu_implicit_baroclinic_wave_4process/*"
98+
artifact_paths: "target_gpu_implicit_baroclinic_wave_4process/output_active/*"
9999
agents:
100100
slurm_gpus_per_task: 1
101101
slurm_cpus_per_task: 4
@@ -112,7 +112,7 @@ steps:
112112
srun --cpu-bind=threads --cpus-per-task=4
113113
julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl
114114
--config_file ${GPU_CONFIG_PATH}gpu_aquaplanet_dyamond_ss_1process.yml
115-
artifact_paths: "gpu_aquaplanet_dyamond_ss_1process/*"
115+
artifact_paths: "gpu_aquaplanet_dyamond_ss_1process/output_active/*"
116116
agents:
117117
slurm_gpus_per_task: 1
118118
slurm_cpus_per_task: 4
@@ -128,7 +128,7 @@ steps:
128128
srun --cpu-bind=threads --cpus-per-task=4
129129
julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl
130130
--config_file ${GPU_CONFIG_PATH}gpu_aquaplanet_dyamond_ss_2process.yml
131-
artifact_paths: "gpu_aquaplanet_dyamond_ss_2process/*"
131+
artifact_paths: "gpu_aquaplanet_dyamond_ss_2process/output_active/*"
132132
agents:
133133
slurm_gpus_per_task: 1
134134
slurm_cpus_per_task: 4
@@ -144,7 +144,7 @@ steps:
144144
srun --cpu-bind=threads --cpus-per-task=4
145145
julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl
146146
--config_file ${GPU_CONFIG_PATH}gpu_aquaplanet_dyamond_ss_4process.yml
147-
artifact_paths: "gpu_aquaplanet_dyamond_ss_4process/*"
147+
artifact_paths: "gpu_aquaplanet_dyamond_ss_4process/output_active/*"
148148
agents:
149149
slurm_gpus_per_task: 1
150150
slurm_cpus_per_task: 4
@@ -160,7 +160,7 @@ steps:
160160
- mkdir -p gpu_aquaplanet_dyamond_ss
161161
- >
162162
julia --color=yes --project=examples post_processing/plot_gpu_strong_scaling.jl gpu_aquaplanet_dyamond_ss
163-
artifact_paths: "gpu_aquaplanet_dyamond_ss/*"
163+
artifact_paths: "gpu_aquaplanet_dyamond_ss/output_active/*"
164164
agents:
165165
slurm_cpus_per_task: 1
166166
slurm_ntasks: 1
@@ -176,7 +176,7 @@ steps:
176176
srun --cpu-bind=threads --cpus-per-task=4
177177
julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl
178178
--config_file ${GPU_CONFIG_PATH}gpu_aquaplanet_dyamond_ws_1process.yml
179-
artifact_paths: "gpu_aquaplanet_dyamond_ws_1process/*"
179+
artifact_paths: "gpu_aquaplanet_dyamond_ws_1process/output_active/*"
180180
agents:
181181
slurm_gpus_per_task: 1
182182
slurm_cpus_per_task: 4
@@ -192,7 +192,7 @@ steps:
192192
srun --cpu-bind=threads --cpus-per-task=4
193193
julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl
194194
--config_file ${GPU_CONFIG_PATH}gpu_aquaplanet_dyamond_ws_2process.yml
195-
artifact_paths: "gpu_aquaplanet_dyamond_ws_2process/*"
195+
artifact_paths: "gpu_aquaplanet_dyamond_ws_2process/output_active/*"
196196
agents:
197197
slurm_gpus_per_task: 1
198198
slurm_cpus_per_task: 4
@@ -208,7 +208,7 @@ steps:
208208
srun --cpu-bind=threads --cpus-per-task=4
209209
julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl
210210
--config_file ${GPU_CONFIG_PATH}gpu_aquaplanet_dyamond_ws_4process.yml
211-
artifact_paths: "gpu_aquaplanet_dyamond_ws_4process/*"
211+
artifact_paths: "gpu_aquaplanet_dyamond_ws_4process/output_active/*"
212212
agents:
213213
slurm_gpus_per_task: 1
214214
slurm_cpus_per_task: 4
@@ -224,7 +224,7 @@ steps:
224224
- mkdir -p gpu_aquaplanet_dyamond_ws
225225
- >
226226
julia --color=yes --project=examples post_processing/plot_gpu_weak_scaling.jl gpu_aquaplanet_dyamond_ws
227-
artifact_paths: "gpu_aquaplanet_dyamond_ws/*"
227+
artifact_paths: "gpu_aquaplanet_dyamond_ws/output_active/*"
228228
agents:
229229
slurm_cpus_per_task: 1
230230
slurm_ntasks: 1
@@ -242,7 +242,7 @@ steps:
242242
nsys profile --trace=nvtx,mpi,cuda,osrt --output=gpu_aquaplanet_diagedmf/report
243243
julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl
244244
--config_file ${GPU_CONFIG_PATH}gpu_aquaplanet_diagedmf.yml
245-
artifact_paths: "gpu_aquaplanet_diagedmf/*"
245+
artifact_paths: "gpu_aquaplanet_diagedmf/output_active/*"
246246
agents:
247247
slurm_gpus_per_task: 1
248248
slurm_cpus_per_task: 4

‎.buildkite/longruns/pipeline.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ steps:
4747
# - label: ":computer: lim ARS zalesak baroclinic wave (ρe_tot) equilmoist high resolution"
4848
# command:
4949
# - srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml
50-
# artifact_paths: "$$JOB_NAME/*"
50+
# artifact_paths: "$$JOB_NAME/output_active/*"
5151
# agents:
5252
# slurm_ntasks: 32
5353
# slurm_mem_per_cpu: 32GB
@@ -58,7 +58,7 @@ steps:
5858
- label: ":computer: SSP baroclinic wave (ρe_tot) equilmoist high resolution centered diff"
5959
command:
6060
- "srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml"
61-
artifact_paths: "$$JOB_NAME/*"
61+
artifact_paths: "$$JOB_NAME/output_active/*"
6262
agents:
6363
slurm_ntasks: 32
6464
slurm_nodes: 2
@@ -85,7 +85,7 @@ steps:
8585
- label: ":computer: low resolution aquaplanet equilmoist clearsky radiation + time-varying insolation + slab ocean"
8686
command:
8787
- srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml
88-
artifact_paths: "$$JOB_NAME/*"
88+
artifact_paths: "$$JOB_NAME/output_active/*"
8989
agents:
9090
slurm_ntasks: 8
9191
slurm_mem_per_cpu: 16GB

‎.buildkite/longruns_gpu/pipeline.yml‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ steps:
4545
- label: ":computer: hydrostatic balance (ρe_tot)"
4646
command:
4747
- srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml
48-
artifact_paths: "$$JOB_NAME/*"
48+
artifact_paths: "$$JOB_NAME/output_active/*"
4949
agents:
5050
slurm_gpus: 1
5151
slurm_time: 12:00:00
@@ -55,7 +55,7 @@ steps:
5555
- label: ":computer: dry baroclinic wave"
5656
command:
5757
- srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml
58-
artifact_paths: "$$JOB_NAME/*"
58+
artifact_paths: "$$JOB_NAME/output_active/*"
5959
agents:
6060
slurm_gpus: 1
6161
slurm_time: 12:00:00
@@ -65,7 +65,7 @@ steps:
6565
- label: ":computer: baroclinic wave equilmoist"
6666
command:
6767
- srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml
68-
artifact_paths: "$$JOB_NAME/*"
68+
artifact_paths: "$$JOB_NAME/output_active/*"
6969
agents:
7070
slurm_gpus: 1
7171
slurm_time: 12:00:00
@@ -75,7 +75,7 @@ steps:
7575
- label: ":computer: dry held-suarez"
7676
command:
7777
- srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml
78-
artifact_paths: "$$JOB_NAME/*"
78+
artifact_paths: "$$JOB_NAME/output_active/*"
7979
agents:
8080
slurm_gpus: 1
8181
slurm_time: 12:00:00
@@ -85,7 +85,7 @@ steps:
8585
- label: ":computer: held-suarez, equilmoist"
8686
command:
8787
- srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml
88-
artifact_paths: "$$JOB_NAME/*"
88+
artifact_paths: "$$JOB_NAME/output_active/*"
8989
agents:
9090
slurm_gpus: 1
9191
slurm_time: 12:00:00
@@ -95,7 +95,7 @@ steps:
9595
- label: ":computer: aquaplanet equilmoist gray radiation + 0M microphysics"
9696
command:
9797
- srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml
98-
artifact_paths: "$$JOB_NAME/*"
98+
artifact_paths: "$$JOB_NAME/output_active/*"
9999
env:
100100
CLIMACORE_DISTRIBUTED: "MPI"
101101
JOB_NAME: "longrun_aquaplanet_rhoe_equil_55km_nz63_gray_0M"
@@ -106,7 +106,7 @@ steps:
106106
- label: ":computer: aquaplanet equilmoist clearsky radiation + diagnostic edmf diffusion only + 0M microphysics"
107107
command:
108108
- srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml
109-
artifact_paths: "$$JOB_NAME/*"
109+
artifact_paths: "$$JOB_NAME/output_active/*"
110110
agents:
111111
slurm_gpus: 1
112112
slurm_time: 12:00:00
@@ -116,7 +116,7 @@ steps:
116116
- label: ":computer: aquaplanet equilmoist clearsky radiation + time-varying insolation + 0M microphysics + slab ocean"
117117
command:
118118
- srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml
119-
artifact_paths: "$$JOB_NAME/*"
119+
artifact_paths: "$$JOB_NAME/output_active/*"
120120
agents:
121121
slurm_gpus: 1
122122
slurm_time: 12:00:00
@@ -126,7 +126,7 @@ steps:
126126
- label: ":computer: held-suarez equilmoist + deep-atmosphere eqns"
127127
command:
128128
- srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml
129-
artifact_paths: "$$JOB_NAME/*"
129+
artifact_paths: "$$JOB_NAME/output_active/*"
130130
env:
131131
CLIMACORE_DISTRIBUTED: "MPI"
132132
JOB_NAME: "longrun_hs_rhoe_equil_55km_nz63_0M_deepatmos"
@@ -137,7 +137,7 @@ steps:
137137
- label: ":computer: aquaplanet equilmoist clearsky radiation + 0M microphysics"
138138
command:
139139
- srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml
140-
artifact_paths: "$$JOB_NAME/*"
140+
artifact_paths: "$$JOB_NAME/output_active/*"
141141
agents:
142142
slurm_gpus: 1
143143
slurm_time: 12:00:00
@@ -147,7 +147,7 @@ steps:
147147
- label: ":computer: aquaplanet equilmoist clearsky radiation + diagnostic edmf + 0M microphysics"
148148
command:
149149
- srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml
150-
artifact_paths: "$$JOB_NAME/*"
150+
artifact_paths: "$$JOB_NAME/output_active/*"
151151
agents:
152152
slurm_gpus: 1
153153
slurm_time: 12:00:00
@@ -157,7 +157,7 @@ steps:
157157
- label: ":computer: aquaplanet equilmoist allsky radiation + diagnostic edmf + 0M microphysics"
158158
command:
159159
- srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml
160-
artifact_paths: "$$JOB_NAME/*"
160+
artifact_paths: "$$JOB_NAME/output_active/*"
161161
agents:
162162
slurm_gpus: 1
163163
slurm_time: 12:00:00
@@ -167,7 +167,7 @@ steps:
167167
- label: ":computer: aquaplanet equilmoist clearsky radiation + 0M microphysics + earth topography"
168168
command:
169169
- srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml
170-
artifact_paths: "$$JOB_NAME/*"
170+
artifact_paths: "$$JOB_NAME/output_active/*"
171171
agents:
172172
slurm_gpus: 1
173173
slurm_time: 12:00:00
@@ -177,7 +177,7 @@ steps:
177177
- label: ":computer: aquaplanet equilmoist clearsky radiation + 0M microphysics + earth topography (SLEVE)"
178178
command:
179179
- srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml
180-
artifact_paths: "$$JOB_NAME/*"
180+
artifact_paths: "$$JOB_NAME/output_active/*"
181181
agents:
182182
slurm_gpus: 1
183183
slurm_time: 6:00:00
@@ -187,7 +187,7 @@ steps:
187187
- label: ":umbrella: aquaplanet equilmoist clearsky radiation + 1M microphysics"
188188
command:
189189
- srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml
190-
artifact_paths: "$$JOB_NAME/*"
190+
artifact_paths: "$$JOB_NAME/output_active/*"
191191
agents:
192192
slurm_gpus: 1
193193
slurm_time: 6:00:00
@@ -201,7 +201,7 @@ steps:
201201
- label: ":computer: aquaplanet dyamond"
202202
command:
203203
- srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml
204-
artifact_paths: "$$JOB_NAME/*"
204+
artifact_paths: "$$JOB_NAME/output_active/*"
205205
agents:
206206
slurm_gpus: 1
207207
slurm_time: 12:00:00
@@ -215,7 +215,7 @@ steps:
215215
- label: ":computer: amip target diagnostic edmf"
216216
command:
217217
- srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml
218-
artifact_paths: "$$JOB_NAME/*"
218+
artifact_paths: "$$JOB_NAME/output_active/*"
219219
agents:
220220
slurm_gpus: 1
221221
slurm_time: 12:00:00

0 commit comments

Comments
 (0)