Skip to content

Commit 8301202

Browse files
nefrathenricinefrathenrici
committed
test longrun
1 parent ca1543c commit 8301202

File tree

1 file changed

+136
-129
lines changed

1 file changed

+136
-129
lines changed

‎.buildkite/longruns/pipeline.yml‎

Lines changed: 136 additions & 129 deletions
Original file line numberDiff line numberDiff line change
@@ -52,121 +52,128 @@ steps:
5252

5353
- wait
5454

55-
- group: "Coupler integration and conservation tests"
56-
57-
steps:
58-
- label: "Aquaplanet: evolving slab ocean"
59-
key: "slabplanet_aqua_evolve_ocean"
60-
command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_evolve_ocean.yml --job_id slabplanet_aqua_evolve_ocean"
61-
artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua_evolve_ocean/artifacts/*"
62-
agents:
63-
slurm_mem: 32GB
64-
65-
- label: "Slabplanet: evolving slab ocean, bucket"
66-
key: "slabplanet_evolve_ocean"
67-
command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_evolve_ocean.yml --job_id slabplanet_evolve_ocean"
68-
artifact_paths: "experiments/ClimaEarth/output/slabplanet_evolve_ocean/artifacts/*"
69-
agents:
70-
slurm_mem: 32GB
71-
72-
- group: "AMIP simulations"
73-
74-
steps:
75-
76-
- label: "GPU AMIP + ED only"
77-
key: "amip_edonly_gpu"
78-
command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_edonly.yml --job_id amip_edonly_gpu"
79-
artifact_paths: "experiments/ClimaEarth/output/amip_edonly_gpu/artifacts/*"
80-
env:
81-
CLIMACOMMS_DEVICE: "CUDA"
82-
agents:
83-
queue: clima
84-
slurm_gpus_per_task: 1
85-
slurm_cpus_per_task: 4
86-
slurm_ntasks: 1
87-
slurm_mem: 32GB
88-
89-
- label: "GPU AMIP + ED only + 1M microphysics"
90-
key: "amip_edonly_1M_gpu"
91-
command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_edonly_1M.yml --job_id amip_edonly_1M_gpu"
92-
artifact_paths: "experiments/ClimaEarth/output/amip_edonly_1M_gpu/artifacts/*"
93-
env:
94-
CLIMACOMMS_DEVICE: "CUDA"
95-
agents:
96-
queue: clima
97-
slurm_gpus_per_task: 1
98-
slurm_cpus_per_task: 4
99-
slurm_ntasks: 1
100-
slurm_mem: 32GB
101-
soft_fail: true
102-
103-
- label: "GPU AMIP + ED only + integrated land"
104-
key: "amip_edonly_integrated_land_gpu" # runs for 4 months only because of instability after that time
105-
command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_edonly_integrated_land.yml --job_id amip_edonly_integrated_land_gpu"
106-
artifact_paths: "experiments/ClimaEarth/output/amip_edonly_integrated_land_gpu/artifacts/*"
107-
env:
108-
CLIMACOMMS_DEVICE: "CUDA"
109-
agents:
110-
queue: clima
111-
slurm_gpus_per_task: 1
112-
slurm_cpus_per_task: 4
113-
slurm_ntasks: 1
114-
slurm_mem: 32GB
115-
116-
- label: "GPU AMIP + diag. EDMF"
117-
key: "amip_diagedmf_gpu"
118-
command:
119-
- "julia --threads=3 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_diagedmf.yml --job_id amip_diagedmf_gpu"
120-
artifact_paths: "experiments/ClimaEarth/output/amip_diagedmf_gpu/artifacts/*"
121-
env:
122-
CLIMACOMMS_DEVICE: "CUDA"
123-
agents:
124-
queue: clima
125-
slurm_gpus_per_task: 1
126-
slurm_cpus_per_task: 4
127-
slurm_ntasks: 1
128-
slurm_mem: 32GB
129-
130-
- label: "GPU AMIP + diag. EDMF + integrated land"
131-
key: "amip_diagedmf_integrated_land_gpu" # runs for 2 months only because of instability after that time
132-
command:
133-
- "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_diagedmf_integrated_land.yml --job_id amip_diagedmf_integrated_land_gpu"
134-
artifact_paths: "experiments/ClimaEarth/output/amip_diagedmf_integrated_land_gpu/artifacts/*"
135-
env:
136-
CLIMACOMMS_DEVICE: "CUDA"
137-
agents:
138-
queue: clima
139-
slurm_gpus_per_task: 1
140-
slurm_cpus_per_task: 4
141-
slurm_ntasks: 1
142-
slurm_mem: 32GB
143-
144-
- group: "CMIP simulations"
145-
146-
steps:
147-
- label: "GPU CMIP: ClimaAtmos + ClimaLand + ClimaOcean + PrescribedSeaIce"
148-
key: "cmip_edonly_land"
149-
command:
150-
- "julia --color=yes --project=experiments/ClimaEarth experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/cmip_edonly_land.yml --job_id cmip_edonly_land"
151-
artifact_paths: "experiments/ClimaEarth/output/cmip_edonly_land/artifacts/*"
152-
env:
153-
CLIMACOMMS_DEVICE: "CUDA"
154-
agents:
155-
queue: clima
156-
slurm_mem: 32GB
157-
slurm_gpus: 1
158-
159-
- label: "GPU CMIP: ClimaAtmos + Bucket + ClimaOcean + PrescribedSeaIce"
160-
key: "cmip_edonly_bucket"
161-
command:
162-
- "julia --color=yes --project=experiments/ClimaEarth experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/cmip_edonly_bucket.yml --job_id cmip_edonly_bucket"
163-
artifact_paths: "experiments/ClimaEarth/output/cmip_edonly_bucket/artifacts/*"
164-
env:
165-
CLIMACOMMS_DEVICE: "CUDA"
166-
agents:
167-
queue: clima
168-
slurm_mem: 32GB
169-
slurm_gpus: 1
55+
# - group: "Coupler integration and conservation tests"
56+
57+
# steps:
58+
# - label: "Aquaplanet: evolving slab ocean"
59+
# key: "slabplanet_aqua_evolve_ocean"
60+
# command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_evolve_ocean.yml --job_id slabplanet_aqua_evolve_ocean"
61+
# artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua_evolve_ocean/artifacts/*"
62+
# agents:
63+
# slurm_mem: 32GB
64+
65+
# - label: "Slabplanet: evolving slab ocean, bucket"
66+
# key: "slabplanet_evolve_ocean"
67+
# command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_evolve_ocean.yml --job_id slabplanet_evolve_ocean"
68+
# artifact_paths: "experiments/ClimaEarth/output/slabplanet_evolve_ocean/artifacts/*"
69+
# agents:
70+
# slurm_mem: 32GB
71+
72+
# - group: "AMIP simulations"
73+
74+
# steps:
75+
76+
# - label: "GPU AMIP + ED only"
77+
# key: "amip_edonly_gpu"
78+
# command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_edonly.yml --job_id amip_edonly_gpu"
79+
# artifact_paths: "experiments/ClimaEarth/output/amip_edonly_gpu/artifacts/*"
80+
# env:
81+
# CLIMACOMMS_CONTEXT: "MPI"
82+
# CLIMACOMMS_DEVICE: "CUDA"
83+
# agents:
84+
# queue: clima
85+
# slurm_gpus_per_task: 1
86+
# slurm_cpus_per_task: 4
87+
# slurm_ntasks: 1
88+
# slurm_mem: 32GB
89+
90+
# - label: "GPU AMIP + ED only + 1M microphysics"
91+
# key: "amip_edonly_1M_gpu"
92+
# command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_edonly_1M.yml --job_id amip_edonly_1M_gpu"
93+
# artifact_paths: "experiments/ClimaEarth/output/amip_edonly_1M_gpu/artifacts/*"
94+
# env:
95+
# CLIMACOMMS_CONTEXT: "MPI"
96+
# CLIMACOMMS_DEVICE: "CUDA"
97+
# agents:
98+
# queue: clima
99+
# slurm_gpus_per_task: 1
100+
# slurm_cpus_per_task: 4
101+
# slurm_ntasks: 1
102+
# slurm_mem: 32GB
103+
# soft_fail: true
104+
105+
# - label: "GPU AMIP + ED only + integrated land"
106+
# key: "amip_edonly_integrated_land_gpu" # runs for 4 months only because of instability after that time
107+
# command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_edonly_integrated_land.yml --job_id amip_edonly_integrated_land_gpu"
108+
# artifact_paths: "experiments/ClimaEarth/output/amip_edonly_integrated_land_gpu/artifacts/*"
109+
# env:
110+
# CLIMACOMMS_CONTEXT: "MPI"
111+
# CLIMACOMMS_DEVICE: "CUDA"
112+
# agents:
113+
# queue: clima
114+
# slurm_gpus_per_task: 1
115+
# slurm_cpus_per_task: 4
116+
# slurm_ntasks: 1
117+
# slurm_mem: 32GB
118+
119+
# - label: "GPU AMIP + diag. EDMF"
120+
# key: "amip_diagedmf_gpu"
121+
# command:
122+
# - "srun --cpu-bind=threads --cpus-per-task=4 julia --threads=3 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_diagedmf.yml --job_id amip_diagedmf_gpu"
123+
# artifact_paths: "experiments/ClimaEarth/output/amip_diagedmf_gpu/artifacts/*"
124+
# env:
125+
# CLIMACOMMS_CONTEXT: "MPI"
126+
# CLIMACOMMS_DEVICE: "CUDA"
127+
# agents:
128+
# queue: clima
129+
# slurm_gpus_per_task: 1
130+
# slurm_cpus_per_task: 4
131+
# slurm_ntasks: 1
132+
# slurm_mem: 32GB
133+
134+
# - label: "GPU AMIP + diag. EDMF + integrated land"
135+
# key: "amip_diagedmf_integrated_land_gpu" # runs for 2 months only because of instability after that time
136+
# command:
137+
# - "srun --cpu-bind=threads --cpus-per-task=4 julia --threads=3 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_diagedmf_integrated_land.yml --job_id amip_diagedmf_integrated_land_gpu"
138+
# artifact_paths: "experiments/ClimaEarth/output/amip_diagedmf_integrated_land_gpu/artifacts/*"
139+
# env:
140+
# CLIMACOMMS_CONTEXT: "MPI"
141+
# CLIMACOMMS_DEVICE: "CUDA"
142+
# agents:
143+
# queue: clima
144+
# slurm_gpus_per_task: 1
145+
# slurm_cpus_per_task: 4
146+
# slurm_ntasks: 1
147+
# slurm_mem: 32GB
148+
149+
# - group: "CMIP simulations"
150+
151+
# steps:
152+
# - label: "GPU CMIP: ClimaAtmos + ClimaLand + ClimaOcean + PrescribedSeaIce"
153+
# key: "cmip_edonly_land"
154+
# command:
155+
# - "julia --color=yes --project=experiments/ClimaEarth experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/cmip_edonly_land.yml --job_id cmip_edonly_land"
156+
# artifact_paths: "experiments/ClimaEarth/output/cmip_edonly_land/artifacts/*"
157+
# env:
158+
# CLIMACOMMS_DEVICE: "CUDA"
159+
# CLIMACOMMS_CONTEXT: "SINGLETON"
160+
# agents:
161+
# queue: clima
162+
# slurm_mem: 32GB
163+
# slurm_gpus: 1
164+
165+
# - label: "GPU CMIP: ClimaAtmos + Bucket + ClimaOcean + PrescribedSeaIce"
166+
# key: "cmip_edonly_bucket"
167+
# command:
168+
# - "julia --color=yes --project=experiments/ClimaEarth experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/cmip_edonly_bucket.yml --job_id cmip_edonly_bucket"
169+
# artifact_paths: "experiments/ClimaEarth/output/cmip_edonly_bucket/artifacts/*"
170+
# env:
171+
# CLIMACOMMS_DEVICE: "CUDA"
172+
# CLIMACOMMS_CONTEXT: "SINGLETON"
173+
# agents:
174+
# queue: clima
175+
# slurm_mem: 32GB
176+
# slurm_gpus: 1
170177

171178
- group: "Calibration experiments"
172179

@@ -181,22 +188,22 @@ steps:
181188
queue: clima
182189
slurm_time: 24:00:00
183190

184-
- wait
191+
# - wait
185192

186-
- group: "Job analysis and reporting"
193+
# - group: "Job analysis and reporting"
187194

188-
steps:
195+
# steps:
189196

190-
# plot job performance history
191-
- label: ":chart_with_downwards_trend: build history"
192-
command:
193-
- build_history main # name of branch to plot
194-
artifact_paths:
195-
- "build_history.html"
197+
# # plot job performance history
198+
# - label: ":chart_with_downwards_trend: build history"
199+
# command:
200+
# - build_history main # name of branch to plot
201+
# artifact_paths:
202+
# - "build_history.html"
196203

197-
- wait
204+
# - wait
198205

199-
- label: ":envelope: Slack report: Slabplanet"
200-
command:
201-
- slack-upload -c C04S6MBSDC4 -f experiments/ClimaEarth/output/slabplanet_evolve_ocean/artifacts/total_energy_bucket.png -n slab_coarse -x "Slabplanet energy conservation"
202-
- slack-upload -c C04S6MBSDC4 -f experiments/ClimaEarth/output/slabplanet_evolve_ocean/artifacts/total_water_bucket.png -n slab_coarse_w -x "Slabplanet water conservation"
206+
# - label: ":envelope: Slack report: Slabplanet"
207+
# command:
208+
# - slack-upload -c C04S6MBSDC4 -f experiments/ClimaEarth/output/slabplanet_evolve_ocean/artifacts/total_energy_bucket.png -n slab_coarse -x "Slabplanet energy conservation"
209+
# - slack-upload -c C04S6MBSDC4 -f experiments/ClimaEarth/output/slabplanet_evolve_ocean/artifacts/total_water_bucket.png -n slab_coarse_w -x "Slabplanet water conservation"

0 commit comments

Comments
 (0)