diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index e3b126f545..5d66066209 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -44,764 +44,772 @@ steps: - label: ":computer: Test reproducibility infrastructure" command: "julia --color=yes --project=.buildkite test/unit_reproducibility_infra.jl" - - group: "Radiation" - steps: - - - label: ":computer: single column radiative equilibrium gray" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/single_column_radiative_equilibrium_gray.yml - --job_id single_column_radiative_equilibrium_gray - artifact_paths: "single_column_radiative_equilibrium_gray/output_active/*" - - - label: ":computer: single column radiative equilibrium clearsky" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/single_column_radiative_equilibrium_clearsky.yml - --job_id single_column_radiative_equilibrium_clearsky - artifact_paths: "single_column_radiative_equilibrium_clearsky/output_active/*" - - - label: ":computer: single column radiative equilibrium clearsky prognostic surface temperature" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/single_column_radiative_equilibrium_clearsky_prognostic_surface_temp.yml - --job_id single_column_radiative_equilibrium_clearsky_prognostic_surface_temp - artifact_paths: "single_column_radiative_equilibrium_clearsky_prognostic_surface_temp/output_active/*" - - - label: ":computer: single column radiative equilibrium allsky idealized clouds varying insolation" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/single_column_radiative_equilibrium_allsky_idealized_clouds.yml - --job_id single_column_radiative_equilibrium_allsky_idealized_clouds - artifact_paths: "single_column_radiative_equilibrium_allsky_idealized_clouds/output_active/*" - - - group: "Precipitation" - steps: - - label: ":umbrella: 1-moment precipitation sanity test single column" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/single_column_precipitation_test.yml - --job_id single_column_precipitation_test - artifact_paths: "single_column_precipitation_test/output_active/*" + # - group: "Radiation" + # steps: + + # - label: ":computer: single column radiative equilibrium gray" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/single_column_radiative_equilibrium_gray.yml + # --job_id single_column_radiative_equilibrium_gray + # artifact_paths: "single_column_radiative_equilibrium_gray/output_active/*" + + # - label: ":computer: single column radiative equilibrium clearsky" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/single_column_radiative_equilibrium_clearsky.yml + # --job_id single_column_radiative_equilibrium_clearsky + # artifact_paths: "single_column_radiative_equilibrium_clearsky/output_active/*" + + # - label: ":computer: single column radiative equilibrium clearsky prognostic surface temperature" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/single_column_radiative_equilibrium_clearsky_prognostic_surface_temp.yml + # --job_id single_column_radiative_equilibrium_clearsky_prognostic_surface_temp + # artifact_paths: "single_column_radiative_equilibrium_clearsky_prognostic_surface_temp/output_active/*" + + # - label: ":computer: single column radiative equilibrium allsky idealized clouds varying insolation" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/single_column_radiative_equilibrium_allsky_idealized_clouds.yml + # --job_id single_column_radiative_equilibrium_allsky_idealized_clouds + # artifact_paths: "single_column_radiative_equilibrium_allsky_idealized_clouds/output_active/*" + + # - group: "Precipitation" + # steps: + # - label: ":umbrella: 1-moment precipitation sanity test single column" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/single_column_precipitation_test.yml + # --job_id single_column_precipitation_test + # artifact_paths: "single_column_precipitation_test/output_active/*" - - label: ":umbrella: 2-moment precipitation sanity test single column" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/single_column_precipitation_2M_test.yml - --job_id single_column_precipitation_2M_test - artifact_paths: "single_column_precipitation_2M_test/output_active/*" - - - group: "Gravity wave" - steps: - - - label: ":computer: non-orographic gravity wave parameterization unit test 3d" - command: "julia --color=yes --project=.buildkite test/parameterized_tendencies/gravity_wave/non_orographic_gravity_wave/nogw_test_3d.jl" - artifact_paths: "nonorographic_gravity_wave_test_3d/*" - agents: - slurm_mem: 20GB - - - label: ":computer: non-orographic gravity wave parameterization test with MiMA output" - command: "julia --color=yes --project=.buildkite test/parameterized_tendencies/gravity_wave/non_orographic_gravity_wave/nogw_test_mima.jl" - artifact_paths: "nonorographic_gravity_wave_test_mima/*" - agents: - slurm_mem: 20GB - - - label: ":computer: non-orographic gravity wave parameterization unit test single column" - command: "julia --color=yes --project=.buildkite test/parameterized_tendencies/gravity_wave/non_orographic_gravity_wave/nogw_test_single_column.jl" - artifact_paths: "nonorographic_gravity_wave_test_single_column/*" - - - label: ":computer: orographic gravity wave parameterization unit test for base flux calculation" - command: "julia --color=yes --project=.buildkite test/parameterized_tendencies/gravity_wave/orographic_gravity_wave/ogwd_baseflux.jl" - artifact_paths: "orographic_gravity_wave_test_baseflux/*" - - - label: ":computer: orographic gravity wave parameterization unit test for 3d calculation" - command: "julia --color=yes --project=.buildkite test/parameterized_tendencies/gravity_wave/orographic_gravity_wave/ogwd_3d.jl" - artifact_paths: "orographic_gravity_wave_test_3d/*" - - - label: ":computer: single column non-orographic gravity wave parameterization" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/single_column_nonorographic_gravity_wave.yml - --job_id single_column_nonorographic_gravity_wave - artifact_paths: "single_column_nonorographic_gravity_wave/*" - - - group: "Column Examples" - steps: - - - label: ":computer: single column hydrostatic balance float64" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/single_column_hydrostatic_balance_ft64.yml - --job_id single_column_hydrostatic_balance_ft64 - - julia --color=yes --project=.buildkite reproducibility_tests/test_mse.jl - --job_id single_column_hydrostatic_balance_ft64 - --out_dir single_column_hydrostatic_balance_ft64/output_active - artifact_paths: "single_column_hydrostatic_balance_ft64/output_active/*" - agents: - slurm_constraint: icelake|cascadelake|skylake|epyc - - - group: "Box Examples" - steps: - - - label: ":computer: Box hydrostatic balance" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/box_hydrostatic_balance.yml - --job_id box_hydrostatic_balance - artifact_paths: "box_hydrostatic_balance/output_active/*" - - - label: ":computer: 3D density current" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/box_density_current_test.yml - --job_id box_density_current_test - artifact_paths: "box_density_current_test/output_active/*" - - - label: ":computer: Box rcemipii with diagnostic edmf" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/rcemipii_box_diagnostic_edmfx.yml - --job_id rcemipii_box_diagnostic_edmfx - artifact_paths: "rcemipii_box_diagnostic_edmfx/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: LES ISDAC in a box" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/les_isdac_box.yml - --job_id les_isdac_box - artifact_paths: "les_isdac_box/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_mem: 16G - slurm_gpus: 1 - soft_fail: true - - - group: "Plane Examples" - steps: - - - label: ":computer: Density current experiment" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/plane_density_current_test.yml - --job_id plane_density_current_test - artifact_paths: "plane_density_current_test/output_active/*" - - - group: "Analytic Tests" - steps: - - label: "GPU: No Topography Test (2D, Float64, Discrete Balance)" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/plane_no_topography_float64_test.yml - --job_id gpu_plane_no_topography_float64_test - artifact_paths: "gpu_plane_no_topography_float64_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "GPU: Cosine Hills Test (2D, Float64)" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/plane_cosine_hills_float64_test.yml - --job_id gpu_plane_cosine_hills_float64_test - artifact_paths: "gpu_plane_cosine_hills_float64_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "GPU: Cosine Hills Test (Extruded 2D, Float64)" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/extruded_plane_cosine_hills_float64_test.yml - --job_id gpu_extruded_plane_cosine_hills_float64_test - artifact_paths: "gpu_extruded_plane_cosine_hills_float64_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 48GB - - - label: "GPU: Cosine Hills Test (3D, Float64)" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/box_cosine_hills_float64_test.yml - --job_id gpu_box_cosine_hills_float64_test - artifact_paths: "gpu_box_cosine_hills_float64_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 48GB - - - label: "GPU: Agnesi Mountain Test (2D, Float64)" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/plane_agnesi_mountain_float64_test.yml - --job_id gpu_plane_agnesi_mountain_float64_test - artifact_paths: "gpu_plane_agnesi_mountain_float64_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "GPU: Schar Mountain Test (2D, Float64)" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/plane_schar_mountain_float64_test.yml - --job_id gpu_plane_schar_mountain_float64_test - artifact_paths: "gpu_plane_schar_mountain_float64_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "GPU: Schar Mountain Test (2D, Float32)" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/plane_schar_mountain_float32_test.yml - --job_id gpu_plane_schar_mountain_float32_test - artifact_paths: "gpu_plane_schar_mountain_float32_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - group: "Conservation check" - steps: - - - label: ":computer: baroclinic wave check conservation" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl --config_file $CONFIG_PATH/baroclinic_wave_conservation.yml - --job_id baroclinic_wave_conservation - artifact_paths: "baroclinic_wave_conservation/output_active/*" - - - label: ":computer: baroclinic wave moist check conservation" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl --config_file $CONFIG_PATH/baroclinic_wave_equil_conservation.yml - --job_id baroclinic_wave_equil_conservation - artifact_paths: "baroclinic_wave_equil_conservation/output_active/*" - - - label: ":computer: baroclinic wave moist check conservation float64" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl --config_file $CONFIG_PATH/baroclinic_wave_equil_conservation_ft64.yml - --job_id baroclinic_wave_equil_conservation_ft64 - artifact_paths: "baroclinic_wave_equil_conservation_ft64/output_active/*" - - - label: ":computer: baroclinic wave moist check conservation float64 sparse autodiff" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl --config_file $CONFIG_PATH/baroclinic_wave_equil_conservation_ft64_sparse_autodiff.yml - --job_id baroclinic_wave_equil_conservation_ft64_sparse_autodiff - artifact_paths: "baroclinic_wave_equil_conservation_ft64_sparse_autodiff/output_active/*" - soft_fail: true - - - label: ":computer: baroclinic wave moist check conservation with sources" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl --config_file $CONFIG_PATH/baroclinic_wave_equil_conservation_source.yml - --job_id baroclinic_wave_equil_conservation_source - artifact_paths: "baroclinic_wave_equil_conservation_source/output_active/*" - agents: - slurm_mem: 16GB - - - group: "Sphere Examples (Dycore)" - steps: - - - label: ":computer: hydrostatic balance float64" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/hydrostatic_balance_ft64.yml - --job_id hydrostatic_balance_ft64 - artifact_paths: "hydrostatic_balance_ft64/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":computer: baroclinic wave" - key: baroclinic_wave - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/baroclinic_wave.yml - --job_id baroclinic_wave - artifact_paths: "baroclinic_wave/output_active/*" - - - label: ":computer: baroclinic wave dense autodiff" - key: baroclinic_wave_dense_autodiff - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/baroclinic_wave_dense_autodiff.yml - --job_id baroclinic_wave_dense_autodiff - artifact_paths: "baroclinic_wave_dense_autodiff/output_active/*" - - - label: ":computer: baroclinic wave sparse autodiff" - key: baroclinic_wave_sparse_autodiff - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/baroclinic_wave_sparse_autodiff.yml - --job_id baroclinic_wave_sparse_autodiff - artifact_paths: "baroclinic_wave_sparse_autodiff/output_active/*" - - - label: ":computer: no lim baroclinic wave equilmoist" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/baroclinic_wave_equil.yml - --job_id baroclinic_wave_equil - - julia --color=yes --project=.buildkite reproducibility_tests/test_mse.jl - --job_id baroclinic_wave_equil - --out_dir baroclinic_wave_equil/output_active - artifact_paths: "baroclinic_wave_equil/output_active/*" - agents: - slurm_constraint: icelake|cascadelake|skylake|epyc - - - label: ":computer: held suarez" - key: held_suarez - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/held_suarez.yml - --job_id held_suarez - artifact_paths: "held_suarez/output_active/*" - - - label: ":computer: no lim held suarez equilmoist" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/held_suarez_equil.yml - --job_id held_suarez_equil - - julia --color=yes --project=.buildkite reproducibility_tests/test_mse.jl - --job_id held_suarez_equil - --out_dir held_suarez_equil/output_active - artifact_paths: "held_suarez_equil/output_active/*" - agents: - slurm_constraint: icelake|cascadelake|skylake|epyc - - - group: "Sphere Examples (Aquaplanet)" - steps: - - - label: ":umbrella: aquaplanet nonequil allsky monin_obukhov varying insol gravity wave (gfdl_restart) high top 1-moment" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/aquaplanet_nonequil_allsky_gw_res.yml - --job_id aquaplanet_nonequil_allsky_gw_res - - julia --color=yes --project=.buildkite reproducibility_tests/test_mse.jl - --job_id aquaplanet_nonequil_allsky_gw_res - --out_dir aquaplanet_nonequil_allsky_gw_res/output_active - artifact_paths: "aquaplanet_nonequil_allsky_gw_res/output_active/*" - agents: - slurm_mem: 20GB - slurm_constraint: icelake|cascadelake|skylake|epyc - - - label: ":umbrella: aquaplanet nonequil allsky monin_obukhov varying insol gravity wave (gfdl_restart) high top 1-moment dense autodiff" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/aquaplanet_nonequil_allsky_gw_res_dense_autodiff.yml - --job_id aquaplanet_nonequil_allsky_gw_res_dense_autodiff - artifact_paths: "aquaplanet_nonequil_allsky_gw_res_dense_autodiff/output_active/*" - agents: - slurm_mem: 20GB - slurm_constraint: icelake|cascadelake|skylake|epyc - - - label: ":umbrella: aquaplanet nonequil allsky monin_obukhov varying insol gravity wave (gfdl_restart) high top 1-moment sparse autodiff" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/aquaplanet_nonequil_allsky_gw_res_sparse_autodiff.yml - --job_id aquaplanet_nonequil_allsky_gw_res_sparse_autodiff - artifact_paths: "aquaplanet_nonequil_allsky_gw_res_sparse_autodiff/output_active/*" - agents: - slurm_mem: 20GB - slurm_constraint: icelake|cascadelake|skylake|epyc - - - label: ":umbrella: aquaplanet nonequil allsky monin_obukhov varying insol gravity wave (gfdl_restart) high top 2-moment" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/aquaplanet_nonequil_allsky_gw_res_2M.yml - --job_id aquaplanet_nonequil_allsky_gw_res_2M - - julia --color=yes --project=.buildkite reproducibility_tests/test_mse.jl - --job_id aquaplanet_nonequil_allsky_gw_res_2M - --out_dir aquaplanet_nonequil_allsky_gw_res_2M/output_active - artifact_paths: "aquaplanet_nonequil_allsky_gw_res_2M/output_active/*" - agents: - slurm_mem: 20GB - slurm_constraint: icelake|cascadelake|skylake|epyc - - - label: ":computer: aquaplanet equil allsky monin_obukhov varying insol gravity wave (raw_topo) high top zonally asymmetric" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/aquaplanet_equil_allsky_gw_raw_zonalasym.yml - --job_id aquaplanet_equil_allsky_gw_raw_zonalasym - - julia --color=yes --project=.buildkite reproducibility_tests/test_mse.jl - --job_id aquaplanet_equil_allsky_gw_raw_zonalasym - --out_dir aquaplanet_equil_allsky_gw_raw_zonalasym/output_active - artifact_paths: "aquaplanet_equil_allsky_gw_raw_zonalasym/output_active/*" - agents: - slurm_mem: 20GB - slurm_constraint: icelake|cascadelake|skylake|epyc - - - label: ":computer: aquaplanet slabocean equilmoist allsky radiation monin_obukhov varying insolation" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/aquaplanet_equil_clearsky_tvinsol_0M_slabocean.yml - --job_id aquaplanet_equil_clearsky_tvinsol_0M_slabocean - artifact_paths: "aquaplanet_equil_clearsky_tvinsol_0M_slabocean/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":computer: aquaplanet slabocean equilmoist allsky radiation monin_obukhov varying insolation float64" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/aquaplanet_equil_clearsky_tvinsol_0M_slabocean_ft64.yml - --job_id aquaplanet_equil_clearsky_tvinsol_0M_slabocean_ft64 - artifact_paths: "aquaplanet_equil_clearsky_tvinsol_0M_slabocean_ft64/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":computer: aquaplanet for coupler" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/aquaplanet_for_coupler.yml - --job_id aquaplanet_for_coupler - artifact_paths: "aquaplanet_for_coupler/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":computer: aquaplanet rcemipii with diagnostic edmf" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/rcemipii_sphere_diagnostic_edmfx.yml - --job_id rcemipii_sphere_diagnostic_edmfx - artifact_paths: "rcemipii_sphere_diagnostic_edmfx/output_active/*" - agents: - slurm_mem: 20GB - - - group: "Sphere Examples (Topography)" - steps: - - - label: ":computer: baroclinic wave topography (dcmip)" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/baroclinic_wave_topography_dcmip_rs.yml - --job_id baroclinic_wave_topography_dcmip_rs - artifact_paths: "baroclinic_wave_topography_dcmip_rs/output_active/*" - - - label: ":computer: Diagnostic Earth surface elevation spectra" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/baroclinic_wave_equil_earth.yml - --job_id baroclinic_wave_equil_earth - artifact_paths: "baroclinic_wave_equil_earth/output_active/*" - - - label: ":computer: Baroclinic wave equilmoist (Hughes2023 double mountain config)" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/baroclinic_wave_hughes2023.yml - --job_id baroclinic_wave_hughes2023 - artifact_paths: "baroclinic_wave_hughes2023/output_active/*" - - - group: "Restarting" - steps: - - - label: ":computer: test restart" - command: > - julia --color=yes --project=.buildkite test/restart.jl - agents: - slurm_mem: 16GB - - - label: ":computer: test restart GPU" - command: > - julia --color=yes --project=.buildkite test/restart.jl - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 16G - - - label: ":computer: test restart" - command: > - julia --color=yes --project=.buildkite test/restart.jl --manytests true - agents: - slurm_mem: 16GB - - - label: ":computer: test restart GPU" - command: > - julia --color=yes --project=.buildkite test/restart.jl --manytests true - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 16G - - # This job occasionally times out. We add a soft file when the agent is - # lost because of timeout - - label: ":computer: test restart MPI" - command: > - srun julia --color=yes --project=.buildkite test/restart.jl - env: - CLIMACOMMS_CONTEXT: "MPI" - timeout_in_minutes: 20 - soft_fail: - - exit_status: -1 - - exit_status: 255 - agents: - slurm_ntasks: 2 - slurm_mem: 24G - - # This job occasionally times out. We add a soft file when the agent is - # lost because of timeout - - label: ":computer: test restart GPU MPI" - command: > - srun julia --color=yes --project=.buildkite test/restart.jl - env: - CLIMACOMMS_CONTEXT: "MPI" - CLIMACOMMS_DEVICE: "CUDA" - timeout_in_minutes: 20 - soft_fail: - - exit_status: -1 - - exit_status: 255 - agents: - slurm_gpus_per_task: 1 - slurm_ntasks: 2 - slurm_mem: 24G - - - group: "MPI Examples" - steps: - - - label: ":computer: Prep restart for MPI" - key: "mpi_baro_wave_make_restart" - command: > - srun julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $MPI_CONFIG_PATH/mpi_make_restart.yml - --job_id mpi_make_restart - artifact_paths: "mpi_make_restart/output_active/*" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 2 - slurm_mem: 16G - - - label: ":computer: Test restart for MPI baroclinic wave" - key: "restart_mpi_baro_wave" - depends_on: "mpi_baro_wave_make_restart" - command: > - tar xvf mpi_make_restart/output_active/hdf5_files.tar -C mpi_make_restart/output_active - - srun julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $MPI_CONFIG_PATH/restart_mpi_baroclinic_wave.yml - --job_id restart_mpi_baroclinic_wave - artifact_paths: "restart_mpi_baroclinic_wave/output_active/*" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 2 - slurm_mem: 16G - timeout_in_minutes: 20 - soft_fail: true - #retry: - # automatic: true - - - label: ":computer: MPI no lim aquaplanet equilmoist clearsky radiation" - command: > - srun julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $MPI_CONFIG_PATH/mpi_aquaplanet_equil_clearsky.yml - --job_id mpi_aquaplanet_equil_clearsky - artifact_paths: "mpi_aquaplanet_equil_clearsky/output_active/*" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 2 - slurm_mem: 16GB - - - label: ":computer: Prep for calling remap pipeline" - key: "prep_remap" - command: > - srun julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $MPI_CONFIG_PATH/prep_remap.yml - --job_id prep_remap - artifact_paths: "prep_remap/output_active/*" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 2 - slurm_mem: 16G - - - label: ":computer: Exercise remap pipeline" - depends_on: "prep_remap" - command: > - tar xvf prep_remap/output_active/hdf5_files.tar -C prep_remap/output_active - - julia --color=yes --project=.buildkite post_processing/remap/remap_pipeline.jl - --data_dir prep_remap/output_active --out_dir remap_pipeline_output - artifact_paths: "remap_pipeline_output/*" - - - group: "EDOnlyEDMFX" - steps: - - - label: ":man_in_business_suit_levitating: EDOnly EDMFX aquaplanet" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/edonly_edmfx_aquaplanet.yml - --job_id edonly_edmfx_aquaplanet - artifact_paths: "edonly_edmfx_aquaplanet/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":man_in_business_suit_levitating: AMIP Target EDOnly nonequil" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/amip_target_edonly_nonequil.yml - --job_id amip_target_edonly_nonequil - artifact_paths: "amip_target_edonly_nonequil/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":man_in_business_suit_levitating: AMIP Target EDOnly nonequil sparse autodiff" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/amip_target_edonly_nonequil_sparse_autodiff.yml - --job_id amip_target_edonly_nonequil_sparse_autodiff - artifact_paths: "amip_target_edonly_nonequil_sparse_autodiff/output_active/*" - agents: - slurm_mem: 40GB - - - group: "Diagnostic EDMFX" - steps: - - - label: ":genie: Diagnostic EDMFX test in a box" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_test_box.yml - --job_id diagnostic_edmfx_test_box - artifact_paths: "diagnostic_edmfx_test_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Diagnostic EDMFX GABLS in a box" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_gabls_box.yml - --job_id diagnostic_edmfx_gabls_box - artifact_paths: "diagnostic_edmfx_gabls_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Diagnostic EDMFX Bomex in a box" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_bomex_box.yml - --job_id diagnostic_edmfx_bomex_box - artifact_paths: "diagnostic_edmfx_bomex_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Diagnostic EDMFX Bomex stretched grid in a box" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_bomex_stretched_box.yml - --job_id diagnostic_edmfx_bomex_stretched_box - artifact_paths: "diagnostic_edmfx_bomex_stretched_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Diagnostic EDMFX DYCOMS_RF01 in a box (explicit)" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_dycoms_rf01_explicit_box.yml - --job_id diagnostic_edmfx_dycoms_rf01_explicit_box - artifact_paths: "diagnostic_edmfx_dycoms_rf01_explicit_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Diagnostic EDMFX DYCOMS_RF01 in a box" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_dycoms_rf01_box.yml - --job_id diagnostic_edmfx_dycoms_rf01_box - artifact_paths: "diagnostic_edmfx_dycoms_rf01_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":umbrella: Diagnostic EDMFX DYCOMS_RF02 in a box" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_dycoms_rf02_box.yml - --job_id diagnostic_edmfx_dycoms_rf02_box - artifact_paths: "diagnostic_edmfx_dycoms_rf02_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":umbrella: Diagnostic EDMFX Rico in a box" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_rico_box.yml - --job_id diagnostic_edmfx_rico_box - artifact_paths: "diagnostic_edmfx_rico_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":umbrella: Diagnostic EDMFX TRMM in a box" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_trmm_box.yml - --job_id diagnostic_edmfx_trmm_box - artifact_paths: "diagnostic_edmfx_trmm_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":umbrella: Diagnostic EDMFX TRMM stretched grid in a box" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_trmm_stretched_box.yml - --job_id diagnostic_edmfx_trmm_stretched_box - artifact_paths: "diagnostic_edmfx_trmm_stretched_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Diagnostic EDMFX TRMM 0M in a box" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_trmm_box_0M.yml - --job_id diagnostic_edmfx_trmm_box_0M - artifact_paths: "diagnostic_edmfx_trmm_box_0M/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Diagnostic EDMFX aquaplanet" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_aquaplanet.yml - --job_id diagnostic_edmfx_aquaplanet - - julia --color=yes --project=.buildkite reproducibility_tests/test_mse.jl - --job_id diagnostic_edmfx_aquaplanet - --out_dir diagnostic_edmfx_aquaplanet/output_active - artifact_paths: "diagnostic_edmfx_aquaplanet/output_active/*" - agents: - slurm_mem: 20GB - slurm_constraint: icelake|cascadelake|skylake|epyc - - - label: ":genie: Diagnostic EDMFX aquaplanet dense autodiff" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_aquaplanet_dense_autodiff.yml - --job_id diagnostic_edmfx_aquaplanet_dense_autodiff - artifact_paths: "diagnostic_edmfx_aquaplanet_dense_autodiff/output_active/*" - agents: - slurm_mem: 20GB - slurm_constraint: icelake|cascadelake|skylake|epyc - - - label: ":genie: Diagnostic EDMFX aquaplanet sparse autodiff" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_aquaplanet_sparse_autodiff.yml - --job_id diagnostic_edmfx_aquaplanet_sparse_autodiff - artifact_paths: "diagnostic_edmfx_aquaplanet_sparse_autodiff/output_active/*" - agents: - slurm_mem: 20GB - slurm_constraint: icelake|cascadelake|skylake|epyc + # - label: ":umbrella: 2-moment precipitation sanity test single column" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/single_column_precipitation_2M_test.yml + # --job_id single_column_precipitation_2M_test + # artifact_paths: "single_column_precipitation_2M_test/output_active/*" + + # - group: "Gravity wave" + # steps: + + # - label: ":computer: non-orographic gravity wave parameterization unit test 3d" + # command: "julia --color=yes --project=.buildkite test/parameterized_tendencies/gravity_wave/non_orographic_gravity_wave/nogw_test_3d.jl" + # artifact_paths: "nonorographic_gravity_wave_test_3d/*" + # agents: + # slurm_mem: 20GB + + # - label: ":computer: non-orographic gravity wave parameterization test with MiMA output" + # command: "julia --color=yes --project=.buildkite test/parameterized_tendencies/gravity_wave/non_orographic_gravity_wave/nogw_test_mima.jl" + # artifact_paths: "nonorographic_gravity_wave_test_mima/*" + # agents: + # slurm_mem: 20GB + + # - label: ":computer: non-orographic gravity wave parameterization unit test single column" + # command: "julia --color=yes --project=.buildkite test/parameterized_tendencies/gravity_wave/non_orographic_gravity_wave/nogw_test_single_column.jl" + # artifact_paths: "nonorographic_gravity_wave_test_single_column/*" + + # - label: ":computer: orographic gravity wave parameterization unit test for base flux calculation" + # command: "julia --color=yes --project=.buildkite test/parameterized_tendencies/gravity_wave/orographic_gravity_wave/ogwd_baseflux.jl" + # artifact_paths: "orographic_gravity_wave_test_baseflux/*" + + # - label: ":computer: orographic gravity wave parameterization unit test for 3d calculation" + # command: "julia --color=yes --project=.buildkite test/parameterized_tendencies/gravity_wave/orographic_gravity_wave/ogwd_3d.jl" + # artifact_paths: "orographic_gravity_wave_test_3d/*" + + # - label: ":computer: single column non-orographic gravity wave parameterization" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/single_column_nonorographic_gravity_wave.yml + # --job_id single_column_nonorographic_gravity_wave + # artifact_paths: "single_column_nonorographic_gravity_wave/*" + + # - group: "Column Examples" + # steps: + + # - label: ":computer: single column hydrostatic balance float64" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/single_column_hydrostatic_balance_ft64.yml + # --job_id single_column_hydrostatic_balance_ft64 + + # julia --color=yes --project=.buildkite reproducibility_tests/test_mse.jl + # --job_id single_column_hydrostatic_balance_ft64 + # --out_dir single_column_hydrostatic_balance_ft64/output_active + # artifact_paths: "single_column_hydrostatic_balance_ft64/output_active/*" + # agents: + # slurm_constraint: icelake|cascadelake|skylake|epyc + + # - group: "Box Examples" + # steps: + + # - label: ":computer: Box hydrostatic balance" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/box_hydrostatic_balance.yml + # --job_id box_hydrostatic_balance + # artifact_paths: "box_hydrostatic_balance/output_active/*" + + # - label: ":computer: 3D density current" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/box_density_current_test.yml + # --job_id box_density_current_test + # artifact_paths: "box_density_current_test/output_active/*" + + # - label: ":computer: Box rcemipii with diagnostic edmf" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/rcemipii_box_diagnostic_edmfx.yml + # --job_id rcemipii_box_diagnostic_edmfx + # artifact_paths: "rcemipii_box_diagnostic_edmfx/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: LES ISDAC in a box" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/les_isdac_box.yml + # --job_id les_isdac_box + # artifact_paths: "les_isdac_box/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_mem: 16G + # slurm_gpus: 1 + # soft_fail: true + + # - group: "Plane Examples" + # steps: + + # - label: ":computer: Density current experiment" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/plane_density_current_test.yml + # --job_id plane_density_current_test + # artifact_paths: "plane_density_current_test/output_active/*" + + # - group: "Analytic Tests" + # steps: + # - label: "GPU: No Topography Test (2D, Float64, Discrete Balance)" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/plane_no_topography_float64_test.yml + # --job_id gpu_plane_no_topography_float64_test + # artifact_paths: "gpu_plane_no_topography_float64_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Cosine Hills Test (2D, Float64)" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/plane_cosine_hills_float64_test.yml + # --job_id gpu_plane_cosine_hills_float64_test + # artifact_paths: "gpu_plane_cosine_hills_float64_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Cosine Hills Test (Extruded 2D, Float64)" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/extruded_plane_cosine_hills_float64_test.yml + # --job_id gpu_extruded_plane_cosine_hills_float64_test + # artifact_paths: "gpu_extruded_plane_cosine_hills_float64_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 48GB + + # - label: "GPU: Cosine Hills Test (3D, Float64)" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/box_cosine_hills_float64_test.yml + # --job_id gpu_box_cosine_hills_float64_test + # artifact_paths: "gpu_box_cosine_hills_float64_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 48GB + + # - label: "GPU: Agnesi Mountain Test (2D, Float64)" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/plane_agnesi_mountain_float64_test.yml + # --job_id gpu_plane_agnesi_mountain_float64_test + # artifact_paths: "gpu_plane_agnesi_mountain_float64_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Schar Mountain Test (2D, Float64)" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/plane_schar_mountain_float64_test.yml + # --job_id gpu_plane_schar_mountain_float64_test + # artifact_paths: "gpu_plane_schar_mountain_float64_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Schar Mountain Test (2D, Float32)" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/plane_schar_mountain_float32_test.yml + # --job_id gpu_plane_schar_mountain_float32_test + # artifact_paths: "gpu_plane_schar_mountain_float32_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - group: "Conservation check" + # steps: + + # - label: ":computer: baroclinic wave check conservation" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl --config_file $CONFIG_PATH/baroclinic_wave_conservation.yml + # --job_id baroclinic_wave_conservation + # artifact_paths: "baroclinic_wave_conservation/output_active/*" + + # - label: ":computer: baroclinic wave moist check conservation" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl --config_file $CONFIG_PATH/baroclinic_wave_equil_conservation.yml + # --job_id baroclinic_wave_equil_conservation + # artifact_paths: "baroclinic_wave_equil_conservation/output_active/*" + + # - label: ":computer: baroclinic wave moist check conservation float64" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl --config_file $CONFIG_PATH/baroclinic_wave_equil_conservation_ft64.yml + # --job_id baroclinic_wave_equil_conservation_ft64 + # artifact_paths: "baroclinic_wave_equil_conservation_ft64/output_active/*" + + # - label: ":computer: baroclinic wave moist check conservation float64 sparse autodiff" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl --config_file $CONFIG_PATH/baroclinic_wave_equil_conservation_ft64_sparse_autodiff.yml + # --job_id baroclinic_wave_equil_conservation_ft64_sparse_autodiff + # artifact_paths: "baroclinic_wave_equil_conservation_ft64_sparse_autodiff/output_active/*" + # soft_fail: true + + # - label: ":computer: baroclinic wave moist check conservation with sources" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl --config_file $CONFIG_PATH/baroclinic_wave_equil_conservation_source.yml + # --job_id baroclinic_wave_equil_conservation_source + # artifact_paths: "baroclinic_wave_equil_conservation_source/output_active/*" + # agents: + # slurm_mem: 16GB + + # - group: "Sphere Examples (Dycore)" + # steps: + + # - label: ":computer: hydrostatic balance float64" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/hydrostatic_balance_ft64.yml + # --job_id hydrostatic_balance_ft64 + # artifact_paths: "hydrostatic_balance_ft64/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":computer: baroclinic wave" + # key: baroclinic_wave + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/baroclinic_wave.yml + # --job_id baroclinic_wave + # artifact_paths: "baroclinic_wave/output_active/*" + + # - label: ":computer: baroclinic wave dense autodiff" + # key: baroclinic_wave_dense_autodiff + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/baroclinic_wave_dense_autodiff.yml + # --job_id baroclinic_wave_dense_autodiff + # artifact_paths: "baroclinic_wave_dense_autodiff/output_active/*" + + # - label: ":computer: baroclinic wave sparse autodiff" + # key: baroclinic_wave_sparse_autodiff + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/baroclinic_wave_sparse_autodiff.yml + # --job_id baroclinic_wave_sparse_autodiff + # artifact_paths: "baroclinic_wave_sparse_autodiff/output_active/*" + + # - label: ":computer: no lim baroclinic wave equilmoist" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/baroclinic_wave_equil.yml + # --job_id baroclinic_wave_equil + + # julia --color=yes --project=.buildkite reproducibility_tests/test_mse.jl + # --job_id baroclinic_wave_equil + # --out_dir baroclinic_wave_equil/output_active + # artifact_paths: "baroclinic_wave_equil/output_active/*" + # agents: + # slurm_constraint: icelake|cascadelake|skylake|epyc + + # - label: ":computer: no lim baroclinic wave nonequilmoist" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/baroclinic_wave_nonequil.yml + # --job_id baroclinic_wave_nonequil + # artifact_paths: "baroclinic_wave_nonequil/output_active/*" + + + # - label: ":computer: held suarez" + # key: held_suarez + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/held_suarez.yml + # --job_id held_suarez + # artifact_paths: "held_suarez/output_active/*" + + # - label: ":computer: no lim held suarez equilmoist" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/held_suarez_equil.yml + # --job_id held_suarez_equil + + # julia --color=yes --project=.buildkite reproducibility_tests/test_mse.jl + # --job_id held_suarez_equil + # --out_dir held_suarez_equil/output_active + # artifact_paths: "held_suarez_equil/output_active/*" + # agents: + # slurm_constraint: icelake|cascadelake|skylake|epyc + + # - group: "Sphere Examples (Aquaplanet)" + # steps: + + # - label: ":umbrella: aquaplanet nonequil allsky monin_obukhov varying insol gravity wave (gfdl_restart) high top 1-moment" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/aquaplanet_nonequil_allsky_gw_res.yml + # --job_id aquaplanet_nonequil_allsky_gw_res + + # julia --color=yes --project=.buildkite reproducibility_tests/test_mse.jl + # --job_id aquaplanet_nonequil_allsky_gw_res + # --out_dir aquaplanet_nonequil_allsky_gw_res/output_active + # artifact_paths: "aquaplanet_nonequil_allsky_gw_res/output_active/*" + # agents: + # slurm_mem: 20GB + # slurm_constraint: icelake|cascadelake|skylake|epyc + + # - label: ":umbrella: aquaplanet nonequil allsky monin_obukhov varying insol gravity wave (gfdl_restart) high top 1-moment dense autodiff" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/aquaplanet_nonequil_allsky_gw_res_dense_autodiff.yml + # --job_id aquaplanet_nonequil_allsky_gw_res_dense_autodiff + # artifact_paths: "aquaplanet_nonequil_allsky_gw_res_dense_autodiff/output_active/*" + # agents: + # slurm_mem: 20GB + # slurm_constraint: icelake|cascadelake|skylake|epyc + + # - label: ":umbrella: aquaplanet nonequil allsky monin_obukhov varying insol gravity wave (gfdl_restart) high top 1-moment sparse autodiff" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/aquaplanet_nonequil_allsky_gw_res_sparse_autodiff.yml + # --job_id aquaplanet_nonequil_allsky_gw_res_sparse_autodiff + # artifact_paths: "aquaplanet_nonequil_allsky_gw_res_sparse_autodiff/output_active/*" + # agents: + # slurm_mem: 20GB + # slurm_constraint: icelake|cascadelake|skylake|epyc + + # - label: ":umbrella: aquaplanet nonequil allsky monin_obukhov varying insol gravity wave (gfdl_restart) high top 2-moment" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/aquaplanet_nonequil_allsky_gw_res_2M.yml + # --job_id aquaplanet_nonequil_allsky_gw_res_2M + + # julia --color=yes --project=.buildkite reproducibility_tests/test_mse.jl + # --job_id aquaplanet_nonequil_allsky_gw_res_2M + # --out_dir aquaplanet_nonequil_allsky_gw_res_2M/output_active + # artifact_paths: "aquaplanet_nonequil_allsky_gw_res_2M/output_active/*" + # agents: + # slurm_mem: 20GB + # slurm_constraint: icelake|cascadelake|skylake|epyc + + # - label: ":computer: aquaplanet equil allsky monin_obukhov varying insol gravity wave (raw_topo) high top zonally asymmetric" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/aquaplanet_equil_allsky_gw_raw_zonalasym.yml + # --job_id aquaplanet_equil_allsky_gw_raw_zonalasym + + # julia --color=yes --project=.buildkite reproducibility_tests/test_mse.jl + # --job_id aquaplanet_equil_allsky_gw_raw_zonalasym + # --out_dir aquaplanet_equil_allsky_gw_raw_zonalasym/output_active + # artifact_paths: "aquaplanet_equil_allsky_gw_raw_zonalasym/output_active/*" + # agents: + # slurm_mem: 20GB + # slurm_constraint: icelake|cascadelake|skylake|epyc + + # - label: ":computer: aquaplanet slabocean equilmoist allsky radiation monin_obukhov varying insolation" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/aquaplanet_equil_clearsky_tvinsol_0M_slabocean.yml + # --job_id aquaplanet_equil_clearsky_tvinsol_0M_slabocean + # artifact_paths: "aquaplanet_equil_clearsky_tvinsol_0M_slabocean/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":computer: aquaplanet slabocean equilmoist allsky radiation monin_obukhov varying insolation float64" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/aquaplanet_equil_clearsky_tvinsol_0M_slabocean_ft64.yml + # --job_id aquaplanet_equil_clearsky_tvinsol_0M_slabocean_ft64 + # artifact_paths: "aquaplanet_equil_clearsky_tvinsol_0M_slabocean_ft64/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":computer: aquaplanet for coupler" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/aquaplanet_for_coupler.yml + # --job_id aquaplanet_for_coupler + # artifact_paths: "aquaplanet_for_coupler/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":computer: aquaplanet rcemipii with diagnostic edmf" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/rcemipii_sphere_diagnostic_edmfx.yml + # --job_id rcemipii_sphere_diagnostic_edmfx + # artifact_paths: "rcemipii_sphere_diagnostic_edmfx/output_active/*" + # agents: + # slurm_mem: 20GB + + # - group: "Sphere Examples (Topography)" + # steps: + + # - label: ":computer: baroclinic wave topography (dcmip)" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/baroclinic_wave_topography_dcmip_rs.yml + # --job_id baroclinic_wave_topography_dcmip_rs + # artifact_paths: "baroclinic_wave_topography_dcmip_rs/output_active/*" + + # - label: ":computer: Diagnostic Earth surface elevation spectra" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/baroclinic_wave_equil_earth.yml + # --job_id baroclinic_wave_equil_earth + # artifact_paths: "baroclinic_wave_equil_earth/output_active/*" + + # - label: ":computer: Baroclinic wave equilmoist (Hughes2023 double mountain config)" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/baroclinic_wave_hughes2023.yml + # --job_id baroclinic_wave_hughes2023 + # artifact_paths: "baroclinic_wave_hughes2023/output_active/*" + + # - group: "Restarting" + # steps: + + # - label: ":computer: test restart" + # command: > + # julia --color=yes --project=.buildkite test/restart.jl + # agents: + # slurm_mem: 16GB + + # - label: ":computer: test restart GPU" + # command: > + # julia --color=yes --project=.buildkite test/restart.jl + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 16G + + # - label: ":computer: test restart" + # command: > + # julia --color=yes --project=.buildkite test/restart.jl --manytests true + # agents: + # slurm_mem: 16GB + + # - label: ":computer: test restart GPU" + # command: > + # julia --color=yes --project=.buildkite test/restart.jl --manytests true + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 16G + + # # This job occasionally times out. We add a soft file when the agent is + # # lost because of timeout + # - label: ":computer: test restart MPI" + # command: > + # srun julia --color=yes --project=.buildkite test/restart.jl + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # timeout_in_minutes: 20 + # soft_fail: + # - exit_status: -1 + # - exit_status: 255 + # agents: + # slurm_ntasks: 2 + # slurm_mem: 24G + + # # This job occasionally times out. We add a soft file when the agent is + # # lost because of timeout + # - label: ":computer: test restart GPU MPI" + # command: > + # srun julia --color=yes --project=.buildkite test/restart.jl + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # CLIMACOMMS_DEVICE: "CUDA" + # timeout_in_minutes: 20 + # soft_fail: + # - exit_status: -1 + # - exit_status: 255 + # agents: + # slurm_gpus_per_task: 1 + # slurm_ntasks: 2 + # slurm_mem: 24G + + # - group: "MPI Examples" + # steps: + + # - label: ":computer: Prep restart for MPI" + # key: "mpi_baro_wave_make_restart" + # command: > + # srun julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $MPI_CONFIG_PATH/mpi_make_restart.yml + # --job_id mpi_make_restart + # artifact_paths: "mpi_make_restart/output_active/*" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_ntasks: 2 + # slurm_mem: 16G + + # - label: ":computer: Test restart for MPI baroclinic wave" + # key: "restart_mpi_baro_wave" + # depends_on: "mpi_baro_wave_make_restart" + # command: > + # tar xvf mpi_make_restart/output_active/hdf5_files.tar -C mpi_make_restart/output_active + + # srun julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $MPI_CONFIG_PATH/restart_mpi_baroclinic_wave.yml + # --job_id restart_mpi_baroclinic_wave + # artifact_paths: "restart_mpi_baroclinic_wave/output_active/*" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_ntasks: 2 + # slurm_mem: 16G + # timeout_in_minutes: 20 + # soft_fail: true + # #retry: + # # automatic: true + + # - label: ":computer: MPI no lim aquaplanet equilmoist clearsky radiation" + # command: > + # srun julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $MPI_CONFIG_PATH/mpi_aquaplanet_equil_clearsky.yml + # --job_id mpi_aquaplanet_equil_clearsky + # artifact_paths: "mpi_aquaplanet_equil_clearsky/output_active/*" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_ntasks: 2 + # slurm_mem: 16GB + + # - label: ":computer: Prep for calling remap pipeline" + # key: "prep_remap" + # command: > + # srun julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $MPI_CONFIG_PATH/prep_remap.yml + # --job_id prep_remap + # artifact_paths: "prep_remap/output_active/*" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_ntasks: 2 + # slurm_mem: 16G + + # - label: ":computer: Exercise remap pipeline" + # depends_on: "prep_remap" + # command: > + # tar xvf prep_remap/output_active/hdf5_files.tar -C prep_remap/output_active + + # julia --color=yes --project=.buildkite post_processing/remap/remap_pipeline.jl + # --data_dir prep_remap/output_active --out_dir remap_pipeline_output + # artifact_paths: "remap_pipeline_output/*" + + # - group: "EDOnlyEDMFX" + # steps: + + # - label: ":man_in_business_suit_levitating: EDOnly EDMFX aquaplanet" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/edonly_edmfx_aquaplanet.yml + # --job_id edonly_edmfx_aquaplanet + # artifact_paths: "edonly_edmfx_aquaplanet/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":man_in_business_suit_levitating: AMIP Target EDOnly nonequil" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/amip_target_edonly_nonequil.yml + # --job_id amip_target_edonly_nonequil + # artifact_paths: "amip_target_edonly_nonequil/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":man_in_business_suit_levitating: AMIP Target EDOnly nonequil sparse autodiff" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/amip_target_edonly_nonequil_sparse_autodiff.yml + # --job_id amip_target_edonly_nonequil_sparse_autodiff + # artifact_paths: "amip_target_edonly_nonequil_sparse_autodiff/output_active/*" + # agents: + # slurm_mem: 40GB + + # - group: "Diagnostic EDMFX" + # steps: + + # - label: ":genie: Diagnostic EDMFX test in a box" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_test_box.yml + # --job_id diagnostic_edmfx_test_box + # artifact_paths: "diagnostic_edmfx_test_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Diagnostic EDMFX GABLS in a box" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_gabls_box.yml + # --job_id diagnostic_edmfx_gabls_box + # artifact_paths: "diagnostic_edmfx_gabls_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Diagnostic EDMFX Bomex in a box" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_bomex_box.yml + # --job_id diagnostic_edmfx_bomex_box + # artifact_paths: "diagnostic_edmfx_bomex_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Diagnostic EDMFX Bomex stretched grid in a box" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_bomex_stretched_box.yml + # --job_id diagnostic_edmfx_bomex_stretched_box + # artifact_paths: "diagnostic_edmfx_bomex_stretched_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Diagnostic EDMFX DYCOMS_RF01 in a box (explicit)" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_dycoms_rf01_explicit_box.yml + # --job_id diagnostic_edmfx_dycoms_rf01_explicit_box + # artifact_paths: "diagnostic_edmfx_dycoms_rf01_explicit_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Diagnostic EDMFX DYCOMS_RF01 in a box" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_dycoms_rf01_box.yml + # --job_id diagnostic_edmfx_dycoms_rf01_box + # artifact_paths: "diagnostic_edmfx_dycoms_rf01_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":umbrella: Diagnostic EDMFX DYCOMS_RF02 in a box" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_dycoms_rf02_box.yml + # --job_id diagnostic_edmfx_dycoms_rf02_box + # artifact_paths: "diagnostic_edmfx_dycoms_rf02_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":umbrella: Diagnostic EDMFX Rico in a box" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_rico_box.yml + # --job_id diagnostic_edmfx_rico_box + # artifact_paths: "diagnostic_edmfx_rico_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":umbrella: Diagnostic EDMFX TRMM in a box" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_trmm_box.yml + # --job_id diagnostic_edmfx_trmm_box + # artifact_paths: "diagnostic_edmfx_trmm_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":umbrella: Diagnostic EDMFX TRMM stretched grid in a box" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_trmm_stretched_box.yml + # --job_id diagnostic_edmfx_trmm_stretched_box + # artifact_paths: "diagnostic_edmfx_trmm_stretched_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Diagnostic EDMFX TRMM 0M in a box" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_trmm_box_0M.yml + # --job_id diagnostic_edmfx_trmm_box_0M + # artifact_paths: "diagnostic_edmfx_trmm_box_0M/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Diagnostic EDMFX aquaplanet" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_aquaplanet.yml + # --job_id diagnostic_edmfx_aquaplanet + + # julia --color=yes --project=.buildkite reproducibility_tests/test_mse.jl + # --job_id diagnostic_edmfx_aquaplanet + # --out_dir diagnostic_edmfx_aquaplanet/output_active + # artifact_paths: "diagnostic_edmfx_aquaplanet/output_active/*" + # agents: + # slurm_mem: 20GB + # slurm_constraint: icelake|cascadelake|skylake|epyc + + # - label: ":genie: Diagnostic EDMFX aquaplanet dense autodiff" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_aquaplanet_dense_autodiff.yml + # --job_id diagnostic_edmfx_aquaplanet_dense_autodiff + # artifact_paths: "diagnostic_edmfx_aquaplanet_dense_autodiff/output_active/*" + # agents: + # slurm_mem: 20GB + # slurm_constraint: icelake|cascadelake|skylake|epyc + + # - label: ":genie: Diagnostic EDMFX aquaplanet sparse autodiff" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_aquaplanet_sparse_autodiff.yml + # --job_id diagnostic_edmfx_aquaplanet_sparse_autodiff + # artifact_paths: "diagnostic_edmfx_aquaplanet_sparse_autodiff/output_active/*" + # agents: + # slurm_mem: 20GB + # slurm_constraint: icelake|cascadelake|skylake|epyc - group: "Prognostic EDMFX" steps: @@ -997,405 +1005,405 @@ steps: slurm_mem: 20GB slurm_constraint: icelake|cascadelake|skylake|epyc - - group: "GPU" - steps: - - - label: "GPU: Gravity waves" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/sphere_nonorographic_gravity_wave.yml - --job_id sphere_nonorographic_gravity_wave - artifact_paths: "sphere_nonorographic_gravity_wave/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 16GB - - - label: "GPU: baroclinic wave" - key: "baroclinic_wave_gpu" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/baroclinic_wave.yml - --job_id baroclinic_wave_gpu - artifact_paths: "baroclinic_wave_gpu/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 16GB - - - label: "GPU: baroclinic wave dense autodiff" - key: "baroclinic_wave_gpu_dense_autodiff" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/baroclinic_wave_dense_autodiff.yml - --job_id baroclinic_wave_gpu_dense_autodiff - artifact_paths: "baroclinic_wave_gpu_dense_autodiff/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 16GB - - - label: "GPU: baroclinic wave sparse autodiff" - key: "baroclinic_wave_gpu_sparse_autodiff" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/baroclinic_wave_sparse_autodiff.yml - --job_id baroclinic_wave_gpu_sparse_autodiff - artifact_paths: "baroclinic_wave_gpu_sparse_autodiff/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 16GB - - - label: "GPU: compare BW with CPU" - command: > - tar xvf baroclinic_wave/output_active/hdf5_files.tar -C baroclinic_wave - - tar xvf baroclinic_wave_gpu/output_active/hdf5_files.tar -C baroclinic_wave_gpu - - julia --color=yes --project=.buildkite post_processing/compare_outputs.jl - --output_folder_1 baroclinic_wave - --output_folder_2 baroclinic_wave_gpu --t_end 10days - --compare_state false - depends_on: - - "baroclinic_wave" - - "baroclinic_wave_gpu" - - - - label: "GPU: baroclinic wave - 2 gpus" - key: "baroclinic_wave_2gpu" - command: - - mkdir -p baroclinic_wave_2gpu - # - > - # srun --cpu-bind=threads --cpus-per-task=4 - # nsys profile --delay 100 --trace=nvtx,cuda,mpi --output=baroclinic_wave_2gpu/output_active/report-%q{PMI_RANK} - # julia --threads=3 --color=yes --project=.buildkite .buildkite/ci_driver.jl - # --config_file ${CONFIG_PATH}/baroclinic_wave.yml - # --job_id baroclinic_wave_2gpu - - > - srun --cpu-bind=threads --cpus-per-task=4 - julia --threads=3 --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file ${CONFIG_PATH}/baroclinic_wave.yml - --job_id baroclinic_wave_2gpu - artifact_paths: "baroclinic_wave_2gpu/output_active/*" - env: - CLIMACOMMS_CONTEXT: "MPI" - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus_per_task: 1 - slurm_cpus_per_task: 4 - slurm_ntasks: 2 - slurm_mem: 16GB - - - label: "GPU: test DYAMOND interpolated initial conditions" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $GPU_CONFIG_PATH/gpu_aquaplanet_dyamond_summer.yml - --job_id gpu_aquaplanet_dyamond_summer - artifact_paths: "gpu_aquaplanet_dyamond_summer/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 16GB + # - group: "GPU" + # steps: + + # - label: "GPU: Gravity waves" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/sphere_nonorographic_gravity_wave.yml + # --job_id sphere_nonorographic_gravity_wave + # artifact_paths: "sphere_nonorographic_gravity_wave/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 16GB + + # - label: "GPU: baroclinic wave" + # key: "baroclinic_wave_gpu" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/baroclinic_wave.yml + # --job_id baroclinic_wave_gpu + # artifact_paths: "baroclinic_wave_gpu/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 16GB + + # - label: "GPU: baroclinic wave dense autodiff" + # key: "baroclinic_wave_gpu_dense_autodiff" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/baroclinic_wave_dense_autodiff.yml + # --job_id baroclinic_wave_gpu_dense_autodiff + # artifact_paths: "baroclinic_wave_gpu_dense_autodiff/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 16GB + + # - label: "GPU: baroclinic wave sparse autodiff" + # key: "baroclinic_wave_gpu_sparse_autodiff" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/baroclinic_wave_sparse_autodiff.yml + # --job_id baroclinic_wave_gpu_sparse_autodiff + # artifact_paths: "baroclinic_wave_gpu_sparse_autodiff/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 16GB + + # - label: "GPU: compare BW with CPU" + # command: > + # tar xvf baroclinic_wave/output_active/hdf5_files.tar -C baroclinic_wave + + # tar xvf baroclinic_wave_gpu/output_active/hdf5_files.tar -C baroclinic_wave_gpu + + # julia --color=yes --project=.buildkite post_processing/compare_outputs.jl + # --output_folder_1 baroclinic_wave + # --output_folder_2 baroclinic_wave_gpu --t_end 10days + # --compare_state false + # depends_on: + # - "baroclinic_wave" + # - "baroclinic_wave_gpu" + + + # - label: "GPU: baroclinic wave - 2 gpus" + # key: "baroclinic_wave_2gpu" + # command: + # - mkdir -p baroclinic_wave_2gpu + # # - > + # # srun --cpu-bind=threads --cpus-per-task=4 + # # nsys profile --delay 100 --trace=nvtx,cuda,mpi --output=baroclinic_wave_2gpu/output_active/report-%q{PMI_RANK} + # # julia --threads=3 --color=yes --project=.buildkite .buildkite/ci_driver.jl + # # --config_file ${CONFIG_PATH}/baroclinic_wave.yml + # # --job_id baroclinic_wave_2gpu + # - > + # srun --cpu-bind=threads --cpus-per-task=4 + # julia --threads=3 --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file ${CONFIG_PATH}/baroclinic_wave.yml + # --job_id baroclinic_wave_2gpu + # artifact_paths: "baroclinic_wave_2gpu/output_active/*" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus_per_task: 1 + # slurm_cpus_per_task: 4 + # slurm_ntasks: 2 + # slurm_mem: 16GB + + # - label: "GPU: test DYAMOND interpolated initial conditions" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $GPU_CONFIG_PATH/gpu_aquaplanet_dyamond_summer.yml + # --job_id gpu_aquaplanet_dyamond_summer + # artifact_paths: "gpu_aquaplanet_dyamond_summer/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 16GB - - label: ":earth_americas: GPU: Diagnostic EDMFX ERA5 Weather Model Initial Condition" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_era5_initial_condition.yml - --job_id diagnostic_edmfx_era5_initial_condition - artifact_paths: "diagnostic_edmfx_era5_initial_condition/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 16GB - - - label: "GPU: EDOnly EDMFX aquaplanet" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/edonly_edmfx_aquaplanet.yml - --job_id edonly_edmfx_aquaplanet_gpu - artifact_paths: "edonly_edmfx_aquaplanet_gpu/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 20GB - - - label: "GPU: Diagnostic EDMFX aquaplanet" - key: "diagnostic_edmfx_aquaplanet_gpu" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_aquaplanet_gpu.yml - --job_id diagnostic_edmfx_aquaplanet_gpu - artifact_paths: "diagnostic_edmfx_aquaplanet_gpu/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 20GB - - - label: "GPU: Prognostic EDMFX aquaplanet" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/prognostic_edmfx_aquaplanet_gpu.yml - --job_id prognostic_edmfx_aquaplanet_gpu - artifact_paths: "prognostic_edmfx_aquaplanet_gpu/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 20GB - - - label: "GPU: Prognostic EDMFX aquaplanet dense autodiff" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/prognostic_edmfx_aquaplanet_gpu_dense_autodiff.yml - --job_id prognostic_edmfx_aquaplanet_gpu_dense_autodiff - artifact_paths: "prognostic_edmfx_aquaplanet_gpu_dense_autodiff/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 20GB - - - label: "GPU: Prognostic EDMFX aquaplanet sparse autodiff" - command: > - julia --color=yes --project=.buildkite .buildkite/ci_driver.jl - --config_file $CONFIG_PATH/prognostic_edmfx_aquaplanet_gpu_sparse_autodiff.yml - --job_id prognostic_edmfx_aquaplanet_gpu_sparse_autodiff - artifact_paths: "prognostic_edmfx_aquaplanet_gpu_sparse_autodiff/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 20GB - - - group: "Benchmarks" - steps: - - - label: ":computer: Benchmark: CPU baroclinic wave moist" - command: > - julia --color=yes --project=.buildkite perf/benchmark_step.jl - --config_file $PERF_CONFIG_PATH/bm_baroclinic_wave_moist.yml - --job_id bm_baroclinic_wave_moist - artifact_paths: "bm_baroclinic_wave_moist/output_active/*" - - - label: ":computer: Benchmark: GPU baroclinic wave moist" - command: > - julia --color=yes --project=.buildkite perf/benchmark_step.jl - --config_file $PERF_CONFIG_PATH/bm_baroclinic_wave_moist.yml - --job_id bm_baroclinic_wave_moist_gpu - artifact_paths: "bm_baroclinic_wave_moist_gpu/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_mem: 16G - slurm_gpus: 1 - - - label: ":computer: Benchmark: CPU default" - command: > - julia --color=yes --project=.buildkite perf/benchmark.jl - --config_file $PERF_CONFIG_PATH/bm_default.yml - --job_id bm_default - agents: - slurm_mem: 24GB - - - label: ":computer: Benchmark: GPU default" - command: > - julia --color=yes --project=.buildkite perf/benchmark.jl - --config_file $PERF_CONFIG_PATH/bm_default.yml - --job_id bm_default_gpu - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_mem: 24GB - slurm_gpus: 1 - - - label: ":computer: Benchmark: GPU diag edmf" - command: > - julia --color=yes --project=.buildkite perf/benchmark.jl - --config_file $PERF_CONFIG_PATH/bm_aquaplanet_diagedmf.yml - --job_id bm_diag_edmf_gpu - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_mem: 24GB - slurm_gpus: 1 - - - label: ":computer: Benchmark: GPU prog edmf" - command: > - julia --color=yes --project=.buildkite perf/benchmark.jl - --config_file $PERF_CONFIG_PATH/bm_aquaplanet_progedmf.yml - --job_id bm_prog_edmf_gpu - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_mem: 24GB - slurm_gpus: 1 - - - group: "Flame graphs" - steps: - - - label: ":fire: Flame graph: gpu job" - command: > - julia --color=yes --project=.buildkite perf/flame.jl - --config_file $PERF_CONFIG_PATH/bm_baroclinic_wave_moist.yml - --job_id flame_baroclinic_wave_moist_gpu - artifact_paths: "flame_baroclinic_wave_moist_gpu/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_mem: 48GB - slurm_gpus: 1 - gres: "gpu:p100:1" - - - label: ":fire: Flame graph: perf target " - command: > - julia --color=yes --project=.buildkite perf/flame.jl - --config_file $PERF_CONFIG_PATH/bm_default.yml - --job_id flame_default - artifact_paths: "flame_default/*" - agents: - slurm_mem: 24GB - - - label: ":fire: Flame graph: 1 moment" - command: > - julia --color=yes --project=.buildkite perf/flame.jl - --config_file $PERF_CONFIG_PATH/bm_default_1m.yml - --job_id flame_default_1m - artifact_paths: "flame_default_1m/*" - agents: - slurm_mem: 24GB - - - label: ":fire: Flame graph: diagnostics" - command: > - julia --color=yes --project=.buildkite perf/flame.jl - --config_file $PERF_CONFIG_PATH/bm_diagnostics.yml - --job_id flame_diagnostics - artifact_paths: "flame_diagnostics/*" - agents: - slurm_mem: 24GB - - - label: ":fire: Flame graph: diagnostics edmf" - command: > - julia --color=yes --project=.buildkite perf/flame.jl - --config_file $PERF_CONFIG_PATH/bm_aquaplanet_diagedmf.yml - --job_id flame_aquaplanet_diagedmf - artifact_paths: "flame_aquaplanet_diagedmf/*" - agents: - slurm_mem: 24GB - - - label: ":fire: Flame graph: prognostic edmf" - command: > - julia --color=yes --project=.buildkite perf/flame.jl - --config_file $PERF_CONFIG_PATH/bm_aquaplanet_progedmf.yml - --job_id flame_aquaplanet_progedmf - artifact_paths: "flame_aquaplanet_progedmf/*" - agents: - slurm_mem: 32GB - - - label: ":fire: Flame graph: prognostic edmf dense autodiff" - command: > - julia --color=yes --project=.buildkite perf/flame.jl - --config_file $PERF_CONFIG_PATH/bm_aquaplanet_progedmf_dense_autodiff.yml - --job_id flame_aquaplanet_progedmf_dense_autodiff - artifact_paths: "flame_aquaplanet_progedmf_dense_autodiff/*" - agents: - slurm_mem: 32GB - - - label: ":fire: Flame graph: prognostic edmf sparse autodiff" - command: > - julia --color=yes --project=.buildkite perf/flame.jl - --config_file $PERF_CONFIG_PATH/bm_aquaplanet_progedmf_sparse_autodiff.yml - --job_id flame_aquaplanet_progedmf_sparse_autodiff - artifact_paths: "flame_aquaplanet_progedmf_sparse_autodiff/*" - agents: - slurm_mem: 32GB - - - label: ":fire: Flame graph: diffusion" - command: > - julia --color=yes --project=.buildkite perf/flame.jl - --config_file $PERF_CONFIG_PATH/bm_diffusion.yml - --job_id flame_diffusion - artifact_paths: "flame_diffusion/*" - agents: - slurm_mem: 24GB - - - label: ":fire: Flame graph: perf target (Callbacks)" - command: > - julia --color=yes --project=.buildkite perf/flame.jl - --config_file $PERF_CONFIG_PATH/bm_callbacks.yml - --job_id flame_callbacks - artifact_paths: "flame_callbacks/*" - agents: - slurm_mem: 24GB - - - label: ":fire: Flame graph: gravity wave" - command: > - julia --color=yes --project=.buildkite perf/flame.jl - --config_file $PERF_CONFIG_PATH/bm_gravity_wave.yml - --job_id flame_gravity_wave - artifact_paths: "flame_gravity_wave/*" - agents: - slurm_mem: 24GB - - - group: "Checkbounds/Inference/Invalidations" - steps: - - # TODO: we should somehow decouple this unit test from the perf env / scripts - # Checkbounds - - label: ":computer: checkbounds" - command: > - julia --color=yes --check-bounds=yes --project=.buildkite perf/benchmark.jl - --config_file $PERF_CONFIG_PATH/bm_default.yml - --job_id checkbounds - artifact_paths: "checkbounds/output_active/*" - agents: - slurm_mem: 20GB - - # Inference - - label: ":rocket: JET n-failures (inference)" - command: > - julia --color=yes --project=.buildkite perf/jet_test_nfailures.jl - --config_file $PERF_CONFIG_PATH/bm_default.yml - --job_id jet_n_failures - agents: - slurm_mem: 24GB - - # Latency - - label: ":mag::rocket: Invalidations" - command: > - julia --color=yes --project=.buildkite perf/invalidations.jl - --config_file $PERF_CONFIG_PATH/bm_default.yml - artifact_paths: "invalidations/*" - agents: - slurm_mem: 24GB - - - wait: ~ - continue_on_failure: true - - - label: ":robot_face: Print new mse tables" - command: "julia --color=yes --project=.buildkite reproducibility_tests/mse_summary.jl" - - - label: ":robot_face: Print new reference counter" - command: "julia --color=yes --project=.buildkite reproducibility_tests/print_new_ref_counter.jl" - - - label: ":bar_chart: Tabulate performance summary" - command: "julia --color=yes --project=.buildkite perf/tabulate_perf_summary.jl" - - # - label: ":chart_with_downwards_trend: build history" - # command: - # - build_history main - # artifact_paths: - # - "build_history.html" - - - wait - - - label: ":robot_face: Move main results" - command: "julia --color=yes --project=.buildkite reproducibility_tests/move_output.jl" + # - label: ":earth_americas: GPU: Diagnostic EDMFX ERA5 Weather Model Initial Condition" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_era5_initial_condition.yml + # --job_id diagnostic_edmfx_era5_initial_condition + # artifact_paths: "diagnostic_edmfx_era5_initial_condition/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 16GB + + # - label: "GPU: EDOnly EDMFX aquaplanet" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/edonly_edmfx_aquaplanet.yml + # --job_id edonly_edmfx_aquaplanet_gpu + # artifact_paths: "edonly_edmfx_aquaplanet_gpu/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 20GB + + # - label: "GPU: Diagnostic EDMFX aquaplanet" + # key: "diagnostic_edmfx_aquaplanet_gpu" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_aquaplanet_gpu.yml + # --job_id diagnostic_edmfx_aquaplanet_gpu + # artifact_paths: "diagnostic_edmfx_aquaplanet_gpu/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 20GB + + # - label: "GPU: Prognostic EDMFX aquaplanet" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/prognostic_edmfx_aquaplanet_gpu.yml + # --job_id prognostic_edmfx_aquaplanet_gpu + # artifact_paths: "prognostic_edmfx_aquaplanet_gpu/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 20GB + + # - label: "GPU: Prognostic EDMFX aquaplanet dense autodiff" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/prognostic_edmfx_aquaplanet_gpu_dense_autodiff.yml + # --job_id prognostic_edmfx_aquaplanet_gpu_dense_autodiff + # artifact_paths: "prognostic_edmfx_aquaplanet_gpu_dense_autodiff/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 20GB + + # - label: "GPU: Prognostic EDMFX aquaplanet sparse autodiff" + # command: > + # julia --color=yes --project=.buildkite .buildkite/ci_driver.jl + # --config_file $CONFIG_PATH/prognostic_edmfx_aquaplanet_gpu_sparse_autodiff.yml + # --job_id prognostic_edmfx_aquaplanet_gpu_sparse_autodiff + # artifact_paths: "prognostic_edmfx_aquaplanet_gpu_sparse_autodiff/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 20GB + + # - group: "Benchmarks" + # steps: + + # - label: ":computer: Benchmark: CPU baroclinic wave moist" + # command: > + # julia --color=yes --project=.buildkite perf/benchmark_step.jl + # --config_file $PERF_CONFIG_PATH/bm_baroclinic_wave_moist.yml + # --job_id bm_baroclinic_wave_moist + # artifact_paths: "bm_baroclinic_wave_moist/output_active/*" + + # - label: ":computer: Benchmark: GPU baroclinic wave moist" + # command: > + # julia --color=yes --project=.buildkite perf/benchmark_step.jl + # --config_file $PERF_CONFIG_PATH/bm_baroclinic_wave_moist.yml + # --job_id bm_baroclinic_wave_moist_gpu + # artifact_paths: "bm_baroclinic_wave_moist_gpu/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_mem: 16G + # slurm_gpus: 1 + + # - label: ":computer: Benchmark: CPU default" + # command: > + # julia --color=yes --project=.buildkite perf/benchmark.jl + # --config_file $PERF_CONFIG_PATH/bm_default.yml + # --job_id bm_default + # agents: + # slurm_mem: 24GB + + # - label: ":computer: Benchmark: GPU default" + # command: > + # julia --color=yes --project=.buildkite perf/benchmark.jl + # --config_file $PERF_CONFIG_PATH/bm_default.yml + # --job_id bm_default_gpu + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_mem: 24GB + # slurm_gpus: 1 + + # - label: ":computer: Benchmark: GPU diag edmf" + # command: > + # julia --color=yes --project=.buildkite perf/benchmark.jl + # --config_file $PERF_CONFIG_PATH/bm_aquaplanet_diagedmf.yml + # --job_id bm_diag_edmf_gpu + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_mem: 24GB + # slurm_gpus: 1 + + # - label: ":computer: Benchmark: GPU prog edmf" + # command: > + # julia --color=yes --project=.buildkite perf/benchmark.jl + # --config_file $PERF_CONFIG_PATH/bm_aquaplanet_progedmf.yml + # --job_id bm_prog_edmf_gpu + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_mem: 24GB + # slurm_gpus: 1 + + # - group: "Flame graphs" + # steps: + + # - label: ":fire: Flame graph: gpu job" + # command: > + # julia --color=yes --project=.buildkite perf/flame.jl + # --config_file $PERF_CONFIG_PATH/bm_baroclinic_wave_moist.yml + # --job_id flame_baroclinic_wave_moist_gpu + # artifact_paths: "flame_baroclinic_wave_moist_gpu/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_mem: 48GB + # slurm_gpus: 1 + # gres: "gpu:p100:1" + + # - label: ":fire: Flame graph: perf target " + # command: > + # julia --color=yes --project=.buildkite perf/flame.jl + # --config_file $PERF_CONFIG_PATH/bm_default.yml + # --job_id flame_default + # artifact_paths: "flame_default/*" + # agents: + # slurm_mem: 24GB + + # - label: ":fire: Flame graph: 1 moment" + # command: > + # julia --color=yes --project=.buildkite perf/flame.jl + # --config_file $PERF_CONFIG_PATH/bm_default_1m.yml + # --job_id flame_default_1m + # artifact_paths: "flame_default_1m/*" + # agents: + # slurm_mem: 24GB + + # - label: ":fire: Flame graph: diagnostics" + # command: > + # julia --color=yes --project=.buildkite perf/flame.jl + # --config_file $PERF_CONFIG_PATH/bm_diagnostics.yml + # --job_id flame_diagnostics + # artifact_paths: "flame_diagnostics/*" + # agents: + # slurm_mem: 24GB + + # - label: ":fire: Flame graph: diagnostics edmf" + # command: > + # julia --color=yes --project=.buildkite perf/flame.jl + # --config_file $PERF_CONFIG_PATH/bm_aquaplanet_diagedmf.yml + # --job_id flame_aquaplanet_diagedmf + # artifact_paths: "flame_aquaplanet_diagedmf/*" + # agents: + # slurm_mem: 24GB + + # - label: ":fire: Flame graph: prognostic edmf" + # command: > + # julia --color=yes --project=.buildkite perf/flame.jl + # --config_file $PERF_CONFIG_PATH/bm_aquaplanet_progedmf.yml + # --job_id flame_aquaplanet_progedmf + # artifact_paths: "flame_aquaplanet_progedmf/*" + # agents: + # slurm_mem: 32GB + + # - label: ":fire: Flame graph: prognostic edmf dense autodiff" + # command: > + # julia --color=yes --project=.buildkite perf/flame.jl + # --config_file $PERF_CONFIG_PATH/bm_aquaplanet_progedmf_dense_autodiff.yml + # --job_id flame_aquaplanet_progedmf_dense_autodiff + # artifact_paths: "flame_aquaplanet_progedmf_dense_autodiff/*" + # agents: + # slurm_mem: 32GB + + # - label: ":fire: Flame graph: prognostic edmf sparse autodiff" + # command: > + # julia --color=yes --project=.buildkite perf/flame.jl + # --config_file $PERF_CONFIG_PATH/bm_aquaplanet_progedmf_sparse_autodiff.yml + # --job_id flame_aquaplanet_progedmf_sparse_autodiff + # artifact_paths: "flame_aquaplanet_progedmf_sparse_autodiff/*" + # agents: + # slurm_mem: 32GB + + # - label: ":fire: Flame graph: diffusion" + # command: > + # julia --color=yes --project=.buildkite perf/flame.jl + # --config_file $PERF_CONFIG_PATH/bm_diffusion.yml + # --job_id flame_diffusion + # artifact_paths: "flame_diffusion/*" + # agents: + # slurm_mem: 24GB + + # - label: ":fire: Flame graph: perf target (Callbacks)" + # command: > + # julia --color=yes --project=.buildkite perf/flame.jl + # --config_file $PERF_CONFIG_PATH/bm_callbacks.yml + # --job_id flame_callbacks + # artifact_paths: "flame_callbacks/*" + # agents: + # slurm_mem: 24GB + + # - label: ":fire: Flame graph: gravity wave" + # command: > + # julia --color=yes --project=.buildkite perf/flame.jl + # --config_file $PERF_CONFIG_PATH/bm_gravity_wave.yml + # --job_id flame_gravity_wave + # artifact_paths: "flame_gravity_wave/*" + # agents: + # slurm_mem: 24GB + + # - group: "Checkbounds/Inference/Invalidations" + # steps: + + # # TODO: we should somehow decouple this unit test from the perf env / scripts + # # Checkbounds + # - label: ":computer: checkbounds" + # command: > + # julia --color=yes --check-bounds=yes --project=.buildkite perf/benchmark.jl + # --config_file $PERF_CONFIG_PATH/bm_default.yml + # --job_id checkbounds + # artifact_paths: "checkbounds/output_active/*" + # agents: + # slurm_mem: 20GB + + # # Inference + # - label: ":rocket: JET n-failures (inference)" + # command: > + # julia --color=yes --project=.buildkite perf/jet_test_nfailures.jl + # --config_file $PERF_CONFIG_PATH/bm_default.yml + # --job_id jet_n_failures + # agents: + # slurm_mem: 24GB + + # # Latency + # - label: ":mag::rocket: Invalidations" + # command: > + # julia --color=yes --project=.buildkite perf/invalidations.jl + # --config_file $PERF_CONFIG_PATH/bm_default.yml + # artifact_paths: "invalidations/*" + # agents: + # slurm_mem: 24GB + + # - wait: ~ + # continue_on_failure: true + + # - label: ":robot_face: Print new mse tables" + # command: "julia --color=yes --project=.buildkite reproducibility_tests/mse_summary.jl" + + # - label: ":robot_face: Print new reference counter" + # command: "julia --color=yes --project=.buildkite reproducibility_tests/print_new_ref_counter.jl" + + # - label: ":bar_chart: Tabulate performance summary" + # command: "julia --color=yes --project=.buildkite perf/tabulate_perf_summary.jl" + + # # - label: ":chart_with_downwards_trend: build history" + # # command: + # # - build_history main + # # artifact_paths: + # # - "build_history.html" + + # - wait + + # - label: ":robot_face: Move main results" + # command: "julia --color=yes --project=.buildkite reproducibility_tests/move_output.jl" diff --git a/src/prognostic_equations/mass_flux_closures.jl b/src/prognostic_equations/mass_flux_closures.jl index 2ecc7bc371..ffe86b34db 100644 --- a/src/prognostic_equations/mass_flux_closures.jl +++ b/src/prognostic_equations/mass_flux_closures.jl @@ -118,6 +118,33 @@ function edmfx_nh_pressure_drag_tendency!( end end +edmfx_vertical_diffusion_tendency!(Yₜ, Y, p, t, turbconv_model) = nothing + +function edmfx_vertical_diffusion_tendency!(Yₜ, Y, p, t, turbconv_model::PrognosticEDMFX) + FT = eltype(p.params) + n = n_mass_flux_subdomains(turbconv_model) + ᶜK_h = p.scratch.ᶜtemp_scalar + @. ᶜK_h = FT(0) + ᶜdivᵥ_mse = Operators.DivergenceF2C( + top = Operators.SetValue(C3(0)), + bottom = Operators.SetValue(C3(0)), + ) + ᶜdivᵥ_q_tot = Operators.DivergenceF2C( + top = Operators.SetValue(C3(0)), + bottom = Operators.SetValue(C3(0)), + ) + + for j in 1:n + ᶜρaʲ = Y.c.sgsʲs.:($j).ρa + ᶜmseʲ = Y.c.sgsʲs.:($j).mse + ᶜq_totʲ = Y.c.sgsʲs.:($j).q_tot + # @. Yₜ.c.sgsʲs.:($$j).mse -= ᶜdivᵥ_mse(-(ᶠinterp(ᶜρaʲ) * ᶠinterp(ᶜK_h) * ᶠgradᵥ(ᶜmseʲ))) / max(ᶜρaʲ, eps(FT)) + # @. Yₜ.c.sgsʲs.:($$j).q_tot -= ᶜdivᵥ_q_tot(-(ᶠinterp(ᶜρaʲ) * ᶠinterp(ᶜK_h) * ᶠgradᵥ(ᶜq_totʲ))) / max(ᶜρaʲ, eps(FT)) + @. Yₜ.c.sgsʲs.:($$j).mse -= ᶜdivᵥ_mse(-(ᶠinterp(ᶜK_h) * ᶠgradᵥ(ᶜmseʲ))) + @. Yₜ.c.sgsʲs.:($$j).q_tot -= ᶜdivᵥ_q_tot(-(ᶠinterp(ᶜK_h) * ᶠgradᵥ(ᶜq_totʲ))) + end +end + """ edmfx_filter_tendency!(Yₜ, Y, p, t, turbconv_model) diff --git a/src/prognostic_equations/remaining_tendency.jl b/src/prognostic_equations/remaining_tendency.jl index 3fe38dcdd5..4ef4f35dc6 100644 --- a/src/prognostic_equations/remaining_tendency.jl +++ b/src/prognostic_equations/remaining_tendency.jl @@ -231,6 +231,7 @@ NVTX.@annotate function additional_tendency!(Yₜ, Y, p, t) if p.atmos.sgs_nh_pressure_mode == Explicit() edmfx_nh_pressure_drag_tendency!(Yₜ, Y, p, t, p.atmos.turbconv_model) end + edmfx_vertical_diffusion_tendency!(Yₜ, Y, p, t, p.atmos.turbconv_model) edmfx_filter_tendency!(Yₜ, Y, p, t, p.atmos.turbconv_model) edmfx_tke_tendency!(Yₜ, Y, p, t, p.atmos.turbconv_model)