Merge pull request #3593 from CliMA/dy/steady_state_tests

Add tests and plots for steady-state velocity

Author: dennisYatunin

.buildkite/ci_driver.jl

@@ -28,7 +28,7 @@ sol_res = CA.solve_atmos!(simulation)
 (; p) = integrator
 
 import ClimaCore
-import ClimaCore: Topologies, Quadratures, Spaces
+import ClimaCore: Topologies, Quadratures, Spaces, Fields
 import ClimaComms
 using SciMLBase
 using PrettyTables
@@ -38,6 +38,8 @@ using ClimaTimeSteppers
 using Test
 import Tar
 import Base.Filesystem: rm
+import Statistics: mean
+import LinearAlgebra: norm_sqr
 include(joinpath(pkgdir(CA), "post_processing", "ci_plots.jl"))
 
 ref_job_id = config.parsed_args["reference_job_id"]
@@ -108,6 +110,58 @@ end
 # Write diagnostics that are in DictWriter to text files
 CA.write_diagnostics_as_txt(simulation)
 
+if config.parsed_args["check_steady_state"]
+    Y_end = integrator.sol.u[end]
+    t_end = integrator.sol.t[end]
+    (; steady_state_velocity, params) = integrator.p
+    (; zd_rayleigh) = params
+    FT = eltype(Y_end)
+
+    @info "Comparing velocity fields to predicted steady state at t = $t_end"
+    ᶜu_normsqr = norm_sqr.(steady_state_velocity.ᶜu)
+    ᶠu_normsqr = norm_sqr.(steady_state_velocity.ᶠu)
+    ᶜuₕ_err_normsqr = norm_sqr.(Y_end.c.uₕ .- CA.C12.(steady_state_velocity.ᶜu))
+    ᶠu₃_err_normsqr = norm_sqr.(Y_end.f.u₃ .- CA.C3.(steady_state_velocity.ᶠu))
+
+    # Average all errors below the sponge layer.
+    ᶜsponge_mask = FT.(Fields.coordinate_field(Y_end.c).z .< zd_rayleigh)
+    ᶠsponge_mask = FT.(Fields.coordinate_field(Y_end.f).z .< zd_rayleigh)
+    ᶜu_rms = sqrt(sum(ᶜu_normsqr .* ᶜsponge_mask) / sum(ᶜsponge_mask))
+    ᶠu_rms = sqrt(sum(ᶠu_normsqr .* ᶠsponge_mask) / sum(ᶠsponge_mask))
+    ᶜuₕ_rmse = sqrt(sum(ᶜuₕ_err_normsqr .* ᶜsponge_mask) / sum(ᶜsponge_mask))
+    ᶠu₃_rmse = sqrt(sum(ᶠu₃_err_normsqr .* ᶠsponge_mask) / sum(ᶠsponge_mask))
+    ᶜuₕ_rel_err = ᶜuₕ_rmse / ᶜu_rms
+    ᶠu₃_rel_err = ᶠu₃_rmse / ᶠu_rms
+
+    # Average the errors on several levels close to the surface.
+    n_levels = 3
+    level_uₕ_rel_errs = map(1:n_levels) do level
+        level_u_rms = sqrt(mean(Fields.level(ᶜu_normsqr, level)))
+        level_uₕ_rmse = sqrt(mean(Fields.level(ᶜuₕ_err_normsqr, level)))
+        level_uₕ_rmse / level_u_rms
+    end
+    level_u₃_rel_errs = map((1:n_levels) .- Fields.half) do level
+        level_u_rms = sqrt(mean(Fields.level(ᶠu_normsqr, level)))
+        level_u₃_rmse = sqrt(mean(Fields.level(ᶠu₃_err_normsqr, level)))
+        level_u₃_rmse / level_u_rms
+    end
+
+    @info "    Absolute RMSE of uₕ below sponge layer: $ᶜuₕ_rmse"
+    @info "    Absolute RMSE of u₃ below sponge layer: $ᶠu₃_rmse"
+    @info "    Relative RMSE of uₕ below sponge layer: $ᶜuₕ_rel_err"
+    @info "    Relative RMSE of u₃ below sponge layer: $ᶠu₃_rel_err"
+    @info "    Relative RMSE of uₕ on $n_levels levels closest to the surface:"
+    @info "        $level_uₕ_rel_errs"
+    @info "    Relative RMSE of u₃ on $n_levels levels closest to the surface:"
+    @info "        $level_u₃_rel_errs"
+
+    if t_end > 24 * 60 * 60
+        # TODO: Float32 simulations currently show significant divergence of uₕ.
+        @test ᶜuₕ_rel_err < (FT == Float32 ? 0.05 : 0.005)
+        @test ᶠu₃_rel_err < 0.0005
+    end
+end
+
 # Conservation checks
 if config.parsed_args["check_conservation"]
     FT = Spaces.undertype(axes(sol.u[end].c.ρ))

.buildkite/pipeline.yml

@@ -178,27 +178,94 @@ steps:
 
   - group: "Plane Examples"
     steps:
-      - label: ":computer: Agnesi linear hydrostatic mountain experiment (uniform)"
+
+      - label: ":computer: Density current experiment"
         command: >
           julia --color=yes --project=.buildkite .buildkite/ci_driver.jl
-          --config_file $CONFIG_PATH/plane_agnesi_mountain_test_uniform.yml
-          --job_id plane_agnesi_mountain_test_uniform
-        artifact_paths: "plane_agnesi_mountain_test_uniform/output_active/*"
+          --config_file $CONFIG_PATH/plane_density_current_test.yml
+          --job_id plane_density_current_test
+        artifact_paths: "plane_density_current_test/output_active/*"
 
-      - label: ":computer: Agnesi linear hydrostatic mountain experiment (stretched)"
+  - group: "Analytic Tests"
+    steps:
+      - label: "GPU: No Topography Test (2D, Float64, Discrete Balance)"
         command: >
           julia --color=yes --project=.buildkite .buildkite/ci_driver.jl
-          --config_file $CONFIG_PATH/plane_agnesi_mountain_test_stretched.yml
-          --job_id plane_agnesi_mountain_test_stretched
-        artifact_paths: "plane_agnesi_mountain_test_stretched/output_active/*"
+          --config_file $CONFIG_PATH/plane_no_topography_float64_test.yml
+          --job_id gpu_plane_no_topography_float64_test
+        artifact_paths: "gpu_plane_no_topography_float64_test/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+        agents:
+          slurm_gpus: 1
 
-      - label: ":computer: Density current experiment"
+      - label: "GPU: Cosine Hills Test (2D, Float64)"
         command: >
           julia --color=yes --project=.buildkite .buildkite/ci_driver.jl
-          --config_file $CONFIG_PATH/plane_density_current_test.yml
-          --job_id plane_density_current_test
-        artifact_paths: "plane_density_current_test/output_active/*"
+          --config_file $CONFIG_PATH/plane_cosine_hills_float64_test.yml
+          --job_id gpu_plane_cosine_hills_float64_test
+        artifact_paths: "gpu_plane_cosine_hills_float64_test/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+        agents:
+          slurm_gpus: 1
+
+      - label: "GPU: Cosine Hills Test (Extruded 2D, Float64)"
+        command: >
+          julia --color=yes --project=.buildkite .buildkite/ci_driver.jl
+          --config_file $CONFIG_PATH/extruded_plane_cosine_hills_float64_test.yml
+          --job_id gpu_extruded_plane_cosine_hills_float64_test
+        artifact_paths: "gpu_extruded_plane_cosine_hills_float64_test/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+        agents:
+          slurm_gpus: 1
+          slurm_mem: 48GB
+
+      - label: "GPU: Cosine Hills Test (3D, Float64)"
+        command: >
+          julia --color=yes --project=.buildkite .buildkite/ci_driver.jl
+          --config_file $CONFIG_PATH/box_cosine_hills_float64_test.yml
+          --job_id gpu_box_cosine_hills_float64_test
+        artifact_paths: "gpu_box_cosine_hills_float64_test/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+        agents:
+          slurm_gpus: 1
+          slurm_mem: 48GB
+
+      - label: "GPU: Agnesi Mountain Test (2D, Float64)"
+        command: >
+          julia --color=yes --project=.buildkite .buildkite/ci_driver.jl
+          --config_file $CONFIG_PATH/plane_agnesi_mountain_float64_test.yml
+          --job_id gpu_plane_agnesi_mountain_float64_test
+        artifact_paths: "gpu_plane_agnesi_mountain_float64_test/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+        agents:
+          slurm_gpus: 1
 
+      - label: "GPU: Schar Mountain Test (2D, Float64)"
+        command: >
+          julia --color=yes --project=.buildkite .buildkite/ci_driver.jl
+          --config_file $CONFIG_PATH/plane_schar_mountain_float64_test.yml
+          --job_id gpu_plane_schar_mountain_float64_test
+        artifact_paths: "gpu_plane_schar_mountain_float64_test/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+        agents:
+          slurm_gpus: 1
+
+      - label: "GPU: Schar Mountain Test (2D, Float32)"
+        command: >
+          julia --color=yes --project=.buildkite .buildkite/ci_driver.jl
+          --config_file $CONFIG_PATH/plane_schar_mountain_float32_test.yml
+          --job_id gpu_plane_schar_mountain_float32_test
+        artifact_paths: "gpu_plane_schar_mountain_float32_test/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+        agents:
+          slurm_gpus: 1
 
   - group: "Conservation check"
     steps:

config/default_configs/default_config.yml

@@ -35,7 +35,7 @@ scalar_hyperdiffusion_coefficient:
   value: 1.5
 # Topography
 topography:
-  help: "Define the surface elevation profile [`NoWarp` (default),`Earth`,`DCMIP200`,`Agnesi`, `Schar`, `Hughes2023`]"
+  help: "Define the surface elevation profile [`NoWarp` (default), `Earth`, `DCMIP200`, `Hughes2023`, `Agnesi`, `Schar`, `Cosine2D`, `Cosine3D`]"
   value: "NoWarp"
 mesh_warp_type:
   help: "Sets the interior mesh warp method [`Linear`, `SLEVE`]"
@@ -182,7 +182,7 @@ surface_temperature:
   help: "Prescribed surface temperature functional form ['ZonallySymmetric' (default), 'ZonallyAsymmetric', 'RCEMIPII']"
   value: "ZonallySymmetric"
 initial_condition:
-  help: "Initial condition [`DryBaroclinicWave`, `MoistBaroclinicWave`, `DecayingProfile`, `IsothermalProfile`, `Bomex`, `DryDensityCurrentProfile`, `AgnesiHProfile`, `ScharProfile`, `RisingThermalBubbleProfile`, `ISDAC`], or a file path for a NetCDF file (read documentation about requirements)."
+  help: "Initial condition [`DryBaroclinicWave`, `MoistBaroclinicWave`, `ConstantBuoyancyFrequencyProfile`, `DecayingProfile`, `IsothermalProfile`, `Bomex`, `DryDensityCurrentProfile`, `RisingThermalBubbleProfile`, `ISDAC`], or a file path for a NetCDF file (read documentation about requirements)."
   value: "DecayingProfile"
 perturb_initstate:
   help: "Add a perturbation to the initial condition [`false`, `true` (default)]"
@@ -226,6 +226,9 @@ reproducibility_test:
 check_conservation:
   help: "Check conservation of mass and energy [`false` (default), `true`]"
   value: false
+check_steady_state:
+  help: "Compare steady-state velocity to analytic solution; only available for certain choices of `topography` [`false` (default), `true`]"
+  value: false
 ls_adv:
   help: "Large-scale advection [`nothing` (default), `Bomex`, `LifeCycleTan2018`, `Rico`, `ARM_SGP`, `GATE_III`]"
   value: ~

config/model_configs/box_cosine_hills_float64_test.yml

@@ -0,0 +1,20 @@
+config: "box"
+FLOAT_TYPE: "Float64"
+initial_condition: "ConstantBuoyancyFrequencyProfile"
+topography: "Cosine3D"
+x_max: 100e3
+y_max: 25e3
+z_max: 21e3
+x_elem: 100
+y_elem: 25
+z_elem: 100
+dz_bottom: 10
+dt: "0.5secs" # CFL is slightly lower than for 2D case
+t_end: "30mins"
+rayleigh_sponge: true
+toml: [toml/steady_state_test.toml]
+check_steady_state: true
+output_default_diagnostics: false
+diagnostics:
+  - short_name: [orog, ua, wa, uapredicted, wapredicted, uaerror, waerror]
+    period: 1mins

config/model_configs/extruded_plane_cosine_hills_float64_test.yml

@@ -0,0 +1,20 @@
+config: "box"
+FLOAT_TYPE: "Float64"
+initial_condition: "ConstantBuoyancyFrequencyProfile"
+topography: "Cosine2D"
+x_max: 100e3
+y_max: 1e3
+z_max: 21e3
+x_elem: 100
+y_elem: 1
+z_elem: 100
+dz_bottom: 10
+dt: "0.5secs" # CFL is slightly lower than for 2D case
+t_end: "9hours"
+rayleigh_sponge: true
+toml: [toml/steady_state_test.toml]
+check_steady_state: true
+output_default_diagnostics: false
+diagnostics:
+  - short_name: [orog, ua, wa, uapredicted, wapredicted, uaerror, waerror]
+    period: 1hours

config/model_configs/plane_agnesi_mountain_float64_test.yml

@@ -0,0 +1,18 @@
+config: "plane"
+FLOAT_TYPE: "Float64"
+initial_condition: "ConstantBuoyancyFrequencyProfile"
+topography: "Agnesi"
+x_max: 100e3
+z_max: 21e3
+x_elem: 100
+z_elem: 100
+dz_bottom: 10
+dt: "0.7secs"
+t_end: "2days"
+rayleigh_sponge: true
+toml: [toml/steady_state_test.toml]
+check_steady_state: true
+output_default_diagnostics: false
+diagnostics:
+  - short_name: [orog, ua, wa, uapredicted, wapredicted, uaerror, waerror]
+    period: 1hours

config/model_configs/plane_agnesi_mountain_test_stretched.yml

@@ -0,0 +1,18 @@
+config: "plane"
+FLOAT_TYPE: "Float64"
+initial_condition: "ConstantBuoyancyFrequencyProfile"
+topography: "Cosine2D"
+x_max: 100e3
+z_max: 21e3
+x_elem: 100
+z_elem: 100
+dz_bottom: 10
+dt: "0.7secs"
+t_end: "2days"
+rayleigh_sponge: true
+toml: [toml/steady_state_test.toml]
+check_steady_state: true
+output_default_diagnostics: false
+diagnostics:
+  - short_name: [orog, ua, wa, uapredicted, wapredicted, uaerror, waerror]
+    period: 1hours

config/model_configs/plane_agnesi_mountain_test_uniform.yml

@@ -0,0 +1,18 @@
+config: "plane"
+FLOAT_TYPE: "Float64"
+initial_condition: "ConstantBuoyancyFrequencyProfile"
+discrete_hydrostatic_balance: true
+x_max: 100e3
+z_max: 21e3
+x_elem: 100
+z_elem: 100
+dz_bottom: 10
+dt: "0.7secs"
+t_end: "2days"
+rayleigh_sponge: true
+toml: [toml/steady_state_test.toml]
+check_steady_state: true
+output_default_diagnostics: false
+diagnostics:
+  - short_name: [orog, ua, wa, uapredicted, wapredicted, uaerror, waerror]
+    period: 1hours