fix config file parsing order

Author: juliasloan25

experiments/ClimaEarth/cli_options.jl

@@ -135,6 +135,8 @@ function argparse_settings()
         default = "PrescribedSurface"
         "--atmos_config_file"
         help = "An optional YAML file used to overwrite the default model parameters."
+        arg_type = String
+        default = nothing
         "--albedo_model"
         help = "Type of albedo model. [`ConstantAlbedo`, `RegressionFunctionAlbedo`, `CouplerAlbedo` (default)]"
         arg_type = String

experiments/ClimaEarth/components/atmosphere/climaatmos.jl

@@ -473,10 +473,10 @@ end
 ### ClimaAtmos.jl model-specific functions (not explicitly required by ClimaCoupler.jl)
 ###
 """
-    get_atmos_config_dict(coupler_dict::Dict, atmos_output_dir)
+    get_atmos_config_dict(coupler_config::Dict, atmos_output_dir)
 
 Returns the specified atmospheric configuration (`atmos_config`) overwitten by arguments
-in the coupler dictionary (`config_dict`).
+in the coupler dictionary (`coupler_config`).
 The returned `atmos_config` dictionary will then be used to set up the atmosphere simulation.
 
 In this function, parameters are overwritten in a specific order, from lowest to highest priority:
@@ -489,20 +489,11 @@ The TOML parameter file to use is chosen using the following priority:
 If a coupler TOML file is provided, it is used. Otherwise we use an atmos TOML
 file if it's provided. If neither is provided, we use a default coupler TOML file.
 """
-function get_atmos_config_dict(coupler_dict::Dict, atmos_output_dir)
-    atmos_config_file = coupler_dict["atmos_config_file"]
-    # override default or specified configs with coupler arguments, and set the correct atmos config_file
-    if isnothing(atmos_config_file)
-        @info "Using Atmos default configuration"
-        atmos_config = merge(CA.default_config_dict(), coupler_dict, Dict("config_file" => atmos_config_file))
-    else
-        @info "Using Atmos configuration from ClimaCoupler in $atmos_config_file"
-        atmos_config = merge(
-            CA.override_default_config(joinpath(pkgdir(ClimaCoupler), atmos_config_file)),
-            coupler_dict,
-            Dict("config_file" => atmos_config_file),
-        )
-    end
+function get_atmos_config_dict(coupler_config::Dict, atmos_output_dir)
+    atmos_config = deepcopy(coupler_config)
+
+    # Rename atmosphere config file from ClimaCoupler convention to ClimaAtmos convention
+    atmos_config["config_file"] = coupler_config["atmos_config_file"]
 
     # use atmos toml if coupler toml is not defined
     # If we can't find the file at the relative path, prepend pkgdir(ClimaAtmos)
@@ -523,25 +514,25 @@ function get_atmos_config_dict(coupler_dict::Dict, atmos_output_dir)
     # Ensure Atmos's own checkpoints are synced up with ClimaCoupler, so that we
     # can pick up from where we have left. NOTE: This should not be needed, but
     # there is no easy way to initialize ClimaAtmos with a different t_start
-    atmos_config["dt_save_state_to_disk"] = coupler_dict["checkpoint_dt"]
+    atmos_config["dt_save_state_to_disk"] = coupler_config["checkpoint_dt"]
 
     # Add all extra atmos diagnostic entries into the vector of atmos diagnostics
     atmos_config["diagnostics"] =
         haskey(atmos_config, "diagnostics") ?
-        vcat(atmos_config["diagnostics"], coupler_dict["extra_atmos_diagnostics"]) :
-        coupler_dict["extra_atmos_diagnostics"]
+        vcat(atmos_config["diagnostics"], coupler_config["extra_atmos_diagnostics"]) :
+        coupler_config["extra_atmos_diagnostics"]
 
     # The Atmos `get_simulation` function expects the atmos config to contains its timestep size
-    # in the `dt` field. If there is a `dt_atmos` field in coupler_dict, we add it to the atmos config as `dt`
-    dt_atmos = haskey(coupler_dict, "dt_atmos") ? coupler_dict["dt_atmos"] : coupler_dict["dt"]
+    # in the `dt` field. If there is a `dt_atmos` field in coupler_config, we add it to the atmos config as `dt`
+    dt_atmos = haskey(coupler_config, "dt_atmos") ? coupler_config["dt_atmos"] : coupler_config["dt"]
     atmos_config["dt"] = dt_atmos
 
     # set restart file to the initial file saved in this location if it is not nothing
     # TODO this is hardcoded and should be fixed once we have a better restart system
     if !isnothing(atmos_config["restart_file"])
         atmos_config["restart_file"] = replace(atmos_config["restart_file"], "active" => "0000")
     end
-    return atmos_config
+    return CA.AtmosConfig(atmos_config)
 end
 
 """

experiments/ClimaEarth/setup_run.jl

@@ -157,7 +157,6 @@ function CoupledSimulation(config_dict::AbstractDict)
     ## Note this step must come after parsing the coupler config dictionary, since
     ##  some parameters are passed from the coupler config to the component model configs
     atmos_config_dict = get_atmos_config_dict(config_dict, atmos_output_dir)
-    (; dt_rad) = get_atmos_args(atmos_config_dict)
 
     ## set unique random seed if desired, otherwise use default
     Random.seed!(random_seed)
@@ -177,12 +176,12 @@ function CoupledSimulation(config_dict::AbstractDict)
             # We only support a round number of seconds
             isinteger(float(t_start)) || error("Cannot restart from a non integer number of seconds")
             t_start_int = Int(float(t_start))
-            atmos_config_dict["t_start"] = "$(t_start_int)secs"
+            atmos_config_dict.parsed_args["t_start"] = "$(t_start_int)secs"
         else
             # There was no `t_start`, so we have to use a workaround for this.
             # This does not support passing the command-line arguments (unless
             # restart_dir is exactly the same as output_dir_root)
-            atmos_config_dict["restart_file"] = climaatmos_restart_path(output_dir_root, restart_t)
+            atmos_config_dict.parsed_args["restart_file"] = climaatmos_restart_path(output_dir_root, restart_t)
         end
 
         @info "Starting from t_start $(t_start)"
@@ -207,7 +206,7 @@ function CoupledSimulation(config_dict::AbstractDict)
     =#
 
     ## init atmos model component
-    atmos_sim = ClimaAtmosSimulation(CA.AtmosConfig(atmos_config_dict))
+    atmos_sim = ClimaAtmosSimulation(atmos_config_dict)
     thermo_params = get_thermo_params(atmos_sim) # TODO: this should be shared by all models #342
 
     #=

experiments/ClimaEarth/user_io/arg_parsing.jl

@@ -14,20 +14,49 @@ mode_name_dict = Dict(
 Read in the configuration file and job ID from the command line.
 A dictionary is constructed from the input configuration file and returned.
 
+Since the atmosphere model also uses a configuration file, we read in the atmosphere
+configuration file specified in the coupler configuration file (if any), and overwrite
+it with the coupler configuration.
+
+The order of priority for overwriting configuration options from lowest to highest is:
+    1. ClimaAtmos defaults
+    2. ClimaCoupler defaults (defined in `experiments/ClimaEarth/cli_options.jl`)
+    3. Command line arguments provided to ClimaCoupler
+    4. ClimaAtmos configuration file (if specified in coupler config file)
+    5. ClimaCoupler configuration file
+
 # Returns
 - `config_dict`: A dictionary mapping configuration keys to the specified settings
 """
 function get_coupler_config_dict(config_file)
-    # Extract the job ID from the command line arguments
-    parsed_args = parse_commandline(argparse_settings())
-    job_id = parsed_args["job_id"]
-
-    # Get the job ID from the config file string if not provided
-    job_id = isnothing(job_id) ? string(split(split(config_file, '/')[end], '.')[1]) : job_id
+    # Get the coupler default configuration dictionary, overwritten by any command line arguments
+    # Typically the command line arguments are only `config_file` and `job_id`
+    coupler_default_cli = parse_commandline(argparse_settings())
+
+    # Extract the job ID from the command line arguments, or from the config file name if not provided
+    job_id = coupler_default_cli["job_id"]
+    coupler_default_cli["job_id"] = isnothing(job_id) ? string(split(split(config_file, '/')[end], '.')[1]) : job_id
+
+    # Load the coupler config file into a dictionary
+    coupler_config_dict = YAML.load_file(config_file)
+
+    # Get ClimaAtmos default configuration dictionary
+    atmos_default = CA.default_config_dict()
+    atmos_config_file = merge(coupler_default_cli, coupler_config_dict)["atmos_config_file"]
+    if isnothing(atmos_config_file)
+        @info "Using Atmos default configuration"
+
+        # Merge the atmos default config, coupler default config + command line inputs,
+        #  and user-provided coupler config
+        config_dict = merge(atmos_default, coupler_default_cli, coupler_config_dict)
+    else
+        @info "Using Atmos configuration from ClimaCoupler in $atmos_config_file"
+        atmos_config_dict = YAML.load_file(joinpath(pkgdir(ClimaCoupler), atmos_config_file))
 
-    # Read in config dictionary from file, overriding the defaults in `parsed_args`
-    config_dict = merge(parsed_args, YAML.load_file(config_file))
-    config_dict["job_id"] = job_id
+        # Merge the atmos default config, coupler default config + command line inputs,
+        #  user-provided atmos config, and user-provided coupler config
+        config_dict = merge(atmos_default, coupler_default_cli, atmos_config_dict, coupler_config_dict)
+    end
 
     # Select the correct timestep for each component model based on which are available
     parse_component_dts!(config_dict)
@@ -159,23 +188,6 @@ function get_coupler_args(config_dict::Dict)
     )
 end
 
-"""
-    get_atmos_args(atmos_config_dict)
-
-Extract the necessary arguments from the atmosphere configuration dictionary.
-
-# Arguments
-- `atmos_config_dict`: A dictionary mapping atmosphere configuration keys to the specified settings
-
-# Returns
-- All arguments needed for the atmosphere simulation
-"""
-function get_atmos_args(atmos_config_dict)
-    dt_rad = atmos_config_dict["dt_rad"]
-    return (; dt_rad)
-end
-
-
 ### Helper functions used in argument parsing ###
 
 """