Skip to content

Commit a24042b

Browse files
committed
Updated to v3.0. Gitlab mirroring is broken, hence this all-in-one update.
1 parent f625d41 commit a24042b

22 files changed

+2177
-534
lines changed

.gitlab-ci.yml

Lines changed: 38 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
image: "mahieronymus/ad_sensitivity:2.12"
1+
image: "mahieronymus/ad_sensitivity:3.0"
22

33
stages:
44
- Static Analysis CPP
55
- build
6-
6+
- test
77

88
lint:
99
stage: Static Analysis CPP
@@ -21,20 +21,52 @@ build:
2121
script:
2222
- mkdir build
2323
- cd build &&
24-
cmake .. -DCMAKE_BUILD_TYPE=release -DTARGET=simulation -DTRUSTED_DATA:BOOL=ON -DB_EIGHT:BOOL=ON -DCCN_AKM:BOOL=ON &&
24+
cmake .. -DCMAKE_BUILD_TYPE=release -DTARGET=simulation -DTRUSTED_DATA:BOOL=ON -DIN_SAT_ADJ:BOOL=ON -DB_EIGHT:BOOL=ON -DCCN_AKM:BOOL=ON &&
2525
make -j4
2626
- cd .. && mkdir build_interface
2727
- cd build_interface &&
28-
cmake .. -DCMAKE_BUILD_TYPE=release -DTARGET=python_interface -DTRUSTED_DATA:BOOL=ON -DB_EIGHT:BOOL=ON -DCCN_AKM:BOOL=ON &&
28+
cmake .. -DCMAKE_BUILD_TYPE=release -DTARGET=python_interface -DTRUSTED_DATA:BOOL=ON -DIN_SAT_ADJ:BOOL=ON -DB_EIGHT:BOOL=ON -DCCN_AKM:BOOL=ON &&
2929
make -j4
3030
- cd .. && mkdir build_regrid
3131
- cd build_regrid &&
32-
cmake .. -DCMAKE_BUILD_TYPE=release -DTARGET=regrid -DCOMPRESSION_LEVEL=6 -DTRUSTED_DATA:BOOL=ON -DB_EIGHT:BOOL=ON -DCCN_AKM:BOOL=ON &&
32+
cmake .. -DCMAKE_BUILD_TYPE=release -DTARGET=regrid -DCOMPRESSION_LEVEL=6 -DTRUSTED_DATA:BOOL=ON -DIN_SAT_ADJ:BOOL=ON -DB_EIGHT:BOOL=ON -DCCN_AKM:BOOL=ON &&
3333
make -j4
3434
- cd .. && mkdir build_loom
3535
- cd build_loom &&
36-
cmake .. -DCMAKE_BUILD_TYPE=release -DTARGET=loom -DCOMPRESS_OUTPUT:BOOL=ON -DCOMPRESSION_LEVEL=6 -DTRUSTED_DATA:BOOL=ON -DB_EIGHT:BOOL=ON -DCCN_AKM:BOOL=ON &&
36+
cmake .. -DCMAKE_BUILD_TYPE=release -DTARGET=loom -DCOMPRESS_OUTPUT:BOOL=ON -DCOMPRESSION_LEVEL=6 -DTRUSTED_DATA:BOOL=ON -DIN_SAT_ADJ:BOOL=ON -DB_EIGHT:BOOL=ON -DCCN_AKM:BOOL=ON &&
3737
make -j4
3838
tags:
3939
- make
4040
allow_failure: false
41+
42+
test:
43+
stage: test
44+
dependencies:
45+
- build
46+
script:
47+
- source /app/venv/bin/activate
48+
- export AD_SENS_HOME=`pwd`
49+
- ls
50+
- pwd
51+
- mkdir -p data/test_files_simulated
52+
- mpirun -n 4 build/bin/./trajectories
53+
-w 20000
54+
-a 3
55+
-t 0
56+
-f 234600
57+
-d 30
58+
-i 1
59+
-b 1
60+
-o data/test_files_simulated/artificial_test.nc
61+
-e 1
62+
-p 500
63+
-g 0
64+
-l data/artificial_test.nc
65+
-s configs/qv_qr_lat_config.json
66+
-u 180
67+
- cd /app/ad_sensitivity_analysis/tests/end_to_end && python test_output.py --input /builds/mahieron/process_plot_scripts/data/test_files_simulated/ --test_nan_dims
68+
--test_physics /builds/mahieron/process_plot_scripts/build_interface/lib/libpython_interface.so --test_phases --test_nan_vars --calc_phases --test_sensitivities --test_amounts --test_saturation
69+
--verbosity 4 --table_path /builds/mahieron/process_plot_scripts/dmin_wetgrowth_lookup.dat
70+
tags:
71+
- test
72+
allow_failure: false

CMakeLists.txt

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -241,7 +241,10 @@ if (LINE_INFO)
241241
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g")
242242
message("${BoldYellow}~~~ Compiling with -g.${ColourReset}")
243243
endif()
244-
244+
if (IN_SAT_ADJ)
245+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DIN_SAT_ADJ")
246+
message("${BoldYellow}~~~ Saturation adjustment is done at the start of each RK4 step and in the end of RK4")
247+
endif()
245248
message(STATUS "Searching for necessary libraries")
246249

247250
include_directories(${AD_SIM_HOME})

README.md

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,15 @@ the time step at which perturbing a parameter is necessary with a random forest
1717
of about 75% for each model parameter and hydrometeor. \
1818
In [another paper](https://doi.org/10.5194/gmd-16-4617-2023) we use [Met.3D](https://met3d.wavestoweather.de/met-3d.html)
1919
to analyze the output further.
20-
This version is used in a third paper where we analyze a WCB case with more than 16,000 trajectories associated
20+
This version is also used in a [third paper](https://doi.org/10.1016/j.jocs.2025.102614) where we analyze a WCB case with more than 16,000 trajectories associated
2121
with an extratropycal cyclone from 18 UTC 03 October 2016 and the following 66 hours.
2222

2323
This repository consists of an implementation of a two-moment scheme (similar to [ICON](https://www.dwd.de/EN/research/weatherforecasting/num_modelling/01_num_weather_prediction_modells/icon_description.html))
2424
with [AD using CoDiPack](https://github.com/scicompkl/codipack) where environment and initial variables are read from [NetCDF](https://www.unidata.ucar.edu/software/netcdf/) files.
2525
Python scripts are used for post-processing and data analysis. For the C++ code we follow the
2626
[doxygen standard](http://www.doxygen.nl/manual/docblocks.html). \
2727
We recommend using the docker image
28-
[mahieronymus/ad_sensitivity:2.12](https://hub.docker.com/repository/docker/mahieronymus/ad_sensitivity/general)
28+
[mahieronymus/ad_sensitivity:3.0](https://hub.docker.com/repository/docker/mahieronymus/ad_sensitivity/general)
2929
which provides the Python scripts for analyzing and plotting the data and all necessary prerequisites.
3030
Alternatively, you may use an [anaconda](https://www.anaconda.com/) environment with `pyproject.toml`.
3131

@@ -51,7 +51,7 @@ C++ Prerequisites
5151
- [NetCDF](https://www.unidata.ucar.edu/software/netcdf/) (v4.9.2)
5252
- [HDF5](https://www.hdfgroup.org/solutions/hdf5/) (v1.12 or above; tested with v1.14.2)
5353
- [Boost](https://www.boost.org/) (1.74.0 or above; tested with v1.81.0)
54-
- [CoDiPack](https://www.scicomp.uni-kl.de/software/codi/) (v2.0 or above: tested with v2.2.0)
54+
- [CoDiPack](https://www.scicomp.uni-kl.de/software/codi/) (v2.0 or above: tested with v3.0.0)
5555
- [CMake](https://cmake.org/) (v3.7.2 or above; tested with v3.26.4)
5656
- [nlohmann/json](https://github.com/nlohmann/json) (v3.9.1 or above; tested with v3.11.2)
5757
- [(optional) PnetCDF](https://parallel-netcdf.github.io/) (v1.12.2; only if you want to use classic NetCDF-files)
@@ -125,7 +125,7 @@ Default: off
125125

126126
Analyzing the output
127127
--------------------
128-
We recommend [our docker image]((https://hub.docker.com/repository/docker/mahieronymus/ad_sensitivity/general)) (v2.12).
128+
We recommend [our docker image]((https://hub.docker.com/repository/docker/mahieronymus/ad_sensitivity/general)) (v2.14).
129129
This docker image provides multiple scripts to analyze the data with Jupyter notebooks.
130130
The data of the sensitivity simulation is stored under `data/simulation/*.nc`.
131131
Within `ad_sensitivity_analysis/interactive/`, there are different functions that can be called with the

configs/Readme.md

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
# Configuration files
2+
There are two types of configuration files:
3+
- Configure gradient output
4+
- Configure ensemble simulations
5+
All configuration files are using the [JSON-format](https://ecma-international.org/publications-and-standards/standards/ecma-404/). This makes the configurations human-readable and easy to generate and read thanks to the extensive support in many languages. This code uses [JSON for Modern C++](https://github.com/nlohmann/json) to load the data.
6+
7+
## Configure gradient output
8+
This configuration is needed whenever a sensitivity analysis is done. It defines which targets, i.e., model state variables, are differentiated using `model_state_variable` and a list of ids of the desired targets. The ids are defined in `include/microphysics/constants.h` and are as follows:
9+
Target | Index
10+
---|----
11+
Pressure | 0
12+
Temperature | 1
13+
Vertical ascent velocity | 2
14+
Saturation | 3
15+
Cloud droplet mixing ratio | 4
16+
Rain droplet mixing | 5
17+
Water vapor mixing ratio | 6
18+
Number of cloud droplets | 7
19+
Number of rain droplets | 8
20+
Ice mixing ratio | 9
21+
Number of ice crystals | 10
22+
Snow mixing ratio | 11
23+
Number of snow particles | 12
24+
Graupel mixing ratio | 13
25+
Number of graupel particles | 14
26+
Hail mixing ratio | 15
27+
Number of hail particles | 16
28+
Ice mixing ratio precipitation | 17
29+
Snow mixing ratio precipitation | 18
30+
Rain mixing ratio precipitation | 19
31+
Graupel mixing ratio precipitation | 20
32+
Hail mixing ratio precipitation | 21
33+
Latent heating | 22
34+
Latent cooling | 23
35+
Ice particles precipitation | 24
36+
Snow particles ratio precipitation | 25
37+
Rain droplets ratio precipitation | 26
38+
Graupel particles ratio precipitation | 27
39+
Hail particles ratio precipitation | 28
40+
41+
The model parameters for which sensitivities are computed are defined by a list `out_params` by name such as `db_ccn_1` or `dcloud_a_geo`. Note the 'd' which usually indicates a derivative. An overview of all model parameters is available in variable `output_grad_idx` in `include/microphysics/constants.h`. You may define your own model parameters here if you add different physics. An example can be seen with `CCN_AKM` which is a preprocessor instruction for the C++-code that defines a different CCN activation.
42+
A full example that calculates the sensitivities of water vapor mixing ratio, latent heating and cooling to several parameters is:
43+
```
44+
{
45+
"model_state_variable": [
46+
6, 22, 23
47+
],
48+
"out_params": [
49+
"db_ccn_1",
50+
"db_ccn_2",
51+
"dc_ccn_1",
52+
"dc_ccn_2"
53+
]
54+
}
55+
```
56+
57+
## Configure ensemble simulations
58+
The ensemble simulations are organized in segments where each segment defines when the perturbed member is started, how many members there are, which parameters are perturbed and how the perturbation is calculated. While it is technically possible to define multiple segments with different perturbations each, it is easier to create multiple configuration files and run each independently when executing it on a cluster.
59+
The conditions for the ensembles are asserted and executed in the function `parameter_check()` in `src/microphysics/trajectories.cpp`.
60+
61+
### When to perturb
62+
The parameter `when_method` defines when to perturb the ensemble member. Possible options are
63+
- `full_perturbation`: Run the perturbation from the beginning.
64+
- `repeated_time`: Start the ensemble every `when_value` seconds.
65+
- `impact_change`: Start an ensemble simulation when the parameter with the highest derivative changes.
66+
- `sign_flip`: Start an ensemble when the gradient of a predefined target to a predefined parameter changes. See below for setting the parameter and target.
67+
- `value`: Start an ensemble if the gradient of a predefined target to a predefined parameter reaches a given value defined in `when_value` (within some tolerance).
68+
Further variables are:
69+
- `duration`: If `when_method` is not `full_perturbation`, you can limit the length of the ensemble simulation with this variable (in seconds).
70+
- `amount`: Define the number of ensemble members (including the unperturbed simulation).
71+
- `params`: A list that defines which parameters are perturbed and how.
72+
Each entry of `params` must define the parmater to be perturbed, how and by how much it is perturbed:
73+
- `name`: Name of the parameter to be perturbed such as `a_ccn_1` or `a_geo`.
74+
- `type`: Type of the parameter such as `model` if it is purely a model parameter such as `a_ccn_1` or the hydrometeor type of the parameter such as `cloud` for `a_geo`.
75+
- `rand_func`: The random number generator or function to be used. Possible options are:
76+
- `fixed`: Perturb by a fixed amount where the parameters is increased by the fixed amount for every second member and reduced otherwise.
77+
- `uniform`: Draw a random number from a uniform distribution.
78+
- `normal`: Draw from a normal distribution.
79+
- `sigma`: (We use $p$ for the parameter value in the following)
80+
- If `rand_func` is `uniform`: The uniform distribution is bound by the [$p - \sigma, p + \sigma$].
81+
- If `rand_func` is `normal`: The normal distribution is defined by $\mathcal{N}(p, \sigma)$ where $\sigma$ is the standard deviation.
82+
- If `rand_func` is `fixed`: The parameter is set to $p - \sigma$ and $p + \sigma$ alternatively for each ensemble member.
83+
- `sigma_perc`: Same as `sigma` but the perturbation is done by a percentage of the parameter value, e.g., for `rand_func` set to `fixed`, the parameter is set to $p - \sigma_{perc} \cdot p$ and $p + \sigma_{perc} \cdot p$ alternatively.
84+
85+
An example where `dcloud_a_geo` is perturbed by 10% where the ensemble is running from start to end with one member an increased and another with a decreased value:
86+
```
87+
{
88+
"segments": [
89+
{
90+
"when_method": "full_perturbation",
91+
"amount": 3,
92+
"params": [
93+
{
94+
"name": "a_geo",
95+
"sigma_perc": 10,
96+
"type": "cloud",
97+
"rand_func": "fixed"
98+
}
99+
]
100+
}
101+
]
102+
}
103+
```
104+
Another example where every 30 minutes an ensemble is started that runs for 30 minutes. There are 64 members with perturbed parameters drawn from a uniform distribution:
105+
```
106+
{
107+
"segments": [
108+
{
109+
"when_method": "repeated_time",
110+
"duration": 1800,
111+
"when_value": 1800,
112+
"amount": 65,
113+
"params": [
114+
{
115+
"name": "a_geo",
116+
"sigma_perc": 10,
117+
"type": "cloud",
118+
"rand_func": "uniform"
119+
}
120+
]
121+
}
122+
]
123+
}
124+
```

configs/james_sens_config.json

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,6 @@
11
{
22
"model_state_variable": [
3-
4, 5, 6, 7, 8, 9,
4-
10, 11, 12, 13, 14,
5-
15, 16, 17, 18, 19,
6-
20, 21, 24, 25, 26,
7-
27, 28
3+
6, 22, 23
84
],
95
"out_params": [
106
"dD_conv_ig",
@@ -15,7 +11,6 @@
1511
"dsnow_a_vel",
1612
"dgraupel_min_x_sedimentation",
1713
"dice_b_geo",
18-
"dinv_z",
1914
"dice_s_vel",
2015
"dgraupel_a_vel",
2116
"dsnow_b_vel",
@@ -125,7 +120,6 @@
125120
"dhail_max_x",
126121
"dice_min_x_evap",
127122
"dd_ccn_2",
128-
"dconst4",
129123
"dk_1_conv",
130124
"drain_rho_v",
131125
"dice_d_crit_c",
@@ -135,8 +129,6 @@
135129
"dice_min_x_depo",
136130
"drain_a_vel",
137131
"dna_orga",
138-
"dconst5",
139-
"dconst0",
140132
"dsnow_min_x_depo",
141133
"dgraupel_min_x_riming",
142134
"dcloud_max_x",
@@ -163,9 +155,7 @@
163155
"dhail_a_ven",
164156
"dcloud_min_x_freezing",
165157
"dr_0",
166-
"dN_avo",
167158
"dsnow_max_x",
168-
"dconst3",
169159
"dsnow_vsedi_max",
170160
"dice_rho_v",
171161
"dgraupel_b_vel",
@@ -191,5 +181,4 @@
191181
"drain_alpha",
192182
"dice_max_x"
193183
]
194-
195184
}

0 commit comments

Comments
 (0)