From 1f3804d249632ab018a2e6a9286d6bcd0c753d77 Mon Sep 17 00:00:00 2001
From: Stefan Poll <s.poll@fz-juelich.de>
Date: Sun, 23 Mar 2025 13:49:17 +0100
Subject: [PATCH 01/21] rewrite download ERA5 based on new CDSAPI

- remove old download scripts
- new download request
- add year, month and output directory as input for download script
---
 mkforcing/download_ERA5.py       | 51 --------------------
 mkforcing/download_ERA5_input.py | 80 ++++++++++++++++++++++++++++++++
 mkforcing/download_ERA5_v2.py    | 50 --------------------
 3 files changed, 80 insertions(+), 101 deletions(-)
 delete mode 100644 mkforcing/download_ERA5.py
 create mode 100755 mkforcing/download_ERA5_input.py
 delete mode 100755 mkforcing/download_ERA5_v2.py

diff --git a/mkforcing/download_ERA5.py b/mkforcing/download_ERA5.py
deleted file mode 100644
index f5a02eb..0000000
--- a/mkforcing/download_ERA5.py
+++ /dev/null
@@ -1,51 +0,0 @@
-import cdsapi
-
-c = cdsapi.Client()
-
-for year in [2017]:
-        for month in range(1,13,1):
-                if month<10:
-                        month="0"+str(month)
-                file="era5_"+str(year)+"_"+str(month)+".nc"
-                c.retrieve(
-    'reanalysis-era5-single-levels',
-    {
-        'product_type': 'reanalysis',
-        'variable': [
-            '10m_u_component_of_wind', '10m_v_component_of_wind', '2m_dewpoint_temperature',
-            '2m_temperature', 'surface_pressure', 'surface_solar_radiation_downwards',
-            'surface_thermal_radiation_downwards', 'total_precipitation',
-        ],
-        'year': year,
-            'month': month,
-        'day': [
-            '01', '02', '03',
-            '04', '05', '06',
-            '07', '08', '09',
-            '10', '11', '12',
-            '13', '14', '15',
-            '16', '17', '18',
-            '19', '20', '21',
-            '22', '23', '24',
-            '25', '26', '27',
-            '28', '29', '30',
-            '31',
-        ],
-        'time': [
-            '00:00', '01:00', '02:00',
-            '03:00', '04:00', '05:00',
-            '06:00', '07:00', '08:00',
-            '09:00', '10:00', '11:00',
-            '12:00', '13:00', '14:00',
-            '15:00', '16:00', '17:00',
-            '18:00', '19:00', '20:00',
-            '21:00', '22:00', '23:00',
-        ],
-        'area': [
-            74, -48, 20,
-            69,
-        ],
-        'format': 'netcdf',
-    },
-                        file)
-
diff --git a/mkforcing/download_ERA5_input.py b/mkforcing/download_ERA5_input.py
new file mode 100755
index 0000000..145df2b
--- /dev/null
+++ b/mkforcing/download_ERA5_input.py
@@ -0,0 +1,80 @@
+#!/usr/bin/env python
+import calendar
+import cdsapi
+import sys
+import os
+
+def generate_days(year, month):
+    # Get the number of days in the given month
+    num_days = calendar.monthrange(year, month)[1]
+
+    # Generate the list of days as integers
+    days = [day for day in range(1, num_days + 1)]
+    
+    return days
+
+def generate_datarequest(year, monthstr, days):
+    
+    # active download client for climate data service (cds)
+    client = cdsapi.Client()
+    
+    # dataset to download rom cds
+    dataset = "reanalysis-era5-single-levels"
+    # request for cds
+    request = {
+        "product_type": ["reanalysis"],
+        "variable": [
+            "surface_pressure",
+            "mean_surface_downward_long_wave_radiation_flux",
+            "mean_surface_downward_short_wave_radiation_flux",
+            "mean_total_precipitation_rate"
+        ],
+        "year": [str(year)],
+        "month": [monthstr],
+        "day": days,
+        "time": [
+            "00:00", "01:00", "02:00",
+            "03:00", "04:00", "05:00",
+            "06:00", "07:00", "08:00",
+            "09:00", "10:00", "11:00",
+            "12:00", "13:00", "14:00",
+            "15:00", "16:00", "17:00",
+            "18:00", "19:00", "20:00",
+            "21:00", "22:00", "23:00"
+        ],
+        "data_format": "netcdf",
+        "download_format": "unarchived",
+        "area": [74, -42, 20, 69]
+    }
+    # filename of downloaded file
+    target = 'download_era5.zip'
+    
+    # Get the data from cds
+    client.retrieve(dataset, request, target)
+
+if __name__ == "__main__":
+    # Check if the correct number of arguments are provided
+    if len(sys.argv) != 4:
+        print("Usage: python download_ERA5_input.py <year> <month> <output_directory>")
+        sys.exit(1)
+
+    # Get the year and month from command-line arguments
+    year = int(sys.argv[1])
+    month = int(sys.argv[2])
+    dirout = sys.argv[3]
+
+    # Ensure the output directory exists, if not, create it
+    if not os.path.exists(dirout):
+        os.makedirs(dirout)
+
+    # change to output directory
+    os.chdir(dirout)
+
+    # Format the month with a leading zero if needed
+    monthstr = f"{month:02d}"
+
+    # Get the list of days for the request
+    days = generate_days(year, month)
+
+    # do download request
+    generate_datarequest(year, monthstr, days)
diff --git a/mkforcing/download_ERA5_v2.py b/mkforcing/download_ERA5_v2.py
deleted file mode 100755
index fd1ef0d..0000000
--- a/mkforcing/download_ERA5_v2.py
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env python3
-import cdsapi
-
-c = cdsapi.Client()
-
-for year in [2017]:
-        for month in range(1,13,1):
-                if month<10:
-                        month="0"+str(month)
-                file="era5_"+str(year)+"_"+str(month)+".nc"
-                c.retrieve(
-    'reanalysis-era5-single-levels',
-    {
-        'product_type': 'reanalysis',
-        'variable': [
-            'surface_pressure', 'mean_surface_downward_long_wave_radiation_flux', 'mean_surface_downward_short_wave_radiation_flux', 'mean_total_precipitation_rate', 'boundary_layer_height'
-        ],
-        'year': year,
-            'month': month,
-        'day': [
-            '01', '02', '03',
-            '04', '05', '06',
-            '07', '08', '09',
-            '10', '11', '12',
-            '13', '14', '15',
-            '16', '17', '18',
-            '19', '20', '21',
-            '22', '23', '24',
-            '25', '26', '27',
-            '28', '29', '30',
-            '31',
-        ],
-        'time': [
-            '00:00', '01:00', '02:00',
-            '03:00', '04:00', '05:00',
-            '06:00', '07:00', '08:00',
-            '09:00', '10:00', '11:00',
-            '12:00', '13:00', '14:00',
-            '15:00', '16:00', '17:00',
-            '18:00', '19:00', '20:00',
-            '21:00', '22:00', '23:00',
-        ],
-        'area': [
-            74, -48, 20,
-            69,
-        ],
-        'format': 'netcdf',
-    },
-                        file)
-

From f331c9ea05ca70397d4b4296a8ae6fca39162b6f Mon Sep 17 00:00:00 2001
From: Stefan Poll <s.poll@fz-juelich.de>
Date: Sun, 23 Mar 2025 20:16:45 +0100
Subject: [PATCH 02/21] adapt ERA5 meteocloud script and wrapper

---
 mkforcing/extract_ERA5_meteocloud.sh          | 25 +++++++++++++--
 mkforcing/extract_ERA5_meteocloud_wrapper.job | 31 +++++++++++++------
 2 files changed, 45 insertions(+), 11 deletions(-)

diff --git a/mkforcing/extract_ERA5_meteocloud.sh b/mkforcing/extract_ERA5_meteocloud.sh
index 3e3a063..df3d8d7 100755
--- a/mkforcing/extract_ERA5_meteocloud.sh
+++ b/mkforcing/extract_ERA5_meteocloud.sh
@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 # load env -> not all CDO are compiled with "-t ecmwf"
+# module use $OTHERSTAGES
 # ml Stages/2022  NVHPC/22.9  ParaStationMPI/5.5.0-1 CDO/2.0.2
 
 if [ -z "$1" ]
@@ -12,9 +13,29 @@ else
    echo "Calculate the year "$iyear
 fi
 
+if [ -z "$2" ]
+then
+   imonth=07
+   echo "Take the default month "$imonth
+else
+   imonth=$(printf %02d $2)
+   echo "Calculate the month "$imonth
+fi
+
+if [ -z "$3" ]
+then
+   outdir=${iyear}-${imonth}
+   echo "Take the default output directory "$outdir
+else
+   outdir=$3
+   echo "Output directory: "$outdir
+fi
+
+cd ${outdir}
+
 for year in ${iyear}
 do
-for month in 01 #02 03 04 05 06 07 08 09 10 11 12 
+for month in ${imonth}
 do
 days_per_month=$(cal ${month} ${year} | awk 'NF {DAYS = $NF}; END {print DAYS}')
 for my_date in $(seq -w 1 ${days_per_month})
@@ -22,7 +43,7 @@ do
 for time in 00 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 do
 
-cdo sellonlatbox,-48,74,20,74 /p/fastdata/slmet/slmet111/met_data/ecmwf/era5/grib/${year}/${month}/${year}${month}${my_date}${time}_ml.grb cut_domain_${year}${month}${my_date}${time}.grb
+cdo sellonlatbox,-48,74,20,74 /p/data1/slmet/met_data/ecmwf/era5/grib/${year}/${month}/${year}${month}${my_date}${time}_ml.grb cut_domain_${year}${month}${my_date}${time}.grb
 
 cdo sellevel,137 cut_domain_${year}${month}${my_date}${time}.grb lower_level_${year}${month}${my_date}${time}.grb
 #cdo -t ecmwf -f nc4 copy lower_level_${month}${my_date}${time}.grb lower_level_${month}${my_date}${time}.nc
diff --git a/mkforcing/extract_ERA5_meteocloud_wrapper.job b/mkforcing/extract_ERA5_meteocloud_wrapper.job
index f383f62..cee16c0 100644
--- a/mkforcing/extract_ERA5_meteocloud_wrapper.job
+++ b/mkforcing/extract_ERA5_meteocloud_wrapper.job
@@ -6,27 +6,40 @@
 #SBATCH --ntasks-per-node=48
 #SBATCH --output=mpiMPMD-out.%j
 #SBATCH --error=mpiMPMD-err.%j
-#SBATCH --time=02:00:00
+#SBATCH --time=12:00:00
 #SBATCH --partition=devel
 #SBATCH --mail-type=NONE
 #SBATCH --account=slts
 
-
+# load environment
 module purge
+module use $OTHERSTAGES
 ml Stages/2022  NVHPC/22.9  ParaStationMPI/5.5.0-1 CDO/2.0.2
 
 RUNDIR=$(pwd)
 
 cd $RUNDIR
 
-for year in `seq 2015 2020`; do
- echo "process year "$year
- srun --exclusive -n 1 ./extract_ERA5_meteocloud.sh $year &
-done
+# start a counter for background jobs
+running_jobs=0
+
+for year in `seq 2017 2020`; do
+ for month in `seq 1 12`; do
+  echo "process year "$year" month "$month
 
-#srun --exclusive -n 1 ./extract_ERA5_meteocloud.sh 2017 &
-#srun --exclusive -n 1 ./extract_ERA5_meteocloud.sh 2018 &
-#srun --exclusive -n 1 ./extract_ERA5_meteocloud.sh 2019 &
+  # increment the running job counter
+  ((running_jobs++))
 
+  # launch job in the background
+  srun --exclusive -n 1 ./extract_ERA5_meteocloud.sh $year $month &
+
+  # if the max number of parallel tasks is reached, wait for a job to finish
+  if ((running_jobs >= SLURM_NTASKS)); then
+     wait -n  # wait for one job to finish before starting another
+     ((running_jobs--))  # decrement the running job counter
+  fi
+
+ done
+done
 
 wait

From 95c1b66f473053df08d8c296d0deeec369df8724 Mon Sep 17 00:00:00 2001
From: Stefan Poll <s.poll@fz-juelich.de>
Date: Mon, 24 Mar 2025 15:45:07 +0100
Subject: [PATCH 03/21] Rewrite prepare ERA5 script

- adapt script to new CDSAPI
- add argument parser for script
- rename of SHUM to QBOT
---
 mkforcing/prepare_ERA5_input.sh | 131 ++++++++++++++++++++++++++++++++
 mkforcing/prepare_ERA5_v2.sh    |  83 --------------------
 2 files changed, 131 insertions(+), 83 deletions(-)
 create mode 100755 mkforcing/prepare_ERA5_input.sh
 delete mode 100755 mkforcing/prepare_ERA5_v2.sh

diff --git a/mkforcing/prepare_ERA5_input.sh b/mkforcing/prepare_ERA5_input.sh
new file mode 100755
index 0000000..9f5d255
--- /dev/null
+++ b/mkforcing/prepare_ERA5_input.sh
@@ -0,0 +1,131 @@
+#!/usr/bin/env bash
+
+# default values of parameters
+lrmp=true
+lmerge=true
+lclm3=false
+# TSMP2/eclm
+pathdata=./
+wgtcaf=/p/scratch/cslts/poll1/sim/euro-cordex/tsmp2_wfe_eur-11u/dta/rmp_gridwgts/wgtdis_era5caf_to_eur11u-189976.nc
+wgtmeteo=/p/scratch/cslts/poll1/sim/euro-cordex/tsmp2_wfe_eur-11u/dta/rmp_gridwgts/wgtdis_era5meteo_to_eur11u-189976.nc
+griddesfile=/p/scratch/cslts/poll1/sim/euro-cordex/tsmp2_wfe_eur-11u/dta/rmp_gridwgts/griddes_eur-11u_189976.txt
+clm3grid=""
+# TSMP1/clm3.5
+#pathdata=/p/project/detectlulcc/ferro1/eclm_static_file_workflow/mkforcing/
+#wgtcaf=/p/project/detectlulcc/poll1/data/script_remap/wgtdis_era5caf_to_eur11-444x432.nc
+#wgtmeteo=/p/project/detectlulcc/poll1/data/script_remap/wgtdis_era5meteo_to_eur11-444x432.nc
+#griddesfile=/p/largedata2/detectdata/CentralDB/projects/z04/detect_grid_specs/web_pep_tmp/EUR-11_TSMP_FZJ-IBG3_444x432_webPEP_sub.txt
+#clm3grid=/p/scratch/cslts/poll1/sim/EUR-11_ECMWF-ERA5_evaluation_r1i1p1_FZJ-COSMO5-01-CLM3-5-0-ParFlow3-12-0_vEXP/geo/TSMP_EUR-11/static/clm/griddata_CLM_EUR-11_TSMP_FZJ-IBG3_CLMPFLDomain_444x432.nc
+iyear=2017
+imonth=07
+tmpdir=tmpdir
+wrkdir=""
+author="Stefan POLL"
+email="s.poll@fz-juelich.de"
+
+# Function to parse input
+parse_arguments() {
+    for arg in "$@"; do
+        key="${arg%%=*}"
+        value="${arg#*=}"
+
+        case "$key" in
+            lrmp) lrmp="$value" ;;
+            lmerge) lmerge="$value" ;;
+            lclm3) lclm3="$value" ;;
+            pathdata) pathdata="$value" ;;
+            wgtcaf) wgtcaf="$value" ;;
+            wgtmeteo) wgtmeteo="$value" ;;
+            griddesfile) griddesfile="$value" ;;
+            clm3grid) clm3grid="$value" ;;
+            tmpdir) tmpdir="$value" ;;
+	    wrkdir) wrkdir="$value" ;;
+            imonth) imonth="$value" ;;
+            iyear) iyear="$value" ;;
+	    author) author="$value" ;;
+	    email) email="$value" ;;
+            *) echo "Warning: Unknown parameter: $key" ;;
+        esac
+    done
+}
+
+# Call the function to parse the input arguments
+# Users needs to make sure for consistent input
+parse_arguments "$@"
+
+#
+#cd $wrkdir
+#mkdir -pv $tmpdir
+
+#
+for year in ${iyear}
+do
+for month in ${imonth}
+do
+
+  # Go into working directory and create temporary directory
+  if [ -z ${wrkdir} ];then
+    wrkdir=${iyear}-${imonth}
+  fi
+  cd $wrkdir
+  mkdir -pv $tmpdir
+
+  if $lrmp; then
+    unzip ${pathdata}/download_era5.zip
+    cdo remap,${griddesfile},${wgtcaf} data_stream-oper_stepType-instant.nc ${tmpdir}/rmp_era5_${year}_${month}_ins.nc
+    cdo remap,${griddesfile},${wgtcaf} data_stream-oper_stepType-avg.nc ${tmpdir}/rmp_era5_${year}_${month}_avg.nc
+#    cdo remap,${griddesfile},${wgtcaf} ${pathdata}/era5_${year}_${month}.nc ${tmpdir}/rmp_era5_${year}_${month}.nc
+    cdo remap,${griddesfile},${wgtmeteo} ${pathdata}/meteocloud_${year}_${month}.nc ${tmpdir}/rmp_meteocloud_${year}_${month}.nc
+  fi
+
+  if $lmerge; then
+
+    cdo expr,'WIND=sqrt(u^2+v^2)' ${tmpdir}/rmp_meteocloud_${year}_${month}.nc ${tmpdir}/${year}_${month}_temp.nc
+    cdo -f nc4c const,10,${tmpdir}/rmp_era5_${year}_${month}_avg.nc ${tmpdir}/${year}_${month}_const.nc
+    ncpdq -U ${tmpdir}/rmp_era5_${year}_${month}_avg.nc ${tmpdir}/${year}_${month}_temp2.nc
+    ncpdq -U ${tmpdir}/rmp_era5_${year}_${month}_ins.nc ${tmpdir}/${year}_${month}_temp7.nc
+    cdo selvar,t,q ${tmpdir}/rmp_meteocloud_${year}_${month}.nc ${tmpdir}/${year}_${month}_temp3.nc
+
+    cdo merge ${tmpdir}/${year}_${month}_const.nc ${tmpdir}/${year}_${month}_temp3.nc ${tmpdir}/${year}_${month}_temp2.nc ${tmpdir}/${year}_${month}_temp.nc ${tmpdir}/${year}_${month}_temp7.nc ${tmpdir}/${year}_${month}_temp4.nc
+
+    ncks -C -x -v hyai,hyam,hybi,hybm ${tmpdir}/${year}_${month}_temp4.nc ${tmpdir}/${year}_${month}_temp5.nc
+    ncwa -O -a lev ${tmpdir}/${year}_${month}_temp5.nc ${year}-${month}.nc
+
+    ncrename -v sp,PSRF -v avg_sdswrf,FSDS -v avg_sdlwrf,FLDS -v avg_tprate,PRECTmms -v const,ZBOT -v t,TBOT -v q,QBOT ${year}-${month}.nc
+#    ncap2 -O -s 'where(FSDS<0.) FSDS=0' ${year}_${month}.nc
+    ncatted -O -a units,ZBOT,m,c,"m" ${year}-${month}.nc
+#    ncrename -d valid_time,time -v valid_time,time ${year}-${month}.nc
+
+    ncatted -O -h -a author,global,m,c,${author} ${year}-${month}.nc
+    ncatted -O -h -a contact,global,m,c,${email} ${year}-${month}.nc
+
+    rm ${tmpdir}/${year}_${month}_temp*nc ${tmpdir}/${year}_${month}_const.nc
+  fi
+
+  ## adaptation for CLM3.5
+  if $lclm3; then
+
+    mv ${year}-${month}.nc ${year}_${month}_tmp.nc
+
+    ## CLM3.5
+    ncrename -v lon,LONGXY ${year}_${month}_tmp.nc
+    ncrename -v lat,LATIXY ${year}_${month}_tmp.nc
+
+    # 
+    cdo selvar,LONE,LATN,LONW,LATS,AREA,EDGEE,EDGEN,EDGEW,EDGES ${clm3grid} ${tmpdir}/${year}_${month}_temp11.nc
+    cdo -O merge ${tmpdir}/${year}_${month}_temp11.nc ${year}_${month}_tmp.nc ${year}-${month}.nc
+
+    ncrename -d rlon,lon ${year}-${month}.nc
+    ncrename -d rlat,lat ${year}-${month}.nc
+
+    ncatted -O -h -a author,global,m,c,${author} ${year}-${month}.nc
+    ncatted -O -h -a contact,global,m,c,${email} ${year}-${month}.nc
+
+    # 
+    rm ${year}_${month}_tmp.nc ${tmpdir}/${year}_${month}_temp11.nc
+
+  fi
+
+done
+done
+
diff --git a/mkforcing/prepare_ERA5_v2.sh b/mkforcing/prepare_ERA5_v2.sh
deleted file mode 100755
index 07f1173..0000000
--- a/mkforcing/prepare_ERA5_v2.sh
+++ /dev/null
@@ -1,83 +0,0 @@
-#!/usr/bin/env bash
-
-# switches
-lrmp=false #true
-lmerge=true #false
-lclm3=true
-
-# 
-pathdata=/p/project/detectlulcc/ferro1/eclm_static_file_workflow/mkforcing/
-wgtcaf=/p/project/detectlulcc/poll1/data/script_remap/wgtdis_era5caf_to_eur11-444x432.nc
-wgtmeteo=/p/project/detectlulcc/poll1/data/script_remap/wgtdis_era5meteo_to_eur11-444x432.nc
-griddesfile=/p/largedata2/detectdata/projects/Z04/detect_grid_specs/web_pep_tmp/griddes_EUR-11_TSMP_FZJ-IBG3_444x432_webPEP_sub.txt
-clm3grid=/p/scratch/cslts/poll1/sim/EUR-11_ECMWF-ERA5_evaluation_r1i1p1_FZJ-COSMO5-01-CLM3-5-0-ParFlow3-12-0_vEXP/geo/TSMP_EUR-11/static/clm/griddata_CLM_EUR-11_TSMP_FZJ-IBG3_CLMPFLDomain_444x432.nc
-tmpdir=tmp_data
-
-# 
-author="Stefan POLL"
-email="s.poll@fz-juelich.de"
-
-#
-mkdir -pv $tmpdir
-
-#
-for year in 2017
-do
-for month in 01 #02 03 04 05 06 07 08 09 10 11 12 
-do
-
-  if $lrmp; then
-    cdo remap,${griddesfile},${wgtcaf} ${pathdata}/era5_${year}_${month}.nc ${tmpdir}/rmp_era5_${year}_${month}.nc
-    cdo remap,${griddesfile},${wgtmeteo} ${pathdata}/meteocloud_${year}_${month}.nc ${tmpdir}/rmp_meteocloud_${year}_${month}.nc
-  fi
-
-  if $lmerge; then
-
-    cdo expr,'WIND=sqrt(u^2+v^2)' ${tmpdir}/rmp_meteocloud_${year}_${month}.nc ${tmpdir}/${year}_${month}_temp.nc
-    cdo -f nc4c const,10,${tmpdir}/rmp_era5_${year}_${month}.nc ${tmpdir}/${year}_${month}_const.nc
-    ncpdq -U ${tmpdir}/rmp_era5_${year}_${month}.nc ${tmpdir}/${year}_${month}_temp2.nc
-    cdo selvar,t,q ${tmpdir}/rmp_meteocloud_${year}_${month}.nc ${tmpdir}/${year}_${month}_temp3.nc
-
-    cdo merge ${tmpdir}/${year}_${month}_const.nc ${tmpdir}/${year}_${month}_temp3.nc ${tmpdir}/${year}_${month}_temp2.nc ${tmpdir}/${year}_${month}_temp.nc ${tmpdir}/${year}_${month}_temp4.nc
-
-    ncks -C -x -v hyai,hyam,hybi,hybm ${tmpdir}/${year}_${month}_temp4.nc ${tmpdir}/${year}_${month}_temp5.nc
-    ncwa -O -a lev ${tmpdir}/${year}_${month}_temp5.nc ${year}_${month}.nc
-
-    ncrename -v sp,PSRF -v msdwswrf,FSDS -v msdwlwrf,FLDS -v mtpr,PRECTmms -v const,ZBOT -v t,TBOT -v q,SHUM ${year}_${month}.nc
-#    ncap2 -O -s 'where(FSDS<0.) FSDS=0' ${year}_${month}.nc
-    ncatted -O -a units,ZBOT,m,c,"m" ${year}_${month}.nc
-
-    ncatted -O -h -a author,global,m,c,${author} ${year}_${month}.nc
-    ncatted -O -h -a contact,global,m,c,${email} ${year}_${month}.nc
-
-    rm ${tmpdir}/${year}_${month}_temp*nc ${tmpdir}/${year}_${month}_const.nc
-  fi
-
-  ## adaptation for CLM3.5
-  if $lclm3; then
-
-    cp ${year}_${month}.nc ${year}_${month}_tmp.nc
-
-    ## CLM3.5
-    ncrename -v lon,LONGXY ${year}_${month}_tmp.nc
-    ncrename -v lat,LATIXY ${year}_${month}_tmp.nc
-    ncrename -v SHUM,QBOT ${year}_${month}_tmp.nc
-
-    # 
-    cdo selvar,LONE,LATN,LONW,LATS,AREA,EDGEE,EDGEN,EDGEW,EDGES ${clm3grid} ${tmpdir}/${year}_${month}_temp11.nc
-    cdo -O merge ${tmpdir}/${year}_${month}_temp11.nc ${year}_${month}_tmp.nc ${year}-${month}.nc
-
-    ncrename -d rlon,lon ${year}-${month}.nc
-    ncrename -d rlat,lat ${year}-${month}.nc
-
-    ncatted -O -h -a author,global,m,c,${author} ${year}-${month}.nc
-    ncatted -O -h -a contact,global,m,c,${email} ${year}-${month}.nc
-
-    # 
-    rm ${year}_${month}_tmp.nc ${tmpdir}/${year}_${month}_temp11.nc
-
-  fi
-
-done
-done
-

From ba833bbf762767400b1b9898c94870bdab8d037d Mon Sep 17 00:00:00 2001
From: Stefan Poll <s.poll@fz-juelich.de>
Date: Mon, 24 Mar 2025 23:30:44 +0100
Subject: [PATCH 04/21] update README and clean-up

---
 README.md                            | 30 +++++++++++++++-------------
 mkforcing/extract_ERA5_meteocloud.sh |  1 -
 mkforcing/prepare_ERA5_input.sh      |  8 ++++----
 3 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/README.md b/README.md
index 387feff..d0eba63 100644
--- a/README.md
+++ b/README.md
@@ -164,23 +164,25 @@ The created surface and domain file have negative longitudes that CLM5 does not
 
 ## Creation of forcing data from ERA5
 
-A possible source of atmospheric forcing for CLM5 is ERA5.
-The folder `mkforcing/` contains two scripts that assist the ERA5 retrieval.
-- `download_ERA5.py` contains a prepared retrieval for the cdsapi python module.
-By modifying the two loops inside the script it is possible to download ERA5 for any timerange.
-However, the script requires that cdsapi is installed with an user specific key.
-More information about the installation can be found [here](https://cds.climate.copernicus.eu/api-how-to).
-- `prepare_ERA5.sh` prepares ERA5 as an input by changing names and modifying units.
-ERA5 has to be regridded to your resolution before the script can be used.
-
-`download_ERA5_v2.py`, `prepare_ERA5_v2.sh` and `extract_ERA5_meteocloud.sh` provide an alternative pathway. [This issue](https://gitlab.jsc.fz-juelich.de/HPSCTerrSys/tsmp-internal-development-tracking/-/issues/36) provides some details. Basically it is safer to extract the lowermost level of temperature, humidity and wind of ERA5 instead of taking 2m-values. The workflow goes like this:
+A possible source of atmospheric forcing for CLM (eCLM, CLM5, CLM3.5) is ERA5. It is safer to extract the lowermost level of temperature, humidity and wind of ERA5 instead of taking mixed 2m-values and 10m values. [This internal issue](https://gitlab.jsc.fz-juelich.de/HPSCTerrSys/tsmp-internal-development-tracking/-/issues/36) provides some details. The `download_ERA5_input.py` can be adapted to download another set of quantities.
 
+The folder `mkforcing/` contains three scripts that assist the ERA5 retrieval.
+- `download_ERA5_input.py` contains a prepared retrieval for the cdsapi python module. However, the script requires that cdsapi is installed with an user specific key.
+More information about the installation and registration can be found [here](https://cds.climate.copernicus.eu/how-to-api).
+Usage:
+`python download_ERA5_input.py <year> <month> <output_directory>`
+Non-JSC users should adapt the download script to include temperature, specific humidity and horizontal wind speed.
+- `extract_ERA5_meteocloud.sh` prepares ERA5 as an input by changing names and modifying units (JSC users only).
+- `prepare_ERA5_input.sh` prepares ERA5 as an input by remapping the ERA5 data, changing names and modifying units. The script is divided into three parts, which could be handled separately. Remapping, merging the data, and special treatment in case CLM3.5 forces data preparation. If remapping is to be used, the remapping weights for the ERA data as well as the grid definition file of the target domain should be created beforehand. The following commands can be used to create the necessary files:
 ```
-bash extract_ERA5_meteocloud.sh
-python download_ERA5_v2.py
-regridding
-bash prepare_ERA5_v2.sh
+cdo gendis,<eclm_domainfile.nc> <era5caf_yyyy_mm.nc> <wgtdis_era5caf_to_domain.nc>
+cdo gendis,<eclm_domainfile.nc> <era5meteo_yyyy_mm.nc> <wgtdis_era5meteo_to_domain.nc>
+cdo griddes,<eclm_domainfile.nc> > <domain_griddef.txt>
 ```
+Usage:
+`sh prepare_ERA5_input.sh iyear=<year> imonth=<month> wgtcaf=<wgtcaf> wgtmeteo=<wgtmeteo> griddesfile=<griddesfile>`
+More options are available, see script for details.
+
 
 Note: This worfklow is not fully tested.
 
diff --git a/mkforcing/extract_ERA5_meteocloud.sh b/mkforcing/extract_ERA5_meteocloud.sh
index df3d8d7..b463d35 100755
--- a/mkforcing/extract_ERA5_meteocloud.sh
+++ b/mkforcing/extract_ERA5_meteocloud.sh
@@ -46,7 +46,6 @@ do
 cdo sellonlatbox,-48,74,20,74 /p/data1/slmet/met_data/ecmwf/era5/grib/${year}/${month}/${year}${month}${my_date}${time}_ml.grb cut_domain_${year}${month}${my_date}${time}.grb
 
 cdo sellevel,137 cut_domain_${year}${month}${my_date}${time}.grb lower_level_${year}${month}${my_date}${time}.grb
-#cdo -t ecmwf -f nc4 copy lower_level_${month}${my_date}${time}.grb lower_level_${month}${my_date}${time}.nc
 cdo -t ecmwf selname,t,u,v,q lower_level_${year}${month}${my_date}${time}.grb variables_lower_level_${year}${month}${my_date}${time}.grb
 
 done
diff --git a/mkforcing/prepare_ERA5_input.sh b/mkforcing/prepare_ERA5_input.sh
index 9f5d255..31c42f0 100755
--- a/mkforcing/prepare_ERA5_input.sh
+++ b/mkforcing/prepare_ERA5_input.sh
@@ -86,7 +86,8 @@ do
     ncpdq -U ${tmpdir}/rmp_era5_${year}_${month}_ins.nc ${tmpdir}/${year}_${month}_temp7.nc
     cdo selvar,t,q ${tmpdir}/rmp_meteocloud_${year}_${month}.nc ${tmpdir}/${year}_${month}_temp3.nc
 
-    cdo merge ${tmpdir}/${year}_${month}_const.nc ${tmpdir}/${year}_${month}_temp3.nc ${tmpdir}/${year}_${month}_temp2.nc ${tmpdir}/${year}_${month}_temp.nc ${tmpdir}/${year}_${month}_temp7.nc ${tmpdir}/${year}_${month}_temp4.nc
+    cdo merge ${tmpdir}/${year}_${month}_const.nc ${tmpdir}/${year}_${month}_temp3.nc ${tmpdir}/${year}_${month}_temp2.nc &
+              ${tmpdir}/${year}_${month}_temp.nc ${tmpdir}/${year}_${month}_temp7.nc ${tmpdir}/${year}_${month}_temp4.nc
 
     ncks -C -x -v hyai,hyam,hybi,hybm ${tmpdir}/${year}_${month}_temp4.nc ${tmpdir}/${year}_${month}_temp5.nc
     ncwa -O -a lev ${tmpdir}/${year}_${month}_temp5.nc ${year}-${month}.nc
@@ -94,7 +95,6 @@ do
     ncrename -v sp,PSRF -v avg_sdswrf,FSDS -v avg_sdlwrf,FLDS -v avg_tprate,PRECTmms -v const,ZBOT -v t,TBOT -v q,QBOT ${year}-${month}.nc
 #    ncap2 -O -s 'where(FSDS<0.) FSDS=0' ${year}_${month}.nc
     ncatted -O -a units,ZBOT,m,c,"m" ${year}-${month}.nc
-#    ncrename -d valid_time,time -v valid_time,time ${year}-${month}.nc
 
     ncatted -O -h -a author,global,m,c,${author} ${year}-${month}.nc
     ncatted -O -h -a contact,global,m,c,${email} ${year}-${month}.nc
@@ -111,7 +111,7 @@ do
     ncrename -v lon,LONGXY ${year}_${month}_tmp.nc
     ncrename -v lat,LATIXY ${year}_${month}_tmp.nc
 
-    # 
+    #
     cdo selvar,LONE,LATN,LONW,LATS,AREA,EDGEE,EDGEN,EDGEW,EDGES ${clm3grid} ${tmpdir}/${year}_${month}_temp11.nc
     cdo -O merge ${tmpdir}/${year}_${month}_temp11.nc ${year}_${month}_tmp.nc ${year}-${month}.nc
 
@@ -121,7 +121,7 @@ do
     ncatted -O -h -a author,global,m,c,${author} ${year}-${month}.nc
     ncatted -O -h -a contact,global,m,c,${email} ${year}-${month}.nc
 
-    # 
+    #
     rm ${year}_${month}_tmp.nc ${tmpdir}/${year}_${month}_temp11.nc
 
   fi

From 1ec7e5cb85ff156fc189626705b17e9601773850 Mon Sep 17 00:00:00 2001
From: Stefan Poll <s.poll@fz-juelich.de>
Date: Fri, 28 Mar 2025 01:15:55 +0100
Subject: [PATCH 05/21] enhance extract_ERA5_meteocloud.sh

- add argument parser
- add input variables and defaults
- add running_jobs functionality
---
 mkforcing/extract_ERA5_meteocloud.sh | 107 +++++++++++++++++++--------
 1 file changed, 77 insertions(+), 30 deletions(-)

diff --git a/mkforcing/extract_ERA5_meteocloud.sh b/mkforcing/extract_ERA5_meteocloud.sh
index b463d35..ee5900f 100755
--- a/mkforcing/extract_ERA5_meteocloud.sh
+++ b/mkforcing/extract_ERA5_meteocloud.sh
@@ -4,54 +4,101 @@
 # module use $OTHERSTAGES
 # ml Stages/2022  NVHPC/22.9  ParaStationMPI/5.5.0-1 CDO/2.0.2
 
-if [ -z "$1" ]
-then
-   iyear=2017
-   echo "Take the default year "$iyear
-else
-   iyear=$1
-   echo "Calculate the year "$iyear
-fi
+function message(){
+if [ -z "${quiet}" ];then
+  echo "$1"
+fi # quiet
+}
 
-if [ -z "$2" ]
-then
-   imonth=07
-   echo "Take the default month "$imonth
-else
-   imonth=$(printf %02d $2)
-   echo "Calculate the month "$imonth
-fi
+# default values of parameters
+iyear=2017
+imonth=07
+ihour=(00 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20 21 22 23)
+outdir=${iyear}-${imonth}
+runpp=1
+area=(-48 74 20 74)
 
-if [ -z "$3" ]
-then
-   outdir=${iyear}-${imonth}
-   echo "Take the default output directory "$outdir
-else
-   outdir=$3
-   echo "Output directory: "$outdir
-fi
+# Function to parse input
+parse_arguments() {
+    for arg in "$@"; do
+        key="${arg%%=*}"
+        value="${arg#*=}"
+
+        case "$key" in
+            quiet) quiet=y;;
+            iyear) iyear="$value" ;;
+            imonth) imonth="$value" ;;
+	    ihour) ihour="$value" ;;
+            outdir) outdir="$value" ;;
+            runpp) runpp="$value" ;;
+	    area) area="$value" ;;
+            *) echo "Warning: Unknown parameter: $key" ;;
+        esac
+    done
+}
+
+# Call the function to parse the input arguments
+# Users needs to make sure for consistent input
+parse_arguments "$@"
+
+message "=========================="
+message "Year: "$iyear
+message "Month: "$imonth
+message "Hours: "$ihour
+message "Selected area W: "${area[0]}
+message "Selected area E: "${area[1]}
+message "Selected area S: "${area[2]}
+message "Selected area N: "${area[3]}
+message "Output directory: "$outdir
+message "Max running procs: "$runpp
+message "=========================="
 
 cd ${outdir}
 
+# start a counter for background jobs
+running_jobs=0
+
 for year in ${iyear}
 do
 for month in ${imonth}
 do
 days_per_month=$(cal ${month} ${year} | awk 'NF {DAYS = $NF}; END {print DAYS}')
-for my_date in $(seq -w 1 ${days_per_month})
+for day in $(seq -w 1 ${days_per_month})
 do
-for time in 00 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20 21 22 23
+for hour in ${ihour}
 do
 
-cdo sellonlatbox,-48,74,20,74 /p/data1/slmet/met_data/ecmwf/era5/grib/${year}/${month}/${year}${month}${my_date}${time}_ml.grb cut_domain_${year}${month}${my_date}${time}.grb
+# increment the running job counter
+running_jobs=$((running_jobs+1))
 
-cdo sellevel,137 cut_domain_${year}${month}${my_date}${time}.grb lower_level_${year}${month}${my_date}${time}.grb
-cdo -t ecmwf selname,t,u,v,q lower_level_${year}${month}${my_date}${time}.grb variables_lower_level_${year}${month}${my_date}${time}.grb
+message "Process "$year"-"$month"-"$day"-"$hour" prun: "$running_jobs
+
+# select domain area
+cdo sellonlatbox,${area[0]},${area[1]},${area[2]},${area[3]} /p/data1/slmet/met_data/ecmwf/era5/grib/${year}/${month}/${year}${month}${day}${hour}_ml.grb cut_domain_${year}${month}${day}${hour}.grb
+# select lowermost model level
+cdo sellevel,137 cut_domain_${year}${month}${day}${hour}.grb lower_level_${year}${month}${day}${hour}.grb
+# select temperature, horizontal wind speed, humidity
+cdo -t ecmwf selname,t,u,v,q lower_level_${year}${month}${day}${hour}.grb variables_lower_level_${year}${month}${day}${hour}.grb
+
+# if the max number of parallel tasks is reached, wait for a job to finish
+if [[ ${running_jobs} -ge ${runpp} ]]; then
+   wait -n  # wait for one job to finish before starting another
+   running_jobs=$((running_jobs-1))  # decrement the running job counter
+fi
 
 done
 done
+
+wait
+
+# merge hourly files to monthly
 cdo merge  variables_lower_level_${year}*.grb meteocloud_${year}_${month}.grb
-rm variables_lower_level_${year}*.grb cut_domain_${year}* lower_level_${year}*
+# transform from grib to netcdf format
 cdo -t ecmwf -f nc4 copy meteocloud_${year}_${month}.grb meteocloud_${year}_${month}.nc
+
+# clean-up
+rm variables_lower_level_${year}*.grb cut_domain_${year}* lower_level_${year}*
+
 done
 done
+

From 926d4b96bb184afe6e7fff493d60bd05faca0966 Mon Sep 17 00:00:00 2001
From: Stefan Poll <s.poll@fz-juelich.de>
Date: Fri, 28 Mar 2025 15:02:01 +0100
Subject: [PATCH 06/21] add download_ERA5 wrapper

---
 mkforcing/download_ERA5_input.py         |  2 +-
 mkforcing/download_ERA5_input_wrapper.sh | 49 ++++++++++++++++++++++++
 2 files changed, 50 insertions(+), 1 deletion(-)
 create mode 100755 mkforcing/download_ERA5_input_wrapper.sh

diff --git a/mkforcing/download_ERA5_input.py b/mkforcing/download_ERA5_input.py
index 145df2b..524ee1c 100755
--- a/mkforcing/download_ERA5_input.py
+++ b/mkforcing/download_ERA5_input.py
@@ -47,7 +47,7 @@ def generate_datarequest(year, monthstr, days):
         "area": [74, -42, 20, 69]
     }
     # filename of downloaded file
-    target = 'download_era5.zip'
+    target = 'download_era5_'+str(year)+'-'+monthstr+'.zip'
     
     # Get the data from cds
     client.retrieve(dataset, request, target)
diff --git a/mkforcing/download_ERA5_input_wrapper.sh b/mkforcing/download_ERA5_input_wrapper.sh
new file mode 100755
index 0000000..fe2d8ed
--- /dev/null
+++ b/mkforcing/download_ERA5_input_wrapper.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# Before using this script CDSAPI has to be configured (see README)
+# Needs to be executed at LOGIN node as connection to "outside" is required
+
+# load environment
+module load Python
+
+# Settings
+start_date="2017-07" # yyyy-mm
+end_date="2018-08"   # yyyy-mm + 1
+out_dir="cdsapidwn"
+
+# Function to parse input
+parse_arguments() {
+    for arg in "$@"; do
+        key="${arg%%=*}"
+        value="${arg#*=}"
+
+        case "$key" in
+            start_date) start_date="$value" ;;
+            end_date) end_date="$value" ;;
+            out_dir) out_dir="$value" ;;
+            *) echo "Warning: Unknown parameter: $key" ;;
+        esac
+    done
+}
+
+# Call the function to parse the input arguments
+# Users needs to make sure for consistent input
+parse_arguments "$@"
+
+
+# create output directory
+mkdir -p $out_dir
+
+# loop over months
+current_date=$start_date
+while [[ "$current_date" < "$end_date" ]]; do
+    echo "Processing month: $current_date"
+
+    year="${current_date%%-*}"
+    month="${current_date#*-}"
+
+    # start download script with data request
+    python download_ERA5_input.py $year $month $out_dir
+
+    # Increment the month
+    current_date=$(date -I -d "$current_date-01 + 1 month" | cut -d'-' -f1,2)
+done

From 75a17553311f30ce8edc1fa3c8a7d7df829223d8 Mon Sep 17 00:00:00 2001
From: Stefan Poll <s.poll@fz-juelich.de>
Date: Mon, 31 Mar 2025 12:17:54 +0200
Subject: [PATCH 07/21] add year/month info to download_era5

---
 mkforcing/download_ERA5_input.py | 2 +-
 mkforcing/prepare_ERA5_input.sh  | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/mkforcing/download_ERA5_input.py b/mkforcing/download_ERA5_input.py
index 524ee1c..a2943f7 100755
--- a/mkforcing/download_ERA5_input.py
+++ b/mkforcing/download_ERA5_input.py
@@ -47,7 +47,7 @@ def generate_datarequest(year, monthstr, days):
         "area": [74, -42, 20, 69]
     }
     # filename of downloaded file
-    target = 'download_era5_'+str(year)+'-'+monthstr+'.zip'
+    target = 'download_era5_'+str(year)+'_'+monthstr+'.zip'
     
     # Get the data from cds
     client.retrieve(dataset, request, target)
diff --git a/mkforcing/prepare_ERA5_input.sh b/mkforcing/prepare_ERA5_input.sh
index 31c42f0..3e27657 100755
--- a/mkforcing/prepare_ERA5_input.sh
+++ b/mkforcing/prepare_ERA5_input.sh
@@ -71,7 +71,7 @@ do
   mkdir -pv $tmpdir
 
   if $lrmp; then
-    unzip ${pathdata}/download_era5.zip
+    /usr/bin/unzip ${pathdata}/download_era5_${year}_${month}.zip
     cdo remap,${griddesfile},${wgtcaf} data_stream-oper_stepType-instant.nc ${tmpdir}/rmp_era5_${year}_${month}_ins.nc
     cdo remap,${griddesfile},${wgtcaf} data_stream-oper_stepType-avg.nc ${tmpdir}/rmp_era5_${year}_${month}_avg.nc
 #    cdo remap,${griddesfile},${wgtcaf} ${pathdata}/era5_${year}_${month}.nc ${tmpdir}/rmp_era5_${year}_${month}.nc
@@ -86,7 +86,7 @@ do
     ncpdq -U ${tmpdir}/rmp_era5_${year}_${month}_ins.nc ${tmpdir}/${year}_${month}_temp7.nc
     cdo selvar,t,q ${tmpdir}/rmp_meteocloud_${year}_${month}.nc ${tmpdir}/${year}_${month}_temp3.nc
 
-    cdo merge ${tmpdir}/${year}_${month}_const.nc ${tmpdir}/${year}_${month}_temp3.nc ${tmpdir}/${year}_${month}_temp2.nc &
+    cdo merge ${tmpdir}/${year}_${month}_const.nc ${tmpdir}/${year}_${month}_temp3.nc ${tmpdir}/${year}_${month}_temp2.nc \
               ${tmpdir}/${year}_${month}_temp.nc ${tmpdir}/${year}_${month}_temp7.nc ${tmpdir}/${year}_${month}_temp4.nc
 
     ncks -C -x -v hyai,hyam,hybi,hybm ${tmpdir}/${year}_${month}_temp4.nc ${tmpdir}/${year}_${month}_temp5.nc

From 05cca5c0ebc47fbf5ccbaa9e998e140a32035d72 Mon Sep 17 00:00:00 2001
From: Stefan Poll <s.poll@fz-juelich.de>
Date: Tue, 1 Apr 2025 17:53:44 +0200
Subject: [PATCH 08/21] replace unzip with python command

 replace unzip with python as unzip is not available at JSC compute nodes.
---
 mkforcing/prepare_ERA5_input.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/mkforcing/prepare_ERA5_input.sh b/mkforcing/prepare_ERA5_input.sh
index 3e27657..89d8fa9 100755
--- a/mkforcing/prepare_ERA5_input.sh
+++ b/mkforcing/prepare_ERA5_input.sh
@@ -71,9 +71,9 @@ do
   mkdir -pv $tmpdir
 
   if $lrmp; then
-    /usr/bin/unzip ${pathdata}/download_era5_${year}_${month}.zip
-    cdo remap,${griddesfile},${wgtcaf} data_stream-oper_stepType-instant.nc ${tmpdir}/rmp_era5_${year}_${month}_ins.nc
-    cdo remap,${griddesfile},${wgtcaf} data_stream-oper_stepType-avg.nc ${tmpdir}/rmp_era5_${year}_${month}_avg.nc
+    python -m zipfile -e ${pathdata}/download_era5_${year}_${month}.zip ${tmpdir}
+    cdo remap,${griddesfile},${wgtcaf} ${tmpdir}/data_stream-oper_stepType-instant.nc ${tmpdir}/rmp_era5_${year}_${month}_ins.nc
+    cdo remap,${griddesfile},${wgtcaf} ${tmpdir}/data_stream-oper_stepType-avg.nc ${tmpdir}/rmp_era5_${year}_${month}_avg.nc
 #    cdo remap,${griddesfile},${wgtcaf} ${pathdata}/era5_${year}_${month}.nc ${tmpdir}/rmp_era5_${year}_${month}.nc
     cdo remap,${griddesfile},${wgtmeteo} ${pathdata}/meteocloud_${year}_${month}.nc ${tmpdir}/rmp_meteocloud_${year}_${month}.nc
   fi

From a737522da5e3f44cc3eb53c8b501b255574e07fe Mon Sep 17 00:00:00 2001
From: Stefan Poll <s.poll@fz-juelich.de>
Date: Mon, 7 Apr 2025 11:59:21 +0200
Subject: [PATCH 09/21] fix hr loop in extract_ERA5_meteocloud.sh

---
 mkforcing/extract_ERA5_meteocloud.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mkforcing/extract_ERA5_meteocloud.sh b/mkforcing/extract_ERA5_meteocloud.sh
index ee5900f..6fce42d 100755
--- a/mkforcing/extract_ERA5_meteocloud.sh
+++ b/mkforcing/extract_ERA5_meteocloud.sh
@@ -65,7 +65,7 @@ do
 days_per_month=$(cal ${month} ${year} | awk 'NF {DAYS = $NF}; END {print DAYS}')
 for day in $(seq -w 1 ${days_per_month})
 do
-for hour in ${ihour}
+for hour in "${ihour[@]}"
 do
 
 # increment the running job counter

From 02dcd7f96d2b711f4e15aa17b3bcdd8964f35645 Mon Sep 17 00:00:00 2001
From: Stefan Poll <s.poll@fz-juelich.de>
Date: Tue, 8 Apr 2025 16:50:22 +0200
Subject: [PATCH 10/21] use ncks for global attributes

---
 mkforcing/prepare_ERA5_input.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/mkforcing/prepare_ERA5_input.sh b/mkforcing/prepare_ERA5_input.sh
index 89d8fa9..e0e954b 100755
--- a/mkforcing/prepare_ERA5_input.sh
+++ b/mkforcing/prepare_ERA5_input.sh
@@ -96,8 +96,8 @@ do
 #    ncap2 -O -s 'where(FSDS<0.) FSDS=0' ${year}_${month}.nc
     ncatted -O -a units,ZBOT,m,c,"m" ${year}-${month}.nc
 
-    ncatted -O -h -a author,global,m,c,${author} ${year}-${month}.nc
-    ncatted -O -h -a contact,global,m,c,${email} ${year}-${month}.nc
+    ncks -O -h --glb author="${author}" ${year}-${month}.nc
+    ncks -O -h --glb contact="${email}" ${year}-${month}.nc
 
     rm ${tmpdir}/${year}_${month}_temp*nc ${tmpdir}/${year}_${month}_const.nc
   fi
@@ -118,8 +118,8 @@ do
     ncrename -d rlon,lon ${year}-${month}.nc
     ncrename -d rlat,lat ${year}-${month}.nc
 
-    ncatted -O -h -a author,global,m,c,${author} ${year}-${month}.nc
-    ncatted -O -h -a contact,global,m,c,${email} ${year}-${month}.nc
+    ncks -O -h --glb author="${author}" ${year}-${month}.nc
+    ncks -O -h --glb contact="${email}" ${year}-${month}.nc
 
     #
     rm ${year}_${month}_tmp.nc ${tmpdir}/${year}_${month}_temp11.nc

From 858634df52d9a224099b622965934674bdd248cb Mon Sep 17 00:00:00 2001
From: Stefan Poll <s.poll@fz-juelich.de>
Date: Tue, 8 Apr 2025 16:58:28 +0200
Subject: [PATCH 11/21] add OpenMP threads to CDO remap in
 prepare_ERA5_input.sh

---
 mkforcing/prepare_ERA5_input.sh | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/mkforcing/prepare_ERA5_input.sh b/mkforcing/prepare_ERA5_input.sh
index e0e954b..5a2d579 100755
--- a/mkforcing/prepare_ERA5_input.sh
+++ b/mkforcing/prepare_ERA5_input.sh
@@ -4,6 +4,7 @@
 lrmp=true
 lmerge=true
 lclm3=false
+ompthd=1
 # TSMP2/eclm
 pathdata=./
 wgtcaf=/p/scratch/cslts/poll1/sim/euro-cordex/tsmp2_wfe_eur-11u/dta/rmp_gridwgts/wgtdis_era5caf_to_eur11u-189976.nc
@@ -33,6 +34,7 @@ parse_arguments() {
             lrmp) lrmp="$value" ;;
             lmerge) lmerge="$value" ;;
             lclm3) lclm3="$value" ;;
+            ompthd) ompthd="$value" ;;
             pathdata) pathdata="$value" ;;
             wgtcaf) wgtcaf="$value" ;;
             wgtmeteo) wgtmeteo="$value" ;;
@@ -72,15 +74,14 @@ do
 
   if $lrmp; then
     python -m zipfile -e ${pathdata}/download_era5_${year}_${month}.zip ${tmpdir}
-    cdo remap,${griddesfile},${wgtcaf} ${tmpdir}/data_stream-oper_stepType-instant.nc ${tmpdir}/rmp_era5_${year}_${month}_ins.nc
-    cdo remap,${griddesfile},${wgtcaf} ${tmpdir}/data_stream-oper_stepType-avg.nc ${tmpdir}/rmp_era5_${year}_${month}_avg.nc
-#    cdo remap,${griddesfile},${wgtcaf} ${pathdata}/era5_${year}_${month}.nc ${tmpdir}/rmp_era5_${year}_${month}.nc
-    cdo remap,${griddesfile},${wgtmeteo} ${pathdata}/meteocloud_${year}_${month}.nc ${tmpdir}/rmp_meteocloud_${year}_${month}.nc
+    cdo -P ${ompthd} remap,${griddesfile},${wgtcaf} ${tmpdir}/data_stream-oper_stepType-instant.nc ${tmpdir}/rmp_era5_${year}_${month}_ins.nc
+    cdo -P ${ompthd} remap,${griddesfile},${wgtcaf} ${tmpdir}/data_stream-oper_stepType-avg.nc ${tmpdir}/rmp_era5_${year}_${month}_avg.nc
+    cdo -P ${ompthd} remap,${griddesfile},${wgtmeteo} ${pathdata}/meteocloud_${year}_${month}.nc ${tmpdir}/rmp_meteocloud_${year}_${month}.nc
   fi
 
   if $lmerge; then
 
-    cdo expr,'WIND=sqrt(u^2+v^2)' ${tmpdir}/rmp_meteocloud_${year}_${month}.nc ${tmpdir}/${year}_${month}_temp.nc
+    cdo -P ${ompthd} expr,'WIND=sqrt(u^2+v^2)' ${tmpdir}/rmp_meteocloud_${year}_${month}.nc ${tmpdir}/${year}_${month}_temp.nc
     cdo -f nc4c const,10,${tmpdir}/rmp_era5_${year}_${month}_avg.nc ${tmpdir}/${year}_${month}_const.nc
     ncpdq -U ${tmpdir}/rmp_era5_${year}_${month}_avg.nc ${tmpdir}/${year}_${month}_temp2.nc
     ncpdq -U ${tmpdir}/rmp_era5_${year}_${month}_ins.nc ${tmpdir}/${year}_${month}_temp7.nc

From 898b5398e2a7d713e7b10e0ee4fcc3dd38505c3f Mon Sep 17 00:00:00 2001
From: Marco van Hulten <Marco.van.Hulten@uni-bonn.de>
Date: Wed, 9 Apr 2025 16:57:11 +0200
Subject: [PATCH 12/21] Cleaned up scripts

- Use python3 in hashbang for cross-platform and rely on it
- Exit on error
- No tab characters
---
 mkforcing/download_ERA5_input.py         | 2 +-
 mkforcing/download_ERA5_input_wrapper.sh | 3 ++-
 mkforcing/extract_ERA5_meteocloud.sh     | 5 +++--
 mkforcing/prepare_ERA5_input.sh          | 7 ++++---
 4 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/mkforcing/download_ERA5_input.py b/mkforcing/download_ERA5_input.py
index a2943f7..5e89edc 100755
--- a/mkforcing/download_ERA5_input.py
+++ b/mkforcing/download_ERA5_input.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 import calendar
 import cdsapi
 import sys
diff --git a/mkforcing/download_ERA5_input_wrapper.sh b/mkforcing/download_ERA5_input_wrapper.sh
index fe2d8ed..aec7e3b 100755
--- a/mkforcing/download_ERA5_input_wrapper.sh
+++ b/mkforcing/download_ERA5_input_wrapper.sh
@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 # Before using this script CDSAPI has to be configured (see README)
 # Needs to be executed at LOGIN node as connection to "outside" is required
+set -eo pipefail
 
 # load environment
 module load Python
@@ -42,7 +43,7 @@ while [[ "$current_date" < "$end_date" ]]; do
     month="${current_date#*-}"
 
     # start download script with data request
-    python download_ERA5_input.py $year $month $out_dir
+    ./download_ERA5_input.py $year $month $out_dir
 
     # Increment the month
     current_date=$(date -I -d "$current_date-01 + 1 month" | cut -d'-' -f1,2)
diff --git a/mkforcing/extract_ERA5_meteocloud.sh b/mkforcing/extract_ERA5_meteocloud.sh
index 6fce42d..71a4130 100755
--- a/mkforcing/extract_ERA5_meteocloud.sh
+++ b/mkforcing/extract_ERA5_meteocloud.sh
@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+set -eo pipefail
 
 # load env -> not all CDO are compiled with "-t ecmwf"
 # module use $OTHERSTAGES
@@ -28,10 +29,10 @@ parse_arguments() {
             quiet) quiet=y;;
             iyear) iyear="$value" ;;
             imonth) imonth="$value" ;;
-	    ihour) ihour="$value" ;;
+            ihour) ihour="$value" ;;
             outdir) outdir="$value" ;;
             runpp) runpp="$value" ;;
-	    area) area="$value" ;;
+            area) area="$value" ;;
             *) echo "Warning: Unknown parameter: $key" ;;
         esac
     done
diff --git a/mkforcing/prepare_ERA5_input.sh b/mkforcing/prepare_ERA5_input.sh
index 5a2d579..c98506b 100755
--- a/mkforcing/prepare_ERA5_input.sh
+++ b/mkforcing/prepare_ERA5_input.sh
@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+set -eo pipefail
 
 # default values of parameters
 lrmp=true
@@ -41,11 +42,11 @@ parse_arguments() {
             griddesfile) griddesfile="$value" ;;
             clm3grid) clm3grid="$value" ;;
             tmpdir) tmpdir="$value" ;;
-	    wrkdir) wrkdir="$value" ;;
+            wrkdir) wrkdir="$value" ;;
             imonth) imonth="$value" ;;
             iyear) iyear="$value" ;;
-	    author) author="$value" ;;
-	    email) email="$value" ;;
+            author) author="$value" ;;
+            email) email="$value" ;;
             *) echo "Warning: Unknown parameter: $key" ;;
         esac
     done

From 4ee3f8d156f1754a75a7f4a967b8bc621e02e5ad Mon Sep 17 00:00:00 2001
From: Marco van Hulten <Marco.van.Hulten@uni-bonn.de>
Date: Wed, 9 Apr 2025 17:02:24 +0200
Subject: [PATCH 13/21] Linked to information and clarified

ATM, the link https://cds.climate.copernicus.eu/how-to-api is deadish.
Referring now also to ECMWF GitHub README.
---
 README.md | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index d0eba63..0d80ae5 100644
--- a/README.md
+++ b/README.md
@@ -167,8 +167,10 @@ The created surface and domain file have negative longitudes that CLM5 does not
 A possible source of atmospheric forcing for CLM (eCLM, CLM5, CLM3.5) is ERA5. It is safer to extract the lowermost level of temperature, humidity and wind of ERA5 instead of taking mixed 2m-values and 10m values. [This internal issue](https://gitlab.jsc.fz-juelich.de/HPSCTerrSys/tsmp-internal-development-tracking/-/issues/36) provides some details. The `download_ERA5_input.py` can be adapted to download another set of quantities.
 
 The folder `mkforcing/` contains three scripts that assist the ERA5 retrieval.
-- `download_ERA5_input.py` contains a prepared retrieval for the cdsapi python module. However, the script requires that cdsapi is installed with an user specific key.
-More information about the installation and registration can be found [here](https://cds.climate.copernicus.eu/how-to-api).
+
+- `download_ERA5_input.py` contains a prepared retrieval for the cdsapi python module.
+The script requires that cdsapi is installed with a user specific key (API access token).
+More information about the installation and registration can be found [here](https://cds.climate.copernicus.eu/how-to-api) and where to put your access token [here](https://github.com/ecmwf/cdsapi?tab=readme-ov-file#install).
 Usage:
 `python download_ERA5_input.py <year> <month> <output_directory>`
 Non-JSC users should adapt the download script to include temperature, specific humidity and horizontal wind speed.

From 62961e7fa2fb4e04d0d23bf9a36855115d3fcbc7 Mon Sep 17 00:00:00 2001
From: Marco van Hulten <Marco.van.Hulten@uni-bonn.de>
Date: Thu, 10 Apr 2025 09:53:41 +0200
Subject: [PATCH 14/21] Corrected info on access to ERA5 data

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 0d80ae5..eece2d5 100644
--- a/README.md
+++ b/README.md
@@ -170,7 +170,7 @@ The folder `mkforcing/` contains three scripts that assist the ERA5 retrieval.
 
 - `download_ERA5_input.py` contains a prepared retrieval for the cdsapi python module.
 The script requires that cdsapi is installed with a user specific key (API access token).
-More information about the installation and registration can be found [here](https://cds.climate.copernicus.eu/how-to-api) and where to put your access token [here](https://github.com/ecmwf/cdsapi?tab=readme-ov-file#install).
+More information about the installation and access can be found [here](https://cds.climate.copernicus.eu/how-to-api) or alternatively [here](https://github.com/ecmwf/cdsapi?tab=readme-ov-file#install).
 Usage:
 `python download_ERA5_input.py <year> <month> <output_directory>`
 Non-JSC users should adapt the download script to include temperature, specific humidity and horizontal wind speed.

From 7fd7b35a9392ed6157f75a4b1e2340e14b28b99a Mon Sep 17 00:00:00 2001
From: Johannes Keller <jo.keller@fz-juelich.de>
Date: Thu, 18 Sep 2025 16:40:48 +0200
Subject: [PATCH 15/21] README updates for `mkforcing/`

---
 README.md | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index eece2d5..22f9518 100644
--- a/README.md
+++ b/README.md
@@ -168,23 +168,39 @@ A possible source of atmospheric forcing for CLM (eCLM, CLM5, CLM3.5) is ERA5. I
 
 The folder `mkforcing/` contains three scripts that assist the ERA5 retrieval.
 
-- `download_ERA5_input.py` contains a prepared retrieval for the cdsapi python module.
+Note: This worfklow is not fully tested.
+
+### Download of ERA5 data
+
+`download_ERA5_input.py` contains a prepared retrieval for the cdsapi python module.
 The script requires that cdsapi is installed with a user specific key (API access token).
+
 More information about the installation and access can be found [here](https://cds.climate.copernicus.eu/how-to-api) or alternatively [here](https://github.com/ecmwf/cdsapi?tab=readme-ov-file#install).
+
 Usage:
+Either directly:
 `python download_ERA5_input.py <year> <month> <output_directory>`
+Or using the wrapper script:
+`./download_ERA5_input_wrapper.sh`
+after changing dates and output directory in the `Settings` section inside this wrapper script.
+
 Non-JSC users should adapt the download script to include temperature, specific humidity and horizontal wind speed.
-- `extract_ERA5_meteocloud.sh` prepares ERA5 as an input by changing names and modifying units (JSC users only).
-- `prepare_ERA5_input.sh` prepares ERA5 as an input by remapping the ERA5 data, changing names and modifying units. The script is divided into three parts, which could be handled separately. Remapping, merging the data, and special treatment in case CLM3.5 forces data preparation. If remapping is to be used, the remapping weights for the ERA data as well as the grid definition file of the target domain should be created beforehand. The following commands can be used to create the necessary files:
+
+### Preparation of ERA5 data I: Names and units
+`extract_ERA5_meteocloud.sh` prepares ERA5 as an input by changing names and modifying units (JSC users only).
+
+### Preparation of ERA5 data II: Remapping
+`prepare_ERA5_input.sh` prepares ERA5 as an input by remapping the ERA5 data, changing names and modifying units. The script is divided into three parts, which could be handled separately. Remapping, merging the data, and special treatment in case CLM3.5 forces data preparation. 
+
+If remapping is to be used, the remapping weights for the ERA data as well as the grid definition file of the target domain should be created beforehand. The following commands can be used to create the necessary files:
 ```
 cdo gendis,<eclm_domainfile.nc> <era5caf_yyyy_mm.nc> <wgtdis_era5caf_to_domain.nc>
 cdo gendis,<eclm_domainfile.nc> <era5meteo_yyyy_mm.nc> <wgtdis_era5meteo_to_domain.nc>
 cdo griddes,<eclm_domainfile.nc> > <domain_griddef.txt>
 ```
+
 Usage:
 `sh prepare_ERA5_input.sh iyear=<year> imonth=<month> wgtcaf=<wgtcaf> wgtmeteo=<wgtmeteo> griddesfile=<griddesfile>`
 More options are available, see script for details.
 
 
-Note: This worfklow is not fully tested.
-

From fcc3bea6bd0c0e7e4da4bb3189cbcc2e3da2b27d Mon Sep 17 00:00:00 2001
From: Johannes Keller <jo.keller@fz-juelich.de>
Date: Thu, 18 Sep 2025 23:10:09 +0200
Subject: [PATCH 16/21] `devel` does not allow for 12h-job

---
 mkforcing/extract_ERA5_meteocloud_wrapper.job | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mkforcing/extract_ERA5_meteocloud_wrapper.job b/mkforcing/extract_ERA5_meteocloud_wrapper.job
index cee16c0..c9d4266 100644
--- a/mkforcing/extract_ERA5_meteocloud_wrapper.job
+++ b/mkforcing/extract_ERA5_meteocloud_wrapper.job
@@ -7,7 +7,7 @@
 #SBATCH --output=mpiMPMD-out.%j
 #SBATCH --error=mpiMPMD-err.%j
 #SBATCH --time=12:00:00
-#SBATCH --partition=devel
+#SBATCH --partition=batch
 #SBATCH --mail-type=NONE
 #SBATCH --account=slts
 

From dcb4c22a12e00ab98ed61778ad0fd18db90967f2 Mon Sep 17 00:00:00 2001
From: Johannes Keller <jo.keller@fz-juelich.de>
Date: Thu, 18 Sep 2025 23:10:37 +0200
Subject: [PATCH 17/21] `seq -w`: month string with width 2 (e.g. 03)

---
 mkforcing/extract_ERA5_meteocloud_wrapper.job | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mkforcing/extract_ERA5_meteocloud_wrapper.job b/mkforcing/extract_ERA5_meteocloud_wrapper.job
index c9d4266..aabfe43 100644
--- a/mkforcing/extract_ERA5_meteocloud_wrapper.job
+++ b/mkforcing/extract_ERA5_meteocloud_wrapper.job
@@ -24,7 +24,7 @@ cd $RUNDIR
 running_jobs=0
 
 for year in `seq 2017 2020`; do
- for month in `seq 1 12`; do
+ for month in `seq -w 1 12`; do
   echo "process year "$year" month "$month
 
   # increment the running job counter

From 46647a5bfd7d761e1023a8745916dc6b862c04a9 Mon Sep 17 00:00:00 2001
From: Johannes Keller <jo.keller@fz-juelich.de>
Date: Thu, 18 Sep 2025 23:11:07 +0200
Subject: [PATCH 18/21] make directory and pass inputs to script

---
 mkforcing/extract_ERA5_meteocloud_wrapper.job | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/mkforcing/extract_ERA5_meteocloud_wrapper.job b/mkforcing/extract_ERA5_meteocloud_wrapper.job
index aabfe43..daae683 100644
--- a/mkforcing/extract_ERA5_meteocloud_wrapper.job
+++ b/mkforcing/extract_ERA5_meteocloud_wrapper.job
@@ -27,11 +27,15 @@ for year in `seq 2017 2020`; do
  for month in `seq -w 1 12`; do
   echo "process year "$year" month "$month
 
+  # make default outdir
+  mkdir $year-$month
+  outdir=$year-$month
+
   # increment the running job counter
   ((running_jobs++))
 
   # launch job in the background
-  srun --exclusive -n 1 ./extract_ERA5_meteocloud.sh $year $month &
+  srun --exclusive -n 1 ./extract_ERA5_meteocloud.sh iyear=$year imonth=$month outdir=$outdir &
 
   # if the max number of parallel tasks is reached, wait for a job to finish
   if ((running_jobs >= SLURM_NTASKS)); then

From aee694e1733dd82f1bb5c50894175f21bb0d4c9a Mon Sep 17 00:00:00 2001
From: Johannes Keller <jo.keller@fz-juelich.de>
Date: Thu, 18 Sep 2025 23:11:22 +0200
Subject: [PATCH 19/21] remove option `e` from `set` command

otherwise the script stopped at `wait` command in first iteration
---
 mkforcing/extract_ERA5_meteocloud.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mkforcing/extract_ERA5_meteocloud.sh b/mkforcing/extract_ERA5_meteocloud.sh
index 71a4130..ffcdf9f 100755
--- a/mkforcing/extract_ERA5_meteocloud.sh
+++ b/mkforcing/extract_ERA5_meteocloud.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -eo pipefail
+set -o pipefail
 
 # load env -> not all CDO are compiled with "-t ecmwf"
 # module use $OTHERSTAGES

From e6ede36e25bf1cecf9f727759de67b91cf4f6238 Mon Sep 17 00:00:00 2001
From: Johannes Keller <jo.keller@fz-juelich.de>
Date: Thu, 18 Sep 2025 23:11:48 +0200
Subject: [PATCH 20/21] README: usage of extract_ERA5_meteocloud wrapper script

---
 README.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/README.md b/README.md
index 22f9518..4075f72 100644
--- a/README.md
+++ b/README.md
@@ -189,6 +189,11 @@ Non-JSC users should adapt the download script to include temperature, specific
 ### Preparation of ERA5 data I: Names and units
 `extract_ERA5_meteocloud.sh` prepares ERA5 as an input by changing names and modifying units (JSC users only).
 
+Usage:
+Running the wrapper job
+`sbatch extract_ERA5_meteocloud_wrapper.job`
+after adapting `year` and `month` loops according to needed dates.
+
 ### Preparation of ERA5 data II: Remapping
 `prepare_ERA5_input.sh` prepares ERA5 as an input by remapping the ERA5 data, changing names and modifying units. The script is divided into three parts, which could be handled separately. Remapping, merging the data, and special treatment in case CLM3.5 forces data preparation. 
 

From 4a3a8e7ae8c15b431a247930e2de242db6bce301 Mon Sep 17 00:00:00 2001
From: Johannes Keller <jo.keller@fz-juelich.de>
Date: Thu, 18 Sep 2025 23:27:30 +0200
Subject: [PATCH 21/21] README: fix, no command for `cdo griddes` command

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 4075f72..24ef08e 100644
--- a/README.md
+++ b/README.md
@@ -201,7 +201,7 @@ If remapping is to be used, the remapping weights for the ERA data as well as th
 ```
 cdo gendis,<eclm_domainfile.nc> <era5caf_yyyy_mm.nc> <wgtdis_era5caf_to_domain.nc>
 cdo gendis,<eclm_domainfile.nc> <era5meteo_yyyy_mm.nc> <wgtdis_era5meteo_to_domain.nc>
-cdo griddes,<eclm_domainfile.nc> > <domain_griddef.txt>
+cdo griddes <eclm_domainfile.nc> > <domain_griddef.txt>
 ```
 
 Usage: