Extension of ESA CCI OZONE CMORizer

environment.yml

@@ -62,6 +62,7 @@ dependencies:
   - seaborn
   - seawater
   - shapely >=2.0.2
+  - webdavclient
   - xarray >=0.12.0
   - xesmf >=0.7.1
   - xgboost >1.6.1  # github.com/ESMValGroup/ESMValTool/issues/2779

environment_osx.yml

@@ -62,6 +62,7 @@ dependencies:
   - seaborn
   - seawater
   - shapely >=2.0.2
+  - webdavclient
   - xarray >=0.12.0
   - xesmf >=0.7.1
   - xgboost >1.6.1  # github.com/ESMValGroup/ESMValTool/issues/2779

esmvaltool/cmorizers/data/cmor_config/ESACCI-OZONE.yml

@@ -5,16 +5,40 @@ attributes:
   tier: 2
   modeling_realm: sat
   project_id: OBS6
-  source: "https://cds.climate.copernicus.eu/datasets/satellite-ozone-v1"
+  source: "https://cds.climate.copernicus.eu/datasets/satellite-ozone-v1 and https://webdav.aeronomie.be/guest/o3_cci/webdata/Nadir_Profiles/L3/IASI_MG_FORLI/"
   reference: "esacci-ozone"
   comment: ""
 
 variables:
-  toz:
+  # Note: Do not change variable names, these are used to handle the different
+  # datasets in the formatting script esacci_ozone.py.
+  toz_gto_ecv:
+    version: L3-GTO-ECV
     mip: AERmon
+    output: toz
     raw: total_ozone_column
-    filename: C3S-L3_OZONE-O3_PRODUCTS-MERGED_UV-MERGED-ALG-MONTHLY-v2000.nc #The filename needs also "YYYYMM-" as prefix added during the cmorization
-  o3:
+    filename: "{year}{month}-C3S-L3_OZONE-O3_PRODUCTS-MERGED_UV-MERGED-ALG-MONTHLY-v2000.nc"
+  o3_sage_omps:
+    version: L3-SAGE-OMPS
     mip: AERmon
+    output: o3
     raw: merged_ozone_concentration
-    filename: C3S-L3_OZONE-O3_PRODUCTS-CONC_MZM-MERGED-ALG-MONTHLY-v0008.nc #The filename needs also "YYYYMM-" as prefix added during the cmorization
+    filename: "{year}{month}-C3S-L3_OZONE-O3_PRODUCTS-CONC_MZM-MERGED-ALG-MONTHLY-v0008.nc"
+  o3_megridop:
+    version: L3-MEGRIDOP
+    mip: AERmon
+    output: o3
+    raw: merged_ozone_concentration
+    filename: "{year}{month}-C3S-L3_OZONE-O3_PRODUCTS-CONC_LLG-MERGED-ALG-MONTHLY-v0005.nc"
+  o3_iasi:
+    version: L3-IASI
+    mip: AERmon
+    output: o3
+    raw: O3_partial_column_profile
+    filename: "IASI_FORLI_O3_MERGED_{year}{month}*_V1.0.nc"
+  toz_iasi:
+    version: L3-IASI
+    mip: AERmon
+    output: toz
+    raw: O3_total_column
+    filename: "IASI_FORLI_O3_MERGED_{year}{month}*_V1.0.nc"

esmvaltool/cmorizers/data/datasets.yml

@@ -522,12 +522,13 @@ datasets:
   ESACCI-OZONE:
     tier: 2
     source: https://cds.climate.copernicus.eu/datasets/satellite-ozone-v1
-    last_access: 2025-02-11
+    last_access: 2025-05-15
     info: |
       Download the data from:
             GTO-ECV total column (variable toz)
             Select the following from the CDS:
             https://cds.climate.copernicus.eu/datasets/satellite-ozone-v1?tab=download
+            Put all files under a single directory (no subdirectories with years).
 
             Processing Level = "Level 3"
             Variable = "Atm. mole content of ozone"
@@ -540,16 +541,36 @@ datasets:
             SAGE-CCI-OMPS (variable o3)
             Select the following options from the same link:
             https://cds.climate.copernicus.eu/datasets/satellite-ozone-v1?tab=download
+            Put all files under a single directory (no subdirectories with years).
 
             Processing Level = "Level 3"
             Variable = "Mole concentration of ozone in air"
-            Vertical aggregation = " Vertical profiles from limb sensors"
+            Vertical aggregation = "Vertical profiles from limb sensors"
             Sensor = " CMZM (Monthly zonal mean merged concentration product from limb
             sensors ACE, GOMOS, MIPAS, OMPS, OSIRIS, SAGE-2 and SCIAMACHY)"
             Year = select all (1984-2022)
             Month = select all (1-12)
             Version = "v0008"
-      Put all files under a single directory (no subdirectories with years).
+
+            MEGRIDOP (variable o3)
+            Select the following options from the same link:
+            https://cds.climate.copernicus.eu/datasets/satellite-ozone-v1?tab=download
+            Put all files under a single directory (no subdirectories with years).
+
+            Processing Level = "Level 3"
+            Variable = "Mole concentration of ozone in air"
+            Vertical aggregation = "Vertical profiles from limb sensors"
+            Sensor = "CLLG (Latitude-longitude gridded merged concentration product from limb sensors GOMOS, MIPAS, OSIRIS and SCIAMACHY)"
+            Year = select all (2001-2024)
+            Month = select all (1-12)
+            Version = "v0005"
+
+            IASI (variables o3, toz)
+            Download from BIRA WebDAV server: https://webdav.aeronomie.be
+            Path: /guest/o3_cci/webdata/Nadir_Profiles/L3/IASI_MG_FORLI/
+            Username: o3_cci_public
+            No password (leave empty)
+            Download each year (yyyy) into separate folders named "IASI_yyyy"
 
   ESACCI-SEAICE:
     tier: 2

esmvaltool/cmorizers/data/downloaders/datasets/esacci_ozone.py

@@ -1,18 +1,27 @@
-"""Script to download ESACCI-OZONE from the CDS."""
+"""Script to download ESACCI-OZONE from the CDS and BIRA WebDAV."""
 
 import gzip
 import logging
+import os
 import shutil
 import zipfile
+from datetime import datetime
 from pathlib import Path
 
 import cdsapi
+import webdav.client as wc
+from dateutil import relativedelta
 
 logger = logging.getLogger(__name__)
 
 
 def download_dataset(
-    config, dataset, dataset_info, start_date, end_date, overwrite
+    config,
+    dataset,
+    dataset_info,
+    start_date,
+    end_date,
+    overwrite,
 ):
     """Download ESACCI-OZONE dataset using CDS API.
 
@@ -22,34 +31,63 @@ def download_dataset(
       the ECMWF account needs to be saved in user's ${HOME} directory.
     - All the files will be saved in ${RAWOBS}/Tier2/ESACCI-OZONE.
     """
-    cds_url = "https://cds.climate.copernicus.eu/api"
-
     if dataset == "ESACCI-OZONE":
+        raw_obs_dir = Path(config["rootpath"]["RAWOBS"][0])
+        output_folder = raw_obs_dir / f"Tier{dataset_info['tier']}" / dataset
+        output_folder.mkdir(parents=True, exist_ok=True)
+
+        cds_url = "https://cds.climate.copernicus.eu/api"
+
+        if start_date is None:
+            gto_year1 = 1995
+            omps_year1 = 1984
+            megridop_year1 = 2001
+        else:
+            gto_year1 = start_date.year
+            omps_year1 = start_date.year
+            megridop_year1 = start_date.year
+        if end_date is None:
+            gto_year2 = 2024
+            omps_year2 = 2023
+            megridop_year2 = 2025
+        else:
+            gto_year2 = end_date.year
+            omps_year2 = end_date.year
+            megridop_year2 = end_date.year
+
         requests = {
-            "toz": {
+            "toz_gto_ecv": {
                 "processing_level": "level_3",
                 "variable": "atmosphere_mole_content_of_ozone",
                 "vertical_aggregation": "total_column",
                 "sensor": ["merged_uv"],
-                "year": [str(y) for y in range(1995, 2024)],
+                "year": [str(y) for y in range(gto_year1, gto_year2)],
                 "month": [f"{m:02d}" for m in range(1, 13)],
                 "version": ["v2000"],
             },
-            "o3": {
+            "o3_sage_omps": {
                 "processing_level": "level_3",
                 "variable": "mole_concentration_of_ozone_in_air",
                 "vertical_aggregation": "vertical_profiles_from_limb_sensors",
                 "sensor": ["cmzm"],
-                "year": [str(y) for y in range(1984, 2023)],
+                "year": [str(y) for y in range(omps_year1, omps_year2)],
                 "month": [f"{m:02d}" for m in range(1, 13)],
                 "version": ["v0008"],
             },
+            "o3_sage_megridop": {
+                "processing_level": "level_3",
+                "variable": "mole_concentration_of_ozone_in_air",
+                "vertical_aggregation": "vertical_profiles_from_limb_sensors",
+                "sensor": ["cllg"],
+                "year": [
+                    str(y) for y in range(megridop_year1, megridop_year2)
+                ],
+                "month": [f"{m:02d}" for m in range(1, 13)],
+                "version": ["v0005"],
+            },
         }
 
-        client = cdsapi.Client(cds_url)
-        raw_obs_dir = Path(config["rootpath"]["RAWOBS"][0])
-        output_folder = raw_obs_dir / f"Tier{dataset_info['tier']}" / dataset
-        output_folder.mkdir(parents=True, exist_ok=True)
+        cds_client = cdsapi.Client(cds_url)
 
         for var_name, request in requests.items():
             logger.info("Downloading %s data to %s", var_name, output_folder)
@@ -58,12 +96,15 @@ def download_dataset(
 
             if file_path.exists() and not overwrite:
                 logger.info(
-                    "File %s already exists. Skipping download.", file_path
+                    "File %s already exists. Skipping download.",
+                    file_path,
                 )
                 continue
 
-            client.retrieve(
-                "satellite-ozone-v1", request, file_path.as_posix()
+            cds_client.retrieve(
+                "satellite-ozone-v1",
+                request,
+                file_path.as_posix(),
             )
 
             # Handle both .gz and .zip files
@@ -80,5 +121,53 @@ def download_dataset(
                     with open(output_folder / file_path.stem, "wb") as f_out:
                         shutil.copyfileobj(f_in, f_out)
 
+        # download IASI data from BIRA WebDAV (IASI data not available on CDS)
+        # all the files will be saved by year (yyyy) in
+        # ${RAWOBS}/Tier2/ESACCI-OZONE/IASI_yyyy
+
+        if start_date is None:
+            start_date = datetime(2008, 1, 1)
+        if end_date is None:
+            end_date = datetime(2023, 12, 31)
+
+        options = {
+            "webdav_hostname": "https://webdav.aeronomie.be",
+            "webdav_login": "o3_cci_public",
+            "webdav_password": "",
+        }
+
+        wd_client = wc.Client(options)
+
+        basepath = "/guest/o3_cci/webdata/Nadir_Profiles/L3/IASI_MG_FORLI/"
+
+        loop_date = start_date
+        while loop_date <= end_date:
+            year = loop_date.year
+
+            # if needed, create local output directory
+            outdir = output_folder / f"IASI_{year}"
+            os.makedirs(outdir, exist_ok=True)
+
+            # directory on WebDAV server to download
+            remotepath = f"{basepath}/{year}"
+            files = wd_client.list(remotepath)
+            info = wd_client.info(remotepath + "/" + files[0])
+            numfiles = len(files)
+            # calculate approx. download volume in Gbytes
+            size = int(info["size"]) * numfiles // 1073741824
+            del files
+
+            loginfo = (
+                f"downloading {numfiles} files for year {year}"
+                f" (approx. {size} Gbytes)"
+            )
+            logger.info(loginfo)
+
+            # synchronize local (output) directory and WebDAV server directory
+            wd_client.pull(remote_directory=remotepath, local_directory=outdir)
+
+            loop_date += relativedelta.relativedelta(years=1)
+
     else:
-        raise ValueError(f"Unknown dataset: {dataset}")
+        errmsg = f"Unknown dataset: {dataset}"
+        raise ValueError(errmsg)