From 2e5c92ea268dfa440d8e43602c0f87d3d3323557 Mon Sep 17 00:00:00 2001 From: Felicity Chun Date: Fri, 7 Mar 2025 14:26:15 +1100 Subject: [PATCH 1/5] cmoriser files --- .../cmorizers/data/cmor_config/CMEMS.yml | 21 ++++ .../data/formatters/datasets/cmems.py | 95 +++++++++++++++++++ esmvaltool/references/cmems.bibtex | 10 ++ 3 files changed, 126 insertions(+) create mode 100644 esmvaltool/cmorizers/data/cmor_config/CMEMS.yml create mode 100644 esmvaltool/cmorizers/data/formatters/datasets/cmems.py create mode 100644 esmvaltool/references/cmems.bibtex diff --git a/esmvaltool/cmorizers/data/cmor_config/CMEMS.yml b/esmvaltool/cmorizers/data/cmor_config/CMEMS.yml new file mode 100644 index 0000000000..764d725aac --- /dev/null +++ b/esmvaltool/cmorizers/data/cmor_config/CMEMS.yml @@ -0,0 +1,21 @@ +--- +# Filename +filename: 'dt_global_allsat_phy_l4.*.nc' + +# Common global attributes for Cmorizer output +attributes: + project_id: OBS6 + dataset_id: CMEMS + version: 'SEALEVEL_GLO_PHY_L4' + tier: 2 + modeling_realm: sat + source: '' + reference: 'cmems' + comment: '' + +# Variables to cmorize +variables: + zos: + mip: Omon + raw: adt + name: sea_surface_height_above_geoid diff --git a/esmvaltool/cmorizers/data/formatters/datasets/cmems.py b/esmvaltool/cmorizers/data/formatters/datasets/cmems.py new file mode 100644 index 0000000000..7e6d4a1c3b --- /dev/null +++ b/esmvaltool/cmorizers/data/formatters/datasets/cmems.py @@ -0,0 +1,95 @@ +"""ESMValTool CMORizer for CMEMS - Sea Level Thematic Assembly Center. + + This is the CMORizer script for Sea Surface Height Above Geoid + +Tier + Tier 2: open dataset. + +Source + https://doi.org/10.48670/moi-00148 + +Last access + 20250220 + +Download and processing instructions + + +""" + +import logging +import os +import re + +import iris + +from esmvalcore.preprocessor import monthly_statistics +from esmvaltool.cmorizers.data import utilities as utils + +logger = logging.getLogger(__name__) + + +def _get_filepaths(in_dir, basename): + """Group daily files.""" + regex = re.compile(basename) + return_ls = [] + + for root, _dir, files in os.walk(in_dir, followlinks=True): + if len(files)>1: + return_files = [] #load and concat each list & process + for filename in files: + if regex.match(filename): + return_files.append(os.path.join(root, filename)) + + return_ls.append(return_files) + + return return_ls + + +def _extract_variable(cmor_info, attrs, file_ls, out_dir): + """Extract variable and aggregate months.""" + var = cmor_info.short_name + standard_name = cmor_info.standard_name + + cube_prepls = iris.cube.CubeList() + for filepaths in file_ls: + cubels = iris.load(filepaths, standard_name) + # logger.info('Number of day files to concatenate: %d', len(filepaths)) + iris.util.equalise_attributes(cubels) + iris.util.unify_time_units(cubels) + cube_mon = cubels.concatenate_cube() #create mean for month + cube_mon = iris.util.squeeze(cube_mon) + cube_prepls.append(monthly_statistics(cube_mon, 'mean')) + + iris.util.equalise_attributes(cube_prepls) + cube = cube_prepls.concatenate_cube() + utils.fix_var_metadata(cube, cmor_info) + cube = utils.fix_coords(cube) + + utils.set_global_atts(cube, attrs) + utils.save_variable(cube, + var, + out_dir, + attrs, + unlimited_dimensions=['time']) + + +def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): + """Cmorization func call.""" + glob_attrs = cfg['attributes'] + cmor_table = cfg['cmor_table'] + + file_ls = _get_filepaths(in_dir, cfg['filename']) + + if len(file_ls) > 0: + logger.info("Found %d input files in '%s'", len(file_ls), in_dir) + else: + logger.info("No files found, basename: %s", cfg['filename']) + + # Run the cmorization + for (var, var_info) in cfg['variables'].items(): + logger.info("CMORizing variable '%s'", var) + glob_attrs['mip'] = var_info['mip'] + cmor_info = cmor_table.get_variable(var_info['mip'], var) + + _extract_variable(cmor_info, glob_attrs, + file_ls, out_dir) diff --git a/esmvaltool/references/cmems.bibtex b/esmvaltool/references/cmems.bibtex new file mode 100644 index 0000000000..5c9f83f0b0 --- /dev/null +++ b/esmvaltool/references/cmems.bibtex @@ -0,0 +1,10 @@ +@misc{https://doi.org/10.48670/moi-00148, + doi = {10.48670/MOI-00148}, + url = {https://resources.marine.copernicus.eu/product-detail/SEALEVEL_GLO_PHY_L4_MY_008_047/INFORMATION}, + author = {{European Union-Copernicus Marine Service}}, + keywords = {oceanography}, + language = {en}, + title = {GLOBAL OCEAN GRIDDED L4 SEA SURFACE HEIGHTS AND DERIVED VARIABLES REPROCESSED (1993-ONGOING)}, + publisher = {Mercator Ocean International}, + year = {2021} +} \ No newline at end of file From 138c394edaa32b78432a7b667a6515ac879b614e Mon Sep 17 00:00:00 2001 From: Felicity Chun Date: Fri, 7 Mar 2025 15:13:13 +1100 Subject: [PATCH 2/5] remove underscore in version --- esmvaltool/cmorizers/data/cmor_config/CMEMS.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvaltool/cmorizers/data/cmor_config/CMEMS.yml b/esmvaltool/cmorizers/data/cmor_config/CMEMS.yml index 764d725aac..25ada3d6de 100644 --- a/esmvaltool/cmorizers/data/cmor_config/CMEMS.yml +++ b/esmvaltool/cmorizers/data/cmor_config/CMEMS.yml @@ -6,7 +6,7 @@ filename: 'dt_global_allsat_phy_l4.*.nc' attributes: project_id: OBS6 dataset_id: CMEMS - version: 'SEALEVEL_GLO_PHY_L4' + version: 'SEALEVEL-GLO-PHY-L4' tier: 2 modeling_realm: sat source: '' From 90f3ad7aea1dbba5302773ee329b97de0fe0e3eb Mon Sep 17 00:00:00 2001 From: Felicity Chun Date: Wed, 15 Oct 2025 14:44:56 +1100 Subject: [PATCH 3/5] update recipe check and datasets yml --- esmvaltool/cmorizers/data/datasets.yml | 8 ++++++++ .../cmorizers/data/formatters/datasets/cmems.py | 9 +++++++-- esmvaltool/recipes/examples/recipe_check_obs.yml | 12 +++++++++++- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/esmvaltool/cmorizers/data/datasets.yml b/esmvaltool/cmorizers/data/datasets.yml index abd5a80fde..6d11f75266 100644 --- a/esmvaltool/cmorizers/data/datasets.yml +++ b/esmvaltool/cmorizers/data/datasets.yml @@ -273,6 +273,14 @@ datasets: https://downloads.psl.noaa.gov/Datasets/cmap/enh/ precip.mon.mean.nc + CMEMS: + tier: 2 + source: https://doi.org/10.48670/moi-00148 + last_access: 2025-02-20 + info: | + Download daily files from: + https://data.marine.copernicus.eu/product/SEALEVEL_GLO_PHY_L4_MY_008_047/services + CowtanWay: tier: 2 source: https://www-users.york.ac.uk/~kdc3/papers/coverage2013/series.html diff --git a/esmvaltool/cmorizers/data/formatters/datasets/cmems.py b/esmvaltool/cmorizers/data/formatters/datasets/cmems.py index 7e6d4a1c3b..18c8eadf0d 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/cmems.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/cmems.py @@ -1,6 +1,8 @@ """ESMValTool CMORizer for CMEMS - Sea Level Thematic Assembly Center. - This is the CMORizer script for Sea Surface Height Above Geoid + This is the CMORizer script for Sea Surface Height Above Geoid. + Copernicus Marine Environment Monitoring Service (CMEMS) + Product: SEALEVEL_GLO_PHY_L4_MY_008_047 Tier Tier 2: open dataset. @@ -12,7 +14,9 @@ 20250220 Download and processing instructions - + Download daily files from: + https://data.marine.copernicus.eu/product/SEALEVEL_GLO_PHY_L4_MY_008_047/services + """ @@ -56,6 +60,7 @@ def _extract_variable(cmor_info, attrs, file_ls, out_dir): # logger.info('Number of day files to concatenate: %d', len(filepaths)) iris.util.equalise_attributes(cubels) iris.util.unify_time_units(cubels) + # if cmor.frequency is mon, check to save out cube_mon = cubels.concatenate_cube() #create mean for month cube_mon = iris.util.squeeze(cube_mon) cube_prepls.append(monthly_statistics(cube_mon, 'mean')) diff --git a/esmvaltool/recipes/examples/recipe_check_obs.yml b/esmvaltool/recipes/examples/recipe_check_obs.yml index d69583fb01..796fcb122e 100644 --- a/esmvaltool/recipes/examples/recipe_check_obs.yml +++ b/esmvaltool/recipes/examples/recipe_check_obs.yml @@ -67,7 +67,17 @@ diagnostics: pr: additional_datasets: - {project: OBS6, dataset: CMAP, mip: Amon, tier: 2, - type: reanaly, version: v1} + type: sat, version: v1} + scripts: null + + + CMEMS: + description: CMEMS check + variables: + zos: + additional_datasets: + - {project: OBS6, dataset: CMEMS, mip: Omon, tier: 2, + type: reanaly, version: SEALEVEL-GLO-PHY-L4} scripts: null From 379dccb007e285a74efdc0702f07eb029d8a3a98 Mon Sep 17 00:00:00 2001 From: Felicity Chun Date: Thu, 16 Oct 2025 14:10:49 +1100 Subject: [PATCH 4/5] format --- .../data/formatters/datasets/cmems.py | 35 +++++++++---------- esmvaltool/references/cmems.bibtex | 4 +-- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/cmems.py b/esmvaltool/cmorizers/data/formatters/datasets/cmems.py index 18c8eadf0d..ec0fb2b4b3 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/cmems.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/cmems.py @@ -38,12 +38,12 @@ def _get_filepaths(in_dir, basename): return_ls = [] for root, _dir, files in os.walk(in_dir, followlinks=True): - if len(files)>1: - return_files = [] #load and concat each list & process + if len(files) > 1: + return_files = [] # load and concat each list & process for filename in files: if regex.match(filename): return_files.append(os.path.join(root, filename)) - + return_ls.append(return_files) return return_ls @@ -61,9 +61,9 @@ def _extract_variable(cmor_info, attrs, file_ls, out_dir): iris.util.equalise_attributes(cubels) iris.util.unify_time_units(cubels) # if cmor.frequency is mon, check to save out - cube_mon = cubels.concatenate_cube() #create mean for month + cube_mon = cubels.concatenate_cube() # create mean for month cube_mon = iris.util.squeeze(cube_mon) - cube_prepls.append(monthly_statistics(cube_mon, 'mean')) + cube_prepls.append(monthly_statistics(cube_mon, "mean")) iris.util.equalise_attributes(cube_prepls) cube = cube_prepls.concatenate_cube() @@ -71,30 +71,27 @@ def _extract_variable(cmor_info, attrs, file_ls, out_dir): cube = utils.fix_coords(cube) utils.set_global_atts(cube, attrs) - utils.save_variable(cube, - var, - out_dir, - attrs, - unlimited_dimensions=['time']) + utils.save_variable( + cube, var, out_dir, attrs, unlimited_dimensions=["time"] + ) def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): """Cmorization func call.""" - glob_attrs = cfg['attributes'] - cmor_table = cfg['cmor_table'] + glob_attrs = cfg["attributes"] + cmor_table = cfg["cmor_table"] - file_ls = _get_filepaths(in_dir, cfg['filename']) + file_ls = _get_filepaths(in_dir, cfg["filename"]) if len(file_ls) > 0: logger.info("Found %d input files in '%s'", len(file_ls), in_dir) else: - logger.info("No files found, basename: %s", cfg['filename']) + logger.info("No files found, basename: %s", cfg["filename"]) # Run the cmorization - for (var, var_info) in cfg['variables'].items(): + for var, var_info in cfg["variables"].items(): logger.info("CMORizing variable '%s'", var) - glob_attrs['mip'] = var_info['mip'] - cmor_info = cmor_table.get_variable(var_info['mip'], var) + glob_attrs["mip"] = var_info["mip"] + cmor_info = cmor_table.get_variable(var_info["mip"], var) - _extract_variable(cmor_info, glob_attrs, - file_ls, out_dir) + _extract_variable(cmor_info, glob_attrs, file_ls, out_dir) diff --git a/esmvaltool/references/cmems.bibtex b/esmvaltool/references/cmems.bibtex index 5c9f83f0b0..aba25aed3f 100644 --- a/esmvaltool/references/cmems.bibtex +++ b/esmvaltool/references/cmems.bibtex @@ -1,4 +1,4 @@ -@misc{https://doi.org/10.48670/moi-00148, +@misc{cmems, doi = {10.48670/MOI-00148}, url = {https://resources.marine.copernicus.eu/product-detail/SEALEVEL_GLO_PHY_L4_MY_008_047/INFORMATION}, author = {{European Union-Copernicus Marine Service}}, @@ -7,4 +7,4 @@ title = {GLOBAL OCEAN GRIDDED L4 SEA SURFACE HEIGHTS AND DERIVED VARIABLES REPROCESSED (1993-ONGOING)}, publisher = {Mercator Ocean International}, year = {2021} -} \ No newline at end of file +} From b86ffde1b98c38a29921928f9fbfd6e452ebf323 Mon Sep 17 00:00:00 2001 From: Felicity Chun Date: Thu, 16 Oct 2025 15:13:20 +1100 Subject: [PATCH 5/5] format python --- esmvaltool/cmorizers/data/formatters/datasets/cmems.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/cmems.py b/esmvaltool/cmorizers/data/formatters/datasets/cmems.py index ec0fb2b4b3..dd3c6f3e50 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/cmems.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/cmems.py @@ -25,8 +25,8 @@ import re import iris - from esmvalcore.preprocessor import monthly_statistics + from esmvaltool.cmorizers.data import utilities as utils logger = logging.getLogger(__name__)