diff --git a/.circleci/config.yml b/.circleci/config.yml index 9765cf6c40..8e5f38b84f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -159,7 +159,7 @@ jobs: working_directory: /esmvaltool docker: - image: condaforge/mambaforge - resource_class: small + resource_class: medium steps: - run: command: | @@ -182,7 +182,7 @@ jobs: # Test building documentation docker: - image: condaforge/mambaforge - resource_class: small + resource_class: medium steps: - checkout - run: diff --git a/.github/workflows/build-and-deploy-on-pypi.yml b/.github/workflows/build-and-deploy-on-pypi.yml index 449d86b3e4..411ad5df63 100644 --- a/.github/workflows/build-and-deploy-on-pypi.yml +++ b/.github/workflows/build-and-deploy-on-pypi.yml @@ -12,12 +12,18 @@ jobs: build-n-publish: name: Build and publish ESMValCore on PyPi runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/project/ESMValCore/ + permissions: + # IMPORTANT: this permission is mandatory for trusted publishing + id-token: write steps: - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up Python 3.11 - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: "3.11" - name: Install pep517 @@ -34,14 +40,10 @@ jobs: --binary --out-dir dist/ . - #- name: Publish distribution 📦 to Test PyPI - # uses: pypa/gh-action-pypi-publish@master - # with: - # password: ${{ secrets.test_pypi_password }} - # repository_url: https://test.pypi.org/legacy/ + # - name: Publish distribution to Test PyPI + # uses: pypa/gh-action-pypi-publish@release/v1 + # with: + # repository-url: https://test.pypi.org/legacy/ - name: Publish distribution 📦 to PyPI if: startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.pypi_password }} diff --git a/.github/workflows/create-condalock-file.yml b/.github/workflows/create-condalock-file.yml index 6638e296ea..f61a43bb8d 100644 --- a/.github/workflows/create-condalock-file.yml +++ b/.github/workflows/create-condalock-file.yml @@ -96,7 +96,4 @@ jobs: automatedPR assignees: valeriupredoi reviewers: valeriupredoi - team-reviewers: | - owners - maintainers draft: false diff --git a/.github/workflows/install-from-condalock-file.yml b/.github/workflows/install-from-condalock-file.yml index d6f3d7f5f3..406e24f7e1 100644 --- a/.github/workflows/install-from-condalock-file.yml +++ b/.github/workflows/install-from-condalock-file.yml @@ -46,7 +46,6 @@ jobs: - run: which python - run: python -V 2>&1 | tee source_install_linux_artifacts_python_${{ matrix.python-version }}/python_version.txt - run: conda create --name esmvaltool-fromlock --file conda-linux-64.lock - - run: conda install pip - run: which python - run: pip --version - run: pip install -e .[develop] diff --git a/.zenodo.json b/.zenodo.json index 31727467bf..df7a49977d 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -178,6 +178,11 @@ "name": "Kazeroni, Rémi", "orcid": "0000-0001-7205-9528" }, + { + "affiliation": "GEOMAR, Germany", + "name": "Hohn, David", + "orcid": "0000-0002-5317-1247" + }, { "affiliation": "DLR, Germany", "name": "Bauer, Julian" @@ -190,6 +195,11 @@ { "affiliation": "Forschungszentrum Juelich, Germany", "name": "Benke, Joerg" + }, + { + "affiliation": "BSC, Spain", + "name": "Martin-Martinez, Eneko", + "orcid": "0000-0002-9213-7818" } ], "description": "ESMValCore: A community tool for pre-processing data from Earth system models in CMIP and running analysis scripts.", diff --git a/CITATION.cff b/CITATION.cff index 70cf872768..d188af5d39 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -199,13 +199,18 @@ authors: affiliation: "Forschungszentrum Juelich (FZJ), Germany" family-names: Benke given-names: Joerg + - + affiliation: "BSC, Spain" + family-names: Martin-Martinez + given-names: Eneko + orcid: "https://orcid.org/0000-0002-9213-7818" cff-version: 1.2.0 -date-released: 2023-11-01 +date-released: 2023-12-19 doi: "10.5281/zenodo.3387139" license: "Apache-2.0" message: "If you use this software, please cite it using these metadata." repository-code: "https://github.com/ESMValGroup/ESMValCore/" title: ESMValCore -version: "v2.10.0rc1" +version: "v2.10.0" ... diff --git a/README.md b/README.md index 20d13f1637..f2039f5e8b 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![codecov](https://codecov.io/gh/ESMValGroup/ESMValCore/branch/main/graph/badge.svg?token=wQnDzguwq6)](https://codecov.io/gh/ESMValGroup/ESMValCore) [![Codacy Badge](https://app.codacy.com/project/badge/Grade/5d496dea9ef64ec68e448a6df5a65783)](https://www.codacy.com/gh/ESMValGroup/ESMValCore?utm_source=github.com&utm_medium=referral&utm_content=ESMValGroup/ESMValCore&utm_campaign=Badge_Grade) [![Docker Build Status](https://img.shields.io/docker/cloud/build/esmvalgroup/esmvalcore)](https://hub.docker.com/r/esmvalgroup/esmvalcore/) -[![Anaconda-Server Badge](https://img.shields.io/badge/Anaconda.org-2.9.0-blue.svg)](https://anaconda.org/conda-forge/esmvalcore) +[![Anaconda-Server Badge](https://img.shields.io/conda/vn/conda-forge/ESMValCore?color=blue&label=conda-forge&logo=conda-forge&logoColor=white)](https://anaconda.org/conda-forge/esmvalcore) [![Github Actions Test](https://github.com/ESMValGroup/ESMValCore/actions/workflows/run-tests.yml/badge.svg)](https://github.com/ESMValGroup/ESMValCore/actions/workflows/run-tests.yml) ![esmvaltoollogo](https://raw.githubusercontent.com/ESMValGroup/ESMValCore/main/doc/figures/ESMValTool-logo-2.png) diff --git a/conda-linux-64.lock b/conda-linux-64.lock index 62887266d7..c2d8d02ef5 100644 --- a/conda-linux-64.lock +++ b/conda-linux-64.lock @@ -1,40 +1,40 @@ # Generated by conda-lock. # platform: linux-64 -# input_hash: 2b5ce1d2957de54bf44f92cef302051df5f4407ef5d8d75d24f94604e083413f +# input_hash: 69ce949889818c504d81d521be29fb77d725352f185b4be94a01411d8bff16a5 @EXPLICIT https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2#d7c89558ba9fa0495403155b64376d81 -https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2023.7.22-hbcca054_0.conda#a73ecd2988327ad4c8f2c331482917f2 +https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2024.2.2-hbcca054_0.conda#2f4327a1cbe7f022401b236e915a5fef https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2#0c96522c6bdaed4b1566d11387caaf45 https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2#34893075a5c9e55cdafac56607368fc6 https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2#4d59c254e01d9cde7957100457e2d5fb -https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-hab24e00_0.tar.bz2#19410c3df09dfb12d1206132a1d357c5 +https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-h77eed37_1.conda#6185f640c43843e5ad6fd1c5372c3f80 https://conda.anaconda.org/conda-forge/noarch/kernel-headers_linux-64-2.6.32-he073ed8_16.conda#7ca122655873935e02c91279c5b03c8c https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-h41732ed_0.conda#7aca3059a1729aa76c597603f10b0dd3 -https://conda.anaconda.org/conda-forge/linux-64/libboost-headers-1.82.0-ha770c72_6.conda#a943dcb8fd22cf23ce901ac84f6538c2 -https://conda.anaconda.org/conda-forge/linux-64/libgcc-devel_linux-64-12.3.0-h8bca6fd_2.conda#ed613582de7b8569fdc53ca141be176a -https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-devel_linux-64-12.3.0-h8bca6fd_2.conda#7268a17e56eb099d1b8869bbbf46de4c -https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.2.0-h7e041cc_2.conda#9172c297304f2a20134fc56c97fbe229 +https://conda.anaconda.org/conda-forge/linux-64/libboost-headers-1.84.0-ha770c72_0.conda#9c595e87653a36aa4d8c71b4e2f7e586 +https://conda.anaconda.org/conda-forge/noarch/libgcc-devel_linux-64-12.3.0-h8bca6fd_105.conda#e12ce6b051085b8f27e239f5e5f5bce5 +https://conda.anaconda.org/conda-forge/noarch/libstdcxx-devel_linux-64-12.3.0-h8bca6fd_105.conda#b3c6062c84a8e172555ee104ea6a01ab +https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.2.0-h7e041cc_5.conda#f6f6600d18a4047b54f803cf708b868a +https://conda.anaconda.org/conda-forge/linux-64/pandoc-3.1.11.1-ha770c72_0.conda#0e2f14aff42adf4675bcd5335d644a5f https://conda.anaconda.org/conda-forge/noarch/poppler-data-0.4.12-hd8ed1ab_0.conda#d8d7293c5b37f39b2ac32940621c6592 https://conda.anaconda.org/conda-forge/linux-64/python_abi-3.11-4_cp311.conda#d786502c97404c94d7d58d258a445a65 -https://conda.anaconda.org/conda-forge/noarch/tzdata-2023c-h71feb2d_0.conda#939e3e74d8be4dac89ce83b20de2492a +https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda#161081fc7cec0bfda0d86d7cb595f8d8 https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-0.tar.bz2#f766549260d6815b0c52253f1fb1bb29 -https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-h807b86a_2.conda#e2042154faafe61969556f28bade94b9 +https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-h807b86a_5.conda#d211c42b9ce49aee3734fdc828731689 https://conda.anaconda.org/conda-forge/noarch/sysroot_linux-64-2.12-he073ed8_16.conda#071ea8dceff4d30ac511f4a2f8437cd1 https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2#73aaf86a425cc6e73fcf236a5a46396d https://conda.anaconda.org/conda-forge/linux-64/binutils_impl_linux-64-2.40-hf600244_0.conda#33084421a8c0af6aef1b439707f7662a https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2#fee5683a3f04bd15cbd8318b096a27ab https://conda.anaconda.org/conda-forge/linux-64/binutils-2.40-hdd6e379_0.conda#ccc940fddbc3fcd3d79cd4c654c4b5c4 https://conda.anaconda.org/conda-forge/linux-64/binutils_linux-64-2.40-hbdbef99_2.conda#adfebae9fdc63a598495dfe3b006973a -https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-h807b86a_2.conda#c28003b0be0494f9a7664389146716ff -https://conda.anaconda.org/conda-forge/linux-64/aws-c-common-0.9.3-hd590300_0.conda#434466e97a4174b0c4de114eb7100550 -https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h7f98852_4.tar.bz2#a1fd65c7ccbf10880423d82bca54eb54 -https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.20.1-hd590300_0.conda#6642e4faa4804be3a0e7edfefbd16595 +https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-h807b86a_5.conda#d4ff227c46917d3b4565302a2bbb276b +https://conda.anaconda.org/conda-forge/linux-64/aws-c-common-0.9.12-hd590300_0.conda#7dbb94ffb9df66406f3101625807cac1 +https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hd590300_5.conda#69b8b6202a07720f448be700e300ccf4 +https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.26.0-hd590300_0.conda#a86d90025198fd411845fc245ebc06c8 https://conda.anaconda.org/conda-forge/linux-64/fribidi-1.0.10-h36c2ea0_0.tar.bz2#ac7bc6a654f8f41b352b38f4051135f8 -https://conda.anaconda.org/conda-forge/linux-64/geos-3.12.0-h59595ed_0.conda#3fdf79ef322c8379ae83be491d805369 +https://conda.anaconda.org/conda-forge/linux-64/geos-3.12.1-h59595ed_0.conda#8c0f4f71f5a59ceb0c6fa9f51501066d https://conda.anaconda.org/conda-forge/linux-64/gettext-0.21.1-h27087fc_0.tar.bz2#14947d8770185e5153fdd04d4673ed37 https://conda.anaconda.org/conda-forge/linux-64/gflags-2.2.2-he1b5a44_1004.tar.bz2#cddaf2c63ea4a5901cf09524c490ecdc https://conda.anaconda.org/conda-forge/linux-64/giflib-5.2.1-h0b41bf4_3.conda#96f3b11872ef6fad973eac856cd2624f -https://conda.anaconda.org/conda-forge/linux-64/gmp-6.2.1-h58526e2_0.tar.bz2#b94cf2db16066b242ebd26db2facbd56 https://conda.anaconda.org/conda-forge/linux-64/graphite2-1.3.13-h58526e2_1001.tar.bz2#8c54672728e8ec6aa6db90cf2806d220 https://conda.anaconda.org/conda-forge/linux-64/icu-73.2-h59595ed_0.conda#cc47e1facc155f91abd89b11e48e72ff https://conda.anaconda.org/conda-forge/linux-64/json-c-0.17-h7ab15ed_0.conda#9961b1f100c3b6852bd97c9233d06979 @@ -45,31 +45,31 @@ https://conda.anaconda.org/conda-forge/linux-64/libaec-1.1.2-h59595ed_1.conda#12 https://conda.anaconda.org/conda-forge/linux-64/libbrotlicommon-1.1.0-hd590300_1.conda#aec6c91c7371c26392a06708a73c70e5 https://conda.anaconda.org/conda-forge/linux-64/libcrc32c-1.1.2-h9c3ff4c_0.tar.bz2#c965a5aa0d5c1c37ffc62dff36e28400 https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.19-hd590300_0.conda#1635570038840ee3f9c71d22aa5b8b6d -https://conda.anaconda.org/conda-forge/linux-64/libev-4.33-h516909a_1.tar.bz2#6f8720dff19e17ce5d48cfe7f3d2f0a3 +https://conda.anaconda.org/conda-forge/linux-64/libev-4.33-hd590300_2.conda#172bf1cd1ff8629f2b1179945ed45055 https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.5.0-hcb278e6_1.conda#6305a3dd2752c76335295da4e581f2fd https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2#d645c6d2ac96843a2bfaccd2d62b3ac3 -https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-13.2.0-ha4646dd_2.conda#78fdab09d9138851dde2b5fe2a11019e -https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.17-h166bdaf_0.tar.bz2#b62b52da46c39ee2bc3c162ac7f1804d +https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-13.2.0-ha4646dd_5.conda#7a6bd7a12a4bd359e2afe6c0fa1acace +https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.17-hd590300_2.conda#d66573916ffcf376178462f1b61c941e https://conda.anaconda.org/conda-forge/linux-64/libjpeg-turbo-3.0.0-hd590300_1.conda#ea25936bb4080d843790b586850f82b8 +https://conda.anaconda.org/conda-forge/linux-64/libnl-3.9.0-hd590300_0.conda#d27c451db4f1d3c983c78167d2fdabc2 https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hd590300_0.conda#30fd6e37fe21f86f4bd26d6ee73eeec7 https://conda.anaconda.org/conda-forge/linux-64/libnuma-2.0.16-h0b41bf4_1.conda#28bfe2cb11357ccc5be21101a6b7ce86 -https://conda.anaconda.org/conda-forge/linux-64/libsanitizer-12.3.0-h0f45ef3_2.conda#4655db64eca78a6fcc4fb654fc1f8d57 +https://conda.anaconda.org/conda-forge/linux-64/libsanitizer-12.3.0-h0f45ef3_5.conda#11d1ceacff40054d5a74b12975d76f20 https://conda.anaconda.org/conda-forge/linux-64/libsodium-1.0.18-h36c2ea0_1.tar.bz2#c3788462a6fbddafdb413a9f9053e58d -https://conda.anaconda.org/conda-forge/linux-64/libtool-2.4.7-h27087fc_0.conda#f204c8ba400ec475452737094fb81d52 https://conda.anaconda.org/conda-forge/linux-64/libutf8proc-2.8.0-h166bdaf_0.tar.bz2#ede4266dc02e875fe1ea77b25dd43747 https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda#40b61aab5c7ba9ff276c41cfffe6b80b https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.3.2-hd590300_0.conda#30de3fd9b3b602f7473f30e684eeea8c +https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda#5aa797f8787fe7a17d1b0821485b5adc https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.13-hd590300_5.conda#f36c115f1ee199da648e0597ec2047ad https://conda.anaconda.org/conda-forge/linux-64/lz4-c-1.9.4-hcb278e6_0.conda#318b08df404f9c9be5712aaa5a6f0bb0 https://conda.anaconda.org/conda-forge/linux-64/lzo-2.10-h516909a_1000.tar.bz2#bb14fcb13341b81d5eb386423b9d2bac -https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.4-hcb278e6_0.conda#681105bccc2a3f7f1a837d47d39c9179 +https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.4-h59595ed_2.conda#7dbaa197d7ba6032caf7ae7f32c1efa0 https://conda.anaconda.org/conda-forge/linux-64/nspr-4.35-h27087fc_0.conda#da0ec11a6454ae19bff5b02ed881a2b1 -https://conda.anaconda.org/conda-forge/linux-64/openssl-3.1.3-hd590300_0.conda#7bb88ce04c8deb9f7d763ae04a1da72f -https://conda.anaconda.org/conda-forge/linux-64/pixman-0.42.2-h59595ed_0.conda#700edd63ccd5fc66b70b1c028cea9a68 +https://conda.anaconda.org/conda-forge/linux-64/openssl-3.2.1-hd590300_0.conda#51a753e64a3027bd7e23a189b1f6e91e +https://conda.anaconda.org/conda-forge/linux-64/pixman-0.43.2-h59595ed_0.conda#71004cbf7924e19c02746ccde9fd7123 https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-h36c2ea0_1001.tar.bz2#22dad4df6e8630e8dff2428f6f6a7036 -https://conda.anaconda.org/conda-forge/linux-64/rdma-core-28.9-h59595ed_1.conda#aeffb7c06b5f65e55e6c637408dc4100 https://conda.anaconda.org/conda-forge/linux-64/snappy-1.1.10-h9fff704_0.conda#e6d228cd0bb74a51dd18f5bfce0b4115 -https://conda.anaconda.org/conda-forge/linux-64/tzcode-2023c-h0b41bf4_0.conda#0c0533894f21c3d35697cb8378d390e2 +https://conda.anaconda.org/conda-forge/linux-64/tzcode-2024a-h3f72095_0.conda#32146e34aaec3745a08b6f49af3f41b0 https://conda.anaconda.org/conda-forge/linux-64/uriparser-0.9.7-hcb278e6_1.conda#2c46deb08ba9b10e90d0a6401ad65deb https://conda.anaconda.org/conda-forge/linux-64/xorg-kbproto-1.0.7-h7f98852_1002.tar.bz2#4b230e8381279d76131116660f5a241a https://conda.anaconda.org/conda-forge/linux-64/xorg-libice-1.1.1-hd590300_0.conda#b462a33c0be1421532f28bfe8f4a7514 @@ -81,99 +81,98 @@ https://conda.anaconda.org/conda-forge/linux-64/xorg-xproto-7.0.31-h7f98852_1007 https://conda.anaconda.org/conda-forge/linux-64/xxhash-0.8.2-hd590300_0.conda#f08fb5c89edfc4aadee1c81d4cfb1fa1 https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2#2161070d867d1b1204ea749c8eec4ef0 https://conda.anaconda.org/conda-forge/linux-64/yaml-0.2.5-h7f98852_2.tar.bz2#4cb3ad778ec2d5a7acbdf254eb1c42ae -https://conda.anaconda.org/conda-forge/linux-64/aws-c-cal-0.6.6-h09139f6_0.conda#289ae014f49bb39e5e0aae277b88dbfe -https://conda.anaconda.org/conda-forge/linux-64/aws-c-compression-0.2.17-h184a658_3.conda#c62775b5028b5a4eda25037f9af7f5b3 -https://conda.anaconda.org/conda-forge/linux-64/aws-c-sdkutils-0.1.12-h184a658_2.conda#ba06d81b81ec3eaf4ee83cd47f808134 -https://conda.anaconda.org/conda-forge/linux-64/aws-checksums-0.1.17-h184a658_2.conda#10fcdbd02ba7fa0827fb8f7d94f8375b +https://conda.anaconda.org/conda-forge/linux-64/aws-c-cal-0.6.9-h14ec70c_3.conda#7da4b84275e63f56d158d6250727a70f +https://conda.anaconda.org/conda-forge/linux-64/aws-c-compression-0.2.17-h572eabf_8.conda#cc6630010cb1211cc15fb348f7c7eb70 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-sdkutils-0.1.14-h572eabf_0.conda#42db61eee93a2c0f918d18bd4422d331 +https://conda.anaconda.org/conda-forge/linux-64/aws-checksums-0.1.17-h572eabf_7.conda#f7323eedc2685a24661cd6b57d7ed321 https://conda.anaconda.org/conda-forge/linux-64/expat-2.5.0-hcb278e6_1.conda#8b9b5aca60558d02ddaa09d599e55920 -https://conda.anaconda.org/conda-forge/linux-64/gcc_impl_linux-64-12.3.0-he2b93b0_2.conda#2f4d8677dc7dd87f93e9abfb2ce86808 +https://conda.anaconda.org/conda-forge/linux-64/gcc_impl_linux-64-12.3.0-he2b93b0_5.conda#e89827619e73df59496c708b94f6f3d5 https://conda.anaconda.org/conda-forge/linux-64/glog-0.6.0-h6f12383_0.tar.bz2#b31f3565cb84435407594e548a2fb7b2 https://conda.anaconda.org/conda-forge/linux-64/hdf4-4.2.15-h2a13503_7.conda#bd77f8da987968ec3927990495dc22e4 https://conda.anaconda.org/conda-forge/linux-64/libbrotlidec-1.1.0-hd590300_1.conda#f07002e225d7a60a694d42a7bf5ff53f https://conda.anaconda.org/conda-forge/linux-64/libbrotlienc-1.1.0-hd590300_1.conda#5fc11c6020d421960607d821310fcd4d https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20191231-he28a2e2_2.tar.bz2#4d331e44109e3f0e19b4cb8f9b82f3e1 https://conda.anaconda.org/conda-forge/linux-64/libevent-2.1.12-hf998b51_1.conda#a1cfcc585f0c42bf8d5546bb1dfb668d -https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-13.2.0-h69a702a_2.conda#e75a75a6eaf6f318dae2631158c46575 +https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-13.2.0-h69a702a_5.conda#e73e9cfd1191783392131e6238bdb3e9 https://conda.anaconda.org/conda-forge/linux-64/libkml-1.3.0-h01aab08_1018.conda#3eb5f16bcc8a02892199aa63555c731f -https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.52.0-h61bc06f_0.conda#613955a50485812985c059e7b269f42e -https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.39-h753d276_0.conda#e1c890aebdebbfbf87e2c917187b4416 -https://conda.anaconda.org/conda-forge/linux-64/libprotobuf-4.24.3-hf27288f_1.conda#5097789a2bc83e697d7509df57f25bfd +https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.58.0-h47da74e_1.conda#700ac6ea6d53d5510591c4344d5c989a +https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.42-h2797004_0.conda#d67729828dc6ff7ba44a61062ad79880 +https://conda.anaconda.org/conda-forge/linux-64/libprotobuf-4.25.1-hf27288f_1.conda#78ad06185133494138cd5e922ed73ac7 https://conda.anaconda.org/conda-forge/linux-64/libre2-11-2023.06.02-h7a70373_0.conda#c0e7eacd9694db3ef5ef2979a7deea70 -https://conda.anaconda.org/conda-forge/linux-64/librttopo-1.1.0-hb58d41b_14.conda#264f9a3a4ea52c8f4d3e8ae1213a3335 -https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.43.2-h2797004_0.conda#4b441a1ee22397d5a27dc1126b849edd +https://conda.anaconda.org/conda-forge/linux-64/librttopo-1.1.0-h8917695_15.conda#20c3c14bc491f30daecaa6f73e2223ae +https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.45.1-h2797004_0.conda#fc4ccadfbf6d4784de88c41704792562 https://conda.anaconda.org/conda-forge/linux-64/libssh2-1.11.0-h0841786_0.conda#1f5a58e686b13bcfde88b93f547d23fe https://conda.anaconda.org/conda-forge/linux-64/libudunits2-2.2.28-h40f5838_3.conda#4bdace082e911a3e1f1f0b721bed5b56 https://conda.anaconda.org/conda-forge/linux-64/libxcb-1.15-h0b41bf4_0.conda#33277193f5b92bad9fdd230eb700929c -https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.11.5-h232c23b_1.conda#f3858448893839820d4bcfb14ad3ecdf +https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.12.5-h232c23b_0.conda#c442ebfda7a475f5e78f1c8e45f1e919 https://conda.anaconda.org/conda-forge/linux-64/libzip-1.10.1-h2629f0a_3.conda#ac79812548e7e8cf61f7b0abdef01d3b -https://conda.anaconda.org/conda-forge/linux-64/pcre2-10.40-hc3806b6_0.tar.bz2#69e2c796349cd9b273890bee0febfe1b +https://conda.anaconda.org/conda-forge/linux-64/pcre2-10.42-hcad00b1_0.conda#679c8961826aa4b50653bce17ee52abe +https://conda.anaconda.org/conda-forge/linux-64/rdma-core-50.0-hd3aeb46_0.conda#4594b391274e38f07c668acb45285a1f https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8228510_1.conda#47d31b792659ce70f470b5c82fdfb7a4 -https://conda.anaconda.org/conda-forge/linux-64/s2n-1.3.54-h06160fa_0.conda#149520612b92991a7de6f17550a19739 -https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-h2797004_0.conda#513336054f884f95d9fd925748f41ef3 -https://conda.anaconda.org/conda-forge/linux-64/ucx-1.15.0-h64cca9d_0.conda#b35b1f1a9fdbf93266c91f297dc9060e +https://conda.anaconda.org/conda-forge/linux-64/s2n-1.4.3-h06160fa_0.conda#860332295eef2ef9fef370f365ea78b4 +https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h4845f30_101.conda#d453b98d9c83e71da0741bb0ff4d76bc https://conda.anaconda.org/conda-forge/linux-64/xorg-libsm-1.2.4-h7391055_0.conda#93ee23f12bc2e684548181256edd2cf6 https://conda.anaconda.org/conda-forge/linux-64/zeromq-4.3.5-h59595ed_0.conda#8851084c192dbc56215ac4e3c9aa30fa https://conda.anaconda.org/conda-forge/linux-64/zlib-1.2.13-hd590300_5.conda#68c34ec6149623be41a1933ab996a209 https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.5-hfc55251_0.conda#04b88013080254850d6c01ed54810589 -https://conda.anaconda.org/conda-forge/linux-64/aws-c-io-0.13.35-hc1ea125_1.conda#11e5683c74d07b2a3e90ddb7ac55e2e0 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-io-0.14.3-h3c8c088_1.conda#12af79204e13550614bc51bb380c32e5 https://conda.anaconda.org/conda-forge/linux-64/blosc-1.21.5-h0f2a231_0.conda#009521b7ed97cca25f8f997f9e745976 https://conda.anaconda.org/conda-forge/linux-64/brotli-bin-1.1.0-hd590300_1.conda#39f910d205726805a958da408ca194ba https://conda.anaconda.org/conda-forge/linux-64/freetype-2.12.1-h267a509_2.conda#9ae35c3d96db2c94ce0cef86efdfa2cb https://conda.anaconda.org/conda-forge/linux-64/gcc-12.3.0-h8d2909c_2.conda#e2f2f81f367e14ca1f77a870bda2fe59 https://conda.anaconda.org/conda-forge/linux-64/gcc_linux-64-12.3.0-h76fc315_2.conda#11517e7b5c910c5b5d6985c0c7eb7f50 -https://conda.anaconda.org/conda-forge/linux-64/gfortran_impl_linux-64-12.3.0-hfcedea8_2.conda#09d48cadff6669068c3bf7ae7dc8ea4a -https://conda.anaconda.org/conda-forge/linux-64/gxx_impl_linux-64-12.3.0-he2b93b0_2.conda#f89b9916afc36fc5562fbfc11330a8a2 +https://conda.anaconda.org/conda-forge/linux-64/gfortran_impl_linux-64-12.3.0-hfcedea8_5.conda#4d72ee7c82f8a9b2ecef4fcefa9acd19 +https://conda.anaconda.org/conda-forge/linux-64/gxx_impl_linux-64-12.3.0-he2b93b0_5.conda#cddba8fd94e52012abea1caad722b9c2 https://conda.anaconda.org/conda-forge/linux-64/krb5-1.21.2-h659d440_0.conda#cd95826dbd331ed1be26bdf401432844 -https://conda.anaconda.org/conda-forge/linux-64/libarchive-3.7.2-h039dbb9_0.conda#611d6c83d1130ea60c916531adfb11db -https://conda.anaconda.org/conda-forge/linux-64/libglib-2.78.0-hebfc3b9_0.conda#e618003da3547216310088478e475945 -https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.24-pthreads_h413a1c8_0.conda#6e4ef6ca28655124dcde9bd500e44c32 +https://conda.anaconda.org/conda-forge/linux-64/libarchive-3.7.2-h2aa1ff5_1.conda#3bf887827d1968275978361a6e405e4f +https://conda.anaconda.org/conda-forge/linux-64/libglib-2.78.3-h783c2da_0.conda#9bd06b12bbfa6fd1740fd23af4b0f0c7 +https://conda.anaconda.org/conda-forge/linux-64/libllvm15-15.0.7-hb3ce162_4.conda#8a35df3cbc0c8b12cc8af9473ae75eef +https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.26-pthreads_h413a1c8_0.conda#760ae35415f5ba8b15d09df5afe8b23a https://conda.anaconda.org/conda-forge/linux-64/libthrift-0.19.0-hb90f79a_1.conda#8cdb7d41faa0260875ba92414c487e2d https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.6.0-ha9c0a0a_2.conda#55ed21669b2015f77c180feb1dd41930 -https://conda.anaconda.org/conda-forge/linux-64/libxslt-1.1.37-h0054252_1.conda#f27960e8873abb5476e96ef33bdbdccd -https://conda.anaconda.org/conda-forge/linux-64/minizip-4.0.1-h0ab5242_5.conda#2f0f7031d8f0f9f6520093009eb3628f -https://conda.anaconda.org/conda-forge/linux-64/nss-3.94-h1d7d5a4_0.conda#7caef74bbfa730e014b20f0852068509 -https://conda.anaconda.org/conda-forge/linux-64/orc-1.9.0-h208142c_3.conda#f983ae19192439116ca5b5589560f167 -https://conda.anaconda.org/conda-forge/linux-64/pandoc-3.1.3-h32600fe_0.conda#8287aeb8462e2d4b235eff788e75919d -https://conda.anaconda.org/conda-forge/linux-64/python-3.11.6-hab00c5b_0_cpython.conda#b0dfbe2fcbfdb097d321bfd50ecddab1 +https://conda.anaconda.org/conda-forge/linux-64/libxslt-1.1.39-h76b75d6_0.conda#e71f31f8cfb0a91439f2086fc8aa0461 +https://conda.anaconda.org/conda-forge/linux-64/minizip-4.0.4-h0ab5242_0.conda#813bc75d9c33ddd9c9d5b8d9c560e152 +https://conda.anaconda.org/conda-forge/linux-64/nss-3.97-h1d7d5a4_0.conda#b916d71a3032416e3f9136090d814472 +https://conda.anaconda.org/conda-forge/linux-64/orc-1.9.2-h7829240_1.conda#306ffb76ce3cdfc539d29fa5b8dd716c +https://conda.anaconda.org/conda-forge/linux-64/python-3.11.7-hab00c5b_1_cpython.conda#27cf681282c11dba7b0b1fd266e8f289 https://conda.anaconda.org/conda-forge/linux-64/re2-2023.06.02-h2873b5e_0.conda#bb2d5e593ef13fe4aff0bc9440f945ae -https://conda.anaconda.org/conda-forge/linux-64/sqlite-3.43.2-h2c6b66d_0.conda#c37b95bcd6c6833dacfd5df0ae2f4303 +https://conda.anaconda.org/conda-forge/linux-64/sqlite-3.45.1-h2c6b66d_0.conda#93acf31b379acebada263b9bce3dc6ed +https://conda.anaconda.org/conda-forge/linux-64/ucx-1.15.0-h75e419f_3.conda#5baf4efbca923cdf73490c62cc7de1e2 https://conda.anaconda.org/conda-forge/linux-64/udunits2-2.2.28-h40f5838_3.conda#6bb8deb138f87c9d48320ac21b87e7a1 https://conda.anaconda.org/conda-forge/linux-64/xorg-libx11-1.8.7-h8ee46fc_0.conda#49e482d882669206653b095f5206c05b -https://conda.anaconda.org/conda-forge/noarch/alabaster-0.7.13-pyhd8ed1ab_0.conda#06006184e203b61d3525f90de394471e +https://conda.anaconda.org/conda-forge/noarch/alabaster-0.7.16-pyhd8ed1ab_0.conda#def531a3ac77b7fb8c21d17bb5d0badb https://conda.anaconda.org/conda-forge/noarch/antlr-python-runtime-4.11.1-pyhd8ed1ab_0.tar.bz2#15109c4977d39ad7aa3423f57243e286 https://conda.anaconda.org/conda-forge/linux-64/atk-1.0-2.38.0-hd4edc92_1.tar.bz2#6c72ec3e660a51736913ef6ea68c454b -https://conda.anaconda.org/conda-forge/noarch/attrs-23.1.0-pyh71513ae_1.conda#3edfead7cedd1ab4400a6c588f3e75f8 -https://conda.anaconda.org/conda-forge/linux-64/aws-c-event-stream-0.3.2-h24cd7f8_3.conda#031e5cce5b151ccd3cb8d101900246d7 -https://conda.anaconda.org/conda-forge/linux-64/aws-c-http-0.7.13-h0561a52_5.conda#ebcdf12a0994c154076a4f0d40ae514c -https://conda.anaconda.org/conda-forge/noarch/backcall-0.2.0-pyh9f0ad1d_0.tar.bz2#6006a6d08a3fa99268a2681c7fb55213 -https://conda.anaconda.org/conda-forge/noarch/backports-1.0-pyhd8ed1ab_3.conda#54ca2e08b3220c148a1d8329c2678e02 +https://conda.anaconda.org/conda-forge/noarch/attrs-23.2.0-pyh71513ae_0.conda#5e4c0743c70186509d1412e03c2d8dfa +https://conda.anaconda.org/conda-forge/linux-64/aws-c-event-stream-0.4.1-h17cd1f3_5.conda#65d1aabc7656d7c08585efd584332235 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-http-0.8.0-hc6da83f_5.conda#a257c3335609a22036947f99a87ca024 https://conda.anaconda.org/conda-forge/linux-64/backports.zoneinfo-0.2.1-py311h38be061_8.conda#5384590f14dfe6ccd02811236afc9f8e https://conda.anaconda.org/conda-forge/linux-64/brotli-1.1.0-hd590300_1.conda#f27a24d46e3ea7b70a1f98e50c62508f https://conda.anaconda.org/conda-forge/linux-64/brotli-python-1.1.0-py311hb755f60_1.conda#cce9e7c3f1c307f2a5fb08a2922d6164 -https://conda.anaconda.org/conda-forge/linux-64/c-compiler-1.6.0-hd590300_0.conda#ea6c792f792bdd7ae6e7e2dee32f0a48 -https://conda.anaconda.org/conda-forge/noarch/certifi-2023.7.22-pyhd8ed1ab_0.conda#7f3dbc9179b4dde7da98dfb151d0ad22 +https://conda.anaconda.org/conda-forge/linux-64/c-compiler-1.7.0-hd590300_0.conda#fad1d0a651bf929c6c16fbf1f6ccfa7c +https://conda.anaconda.org/conda-forge/noarch/certifi-2024.2.2-pyhd8ed1ab_0.conda#0876280e409658fc6f9e75d035960333 https://conda.anaconda.org/conda-forge/noarch/cfgv-3.3.1-pyhd8ed1ab_0.tar.bz2#ebb5f5f7dc4f1a3780ef7ea7738db08c -https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.3.0-pyhd8ed1ab_0.conda#fef8ef5f0a54546b9efee39468229917 +https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.3.2-pyhd8ed1ab_0.conda#7f4a9e3fcff3f6356ae99244a014da6a https://conda.anaconda.org/conda-forge/noarch/click-8.1.7-unix_pyh707e725_0.conda#f3ad426304898027fc619827ff428eca https://conda.anaconda.org/conda-forge/noarch/cloudpickle-3.0.0-pyhd8ed1ab_0.conda#753d29fe41bb881e4b9c004f0abf973f https://conda.anaconda.org/conda-forge/noarch/codespell-2.2.6-pyhd8ed1ab_0.conda#a206349b7bb7475ae580f987cb425bdd https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_0.tar.bz2#3faab06a954c2a04039983f2c4a50d99 https://conda.anaconda.org/conda-forge/noarch/cycler-0.12.1-pyhd8ed1ab_0.conda#5cd86562580f274031ede6aa6aa24441 -https://conda.anaconda.org/conda-forge/linux-64/cython-3.0.4-py311hb755f60_0.conda#6dc0be74820d94a0606595130218981c +https://conda.anaconda.org/conda-forge/linux-64/cython-3.0.8-py311hb755f60_0.conda#28778bfea41b0f34141208783882649b https://conda.anaconda.org/conda-forge/noarch/decorator-5.1.1-pyhd8ed1ab_0.tar.bz2#43afe5ab04e35e17ba28649471dd7364 https://conda.anaconda.org/conda-forge/noarch/defusedxml-0.7.1-pyhd8ed1ab_0.tar.bz2#961b3a227b437d82ad7054484cfa71b2 -https://conda.anaconda.org/conda-forge/noarch/dill-0.3.7-pyhd8ed1ab_0.conda#5e4f3466526c52bc9af2d2353a1460bd -https://conda.anaconda.org/conda-forge/noarch/distlib-0.3.7-pyhd8ed1ab_0.conda#12d8aae6994f342618443a8f05c652a0 -https://conda.anaconda.org/conda-forge/linux-64/docutils-0.20.1-py311h38be061_2.conda#33f8066e53679dd4be2355fec849bf01 +https://conda.anaconda.org/conda-forge/noarch/dill-0.3.8-pyhd8ed1ab_0.conda#78745f157d56877a2c6e7b386f66f3e2 +https://conda.anaconda.org/conda-forge/noarch/distlib-0.3.8-pyhd8ed1ab_0.conda#db16c66b759a64dc5183d69cc3745a52 +https://conda.anaconda.org/conda-forge/linux-64/docutils-0.20.1-py311h38be061_3.conda#1c33f55e5cdcc2a2b973c432b5225bfe https://conda.anaconda.org/conda-forge/noarch/dodgy-0.2.1-py_0.tar.bz2#62a69d073f7446c90f417b0787122f5b https://conda.anaconda.org/conda-forge/noarch/entrypoints-0.4-pyhd8ed1ab_0.tar.bz2#3cf04868fee0a029769bd41f4b2fbf2d -https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.1.3-pyhd8ed1ab_0.conda#e6518222753f519e911e83136d2158d9 +https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.2.0-pyhd8ed1ab_2.conda#8d652ea2ee8eaee02ed8dc820bc794aa https://conda.anaconda.org/conda-forge/noarch/execnet-2.0.2-pyhd8ed1ab_0.conda#67de0d8241e1060a479e3c37793e26f9 -https://conda.anaconda.org/conda-forge/noarch/executing-1.2.0-pyhd8ed1ab_0.tar.bz2#4c1bc140e2be5c8ba6e3acab99e25c50 -https://conda.anaconda.org/conda-forge/noarch/filelock-3.12.4-pyhd8ed1ab_0.conda#5173d4b8267a0699a43d73231e0b6596 +https://conda.anaconda.org/conda-forge/noarch/executing-2.0.1-pyhd8ed1ab_0.conda#e16be50e378d8a4533b989035b196ab8 +https://conda.anaconda.org/conda-forge/noarch/filelock-3.13.1-pyhd8ed1ab_0.conda#0c1729b74a8152fde6a38ba0a2ab9f45 https://conda.anaconda.org/conda-forge/linux-64/fontconfig-2.14.2-h14ed4e7_0.conda#0f69b688f52ff6da70bccb7ff7001d1d https://conda.anaconda.org/conda-forge/linux-64/freexl-2.0.0-h743c826_0.conda#12e6988845706b2cfbc3bc35c9a61a95 -https://conda.anaconda.org/conda-forge/noarch/fsspec-2023.9.2-pyh1a96a4e_0.conda#9d15cd3a0e944594ab528da37dc72ecc +https://conda.anaconda.org/conda-forge/noarch/fsspec-2024.2.0-pyhca7485f_0.conda#fad86b90138cf5d82c6f5a2ed6e683d9 https://conda.anaconda.org/conda-forge/linux-64/gdk-pixbuf-2.42.10-h829c605_4.conda#252a696860674caf7a855e16f680d63a https://conda.anaconda.org/conda-forge/noarch/geographiclib-1.52-pyhd8ed1ab_0.tar.bz2#6880e7100ebae550a33ce26663316d85 https://conda.anaconda.org/conda-forge/linux-64/gfortran-12.3.0-h499e0f7_2.conda#0558a8c44eb7a18e6682bd3a8ae6dcab @@ -182,58 +181,57 @@ https://conda.anaconda.org/conda-forge/linux-64/gts-0.7.6-h977cf35_4.conda#4d8df https://conda.anaconda.org/conda-forge/linux-64/gxx-12.3.0-h8d2909c_2.conda#673bac341be6b90ef9e8abae7e52ca46 https://conda.anaconda.org/conda-forge/linux-64/gxx_linux-64-12.3.0-h8a814eb_2.conda#f517b1525e9783849bd56a5dc45a9960 https://conda.anaconda.org/conda-forge/noarch/humanfriendly-10.0-pyhd8ed1ab_6.conda#2ed1fe4b9079da97c44cfe9c2e5078fd -https://conda.anaconda.org/conda-forge/noarch/idna-3.4-pyhd8ed1ab_0.tar.bz2#34272b248891bddccc64479f9a7fffed +https://conda.anaconda.org/conda-forge/noarch/idna-3.6-pyhd8ed1ab_0.conda#1a76f09108576397c41c0b0c5bd84134 https://conda.anaconda.org/conda-forge/noarch/imagesize-1.4.1-pyhd8ed1ab_0.tar.bz2#7de5386c8fea29e76b303f37dde4c352 https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_0.conda#f800d2da156d08e289b14e87e43c1ae5 https://conda.anaconda.org/conda-forge/noarch/itsdangerous-2.1.2-pyhd8ed1ab_0.tar.bz2#3c3de74912f11d2b590184f03c7cd09b https://conda.anaconda.org/conda-forge/linux-64/kiwisolver-1.4.5-py311h9547e67_1.conda#2c65bdf442b0d37aad080c8a4e0d452f -https://conda.anaconda.org/conda-forge/linux-64/lazy-object-proxy-1.9.0-py311h459d7ec_1.conda#7cc99d87755a9e64586a6004c5f0f534 -https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.15-hb7c19ff_3.conda#e96637dd92c5f340215c753a5c9a22d7 -https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-19_linux64_openblas.conda#420f4e9be59d0dc9133a0f43f7bab3f3 -https://conda.anaconda.org/conda-forge/linux-64/libcurl-8.4.0-hca28451_0.conda#1158ac1d2613b28685644931f11ee807 -https://conda.anaconda.org/conda-forge/linux-64/libgrpc-1.58.1-he06187c_2.conda#42f5e2ba0d41ba270afd3eb5c725ccf5 -https://conda.anaconda.org/conda-forge/linux-64/libpq-16.0-hfc447b1_1.conda#e4a9a5ba40123477db33e02a78dffb01 +https://conda.anaconda.org/conda-forge/linux-64/lazy-object-proxy-1.10.0-py311h459d7ec_0.conda#d39020c78fd00ed774ff9c876e8aba07 +https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.16-hb7c19ff_0.conda#51bb7010fc86f70eee639b4bb7a894f5 +https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-21_linux64_openblas.conda#0ac9f44fc096772b0aa092119b00c3ca +https://conda.anaconda.org/conda-forge/linux-64/libcurl-8.5.0-hca28451_0.conda#7144d5a828e2cae218e0e3c98d8a0aeb +https://conda.anaconda.org/conda-forge/linux-64/libgrpc-1.60.0-h74775cd_1.conda#e5dac7b919ed16dbcf9dc0f512cb68c0 +https://conda.anaconda.org/conda-forge/linux-64/libpq-16.2-h33b98f1_0.conda#fe0e297faf462ee579c95071a5211665 https://conda.anaconda.org/conda-forge/linux-64/libwebp-1.3.2-h658648e_1.conda#0ebb65e8d86843865796c7c95a941f34 https://conda.anaconda.org/conda-forge/noarch/locket-1.0.0-pyhd8ed1ab_0.tar.bz2#91e27ef3d05cc772ce627e51cff111c4 -https://conda.anaconda.org/conda-forge/linux-64/lxml-4.9.3-py311h1a07684_1.conda#aab51e50d994e58efdfa5382139b0468 -https://conda.anaconda.org/conda-forge/linux-64/lz4-4.3.2-py311h38e4bf4_1.conda#f8e0b648d77bbe44d1fe8af8cc56a590 -https://conda.anaconda.org/conda-forge/linux-64/markupsafe-2.1.3-py311h459d7ec_1.conda#71120b5155a0c500826cf81536721a15 +https://conda.anaconda.org/conda-forge/linux-64/lxml-5.1.0-py311h9691dec_0.conda#cee803b62c62e5f3326be31e57161ff5 +https://conda.anaconda.org/conda-forge/linux-64/lz4-4.3.3-py311h38e4bf4_0.conda#3910c815fc788621f88b2bdc0fa9f0a6 +https://conda.anaconda.org/conda-forge/linux-64/markupsafe-2.1.5-py311h459d7ec_0.conda#a322b4185121935c871d201ae00ac143 https://conda.anaconda.org/conda-forge/noarch/mccabe-0.7.0-pyhd8ed1ab_0.tar.bz2#34fc335fc50eef0b5ea708f2b5f54e0c -https://conda.anaconda.org/conda-forge/noarch/mistune-3.0.1-pyhd8ed1ab_0.conda#1dad8397c94e4de97a70de552a7dcf49 -https://conda.anaconda.org/conda-forge/linux-64/msgpack-python-1.0.6-py311h9547e67_0.conda#e826b71bf3dc8c91ee097663e2bcface -https://conda.anaconda.org/conda-forge/noarch/munch-4.0.0-pyhd8ed1ab_0.conda#376b32e8f9d3eacbd625f37d39bd507d +https://conda.anaconda.org/conda-forge/noarch/mistune-3.0.2-pyhd8ed1ab_0.conda#5cbee699846772cc939bef23a0d524ed +https://conda.anaconda.org/conda-forge/linux-64/msgpack-python-1.0.7-py311h9547e67_0.conda#3ac85c6c226e2a2e4b17864fc2ca88ff https://conda.anaconda.org/conda-forge/noarch/munkres-1.1.4-pyh9f0ad1d_0.tar.bz2#2ba8498c1018c1e9c61eb99b973dfe19 https://conda.anaconda.org/conda-forge/noarch/mypy_extensions-1.0.0-pyha770c72_0.conda#4eccaeba205f0aed9ac3a9ea58568ca3 -https://conda.anaconda.org/conda-forge/noarch/networkx-3.2-pyhd8ed1ab_1.conda#522039fb968d6d0a10e872e6f3856f53 +https://conda.anaconda.org/conda-forge/noarch/networkx-3.2.1-pyhd8ed1ab_0.conda#425fce3b531bed6ec3c74fab3e5f0a1c https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.0-h488ebb8_3.conda#128c25b7fe6a25286a48f3a6a9b5b6f3 https://conda.anaconda.org/conda-forge/noarch/packaging-23.2-pyhd8ed1ab_0.conda#79002079284aa895f883c6b7f3f88fd6 https://conda.anaconda.org/conda-forge/noarch/pandocfilters-1.5.0-pyhd8ed1ab_0.tar.bz2#457c2c8c08e54905d6954e79cb5b5db9 https://conda.anaconda.org/conda-forge/noarch/parso-0.8.3-pyhd8ed1ab_0.tar.bz2#17a565a0c3899244e938cdf417e7b094 -https://conda.anaconda.org/conda-forge/noarch/pathspec-0.11.2-pyhd8ed1ab_0.conda#e41debb259e68490e3ab81e46b639ab6 +https://conda.anaconda.org/conda-forge/noarch/pathspec-0.12.1-pyhd8ed1ab_0.conda#17064acba08d3686f1135b5ec1b32b12 https://conda.anaconda.org/conda-forge/noarch/pickleshare-0.7.5-py_1003.tar.bz2#415f0ebb6198cc2801c73438a9fb5761 https://conda.anaconda.org/conda-forge/noarch/pkgutil-resolve-name-1.3.10-pyhd8ed1ab_1.conda#405678b942f2481cecdb3e010f4925d9 -https://conda.anaconda.org/conda-forge/noarch/pluggy-1.3.0-pyhd8ed1ab_0.conda#2390bd10bed1f3fdc7a537fb5a447d8d -https://conda.anaconda.org/conda-forge/linux-64/psutil-5.9.5-py311h459d7ec_1.conda#490d7fa8675afd1aa6f1b2332d156a45 +https://conda.anaconda.org/conda-forge/noarch/platformdirs-4.2.0-pyhd8ed1ab_0.conda#a0bc3eec34b0fab84be6b2da94e98e20 +https://conda.anaconda.org/conda-forge/noarch/pluggy-1.4.0-pyhd8ed1ab_0.conda#139e9feb65187e916162917bb2484976 +https://conda.anaconda.org/conda-forge/linux-64/psutil-5.9.8-py311h459d7ec_0.conda#9bc62d25dcf64eec484974a3123c9d57 https://conda.anaconda.org/conda-forge/noarch/ptyprocess-0.7.0-pyhd3deb0d_0.tar.bz2#359eeb6536da0e687af562ed265ec263 https://conda.anaconda.org/conda-forge/noarch/pure_eval-0.2.2-pyhd8ed1ab_0.tar.bz2#6784285c7e55cb7212efabc79e4c2883 -https://conda.anaconda.org/conda-forge/noarch/py-1.11.0-pyh6c4a22f_0.tar.bz2#b4613d7e7a493916d867842a6a148054 https://conda.anaconda.org/conda-forge/noarch/pycodestyle-2.9.1-pyhd8ed1ab_0.tar.bz2#0191dd7efe1a94262812770183b68892 https://conda.anaconda.org/conda-forge/noarch/pycparser-2.21-pyhd8ed1ab_0.tar.bz2#076becd9e05608f8dc72757d5f3a91ff https://conda.anaconda.org/conda-forge/noarch/pyflakes-2.5.0-pyhd8ed1ab_0.tar.bz2#1b3bef4313288ae8d35b1dfba4cd84a3 -https://conda.anaconda.org/conda-forge/noarch/pygments-2.16.1-pyhd8ed1ab_0.conda#40e5cb18165466773619e5c963f00a7b +https://conda.anaconda.org/conda-forge/noarch/pygments-2.17.2-pyhd8ed1ab_0.conda#140a7f159396547e9799aa98f9f0742e https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.1.1-pyhd8ed1ab_0.conda#176f7d56f0cfe9008bdf1bccd7de02fb https://conda.anaconda.org/conda-forge/noarch/pyshp-2.3.1-pyhd8ed1ab_0.tar.bz2#92a889dc236a5197612bc85bee6d7174 https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha2e5f31_6.tar.bz2#2a7de29fb590ca14b5243c4c812c8025 -https://conda.anaconda.org/conda-forge/noarch/python-fastjsonschema-2.18.1-pyhd8ed1ab_0.conda#305141cff54af2f90e089d868fffce28 -https://conda.anaconda.org/conda-forge/noarch/python-tzdata-2023.3-pyhd8ed1ab_0.conda#2590495f608a63625e165915fb4e2e34 +https://conda.anaconda.org/conda-forge/noarch/python-fastjsonschema-2.19.1-pyhd8ed1ab_0.conda#4d3ceee3af4b0f9a1f48f57176bf8625 +https://conda.anaconda.org/conda-forge/noarch/python-tzdata-2023.4-pyhd8ed1ab_0.conda#c79cacf8a06a51552fc651652f170208 https://conda.anaconda.org/conda-forge/linux-64/python-xxhash-3.4.1-py311h459d7ec_0.conda#60b5332b3989fda37884b92c7afd6a91 -https://conda.anaconda.org/conda-forge/noarch/pytz-2023.3.post1-pyhd8ed1ab_0.conda#c93346b446cd08c169d843ae5fc0da97 +https://conda.anaconda.org/conda-forge/noarch/pytz-2024.1-pyhd8ed1ab_0.conda#3eeeeb9e4827ace8c0c1419c85d590ad https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0.1-py311h459d7ec_1.conda#52719a74ad130de8fb5d047dc91f247a -https://conda.anaconda.org/conda-forge/linux-64/pyzmq-25.1.1-py311h75c88c4_1.conda#b858421f6a3052950c33aecd44a905cb -https://conda.anaconda.org/conda-forge/linux-64/rpds-py-0.10.6-py311h46250e7_0.conda#b7237977dbca779aaaab65f42f5558d9 +https://conda.anaconda.org/conda-forge/linux-64/pyzmq-25.1.2-py311h34ded2d_0.conda#819aa640a0493d4b52faf938e94d129e +https://conda.anaconda.org/conda-forge/linux-64/rpds-py-0.17.1-py311h46250e7_0.conda#a206e8c500a27fa82adae7c2f1929675 https://conda.anaconda.org/conda-forge/noarch/semver-3.0.2-pyhd8ed1ab_0.conda#5efb3fccda53974aed800b6d575f72ed https://conda.anaconda.org/conda-forge/noarch/setoptconf-tmp-0.3.1-pyhd8ed1ab_0.tar.bz2#af3e36d4effb85b9b9f93cd1db0963df -https://conda.anaconda.org/conda-forge/noarch/setuptools-68.2.2-pyhd8ed1ab_0.conda#fc2166155db840c634a1291a5c35a709 +https://conda.anaconda.org/conda-forge/noarch/setuptools-69.0.3-pyhd8ed1ab_0.conda#40695fdfd15a92121ed2922900d0308b https://conda.anaconda.org/conda-forge/noarch/six-1.16.0-pyh6c4a22f_0.tar.bz2#e5f25f8dbc060e9a8d912e432202afc2 https://conda.anaconda.org/conda-forge/noarch/smmap-5.0.0-pyhd8ed1ab_0.tar.bz2#62f26a3d1387acee31322208f0cfa3e0 https://conda.anaconda.org/conda-forge/noarch/snowballstemmer-2.2.0-pyhd8ed1ab_0.tar.bz2#4d22a9315e78c6827f806065957d566e @@ -241,191 +239,198 @@ https://conda.anaconda.org/conda-forge/noarch/sortedcontainers-2.4.0-pyhd8ed1ab_ https://conda.anaconda.org/conda-forge/noarch/soupsieve-2.5-pyhd8ed1ab_1.conda#3f144b2c34f8cb5a9abd9ed23a39c561 https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-jsmath-1.0.1-pyhd8ed1ab_0.conda#da1d979339e2714c30a8e806a33ec087 https://conda.anaconda.org/conda-forge/noarch/sqlparse-0.4.4-pyhd8ed1ab_0.conda#2e2f31b3b1c866c29636377e14f8c4c6 -https://conda.anaconda.org/conda-forge/noarch/tblib-2.0.0-pyhd8ed1ab_0.conda#f5580336fe091d46f9a2ea97da044550 -https://conda.anaconda.org/conda-forge/noarch/termcolor-2.3.0-pyhd8ed1ab_0.conda#440d508f025b1692168caaf436504af3 +https://conda.anaconda.org/conda-forge/noarch/tblib-3.0.0-pyhd8ed1ab_0.conda#04eedddeb68ad39871c8127dd1c21f4f +https://conda.anaconda.org/conda-forge/noarch/termcolor-2.4.0-pyhd8ed1ab_0.conda#a5033708ad9283907c3b1bc1f90d0d0d https://conda.anaconda.org/conda-forge/noarch/toml-0.10.2-pyhd8ed1ab_0.tar.bz2#f832c45a477c78bebd107098db465095 https://conda.anaconda.org/conda-forge/noarch/tomli-2.0.1-pyhd8ed1ab_0.tar.bz2#5844808ffab9ebdb694585b50ba02a96 -https://conda.anaconda.org/conda-forge/noarch/tomlkit-0.12.1-pyha770c72_0.conda#62f5b331c53d73e2f6c4c130b53518a0 -https://conda.anaconda.org/conda-forge/noarch/toolz-0.12.0-pyhd8ed1ab_0.tar.bz2#92facfec94bc02d6ccf42e7173831a36 +https://conda.anaconda.org/conda-forge/noarch/tomlkit-0.12.3-pyha770c72_0.conda#074d0ce7a6261ab8b497c3518796ef3e +https://conda.anaconda.org/conda-forge/noarch/toolz-0.12.1-pyhd8ed1ab_0.conda#2fcb582444635e2c402e8569bb94e039 https://conda.anaconda.org/conda-forge/linux-64/tornado-6.3.3-py311h459d7ec_1.conda#a700fcb5cedd3e72d0c75d095c7a6eda -https://conda.anaconda.org/conda-forge/noarch/traitlets-5.11.2-pyhd8ed1ab_0.conda#bd3f90f7551e1cffb1f402880eb2cef1 +https://conda.anaconda.org/conda-forge/noarch/traitlets-5.14.1-pyhd8ed1ab_0.conda#1c6acfdc7ecbfe09954c4216da99c146 https://conda.anaconda.org/conda-forge/noarch/types-pyyaml-6.0.12.12-pyhd8ed1ab_0.conda#0cb14c80f66937df894d60626dd1921f -https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.8.0-pyha770c72_0.conda#5b1be40a26d10a06f6d4f1f9e19fa0c7 -https://conda.anaconda.org/conda-forge/linux-64/ujson-5.8.0-py311hb755f60_0.conda#91e67c62c48444e4efc08fb61835abe8 +https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.9.0-pyha770c72_0.conda#a92a6440c3fe7052d63244f3aba2a4a7 +https://conda.anaconda.org/conda-forge/linux-64/ujson-5.9.0-py311hb755f60_0.conda#36dda52dc99a4fb9cadd3b738ec24848 https://conda.anaconda.org/conda-forge/noarch/untokenize-0.1.1-py_0.tar.bz2#1447ead40f2a01733a9c8dfc32988375 +https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.2.13-pyhd8ed1ab_0.conda#68f0738df502a14213624b288c60c9ad https://conda.anaconda.org/conda-forge/noarch/webencodings-0.5.1-pyhd8ed1ab_2.conda#daf5160ff9cde3a468556965329085b9 https://conda.anaconda.org/conda-forge/noarch/webob-1.8.7-pyhd8ed1ab_0.tar.bz2#a8192f3585f341ea66c60c189580ac67 -https://conda.anaconda.org/conda-forge/noarch/wheel-0.41.2-pyhd8ed1ab_0.conda#1ccd092478b3e0ee10d7a891adbf8a4f -https://conda.anaconda.org/conda-forge/linux-64/wrapt-1.15.0-py311h459d7ec_1.conda#f4d770a09066aaa313b5cc22c0f6e9d1 +https://conda.anaconda.org/conda-forge/noarch/wheel-0.42.0-pyhd8ed1ab_0.conda#1cdea58981c5cbc17b51973bcaddcea7 +https://conda.anaconda.org/conda-forge/linux-64/wrapt-1.16.0-py311h459d7ec_0.conda#6669b5529d206c1f880b642cdd17ae05 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.4-h0b41bf4_2.conda#82b6df12252e6f32402b96dacc656fec https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.11-hd590300_0.conda#ed67c36f215b310412b2af935bf3e530 -https://conda.anaconda.org/conda-forge/noarch/xyzservices-2023.10.0-pyhd8ed1ab_0.conda#9c6fe7db9c9133ade38b9a5011103243 +https://conda.anaconda.org/conda-forge/noarch/xyzservices-2023.10.1-pyhd8ed1ab_0.conda#1e0d85c0e2fef9539218da185b285f54 https://conda.anaconda.org/conda-forge/noarch/zict-3.0.0-pyhd8ed1ab_0.conda#cf30c2c15b82aacb07f9c09e28ff2275 https://conda.anaconda.org/conda-forge/noarch/zipp-3.17.0-pyhd8ed1ab_0.conda#2e4d6bc0b14e10f895fc6791a7d9b26a https://conda.anaconda.org/conda-forge/noarch/accessible-pygments-0.0.4-pyhd8ed1ab_0.conda#46a2e6e3dfa718ce3492018d5a110dd6 https://conda.anaconda.org/conda-forge/noarch/asgiref-3.7.2-pyhd8ed1ab_0.conda#596932155bf88bb6837141550cb721b0 https://conda.anaconda.org/conda-forge/linux-64/astroid-2.15.8-py311h38be061_0.conda#46d70fcb74472aab178991f0231ee3c6 -https://conda.anaconda.org/conda-forge/noarch/asttokens-2.4.0-pyhd8ed1ab_0.conda#056f04e51dd63337e8d7c425c18c86f1 -https://conda.anaconda.org/conda-forge/linux-64/aws-c-auth-0.7.4-hdf831e0_4.conda#f2b788b963b846fbfc322787bc79fe4c -https://conda.anaconda.org/conda-forge/linux-64/aws-c-mqtt-0.9.7-h6e0f915_1.conda#976997418755ab56d0cc3b4c93e10ebd -https://conda.anaconda.org/conda-forge/noarch/babel-2.13.0-pyhd8ed1ab_0.conda#22541af7a9eb59fc6afcadb7ecdf9219 -https://conda.anaconda.org/conda-forge/noarch/backports.functools_lru_cache-1.6.5-pyhd8ed1ab_0.conda#6b1b907661838a75d067a22f87996b2e -https://conda.anaconda.org/conda-forge/noarch/beautifulsoup4-4.12.2-pyha770c72_0.conda#a362ff7d976217f8fa78c0f1c4f59717 +https://conda.anaconda.org/conda-forge/noarch/asttokens-2.4.1-pyhd8ed1ab_0.conda#5f25798dcefd8252ce5f9dc494d5f571 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-auth-0.7.15-h70caa3e_0.conda#ac982c47c6439386e10afb4e0fe0d3e0 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-mqtt-0.10.1-h0ef3971_3.conda#5f80f11865fad4cc684f1007170df6ec +https://conda.anaconda.org/conda-forge/linux-64/azure-core-cpp-1.10.3-h91d86a7_1.conda#c05a913b8203d14b4a91c54d57b52282 +https://conda.anaconda.org/conda-forge/noarch/babel-2.14.0-pyhd8ed1ab_0.conda#9669586875baeced8fc30c0826c3270e +https://conda.anaconda.org/conda-forge/noarch/beautifulsoup4-4.12.3-pyha770c72_0.conda#332493000404d8411859539a5a630865 https://conda.anaconda.org/conda-forge/noarch/bleach-6.1.0-pyhd8ed1ab_0.conda#0ed9d7c0e9afa7c025807a9a8136ea3e https://conda.anaconda.org/conda-forge/linux-64/cairo-1.18.0-h3faef2a_0.conda#f907bb958910dc404647326ca80c263e -https://conda.anaconda.org/conda-forge/noarch/cattrs-23.1.2-pyhd8ed1ab_0.conda#e554f60477143949704bf470f66a81e7 https://conda.anaconda.org/conda-forge/linux-64/cffi-1.16.0-py311hb3a22ac_0.conda#b3469563ac5e808b0cd92810d0697043 -https://conda.anaconda.org/conda-forge/linux-64/cfitsio-4.3.0-hbdc6101_0.conda#797554b8b7603011e8677884381fbcc5 +https://conda.anaconda.org/conda-forge/linux-64/cfitsio-4.3.1-hbdc6101_0.conda#dcea02841b33a9c49f74ca9328de919a https://conda.anaconda.org/conda-forge/noarch/click-plugins-1.1.1-py_0.tar.bz2#4fd2c6b53934bd7d96d1f3fdaf99b79f https://conda.anaconda.org/conda-forge/noarch/cligj-0.7.2-pyhd8ed1ab_1.tar.bz2#a29b7c141d6b2de4bb67788a5f107734 -https://conda.anaconda.org/conda-forge/linux-64/coverage-7.3.2-py311h459d7ec_0.conda#7b3145fed7adc7c63a0e08f6f29f5480 -https://conda.anaconda.org/conda-forge/linux-64/cxx-compiler-1.6.0-h00ab1b0_0.conda#364c6ae36c4e36fcbd4d273cf4db78af -https://conda.anaconda.org/conda-forge/linux-64/cytoolz-0.12.2-py311h459d7ec_1.conda#afe341dbe834ae76d2c23157ff00e633 +https://conda.anaconda.org/conda-forge/linux-64/coverage-7.4.1-py311h459d7ec_0.conda#9caf3270065a2d40fd9a443ba1568e96 +https://conda.anaconda.org/conda-forge/linux-64/cxx-compiler-1.7.0-h00ab1b0_0.conda#b4537c98cb59f8725b0e1e65816b4a28 +https://conda.anaconda.org/conda-forge/linux-64/cytoolz-0.12.3-py311h459d7ec_0.conda#13d385f635d7fbe9acc93600f67a6cb4 https://conda.anaconda.org/conda-forge/noarch/docformatter-1.7.5-pyhd8ed1ab_0.conda#3a941b6083e945aa87e739a9b85c82e9 https://conda.anaconda.org/conda-forge/noarch/fire-0.5.0-pyhd8ed1ab_0.conda#9fd22aae8d2f319e80f68b295ab91d64 -https://conda.anaconda.org/conda-forge/linux-64/fonttools-4.43.1-py311h459d7ec_0.conda#ac995b680de3bdce2531c553b27dfe7e -https://conda.anaconda.org/conda-forge/linux-64/fortran-compiler-1.6.0-heb67821_0.conda#b65c49dda97ae497abcbdf3a8ba0018f -https://conda.anaconda.org/conda-forge/noarch/geopy-2.4.0-pyhd8ed1ab_0.conda#90faaa7eaeba3cc877074c0916efe30c +https://conda.anaconda.org/conda-forge/linux-64/fonttools-4.48.1-py311h459d7ec_0.conda#36363685b6e56682b1b256eb0ad503f6 +https://conda.anaconda.org/conda-forge/linux-64/fortran-compiler-1.7.0-heb67821_0.conda#7ef7c0f111dad1c8006504a0f1ccd820 +https://conda.anaconda.org/conda-forge/noarch/geopy-2.4.1-pyhd8ed1ab_0.conda#c75621ce68f6570fff9a6734cf21c9a7 https://conda.anaconda.org/conda-forge/noarch/gitdb-4.0.11-pyhd8ed1ab_0.conda#623b19f616f2ca0c261441067e18ae40 -https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.14.2-nompi_h4f84152_100.conda#2de6a9bc8083b49f09b2f6eb28d3ba3c -https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-6.8.0-pyha770c72_0.conda#4e9f59a060c3be52bc4ddc46ee9b6946 -https://conda.anaconda.org/conda-forge/noarch/importlib_resources-6.1.0-pyhd8ed1ab_0.conda#48b0d98e0c0ec810d3ccc2a0926c8c0e +https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.14.3-nompi_h4f84152_100.conda#d471a5c3abc984b662d9bae3bb7fd8a5 +https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-7.0.1-pyha770c72_0.conda#746623a787e06191d80a2133e5daff17 +https://conda.anaconda.org/conda-forge/noarch/importlib_resources-6.1.1-pyhd8ed1ab_0.conda#3d5fa25cf42f3f32a12b2d874ace8574 https://conda.anaconda.org/conda-forge/noarch/isodate-0.6.1-pyhd8ed1ab_0.tar.bz2#4a62c93c1b5c0b920508ae3fd285eaf5 -https://conda.anaconda.org/conda-forge/noarch/isort-5.12.0-pyhd8ed1ab_1.conda#07ed3421bad60867234c7a9282ea39d4 +https://conda.anaconda.org/conda-forge/noarch/isort-5.13.2-pyhd8ed1ab_0.conda#1d25ed2b95b92b026aaa795eabec8d91 https://conda.anaconda.org/conda-forge/noarch/jedi-0.19.1-pyhd8ed1ab_0.conda#81a3be0b2023e1ea8555781f0ad904a2 -https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.2-pyhd8ed1ab_1.tar.bz2#c8490ed5c70966d232fdd389d0dbed37 -https://conda.anaconda.org/conda-forge/noarch/jupyterlab_pygments-0.2.2-pyhd8ed1ab_0.tar.bz2#243f63592c8e449f40cd42eb5cf32f40 +https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.3-pyhd8ed1ab_0.conda#e7d8df6509ba635247ff9aea31134262 +https://conda.anaconda.org/conda-forge/linux-64/jupyter_core-5.7.1-py311h38be061_0.conda#175a430872841f7c351879f4c4c85b9e +https://conda.anaconda.org/conda-forge/noarch/jupyterlab_pygments-0.3.0-pyhd8ed1ab_1.conda#afcd1b53bcac8844540358e33f33d28f https://conda.anaconda.org/conda-forge/noarch/latexcodec-2.0.1-pyh9f0ad1d_0.tar.bz2#8d67904973263afd2985ba56aa2d6bb4 -https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-19_linux64_openblas.conda#d12374af44575413fbbd4a217d46ea33 +https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-21_linux64_openblas.conda#4a3816d06451c4946e2db26b86472cb6 https://conda.anaconda.org/conda-forge/linux-64/libgd-2.3.3-h119a65a_9.conda#cfebc557e54905dadc355c0e9f003004 -https://conda.anaconda.org/conda-forge/linux-64/libgoogle-cloud-2.12.0-h19a6dae_3.conda#cb26f6b7184480053106ea4713a52daf -https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-19_linux64_openblas.conda#9f100edf65436e3eabc2a51fc00b2c37 +https://conda.anaconda.org/conda-forge/linux-64/libgoogle-cloud-2.12.0-hef10d8f_5.conda#055e2266d27f0e2290cf0a6ad668a225 +https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-21_linux64_openblas.conda#1a42f305615c3867684e049e85927531 https://conda.anaconda.org/conda-forge/noarch/logilab-common-1.7.3-py_0.tar.bz2#6eafcdf39a7eb90b6d951cfff59e8d3b https://conda.anaconda.org/conda-forge/noarch/matplotlib-inline-0.1.6-pyhd8ed1ab_0.tar.bz2#b21613793fcc81d944c76c9f2864a7de -https://conda.anaconda.org/conda-forge/linux-64/mypy-1.6.1-py311h459d7ec_0.conda#d3e5072b7ff746e9f9b44f0f52de6727 +https://conda.anaconda.org/conda-forge/linux-64/mypy-1.8.0-py311h459d7ec_0.conda#93b7b2391a045cea0d97772f550f1d77 https://conda.anaconda.org/conda-forge/noarch/nested-lookup-0.2.25-pyhd8ed1ab_1.tar.bz2#2f59daeb14581d41b1e2dda0895933b2 https://conda.anaconda.org/conda-forge/noarch/nodeenv-1.8.0-pyhd8ed1ab_0.conda#2a75b296096adabbabadd5e9782e5fcc https://conda.anaconda.org/conda-forge/noarch/partd-1.4.1-pyhd8ed1ab_0.conda#acf4b7c0bcd5fa3b0e05801c4d2accd6 -https://conda.anaconda.org/conda-forge/noarch/pexpect-4.8.0-pyh1a96a4e_2.tar.bz2#330448ce4403cc74990ac07c555942a1 -https://conda.anaconda.org/conda-forge/linux-64/pillow-10.1.0-py311ha6c5da5_0.conda#83a988daf5c49e57f7d2086fb6781fe8 -https://conda.anaconda.org/conda-forge/noarch/pip-23.3-pyhd8ed1ab_0.conda#a06f102f59c8e3bb8b3e46e71c384709 -https://conda.anaconda.org/conda-forge/linux-64/postgresql-16.0-h8972f4a_1.conda#6ce1ab5480d3aa4308654971ac5f731b -https://conda.anaconda.org/conda-forge/linux-64/proj-9.3.0-h1d62c97_2.conda#b5e57a0c643da391bef850922963eece +https://conda.anaconda.org/conda-forge/noarch/pexpect-4.9.0-pyhd8ed1ab_0.conda#629f3203c99b32e0988910c93e77f3b6 +https://conda.anaconda.org/conda-forge/linux-64/pillow-10.2.0-py311ha6c5da5_0.conda#a5ccd7f2271f28b7d2de0b02b64e3796 +https://conda.anaconda.org/conda-forge/noarch/pip-24.0-pyhd8ed1ab_0.conda#f586ac1e56c8638b64f9c8122a7b8a67 +https://conda.anaconda.org/conda-forge/linux-64/postgresql-16.2-h7387d8b_0.conda#4e86738066b4966f0357f661b3691cae +https://conda.anaconda.org/conda-forge/linux-64/proj-9.3.1-h1d62c97_0.conda#44ec51d0857d9be26158bb85caa74fdb +https://conda.anaconda.org/conda-forge/noarch/prompt-toolkit-3.0.42-pyha770c72_0.conda#0bf64bf10eee21f46ac83c161917fa86 https://conda.anaconda.org/conda-forge/noarch/pydocstyle-6.3.0-pyhd8ed1ab_0.conda#7e23a61a7fbaedfef6eb0e1ac775c8e5 -https://conda.anaconda.org/conda-forge/noarch/pytest-7.4.2-pyhd8ed1ab_0.conda#6dd662ff5ac9a783e5c940ce9f3fe649 +https://conda.anaconda.org/conda-forge/noarch/pytest-8.0.0-pyhd8ed1ab_0.conda#5ba1cc5b924226349d4a49fb547b7579 https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.8.2-pyhd8ed1ab_0.tar.bz2#dd999d1cc9f79e67dbb855c8924c7984 -https://conda.anaconda.org/conda-forge/noarch/referencing-0.30.2-pyhd8ed1ab_0.conda#a33161b983172ba6ef69d5fc850650cd +https://conda.anaconda.org/conda-forge/noarch/referencing-0.33.0-pyhd8ed1ab_0.conda#bc415a1c6cf049166215d6b596e0fcbe https://conda.anaconda.org/conda-forge/noarch/tinycss2-1.2.1-pyhd8ed1ab_0.tar.bz2#7234c9eefff659501cd2fe0d2ede4d48 -https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.8.0-hd8ed1ab_0.conda#384462e63262a527bda564fa2d9126c0 +https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.9.0-hd8ed1ab_0.conda#c16524c1b7227dc80b36b4fa6f77cc86 https://conda.anaconda.org/conda-forge/noarch/url-normalize-1.4.3-pyhd8ed1ab_0.tar.bz2#7c4076e494f0efe76705154ac9302ba6 -https://conda.anaconda.org/conda-forge/noarch/urllib3-2.0.7-pyhd8ed1ab_0.conda#270e71c14d37074b1d066ee21cf0c4a6 -https://conda.anaconda.org/conda-forge/linux-64/xerces-c-3.2.4-hac6953d_3.conda#297e6a75dc1b6a440cd341a85eab8a00 +https://conda.anaconda.org/conda-forge/noarch/urllib3-2.2.0-pyhd8ed1ab_0.conda#6a7e0694921f668a030d52f0c47baebd +https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.25.0-pyhd8ed1ab_0.conda#c119653cba436d8183c27bf6d190e587 +https://conda.anaconda.org/conda-forge/linux-64/xerces-c-3.2.5-hac6953d_0.conda#63b80ca78d29380fe69e69412dcbe4ac https://conda.anaconda.org/conda-forge/noarch/yamale-4.0.4-pyh6c4a22f_0.tar.bz2#cc9f59f147740d88679bf1bd94dbe588 -https://conda.anaconda.org/conda-forge/noarch/yamllint-1.32.0-pyhd8ed1ab_0.conda#6d2425548b0293a225ca4febd80feaa3 -https://conda.anaconda.org/conda-forge/linux-64/aws-c-s3-0.3.18-hd871755_2.conda#dd4fc5bb1ec65369c6291e0a311bb31d -https://conda.anaconda.org/conda-forge/linux-64/compilers-1.6.0-ha770c72_0.conda#e2259de4640a51a28c21931ae98e4975 -https://conda.anaconda.org/conda-forge/linux-64/cryptography-41.0.4-py311h63ff55d_0.conda#2b14cd05541532521196b0d2e0291ecf -https://conda.anaconda.org/conda-forge/noarch/django-4.2.6-pyhd8ed1ab_0.conda#bebc7fbd0b86a4e28a931c306cd5324e +https://conda.anaconda.org/conda-forge/noarch/yamllint-1.34.0-pyhd8ed1ab_0.conda#262273faaed1e4bdf0c61209c38abd15 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-s3-0.5.0-h1b46bed_2.conda#cbbdaaec72d302636a64a3fcaa3a72c7 +https://conda.anaconda.org/conda-forge/linux-64/azure-storage-common-cpp-12.5.0-hb858b4b_2.conda#19f23b45d1925a9a8f701a3f6f9cce4f +https://conda.anaconda.org/conda-forge/noarch/cattrs-23.2.3-pyhd8ed1ab_0.conda#91fc4700dcce4a46d439900a132fe4e5 +https://conda.anaconda.org/conda-forge/linux-64/compilers-1.7.0-ha770c72_0.conda#81458b3aed8ab8711951ec3c0c04e097 +https://conda.anaconda.org/conda-forge/linux-64/cryptography-42.0.2-py311hcb13ee4_0.conda#c61fd9e9fcfa599ea5a8b1de42b147a8 +https://conda.anaconda.org/conda-forge/noarch/django-5.0.2-pyhd8ed1ab_0.conda#596031b6473e1fead388589348472748 https://conda.anaconda.org/conda-forge/noarch/flake8-5.0.4-pyhd8ed1ab_0.tar.bz2#8079ea7dec0a917dd0cb6c257f7ea9ea -https://conda.anaconda.org/conda-forge/linux-64/geotiff-1.7.1-hf074850_14.conda#1d53ee057d8481bd2b4c2c34c8e92aac -https://conda.anaconda.org/conda-forge/noarch/gitpython-3.1.40-pyhd8ed1ab_0.conda#6bf74c3b7c13079a91d4bd3da51cefcf -https://conda.anaconda.org/conda-forge/linux-64/harfbuzz-8.2.1-h3d44ed6_0.conda#98db5f8813f45e2b29766aff0e4a499c -https://conda.anaconda.org/conda-forge/noarch/importlib_metadata-6.8.0-hd8ed1ab_0.conda#b279b07ce18058034e5b3606ba103a8b -https://conda.anaconda.org/conda-forge/noarch/jsonschema-specifications-2023.7.1-pyhd8ed1ab_0.conda#7c27ea1bdbe520bb830dcadd59f55cbf -https://conda.anaconda.org/conda-forge/linux-64/kealib-1.5.2-hcd42e92_1.conda#b04c039f0bd511533a0d8bc8a7b6835e -https://conda.anaconda.org/conda-forge/linux-64/libnetcdf-4.9.2-nompi_h80fb2b6_112.conda#a19fa6cacf80c8a366572853d5890eb4 -https://conda.anaconda.org/conda-forge/linux-64/libspatialite-5.1.0-h090f1da_0.conda#c4360eaa543bb3bcbb9cd135eb6fb0fc -https://conda.anaconda.org/conda-forge/linux-64/numpy-1.26.0-py311h64a7726_0.conda#bf16a9f625126e378302f08e7ed67517 -https://conda.anaconda.org/conda-forge/noarch/platformdirs-3.11.0-pyhd8ed1ab_0.conda#8f567c0a74aa44cf732f15773b4083b0 -https://conda.anaconda.org/conda-forge/linux-64/poppler-23.10.0-h590f24d_0.conda#06b4a80e2cc3974e55f83e2115e2e90a +https://conda.anaconda.org/conda-forge/linux-64/geotiff-1.7.1-h6b2125f_15.conda#218a726155bd9ae1787b26054eed8566 +https://conda.anaconda.org/conda-forge/noarch/gitpython-3.1.41-pyhd8ed1ab_0.conda#84874a90c312088f7b5e63402fc44a58 +https://conda.anaconda.org/conda-forge/linux-64/harfbuzz-8.3.0-h3d44ed6_0.conda#5a6f6c00ef982a9bc83558d9ac8f64a0 +https://conda.anaconda.org/conda-forge/noarch/importlib_metadata-7.0.1-hd8ed1ab_0.conda#4a2f43a20fa404b998859c6a470ba316 +https://conda.anaconda.org/conda-forge/noarch/jsonschema-specifications-2023.12.1-pyhd8ed1ab_0.conda#a0e4efb5f35786a05af4809a2fb1f855 +https://conda.anaconda.org/conda-forge/linux-64/kealib-1.5.3-h2f55d51_0.conda#f7e7077802927590efc8bf7328208f12 +https://conda.anaconda.org/conda-forge/linux-64/libnetcdf-4.9.2-nompi_h9612171_113.conda#b2414908e43c442ddc68e6148774a304 +https://conda.anaconda.org/conda-forge/linux-64/libspatialite-5.1.0-h7bd4643_4.conda#127d36f9ee392fa81b45e81867ce30ab +https://conda.anaconda.org/conda-forge/linux-64/numpy-1.26.4-py311h64a7726_0.conda#a502d7aad449a1206efb366d6a12c52d +https://conda.anaconda.org/conda-forge/linux-64/poppler-24.02.0-h590f24d_0.conda#7e715c1572de09d6106c5a31fa70ffca https://conda.anaconda.org/conda-forge/noarch/pybtex-0.24.0-pyhd8ed1ab_2.tar.bz2#2099b86a7399c44c0c61cdb6de6915ba -https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.6.1-py311h1facc83_2.conda#8298afb85a731b02dac82e02b6e13ae0 +https://conda.anaconda.org/conda-forge/noarch/pylint-2.17.7-pyhd8ed1ab_0.conda#3cab6aee60038b3f621bce3e50f52bed +https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.6.1-py311hca0b8b9_5.conda#cac429fcb9126d5e6f02c8ba61c2a811 https://conda.anaconda.org/conda-forge/noarch/pytest-cov-4.1.0-pyhd8ed1ab_0.conda#06eb685a3a0b146347a58dda979485da -https://conda.anaconda.org/conda-forge/noarch/pytest-env-1.0.1-pyhd8ed1ab_0.conda#9da651d84c73bac482cae51613a4d4d6 -https://conda.anaconda.org/conda-forge/noarch/pytest-metadata-3.0.0-pyhd8ed1ab_1.conda#8bdcc0f401561213821bf67513abeeff +https://conda.anaconda.org/conda-forge/noarch/pytest-env-1.1.3-pyhd8ed1ab_0.conda#1dbdf019d740419852c4a7803fff49d9 +https://conda.anaconda.org/conda-forge/noarch/pytest-metadata-3.1.0-pyhd8ed1ab_0.conda#d87474b01a3e2b8e919a24b922463056 https://conda.anaconda.org/conda-forge/noarch/pytest-mock-3.12.0-pyhd8ed1ab_0.conda#ac9fedc9a0c397f2318e82525491dd83 -https://conda.anaconda.org/conda-forge/noarch/pytest-mypy-0.8.0-pyhd8ed1ab_0.tar.bz2#4e81c96e5f875c09e5b9f999035b9d8e -https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.3.1-pyhd8ed1ab_0.conda#816073bb54ef59f33f0f26c14f88311b +https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.5.0-pyhd8ed1ab_0.conda#d5f595da2daead898ca958ac62f0307b https://conda.anaconda.org/conda-forge/noarch/rdflib-7.0.0-pyhd8ed1ab_0.conda#44d14ef95495b3d4438f28998e0296a9 https://conda.anaconda.org/conda-forge/noarch/requests-2.31.0-pyhd8ed1ab_0.conda#a30144e4156cdbb236f99ebb49828f8b https://conda.anaconda.org/conda-forge/noarch/requirements-detector-1.2.2-pyhd8ed1ab_0.conda#6626918380d99292df110f3c91b6e5ec https://conda.anaconda.org/conda-forge/noarch/stack_data-0.6.2-pyhd8ed1ab_0.conda#e7df0fdd404616638df5ece6e69ba7af -https://conda.anaconda.org/conda-forge/linux-64/tiledb-2.16.3-h8c794c1_3.conda#7de728789b0aba16018f726dc5ddbec2 -https://conda.anaconda.org/conda-forge/noarch/types-requests-2.31.0.10-pyhd8ed1ab_0.conda#84dca7318667fa091bbfd8724d22ea99 +https://conda.anaconda.org/conda-forge/noarch/types-requests-2.31.0.20240125-pyhd8ed1ab_0.conda#0200f9f9bb98c12a172ae49ef7d83d60 https://conda.anaconda.org/conda-forge/linux-64/ukkonen-1.0.1-py311h9547e67_4.conda#586da7df03b68640de14dc3e8bcbf76f -https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.2.8-pyhd8ed1ab_0.conda#367386d2575a0e62412448eda1012efd -https://conda.anaconda.org/conda-forge/linux-64/aws-crt-cpp-0.24.3-ha4d64e0_3.conda#14f07bbda88ce2f97b71c2f047ca76ad -https://conda.anaconda.org/conda-forge/linux-64/cftime-1.6.2-py311h1f0f07a_2.conda#571c0c47e8dbcf03577935ac818b6696 -https://conda.anaconda.org/conda-forge/linux-64/contourpy-1.1.1-py311h9547e67_1.conda#52d3de443952d33c5cee6b24b172ce96 -https://conda.anaconda.org/conda-forge/noarch/dask-core-2023.10.0-pyhd8ed1ab_0.conda#6b415b2cd4a106b8eb1632be126ede81 +https://conda.anaconda.org/conda-forge/noarch/yapf-0.40.1-pyhd8ed1ab_0.conda#f269942e802d5e148632143d4c37acc9 +https://conda.anaconda.org/conda-forge/linux-64/aws-crt-cpp-0.26.1-h33f84b2_9.conda#feff02dd51629ad703677c28eaf03a1e +https://conda.anaconda.org/conda-forge/linux-64/azure-storage-blobs-cpp-12.10.0-h00ab1b0_0.conda#64eec459779f01803594f5272cdde23c +https://conda.anaconda.org/conda-forge/linux-64/cftime-1.6.3-py311h1f0f07a_0.conda#b7e6d52b39e199238c3400cafaabafb3 +https://conda.anaconda.org/conda-forge/linux-64/contourpy-1.2.0-py311h9547e67_0.conda#40828c5b36ef52433e21f89943e09f33 +https://conda.anaconda.org/conda-forge/noarch/dask-core-2024.2.0-pyhd8ed1ab_0.conda#5973bc565e2aea620c3a431cafdde032 https://conda.anaconda.org/conda-forge/noarch/flake8-polyfill-1.0.2-py_0.tar.bz2#a53db35e3d07f0af2eccd59c2a00bffe -https://conda.anaconda.org/conda-forge/noarch/identify-2.5.30-pyhd8ed1ab_0.conda#b7a2e3bb89bda8c69839485c20aabadf -https://conda.anaconda.org/conda-forge/noarch/jsonschema-4.19.1-pyhd8ed1ab_0.conda#78aff5d2af74e6537c1ca73017f01f4f -https://conda.anaconda.org/conda-forge/linux-64/jupyter_core-5.4.0-py311h38be061_0.conda#eb4ab2a92a2cc9715226382abb8ebaee -https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.7.2-h6f3d308_7.conda#f9555eada0412c9f3dd4b34f5afecf5b -https://conda.anaconda.org/conda-forge/linux-64/netcdf-fortran-4.6.1-nompi_hacb5139_102.conda#487a1c19dd3eacfd055ad614e9acde87 -https://conda.anaconda.org/conda-forge/linux-64/pandas-2.1.1-py311h320fe9a_1.conda#a4371a95a8ae703a22949af28467b93d +https://conda.anaconda.org/conda-forge/noarch/identify-2.5.34-pyhd8ed1ab_0.conda#048ba98aa7b16ef0d8866f8c87d7c5b8 +https://conda.anaconda.org/conda-forge/noarch/ipython-8.21.0-pyh707e725_0.conda#371344fdbdf9c70cfe9adb512a8cbca6 +https://conda.anaconda.org/conda-forge/noarch/jsonschema-4.21.1-pyhd8ed1ab_0.conda#8a3a3d01629da20befa340919e3dd2c4 +https://conda.anaconda.org/conda-forge/noarch/jupyter_client-8.6.0-pyhd8ed1ab_0.conda#6bd3f1069cdebb44c7ae9efb900e312d +https://conda.anaconda.org/conda-forge/linux-64/netcdf-fortran-4.6.1-nompi_hacb5139_103.conda#50f05f98d084805642d24dff910e11e8 +https://conda.anaconda.org/conda-forge/linux-64/pandas-2.1.4-py311h320fe9a_0.conda#e44ccb61b6621bf3f8053ae66eba7397 https://conda.anaconda.org/conda-forge/linux-64/pango-1.50.14-ha41ecd1_2.conda#1a66c10f6a0da3dbd2f3a68127e7f6a0 -https://conda.anaconda.org/conda-forge/noarch/pooch-1.7.0-pyhd8ed1ab_4.conda#3cdaf7af08850933662b1e228bc6b5bc -https://conda.anaconda.org/conda-forge/noarch/prompt-toolkit-3.0.39-pyha770c72_0.conda#a4986c6bb5b0d05a38855b0880a5f425 -https://conda.anaconda.org/conda-forge/noarch/pylint-2.17.7-pyhd8ed1ab_0.conda#3cab6aee60038b3f621bce3e50f52bed -https://conda.anaconda.org/conda-forge/noarch/pyopenssl-23.2.0-pyhd8ed1ab_1.conda#34f7d568bf59d18e3fef8c405cbece21 -https://conda.anaconda.org/conda-forge/noarch/pytest-html-3.2.0-pyhd8ed1ab_1.tar.bz2#d5c7a941dfbceaab4b172a56d7918eb0 -https://conda.anaconda.org/conda-forge/noarch/requests-cache-1.1.0-pyhd8ed1ab_0.conda#57b89064c125bb9d0e533e018c3eb17a -https://conda.anaconda.org/conda-forge/linux-64/scipy-1.11.3-py311h64a7726_1.conda#e4b4d3b764e2d029477d0db88248a8b5 -https://conda.anaconda.org/conda-forge/linux-64/shapely-2.0.2-py311he06c224_0.conda#c90e2469d7512f3bba893533a82d7a02 -https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.24.4-pyhd8ed1ab_0.conda#c3feaf947264a59a125e8c26e98c3c5a -https://conda.anaconda.org/conda-forge/noarch/yapf-0.40.1-pyhd8ed1ab_0.conda#f269942e802d5e148632143d4c37acc9 -https://conda.anaconda.org/conda-forge/linux-64/aws-sdk-cpp-1.11.182-h10f61e1_0.conda#bcdd6943a6be3aa15df3bfea9264e306 -https://conda.anaconda.org/conda-forge/noarch/bokeh-3.3.0-pyhd8ed1ab_0.conda#5d6ff9d18f0b611a7dc131f4a7444c2e +https://conda.anaconda.org/conda-forge/noarch/pooch-1.8.0-pyhd8ed1ab_0.conda#134b2b57b7865d2316a7cce1915a51ed +https://conda.anaconda.org/conda-forge/noarch/pylint-plugin-utils-0.7-pyhd8ed1ab_0.tar.bz2#1657976383aee04dbb3ae3bdf654bb58 +https://conda.anaconda.org/conda-forge/noarch/pyopenssl-24.0.0-pyhd8ed1ab_0.conda#b50aec2c744a5c493c09cce9e2e7533e +https://conda.anaconda.org/conda-forge/noarch/pytest-html-4.1.1-pyhd8ed1ab_0.conda#4d2040212307d18392a2687772b3a96d +https://conda.anaconda.org/conda-forge/noarch/requests-cache-1.1.1-pyhd8ed1ab_0.conda#29bf13210ee541c59166cea092b91080 +https://conda.anaconda.org/conda-forge/linux-64/scipy-1.12.0-py311h64a7726_2.conda#24ca5107ab75c5521067b8ba505dfae5 +https://conda.anaconda.org/conda-forge/linux-64/shapely-2.0.2-py311h2032efe_1.conda#4ba860ff851768615b1a25b788022750 +https://conda.anaconda.org/conda-forge/linux-64/aws-sdk-cpp-1.11.242-h65f022c_0.conda#09b53fbd76044de441d25261840821ac +https://conda.anaconda.org/conda-forge/noarch/bokeh-3.3.4-pyhd8ed1ab_0.conda#6cc92bba68b7bb5a3b180e96508f9480 https://conda.anaconda.org/conda-forge/linux-64/cf-units-3.2.0-py311h1f0f07a_4.conda#1e105c1a8ea2163507726144b401eb1b -https://conda.anaconda.org/conda-forge/noarch/distributed-2023.10.0-pyhd8ed1ab_0.conda#3ecca3ba0a497f06e697777464a2cae9 +https://conda.anaconda.org/conda-forge/noarch/distributed-2024.2.0-pyhd8ed1ab_0.conda#81c14e12f44f94613fe5922403e32341 https://conda.anaconda.org/conda-forge/linux-64/esmf-8.4.2-nompi_h9e768e6_3.conda#c330e87e698bae8e7381c0315cf25dd0 -https://conda.anaconda.org/conda-forge/linux-64/gdal-3.7.2-py311h815a124_7.conda#e68c90c9490e1621ff4a7871849f03b2 -https://conda.anaconda.org/conda-forge/linux-64/gtk2-2.24.33-h90689f9_2.tar.bz2#957a0255ab58aaf394a91725d73ab422 -https://conda.anaconda.org/conda-forge/noarch/jupyter_client-8.4.0-pyhd8ed1ab_0.conda#554496685357ab0d69676cab8e8fb594 -https://conda.anaconda.org/conda-forge/linux-64/librsvg-2.56.3-h98fae49_0.conda#620e754f4344f4c27259ff460a2b9c50 -https://conda.anaconda.org/conda-forge/linux-64/matplotlib-base-3.8.0-py311h54ef318_2.conda#5655371cc61b8c31c369a7e709acb294 -https://conda.anaconda.org/conda-forge/noarch/myproxyclient-2.1.0-pyhd8ed1ab_2.tar.bz2#363b0816e411feb0df925d4f224f026a +https://conda.anaconda.org/conda-forge/linux-64/gtk2-2.24.33-h7f000aa_3.conda#0abfa7f9241a0f4fd732bc15773cfb0c +https://conda.anaconda.org/conda-forge/linux-64/librsvg-2.56.3-he3f83f7_1.conda#03bd1ddcc942867a19528877143b9852 +https://conda.anaconda.org/conda-forge/linux-64/matplotlib-base-3.8.2-py311h54ef318_0.conda#9f80753bc008bfc9b95f39d9ff9f1694 +https://conda.anaconda.org/conda-forge/noarch/myproxyclient-2.1.1-pyhd8ed1ab_0.conda#bcdbeb2b693eba886583a907840c6421 https://conda.anaconda.org/conda-forge/noarch/nbformat-5.9.2-pyhd8ed1ab_0.conda#61ba076de6530d9301a0053b02f093d2 -https://conda.anaconda.org/conda-forge/linux-64/netcdf4-1.6.4-nompi_py311he8ad708_103.conda#97b45ba4ff4e46a07dd6c60040256538 +https://conda.anaconda.org/conda-forge/linux-64/netcdf4-1.6.5-nompi_py311he8ad708_100.conda#597b1ad6cb7011b7561c20ea30295cae https://conda.anaconda.org/conda-forge/noarch/pep8-naming-0.10.0-pyh9f0ad1d_0.tar.bz2#b3c5536e4f9f58a4b16adb6f1e11732d -https://conda.anaconda.org/conda-forge/noarch/pre-commit-3.5.0-pyha770c72_0.conda#964e3d762e427661c59263435a14c492 -https://conda.anaconda.org/conda-forge/noarch/prompt_toolkit-3.0.39-hd8ed1ab_0.conda#4bbbe67d5df19db30f04b8e344dc9976 -https://conda.anaconda.org/conda-forge/noarch/pylint-plugin-utils-0.7-pyhd8ed1ab_0.tar.bz2#1657976383aee04dbb3ae3bdf654bb58 +https://conda.anaconda.org/conda-forge/noarch/pre-commit-3.6.1-pyha770c72_0.conda#4efd2c755bf2079e5651e57e1999db6c +https://conda.anaconda.org/conda-forge/noarch/pylint-celery-0.3-py_1.tar.bz2#e29456a611a62d3f26105a2f9c68f759 +https://conda.anaconda.org/conda-forge/noarch/pylint-django-2.5.3-pyhd8ed1ab_0.tar.bz2#00d8853fb1f87195722ea6a582cc9b56 +https://conda.anaconda.org/conda-forge/noarch/pylint-flask-0.6-py_0.tar.bz2#5a9afd3d0a61b08d59eed70fab859c1b https://conda.anaconda.org/conda-forge/linux-64/python-stratify-0.3.0-py311h1f0f07a_1.conda#cd36a89a048ad2bcc6d8b43f648fb1d0 -https://conda.anaconda.org/conda-forge/noarch/xarray-2023.10.1-pyhd8ed1ab_0.conda#9b20e5d68eea6878a0a6fc57a3043889 -https://conda.anaconda.org/conda-forge/linux-64/cartopy-0.22.0-py311h320fe9a_0.conda#1271b2375735e2aaa6d6770dbe2ad087 -https://conda.anaconda.org/conda-forge/noarch/cf_xarray-0.8.4-pyhd8ed1ab_0.conda#18472f8f9452f962fe0bcb1b8134b494 +https://conda.anaconda.org/conda-forge/linux-64/tiledb-2.19.1-h4386cac_0.conda#8d16e7b2529607a12aa6722c7a7c7356 +https://conda.anaconda.org/conda-forge/noarch/xarray-2024.1.1-pyhd8ed1ab_0.conda#38b5de3877d1f28089b231d24622dd64 +https://conda.anaconda.org/conda-forge/linux-64/cartopy-0.22.0-py311h320fe9a_1.conda#10d1806e20da040c58c36deddf51c70c +https://conda.anaconda.org/conda-forge/noarch/cf_xarray-0.8.9-pyhd8ed1ab_0.conda#f9f82c2c3d1b3588e8ab34976f98af91 https://conda.anaconda.org/conda-forge/noarch/dask-jobqueue-0.8.2-pyhd8ed1ab_0.conda#cc344a296a41369bcb05f7216661cec8 -https://conda.anaconda.org/conda-forge/noarch/esgf-pyclient-0.3.1-pyh1a96a4e_2.tar.bz2#64068564a9c2932bf30e9b4ec567927d +https://conda.anaconda.org/conda-forge/noarch/esgf-pyclient-0.3.1-pyhca7485f_3.conda#1d43833138d38ad8324700ce45a7099a https://conda.anaconda.org/conda-forge/noarch/esmpy-8.4.2-pyhc1e730c_4.conda#ddcf387719b2e44df0cc4dd467643951 -https://conda.anaconda.org/conda-forge/linux-64/fiona-1.9.5-py311hbac4ec9_0.conda#786d3808394b1bdfd3f41f2e2c67279e -https://conda.anaconda.org/conda-forge/linux-64/graphviz-8.1.0-h28d9a01_0.conda#33628e0e3de7afd2c8172f76439894cb -https://conda.anaconda.org/conda-forge/noarch/ipython-8.16.1-pyh0d859eb_0.conda#7e52cb0dbf01b90365bfe433ec8bd3c0 -https://conda.anaconda.org/conda-forge/linux-64/libarrow-13.0.0-he1fec18_11_cpu.conda#4bbdac31e0b1fb5299beb9b92f7abccd +https://conda.anaconda.org/conda-forge/linux-64/graphviz-9.0.0-h78e8752_1.conda#a3f4cd4a512ec5db35ffbf25ba11f537 +https://conda.anaconda.org/conda-forge/linux-64/libarrow-15.0.0-he2c5238_2_cpu.conda#cd7cd1c21dc42befdbb44b5afe2cd048 +https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.8.3-h80d7d79_2.conda#b0c8bd68d3119c8e3a355ff9417ceff1 https://conda.anaconda.org/conda-forge/noarch/nbclient-0.8.0-pyhd8ed1ab_0.conda#e78da91cf428faaf05701ce8cc8f2f9b https://conda.anaconda.org/conda-forge/noarch/nc-time-axis-1.4.1-pyhd8ed1ab_0.tar.bz2#281b58948bf60a2582de9e548bcc5369 -https://conda.anaconda.org/conda-forge/noarch/pylint-celery-0.3-py_1.tar.bz2#e29456a611a62d3f26105a2f9c68f759 -https://conda.anaconda.org/conda-forge/noarch/pylint-django-2.5.3-pyhd8ed1ab_0.tar.bz2#00d8853fb1f87195722ea6a582cc9b56 -https://conda.anaconda.org/conda-forge/noarch/pylint-flask-0.6-py_0.tar.bz2#5a9afd3d0a61b08d59eed70fab859c1b -https://conda.anaconda.org/conda-forge/noarch/iris-3.7.0-pyha770c72_0.conda#dccc1f660bf455c239adaabf56b91dc9 -https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.9.2-pyhd8ed1ab_0.conda#01e4314c780ca73759c694ce3ece281f https://conda.anaconda.org/conda-forge/noarch/prospector-1.10.3-pyhd8ed1ab_0.conda#f551d4d859a1d70c6abff8310a655481 -https://conda.anaconda.org/conda-forge/noarch/py-cordex-0.6.4-pyhd8ed1ab_0.conda#a30ffe78645b5777aa21d3e0f2d2a015 -https://conda.anaconda.org/conda-forge/linux-64/pyarrow-13.0.0-py311h39c9aba_11_cpu.conda#1d1fcce6ad64a319f6c8ad217313f52d -https://conda.anaconda.org/conda-forge/linux-64/pydot-1.4.2-py311h38be061_4.conda#5c223cb0d9c05552bf9d1586a92720b2 -https://conda.anaconda.org/conda-forge/noarch/dask-2023.10.0-pyhd8ed1ab_0.conda#0c85f5482dd283c2b9ca68d331edb471 -https://conda.anaconda.org/conda-forge/noarch/nbconvert-pandoc-7.9.2-pyhd8ed1ab_0.conda#1ad46253f2eb46fb8c9b2c22b7ca012f +https://conda.anaconda.org/conda-forge/linux-64/gdal-3.8.3-py311h8be719e_2.conda#4b587b3a15d00f6ec7fdecfe2d805140 +https://conda.anaconda.org/conda-forge/noarch/iris-3.7.0-pyha770c72_0.conda#dccc1f660bf455c239adaabf56b91dc9 +https://conda.anaconda.org/conda-forge/linux-64/libarrow-acero-15.0.0-h59595ed_2_cpu.conda#85d3e05ea2b427e879e486f09fb8cf54 +https://conda.anaconda.org/conda-forge/linux-64/libarrow-flight-15.0.0-hdc44a87_2_cpu.conda#41f4c79b79d6c13ffb7abc71ab4f0c54 +https://conda.anaconda.org/conda-forge/linux-64/libarrow-gandiva-15.0.0-hacb8726_2_cpu.conda#d8415318348d02dad131144d6fc151ec +https://conda.anaconda.org/conda-forge/linux-64/libparquet-15.0.0-h352af49_2_cpu.conda#8d99909e413b67872996d46093dda024 +https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.16.0-pyhd8ed1ab_0.conda#de2255e7a38fad6eaf457739c6599413 +https://conda.anaconda.org/conda-forge/noarch/py-cordex-0.6.6-pyhd8ed1ab_0.conda#255f9eac03143526c8aed41d1d091c63 +https://conda.anaconda.org/conda-forge/linux-64/pydot-2.0.0-py311h38be061_0.conda#cdfd23a54a18f3c8d5320d7717f4ed52 +https://conda.anaconda.org/conda-forge/linux-64/fiona-1.9.5-py311hf8e0aa6_3.conda#a5277325e005e9d014eca37187b3f4a2 +https://conda.anaconda.org/conda-forge/linux-64/libarrow-dataset-15.0.0-h59595ed_2_cpu.conda#3cda69f7af9b2341e3ee0fb602861726 +https://conda.anaconda.org/conda-forge/linux-64/libarrow-flight-sql-15.0.0-hfbc7f12_2_cpu.conda#d779a1334ac200d6e9c04ee6bfa2a6af +https://conda.anaconda.org/conda-forge/noarch/nbconvert-pandoc-7.16.0-pyhd8ed1ab_0.conda#28dde45c295b3f110bc6bb425472137b https://conda.anaconda.org/conda-forge/noarch/prov-2.0.0-pyhd3deb0d_0.tar.bz2#aa9b3ad140f6c0668c646f32e20ccf82 -https://conda.anaconda.org/conda-forge/noarch/iris-esmf-regrid-0.8.0-pyhd8ed1ab_0.conda#56e85460d22fa7d4fb06300f785dd1e1 -https://conda.anaconda.org/conda-forge/noarch/nbconvert-7.9.2-pyhd8ed1ab_0.conda#16ccaddfcfa0a1a606a9ecf6a52d6c11 +https://conda.anaconda.org/conda-forge/linux-64/libarrow-substrait-15.0.0-hfbc7f12_2_cpu.conda#21de7bd8fd4568ccb232bc7bfbf3d112 +https://conda.anaconda.org/conda-forge/noarch/nbconvert-7.16.0-pyhd8ed1ab_0.conda#342ba1099325da21a811e80397006461 +https://conda.anaconda.org/conda-forge/linux-64/pyarrow-15.0.0-py311h39c9aba_2_cpu.conda#6c70dd512802dfc4114b2df223a883bd +https://conda.anaconda.org/conda-forge/noarch/pyarrow-hotfix-0.6-pyhd8ed1ab_0.conda#ccc06e6ef2064ae129fab3286299abda +https://conda.anaconda.org/conda-forge/noarch/dask-2024.2.0-pyhd8ed1ab_0.conda#085d464298ca31a98193af99ee5e75e7 +https://conda.anaconda.org/conda-forge/noarch/iris-esmf-regrid-0.9.0-pyhd8ed1ab_0.conda#570f2c6e387fd6dac5356a5152f91b3f https://conda.anaconda.org/conda-forge/noarch/autodocsumm-0.2.6-pyhd8ed1ab_0.tar.bz2#4409dd7e06a62c3b2aa9e96782c49c6d https://conda.anaconda.org/conda-forge/noarch/nbsphinx-0.9.3-pyhd8ed1ab_0.conda#0dbaa7d08d3d79b2a1a4dd6a02cc4581 -https://conda.anaconda.org/conda-forge/noarch/pydata-sphinx-theme-0.14.1-pyhd8ed1ab_0.conda#78153addf629c51fab775ef360012ca3 -https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-applehelp-1.0.7-pyhd8ed1ab_0.conda#aebfabcb60c33a89c1f9290cab49bc93 -https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-devhelp-1.0.5-pyhd8ed1ab_0.conda#ebf08f5184d8eaa486697bc060031953 -https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-htmlhelp-2.0.4-pyhd8ed1ab_0.conda#a9a89000dfd19656ad004b937eeb6828 -https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-qthelp-1.0.6-pyhd8ed1ab_0.conda#cf5c9649272c677a964a7313279e3a9b +https://conda.anaconda.org/conda-forge/noarch/pydata-sphinx-theme-0.15.2-pyhd8ed1ab_0.conda#ce99859070b0e17ccc63234ca58f3ed8 +https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-applehelp-1.0.8-pyhd8ed1ab_0.conda#611a35a27914fac3aa37611a6fe40bb5 +https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-devhelp-1.0.6-pyhd8ed1ab_0.conda#d7e4954df0d3aea2eacc7835ad12671d +https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-htmlhelp-2.0.5-pyhd8ed1ab_0.conda#7e1e7437273682ada2ed5e9e9714b140 +https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-qthelp-1.0.7-pyhd8ed1ab_0.conda#26acae54b06f178681bfb551760f5dd1 https://conda.anaconda.org/conda-forge/noarch/sphinx-7.2.6-pyhd8ed1ab_0.conda#bbfd1120d1824d2d073bc65935f0e4c0 -https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-serializinghtml-1.1.9-pyhd8ed1ab_0.conda#0612e497d7860728f2cda421ea2aec09 +https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-serializinghtml-1.1.10-pyhd8ed1ab_0.conda#e507335cb4ca9cff4c3d0fa9cdab255e diff --git a/doc/api/esmvalcore.regridding_schemes.rst b/doc/api/esmvalcore.regridding_schemes.rst new file mode 100644 index 0000000000..960b8c4b7d --- /dev/null +++ b/doc/api/esmvalcore.regridding_schemes.rst @@ -0,0 +1,23 @@ +.. _regridding_schemes: + +Regridding schemes +================== + +Iris natively supports data regridding with its :meth:`iris.cube.Cube.regrid` +method and a set of predefined regridding schemes provided in the +:mod:`~iris.analysis` module (further details are given on `this +`__ +page). +Here, further regridding schemes are provided that are compatible with +:meth:`iris.cube.Cube.regrid`. + +Example: + +.. code:: python + + from esmvalcore.preprocessor.regrid_schemes import ESMPyAreaWeighted + + regridded_cube = cube.regrid(target_grid, ESMPyAreaWeighted()) + +.. automodule:: esmvalcore.preprocessor.regrid_schemes + :no-show-inheritance: diff --git a/doc/api/esmvalcore.rst b/doc/api/esmvalcore.rst index e1490a08d8..d160246243 100644 --- a/doc/api/esmvalcore.rst +++ b/doc/api/esmvalcore.rst @@ -17,5 +17,6 @@ library. This section documents the public API of ESMValCore. esmvalcore.iris_helpers esmvalcore.local esmvalcore.preprocessor + esmvalcore.regridding_schemes esmvalcore.typing esmvalcore.experimental diff --git a/doc/changelog.rst b/doc/changelog.rst index daa5dae664..7160637390 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -9,138 +9,154 @@ v2.10.0 ------- Highlights -TODO: add highlights +- All statistics preprocessors support the same operators and have a common + :ref:`documentation `. In addition, arbitrary keyword arguments + for the statistical operation can be directly given to the preprocessor. + +- The output webpage generated by the tool now looks better and provides + methods to select and filter the output. + +- Improved computational efficiency: + + - Automatic rechunking between preprocessor steps to keep the + `graph size smaller `_ + and the `chunk size optimal `__. + - Reduce the size of the dask graph created by :func:`esmvalcore.preprocessor.anomalies`. + - Preprocessors :func:`esmvalcore.preprocessor.mask_above_threshold`, + :func:`esmvalcore.preprocessor.mask_below_threshold`, + :func:`esmvalcore.preprocessor.mask_inside_range`, + :func:`esmvalcore.preprocessor.mask_outside_range` are now lazy. + - Lazy coordinates bounds are no longer loaded into memory by the CMOR checks and fixes. This release includes Backwards incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- Remove the deprecated option ``use_legacy_supplementaries`` (`#2202 `__) `Bouwe Andela `__ +- Remove the deprecated option ``use_legacy_supplementaries`` (:pull:`2202`) by :user:`bouweandela` - The recommended upgrade procedure is to remove ``use_legacy_supplementaries`` from config-user.yml (if it was there) and remove any mention of ``fx_variables`` from the recipe. If automatically defining the required supplementary variables does not work, define them in the variable or (``additional_``) ``datasets`` section as described in :ref:`supplementary_variables`. -- Use smarter (units-aware) weights (`#2139 `__) `Manuel Schlund `__ - +- Use smarter (units-aware) weights (:pull:`2139`) by :user:`schlunma` + - Some preprocessors handle units better. For details, see the pull request. -- Removed deprecated configuration option ``offline`` (`#2213 `__) `Manuel Schlund `__ +- Removed deprecated configuration option ``offline`` (:pull:`2213`) by :user:`schlunma` - In :ref:`changelog-v2-8-0`, we replaced the old ``offline`` configuration option. From this version on, it stops working. Please refer to :ref:`changelog-v2-8-0` for upgrade instructions. -- Fix issue with CORDEX datasets requiring different dataset tags for downloads and fixes (`#2066 `__) `Joakim Löw `__ - +- Fix issue with CORDEX datasets requiring different dataset tags for downloads and fixes (:pull:`2066`) by :user:`ljoakim` + - Due to the different facets for CORDEX datasets, there was an inconsistency in the fixing mechanism. This change requires changes to existing recipes that use CORDEX datasets. Please refer to the pull request for detailed update instructions. -- Added new operators for statistics preprocessor (e.g., ``'percentile'``) and allowed passing additional arguments (`#2191 `__) `Manuel Schlund `__ - - - This harmonizes the operators for all statistics preprocessors. From this version, the new names can be used; the old arguments will stop working from - version 2.12.0. Please refer to :ref:`stat_preprocs` for a detailed description. - - For the following changes, no user change is necessary - - - Remove deprecated way of calling :func:`~esmvalcore.cmor.table.read_cmor_tables` (`#2201 `__) `Bouwe Andela `__ - - Remove deprecated callback argument from preprocessor ``load`` function (`#2207 `__) `Bouwe Andela `__ + - Remove deprecated way of calling :func:`~esmvalcore.cmor.table.read_cmor_tables` (:pull:`2201`) by :user:`bouweandela` - - Remove deprecated preprocessor function `cleanup` (`#2215 `__) `Bouwe Andela `__ + - Remove deprecated callback argument from preprocessor ``load`` function (:pull:`2207`) by :user:`bouweandela` + - Remove deprecated preprocessor function `cleanup` (:pull:`2215`) by :user:`bouweandela` Deprecations ~~~~~~~~~~~~ -- Clearly separate fixes and CMOR checks (`#2157 `__) `Manuel Schlund `__ +- Clearly separate fixes and CMOR checks (:pull:`2157`) by :user:`schlunma` +- Added new operators for statistics preprocessor (e.g., ``'percentile'``) and allowed passing additional arguments (:pull:`2191`) by :user:`schlunma` + + - This harmonizes the operators for all statistics preprocessors. From this version, the new names can be used; the old arguments will stop working from + version 2.12.0. Please refer to :ref:`stat_preprocs` for a detailed description. Bug fixes ~~~~~~~~~ -- Re-add correctly region-extracted cell measures and ancillary variables after :ref:`extract_region` (`#2166 `__) `Valeriu Predoi `__, `Manuel Schlund `__ +- Re-add correctly region-extracted cell measures and ancillary variables after :ref:`extract_region` (:pull:`2166`) by :user:`valeriupredoi`, :user:`schlunma` - Fix sorting of datasets - - Fix sorting of ensemble members in :func:`~esmvalcore.dataset.datasets_to_recipe` (`#2095 `__) `Bouwe Andela `__ - - Fix a problem with sorting datasets that have a mix of facet types (`#2238 `__) `Bouwe Andela `__ - - Avoid a crash if dataset has supplementary variables (`#2198 `__) `Bouwe Andela `__ + - Fix sorting of ensemble members in :func:`~esmvalcore.dataset.datasets_to_recipe` (:pull:`2095`) by :user:`bouweandela` + - Fix a problem with sorting datasets that have a mix of facet types (:pull:`2238`) by :user:`bouweandela` + - Avoid a crash if dataset has supplementary variables (:pull:`2198`) by :user:`bouweandela` CMOR standard ~~~~~~~~~~~~~ -- ERA5 on-the-fly CMORizer: changed sign of variables ``evspsbl`` and ``evspsblpot`` (`#2115 `__) `katjaweigel `__ -- Add ``ch4`` surface custom cmor table entry (`#2168 `__) `Birgit Hassler `__ -- Add CMIP3 institutes names used at NCI (`#2152 `__) `Romain Beucher `__ -- Added :func:`~esmvalcore.cmor.fixes.get_time_bounds` and :func:`~esmvalcore.cmor.fixes.get_next_month` to public API (`#2214 `__) `Manuel Schlund `__ +- ERA5 on-the-fly CMORizer: changed sign of variables ``evspsbl`` and ``evspsblpot`` (:pull:`2115`) by :user:`katjaweigel` +- Add ``ch4`` surface custom cmor table entry (:pull:`2168`) by :user:`hb326` +- Add CMIP3 institutes names used at NCI (:pull:`2152`) by :user:`rbeucher` +- Added :func:`~esmvalcore.cmor.fixes.get_time_bounds` and :func:`~esmvalcore.cmor.fixes.get_next_month` to public API (:pull:`2214`) by :user:`schlunma` - Improve concatenation checks - - Relax concatenation checks for ``--check_level=relax`` and ``--check_level=ignore`` (`#2144 `__) `sloosvel `__ - - Fix ``concatenate`` preprocessor function (`#2240 `__) `Bouwe Andela `__ - - Fix time overlap handling in concatenation (`#2247 `__) `Klaus Zimmermann `__ + - Relax concatenation checks for ``--check_level=relax`` and ``--check_level=ignore`` (:pull:`2144`) by :user:`sloosvel` + - Fix ``concatenate`` preprocessor function (:pull:`2240`) by :user:`bouweandela` + - Fix time overlap handling in concatenation (:pull:`2247`) by :user:`zklaus` Computational performance improvements ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- Make :ref:`threshold_masking` preprocessors lazy (`#2169 `__) `Jörg Benke `__ +- Make :ref:`threshold_masking` preprocessors lazy (:pull:`2169`) by :user:`joergbenke` - - Restored usage of numpy in `_mask_with_shp` (`#2209 `__) `Jörg Benke `__ -- Call coord.core_bounds() instead of coord.bounds in ``check.py`` (`#2146 `__) `sloosvel `__ -- Rechunk between preprocessor steps (`#2205 `__) `Bouwe Andela `__ -- Reduce the size of the dask graph created by the ``anomalies`` preprocessor function (`#2200 `__) `Bouwe Andela `__ + - Restored usage of numpy in `_mask_with_shp` (:pull:`2209`) by :user:`joergbenke` +- Do not realize lazy coordinate bounds in CMOR check (:pull:`2146`) by :user:`sloosvel` +- Rechunk between preprocessor steps (:pull:`2205`) by :user:`bouweandela` +- Reduce the size of the dask graph created by the ``anomalies`` preprocessor function (:pull:`2200`) by :user:`bouweandela` Documentation ~~~~~~~~~~~~~ -- Add reference to release v2.9.0 in the changelog (`#2130 `__) `Rémi Kazeroni `__ -- Add merge instructions to release instructions (`#2131 `__) `Klaus Zimmermann `__ -- Update `mamba` before building environment during Readthedocs build (`#2149 `__) `Valeriu Predoi `__ -- Ensure compatible zstandard and zstd versions for .conda support (`#2204 `__) `Klaus Zimmermann `__ -- Remove outdated documentation (`#2210 `__) `Bouwe Andela `__ -- Remove meercode badge from README because their API is broken (`#2224 `__) `Valeriu Predoi `__ -- Correct usage help text of version command (`#2232 `__) `James Frost `__ -- Add ``navigation_with_keys: False`` to ``html_theme_options`` in Readthedocs ``conf.py`` (`#2245 `__) `Valeriu Predoi `__ -- Replace squarey badge with roundy shield for Anaconda sticker in README (`#2233 `__) `Valeriu Predoi `__ +- Add reference to release v2.9.0 in the changelog (:pull:`2130`) by :user:`remi-kazeroni` +- Add merge instructions to release instructions (:pull:`2131`) by :user:`zklaus` +- Update `mamba` before building environment during Readthedocs build (:pull:`2149`) by :user:`valeriupredoi` +- Ensure compatible zstandard and zstd versions for .conda support (:pull:`2204`) by :user:`zklaus` +- Remove outdated documentation (:pull:`2210`) by :user:`bouweandela` +- Remove meercode badge from README because their API is broken (:pull:`2224`) by :user:`valeriupredoi` +- Correct usage help text of version command (:pull:`2232`) by :user:`jfrost-mo` +- Add ``navigation_with_keys: False`` to ``html_theme_options`` in Readthedocs ``conf.py`` (:pull:`2245`) by :user:`valeriupredoi` +- Replace squarey badge with roundy shield for Anaconda sticker in README (:pull:`2233`, :pull:`2260`) by :user:`valeriupredoi` Fixes for datasets ~~~~~~~~~~~~~~~~~~ -- Updated doc about fixes and added type hints to fix functions (`#2160 `__) `Manuel Schlund `__ +- Updated doc about fixes and added type hints to fix functions (:pull:`2160`) by :user:`schlunma` Installation ~~~~~~~~~~~~ -- Clean-up how pins are written in conda environment file (`#2125 `__) `Valeriu Predoi `__ -- Use importlib.metadata instead of deprecated pkg_resources (`#2096 `__) `Bouwe Andela `__ -- Pin shapely to >=2.0 (`#2075 `__) `Valeriu Predoi `__ +- Clean-up how pins are written in conda environment file (:pull:`2125`) by :user:`valeriupredoi` +- Use importlib.metadata instead of deprecated pkg_resources (:pull:`2096`) by :user:`bouweandela` +- Pin shapely to >=2.0 (:pull:`2075`) by :user:`valeriupredoi` +- Pin Python to <3.12 in conda environment (:pull:`2272`) by :user:`bouweandela` Preprocessor ~~~~~~~~~~~~ -- Improve preprocessor output sorting code (`#2111 `__) `Bouwe Andela `__ -- Preprocess datasets in the same order as they are listed in the recipe (`#2103 `__) `Bouwe Andela `__ +- Improve preprocessor output sorting code (:pull:`2111`) by :user:`bouweandela` +- Preprocess datasets in the same order as they are listed in the recipe (:pull:`2103`) by :user:`bouweandela` Automatic testing ~~~~~~~~~~~~~~~~~ -- [Github Actions] Compress all bash shell setters into one default option per workflow (`#2126 `__) `Valeriu Predoi `__ -- [Github Actions] Fix Monitor Tests Github Action (`#2135 `__) `Valeriu Predoi `__ -- [condalock] update conda lock file (`#2141 `__) `Valeriu Predoi `__ -- [Condalock] make sure mamba/conda are at latest version by forcing a pinned mamba install (`#2136 `__) `Valeriu Predoi `__ -- Update code coverage orbs (`#2206 `__) `Bouwe Andela `__ -- Revisit the comment-triggered Github Actions test (`#2243 `__) `Valeriu Predoi `__ -- Remove workflow that runs Github Actions tests from PR comment (`#2244 `__) `Valeriu Predoi `__ +- [Github Actions] Compress all bash shell setters into one default option per workflow (:pull:`2126`) by :user:`valeriupredoi` +- [Github Actions] Fix Monitor Tests Github Action (:pull:`2135`) by :user:`valeriupredoi` +- [condalock] update conda lock file (:pull:`2141`) by :user:`valeriupredoi` +- [Condalock] make sure mamba/conda are at latest version by forcing a pinned mamba install (:pull:`2136`) by :user:`valeriupredoi` +- Update code coverage orbs (:pull:`2206`) by :user:`bouweandela` +- Revisit the comment-triggered Github Actions test (:pull:`2243`) by :user:`valeriupredoi` +- Remove workflow that runs Github Actions tests from PR comment (:pull:`2244`) by :user:`valeriupredoi` Improvements ~~~~~~~~~~~~ -- Merge v2.9.x into main (`#2128 `__) `Manuel Schlund `__ -- Fix typo in citation file (`#2182 `__) `Bouwe Andela `__ -- Cleaned and extended function that extracts datetimes from paths (`#2181 `__) `Manuel Schlund `__ -- Add file encoding (and some read modes) at open file step (`#2219 `__) `Valeriu Predoi `__ -- Check type of argument passed to :func:`~esmvalcore.cmor.table.read_cmor_tables` (`#2217 `__) `Valeriu Predoi `__ -- Dynamic HTML output for monitoring (`#2062 `__) `Brei Soliño `__ - +- Merge v2.9.x into main (:pull:`2128`) by :user:`schlunma` +- Fix typo in citation file (:pull:`2182`) by :user:`bouweandela` +- Cleaned and extended function that extracts datetimes from paths (:pull:`2181`) by :user:`schlunma` +- Add file encoding (and some read modes) at open file step (:pull:`2219`) by :user:`valeriupredoi` +- Check type of argument passed to :func:`~esmvalcore.cmor.table.read_cmor_tables` (:pull:`2217`) by :user:`valeriupredoi` +- Dynamic HTML output for monitoring (:pull:`2062`) by :user:`bsolino` +- Use PyPI's trusted publishers authentication (:pull:`2269`) by :user:`valeriupredoi` .. _changelog-v2-9-0: @@ -156,15 +172,14 @@ which can Configuration examples and advice are available in :ref:`our documentation `. More work on improving the computational performance is planned, so please share -your experiences, good and bad, with this new feature in -`ESMValGroup/ESMValCore#1763 `__. +your experiences, good and bad, with this new feature in :discussion:`1763`. This release includes Backwards incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- Remove deprecated configuration options (`#2056 `__) `Bouwe Andela `__ +- Remove deprecated configuration options (:pull:`2056`) by :user:`bouweandela` - The module ``esmvalcore.experimental.config`` has been removed. To upgrade, import the module from :mod:`esmvalcore.config` instead. @@ -178,52 +193,52 @@ Backwards incompatible changes Bug fixes ~~~~~~~~~ -- Respect ``ignore_warnings`` settings from the :ref:`project configuration ` in :func:`esmvalcore.dataset.Dataset.load` (`#2046 `__) `Manuel Schlund `__ -- Fixed usage of custom location for :ref:`custom CMOR tables ` (`#2052 `__) `Manuel Schlund `__ -- Fix issue with writing index.html when :ref:`running a recipe ` with ``--resume-from`` (`#2055 `__) `Bouwe Andela `__ -- Fixed bug in ICON CMORizer that lead to shifted time coordinates (`#2038 `__) `Manuel Schlund `__ -- Include ``-`` in allowed characters for bibtex references (`#2097 `__) `Alistair Sellar `__ -- Do not raise an exception if the requested version of a file is not available for all matching files on ESGF (`#2105 `__) `Bouwe Andela `__ +- Respect ``ignore_warnings`` settings from the :ref:`project configuration ` in :func:`esmvalcore.dataset.Dataset.load` (:pull:`2046`) by :user:`schlunma` +- Fixed usage of custom location for :ref:`custom CMOR tables ` (:pull:`2052`) by :user:`schlunma` +- Fix issue with writing index.html when :ref:`running a recipe ` with ``--resume-from`` (:pull:`2055`) by :user:`bouweandela` +- Fixed bug in ICON CMORizer that lead to shifted time coordinates (:pull:`2038`) by :user:`schlunma` +- Include ``-`` in allowed characters for bibtex references (:pull:`2097`) by :user:`alistairsellar` +- Do not raise an exception if the requested version of a file is not available for all matching files on ESGF (:pull:`2105`) by :user:`bouweandela` Computational performance improvements ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- Add support for :ref:`configuring Dask distributed ` (`#2049 `__, `#2122 `__) `Bouwe Andela `__ -- Make :func:`esmvalcore.preprocessor.extract_levels` lazy (`#1761 `__) `Bouwe Andela `__ -- Lazy implementation of :func:`esmvalcore.preprocessor.multi_model_statistics` and :func:`esmvalcore.preprocessor.ensemble_statistics` (`#968 `__ and `#2087 `__) `Peter Kalverla `__ -- Avoid realizing data in preprocessor function :func:`esmvalcore.preprocessor.concatenate` when cubes overlap (`#2109 `__) `Bouwe Andela `__ +- Add support for :ref:`configuring Dask distributed ` (:pull:`2049`, :pull:`2122`) by :user:`bouweandela` +- Make :func:`esmvalcore.preprocessor.extract_levels` lazy (:pull:`1761`) by :user:`bouweandela` +- Lazy implementation of :func:`esmvalcore.preprocessor.multi_model_statistics` and :func:`esmvalcore.preprocessor.ensemble_statistics` (:pull:`968` and :pull:`2087`) by :user:`Peter9192` +- Avoid realizing data in preprocessor function :func:`esmvalcore.preprocessor.concatenate` when cubes overlap (:pull:`2109`) by :user:`bouweandela` Documentation ~~~~~~~~~~~~~ -- Remove unneeded sphinxcontrib extension (`#2047 `__) `Valeriu Predoi `__ -- Show ESMValTool logo on `PyPI webpage `__ (`#2065 `__) `Valeriu Predoi `__ -- Fix gitter badge in README (`#2118 `__) `Rémi Kazeroni `__ -- Add changelog for v2.9.0 (`#2088 `__ and `#2123 `__) `Bouwe Andela `__ +- Remove unneeded sphinxcontrib extension (:pull:`2047`) by :user:`valeriupredoi` +- Show ESMValTool logo on `PyPI webpage `__ (:pull:`2065`) by :user:`valeriupredoi` +- Fix gitter badge in README (:pull:`2118`) by :user:`remi-kazeroni` +- Add changelog for v2.9.0 (:pull:`2088` and :pull:`2123`) by :user:`bouweandela` Fixes for datasets ~~~~~~~~~~~~~~~~~~ -- Pass the :obj:`esmvalcore.config.Session` to fixes (`#1988 `__) `Manuel Schlund `__ -- ICON: Allowed specifying vertical grid information in recipe (`#2067 `__) `Manuel Schlund `__ -- Allow specifying ``raw_units`` for CESM2, EMAC, and ICON CMORizers (`#2043 `__) `Manuel Schlund `__ -- ICON: allow specifying horizontal grid file in recipe/extra facets (`#2078 `__) `Manuel Schlund `__ -- Fix tas/tos CMIP6: FIO, KACE, MIROC, IITM (`#2061 `__) `Pep Cos `__ -- Add fix for EC-Earth3-Veg tos calendar (`#2100 `__) `Bouwe Andela `__ -- Correct GISS-E2-1-G ``tos`` units (`#2099 `__) `Bouwe Andela `__ +- Pass the :obj:`esmvalcore.config.Session` to fixes (:pull:`1988`) by :user:`schlunma` +- ICON: Allowed specifying vertical grid information in recipe (:pull:`2067`) by :user:`schlunma` +- Allow specifying ``raw_units`` for CESM2, EMAC, and ICON CMORizers (:pull:`2043`) by :user:`schlunma` +- ICON: allow specifying horizontal grid file in recipe/extra facets (:pull:`2078`) by :user:`schlunma` +- Fix tas/tos CMIP6: FIO, KACE, MIROC, IITM (:pull:`2061`) by :user:`pepcos` +- Add fix for EC-Earth3-Veg tos calendar (:pull:`2100`) by :user:`bouweandela` +- Correct GISS-E2-1-G ``tos`` units (:pull:`2099`) by :user:`bouweandela` Installation ~~~~~~~~~~~~ -- Drop support for Python 3.8 (`#2053 `__) `Bouwe Andela `__ -- Add python 3.11 to Github Actions package (conda and PyPI) installation tests (`#2083 `__) `Valeriu Predoi `__ -- Remove ``with_mypy`` or ``with-mypy`` optional tool for prospector (`#2108 `__) `Valeriu Predoi `__ +- Drop support for Python 3.8 (:pull:`2053`) by :user:`bouweandela` +- Add python 3.11 to Github Actions package (conda and PyPI) installation tests (:pull:`2083`) by :user:`valeriupredoi` +- Remove ``with_mypy`` or ``with-mypy`` optional tool for prospector (:pull:`2108`) by :user:`valeriupredoi` Preprocessor ~~~~~~~~~~~~ -- Added ``period='hourly'`` for :func:`esmvalcore.preprocessor.climate_statistics` and :func:`esmvalcore.preprocessor.anomalies` (`#2068 `__) `Manuel Schlund `__ -- Support IPCC AR6 regions in :func:`esmvalcore.preprocessor.extract_shape` (`#2008 `__) `Manuel Schlund `__ +- Added ``period='hourly'`` for :func:`esmvalcore.preprocessor.climate_statistics` and :func:`esmvalcore.preprocessor.anomalies` (:pull:`2068`) by :user:`schlunma` +- Support IPCC AR6 regions in :func:`esmvalcore.preprocessor.extract_shape` (:pull:`2008`) by :user:`schlunma` .. _changelog-v2-8-1: @@ -239,37 +254,37 @@ This release includes: Bug fixes ~~~~~~~~~ -- Pin numpy !=1.24.3 (`#2011 `__) `Valeriu Predoi `__ -- Fix a bug in recording provenance for the ``mask_multimodel`` preprocessor (`#1984 `__) `Manuel Schlund `__ -- Fix ICON hourly data rounding issues (`#2022 `__) `Julian Bauer `__ -- Use the default SSL context when using the ``extract_location`` preprocessor (`#2023 `__) `Emma Hogan `__ -- Make time-related CMOR fixes work with time dimensions `time1`, `time2`, `time3` (`#1971 `__) `Manuel Schlund `__ -- Always create a cache directory for storing ICON grid files (`#2030 `__) `Manuel Schlund `__ -- Fixed altitude <--> pressure level conversion for masked arrays in the ``extract_levels`` preprocessor (`#1999 `__) `Manuel Schlund `__ -- Allowed ignoring of scalar time coordinates in the ``multi_model_statistics`` preprocessor (`#1961 `__) `Manuel Schlund `__ +- Pin numpy !=1.24.3 (:pull:`2011`) by :user:`valeriupredoi` +- Fix a bug in recording provenance for the ``mask_multimodel`` preprocessor (:pull:`1984`) by :user:`schlunma` +- Fix ICON hourly data rounding issues (:pull:`2022`) by :user:`BauerJul` +- Use the default SSL context when using the ``extract_location`` preprocessor (:pull:`2023`) by :user:`ehogan` +- Make time-related CMOR fixes work with time dimensions `time1`, `time2`, `time3` (:pull:`1971`) by :user:`schlunma` +- Always create a cache directory for storing ICON grid files (:pull:`2030`) by :user:`schlunma` +- Fixed altitude <--> pressure level conversion for masked arrays in the ``extract_levels`` preprocessor (:pull:`1999`) by :user:`schlunma` +- Allowed ignoring of scalar time coordinates in the ``multi_model_statistics`` preprocessor (:pull:`1961`) by :user:`schlunma` Fixes for datasets ~~~~~~~~~~~~~~~~~~ -- Add support for hourly ICON data (`#1990 `__) `Julian Bauer `__ -- Fix areacello in BCC-CSM2-MR (`#1993 `__) `Rémi Kazeroni `__ +- Add support for hourly ICON data (:pull:`1990`) by :user:`BauerJul` +- Fix areacello in BCC-CSM2-MR (:pull:`1993`) by :user:`remi-kazeroni` Installation ~~~~~~~~~~~~ -- Add support for Python=3.11 (`#1832 `__) `Valeriu Predoi `__ -- Modernize conda lock file creation workflow with mamba, Mambaforge etc (`#2027 `__) `Valeriu Predoi `__ -- Pin `libnetcdf!=4.9.1` (`#2072 `__) `Rémi Kazeroni `__ +- Add support for Python=3.11 (:pull:`1832`) by :user:`valeriupredoi` +- Modernize conda lock file creation workflow with mamba, Mambaforge etc (:pull:`2027`) by :user:`valeriupredoi` +- Pin `libnetcdf!=4.9.1` (:pull:`2072`) by :user:`remi-kazeroni` Documentation ~~~~~~~~~~~~~ -- Add changelog for v2.8.1 (`#2079 `__) `Bouwe Andela `__ +- Add changelog for v2.8.1 (:pull:`2079`) by :user:`bouweandela` Automatic testing ~~~~~~~~~~~~~~~~~ -- Use mocked `geopy.geocoders.Nominatim` to avoid `ReadTimeoutError` (`#2005 `__) `Manuel Schlund `__ -- Update pre-commit hooks (`#2020 `__) `Bouwe Andela `__ +- Use mocked `geopy.geocoders.Nominatim` to avoid `ReadTimeoutError` (:pull:`2005`) by :user:`schlunma` +- Update pre-commit hooks (:pull:`2020`) by :user:`bouweandela` .. _changelog-v2-8-0: @@ -282,22 +297,22 @@ Highlights - ESMValCore now supports wildcards in recipes and offers improved support for ancillary variables and dataset versioning thanks to contributions by - `Bouwe Andela `__. For details, see + :user:`bouweandela`. For details, see :ref:`Automatically populating a recipe with all available datasets ` and :ref:`Defining supplementary variables `. - Support for CORDEX datasets in a rotated pole coordinate system has been - added by `sloosvel `__. + added by :user:`sloosvel`. - Native :ref:`ICON ` output is now made UGRID-compliant on-the-fly to unlock the use of more sophisticated regridding algorithms, - thanks to `Manuel Schlund `__. + thanks to :user:`schlunma`. - The Python API has been extended with the addition of three modules: :mod:`esmvalcore.config`, :mod:`esmvalcore.dataset`, and :mod:`esmvalcore.local`, all these features courtesy of - `Bouwe Andela `__. For details, see our new + :user:`bouweandela`. For details, see our new example :doc:`example-notebooks`. - The preprocessor :func:`~esmvalcore.preprocessor.multi_model_statistics` has been extended to support more use-cases thanks to contributions by - `Manuel Schlund `__. For details, see + :user:`schlunma`. For details, see :ref:`Multi-model statistics `. This release includes: @@ -307,9 +322,7 @@ Backwards incompatible changes Please read the descriptions of the linked pull requests for detailed upgrade instructions. - The algorithm for automatically defining the ancillary variables and cell - measures has been improved - (`#1609 `_) - `Bouwe Andela `__. + measures has been improved (:pull:`1609`) by :user:`bouweandela`. If this does not work as expected, more examples of how to adapt your recipes are given `here `__ @@ -317,167 +330,161 @@ Please read the descriptions of the linked pull requests for detailed upgrade in :ref:`recipe documentation ` and the :ref:`preprocessor documentation `. - Remove deprecated features scheduled for removal in v2.8.0 or earlier - (`#1826 `__) - `Manuel Schlund `__. + (:pull:`1826`) by :user:`schlunma`. Removed ``esmvalcore.iris_helpers.var_name_constraint`` (has been deprecated in v2.6.0; please use :class:`iris.NameConstraint` with the keyword argument ``var_name`` instead) and the option ``always_use_ne_mask`` for :func:`esmvalcore.preprocessor.mask_landsea` (has been deprecated in v2.5.0; the same behavior can now be achieved by specifying ``supplementary_variables``. - No files will be found if a non-existent version of a dataset is specified - (`#1835 `_) - `Bouwe Andela `__. If a ``version`` of a + (:pull:`1835`) by :user:`bouweandela`. If a ``version`` of a dataset is specified in the recipe, the tool will now search for exactly that version, instead of simply using the latest version. Therefore, it is necessary to make sure that the version number in the directory tree matches with the version number in the recipe to find the files. - The default filename template for obs4MIPs has been updated to better match - filenames used in this project in - (`#1866 `__) - `Bouwe Andela `__. This may cause issues - if you are storing all the files for obs4MIPs in a directory with no - subdirectories per dataset. + filenames used in this project in (:pull:`1866`) by :user:`bouweandela`. This + may cause issues if you are storing all the files for obs4MIPs in a + directory with no subdirectories per dataset. Deprecations ~~~~~~~~~~~~ Please read the descriptions of the linked pull requests for detailed upgrade instructions. - Various configuration related options that are now available through - :mod:`esmvalcore.config` have been deprecated (`#1769 `_) `Bouwe Andela `__. + :mod:`esmvalcore.config` have been deprecated (:pull:`1769`) by :user:`bouweandela`. - The ``fx_variables`` preprocessor argument and related features have been - deprecated (`#1609`_) - `Bouwe Andela `__. - See `here `__ - for more information. + deprecated (:pull:`1609`) by :user:`bouweandela`. + See :pull:`1609#Deprecations` for more information. - Combined ``offline`` and ``always_search_esgf`` into a single option ``search_esgf`` - (`#1935 `_) - `Manuel Schlund `__. The configuration + (:pull:`1935`) + :user:`schlunma`. The configuration option/command line argument ``offline`` has been deprecated in favor of ``search_esgf``. The previous ``offline: true`` is now ``search_esgf: never`` (the default); the previous ``offline: false`` is now ``search_esgf: when_missing``. More details on how to adapt your workflow - regarding these new options are given in `#1935`_ and the + regarding these new options are given in :pull:`1935` and the `documentation `__. -- :func:`esmvalcore.preprocessor.cleanup` has been deprecated (`#1949 `_) - `Manuel Schlund `__. Please do not use this +- :func:`esmvalcore.preprocessor.cleanup` has been deprecated (:pull:`1949`) + :user:`schlunma`. Please do not use this anymore in the recipe (it is not necessary). Python API ~~~~~~~~~~ -- Support searching ESGF for a specific version of a file and add :obj:`esmvalcore.esgf.ESGFFile.facets` (`#1822 `__) `Bouwe Andela `__ -- Fix issues with searching for files on ESGF (`#1863 `__) `Bouwe Andela `__ -- Move the :mod:`esmvalcore.experimental.config` module to :mod:`esmvalcore.config` (`#1769`_) `Bouwe Andela `__ -- Add :mod:`esmvalcore.local`, a module to search data on the local filesystem (`#1835`_) `Bouwe Andela `__ -- Add :mod:`esmvalcore.dataset` module (`#1877 `__) `Bouwe Andela `__ +- Support searching ESGF for a specific version of a file and add :obj:`esmvalcore.esgf.ESGFFile.facets` (:pull:`1822`) by :user:`bouweandela` +- Fix issues with searching for files on ESGF (:pull:`1863`) by :user:`bouweandela` +- Move the :mod:`esmvalcore.experimental.config` module to :mod:`esmvalcore.config` (:pull:`1769`) by :user:`bouweandela` +- Add :mod:`esmvalcore.local`, a module to search data on the local filesystem (:pull:`#1835`) by :user:`bouweandela` +- Add :mod:`esmvalcore.dataset` module (:pull:`1877`) by :user:`bouweandela` Bug fixes ~~~~~~~~~ -- Import from :mod:`esmvalcore.config` in the :mod:`esmvalcore.experimental` module (`#1816 `__) `Bouwe Andela `__ -- Added scalar coords of input cubes to output of esmpy_regrid (`#1811 `__) `Manuel Schlund `__ -- Fix severe bug in :func:`esmvalcore.preprocessor.mask_fillvalues` (`#1823 `__) `Manuel Schlund `__ -- Fix LWP of ICON on-the-fly CMORizer (`#1839 `__) `Manuel Schlund `__ -- Fixed issue in irregular regridding regarding scalar coordinates (`#1845 `__) `Manuel Schlund `__ -- Update product attributes and `metadata.yml` with cube metadata before saving files (`#1837 `__) `Manuel Schlund `__ -- Remove an extra space character from a filename (`#1883 `__) `Bouwe Andela `__ -- Improve resilience of ESGF search (`#1869 `__) `Bouwe Andela `__ -- Fix issue with no files found if timerange start/end differs in length (`#1880 `__) `Bouwe Andela `__ -- Add `driver` and `sub_experiment` tags to generate dataset aliases (`#1886 `__) `sloosvel `__ -- Fixed time points of native CESM2 output (`#1772 `__) `Manuel Schlund `__ -- Fix type hints for Python versions < 3.10 (`#1897 `__) `Bouwe Andela `__ -- Fixed `set_range_in_0_360` for dask arrays (`#1919 `__) `Manuel Schlund `__ -- Made equalized attributes in concatenated cubes consistent across runs (`#1783 `__) `Manuel Schlund `__ -- Fix issue with reading dates from files (`#1936 `__) `Bouwe Andela `__ -- Add institute name used on ESGF for CMIP5 CanAM4, CanCM4, and CanESM2 (`#1937 `__) `Bouwe Andela `__ -- Fix issue where data was not loaded and saved (`#1962 `__) `Bouwe Andela `__ -- Fix type hints for Python 3.8 (`#1795 `__) `Bouwe Andela `__ -- Update the institute facet of the CSIRO-Mk3L-1-2 model (`#1966 `__) `Rémi Kazeroni `__ -- Fixed race condition that may result in errors in :func:`esmvalcore.preprocessor.cleanup` (`#1949`_) `Manuel Schlund `__ -- Update notebook so it uses supplementaries instead of ancillaries (`#1945 `__) `Bouwe Andela `__ +- Import from :mod:`esmvalcore.config` in the :mod:`esmvalcore.experimental` module (:pull:`1816`) by :user:`bouweandela` +- Added scalar coords of input cubes to output of esmpy_regrid (:pull:`1811`) by :user:`schlunma` +- Fix severe bug in :func:`esmvalcore.preprocessor.mask_fillvalues` (:pull:`1823`) by :user:`schlunma` +- Fix LWP of ICON on-the-fly CMORizer (:pull:`1839`) by :user:`schlunma` +- Fixed issue in irregular regridding regarding scalar coordinates (:pull:`1845`) by :user:`schlunma` +- Update product attributes and `metadata.yml` with cube metadata before saving files (:pull:`1837`) by :user:`schlunma` +- Remove an extra space character from a filename (:pull:`1883`) by :user:`bouweandela` +- Improve resilience of ESGF search (:pull:`1869`) by :user:`bouweandela` +- Fix issue with no files found if timerange start/end differs in length (:pull:`1880`) by :user:`bouweandela` +- Add `driver` and `sub_experiment` tags to generate dataset aliases (:pull:`1886`) by :user:`sloosvel` +- Fixed time points of native CESM2 output (:pull:`1772`) by :user:`schlunma` +- Fix type hints for Python versions < 3.10 (:pull:`1897`) by :user:`bouweandela` +- Fixed `set_range_in_0_360` for dask arrays (:pull:`1919`) by :user:`schlunma` +- Made equalized attributes in concatenated cubes consistent across runs (:pull:`1783`) by :user:`schlunma` +- Fix issue with reading dates from files (:pull:`1936`) by :user:`bouweandela` +- Add institute name used on ESGF for CMIP5 CanAM4, CanCM4, and CanESM2 (:pull:`1937`) by :user:`bouweandela` +- Fix issue where data was not loaded and saved (:pull:`1962`) by :user:`bouweandela` +- Fix type hints for Python 3.8 (:pull:`1795`) by :user:`bouweandela` +- Update the institute facet of the CSIRO-Mk3L-1-2 model (:pull:`1966`) by :user:`remi-kazeroni` +- Fixed race condition that may result in errors in :func:`esmvalcore.preprocessor.cleanup` (:pull:`1949`) by :user:`schlunma` +- Update notebook so it uses supplementaries instead of ancillaries (:pull:`1945`) by :user:`bouweandela` Documentation ~~~~~~~~~~~~~ -- Fix anaconda badge in README (`#1759 `__) `Valeriu Predoi `__ -- Fix mistake in the documentation of :obj:`esmvalcore.esgf.find_files` (`#1784 `__) `Bouwe Andela `__ -- Support linking to "stable" ESMValTool version on readthedocs (`#1608 `__) `Bouwe Andela `__ -- Updated ICON doc with information on usage of extract_levels preprocessor (`#1903 `__) `Manuel Schlund `__ -- Add changelog for latest released version v2.7.1 (`#1905 `__) `Valeriu Predoi `__ -- Update `preprocessor.rst` due to renaming of NCEP dataset to NCEP-NCAR-R1 (`#1908 `__) `Birgit Hassler `__ -- Replace timerange nested lists in docs with overview table (`#1940 `__) `Klaus Zimmermann `__ -- Updated section "backward compatibility" in `contributing.rst` (`#1918 `__) `Axel Lauer `__ -- Add link to ESMValTool release procedure steps (`#1957 `__) `Rémi Kazeroni `__ -- Synchronize documentation table of contents with ESMValTool (`#1958 `__) `Bouwe Andela `__ +- Fix anaconda badge in README (:pull:`1759`) by :user:`valeriupredoi` +- Fix mistake in the documentation of :obj:`esmvalcore.esgf.find_files` (:pull:`1784`) by :user:`bouweandela` +- Support linking to "stable" ESMValTool version on readthedocs (:pull:`1608`) by :user:`bouweandela` +- Updated ICON doc with information on usage of extract_levels preprocessor (:pull:`1903`) by :user:`schlunma` +- Add changelog for latest released version v2.7.1 (:pull:`1905`) by :user:`valeriupredoi` +- Update `preprocessor.rst` due to renaming of NCEP dataset to NCEP-NCAR-R1 (:pull:`1908`) by :user:`hb326` +- Replace timerange nested lists in docs with overview table (:pull:`1940`) by :user:`zklaus` +- Updated section "backward compatibility" in `contributing.rst` (:pull:`1918`) by :user:`axel-lauer` +- Add link to ESMValTool release procedure steps (:pull:`1957`) by :user:`remi-kazeroni` +- Synchronize documentation table of contents with ESMValTool (:pull:`1958`) by :user:`bouweandela` Improvements ~~~~~~~~~~~~ -- Support wildcards in the recipe and improve support for ancillary variables and dataset versioning (`#1609`_) `Bouwe Andela `__. More details on how to adapt your recipes are given in the corresponding pull request description and in the corresponding sections of the `recipe documentation `__ and the `preprocessor documentation `__. -- Create a session directory with suffix "-1", "-2", etc if it already exists (`#1818 `__) `Bouwe Andela `__ -- Message for users when they use esmvaltool executable from esmvalcore only (`#1831 `__) `Valeriu Predoi `__ -- Order recipe output in index.html (`#1899 `__) `Bouwe Andela `__ -- Improve reading facets from ESGF search results (`#1920 `__) `Bouwe Andela `__ +- Support wildcards in the recipe and improve support for ancillary variables and dataset versioning (:pull:`1609`) by :user:`bouweandela`. More details on how to adapt your recipes are given in the corresponding pull request description and in the corresponding sections of the `recipe documentation `__ and the `preprocessor documentation `__. +- Create a session directory with suffix "-1", "-2", etc if it already exists (:pull:`1818`) by :user:`bouweandela` +- Message for users when they use esmvaltool executable from esmvalcore only (:pull:`1831`) by :user:`valeriupredoi` +- Order recipe output in index.html (:pull:`1899`) by :user:`bouweandela` +- Improve reading facets from ESGF search results (:pull:`1920`) by :user:`bouweandela` Fixes for datasets ~~~~~~~~~~~~~~~~~~ -- Fix rotated coordinate grids and `tas` and `pr` for CORDEX datasets (`#1765 `__) `sloosvel `__ -- Made ICON output UGRID-compliant (on-the-fly) (`#1664 `__) `Manuel Schlund `__ -- Fix automatic download of ICON grid file and make ICON UGRIDization optional (`default: true`) (`#1922 `__) `Manuel Schlund `__ -- Add siconc fixes for EC-Earth3-Veg and EC-Earth3-Veg-LR models (`#1771 `__) `Evgenia Galytska `__ -- Fix siconc in KIOST-ESM (`#1829 `__) `Lisa Bock `__ -- Extension of ERA5 CMORizer (variable cl) (`#1850 `__) `Axel Lauer `__ -- Add standard variable names for EMAC (`#1853 `__) `FranziskaWinterstein `__ -- Fix for FGOALS-f3-L clt (`#1928 `__) `Lisa Bock `__ +- Fix rotated coordinate grids and `tas` and `pr` for CORDEX datasets (:pull:`1765`) by :user:`sloosvel` +- Made ICON output UGRID-compliant (on-the-fly) (:pull:`1664`) by :user:`schlunma` +- Fix automatic download of ICON grid file and make ICON UGRIDization optional (`default: true`) (:pull:`1922`) by :user:`schlunma` +- Add siconc fixes for EC-Earth3-Veg and EC-Earth3-Veg-LR models (:pull:`1771`) by :user:`egalytska` +- Fix siconc in KIOST-ESM (:pull:`1829`) by :user:`LisaBock` +- Extension of ERA5 CMORizer (variable cl) (:pull:`1850`) by :user:`axel-lauer` +- Add standard variable names for EMAC (:pull:`1853`) by :user:`FranziskaWinterstein` +- Fix for FGOALS-f3-L clt (:pull:`1928`) by :user:`LisaBock` Installation ~~~~~~~~~~~~ -- Add all deps to the conda-forge environment and suppress installing and reinstalling deps with pip at readthedocs builds (`#1786 `__) `Valeriu Predoi `__ -- Pin netCDF4<1.6.1 (`#1805 `__) `Bouwe Andela `__ -- Unpin NetCF4 (`#1814 `__) `Valeriu Predoi `__ -- Unpin flake8 (`#1820 `__) `Valeriu Predoi `__ -- Add iris-esmf-regrid as a dependency (`#1809 `__) `sloosvel `__ -- Pin esmpy<8.4 (`#1871 `__) `Klaus Zimmermann `__ -- Update esmpy import for ESMF v8.4.0 (`#1876 `__) `Bouwe Andela `__ +- Add all deps to the conda-forge environment and suppress installing and reinstalling deps with pip at readthedocs builds (:pull:`1786`) by :user:`valeriupredoi` +- Pin netCDF4<1.6.1 (:pull:`1805`) by :user:`bouweandela` +- Unpin NetCF4 (:pull:`1814`) by :user:`valeriupredoi` +- Unpin flake8 (:pull:`1820`) by :user:`valeriupredoi` +- Add iris-esmf-regrid as a dependency (:pull:`1809`) by :user:`sloosvel` +- Pin esmpy<8.4 (:pull:`1871`) by :user:`zklaus` +- Update esmpy import for ESMF v8.4.0 (:pull:`1876`) by :user:`bouweandela` Preprocessor ~~~~~~~~~~~~ -- Allow :func:`esmvalcore.preprocessor.multi_model_statistics` on cubes with arbitrary dimensions (`#1808 `__) `Manuel Schlund `__ -- Smarter removal of coordinate metadata in :func:`esmvalcore.preprocessor.multi_model_statistics` preprocessor (`#1813 `__) `Manuel Schlund `__ -- Allowed usage of :func:`esmvalcore.preprocessor.multi_model_statistics` on single cubes/products (`#1849 `__) `Manuel Schlund `__ -- Allowed usage of :func:`esmvalcore.preprocessor.multi_model_statistics` on cubes with identical ``name()`` and ``units`` (but e.g. different long_name) (`#1921 `__) `Manuel Schlund `__ -- Allowed ignoring scalar coordinates in :func:`esmvalcore.preprocessor.multi_model_statistics` (`#1934 `__) `Manuel Schlund `__ -- Refactored :func:`esmvalcore.preprocessor.regrid` and removed unnecessary code not needed anymore due to new iris version (`#1898 `__) `Manuel Schlund `__ -- Do not realise coordinates during CMOR check (`#1912 `__) `sloosvel `__ -- Make :func:`esmvalcore.preprocessor.extract_volume` work with closed and mixed intervals and allow nearest value selection (`#1930 `__) `sloosvel `__ +- Allow :func:`esmvalcore.preprocessor.multi_model_statistics` on cubes with arbitrary dimensions (:pull:`1808`) by :user:`schlunma` +- Smarter removal of coordinate metadata in :func:`esmvalcore.preprocessor.multi_model_statistics` preprocessor (:pull:`1813`) by :user:`schlunma` +- Allowed usage of :func:`esmvalcore.preprocessor.multi_model_statistics` on single cubes/products (:pull:`1849`) by :user:`schlunma` +- Allowed usage of :func:`esmvalcore.preprocessor.multi_model_statistics` on cubes with identical ``name()`` and ``units`` (but e.g. different long_name) (:pull:`1921`) by :user:`schlunma` +- Allowed ignoring scalar coordinates in :func:`esmvalcore.preprocessor.multi_model_statistics` (:pull:`1934`) by :user:`schlunma` +- Refactored :func:`esmvalcore.preprocessor.regrid` and removed unnecessary code not needed anymore due to new iris version (:pull:`1898`) by :user:`schlunma` +- Do not realise coordinates during CMOR check (:pull:`1912`) by :user:`sloosvel` +- Make :func:`esmvalcore.preprocessor.extract_volume` work with closed and mixed intervals and allow nearest value selection (:pull:`1930`) by :user:`sloosvel` Release ~~~~~~~ -- Changelog for `v2.8.0rc1` (`#1952 `__) `Rémi Kazeroni `__ -- Increase version number for ESMValCore `v2.8.0rc1` (`#1955 `__) `Rémi Kazeroni `__ -- Changelog for `v2.8.0rc2` (`#1959 `__) `Rémi Kazeroni `__ -- Increase version number for ESMValCore `v2.8.0rc2` (`#1973 `__) `Rémi Kazeroni `__ -- Changelog for `v2.8.0` (`#1978 `__) `Rémi Kazeroni `__ -- Increase version number for ESMValCore `v2.8.0` (`#1983 `__) `Rémi Kazeroni `__ +- Changelog for `v2.8.0rc1` (:pull:`1952`) by :user:`remi-kazeroni` +- Increase version number for ESMValCore `v2.8.0rc1` (:pull:`1955`) by :user:`remi-kazeroni` +- Changelog for `v2.8.0rc2` (:pull:`1959`) by :user:`remi-kazeroni` +- Increase version number for ESMValCore `v2.8.0rc2` (:pull:`1973`) by :user:`remi-kazeroni` +- Changelog for `v2.8.0` (:pull:`1978`) by :user:`remi-kazeroni` +- Increase version number for ESMValCore `v2.8.0` (:pull:`1983`) by :user:`remi-kazeroni` Automatic testing ~~~~~~~~~~~~~~~~~ -- Set implicit optional to true in `mypy` config to avert side effects and test fails from new mypy version (`#1790 `__) `Valeriu Predoi `__ -- Remove duplicate `implicit_optional = True` line in ``setup.cfg`` (`#1791 `__) `Valeriu Predoi `__ -- Fix failing test due to missing sample data (`#1797 `__) `Bouwe Andela `__ -- Remove outdated cmor_table facet from data finder tests (`#1798 `__) `Bouwe Andela `__ -- Modernize tests for :func:`esmvalcore.preprocessor.save` (`#1799 `__) `Bouwe Andela `__ -- No more sequential tests since SegFaults were not noticed anymore (`#1819 `__) `Valeriu Predoi `__ -- Update pre-commit configuration (`#1821 `__) `Bouwe Andela `__ -- Updated URL of ICON grid file used for testing (`#1914 `__) `Manuel Schlund `__ +- Set implicit optional to true in `mypy` config to avert side effects and test fails from new mypy version (:pull:`1790`) by :user:`valeriupredoi` +- Remove duplicate `implicit_optional = True` line in ``setup.cfg`` (:pull:`1791`) by :user:`valeriupredoi` +- Fix failing test due to missing sample data (:pull:`1797`) by :user:`bouweandela` +- Remove outdated cmor_table facet from data finder tests (:pull:`1798`) by :user:`bouweandela` +- Modernize tests for :func:`esmvalcore.preprocessor.save` (:pull:`1799`) by :user:`bouweandela` +- No more sequential tests since SegFaults were not noticed anymore (:pull:`1819`) by :user:`valeriupredoi` +- Update pre-commit configuration (:pull:`1821`) by :user:`bouweandela` +- Updated URL of ICON grid file used for testing (:pull:`1914`) by :user:`schlunma` Variable Derivation ~~~~~~~~~~~~~~~~~~~ -- Add derivation of sea ice extent (`#1695 `__) `sloosvel `__ +- Add derivation of sea ice extent (:pull:`1695`) by :user:`sloosvel` .. _changelog-v2-7-1: @@ -493,19 +500,19 @@ This is a bugfix release where we unpin `cf-units` to allow the latest `iris=3.4 Installation ~~~~~~~~~~~~ -- Set the version number on the development branches to one minor version more than the previous release (`#1854 `__) `Bouwe Andela `__ -- Unpin cf-units (`#1770 `__) `Bouwe Andela `__ +- Set the version number on the development branches to one minor version more than the previous release (:pull:`1854`) by :user:`bouweandela` +- Unpin cf-units (:pull:`1770`) by :user:`bouweandela` Bug fixes ~~~~~~~~~ -- Improve error handling if an esgf index node is offline (`#1834 `__) `Bouwe Andela `__ +- Improve error handling if an esgf index node is offline (:pull:`1834`) by :user:`bouweandela` Automatic testing ~~~~~~~~~~~~~~~~~ -- Removed unnecessary test that fails with iris 3.4.0 (`#1846 `__) `Manuel Schlund `__ -- Update CEDA ESGF index node hostname (`#1838 `__) `Valeriu Predoi `__ +- Removed unnecessary test that fails with iris 3.4.0 (:pull:`1846`) by :user:`schlunma` +- Update CEDA ESGF index node hostname (:pull:`1838`) by :user:`valeriupredoi` .. _changelog-v2-7-0: @@ -516,67 +523,67 @@ v2.7.0 Highlights ~~~~~~~~~~ -- We have a new preprocessor function called `'rolling_window_statistics' `__ implemented by `Liza Malinina `__ -- We have improved the support for native models, refactored native model fixes by adding common base class `NativeDatasetFix`, changed default DRS for reading native ICON output, and added tests for input/output filenames for `ICON `__ and `EMAC `__ on-the-fly CMORizer, all these features courtesy of `Manuel Schlund `__ -- Performance of preprocessor functions that use time dimensions has been sped up by **two orders of magnitude** thanks to contributions by `Bouwe Andela `__ +- We have a new preprocessor function called `'rolling_window_statistics' `__ implemented by :user:`malininae` +- We have improved the support for native models, refactored native model fixes by adding common base class `NativeDatasetFix`, changed default DRS for reading native ICON output, and added tests for input/output filenames for `ICON `__ and `EMAC `__ on-the-fly CMORizer, all these features courtesy of :user:`schlunma` +- Performance of preprocessor functions that use time dimensions has been sped up by **two orders of magnitude** thanks to contributions by :user:`bouweandela` This release includes: Backwards incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- Change default DRS for reading native ICON output (`#1705 `__) `Manuel Schlund `__ +- Change default DRS for reading native ICON output (:pull:`1705`) by :user:`schlunma` Bug fixes ~~~~~~~~~ -- Add support for regions stored as MultiPolygon to extract_shape preprocessor (`#1670 `__) `Bouwe Andela `__ -- Fixed type annotations for Python 3.8 (`#1700 `__) `Manuel Schlund `__ -- Core `_io.concatenate()` may fail due to case when one of the cubes is scalar - this fixes that (`#1715 `__) `Valeriu Predoi `__ -- Pick up esmvalcore badge instead of esmvaltool one in README (`#1749 `__) `Valeriu Predoi `__ -- Restore support for scalar cubes to time selection preprocessor functions (`#1750 `__) `Bouwe Andela `__ -- Fix calculation of precipitation flux in EMAC on-the-fly CMORizer (`#1755 `__) `Manuel Schlund `__ +- Add support for regions stored as MultiPolygon to extract_shape preprocessor (:pull:`1670`) by :user:`bouweandela` +- Fixed type annotations for Python 3.8 (:pull:`1700`) by :user:`schlunma` +- Core `_io.concatenate()` may fail due to case when one of the cubes is scalar - this fixes that (:pull:`1715`) by :user:`valeriupredoi` +- Pick up esmvalcore badge instead of esmvaltool one in README (:pull:`1749`) by :user:`valeriupredoi` +- Restore support for scalar cubes to time selection preprocessor functions (:pull:`1750`) by :user:`bouweandela` +- Fix calculation of precipitation flux in EMAC on-the-fly CMORizer (:pull:`1755`) by :user:`schlunma` Deprecations ~~~~~~~~~~~~ -- Remove deprecation warning for regrid schemes already deprecated for v2.7.0 (`#1753 `__) `Valeriu Predoi `__ +- Remove deprecation warning for regrid schemes already deprecated for v2.7.0 (:pull:`1753`) by :user:`valeriupredoi` Documentation ~~~~~~~~~~~~~ -- Add Met Office Installation Method (`#1692 `__) `mo-tgeddes `__ -- Add MO-paths to config file (`#1709 `__) `mo-tgeddes `__ -- Update MO obs4MIPs paths in the user configuration file (`#1734 `__) `mo-tgeddes `__ -- Update `Making a release` section of the documentation (`#1689 `__) `sloosvel `__ -- Added changelog for v2.7.0 (`#1746 `__) `Valeriu Predoi `__ -- update CITATION.cff file with 2.7.0 release info (`#1757 `__) `Valeriu Predoi `__ +- Add Met Office Installation Method (:pull:`1692`) by :user:`mo-tgeddes` +- Add MO-paths to config file (:pull:`1709`) by :user:`mo-tgeddes` +- Update MO obs4MIPs paths in the user configuration file (:pull:`1734`) by :user:`mo-tgeddes` +- Update `Making a release` section of the documentation (:pull:`1689`) by :user:`sloosvel` +- Added changelog for v2.7.0 (:pull:`1746`) by :user:`valeriupredoi` +- update CITATION.cff file with 2.7.0 release info (:pull:`1757`) by :user:`valeriupredoi` Improvements ~~~~~~~~~~~~ -- New preprocessor function 'rolling_window_statistics' (`#1702 `__) `Liza Malinina `__ -- Remove `pytest_flake8` plugin and use `flake8` instead (`#1722 `__) `Valeriu Predoi `__ -- Added CESM2 CMORizer (`#1678 `__) `Manuel Schlund `__ -- Speed up functions that use time dimension (`#1713 `__) `Bouwe Andela `__ -- Modernize and minimize pylint configuration (`#1726 `__) `Bouwe Andela `__ +- New preprocessor function 'rolling_window_statistics' (:pull:`1702`) by :user:`malininae` +- Remove `pytest_flake8` plugin and use `flake8` instead (:pull:`1722`) by :user:`valeriupredoi` +- Added CESM2 CMORizer (:pull:`1678`) by :user:`schlunma` +- Speed up functions that use time dimension (:pull:`1713`) by :user:`bouweandela` +- Modernize and minimize pylint configuration (:pull:`1726`) by :user:`bouweandela` Fixes for datasets ~~~~~~~~~~~~~~~~~~ -- Refactored native model fixes by adding common base class `NativeDatasetFix` (`#1694 `__) `Manuel Schlund `__ +- Refactored native model fixes by adding common base class `NativeDatasetFix` (:pull:`1694`) by :user:`schlunma` Installation ~~~~~~~~~~~~ -- Pin `netCDF4 != 1.6.1` since that seems to throw a flurry of Segmentation Faults (`#1724 `__) `Valeriu Predoi `__ +- Pin `netCDF4 != 1.6.1` since that seems to throw a flurry of Segmentation Faults (:pull:`1724`) by :user:`valeriupredoi` Automatic testing ~~~~~~~~~~~~~~~~~ -- Pin `flake8<5.0.0` since Circle CI tests are failing copiously (`#1698 `__) `Valeriu Predoi `__ -- Added tests for input/output filenames for ICON and EMAC on-the-fly CMORizer (`#1718 `__) `Manuel Schlund `__ -- Fix failed tests for Python<3.10 resulting from typing (`#1748 `__) `Manuel Schlund `__ +- Pin `flake8<5.0.0` since Circle CI tests are failing copiously (:pull:`1698`) by :user:`valeriupredoi` +- Added tests for input/output filenames for ICON and EMAC on-the-fly CMORizer (:pull:`1718`) by :user:`schlunma` +- Fix failed tests for Python<3.10 resulting from typing (:pull:`1748`) by :user:`schlunma` .. _changelog-v2-6-0: @@ -594,109 +601,109 @@ This release includes Deprecations ~~~~~~~~~~~~ -- Deprecate the function `esmvalcore.var_name_constraint` (`#1592 `__) `Manuel Schlund `__. This function is scheduled for removal in v2.8.0. Please use :class:`iris.NameConstraint` with the keyword argument `var_name` instead: this is an exact replacement. +- Deprecate the function `esmvalcore.var_name_constraint` (:pull:`1592`) by :user:`schlunma`. This function is scheduled for removal in v2.8.0. Please use :class:`iris.NameConstraint` with the keyword argument `var_name` instead: this is an exact replacement. Bug fixes ~~~~~~~~~ -- Added `start_year` and `end_year` attributes to derived variables (`#1547 `__) `Manuel Schlund `__ -- Show all results on recipe results webpage (`#1560 `__) `Bouwe Andela `__ -- Regridding regular grids with similar coordinates (`#1567 `__) `Tomas Lovato `__ -- Fix timerange wildcard search when deriving variables or downloading files (`#1562 `__) `sloosvel `__ -- Fix `force_derivation` bug (`#1627 `__) `sloosvel `__ -- Correct `build-and-deploy-on-pypi` action (`#1634 `__) `sloosvel `__ -- Apply `clip_timerange` to time dependent fx variables (`#1603 `__) `sloosvel `__ -- Correctly handle requests.exceptions.ConnectTimeout when an ESGF index node is offline (`#1638 `__) `Bouwe Andela `__ +- Added `start_year` and `end_year` attributes to derived variables (:pull:`1547`) by :user:`schlunma` +- Show all results on recipe results webpage (:pull:`1560`) by :user:`bouweandela` +- Regridding regular grids with similar coordinates (:pull:`1567`) by :user:`tomaslovato` +- Fix timerange wildcard search when deriving variables or downloading files (:pull:`1562`) by :user:`sloosvel` +- Fix `force_derivation` bug (:pull:`1627`) by :user:`sloosvel` +- Correct `build-and-deploy-on-pypi` action (:pull:`1634`) by :user:`sloosvel` +- Apply `clip_timerange` to time dependent fx variables (:pull:`1603`) by :user:`sloosvel` +- Correctly handle requests.exceptions.ConnectTimeout when an ESGF index node is offline (:pull:`1638`) by :user:`bouweandela` CMOR standard ~~~~~~~~~~~~~ -- Added custom CMOR tables used for EMAC CMORizer (`#1599 `__) `Manuel Schlund `__ -- Extended ICON CMORizer (`#1549 `__) `Manuel Schlund `__ -- Add CMOR check exception for a basin coord named sector (`#1612 `__) `David Hohn `__ -- Custom user-defined location for custom CMOR tables (`#1625 `__) `Manuel Schlund `__ +- Added custom CMOR tables used for EMAC CMORizer (:pull:`1599`) by :user:`schlunma` +- Extended ICON CMORizer (:pull:`1549`) by :user:`schlunma` +- Add CMOR check exception for a basin coord named sector (:pull:`1612`) by :user:`dhohn` +- Custom user-defined location for custom CMOR tables (:pull:`1625`) by :user:`schlunma` Containerization ~~~~~~~~~~~~~~~~ -- Remove update command in Dockerfile (`#1630 `__) `sloosvel `__ +- Remove update command in Dockerfile (:pull:`1630`) by :user:`sloosvel` Community ~~~~~~~~~ -- Add David Hohn to contributors' list (`#1586 `__) `Valeriu Predoi `__ +- Add David Hohn to contributors' list (:pull:`1586`) by :user:`valeriupredoi` Documentation ~~~~~~~~~~~~~ -- [Github Actions Docs] Full explanation on how to use the GA test triggered by PR comment and added docs link for GA hosted runners (`#1553 `__) `Valeriu Predoi `__ -- Update the command for building the documentation (`#1556 `__) `Bouwe Andela `__ -- Update documentation on running the tool (`#1400 `__) `Bouwe Andela `__ -- Add support for DKRZ-Levante (`#1558 `__) `Rémi Kazeroni `__ -- Improved documentation on native dataset support (`#1559 `__) `Manuel Schlund `__ -- Tweak `extract_point` preprocessor: explain what it returns if one point coord outside cube and add explicit test (`#1584 `__) `Valeriu Predoi `__ -- Update CircleCI, readthedocs, and Docker configuration (`#1588 `__) `Bouwe Andela `__ -- Remove support for Mistral in `config-user.yml` (`#1620 `__) `Rémi Kazeroni `__ -- Add changelog for v2.6.0rc1 (`#1633 `__) `sloosvel `__ -- Add a note on transferring permissions to the release manager (`#1645 `__) `Bouwe Andela `__ -- Add documentation on building and uploading Docker images (`#1644 `__) `Bouwe Andela `__ -- Update documentation on ESMValTool module at DKRZ (`#1647 `__) `Rémi Kazeroni `__ -- Expanded information on deprecations in changelog (`#1658 `__) `Manuel Schlund `__ +- [Github Actions Docs] Full explanation on how to use the GA test triggered by PR comment and added docs link for GA hosted runners (:pull:`1553`) by :user:`valeriupredoi` +- Update the command for building the documentation (:pull:`1556`) by :user:`bouweandela` +- Update documentation on running the tool (:pull:`1400`) by :user:`bouweandela` +- Add support for DKRZ-Levante (:pull:`1558`) by :user:`remi-kazeroni` +- Improved documentation on native dataset support (:pull:`1559`) by :user:`schlunma` +- Tweak `extract_point` preprocessor: explain what it returns if one point coord outside cube and add explicit test (:pull:`1584`) by :user:`valeriupredoi` +- Update CircleCI, readthedocs, and Docker configuration (:pull:`1588`) by :user:`bouweandela` +- Remove support for Mistral in `config-user.yml` (:pull:`1620`) by :user:`remi-kazeroni` +- Add changelog for v2.6.0rc1 (:pull:`1633`) by :user:`sloosvel` +- Add a note on transferring permissions to the release manager (:pull:`1645`) by :user:`bouweandela` +- Add documentation on building and uploading Docker images (:pull:`1644`) by :user:`bouweandela` +- Update documentation on ESMValTool module at DKRZ (:pull:`1647`) by :user:`remi-kazeroni` +- Expanded information on deprecations in changelog (:pull:`1658`) by :user:`schlunma` Improvements ~~~~~~~~~~~~ -- Removed trailing whitespace in custom CMOR tables (`#1564 `__) `Manuel Schlund `__ -- Try searching multiple ESGF index nodes (`#1561 `__) `Bouwe Andela `__ -- Add CMIP6 `amoc` derivation case and add a test (`#1577 `__) `Valeriu Predoi `__ -- Added EMAC CMORizer (`#1554 `__) `Manuel Schlund `__ -- Improve performance of `volume_statistics` (`#1545 `__) `sloosvel `__ +- Removed trailing whitespace in custom CMOR tables (:pull:`1564`) by :user:`schlunma` +- Try searching multiple ESGF index nodes (:pull:`1561`) by :user:`bouweandela` +- Add CMIP6 `amoc` derivation case and add a test (:pull:`1577`) by :user:`valeriupredoi` +- Added EMAC CMORizer (:pull:`1554`) by :user:`schlunma` +- Improve performance of `volume_statistics` (:pull:`1545`) by :user:`sloosvel` Fixes for datasets ~~~~~~~~~~~~~~~~~~ -- Fixes of ocean variables in multiple CMIP6 datasets (`#1566 `__) `Tomas Lovato `__ -- Ensure lat/lon bounds in FGOALS-l3 atmos variables are contiguous (`#1571 `__) `sloosvel `__ -- Added `AllVars` fix for CMIP6's ICON-ESM-LR (`#1582 `__) `Manuel Schlund `__ +- Fixes of ocean variables in multiple CMIP6 datasets (:pull:`1566`) by :user:`tomaslovato` +- Ensure lat/lon bounds in FGOALS-l3 atmos variables are contiguous (:pull:`1571`) by :user:`sloosvel` +- Added `AllVars` fix for CMIP6's ICON-ESM-LR (:pull:`1582`) by :user:`schlunma` Installation ~~~~~~~~~~~~ -- Removed `package/meta.yml` (`#1540 `__) `Manuel Schlund `__ -- Pinned iris>=3.2.1 (`#1552 `__) `Manuel Schlund `__ -- Use setuptools-scm to automatically generate the version number (`#1578 `__) `Bouwe Andela `__ -- Pin cf-units to lower than 3.1.0 to temporarily avoid changes within new version related to calendars (`#1659 `__) `Valeriu Predoi `__ +- Removed `package/meta.yml` (:pull:`1540`) by :user:`schlunma` +- Pinned iris>=3.2.1 (:pull:`1552`) by :user:`schlunma` +- Use setuptools-scm to automatically generate the version number (:pull:`1578`) by :user:`bouweandela` +- Pin cf-units to lower than 3.1.0 to temporarily avoid changes within new version related to calendars (:pull:`1659`) by :user:`valeriupredoi` Preprocessor ~~~~~~~~~~~~ -- Allowed special case for unit conversion of precipitation (`kg m-2 s-1` <--> `mm day-1`) (`#1574 `__) `Manuel Schlund `__ -- Add general `extract_coordinate_points` preprocessor (`#1581 `__) `sloosvel `__ -- Add preprocessor `accumulate_coordinate` (`#1281 `__) `Javier Vegas-Regidor `__ -- Add `axis_statistics` and improve `depth_integration` (`#1589 `__) `sloosvel `__ +- Allowed special case for unit conversion of precipitation (`kg m-2 s-1` <--> `mm day-1`) (:pull:`1574`) by :user:`schlunma` +- Add general `extract_coordinate_points` preprocessor (:pull:`1581`) by :user:`sloosvel` +- Add preprocessor `accumulate_coordinate` (:pull:`1281`) by :user:`jvegreg` +- Add `axis_statistics` and improve `depth_integration` (:pull:`1589`) by :user:`sloosvel` Release ~~~~~~~ -- Increase version number for ESMValCore v2.6.0rc1 (`#1632 `__) `sloosvel `__ -- Update changelog and version for 2.6rc3 (`#1646 `__) `sloosvel `__ -- Add changelog for rc4 (`#1662 `__) `sloosvel `__ +- Increase version number for ESMValCore v2.6.0rc1 (:pull:`1632`) by :user:`sloosvel` +- Update changelog and version for 2.6rc3 (:pull:`1646`) by :user:`sloosvel` +- Add changelog for rc4 (:pull:`1662`) by :user:`sloosvel` Automatic testing ~~~~~~~~~~~~~~~~~ -- Refresh CircleCI cache weekly (`#1597 `__) `Bouwe Andela `__ -- Use correct cache restore key on CircleCI (`#1598 `__) `Bouwe Andela `__ -- Install git and ssh before checking out code on CircleCI (`#1601 `__) `Bouwe Andela `__ -- Fetch all history in Github Action tests (`#1622 `__) `sloosvel `__ -- Test Github Actions dashboard badge from meercode.io (`#1640 `__) `Valeriu Predoi `__ -- Improve esmvalcore.esgf unit test (`#1650 `__) `Bouwe Andela `__ +- Refresh CircleCI cache weekly (:pull:`1597`) by :user:`bouweandela` +- Use correct cache restore key on CircleCI (:pull:`1598`) by :user:`bouweandela` +- Install git and ssh before checking out code on CircleCI (:pull:`1601`) by :user:`bouweandela` +- Fetch all history in Github Action tests (:pull:`1622`) by :user:`sloosvel` +- Test Github Actions dashboard badge from meercode.io (:pull:`1640`) by :user:`valeriupredoi` +- Improve esmvalcore.esgf unit test (:pull:`1650`) by :user:`bouweandela` Variable Derivation ~~~~~~~~~~~~~~~~~~~ -- Added derivation of `hfns` (`#1594 `__) `Manuel Schlund `__ +- Added derivation of `hfns` (:pull:`1594`) by :user:`schlunma` .. _changelog-v2-5-0: @@ -715,115 +722,115 @@ This release includes Backwards incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- Update Cordex section in `config-developer.yml` (`#1303 `__) `francesco-cmcc `__. This changes the naming convention of ESMValCore's output files from CORDEX dataset. This only affects recipes that use CORDEX data. Most likely, no changes in diagnostics are necessary; however, if code relies on the specific naming convention of files, it might need to be adapted. -- Dropped Python 3.7 (`#1530 `__) `Manuel Schlund `__. ESMValCore v2.5.0 dropped support for Python 3.7. From now on Python >=3.8 is required to install ESMValCore. The main reason for this is that conda-forge dropped support for Python 3.7 for OSX and arm64 (more details are given `here `__). +- Update Cordex section in `config-developer.yml` (:pull:`1303`) by :user:`francesco-cmcc`. This changes the naming convention of ESMValCore's output files from CORDEX dataset. This only affects recipes that use CORDEX data. Most likely, no changes in diagnostics are necessary; however, if code relies on the specific naming convention of files, it might need to be adapted. +- Dropped Python 3.7 (:pull:`1530`) by :user:`schlunma`. ESMValCore v2.5.0 dropped support for Python 3.7. From now on Python >=3.8 is required to install ESMValCore. The main reason for this is that conda-forge dropped support for Python 3.7 for OSX and arm64 (more details are given `here `__). Bug fixes ~~~~~~~~~ -- Fix `extract_shape` when fx vars are present (`#1403 `__) `sloosvel `__ -- Added support of `extra_facets` to fx variables added by the preprocessor (`#1399 `__) `Manuel Schlund `__ -- Augmented input for derived variables with extra_facets (`#1412 `__) `Manuel Schlund `__ -- Correctly use masked arrays after `unstructured_nearest` regridding (`#1414 `__) `Manuel Schlund `__ -- Fixing the broken derivation script for XCH4 (and XCO2) (`#1428 `__) `Birgit Hassler `__ -- Ignore `.pymon-journal` file in test discovery (`#1436 `__) `Valeriu Predoi `__ -- Fixed bug that caused automatic download to fail in rare cases (`#1442 `__) `Manuel Schlund `__ -- Add new `JULIA_LOAD_PATH` to diagnostic task test (`#1444 `__) `Valeriu Predoi `__ -- Fix provenance file permissions (`#1468 `__) `Bouwe Andela `__ -- Fixed usage of `statistics=std_dev` option in multi-model statistics preprocessors (`#1478 `__) `Manuel Schlund `__ -- Removed scalar coordinates `p0` and `ptop` prior to merge in `multi_model_statistics` (`#1471 `__) `Axel Lauer `__ -- Added `dataset` and `alias` attributes to `multi_model_statistics` output (`#1483 `__) `Manuel Schlund `__ -- Fixed issues with multi-model-statistics timeranges (`#1486 `__) `Manuel Schlund `__ -- Fixed output messages for CMOR logging (`#1494 `__) `Manuel Schlund `__ -- Fixed `clip_timerange` if only a single time point is extracted (`#1497 `__) `Manuel Schlund `__ -- Fixed chunking in `multi_model_statistics` (`#1500 `__) `Manuel Schlund `__ -- Fixed renaming of auxiliary coordinates in `multi_model_statistics` if coordinates are equal (`#1502 `__) `Manuel Schlund `__ -- Fixed timerange selection for automatic downloads (`#1517 `__) `Manuel Schlund `__ -- Fixed chunking in `multi_model_statistics` (`#1524 `__) `Manuel Schlund `__ +- Fix `extract_shape` when fx vars are present (:pull:`1403`) by :user:`sloosvel` +- Added support of `extra_facets` to fx variables added by the preprocessor (:pull:`1399`) by :user:`schlunma` +- Augmented input for derived variables with extra_facets (:pull:`1412`) by :user:`schlunma` +- Correctly use masked arrays after `unstructured_nearest` regridding (:pull:`1414`) by :user:`schlunma` +- Fixing the broken derivation script for XCH4 (and XCO2) (:pull:`1428`) by :user:`hb326` +- Ignore `.pymon-journal` file in test discovery (:pull:`1436`) by :user:`valeriupredoi` +- Fixed bug that caused automatic download to fail in rare cases (:pull:`1442`) by :user:`schlunma` +- Add new `JULIA_LOAD_PATH` to diagnostic task test (:pull:`1444`) by :user:`valeriupredoi` +- Fix provenance file permissions (:pull:`1468`) by :user:`bouweandela` +- Fixed usage of `statistics=std_dev` option in multi-model statistics preprocessors (:pull:`1478`) by :user:`schlunma` +- Removed scalar coordinates `p0` and `ptop` prior to merge in `multi_model_statistics` (:pull:`1471`) by :user:`axel-lauer` +- Added `dataset` and `alias` attributes to `multi_model_statistics` output (:pull:`1483`) by :user:`schlunma` +- Fixed issues with multi-model-statistics timeranges (:pull:`1486`) by :user:`schlunma` +- Fixed output messages for CMOR logging (:pull:`1494`) by :user:`schlunma` +- Fixed `clip_timerange` if only a single time point is extracted (:pull:`1497`) by :user:`schlunma` +- Fixed chunking in `multi_model_statistics` (:pull:`1500`) by :user:`schlunma` +- Fixed renaming of auxiliary coordinates in `multi_model_statistics` if coordinates are equal (:pull:`1502`) by :user:`schlunma` +- Fixed timerange selection for automatic downloads (:pull:`1517`) by :user:`schlunma` +- Fixed chunking in `multi_model_statistics` (:pull:`1524`) by :user:`schlunma` Deprecations ~~~~~~~~~~~~ -- Renamed vertical regridding schemes (`#1429 `__) `Manuel Schlund `__. Old regridding schemes are supported until v2.7.0. For details, see :ref:`Vertical interpolation schemes `. +- Renamed vertical regridding schemes (:pull:`1429`) by :user:`schlunma`. Old regridding schemes are supported until v2.7.0. For details, see :ref:`Vertical interpolation schemes `. Documentation ~~~~~~~~~~~~~ -- Remove duplicate entries in changelog (`#1391 `__) `Klaus Zimmermann `__ -- Documentation on how to use HPC central installations (`#1409 `__) `Valeriu Predoi `__ -- Correct brackets in preprocessor documentation for list of seasons (`#1420 `__) `Bouwe Andela `__ -- Add Python=3.10 to package info, update Circle CI auto install and documentation for Python=3.10 (`#1432 `__) `Valeriu Predoi `__ -- Reverted unintentional change in `.zenodo.json` (`#1452 `__) `Manuel Schlund `__ -- Synchronized config-user.yml with version from ESMValTool (`#1453 `__) `Manuel Schlund `__ -- Solved issues in configuration files (`#1457 `__) `Manuel Schlund `__ -- Add direct link to download conda lock file in the install documentation (`#1462 `__) `Valeriu Predoi `__ -- CITATION.cff fix and automatic validation of citation metadata (`#1467 `__) `Valeriu Predoi `__ -- Updated documentation on how to deprecate features (`#1426 `__) `Manuel Schlund `__ -- Added reference hook to conda lock in documentation install section (`#1473 `__) `Valeriu Predoi `__ -- Increased ESMValCore version to 2.5.0rc1 (`#1477 `__) `Manuel Schlund `__ -- Added changelog for v2.5.0 release (`#1476 `__) `Manuel Schlund `__ -- Increased ESMValCore version to 2.5.0rc2 (`#1487 `__) `Manuel Schlund `__ -- Added some authors to citation and zenodo files (`#1488 `__) `SarahAlidoost `__ -- Restored `scipy` intersphinx mapping (`#1491 `__) `Manuel Schlund `__ -- Increased ESMValCore version to 2.5.0rc3 (`#1504 `__) `Manuel Schlund `__ -- Fix download instructions for the MSWEP dataset (`#1506 `__) `Rémi Kazeroni `__ -- Documentation updated for the new cmorizer framework (`#1417 `__) `Rémi Kazeroni `__ -- Added tests for duplicates in changelog and removed duplicates (`#1508 `__) `Manuel Schlund `__ -- Increased ESMValCore version to 2.5.0rc4 (`#1519 `__) `Manuel Schlund `__ -- Add Github Actions Test badge in README (`#1526 `__) `Valeriu Predoi `__ -- Increased ESMValCore version to 2.5.0rc5 (`#1529 `__) `Manuel Schlund `__ -- Increased ESMValCore version to 2.5.0rc6 (`#1532 `__) `Manuel Schlund `__ +- Remove duplicate entries in changelog (:pull:`1391`) by :user:`zklaus` +- Documentation on how to use HPC central installations (:pull:`1409`) by :user:`valeriupredoi` +- Correct brackets in preprocessor documentation for list of seasons (:pull:`1420`) by :user:`bouweandela` +- Add Python=3.10 to package info, update Circle CI auto install and documentation for Python=3.10 (:pull:`1432`) by :user:`valeriupredoi` +- Reverted unintentional change in `.zenodo.json` (:pull:`1452`) by :user:`schlunma` +- Synchronized config-user.yml with version from ESMValTool (:pull:`1453`) by :user:`schlunma` +- Solved issues in configuration files (:pull:`1457`) by :user:`schlunma` +- Add direct link to download conda lock file in the install documentation (:pull:`1462`) by :user:`valeriupredoi` +- CITATION.cff fix and automatic validation of citation metadata (:pull:`1467`) by :user:`valeriupredoi` +- Updated documentation on how to deprecate features (:pull:`1426`) by :user:`schlunma` +- Added reference hook to conda lock in documentation install section (:pull:`1473`) by :user:`valeriupredoi` +- Increased ESMValCore version to 2.5.0rc1 (:pull:`1477`) by :user:`schlunma` +- Added changelog for v2.5.0 release (:pull:`1476`) by :user:`schlunma` +- Increased ESMValCore version to 2.5.0rc2 (:pull:`1487`) by :user:`schlunma` +- Added some authors to citation and zenodo files (:pull:`1488`) by :user:`SarahAlidoost` +- Restored `scipy` intersphinx mapping (:pull:`1491`) by :user:`schlunma` +- Increased ESMValCore version to 2.5.0rc3 (:pull:`1504`) by :user:`schlunma` +- Fix download instructions for the MSWEP dataset (:pull:`1506`) by :user:`remi-kazeroni` +- Documentation updated for the new cmorizer framework (:pull:`1417`) by :user:`remi-kazeroni` +- Added tests for duplicates in changelog and removed duplicates (:pull:`1508`) by :user:`schlunma` +- Increased ESMValCore version to 2.5.0rc4 (:pull:`1519`) by :user:`schlunma` +- Add Github Actions Test badge in README (:pull:`1526`) by :user:`valeriupredoi` +- Increased ESMValCore version to 2.5.0rc5 (:pull:`1529`) by :user:`schlunma` +- Increased ESMValCore version to 2.5.0rc6 (:pull:`1532`) by :user:`schlunma` Fixes for datasets ~~~~~~~~~~~~~~~~~~ -- Added fix for AIRS v2.1 (obs4mips) (`#1472 `__) `Axel Lauer `__ +- Added fix for AIRS v2.1 (obs4mips) (:pull:`1472`) by :user:`axel-lauer` Preprocessor ~~~~~~~~~~~~ -- Added bias preprocessor (`#1406 `__) `Manuel Schlund `__ -- Improve error messages when a preprocessor is failing (`#1408 `__) `Manuel Schlund `__ -- Added option to explicitly not use fx variables in preprocessors (`#1416 `__) `Manuel Schlund `__ -- Add `extract_location` preprocessor to extract town, city, mountains etc - anything specifiable by a location (`#1251 `__) `Javier Vegas-Regidor `__ -- Add ensemble statistics preprocessor and 'groupby' option for multimodel (`#673 `__) `sloosvel `__ -- Generic regridding preprocessor (`#1448 `__) `Klaus Zimmermann `__ +- Added bias preprocessor (:pull:`1406`) by :user:`schlunma` +- Improve error messages when a preprocessor is failing (:pull:`1408`) by :user:`schlunma` +- Added option to explicitly not use fx variables in preprocessors (:pull:`1416`) by :user:`schlunma` +- Add `extract_location` preprocessor to extract town, city, mountains etc - anything specifiable by a location (:pull:`1251`) by :user:`jvegreg` +- Add ensemble statistics preprocessor and 'groupby' option for multimodel (:pull:`673`) by :user:`sloosvel` +- Generic regridding preprocessor (:pull:`1448`) by :user:`zklaus` Automatic testing ~~~~~~~~~~~~~~~~~ -- Add `pandas` as dependency :panda_face: (`#1402 `__) `Valeriu Predoi `__ -- Fixed tests for python 3.7 (`#1410 `__) `Manuel Schlund `__ -- Remove accessing `.xml()` cube method from test (`#1419 `__) `Valeriu Predoi `__ -- Remove flag to use pip 2020 solver from Github Action pip install command on OSX (`#1357 `__) `Valeriu Predoi `__ -- Add Python=3.10 to Github Actions and switch to Python=3.10 for the Github Action that builds the PyPi package (`#1430 `__) `Valeriu Predoi `__ -- Pin `flake8<4` to keep getting relevant error traces when tests fail with FLAKE8 issues (`#1434 `__) `Valeriu Predoi `__ -- Implementing conda lock (`#1164 `__) `Valeriu Predoi `__ -- Relocate `pytest-monitor` outputted database `.pymon` so `.pymon-journal` file should not be looked for by `pytest` (`#1441 `__) `Valeriu Predoi `__ -- Switch to Mambaforge in Github Actions tests (`#1438 `__) `Valeriu Predoi `__ -- Turn off conda lock file creation on any push on `main` branch from Github Action test (`#1489 `__) `Valeriu Predoi `__ -- Add DRS path test for IPSLCM files (`#1490 `__) `Stéphane Sénési `__ -- Add a test module that runs tests of `iris` I/O every time we notice serious bugs there (`#1510 `__) `Valeriu Predoi `__ -- [Github Actions] Trigger Github Actions tests (`run-tests.yml` workflow) from a comment in a PR (`#1520 `__) `Valeriu Predoi `__ +- Add `pandas` as dependency :panda_face: (:pull:`1402`) by :user:`valeriupredoi` +- Fixed tests for python 3.7 (:pull:`1410`) by :user:`schlunma` +- Remove accessing `.xml()` cube method from test (:pull:`1419`) by :user:`valeriupredoi` +- Remove flag to use pip 2020 solver from Github Action pip install command on OSX (:pull:`1357`) by :user:`valeriupredoi` +- Add Python=3.10 to Github Actions and switch to Python=3.10 for the Github Action that builds the PyPi package (:pull:`1430`) by :user:`valeriupredoi` +- Pin `flake8<4` to keep getting relevant error traces when tests fail with FLAKE8 issues (:pull:`1434`) by :user:`valeriupredoi` +- Implementing conda lock (:pull:`1164`) by :user:`valeriupredoi` +- Relocate `pytest-monitor` outputted database `.pymon` so `.pymon-journal` file should not be looked for by `pytest` (:pull:`1441`) by :user:`valeriupredoi` +- Switch to Mambaforge in Github Actions tests (:pull:`1438`) by :user:`valeriupredoi` +- Turn off conda lock file creation on any push on `main` branch from Github Action test (:pull:`1489`) by :user:`valeriupredoi` +- Add DRS path test for IPSLCM files (:pull:`1490`) by :user:`senesis` +- Add a test module that runs tests of `iris` I/O every time we notice serious bugs there (:pull:`1510`) by :user:`valeriupredoi` +- [Github Actions] Trigger Github Actions tests (`run-tests.yml` workflow) from a comment in a PR (:pull:`1520`) by :user:`valeriupredoi` - Update Linux condalock file (various pull requests) github-actions[bot] Installation ~~~~~~~~~~~~ -- Move `nested-lookup` dependency to `environment.yml` to be installed from conda-forge instead of PyPi (`#1481 `__) `Valeriu Predoi `__ -- Pinned `iris` (`#1511 `__) `Manuel Schlund `__ -- Updated dependencies (`#1521 `__) `Manuel Schlund `__ -- Pinned iris<3.2.0 (`#1525 `__) `Manuel Schlund `__ +- Move `nested-lookup` dependency to `environment.yml` to be installed from conda-forge instead of PyPi (:pull:`1481`) by :user:`valeriupredoi` +- Pinned `iris` (:pull:`1511`) by :user:`schlunma` +- Updated dependencies (:pull:`1521`) by :user:`schlunma` +- Pinned iris<3.2.0 (:pull:`1525`) by :user:`schlunma` Improvements ~~~~~~~~~~~~ -- Allow to load all files, first X years or last X years in an experiment (`#1133 `__) `sloosvel `__ -- Filter tasks earlier (`#1264 `__) `Javier Vegas-Regidor `__ -- Added earlier validation for command line arguments (`#1435 `__) `Manuel Schlund `__ -- Remove `profile_diagnostic` from diagnostic settings and increase test coverage of `_task.py` (`#1404 `__) `Valeriu Predoi `__ -- Add `output2` to the `product` extra facet of CMIP5 data (`#1514 `__) `Rémi Kazeroni `__ -- Speed up ESGF search (`#1512 `__) `Bouwe Andela `__ +- Allow to load all files, first X years or last X years in an experiment (:pull:`1133`) by :user:`sloosvel` +- Filter tasks earlier (:pull:`1264`) by :user:`jvegreg` +- Added earlier validation for command line arguments (:pull:`1435`) by :user:`schlunma` +- Remove `profile_diagnostic` from diagnostic settings and increase test coverage of `_task.py` (:pull:`1404`) by :user:`valeriupredoi` +- Add `output2` to the `product` extra facet of CMIP5 data (:pull:`1514`) by :user:`remi-kazeroni` +- Speed up ESGF search (:pull:`1512`) by :user:`bouweandela` .. _changelog-v2-4-0: @@ -842,108 +849,108 @@ This release includes Bug fixes ~~~~~~~~~ -- Crop on the ID-selected region(s) and not on the whole shapefile (`#1151 `__) `Stef Smeets `__ -- Add 'comment' to list of removed attributes (`#1244 `__) `Peter Kalverla `__ -- Speed up multimodel statistics and fix bug in peak computation (`#1301 `__) `Bouwe Andela `__ -- No longer make plots of provenance (`#1307 `__) `Bouwe Andela `__ -- No longer embed provenance in output files (`#1306 `__) `Bouwe Andela `__ -- Removed automatic addition of areacello to obs4mips datasets (`#1316 `__) `Manuel Schlund `__ -- Pin docutils <0.17 to fix bullet lists on readthedocs (`#1320 `__) `Klaus Zimmermann `__ -- Fix obs4MIPs capitalization (`#1328 `__) `Bouwe Andela `__ -- Fix Python 3.7 tests (`#1330 `__) `Bouwe Andela `__ -- Handle fx variables in `extract_levels` and some time operations (`#1269 `__) `sloosvel `__ -- Refactored mask regridding for irregular grids (fixes #772) (`#865 `__) `Klaus Zimmermann `__ -- Fix `da.broadcast_to` call when the fx cube has different shape than target data cube (`#1350 `__) `Valeriu Predoi `__ -- Add tests for _aggregate_time_fx (`#1354 `__) `sloosvel `__ -- Fix extra facets (`#1360 `__) `Bouwe Andela `__ -- Pin pip!=21.3 to avoid pypa/pip#10573 with editable installs (`#1359 `__) `Klaus Zimmermann `__ -- Add a custom `date2num` function to deal with changes in cftime (`#1373 `__) `Klaus Zimmermann `__ -- Removed custom version of `AtmosphereSigmaFactory` (`#1382 `__) `Manuel Schlund `__ +- Crop on the ID-selected region(s) and not on the whole shapefile (:pull:`1151`) by :user:`stefsmeets` +- Add 'comment' to list of removed attributes (:pull:`1244`) by :user:`Peter9192` +- Speed up multimodel statistics and fix bug in peak computation (:pull:`1301`) by :user:`bouweandela` +- No longer make plots of provenance (:pull:`1307`) by :user:`bouweandela` +- No longer embed provenance in output files (:pull:`1306`) by :user:`bouweandela` +- Removed automatic addition of areacello to obs4mips datasets (:pull:`1316`) by :user:`schlunma` +- Pin docutils <0.17 to fix bullet lists on readthedocs (:pull:`1320`) by :user:`zklaus` +- Fix obs4MIPs capitalization (:pull:`1328`) by :user:`bouweandela` +- Fix Python 3.7 tests (:pull:`1330`) by :user:`bouweandela` +- Handle fx variables in `extract_levels` and some time operations (:pull:`1269`) by :user:`sloosvel` +- Refactored mask regridding for irregular grids (fixes #772) (:pull:`865`) by :user:`zklaus` +- Fix `da.broadcast_to` call when the fx cube has different shape than target data cube (:pull:`1350`) by :user:`valeriupredoi` +- Add tests for _aggregate_time_fx (:pull:`1354`) by :user:`sloosvel` +- Fix extra facets (:pull:`1360`) by :user:`bouweandela` +- Pin pip!=21.3 to avoid pypa/pip#10573 with editable installs (:pull:`1359`) by :user:`zklaus` +- Add a custom `date2num` function to deal with changes in cftime (:pull:`1373`) by :user:`zklaus` +- Removed custom version of `AtmosphereSigmaFactory` (:pull:`1382`) by :user:`schlunma` Deprecations ~~~~~~~~~~~~ -- Remove write_netcdf and write_plots from config-user.yml (`#1300 `__) `Bouwe Andela `__ +- Remove write_netcdf and write_plots from config-user.yml (:pull:`1300`) by :user:`bouweandela` Documentation ~~~~~~~~~~~~~ -- Add link to plot directory in index.html (`#1256 `__) `Stef Smeets `__ -- Work around issue with yapf not following PEP8 (`#1277 `__) `Bouwe Andela `__ -- Update the core development team (`#1278 `__) `Bouwe Andela `__ -- Update the documentation of the provenance interface (`#1305 `__) `Bouwe Andela `__ -- Update version number to first release candidate 2.4.0rc1 (`#1363 `__) `Klaus Zimmermann `__ -- Update to new ESMValTool logo (`#1374 `__) `Klaus Zimmermann `__ -- Update version number for third release candidate 2.4.0rc3 (`#1384 `__) `Klaus Zimmermann `__ -- Update changelog for 2.4.0rc3 (`#1385 `__) `Klaus Zimmermann `__ -- Update version number to final 2.4.0 release (`#1389 `__) `Klaus Zimmermann `__ -- Update changelog for 2.4.0 (`#1366 `__) `Klaus Zimmermann `__ +- Add link to plot directory in index.html (:pull:`1256`) by :user:`stefsmeets` +- Work around issue with yapf not following PEP8 (:pull:`1277`) by :user:`bouweandela` +- Update the core development team (:pull:`1278`) by :user:`bouweandela` +- Update the documentation of the provenance interface (:pull:`1305`) by :user:`bouweandela` +- Update version number to first release candidate 2.4.0rc1 (:pull:`1363`) by :user:`zklaus` +- Update to new ESMValTool logo (:pull:`1374`) by :user:`zklaus` +- Update version number for third release candidate 2.4.0rc3 (:pull:`1384`) by :user:`zklaus` +- Update changelog for 2.4.0rc3 (:pull:`1385`) by :user:`zklaus` +- Update version number to final 2.4.0 release (:pull:`1389`) by :user:`zklaus` +- Update changelog for 2.4.0 (:pull:`1366`) by :user:`zklaus` Fixes for datasets ~~~~~~~~~~~~~~~~~~ -- Add fix for differing latitude coordinate between historical and ssp585 in MPI-ESM1-2-HR r2i1p1f1 (`#1292 `__) `Bouwe Andela `__ -- Add fixes for time and latitude coordinate of EC-Earth3 r3i1p1f1 (`#1290 `__) `Bouwe Andela `__ -- Apply latitude fix to all CCSM4 variables (`#1295 `__) `Bouwe Andela `__ -- Fix lat and lon bounds for FGOALS-g3 mrsos (`#1289 `__) `Thomas Crocker `__ -- Add grid fix for tos in fgoals-f3-l (`#1326 `__) `sloosvel `__ -- Add fix for CIESM pr (`#1344 `__) `Bouwe Andela `__ -- Fix DRS for IPSLCM : split attribute 'freq' into : 'out' and 'freq' (`#1304 `__) `Stéphane Sénési - work `__ +- Add fix for differing latitude coordinate between historical and ssp585 in MPI-ESM1-2-HR r2i1p1f1 (:pull:`1292`) by :user:`bouweandela` +- Add fixes for time and latitude coordinate of EC-Earth3 r3i1p1f1 (:pull:`1290`) by :user:`bouweandela` +- Apply latitude fix to all CCSM4 variables (:pull:`1295`) by :user:`bouweandela` +- Fix lat and lon bounds for FGOALS-g3 mrsos (:pull:`1289`) by :user:`thomascrocker` +- Add grid fix for tos in fgoals-f3-l (:pull:`1326`) by :user:`sloosvel` +- Add fix for CIESM pr (:pull:`1344`) by :user:`bouweandela` +- Fix DRS for IPSLCM : split attribute 'freq' into : 'out' and 'freq' (:pull:`1304`) by :user:`senesis` CMOR standard ~~~~~~~~~~~~~ -- Remove history attribute from coords (`#1276 `__) `Javier Vegas-Regidor `__ -- Increased flexibility of CMOR checks for datasets with generic alevel coordinates (`#1032 `__) `Manuel Schlund `__ -- Automatically fix small deviations in vertical levels (`#1177 `__) `Bouwe Andela `__ -- Adding standard names to the custom tables of the `rlns` and `rsns` variables (`#1386 `__) `Rémi Kazeroni `__ +- Remove history attribute from coords (:pull:`1276`) by :user:`jvegreg` +- Increased flexibility of CMOR checks for datasets with generic alevel coordinates (:pull:`1032`) by :user:`schlunma` +- Automatically fix small deviations in vertical levels (:pull:`1177`) by :user:`bouweandela` +- Adding standard names to the custom tables of the `rlns` and `rsns` variables (:pull:`1386`) by :user:`remi-kazeroni` Preprocessor ~~~~~~~~~~~~ -- Implemented fully lazy climate_statistics (`#1194 `__) `Manuel Schlund `__ -- Run the multimodel statistics preprocessor last (`#1299 `__) `Bouwe Andela `__ +- Implemented fully lazy climate_statistics (:pull:`1194`) by :user:`schlunma` +- Run the multimodel statistics preprocessor last (:pull:`1299`) by :user:`bouweandela` Automatic testing ~~~~~~~~~~~~~~~~~ -- Improving test coverage for _task.py (`#514 `__) `Valeriu Predoi `__ -- Upload coverage to codecov (`#1190 `__) `Bouwe Andela `__ -- Improve codecov status checks (`#1195 `__) `Bouwe Andela `__ -- Fix curl install in CircleCI (`#1228 `__) `Javier Vegas-Regidor `__ -- Drop support for Python 3.6 (`#1200 `__) `Valeriu Predoi `__ -- Allow more recent version of `scipy` (`#1182 `__) `Manuel Schlund `__ -- Speed up conda build `conda_build` Circle test by using `mamba` solver via `boa` (and use it for Github Actions test too) (`#1243 `__) `Valeriu Predoi `__ -- Fix numpy deprecation warnings (`#1274 `__) `Bouwe Andela `__ -- Unpin upper bound for iris (previously was at <3.0.4) (`#1275 `__) `Valeriu Predoi `__ -- Modernize `conda_install` test on Circle CI by installing from conda-forge with Python 3.9 and change install instructions in documentation (`#1280 `__) `Valeriu Predoi `__ -- Run a nightly Github Actions workflow to monitor tests memory per test (configurable for other metrics too) (`#1284 `__) `Valeriu Predoi `__ -- Speed up tests of tasks (`#1302 `__) `Bouwe Andela `__ -- Fix upper case to lower case variables and functions for flake compliance in `tests/unit/preprocessor/_regrid/test_extract_levels.py` (`#1347 `__) `Valeriu Predoi `__ -- Cleaned up a bit Github Actions workflows (`#1345 `__) `Valeriu Predoi `__ -- Update circleci jobs: renaming tests to more descriptive names and removing conda build test (`#1351 `__) `Klaus Zimmermann `__ -- Pin iris to latest `>=3.1.0` (`#1341 `__) `Valeriu Predoi `__ +- Improving test coverage for _task.py (:pull:`514`) by :user:`valeriupredoi` +- Upload coverage to codecov (:pull:`1190`) by :user:`bouweandela` +- Improve codecov status checks (:pull:`1195`) by :user:`bouweandela` +- Fix curl install in CircleCI (:pull:`1228`) by :user:`jvegreg` +- Drop support for Python 3.6 (:pull:`1200`) by :user:`valeriupredoi` +- Allow more recent version of `scipy` (:pull:`1182`) by :user:`schlunma` +- Speed up conda build `conda_build` Circle test by using `mamba` solver via `boa` (and use it for Github Actions test too) (:pull:`1243`) by :user:`valeriupredoi` +- Fix numpy deprecation warnings (:pull:`1274`) by :user:`bouweandela` +- Unpin upper bound for iris (previously was at <3.0.4) (:pull:`1275`) by :user:`valeriupredoi` +- Modernize `conda_install` test on Circle CI by installing from conda-forge with Python 3.9 and change install instructions in documentation (:pull:`1280`) by :user:`valeriupredoi` +- Run a nightly Github Actions workflow to monitor tests memory per test (configurable for other metrics too) (:pull:`1284`) by :user:`valeriupredoi` +- Speed up tests of tasks (:pull:`1302`) by :user:`bouweandela` +- Fix upper case to lower case variables and functions for flake compliance in `tests/unit/preprocessor/_regrid/test_extract_levels.py` (:pull:`1347`) by :user:`valeriupredoi` +- Cleaned up a bit Github Actions workflows (:pull:`1345`) by :user:`valeriupredoi` +- Update circleci jobs: renaming tests to more descriptive names and removing conda build test (:pull:`1351`) by :user:`zklaus` +- Pin iris to latest `>=3.1.0` (:pull:`1341`) by :user:`valeriupredoi` Installation ~~~~~~~~~~~~ -- Pin esmpy to anything but 8.1.0 since that particular one changes the CPU affinity (`#1310 `__) `Valeriu Predoi `__ +- Pin esmpy to anything but 8.1.0 since that particular one changes the CPU affinity (:pull:`1310`) by :user:`valeriupredoi` Improvements ~~~~~~~~~~~~ -- Add a more friendly and useful message when using default config file (`#1233 `__) `Valeriu Predoi `__ -- Replace os.walk by glob.glob in data finder (only look for data in the specified locations) (`#1261 `__) `Bouwe Andela `__ -- Machine-specific directories for auxiliary data in the `config-user.yml` file (`#1268 `__) `Rémi Kazeroni `__ -- Add an option to download missing data from ESGF (`#1217 `__) `Bouwe Andela `__ -- Speed up provenance recording (`#1327 `__) `Bouwe Andela `__ -- Improve results web page (`#1332 `__) `Bouwe Andela `__ -- Move institutes from config-developer.yml to default extra facets config and add wildcard support for extra facets (`#1259 `__) `Bouwe Andela `__ -- Add support for re-using preprocessor output from previous runs (`#1321 `__) `Bouwe Andela `__ -- Log fewer messages to screen and hide stack trace for known recipe errors (`#1296 `__) `Bouwe Andela `__ -- Log ESMValCore and ESMValTool versions when running (`#1263 `__) `Javier Vegas-Regidor `__ -- Add "grid" as a tag to the output file template for CMIP6 (`#1356 `__) `Klaus Zimmermann `__ -- Implemented ICON project to read native ICON model output (`#1079 `__) `Brei Soliño `__ +- Add a more friendly and useful message when using default config file (:pull:`1233`) by :user:`valeriupredoi` +- Replace os.walk by glob.glob in data finder (only look for data in the specified locations) (:pull:`1261`) by :user:`bouweandela` +- Machine-specific directories for auxiliary data in the `config-user.yml` file (:pull:`1268`) by :user:`remi-kazeroni` +- Add an option to download missing data from ESGF (:pull:`1217`) by :user:`bouweandela` +- Speed up provenance recording (:pull:`1327`) by :user:`bouweandela` +- Improve results web page (:pull:`1332`) by :user:`bouweandela` +- Move institutes from config-developer.yml to default extra facets config and add wildcard support for extra facets (:pull:`1259`) by :user:`bouweandela` +- Add support for re-using preprocessor output from previous runs (:pull:`1321`) by :user:`bouweandela` +- Log fewer messages to screen and hide stack trace for known recipe errors (:pull:`1296`) by :user:`bouweandela` +- Log ESMValCore and ESMValTool versions when running (:pull:`1263`) by :user:`jvegreg` +- Add "grid" as a tag to the output file template for CMIP6 (:pull:`1356`) by :user:`zklaus` +- Implemented ICON project to read native ICON model output (:pull:`1079`) by :user:`bsolino` .. _changelog-v2-3-1: @@ -956,45 +963,45 @@ This release includes Bug fixes ~~~~~~~~~ -- Update config-user.yml template with correct drs entries for CEDA-JASMIN (`#1184 `__) `Valeriu Predoi `__ -- Enhancing MIROC5 fix for hfls and evspsbl (`#1192 `__) `katjaweigel `__ -- Fix alignment of daily data with inconsistent calendars in multimodel statistics (`#1212 `__) `Peter Kalverla `__ -- Pin cf-units, remove github actions test for Python 3.6 and fix test_access1_0 and test_access1_3 to use cf-units for comparisons (`#1197 `__) `Valeriu Predoi `__ -- Fixed search for fx files when no ``mip`` is given (`#1216 `__) `Manuel Schlund `__ -- Make sure climate statistics always returns original dtype (`#1237 `__) `Klaus Zimmermann `__ -- Bugfix for regional regridding when non-integer range is passed (`#1231 `__) `Stef Smeets `__ -- Make sure area_statistics preprocessor always returns original dtype (`#1239 `__) `Klaus Zimmermann `__ -- Add "." (dot) as allowed separation character for the time range group (`#1248 `__) `Klaus Zimmermann `__ +- Update config-user.yml template with correct drs entries for CEDA-JASMIN (:pull:`1184`) by :user:`valeriupredoi` +- Enhancing MIROC5 fix for hfls and evspsbl (:pull:`1192`) by :user:`katjaweigel` +- Fix alignment of daily data with inconsistent calendars in multimodel statistics (:pull:`1212`) by :user:`Peter9192` +- Pin cf-units, remove github actions test for Python 3.6 and fix test_access1_0 and test_access1_3 to use cf-units for comparisons (:pull:`1197`) by :user:`valeriupredoi` +- Fixed search for fx files when no ``mip`` is given (:pull:`1216`) by :user:`schlunma` +- Make sure climate statistics always returns original dtype (:pull:`1237`) by :user:`zklaus` +- Bugfix for regional regridding when non-integer range is passed (:pull:`1231`) by :user:`stefsmeets` +- Make sure area_statistics preprocessor always returns original dtype (:pull:`1239`) by :user:`zklaus` +- Add "." (dot) as allowed separation character for the time range group (:pull:`1248`) by :user:`zklaus` Documentation ~~~~~~~~~~~~~ -- Add a link to the instructions to use pre-installed versions on HPC clusters (`#1186 `__) `Rémi Kazeroni `__ -- Bugfix release: set version to 2.3.1 (`#1253 `__) `Klaus Zimmermann `__ +- Add a link to the instructions to use pre-installed versions on HPC clusters (:pull:`1186`) by :user:`remi-kazeroni` +- Bugfix release: set version to 2.3.1 (:pull:`1253`) by :user:`zklaus` Fixes for datasets ~~~~~~~~~~~~~~~~~~ -- Set circular attribute in MCM-UA-1-0 fix (`#1178 `__) `sloosvel `__ -- Fixed time coordinate of MIROC-ESM (`#1188 `__) `Manuel Schlund `__ +- Set circular attribute in MCM-UA-1-0 fix (:pull:`1178`) by :user:`sloosvel` +- Fixed time coordinate of MIROC-ESM (:pull:`1188`) by :user:`schlunma` Preprocessor ~~~~~~~~~~~~ -- Filter warnings about collapsing multi-model dimension in multimodel statistics preprocessor function (`#1215 `__) `Bouwe Andela `__ -- Remove fx variables before computing multimodel statistics (`#1220 `__) `sloosvel `__ +- Filter warnings about collapsing multi-model dimension in multimodel statistics preprocessor function (:pull:`1215`) by :user:`bouweandela` +- Remove fx variables before computing multimodel statistics (:pull:`1220`) by :user:`sloosvel` Installation ~~~~~~~~~~~~ -- Pin lower bound for iris to 3.0.2 (`#1206 `__) `Valeriu Predoi `__ -- Pin `iris<3.0.4` to ensure we still (sort of) support Python 3.6 (`#1252 `__) `Valeriu Predoi `__ +- Pin lower bound for iris to 3.0.2 (:pull:`1206`) by :user:`valeriupredoi` +- Pin `iris<3.0.4` to ensure we still (sort of) support Python 3.6 (:pull:`1252`) by :user:`valeriupredoi` Improvements ~~~~~~~~~~~~ -- Add test to verify behaviour for scalar height coord for tas in multi-model (`#1209 `__) `Peter Kalverla `__ -- Sort missing years in "No input data available for years" message (`#1225 `__) `Lee de Mora `__ +- Add test to verify behaviour for scalar height coord for tas in multi-model (:pull:`1209`) by :user:`Peter9192` +- Sort missing years in "No input data available for years" message (:pull:`1225`) by :user:`ledm` .. _changelog-v2-3-0: @@ -1007,24 +1014,24 @@ This release includes Bug fixes ~~~~~~~~~ -- Extend preprocessor multi_model_statistics to handle data with "altitude" coordinate (`#1010 `__) `Axel Lauer `__ -- Remove scripts included with CMOR tables (`#1011 `__) `Bouwe Andela `__ -- Avoid side effects in extract_season (`#1019 `__) `Javier Vegas-Regidor `__ -- Use nearest scheme to avoid interpolation errors with masked data in regression test (`#1021 `__) `Stef Smeets `__ -- Move _get_time_bounds from preprocessor._time to cmor.check to avoid circular import with cmor module (`#1037 `__) `Valeriu Predoi `__ -- Fix test that makes conda build fail (`#1046 `__) `Valeriu Predoi `__ -- Fix 'positive' attribute for rsns/rlns variables (`#1051 `__) `Lukas Brunner `__ -- Added preprocessor mask_multimodel (`#767 `__) `Manuel Schlund `__ -- Fix bug when fixing bounds after fixing longitude values (`#1057 `__) `sloosvel `__ -- Run conda build parallel AND sequential tests (`#1065 `__) `Valeriu Predoi `__ -- Add key to id_prop (`#1071 `__) `Lukas Brunner `__ -- Fix bounds after reversing coordinate values (`#1061 `__) `sloosvel `__ -- Fixed --skip-nonexistent option (`#1093 `__) `Manuel Schlund `__ -- Do not consider CMIP5 variable sit to be the same as sithick from CMIP6 (`#1033 `__) `Bouwe Andela `__ -- Improve finding date range in filenames (enforces separators) (`#1145 `__) `Stéphane Sénési - work `__ -- Review fx handling (`#1147 `__) `sloosvel `__ -- Fix lru cache decorator with explicit call to method (`#1172 `__) `Valeriu Predoi `__ -- Update _volume.py (`#1174 `__) `Lee de Mora `__ +- Extend preprocessor multi_model_statistics to handle data with "altitude" coordinate (:pull:`1010`) by :user:`axel-lauer` +- Remove scripts included with CMOR tables (:pull:`1011`) by :user:`bouweandela` +- Avoid side effects in extract_season (:pull:`1019`) by :user:`jvegreg` +- Use nearest scheme to avoid interpolation errors with masked data in regression test (:pull:`1021`) by :user:`stefsmeets` +- Move _get_time_bounds from preprocessor._time to cmor.check to avoid circular import with cmor module (:pull:`1037`) by :user:`valeriupredoi` +- Fix test that makes conda build fail (:pull:`1046`) by :user:`valeriupredoi` +- Fix 'positive' attribute for rsns/rlns variables (:pull:`1051`) by :user:`lukasbrunner` +- Added preprocessor mask_multimodel (:pull:`767`) by :user:`schlunma` +- Fix bug when fixing bounds after fixing longitude values (:pull:`1057`) by :user:`sloosvel` +- Run conda build parallel AND sequential tests (:pull:`1065`) by :user:`valeriupredoi` +- Add key to id_prop (:pull:`1071`) by :user:`lukasbrunner` +- Fix bounds after reversing coordinate values (:pull:`1061`) by :user:`sloosvel` +- Fixed --skip-nonexistent option (:pull:`1093`) by :user:`schlunma` +- Do not consider CMIP5 variable sit to be the same as sithick from CMIP6 (:pull:`1033`) by :user:`bouweandela` +- Improve finding date range in filenames (enforces separators) (:pull:`1145`) by :user:`senesis` +- Review fx handling (:pull:`1147`) by :user:`sloosvel` +- Fix lru cache decorator with explicit call to method (:pull:`1172`) by :user:`valeriupredoi` +- Update _volume.py (:pull:`1174`) by :user:`ledm` Deprecations ~~~~~~~~~~~~ @@ -1034,53 +1041,53 @@ Deprecations Documentation ~~~~~~~~~~~~~ -- Final changelog for 2.3.0 (`#1163 `__) `Klaus Zimmermann `__ -- Set version to 2.3.0 (`#1162 `__) `Klaus Zimmermann `__ -- Fix documentation build (`#1006 `__) `Bouwe Andela `__ -- Add labels required for linking from ESMValTool docs (`#1038 `__) `Bouwe Andela `__ -- Update contribution guidelines (`#1047 `__) `Bouwe Andela `__ -- Fix basestring references in documentation (`#1106 `__) `Javier Vegas-Regidor `__ -- Updated references master to main (`#1132 `__) `Axel Lauer `__ -- Add instructions how to use the central installation at DKRZ-Mistral (`#1155 `__) `Rémi Kazeroni `__ +- Final changelog for 2.3.0 (:pull:`1163`) by :user:`zklaus` +- Set version to 2.3.0 (:pull:`1162`) by :user:`zklaus` +- Fix documentation build (:pull:`1006`) by :user:`bouweandela` +- Add labels required for linking from ESMValTool docs (:pull:`1038`) by :user:`bouweandela` +- Update contribution guidelines (:pull:`1047`) by :user:`bouweandela` +- Fix basestring references in documentation (:pull:`1106`) by :user:`jvegreg` +- Updated references master to main (:pull:`1132`) by :user:`axel-lauer` +- Add instructions how to use the central installation at DKRZ-Mistral (:pull:`1155`) by :user:`remi-kazeroni` Fixes for datasets ~~~~~~~~~~~~~~~~~~ -- Added fixes for various CMIP5 datasets, variable cl (3-dim cloud fraction) (`#1017 `__) `Axel Lauer `__ -- Added fixes for hybrid level coordinates of CESM2 models (`#882 `__) `Manuel Schlund `__ -- Extending LWP fix for CMIP6 models (`#1049 `__) `Axel Lauer `__ -- Add fixes for the net & up radiation variables from ERA5 (`#1052 `__) `Lukas Brunner `__ -- Add derived variable rsus (`#1053 `__) `Lukas Brunner `__ -- Supported `mip`-level fixes (`#1095 `__) `Manuel Schlund `__ -- Fix erroneous use of `grid_latitude` and `grid_longitude` and cleaned ocean grid fixes (`#1092 `__) `Manuel Schlund `__ -- Fix for pr of miroc5 (`#1110 `__) `Rémi Kazeroni `__ -- Ocean depth fix for cnrm_esm2_1, gfdl_esm4, ipsl_cm6a_lr datasets + mcm_ua_1_0 (`#1098 `__) `Tomas Lovato `__ -- Fix for uas variable of the MCM_UA_1_0 dataset (`#1102 `__) `Rémi Kazeroni `__ -- Fixes for sos and siconc of BCC models (`#1090 `__) `Rémi Kazeroni `__ -- Run fgco2 fix for all CESM2 models (`#1108 `__) `Lisa Bock `__ -- Fixes for the siconc variable of CMIP6 models (`#1105 `__) `Rémi Kazeroni `__ -- Fix wrong sign for land surface flux (`#1113 `__) `Lisa Bock `__ -- Fix for pr of EC_EARTH (`#1116 `__) `Rémi Kazeroni `__ +- Added fixes for various CMIP5 datasets, variable cl (3-dim cloud fraction) (:pull:`1017`) by :user:`axel-lauer` +- Added fixes for hybrid level coordinates of CESM2 models (:pull:`882`) by :user:`schlunma` +- Extending LWP fix for CMIP6 models (:pull:`1049`) by :user:`axel-lauer` +- Add fixes for the net & up radiation variables from ERA5 (:pull:`1052`) by :user:`lukasbrunner` +- Add derived variable rsus (:pull:`1053`) by :user:`lukasbrunner` +- Supported `mip`-level fixes (:pull:`1095`) by :user:`schlunma` +- Fix erroneous use of `grid_latitude` and `grid_longitude` and cleaned ocean grid fixes (:pull:`1092`) by :user:`schlunma` +- Fix for pr of miroc5 (:pull:`1110`) by :user:`remi-kazeroni` +- Ocean depth fix for cnrm_esm2_1, gfdl_esm4, ipsl_cm6a_lr datasets + mcm_ua_1_0 (:pull:`1098`) by :user:`tomaslovato` +- Fix for uas variable of the MCM_UA_1_0 dataset (:pull:`1102`) by :user:`remi-kazeroni` +- Fixes for sos and siconc of BCC models (:pull:`1090`) by :user:`remi-kazeroni` +- Run fgco2 fix for all CESM2 models (:pull:`1108`) by :user:`LisaBock` +- Fixes for the siconc variable of CMIP6 models (:pull:`1105`) by :user:`remi-kazeroni` +- Fix wrong sign for land surface flux (:pull:`1113`) by :user:`LisaBock` +- Fix for pr of EC_EARTH (:pull:`1116`) by :user:`remi-kazeroni` CMOR standard ~~~~~~~~~~~~~ -- Format cmor related files (`#976 `__) `Javier Vegas-Regidor `__ -- Check presence of time bounds and guess them if needed (`#849 `__) `sloosvel `__ -- Add custom variable "tasaga" (`#1118 `__) `Lisa Bock `__ -- Find files for CMIP6 DCPP startdates (`#771 `__) `sloosvel `__ +- Format cmor related files (:pull:`976`) by :user:`jvegreg` +- Check presence of time bounds and guess them if needed (:pull:`849`) by :user:`sloosvel` +- Add custom variable "tasaga" (:pull:`1118`) by :user:`LisaBock` +- Find files for CMIP6 DCPP startdates (:pull:`771`) by :user:`sloosvel` Preprocessor ~~~~~~~~~~~~ -- Update tests for multimodel statistics preprocessor (`#1023 `__) `Stef Smeets `__ -- Raise in extract_season and extract_month if result is None (`#1041 `__) `Javier Vegas-Regidor `__ -- Allow selection of shapes in extract_shape (`#764 `__) `Javier Vegas-Regidor `__ -- Add option for regional regridding to regrid preprocessor (`#1034 `__) `Stef Smeets `__ -- Load fx variables as cube cell measures / ancillary variables (`#999 `__) `sloosvel `__ -- Check horizontal grid before regridding (`#507 `__) `Benjamin Müller `__ -- Clip irregular grids (`#245 `__) `Bouwe Andela `__ -- Use native iris functions in multi-model statistics (`#1150 `__) `Peter Kalverla `__ +- Update tests for multimodel statistics preprocessor (:pull:`1023`) by :user:`stefsmeets` +- Raise in extract_season and extract_month if result is None (:pull:`1041`) by :user:`jvegreg` +- Allow selection of shapes in extract_shape (:pull:`764`) by :user:`jvegreg` +- Add option for regional regridding to regrid preprocessor (:pull:`1034`) by :user:`stefsmeets` +- Load fx variables as cube cell measures / ancillary variables (:pull:`999`) by :user:`sloosvel` +- Check horizontal grid before regridding (:pull:`507`) by :user:`BenMGeo` +- Clip irregular grids (:pull:`245`) by :user:`bouweandela` +- Use native iris functions in multi-model statistics (:pull:`1150`) by :user:`Peter9192` Notebook API (experimental) ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1090,38 +1097,38 @@ Notebook API (experimental) Automatic testing ~~~~~~~~~~~~~~~~~ -- Report coverage for tests that run on any pull request (`#994 `__) `Bouwe Andela `__ -- Install ESMValTool sample data from PyPI (`#998 `__) `Javier Vegas-Regidor `__ -- Fix tests for multi-processing with spawn method (i.e. macOSX with Python>3.8) (`#1003 `__) `Barbara Vreede `__ -- Switch to running the Github Action test workflow every 3 hours in single thread mode to observe if Segmentation Faults occur (`#1022 `__) `Valeriu Predoi `__ -- Revert to original Github Actions test workflow removing the 3-hourly test run with -n 1 (`#1025 `__) `Valeriu Predoi `__ -- Avoid stale cache for multimodel statistics regression tests (`#1030 `__) `Bouwe Andela `__ -- Add newer Python versions in OSX to Github Actions (`#1035 `__) `Barbara Vreede `__ -- Add tests for type annotations with mypy (`#1042 `__) `Stef Smeets `__ -- Run problematic cmor tests sequentially to avoid segmentation faults on CircleCI (`#1064 `__) `Valeriu Predoi `__ -- Test installation of esmvalcore from conda-forge (`#1075 `__) `Valeriu Predoi `__ -- Added additional test cases for integration tests of data_finder.py (`#1087 `__) `Manuel Schlund `__ -- Pin cf-units and fix tests (cf-units>=2.1.5) (`#1140 `__) `Valeriu Predoi `__ -- Fix failing CircleCI tests (`#1167 `__) `Bouwe Andela `__ -- Fix test failing due to fx files chosen differently on different OS's (`#1169 `__) `Valeriu Predoi `__ -- Compare datetimes instead of strings in _fixes/cmip5/test_access1_X.py (`#1173 `__) `Valeriu Predoi `__ -- Pin Python to 3.9 in environment.yml on CircleCI and skip mypy tests in conda build (`#1176 `__) `Bouwe Andela `__ +- Report coverage for tests that run on any pull request (:pull:`994`) by :user:`bouweandela` +- Install ESMValTool sample data from PyPI (:pull:`998`) by :user:`jvegreg` +- Fix tests for multi-processing with spawn method (i.e. macOSX with Python>3.8) (:pull:`1003`) by :user:`bvreede` +- Switch to running the Github Action test workflow every 3 hours in single thread mode to observe if Segmentation Faults occur (:pull:`1022`) by :user:`valeriupredoi` +- Revert to original Github Actions test workflow removing the 3-hourly test run with -n 1 (:pull:`1025`) by :user:`valeriupredoi` +- Avoid stale cache for multimodel statistics regression tests (:pull:`1030`) by :user:`bouweandela` +- Add newer Python versions in OSX to Github Actions (:pull:`1035`) by :user:`bvreede` +- Add tests for type annotations with mypy (:pull:`1042`) by :user:`stefsmeets` +- Run problematic cmor tests sequentially to avoid segmentation faults on CircleCI (:pull:`1064`) by :user:`valeriupredoi` +- Test installation of esmvalcore from conda-forge (:pull:`1075`) by :user:`valeriupredoi` +- Added additional test cases for integration tests of data_finder.py (:pull:`1087`) by :user:`schlunma` +- Pin cf-units and fix tests (cf-units>=2.1.5) (:pull:`1140`) by :user:`valeriupredoi` +- Fix failing CircleCI tests (:pull:`1167`) by :user:`bouweandela` +- Fix test failing due to fx files chosen differently on different OS's (:pull:`1169`) by :user:`valeriupredoi` +- Compare datetimes instead of strings in _fixes/cmip5/test_access1_X.py (:pull:`1173`) by :user:`valeriupredoi` +- Pin Python to 3.9 in environment.yml on CircleCI and skip mypy tests in conda build (:pull:`1176`) by :user:`bouweandela` Installation ~~~~~~~~~~~~ -- Update yamale to version 3 (`#1059 `__) `Klaus Zimmermann `__ +- Update yamale to version 3 (:pull:`1059`) by :user:`zklaus` Improvements ~~~~~~~~~~~~ -- Refactor diagnostics / tags management (`#939 `__) `Stef Smeets `__ -- Support multiple paths in input_dir (`#1000 `__) `Javier Vegas-Regidor `__ -- Generate HTML report with recipe output (`#991 `__) `Stef Smeets `__ -- Add timeout to requests.get in _citation.py (`#1091 `__) `SarahAlidoost `__ -- Add SYNDA drs for CMIP5 and CMIP6 (closes #582) (`#583 `__) `Klaus Zimmermann `__ -- Add basic support for variable mappings (`#1124 `__) `Klaus Zimmermann `__ -- Handle IPSL-CM6 (`#1153 `__) `Stéphane Sénési - work `__ +- Refactor diagnostics / tags management (:pull:`939`) by :user:`stefsmeets` +- Support multiple paths in input_dir (:pull:`1000`) by :user:`jvegreg` +- Generate HTML report with recipe output (:pull:`991`) by :user:`stefsmeets` +- Add timeout to requests.get in _citation.py (:pull:`1091`) by :user:`SarahAlidoost` +- Add SYNDA drs for CMIP5 and CMIP6 (closes #582) (:pull:`583`) by :user:`zklaus` +- Add basic support for variable mappings (:pull:`1124`) by :user:`zklaus` +- Handle IPSL-CM6 (:pull:`1153`) by :user:`senesis` .. _changelog-v2-2-0: @@ -1146,126 +1153,126 @@ This release includes Bug fixes ~~~~~~~~~ -- Fix path settings for DKRZ/Mistral (`#852 `__) `Bouwe Andela `__ -- Change logic for calling the diagnostic script to avoid problems with scripts where the executable bit is accidentally set (`#877 `__) `Bouwe Andela `__ -- Fix overwriting in generic level check (`#886 `__) `sloosvel `__ -- Add double quotes to script args in rerun screen message when using vprof profiling (`#897 `__) `Valeriu Predoi `__ -- Simplify time handling in multi-model statistics preprocessor (`#685 `__) `Peter Kalverla `__ -- Fix links to Iris documentation (`#966 `__) `Javier Vegas-Regidor `__ -- Bugfix: Fix units for MSWEP data (`#986 `__) `Stef Smeets `__ +- Fix path settings for DKRZ/Mistral (:pull:`852`) by :user:`bouweandela` +- Change logic for calling the diagnostic script to avoid problems with scripts where the executable bit is accidentally set (:pull:`877`) by :user:`bouweandela` +- Fix overwriting in generic level check (:pull:`886`) by :user:`sloosvel` +- Add double quotes to script args in rerun screen message when using vprof profiling (:pull:`897`) by :user:`valeriupredoi` +- Simplify time handling in multi-model statistics preprocessor (:pull:`685`) by :user:`Peter9192` +- Fix links to Iris documentation (:pull:`966`) by :user:`jvegreg` +- Bugfix: Fix units for MSWEP data (:pull:`986`) by :user:`stefsmeets` Deprecations ~~~~~~~~~~~~ -- Deprecate defining write_plots and write_netcdf in config-user file (`#808 `__) `Bouwe Andela `__ +- Deprecate defining write_plots and write_netcdf in config-user file (:pull:`808`) by :user:`bouweandela` Documentation ~~~~~~~~~~~~~ -- Fix numbering of steps in release instructions (`#838 `__) `Bouwe Andela `__ -- Add labels to changelogs of individual versions for easy reference (`#899 `__) `Klaus Zimmermann `__ -- Make CircleCI badge specific to main branch (`#902 `__) `Bouwe Andela `__ -- Fix docker build badge url (`#906 `__) `Stef Smeets `__ -- Update github PR template (`#909 `__) `Stef Smeets `__ -- Refer to ESMValTool GitHub discussions page in the error message (`#900 `__) `Bouwe Andela `__ -- Support automatically closing issues (`#922 `__) `Bouwe Andela `__ -- Fix checkboxes in PR template (`#931 `__) `Stef Smeets `__ -- Change in config-user defaults and documentation with new location for esmeval OBS data on JASMIN (`#958 `__) `Valeriu Predoi `__ -- Update Core Team info (`#942 `__) `Axel Lauer `__ -- Update iris documentation URL for sphinx (`#964 `__) `Bouwe Andela `__ -- Set version to 2.2.0 (`#977 `__) `Javier Vegas-Regidor `__ -- Add first draft of v2.2.0 changelog (`#983 `__) `Javier Vegas-Regidor `__ -- Add checkbox in PR template to assign labels (`#985 `__) `Javier Vegas-Regidor `__ -- Update install.rst (`#848 `__) `bascrezee `__ -- Change the order of the publication steps (`#984 `__) `Javier Vegas-Regidor `__ -- Add instructions how to use esmvaltool from HPC central installations (`#841 `__) `Valeriu Predoi `__ +- Fix numbering of steps in release instructions (:pull:`838`) by :user:`bouweandela` +- Add labels to changelogs of individual versions for easy reference (:pull:`899`) by :user:`zklaus` +- Make CircleCI badge specific to main branch (:pull:`902`) by :user:`bouweandela` +- Fix docker build badge url (:pull:`906`) by :user:`stefsmeets` +- Update github PR template (:pull:`909`) by :user:`stefsmeets` +- Refer to ESMValTool GitHub discussions page in the error message (:pull:`900`) by :user:`bouweandela` +- Support automatically closing issues (:pull:`922`) by :user:`bouweandela` +- Fix checkboxes in PR template (:pull:`931`) by :user:`stefsmeets` +- Change in config-user defaults and documentation with new location for esmeval OBS data on JASMIN (:pull:`958`) by :user:`valeriupredoi` +- Update Core Team info (:pull:`942`) by :user:`axel-lauer` +- Update iris documentation URL for sphinx (:pull:`964`) by :user:`bouweandela` +- Set version to 2.2.0 (:pull:`977`) by :user:`jvegreg` +- Add first draft of v2.2.0 changelog (:pull:`983`) by :user:`jvegreg` +- Add checkbox in PR template to assign labels (:pull:`985`) by :user:`jvegreg` +- Update install.rst (:pull:`848`) by :user:`bascrezee` +- Change the order of the publication steps (:pull:`984`) by :user:`jvegreg` +- Add instructions how to use esmvaltool from HPC central installations (:pull:`841`) by :user:`valeriupredoi` Fixes for datasets ~~~~~~~~~~~~~~~~~~ -- Fixing unit for derived variable rsnstcsnorm to prevent overcorrection2 (`#846 `__) `katjaweigel `__ -- Cmip6 fix awi cm 1 1 mr (`#822 `__) `mwjury `__ -- Cmip6 fix ec earth3 veg (`#836 `__) `mwjury `__ -- Changed latitude longitude fix from Tas to AllVars. (`#916 `__) `katjaweigel `__ -- Fix for precipitation (pr) to use ERA5-Land cmorizer (`#879 `__) `katjaweigel `__ -- Cmip6 fix ec earth3 (`#837 `__) `mwjury `__ -- Cmip6_fix_fgoals_f3_l_Amon_time_bnds (`#831 `__) `mwjury `__ -- Fix for FGOALS-f3-L sftlf (`#667 `__) `mwjury `__ -- Improve ACCESS-CM2 and ACCESS-ESM1-5 fixes and add CIESM and CESM2-WACCM-FV2 fixes for cl, clw and cli (`#635 `__) `Axel Lauer `__ -- Add fixes for cl, cli, clw and tas for several CMIP6 models (`#955 `__) `Manuel Schlund `__ -- Dataset fixes for MSWEP (`#969 `__) `Stef Smeets `__ -- Dataset fixes for: ACCESS-ESM1-5, CanESM5, CanESM5 for carbon cycle (`#947 `__) `Bettina Gier `__ -- Fixes for KIOST-ESM (CMIP6) (`#904 `__) `Rémi Kazeroni `__ -- Fixes for AWI-ESM-1-1-LR (CMIP6, piControl) (`#911 `__) `Rémi Kazeroni `__ +- Fixing unit for derived variable rsnstcsnorm to prevent overcorrection2 (:pull:`846`) by :user:`katjaweigel` +- Cmip6 fix awi cm 1 1 mr (:pull:`822`) by :user:`mwjury` +- Cmip6 fix ec earth3 veg (:pull:`836`) by :user:`mwjury` +- Changed latitude longitude fix from Tas to AllVars. (:pull:`916`) by :user:`katjaweigel` +- Fix for precipitation (pr) to use ERA5-Land cmorizer (:pull:`879`) by :user:`katjaweigel` +- Cmip6 fix ec earth3 (:pull:`837`) by :user:`mwjury` +- Cmip6_fix_fgoals_f3_l_Amon_time_bnds (:pull:`831`) by :user:`mwjury` +- Fix for FGOALS-f3-L sftlf (:pull:`667`) by :user:`mwjury` +- Improve ACCESS-CM2 and ACCESS-ESM1-5 fixes and add CIESM and CESM2-WACCM-FV2 fixes for cl, clw and cli (:pull:`635`) by :user:`axel-lauer` +- Add fixes for cl, cli, clw and tas for several CMIP6 models (:pull:`955`) by :user:`schlunma` +- Dataset fixes for MSWEP (:pull:`969`) by :user:`stefsmeets` +- Dataset fixes for: ACCESS-ESM1-5, CanESM5, CanESM5 for carbon cycle (:pull:`947`) by :user:`bettina-gier` +- Fixes for KIOST-ESM (CMIP6) (:pull:`904`) by :user:`remi-kazeroni` +- Fixes for AWI-ESM-1-1-LR (CMIP6, piControl) (:pull:`911`) by :user:`remi-kazeroni` CMOR standard ~~~~~~~~~~~~~ -- CMOR check generic level coordinates in CMIP6 (`#598 `__) `sloosvel `__ -- Update CMIP6 tables to 6.9.33 (`#919 `__) `Javier Vegas-Regidor `__ -- Adding custom variables for tas uncertainty (`#924 `__) `Lisa Bock `__ -- Remove monotonicity coordinate check for unstructured grids (`#965 `__) `Javier Vegas-Regidor `__ +- CMOR check generic level coordinates in CMIP6 (:pull:`598`) by :user:`sloosvel` +- Update CMIP6 tables to 6.9.33 (:pull:`919`) by :user:`jvegreg` +- Adding custom variables for tas uncertainty (:pull:`924`) by :user:`LisaBock` +- Remove monotonicity coordinate check for unstructured grids (:pull:`965`) by :user:`jvegreg` Preprocessor ~~~~~~~~~~~~ -- Added clip_start_end_year preprocessor (`#796 `__) `Manuel Schlund `__ -- Add support for downloading CMIP6 data with Synda (`#699 `__) `Bouwe Andela `__ -- Add multimodel tests using real data (`#856 `__) `Stef Smeets `__ -- Add plev/altitude conversion to extract_levels (`#892 `__) `Axel Lauer `__ -- Add possibility of custom season extraction. (`#247 `__) `mwjury `__ -- Adding the ability to derive xch4 (`#783 `__) `Birgit Hassler `__ -- Add preprocessor function to resample time and compute x-hourly statistics (`#696 `__) `Javier Vegas-Regidor `__ -- Fix duplication in preprocessors DEFAULT_ORDER introduced in #696 (`#973 `__) `Javier Vegas-Regidor `__ -- Use consistent precision in multi-model statistics calculation and update reference data for tests (`#941 `__) `Peter Kalverla `__ -- Refactor multi-model statistics code to facilitate ensemble stats and lazy evaluation (`#949 `__) `Peter Kalverla `__ -- Add option to exclude input cubes in output of multimodel statistics to solve an issue introduced by #949 (`#978 `__) `Peter Kalverla `__ +- Added clip_start_end_year preprocessor (:pull:`796`) by :user:`schlunma` +- Add support for downloading CMIP6 data with Synda (:pull:`699`) by :user:`bouweandela` +- Add multimodel tests using real data (:pull:`856`) by :user:`stefsmeets` +- Add plev/altitude conversion to extract_levels (:pull:`892`) by :user:`axel-lauer` +- Add possibility of custom season extraction. (:pull:`247`) by :user:`mwjury` +- Adding the ability to derive xch4 (:pull:`783`) by :user:`hb326` +- Add preprocessor function to resample time and compute x-hourly statistics (:pull:`696`) by :user:`jvegreg` +- Fix duplication in preprocessors DEFAULT_ORDER introduced in #696 (:pull:`973`) by :user:`jvegreg` +- Use consistent precision in multi-model statistics calculation and update reference data for tests (:pull:`941`) by :user:`Peter9192` +- Refactor multi-model statistics code to facilitate ensemble stats and lazy evaluation (:pull:`949`) by :user:`Peter9192` +- Add option to exclude input cubes in output of multimodel statistics to solve an issue introduced by #949 (:pull:`978`) by :user:`Peter9192` Automatic testing ~~~~~~~~~~~~~~~~~ -- Pin cftime>=1.3.0 to have newer string formatting in and fix two tests (`#878 `__) `Valeriu Predoi `__ -- Switched miniconda conda setup hooks for Github Actions workflows (`#873 `__) `Valeriu Predoi `__ -- Add test for latest version resolver (`#874 `__) `Stef Smeets `__ -- Update codacy coverage reporter to fix coverage (`#905 `__) `Niels Drost `__ -- Avoid hardcoded year in tests and add improvement to plev test case (`#921 `__) `Bouwe Andela `__ -- Pin scipy to less than 1.6.0 until ESMValGroup/ESMValCore/issues/927 gets resolved (`#928 `__) `Valeriu Predoi `__ -- Github Actions: change time when conda install test runs (`#930 `__) `Valeriu Predoi `__ -- Remove redundant test line from test_utils.py (`#935 `__) `Valeriu Predoi `__ -- Removed netCDF4 package from integration tests of fixes (`#938 `__) `Manuel Schlund `__ -- Use new conda environment for installing ESMValCore in Docker containers (`#951 `__) `Bouwe Andela `__ +- Pin cftime>=1.3.0 to have newer string formatting in and fix two tests (:pull:`878`) by :user:`valeriupredoi` +- Switched miniconda conda setup hooks for Github Actions workflows (:pull:`873`) by :user:`valeriupredoi` +- Add test for latest version resolver (:pull:`874`) by :user:`stefsmeets` +- Update codacy coverage reporter to fix coverage (:pull:`905`) by :user:`nielsdrost` +- Avoid hardcoded year in tests and add improvement to plev test case (:pull:`921`) by :user:`bouweandela` +- Pin scipy to less than 1.6.0 until :issue:`927` gets resolved (:pull:`928`) by :user:`valeriupredoi` +- Github Actions: change time when conda install test runs (:pull:`930`) by :user:`valeriupredoi` +- Remove redundant test line from test_utils.py (:pull:`935`) by :user:`valeriupredoi` +- Removed netCDF4 package from integration tests of fixes (:pull:`938`) by :user:`schlunma` +- Use new conda environment for installing ESMValCore in Docker containers (:pull:`951`) by :user:`bouweandela` Notebook API (experimental) ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- Implement importable config object in experimental API submodule (`#868 `__) `Stef Smeets `__ -- Add loading and running recipes to the notebook API (`#907 `__) `Stef Smeets `__ -- Add displaying and loading of recipe output to the notebook API (`#957 `__) `Stef Smeets `__ -- Add functionality to run single diagnostic task to notebook API (`#962 `__) `Stef Smeets `__ +- Implement importable config object in experimental API submodule (:pull:`868`) by :user:`stefsmeets` +- Add loading and running recipes to the notebook API (:pull:`907`) by :user:`stefsmeets` +- Add displaying and loading of recipe output to the notebook API (:pull:`957`) by :user:`stefsmeets` +- Add functionality to run single diagnostic task to notebook API (:pull:`962`) by :user:`stefsmeets` Improvements ~~~~~~~~~~~~ -- Create CODEOWNERS file (`#809 `__) `Javier Vegas-Regidor `__ -- Remove code needed for Python <3.6 (`#844 `__) `Bouwe Andela `__ -- Add requests as a dependency (`#850 `__) `Bouwe Andela `__ -- Pin Python to less than 3.9 (`#870 `__) `Valeriu Predoi `__ -- Remove numba dependency (`#880 `__) `Manuel Schlund `__ -- Add Listing and finding recipes to the experimental notebook API (`#901 `__) `Stef Smeets `__ -- Skip variables that don't have dataset or additional_dataset keys (`#860 `__) `Valeriu Predoi `__ -- Refactor logging configuration (`#933 `__) `Stef Smeets `__ -- Xco2 derivation (`#913 `__) `Bettina Gier `__ -- Working environment for Python 3.9 (pin to !=3.9.0) (`#885 `__) `Valeriu Predoi `__ -- Print source file when using config get_config_user command (`#960 `__) `Valeriu Predoi `__ -- Switch to Iris 3 (`#819 `__) `Stef Smeets `__ -- Refactor tasks (`#959 `__) `Stef Smeets `__ -- Restore task summary in debug log after #959 (`#981 `__) `Bouwe Andela `__ -- Pin pre-commit hooks (`#974 `__) `Stef Smeets `__ -- Improve error messages when data is missing (`#917 `__) `Javier Vegas-Regidor `__ -- Set remove_preproc_dir to false in default config-user (`#979 `__) `Valeriu Predoi `__ -- Move fiona to be installed from conda forge (`#987 `__) `Valeriu Predoi `__ -- Re-added fiona in setup.py (`#990 `__) `Valeriu Predoi `__ +- Create CODEOWNERS file (:pull:`809`) by :user:`jvegreg` +- Remove code needed for Python <3.6 (:pull:`844`) by :user:`bouweandela` +- Add requests as a dependency (:pull:`850`) by :user:`bouweandela` +- Pin Python to less than 3.9 (:pull:`870`) by :user:`valeriupredoi` +- Remove numba dependency (:pull:`880`) by :user:`schlunma` +- Add Listing and finding recipes to the experimental notebook API (:pull:`901`) by :user:`stefsmeets` +- Skip variables that don't have dataset or additional_dataset keys (:pull:`860`) by :user:`valeriupredoi` +- Refactor logging configuration (:pull:`933`) by :user:`stefsmeets` +- Xco2 derivation (:pull:`913`) by :user:`bettina-gier` +- Working environment for Python 3.9 (pin to !=3.9.0) (:pull:`885`) by :user:`valeriupredoi` +- Print source file when using config get_config_user command (:pull:`960`) by :user:`valeriupredoi` +- Switch to Iris 3 (:pull:`819`) by :user:`stefsmeets` +- Refactor tasks (:pull:`959`) by :user:`stefsmeets` +- Restore task summary in debug log after #959 (:pull:`981`) by :user:`bouweandela` +- Pin pre-commit hooks (:pull:`974`) by :user:`stefsmeets` +- Improve error messages when data is missing (:pull:`917`) by :user:`jvegreg` +- Set remove_preproc_dir to false in default config-user (:pull:`979`) by :user:`valeriupredoi` +- Move fiona to be installed from conda forge (:pull:`987`) by :user:`valeriupredoi` +- Re-added fiona in setup.py (:pull:`990`) by :user:`valeriupredoi` .. _changelog-v2-1-0: @@ -1277,57 +1284,57 @@ This release includes Bug fixes ~~~~~~~~~ -- Set unit=1 if anomalies are standardized (`#727 `__) `bascrezee `__ -- Fix crash for FGOALS-g2 variables without longitude coordinate (`#729 `__) `Bouwe Andela `__ -- Improve variable alias management (`#595 `__) `Javier Vegas-Regidor `__ -- Fix area_statistics fx files loading (`#798 `__) `Javier Vegas-Regidor `__ -- Fix units after derivation (`#754 `__) `Manuel Schlund `__ +- Set unit=1 if anomalies are standardized (:pull:`727`) by :user:`bascrezee` +- Fix crash for FGOALS-g2 variables without longitude coordinate (:pull:`729`) by :user:`bouweandela` +- Improve variable alias management (:pull:`595`) by :user:`jvegreg` +- Fix area_statistics fx files loading (:pull:`798`) by :user:`jvegreg` +- Fix units after derivation (:pull:`754`) by :user:`schlunma` Documentation ~~~~~~~~~~~~~ -- Update v2.0.0 release notes with final additions (`#722 `__) `Bouwe Andela `__ -- Update package description in setup.py (`#725 `__) `Mattia Righi `__ -- Add installation instructions for pip installation (`#735 `__) `Bouwe Andela `__ -- Improve config-user documentation (`#740 `__) `Bouwe Andela `__ -- Update the zenodo file with contributors (`#807 `__) `Valeriu Predoi `__ -- Improve command line run documentation (`#721 `__) `Javier Vegas-Regidor `__ -- Update the zenodo file with contributors (continued) (`#810 `__) `Valeriu Predoi `__ +- Update v2.0.0 release notes with final additions (:pull:`722`) by :user:`bouweandela` +- Update package description in setup.py (:pull:`725`) by :user:`mattiarighi` +- Add installation instructions for pip installation (:pull:`735`) by :user:`bouweandela` +- Improve config-user documentation (:pull:`740`) by :user:`bouweandela` +- Update the zenodo file with contributors (:pull:`807`) by :user:`valeriupredoi` +- Improve command line run documentation (:pull:`721`) by :user:`jvegreg` +- Update the zenodo file with contributors (continued) (:pull:`810`) by :user:`valeriupredoi` Improvements ~~~~~~~~~~~~ -- Reduce size of docker image (`#723 `__) `Javier Vegas-Regidor `__ -- Add 'test' extra to installation, used by docker development tag (`#733 `__) `Bouwe Andela `__ -- Correct dockerhub link (`#736 `__) `Bouwe Andela `__ -- Create action-install-from-pypi.yml (`#734 `__) `Valeriu Predoi `__ -- Add pre-commit for linting/formatting (`#766 `__) `Stef Smeets `__ -- Run tests in parallel and when building conda package (`#745 `__) `Bouwe Andela `__ -- Readable exclude pattern for pre-commit (`#770 `__) `Stef Smeets `__ -- Github Actions Tests (`#732 `__) `Valeriu Predoi `__ -- Remove isort setup to fix formatting conflict with yapf (`#778 `__) `Stef Smeets `__ -- Fix yapf-isort import formatting conflict (Fixes #777) (`#784 `__) `Stef Smeets `__ -- Sorted output for `esmvaltool recipes list` (`#790 `__) `Stef Smeets `__ -- Replace vmprof with vprof (`#780 `__) `Valeriu Predoi `__ -- Update CMIP6 tables to 6.9.32 (`#706 `__) `Javier Vegas-Regidor `__ -- Default config-user path now set in config-user read function (`#791 `__) `Javier Vegas-Regidor `__ -- Add custom variable lweGrace (`#692 `__) `bascrezee `__ -- Create Github Actions workflow to build and deploy on Test PyPi and PyPi (`#820 `__) `Valeriu Predoi `__ -- Build and publish the esmvalcore package to conda via Github Actions workflow (`#825 `__) `Valeriu Predoi `__ +- Reduce size of docker image (:pull:`723`) by :user:`jvegreg` +- Add 'test' extra to installation, used by docker development tag (:pull:`733`) by :user:`bouweandela` +- Correct dockerhub link (:pull:`736`) by :user:`bouweandela` +- Create action-install-from-pypi.yml (:pull:`734`) by :user:`valeriupredoi` +- Add pre-commit for linting/formatting (:pull:`766`) by :user:`stefsmeets` +- Run tests in parallel and when building conda package (:pull:`745`) by :user:`bouweandela` +- Readable exclude pattern for pre-commit (:pull:`770`) by :user:`stefsmeets` +- Github Actions Tests (:pull:`732`) by :user:`valeriupredoi` +- Remove isort setup to fix formatting conflict with yapf (:pull:`778`) by :user:`stefsmeets` +- Fix yapf-isort import formatting conflict (Fixes #777) (:pull:`784`) by :user:`stefsmeets` +- Sorted output for `esmvaltool recipes list` (:pull:`790`) by :user:`stefsmeets` +- Replace vmprof with vprof (:pull:`780`) by :user:`valeriupredoi` +- Update CMIP6 tables to 6.9.32 (:pull:`706`) by :user:`jvegreg` +- Default config-user path now set in config-user read function (:pull:`791`) by :user:`jvegreg` +- Add custom variable lweGrace (:pull:`692`) by :user:`bascrezee` +- Create Github Actions workflow to build and deploy on Test PyPi and PyPi (:pull:`820`) by :user:`valeriupredoi` +- Build and publish the esmvalcore package to conda via Github Actions workflow (:pull:`825`) by :user:`valeriupredoi` Fixes for datasets ~~~~~~~~~~~~~~~~~~ -- Fix cmip6 models (`#629 `__) `npgillett `__ -- Fix siconca variable in EC-Earth3 and EC-Earth3-Veg models in amip simulation (`#702 `__) `Evgenia Galytska `__ +- Fix cmip6 models (:pull:`629`) by :user:`npgillett` +- Fix siconca variable in EC-Earth3 and EC-Earth3-Veg models in amip simulation (:pull:`702`) by :user:`egalytska` Preprocessor ~~~~~~~~~~~~ -- Move cmor_check_data to early in preprocessing chain (`#743 `__) `Bouwe Andela `__ -- Add RMS iris analysis operator to statistics preprocessor functions (`#747 `__) `Pep Cos `__ -- Add surface chlorophyll concentration as a derived variable (`#720 `__) `sloosvel `__ -- Use dask to reduce memory consumption of extract_levels for masked data (`#776 `__) `Valeriu Predoi `__ +- Move cmor_check_data to early in preprocessing chain (:pull:`743`) by :user:`bouweandela` +- Add RMS iris analysis operator to statistics preprocessor functions (:pull:`747`) by :user:`pcosbsc` +- Add surface chlorophyll concentration as a derived variable (:pull:`720`) by :user:`sloosvel` +- Use dask to reduce memory consumption of extract_levels for masked data (:pull:`776`) by :user:`valeriupredoi` .. _changelog-v2-0-0: @@ -1339,81 +1346,81 @@ This release includes Bug fixes ~~~~~~~~~ -- Fixed derivation of co2s (`#594 `__) `Manuel Schlund `__ -- Padding while cropping needs to stay within sane bounds for shapefiles that span the whole Earth (`#626 `__) `Valeriu Predoi `__ -- Fix concatenation of a single cube (`#655 `__) `Bouwe Andela `__ -- Fix mask fx dict handling not to fail if empty list in values (`#661 `__) `Valeriu Predoi `__ -- Preserve metadata during anomalies computation when using iris cubes difference (`#652 `__) `Valeriu Predoi `__ -- Avoid crashing when there is directory 'esmvaltool' in the current working directory (`#672 `__) `Valeriu Predoi `__ -- Solve bug in ACCESS1 dataset fix for calendar. (`#671 `__) `Peter Kalverla `__ -- Fix the syntax for adding multiple ensemble members from the same dataset (`#678 `__) `SarahAlidoost `__ -- Fix bug that made preprocessor with fx files fail in rare cases (`#670 `__) `Manuel Schlund `__ -- Add support for string coordinates (`#657 `__) `Javier Vegas-Regidor `__ -- Fixed the shape extraction to account for wraparound shapefile coords (`#319 `__) `Valeriu Predoi `__ -- Fixed bug in time weights calculation (`#695 `__) `Manuel Schlund `__ -- Fix diagnostic filter (`#713 `__) `Javier Vegas-Regidor `__ +- Fixed derivation of co2s (:pull:`594`) by :user:`schlunma` +- Padding while cropping needs to stay within sane bounds for shapefiles that span the whole Earth (:pull:`626`) by :user:`valeriupredoi` +- Fix concatenation of a single cube (:pull:`655`) by :user:`bouweandela` +- Fix mask fx dict handling not to fail if empty list in values (:pull:`661`) by :user:`valeriupredoi` +- Preserve metadata during anomalies computation when using iris cubes difference (:pull:`652`) by :user:`valeriupredoi` +- Avoid crashing when there is directory 'esmvaltool' in the current working directory (:pull:`672`) by :user:`valeriupredoi` +- Solve bug in ACCESS1 dataset fix for calendar. (:pull:`671`) by :user:`Peter9192` +- Fix the syntax for adding multiple ensemble members from the same dataset (:pull:`678`) by :user:`SarahAlidoost` +- Fix bug that made preprocessor with fx files fail in rare cases (:pull:`670`) by :user:`schlunma` +- Add support for string coordinates (:pull:`657`) by :user:`jvegreg` +- Fixed the shape extraction to account for wraparound shapefile coords (:pull:`319`) by :user:`valeriupredoi` +- Fixed bug in time weights calculation (:pull:`695`) by :user:`schlunma` +- Fix diagnostic filter (:pull:`713`) by :user:`jvegreg` Documentation ~~~~~~~~~~~~~ -- Add pandas as a requirement for building the documentation (`#607 `__) `Bouwe Andela `__ -- Document default order in which preprocessor functions are applied (`#633 `__) `Bouwe Andela `__ -- Add pointers about data loading and CF standards to documentation (`#571 `__) `Valeriu Predoi `__ -- Config file populated with site-specific data paths examples (`#619 `__) `Valeriu Predoi `__ -- Update Codacy badges (`#643 `__) `Bouwe Andela `__ -- Update copyright info on readthedocs (`#668 `__) `Bouwe Andela `__ -- Updated references to documentation (now docs.esmvaltool.org) (`#675 `__) `Axel Lauer `__ -- Add all European grants to Zenodo (`#680 `__) `Bouwe Andela `__ -- Update Sphinx to v3 or later (`#683 `__) `Bouwe Andela `__ -- Increase version to 2.0.0 and add release notes (`#691 `__) `Bouwe Andela `__ -- Update setup.py and README.md for use on PyPI (`#693 `__) `Bouwe Andela `__ -- Suggested Documentation changes (`#690 `__) `Steve Smith `__ +- Add pandas as a requirement for building the documentation (:pull:`607`) by :user:`bouweandela` +- Document default order in which preprocessor functions are applied (:pull:`633`) by :user:`bouweandela` +- Add pointers about data loading and CF standards to documentation (:pull:`571`) by :user:`valeriupredoi` +- Config file populated with site-specific data paths examples (:pull:`619`) by :user:`valeriupredoi` +- Update Codacy badges (:pull:`643`) by :user:`bouweandela` +- Update copyright info on readthedocs (:pull:`668`) by :user:`bouweandela` +- Updated references to documentation (now docs.esmvaltool.org) (:pull:`675`) by :user:`axel-lauer` +- Add all European grants to Zenodo (:pull:`680`) by :user:`bouweandela` +- Update Sphinx to v3 or later (:pull:`683`) by :user:`bouweandela` +- Increase version to 2.0.0 and add release notes (:pull:`691`) by :user:`bouweandela` +- Update setup.py and README.md for use on PyPI (:pull:`693`) by :user:`bouweandela` +- Suggested Documentation changes (:pull:`690`) by :user:`ssmithClimate` Improvements ~~~~~~~~~~~~ -- Reduce the size of conda package (`#606 `__) `Bouwe Andela `__ -- Add a few unit tests for DiagnosticTask (`#613 `__) `Bouwe Andela `__ -- Make ncl or R tests not fail if package not installed (`#610 `__) `Valeriu Predoi `__ -- Pin flake8<3.8.0 (`#623 `__) `Valeriu Predoi `__ -- Log warnings for likely errors in provenance record (`#592 `__) `Bouwe Andela `__ -- Unpin flake8 (`#646 `__) `Bouwe Andela `__ -- More flexible native6 default DRS (`#645 `__) `Bouwe Andela `__ -- Try to use the same python for running diagnostics as for esmvaltool (`#656 `__) `Bouwe Andela `__ -- Fix test for lower python version and add note on lxml (`#659 `__) `Valeriu Predoi `__ -- Added 1m deep average soil moisture variable (`#664 `__) `bascrezee `__ -- Update docker recipe (`#603 `__) `Javier Vegas-Regidor `__ -- Improve command line interface (`#605 `__) `Javier Vegas-Regidor `__ -- Remove utils directory (`#697 `__) `Bouwe Andela `__ -- Avoid pytest version that crashes (`#707 `__) `Bouwe Andela `__ -- Options arg in read_config_user_file now optional (`#716 `__) `Javier Vegas-Regidor `__ -- Produce a readable warning if ancestors are a string instead of a list. (`#711 `__) `katjaweigel `__ -- Pin Yamale to v2 (`#718 `__) `Bouwe Andela `__ -- Expanded cmor public API (`#714 `__) `Manuel Schlund `__ +- Reduce the size of conda package (:pull:`606`) by :user:`bouweandela` +- Add a few unit tests for DiagnosticTask (:pull:`613`) by :user:`bouweandela` +- Make ncl or R tests not fail if package not installed (:pull:`610`) by :user:`valeriupredoi` +- Pin flake8<3.8.0 (:pull:`623`) by :user:`valeriupredoi` +- Log warnings for likely errors in provenance record (:pull:`592`) by :user:`bouweandela` +- Unpin flake8 (:pull:`646`) by :user:`bouweandela` +- More flexible native6 default DRS (:pull:`645`) by :user:`bouweandela` +- Try to use the same python for running diagnostics as for esmvaltool (:pull:`656`) by :user:`bouweandela` +- Fix test for lower python version and add note on lxml (:pull:`659`) by :user:`valeriupredoi` +- Added 1m deep average soil moisture variable (:pull:`664`) by :user:`bascrezee` +- Update docker recipe (:pull:`603`) by :user:`jvegreg` +- Improve command line interface (:pull:`605`) by :user:`jvegreg` +- Remove utils directory (:pull:`697`) by :user:`bouweandela` +- Avoid pytest version that crashes (:pull:`707`) by :user:`bouweandela` +- Options arg in read_config_user_file now optional (:pull:`716`) by :user:`jvegreg` +- Produce a readable warning if ancestors are a string instead of a list. (:pull:`711`) by :user:`katjaweigel` +- Pin Yamale to v2 (:pull:`718`) by :user:`bouweandela` +- Expanded cmor public API (:pull:`714`) by :user:`schlunma` Fixes for datasets ~~~~~~~~~~~~~~~~~~ -- Added various fixes for hybrid height coordinates (`#562 `__) `Manuel Schlund `__ -- Extended fix for cl-like variables of CESM2 models (`#604 `__) `Manuel Schlund `__ -- Added fix to convert "geopotential" to "geopotential height" for ERA5 (`#640 `__) `Evgenia Galytska `__ -- Do not fix longitude values if they are too far from valid range (`#636 `__) `Javier Vegas-Regidor `__ +- Added various fixes for hybrid height coordinates (:pull:`562`) by :user:`schlunma` +- Extended fix for cl-like variables of CESM2 models (:pull:`604`) by :user:`schlunma` +- Added fix to convert "geopotential" to "geopotential height" for ERA5 (:pull:`640`) by :user:`egalytska` +- Do not fix longitude values if they are too far from valid range (:pull:`636`) by :user:`jvegreg` Preprocessor ~~~~~~~~~~~~ -- Implemented concatenation of cubes with derived coordinates (`#546 `__) `Manuel Schlund `__ -- Fix derived variable ctotal calculation depending on project and standard name (`#620 `__) `Valeriu Predoi `__ -- State of the art FX variables handling without preprocessing (`#557 `__) `Valeriu Predoi `__ -- Add max, min and std operators to multimodel (`#602 `__) `Javier Vegas-Regidor `__ -- Added preprocessor to extract amplitude of cycles (`#597 `__) `Manuel Schlund `__ -- Overhaul concatenation and allow for correct concatenation of multiple overlapping datasets (`#615 `__) `Valeriu Predoi `__ -- Change volume stats to handle and output masked array result (`#618 `__) `Valeriu Predoi `__ -- Area_weights for cordex in area_statistics (`#631 `__) `mwjury `__ -- Accept cubes as input in multimodel (`#637 `__) `sloosvel `__ -- Make multimodel work correctly with yearly data (`#677 `__) `Valeriu Predoi `__ -- Optimize time weights in time preprocessor for climate statistics (`#684 `__) `Valeriu Predoi `__ -- Add percentiles to multi-model stats (`#679 `__) `Peter Kalverla `__ +- Implemented concatenation of cubes with derived coordinates (:pull:`546`) by :user:`schlunma` +- Fix derived variable ctotal calculation depending on project and standard name (:pull:`620`) by :user:`valeriupredoi` +- State of the art FX variables handling without preprocessing (:pull:`557`) by :user:`valeriupredoi` +- Add max, min and std operators to multimodel (:pull:`602`) by :user:`jvegreg` +- Added preprocessor to extract amplitude of cycles (:pull:`597`) by :user:`schlunma` +- Overhaul concatenation and allow for correct concatenation of multiple overlapping datasets (:pull:`615`) by :user:`valeriupredoi` +- Change volume stats to handle and output masked array result (:pull:`618`) by :user:`valeriupredoi` +- Area_weights for cordex in area_statistics (:pull:`631`) by :user:`mwjury` +- Accept cubes as input in multimodel (:pull:`637`) by :user:`sloosvel` +- Make multimodel work correctly with yearly data (:pull:`677`) by :user:`valeriupredoi` +- Optimize time weights in time preprocessor for climate statistics (:pull:`684`) by :user:`valeriupredoi` +- Add percentiles to multi-model stats (:pull:`679`) by :user:`Peter9192` .. _changelog-v2-0-0b9: @@ -1425,21 +1432,21 @@ This release includes Bug fixes ~~~~~~~~~ -- Cast dtype float32 to output from zonal and meridional area preprocessors (`#581 `__) `Valeriu Predoi `__ +- Cast dtype float32 to output from zonal and meridional area preprocessors (:pull:`581`) by :user:`valeriupredoi` Improvements ~~~~~~~~~~~~ -- Unpin on Python<3.8 for conda package (run) (`#570 `__) `Valeriu Predoi `__ -- Update pytest installation marker (`#572 `__) `Bouwe Andela `__ -- Remove vmrh2o (`#573 `__) `Mattia Righi `__ -- Restructure documentation (`#575 `__) `Bouwe Andela `__ -- Fix mask in land variables for CCSM4 (`#579 `__) `Klaus Zimmermann `__ -- Fix derive scripts wrt required method (`#585 `__) `Klaus Zimmermann `__ -- Check coordinates do not have repeated standard names (`#558 `__) `Javier Vegas-Regidor `__ -- Added derivation script for co2s (`#587 `__) `Manuel Schlund `__ -- Adapted custom co2s table to match CMIP6 version (`#588 `__) `Manuel Schlund `__ -- Increase version to v2.0.0b9 (`#593 `__) `Bouwe Andela `__ -- Add a method to save citation information (`#402 `__) `SarahAlidoost `__ +- Unpin on Python<3.8 for conda package (run) (:pull:`570`) by :user:`valeriupredoi` +- Update pytest installation marker (:pull:`572`) by :user:`bouweandela` +- Remove vmrh2o (:pull:`573`) by :user:`mattiarighi` +- Restructure documentation (:pull:`575`) by :user:`bouweandela` +- Fix mask in land variables for CCSM4 (:pull:`579`) by :user:`zklaus` +- Fix derive scripts wrt required method (:pull:`585`) by :user:`zklaus` +- Check coordinates do not have repeated standard names (:pull:`558`) by :user:`jvegreg` +- Added derivation script for co2s (:pull:`587`) by :user:`schlunma` +- Adapted custom co2s table to match CMIP6 version (:pull:`588`) by :user:`schlunma` +- Increase version to v2.0.0b9 (:pull:`593`) by :user:`bouweandela` +- Add a method to save citation information (:pull:`402`) by :user:`SarahAlidoost` For older releases, see the release notes on https://github.com/ESMValGroup/ESMValCore/releases. diff --git a/doc/conf.py b/doc/conf.py index 1613a0133b..3f443ced03 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -60,6 +60,7 @@ 'nbsphinx', 'sphinx.ext.autodoc', 'sphinx.ext.doctest', + 'sphinx.ext.extlinks', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', @@ -450,6 +451,36 @@ 'scipy': ('https://docs.scipy.org/doc/scipy/', None), } +# -- Extlinks extension ------------------------------------------------------- +# See https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html + +extlinks = { + "discussion": ( + "https://github.com/ESMValGroup/ESMValCore/discussions/%s", + "Discussion #%s", + ), + "issue": ( + "https://github.com/ESMValGroup/ESMValCore/issues/%s", + "Issue #%s", + ), + "pull": ( + "https://github.com/ESMValGroup/ESMValCore/pull/%s", + "Pull request #%s", + ), + "release": ( + "https://github.com/ESMValGroup/ESMValCore/releases/tag/%s", + "ESMValCore %s", + ), + "team": ( + "https://github.com/orgs/ESMValGroup/teams/%s", + "@ESMValGroup/%s", + ), + "user": ( + "https://github.com/%s", + "@%s", + ), +} + # -- Custom Document processing ---------------------------------------------- sys.path.append(os.path.dirname(__file__)) diff --git a/doc/quickstart/configure.rst b/doc/quickstart/configure.rst index 4a3642f1ca..fadd145b24 100644 --- a/doc/quickstart/configure.rst +++ b/doc/quickstart/configure.rst @@ -243,8 +243,7 @@ Extensive documentation on setting up Dask Clusters is available If not all preprocessor functions support lazy data, computational performance may be best with the default scheduler. - See `issue #674 `_ for - progress on making all preprocessor functions lazy. + See :issue:`674` for progress on making all preprocessor functions lazy. **Example configurations** @@ -383,8 +382,8 @@ corresponding command line arguments ``--search_esgf=when_missing`` or tool by pressing the ``Ctrl`` and ``C`` keys on your keyboard simultaneously several times, edit the recipe so it contains fewer datasets and try again. -For downloading some files (e.g. those produced by the CORDEX project), -you need to log in to be able to download the data. +For downloading some files, you may need to log in to be able to download the +data. See the `ESGF user guide `_ @@ -684,7 +683,8 @@ related to CMOR table settings available: extended with variables from the :ref:`custom_cmor_tables` (by default loaded from the ``esmvalcore/cmor/tables/custom`` directory) and it is possible to use variables with a ``mip`` which is different from the MIP table in which - they are defined. + they are defined. Note that this option is always enabled for + :ref:`derived ` variables. * ``cmor_path``: path to the CMOR table. Relative paths are with respect to `esmvalcore/cmor/tables`_. Defaults to the value provided in ``cmor_type`` written in lower case. @@ -699,11 +699,18 @@ Custom CMOR tables As mentioned in the previous section, the CMOR tables of projects that use ``cmor_strict: false`` will be extended with custom CMOR tables. -By default, these are loaded from `esmvalcore/cmor/tables/custom +For derived variables (the ones with ``derive: true`` in the recipe), the +custom CMOR tables will always be considered. +By default, these custom tables are loaded from `esmvalcore/cmor/tables/custom `_. However, by using the special project ``custom`` in the ``config-developer.yml`` file with the option ``cmor_path``, a custom location -for these custom CMOR tables can be specified: +for these custom CMOR tables can be specified. +In this case, the default custom tables are extended with those entries from +the custom location (in case of duplication, the custom location tables take +precedence). + +Example: .. code-block:: yaml @@ -743,11 +750,10 @@ Example for the file ``CMOR_asr.dat``: !---------------------------------- ! -It is also possible to use a special coordinates file ``CMOR_coordinates.dat``. -If this is not present in the custom directory, the one from the default -directory (`esmvalcore/cmor/tables/custom/CMOR_coordinates.dat -`_) -is used. +It is also possible to use a special coordinates file ``CMOR_coordinates.dat``, +which will extend the entries from the default one +(`esmvalcore/cmor/tables/custom/CMOR_coordinates.dat +`_). .. _filterwarnings_config-developer: diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index 5062d9fe15..31ee262ef5 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -121,13 +121,13 @@ ERA5 - Supported variables: ``cl``, ``clt``, ``evspsbl``, ``evspsblpot``, ``mrro``, ``pr``, ``prsn``, ``ps``, ``psl``, ``ptype``, ``rls``, ``rlds``, ``rsds``, ``rsdt``, ``rss``, ``uas``, ``vas``, ``tas``, ``tasmax``, ``tasmin``, ``tdps``, ``ts``, ``tsn`` (``E1hr``/``Amon``), ``orog`` (``fx``) - Tier: 3 -.. note:: According to the description of Evapotranspiration and potential Evapotranspiration on the Copernicus page +.. note:: According to the description of Evapotranspiration and potential Evapotranspiration on the Copernicus page (https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-monthly-means?tab=overview): - "The ECMWF Integrated Forecasting System (IFS) convention is that downward fluxes are positive. + "The ECMWF Integrated Forecasting System (IFS) convention is that downward fluxes are positive. Therefore, negative values indicate evaporation and positive values indicate condensation." - + In the CMOR table, these fluxes are defined as positive, if they go from the surface into the atmosphere: - "Evaporation at surface (also known as evapotranspiration): flux of water into the atmosphere due to conversion + "Evaporation at surface (also known as evapotranspiration): flux of water into the atmosphere due to conversion of both liquid and solid phases to vapor (from underlying surface and vegetation)." Therefore, the ERA5 (and ERA5-Land) CMORizer switches the signs of ``evspsbl`` and ``evspsblpot`` to be compatible with the CMOR standard used e.g. by the CMIP models. @@ -398,7 +398,7 @@ The UGRID conventions provide a standardized format to store data on unstructured grids, which is required by many software packages or tools to work correctly. An example is the horizontal regridding of native ICON data to a regular grid. -While the built-in :ref:`unstructured_nearest scheme ` can handle unstructured grids not in UGRID format, using more complex regridding algorithms (for example provided by the :doc:`iris-esmf-regrid:index` package through :ref:`generic regridding @@ -420,7 +420,7 @@ This automatic UGRIDization is enabled by default, but can be switched off with the facet ``ugrid: false`` in the recipe or the extra facets (see below). This is useful for diagnostics that do not support input data in UGRID format (yet) like the :ref:`Psyplot diagnostic ` or -if you want to use the built-in :ref:`unstructured_nearest scheme ` regridding scheme. For 3D ICON variables, ESMValCore tries to add the pressure level information diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index 5c4b14e351..638a9c6c1e 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -201,13 +201,15 @@ the ozone 3D field. In this case, a derivation function is provided to vertically integrate the ozone and obtain total column ozone for direct comparison with the observations. -To contribute a new derived variable, it is also necessary to define a name for -it and to provide the corresponding CMOR table. This is to guarantee the proper -metadata definition is attached to the derived data. Such custom CMOR tables -are collected as part of the `ESMValCore package -`_. By default, the variable -derivation will be applied only if the variable is not already available in the -input data, but the derivation can be forced by setting the appropriate flag. +The tool will also look in other ``mip`` tables for the same ``project`` to find +the definition of derived variables. To contribute a completely new derived +variable, it is necessary to define a name for it and to provide the +corresponding CMOR table. This is to guarantee the proper metadata definition +is attached to the derived data. Such custom CMOR tables are collected as part +of the `ESMValCore package `_. +By default, the variable derivation will be applied only if the variable is not +already available in the input data, but the derivation can be forced by +setting the ``force_derivation`` flag. .. code-block:: yaml @@ -896,43 +898,27 @@ third party regridding schemes designed for use with :doc:`Iris Built-in regridding schemes ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The schemes used for the interpolation and extrapolation operations needed by -the horizontal regridding functionality directly map to their corresponding -implementations in :mod:`iris`: - -* ``linear``: Linear interpolation without extrapolation, i.e., extrapolation - points will be masked even if the source data is not a masked array (uses - ``Linear(extrapolation_mode='mask')``, see :obj:`iris.analysis.Linear`). -* ``linear_extrapolate``: Linear interpolation with extrapolation, i.e., - extrapolation points will be calculated by extending the gradient of the - closest two points (uses ``Linear(extrapolation_mode='extrapolate')``, see - :obj:`iris.analysis.Linear`). -* ``nearest``: Nearest-neighbour interpolation without extrapolation, i.e., - extrapolation points will be masked even if the source data is not a masked - array (uses ``Nearest(extrapolation_mode='mask')``, see - :obj:`iris.analysis.Nearest`). -* ``area_weighted``: Area-weighted regridding (uses ``AreaWeighted()``, see - :obj:`iris.analysis.AreaWeighted`). -* ``unstructured_nearest``: Nearest-neighbour interpolation for unstructured - grids (uses ``UnstructuredNearest()``, see - :obj:`iris.analysis.UnstructuredNearest`). +* ``linear``: Bilinear regridding. + For source data on a regular grid, uses :obj:`~iris.analysis.Linear` with + `extrapolation_mode='mask'`. + For source data on an irregular grid, uses + :class:`~esmvalcore.preprocessor.regrid_schemes.ESMPyLinear`. + Source data on an unstructured grid is not supported, yet. +* ``nearest``: Nearest-neighbor regridding. + For source data on a regular grid, uses :obj:`~iris.analysis.Nearest` with + `extrapolation_mode='mask'`. + For source data on an irregular grid, uses + :class:`~esmvalcore.preprocessor.regrid_schemes.ESMPyNearest`. + For source data on an unstructured grid, uses + :class:`~esmvalcore.preprocessor.regrid_schemes.UnstructuredNearest`. +* ``area_weighted``: First-order conservative (area-weighted) regridding. + For source data on a regular grid, uses :obj:`~iris.analysis.AreaWeighted`. + For source data on an irregular grid, uses + :class:`~esmvalcore.preprocessor.regrid_schemes.ESMPyAreaWeighted`. + Source data on an unstructured grid is not supported, yet. See also :func:`esmvalcore.preprocessor.regrid` -.. note:: - - Controlling the extrapolation mode allows us to avoid situations where - extrapolating values makes little physical sense (e.g. extrapolating beyond - the last data point). - -.. note:: - - The regridding mechanism is (at the moment) done with fully realized data in - memory, so depending on how fine the target grid is, it may use a rather - large amount of memory. Empirically target grids of up to ``0.5x0.5`` - degrees should not produce any memory-related issues, but be advised that - for resolutions of ``< 0.5`` degrees the regridding becomes very slow and - will use a lot of memory. .. _generic regridding schemes: @@ -969,6 +955,28 @@ tolerance. reference: iris.analysis:AreaWeighted mdtol: 0.7 +Another example is bilinear regridding with extrapolation. +This can be achieved with the :class:`iris.analysis.Linear` scheme and the +``extrapolation_mode`` keyword. +Extrapolation points will be calculated by extending the gradient of the +closest two points. + +.. code-block:: yaml + + preprocessors: + regrid_preprocessor: + regrid: + target_grid: 2.5x2.5 + scheme: + reference: iris.analysis:Linear + extrapolation_mode: extrapolate + +.. note:: + + Controlling the extrapolation mode allows us to avoid situations where + extrapolating values makes little physical sense (e.g. extrapolating beyond + the last data point). + The value of the ``reference`` key has two parts that are separated by a ``:`` with no surrounding spaces. The first part is an importable Python module, the second refers to the scheme, i.e. some callable that will be called @@ -1219,6 +1227,7 @@ The ``_time.py`` module contains the following preprocessor functions: * regrid_time_: Aligns the time axis of each dataset to have common time points and calendars. * timeseries_filter_: Allows application of a filter to the time-series data. +* local_solar_time_: Convert cube with UTC time to local solar time. Statistics functions are applied by default in the order they appear in the list. For example, the following example applied to hourly data will retrieve @@ -1653,6 +1662,55 @@ Examples: See also :func:`esmvalcore.preprocessor.timeseries_filter`. +.. _local_solar_time: + +``local_solar_time`` +-------------------- + +Many variables in the Earth system show a strong diurnal cycle. +The reason for that is of course Earth's rotation around its own axis, which +leads to a diurnal cycle of the incoming solar radiation. +While UTC time is a very good absolute time measure, it is not really suited to +analyze diurnal cycles over larger regions. +For example, diurnal cycles over Russia and the USA are phase-shifted by ~180° += 12 hr in UTC time. + +This is where the `local solar time (LST) +`__ comes into play: +For a given location, 12:00 noon LST is defined as the moment when the sun +reaches its highest point in the sky. +By using this definition based on the origin of the diurnal cycle (the sun), we +can directly compare diurnal cycles across the globe. +LST is mainly determined by the longitude of a location, but due to the +eccentricity of Earth's orbit, it also depends on the day of year (see +`equation of time `__). +However, this correction is at most ~15 min, which is usually smaller than the +highest frequency output of CMIP6 models (1 hr) and smaller than the time scale +for diurnal evolution of meteorological phenomena (which is in the order of +hours, not minutes). +Thus, instead, we use the **mean** LST, which solely depends on longitude: + +.. math:: + + LST = UTC + 12 \cdot \frac{lon}{180°} + +where the times are given in hours and `lon` in degrees in the interval [-180, +180]. +To transform data from UTC to LST, this preprocessor shifts data along the time +axis based on the longitude. + +This preprocessor does not need any additional parameters. + +Example: + +.. code-block:: yaml + + calculate_local_solar_time: + local_solar_time: + +See also :func:`esmvalcore.preprocessor.local_solar_time`. + + .. _area operations: Area manipulation @@ -1904,11 +1962,15 @@ along the longitude coordinate. Parameters: * `operator`: Operation to apply. See :ref:`stat_preprocs` for more details on supported statistics. + * `normalize`: If given, do not return the statistics cube itself, but + rather, the input cube, normalized with the statistics cube. Can either + be `subtract` (statistics cube is subtracted from the input cube) or + `divide` (input cube is divided by the statistics cube). * Other parameters are directly passed to the `operator` as keyword arguments. See :ref:`stat_preprocs` for more details. -See also :func:`esmvalcore.preprocessor.zonal_means`. +See also :func:`esmvalcore.preprocessor.zonal_statistics`. ``meridional_statistics`` @@ -1921,11 +1983,15 @@ argument: Parameters: * `operator`: Operation to apply. See :ref:`stat_preprocs` for more details on supported statistics. + * `normalize`: If given, do not return the statistics cube itself, but + rather, the input cube, normalized with the statistics cube. Can either + be `subtract` (statistics cube is subtracted from the input cube) or + `divide` (input cube is divided by the statistics cube). * Other parameters are directly passed to the `operator` as keyword arguments. See :ref:`stat_preprocs` for more details. -See also :func:`esmvalcore.preprocessor.meridional_means`. +See also :func:`esmvalcore.preprocessor.meridional_statistics`. .. _area_statistics: @@ -1933,10 +1999,6 @@ See also :func:`esmvalcore.preprocessor.meridional_means`. ``area_statistics`` ------------------- -This function calculates statistics over a region. -It takes one argument, ``operator``, which is the name of the operation to -apply. - This function can be used to apply several different operations in the horizontal plane: for example, mean, sum, standard deviation, median, variance, minimum, maximum and root mean square. @@ -1955,6 +2017,33 @@ The required supplementary variable, either ``areacella`` for atmospheric variables or ``areacello`` for ocean variables, can be attached to the main dataset as described in :ref:`supplementary_variables`. +Parameters: + * `operator`: Operation to apply. + See :ref:`stat_preprocs` for more details on supported statistics. + * `normalize`: If given, do not return the statistics cube itself, but + rather, the input cube, normalized with the statistics cube. Can either + be `subtract` (statistics cube is subtracted from the input cube) or + `divide` (input cube is divided by the statistics cube). + * Other parameters are directly passed to the `operator` as keyword + arguments. + See :ref:`stat_preprocs` for more details. + +Examples: +* Calculate global mean: + + .. code-block:: yaml + + area_statistics: + operator: mean + +* Subtract global mean from dataset: + + .. code-block:: yaml + + area_statistics: + operator: mean + normalize: subtract + See also :func:`esmvalcore.preprocessor.area_statistics`. @@ -2042,9 +2131,6 @@ See also :func:`esmvalcore.preprocessor.extract_volume`. This function calculates the volume-weighted average across three dimensions, but maintains the time dimension. -This function takes the argument: `operator`, which defines the operation to -apply over the volume. -At the moment, only `mean` is supported. By default, the `mean` operation is weighted by the grid cell volumes. For weighted statistics, this function requires a cell volume `cell measure`_, @@ -2055,6 +2141,18 @@ dataset as described in :ref:`supplementary_variables`. No depth coordinate is required as this is determined by Iris. +Parameters: + * `operator`: Operation to apply. + At the moment, only `mean` is supported. + See :ref:`stat_preprocs` for more details on supported statistics. + * `normalize`: If given, do not return the statistics cube itself, but + rather, the input cube, normalized with the statistics cube. Can either + be `subtract` (statistics cube is subtracted from the input cube) or + `divide` (input cube is divided by the statistics cube). + * Other parameters are directly passed to the `operator` as keyword + arguments. + See :ref:`stat_preprocs` for more details. + See also :func:`esmvalcore.preprocessor.volume_statistics`. .. _axis_statistics: @@ -2070,6 +2168,10 @@ Takes arguments: Possible values for the axis are `x`, `y`, `z`, `t`. * `operator`: Operation to apply. See :ref:`stat_preprocs` for more details on supported statistics. + * `normalize`: If given, do not return the statistics cube itself, but + rather, the input cube, normalized with the statistics cube. Can either + be `subtract` (statistics cube is subtracted from the input cube) or + `divide` (input cube is divided by the statistics cube). * Other parameters are directly passed to the `operator` as keyword arguments. See :ref:`stat_preprocs` for more details. @@ -2283,6 +2385,8 @@ Currently, the following special conversions are supported: * ``precipitation_flux`` (``kg m-2 s-1``) -- ``lwe_precipitation_rate`` (``mm day-1``) +* ``equivalent_thickness_at_stp_of_atmosphere_ozone_content`` (``m``) -- + ``equivalent_thickness_at_stp_of_atmosphere_ozone_content`` (``DU``) .. hint:: Names in the list correspond to ``standard_names`` of the input data. @@ -2332,9 +2436,9 @@ The bias module contains the following preprocessor functions: ``bias`` -------- -This function calculates biases with respect to a given reference dataset. For -this, exactly one input dataset needs to be declared as ``reference_for_bias: -true`` in the recipe, e.g., +This function calculates biases with respect to a given reference dataset. +For this, exactly one input dataset needs to be declared as +``reference_for_bias: true`` in the recipe, e.g., .. code-block:: yaml @@ -2346,34 +2450,37 @@ true`` in the recipe, e.g., reference_for_bias: true} In the example above, ERA-Interim is used as reference dataset for the bias -calculation. For this preprocessor, all input datasets need to have identical -dimensional coordinates. This can for example be ensured with the preprocessors -:func:`esmvalcore.preprocessor.regrid` and/or -:func:`esmvalcore.preprocessor.regrid_time`. - -The ``bias`` preprocessor supports 4 optional arguments: - - * ``bias_type`` (:obj:`str`, default: ``'absolute'``): Bias type that is - calculated. Can be ``'absolute'`` (i.e., calculate bias for dataset - :math:`X` and reference :math:`R` as :math:`X - R`) or ``relative`` (i.e, - calculate bias as :math:`\frac{X - R}{R}`). - * ``denominator_mask_threshold`` (:obj:`float`, default: ``1e-3``): - Threshold to mask values close to zero in the denominator (i.e., the - reference dataset) during the calculation of relative biases. All values - in the reference dataset with absolute value less than the given threshold - are masked out. This setting is ignored when ``bias_type`` is set to - ``'absolute'``. Please note that for some variables with very small - absolute values (e.g., carbon cycle fluxes, which are usually :math:`< - 10^{-6}` kg m :math:`^{-2}` s :math:`^{-1}`) it is absolutely essential to - change the default value in order to get reasonable results. - * ``keep_reference_dataset`` (:obj:`bool`, default: ``False``): If - ``True``, keep the reference dataset in the output. If ``False``, drop the - reference dataset. - * ``exclude`` (:obj:`list` of :obj:`str`): Exclude specific datasets from - this preprocessor. Note that this option is only available in the recipe, - not when using :func:`esmvalcore.preprocessor.bias` directly (e.g., in - another python script). If the reference dataset has been excluded, an - error is raised. +calculation. +The reference dataset needs to be broadcastable to all other datasets. +This supports `iris' rich broadcasting abilities +`__. +To ensure this, the preprocessors :func:`esmvalcore.preprocessor.regrid` and/or +:func:`esmvalcore.preprocessor.regrid_time` might be helpful. + +The ``bias`` preprocessor supports 4 optional arguments in the recipe: + +* ``bias_type`` (:obj:`str`, default: ``'absolute'``): Bias type that is + calculated. Can be ``'absolute'`` (i.e., calculate bias for dataset + :math:`X` and reference :math:`R` as :math:`X - R`) or ``relative`` (i.e., + calculate bias as :math:`\frac{X - R}{R}`). +* ``denominator_mask_threshold`` (:obj:`float`, default: ``1e-3``): + Threshold to mask values close to zero in the denominator (i.e., the + reference dataset) during the calculation of relative biases. All values + in the reference dataset with absolute value less than the given threshold + are masked out. This setting is ignored when ``bias_type`` is set to + ``'absolute'``. Please note that for some variables with very small + absolute values (e.g., carbon cycle fluxes, which are usually :math:`< + 10^{-6}` kg m :math:`^{-2}` s :math:`^{-1}`) it is absolutely essential to + change the default value in order to get reasonable results. +* ``keep_reference_dataset`` (:obj:`bool`, default: ``False``): If + ``True``, keep the reference dataset in the output. If ``False``, drop the + reference dataset. +* ``exclude`` (:obj:`list` of :obj:`str`): Exclude specific datasets from + this preprocessor. Note that this option is only available in the recipe, + not when using :func:`esmvalcore.preprocessor.bias` directly (e.g., in + another python script). If the reference dataset has been excluded, an + error is raised. Example: diff --git a/environment.yml b/environment.yml index cb066cba09..ede6337b73 100644 --- a/environment.yml +++ b/environment.yml @@ -20,7 +20,7 @@ dependencies: - geopy - humanfriendly - importlib_metadata # required for Python < 3.10 - - iris >=3.6.0 + - iris >=3.6.1 - iris-esmf-regrid >=0.7.0 - isodate - jinja2 @@ -30,14 +30,14 @@ dependencies: - netcdf4 - numpy !=1.24.3 - packaging - - pandas + - pandas !=2.2.0 # github.com/ESMValGroup/ESMValCore/pull/2305 - pillow - pip !=21.3 - prov - psutil - py-cordex - pybtex - - python >=3.9 + - python >=3.9,<3.12 - python-stratify >=0.3 - pyyaml - requests @@ -58,7 +58,6 @@ dependencies: - pytest-env - pytest-html !=2.1.0 - pytest-metadata >=1.5.1 - - pytest-mypy - pytest-mock - pytest-xdist # Not on conda-forge - ESMValTool_sample_data==0.0.3 diff --git a/esmvalcore/_main.py b/esmvalcore/_main.py index afb0cc0476..1fa4e80ce5 100755 --- a/esmvalcore/_main.py +++ b/esmvalcore/_main.py @@ -353,8 +353,10 @@ def run(self, Recipe to run, as either the name of an installed recipe or the path to a non-installed one. config_file: str, optional - Configuration file to use. If not provided the file - ${HOME}/.esmvaltool/config-user.yml will be used. + Configuration file to use. Can be given as absolute or relative + path. In the latter case, search in the current working directory + and `${HOME}/.esmvaltool` (in that order). If not provided, the + file `${HOME}/.esmvaltool/config-user.yml` will be used. resume_from: list(str), optional Resume one or more previous runs by using preprocessor output files from these output directories. @@ -383,9 +385,15 @@ def run(self, """ from .config import CFG + # At this point, --config_file is already parsed if a valid file has + # been given (see + # https://github.com/ESMValGroup/ESMValCore/issues/2280), but no error + # has been raised if the file does not exist. Thus, reload the file + # here with `load_from_file` to make sure a proper error is raised. + CFG.load_from_file(config_file) + recipe = self._get_recipe(recipe) - CFG.load_from_file(config_file) session = CFG.start_session(recipe.stem) if check_level is not None: session['check_level'] = check_level diff --git a/esmvalcore/_recipe/check.py b/esmvalcore/_recipe/check.py index 4b1b36cb60..48679585a2 100644 --- a/esmvalcore/_recipe/check.py +++ b/esmvalcore/_recipe/check.py @@ -17,6 +17,12 @@ from esmvalcore.local import _get_start_end_year, _parse_period from esmvalcore.preprocessor import TIME_PREPROCESSORS, PreprocessingTask from esmvalcore.preprocessor._multimodel import _get_operator_and_kwargs +from esmvalcore.preprocessor._regrid import ( + HORIZONTAL_SCHEMES_IRREGULAR, + HORIZONTAL_SCHEMES_REGULAR, + HORIZONTAL_SCHEMES_UNSTRUCTURED, + _load_generic_scheme, +) from esmvalcore.preprocessor._shared import get_iris_aggregator from esmvalcore.preprocessor._supplementary_vars import ( PREPROCESSOR_SUPPLEMENTARIES, @@ -64,7 +70,7 @@ def diagnostics(diags): for name, diagnostic in diags.items(): if 'scripts' not in diagnostic: raise RecipeError( - "Missing scripts section in diagnostic {}".format(name)) + f"Missing scripts section in diagnostic '{name}'.") variable_names = tuple(diagnostic.get('variables', {})) scripts = diagnostic.get('scripts') if scripts is None: @@ -72,13 +78,13 @@ def diagnostics(diags): for script_name, script in scripts.items(): if script_name in variable_names: raise RecipeError( - "Invalid script name {} encountered in diagnostic {}: " - "scripts cannot have the same name as variables.".format( - script_name, name)) + f"Invalid script name '{script_name}' encountered " + f"in diagnostic '{name}': scripts cannot have the " + "same name as variables.") if not script.get('script'): raise RecipeError( - "No script defined for script {} in diagnostic {}".format( - script_name, name)) + f"No script defined for script '{script_name}' in " + f"diagnostic '{name}'.") def duplicate_datasets( @@ -89,27 +95,31 @@ def duplicate_datasets( """Check for duplicate datasets.""" if not datasets: raise RecipeError( - "You have not specified any dataset or additional_dataset groups " - f"for variable {variable_group} in diagnostic {diagnostic}.") + "You have not specified any dataset or additional_dataset " + f"groups for variable '{variable_group}' in diagnostic " + f"'{diagnostic}'.") checked_datasets_ = [] for dataset in datasets: if dataset in checked_datasets_: raise RecipeError( - f"Duplicate dataset {dataset} for variable {variable_group} " - f"in diagnostic {diagnostic}.") + f"Duplicate dataset\n{pformat(dataset)}\nfor variable " + f"'{variable_group}' in diagnostic '{diagnostic}'.") checked_datasets_.append(dataset) -def variable(var: dict[str, Any], required_keys: Iterable[str]): +def variable( + var: dict[str, Any], + required_keys: Iterable[str], + diagnostic: str, + variable_group: str +) -> None: """Check variables as derived from recipe.""" required = set(required_keys) missing = required - set(var) if missing: raise RecipeError( - f"Missing keys {missing} in\n" - f"{pformat(var)}\n" - "for variable {var['variable_group']} in diagnostic " - f"{var['diagnostic']}") + f"Missing keys {missing} in\n{pformat(var)}\nfor variable " + f"'{variable_group}' in diagnostic '{diagnostic}'.") def _log_data_availability_errors(dataset): @@ -385,6 +395,19 @@ def differing_timeranges(timeranges, required_vars): "Set `timerange` to a common value.") +def bias_type(settings: dict) -> None: + """Check that bias_type for bias preprocessor is valid.""" + if 'bias' not in settings: + return + valid_options = ('absolute', 'relative') + user_bias_type = settings['bias'].get('bias_type', 'absolute') + if user_bias_type not in valid_options: + raise RecipeError( + f"Expected one of {valid_options} for `bias_type`, got " + f"'{user_bias_type}'" + ) + + def reference_for_bias_preproc(products): """Check that exactly one reference dataset for bias preproc is given.""" step = 'bias' @@ -473,3 +496,50 @@ def _check_mm_stat(step, step_settings): raise RecipeError( f"Invalid options for {step}: {exc}" ) + + +def regridding_schemes(settings: dict): + """Check :obj:`str` regridding schemes.""" + if 'regrid' not in settings: + return + + # Note: If 'scheme' is missing, this will be detected in + # PreprocessorFile.check() later + scheme = settings['regrid'].get('scheme') + + # Check built-in regridding schemes (given as str) + if isinstance(scheme, str): + scheme = settings['regrid']['scheme'] + + # Also allow deprecated 'linear_extrapolate' and 'unstructured_nearest' + # schemes (the corresponding deprecation warnings will be raised in the + # regrid() preprocessor) TODO: Remove in v2.13.0 + if scheme in ('linear_extrapolate', 'unstructured_nearest'): + return + + allowed_regridding_schemes = list( + set( + list(HORIZONTAL_SCHEMES_IRREGULAR) + + list(HORIZONTAL_SCHEMES_REGULAR) + + list(HORIZONTAL_SCHEMES_UNSTRUCTURED) + ) + ) + if scheme not in allowed_regridding_schemes: + raise RecipeError( + f"Got invalid built-in regridding scheme '{scheme}', expected " + f"one of {allowed_regridding_schemes} or a generic scheme " + f"(see https://docs.esmvaltool.org/projects/ESMValCore/en/" + f"latest/recipe/preprocessor.html#generic-regridding-schemes)." + ) + + # Check generic regridding schemes (given as dict) + if isinstance(scheme, dict): + try: + _load_generic_scheme(scheme) + except ValueError as exc: + raise RecipeError( + f"Failed to load generic regridding scheme: {str(exc)} See " + f"https://docs.esmvaltool.org/projects/ESMValCore/en/latest" + f"/recipe/preprocessor.html#generic-regridding-schemes for " + f"details." + ) diff --git a/esmvalcore/_recipe/recipe.py b/esmvalcore/_recipe/recipe.py index 1613f6cf6d..4369a7d4ac 100644 --- a/esmvalcore/_recipe/recipe.py +++ b/esmvalcore/_recipe/recipe.py @@ -250,25 +250,42 @@ def _add_to_download_list(dataset): def _schedule_for_download(datasets): - """Schedule files for download and show the list of files in the log.""" + """Schedule files for download.""" for dataset in datasets: _add_to_download_list(dataset) for supplementary_ds in dataset.supplementaries: _add_to_download_list(supplementary_ds) - files = list(dataset.files) - for supplementary_ds in dataset.supplementaries: - files.extend(supplementary_ds.files) + +def _log_input_files(datasets: Iterable[Dataset]) -> None: + """Show list of files in log (including supplementaries).""" + for dataset in datasets: + # Only log supplementary variables if present + supplementary_files_str = "" + if dataset.supplementaries: + for sup_ds in dataset.supplementaries: + supplementary_files_str += ( + f"\nwith files for supplementary variable " + f"{sup_ds['short_name']}:\n{_get_files_str(sup_ds)}" + ) logger.debug( - "Using input files for variable %s of dataset %s:\n%s", + "Using input files for variable %s of dataset %s:\n%s%s", dataset.facets['short_name'], - dataset.facets['alias'].replace('_', ' '), - '\n'.join(f'{f} (will be downloaded)' if not f.exists() else str(f) - for f in files), + dataset.facets['alias'].replace('_', ' '), # type: ignore + _get_files_str(dataset), + supplementary_files_str ) +def _get_files_str(dataset: Dataset) -> str: + """Get nice string representation of all files of a dataset.""" + return '\n'.join( + f' {f}' if f.exists() # type: ignore + else f' {f} (will be downloaded)' for f in dataset.files + ) + + def _check_input_files(input_datasets: Iterable[Dataset]) -> set[str]: """Check that the required input files are available.""" missing = set() @@ -517,6 +534,7 @@ def _get_preprocessor_products( _set_version(dataset, input_datasets) USED_DATASETS.append(dataset) _schedule_for_download(input_datasets) + _log_input_files(input_datasets) logger.info("Found input files for %s", dataset.summary(shorten=True)) filename = _get_output_file( @@ -638,6 +656,8 @@ def _update_preproc_functions(settings, dataset, datasets, missing_vars): if dataset.facets.get('frequency') == 'fx': check.check_for_temporal_preprocs(settings) check.statistics_preprocessors(settings) + check.regridding_schemes(settings) + check.bias_type(settings) def _get_preprocessor_task(datasets, profiles, task_name): diff --git a/esmvalcore/_recipe/recipe_schema.yml b/esmvalcore/_recipe/recipe_schema.yml index ed61c4ad43..dc254a0d52 100644 --- a/esmvalcore/_recipe/recipe_schema.yml +++ b/esmvalcore/_recipe/recipe_schema.yml @@ -46,7 +46,7 @@ variable: # TODO: add preprocessor item diagnostic: - scripts: any(null(), map(include('script'))) + scripts: any(null(), map(include('script')), required=False) additional_datasets: list(include('dataset'), required=False) title: str(required=False) description: str(required=False) @@ -55,4 +55,4 @@ diagnostic: variables: map(include('variable'), null(), required=False) script: - script: str() + script: str(required=False) diff --git a/esmvalcore/_recipe/to_datasets.py b/esmvalcore/_recipe/to_datasets.py index 56d9d44221..fd286ea81b 100644 --- a/esmvalcore/_recipe/to_datasets.py +++ b/esmvalcore/_recipe/to_datasets.py @@ -244,6 +244,7 @@ def _get_dataset_facets_from_recipe( recipe_variable: dict[str, Any], recipe_dataset: dict[str, Any], profiles: dict[str, Any], + diagnostic_name: str, session: Session, ) -> tuple[Facets, list[Facets]]: """Read the facets for a single dataset definition from the recipe.""" @@ -286,6 +287,8 @@ def _get_dataset_facets_from_recipe( 'dataset', 'project', ), + diagnostic=diagnostic_name, + variable_group=variable_group ) preprocessor = facets.get('preprocessor', 'default') @@ -329,6 +332,7 @@ def _get_facets_from_recipe( recipe_variable=recipe_variable, recipe_dataset=recipe_dataset, profiles=profiles, + diagnostic_name=diagnostic_name, session=session, ) @@ -483,8 +487,12 @@ def _get_input_datasets(dataset: Dataset) -> list[Dataset]: # idea: add option to specify facets in list of dicts that is value of # 'derive' in the recipe and use that instead of get_required? for input_facets in required_vars: - input_dataset = dataset.copy(**input_facets) - _update_cmor_facets(input_dataset.facets, override=True) + input_dataset = dataset.copy() + keep = {'alias', 'recipe_dataset_index', *dataset.minimal_facets} + input_dataset.facets = { + k: v for k, v in input_dataset.facets.items() if k in keep + } + input_dataset.facets.update(input_facets) input_dataset.augment_facets() _fix_cmip5_fx_ensemble(input_dataset) if input_facets.get('optional') and not input_dataset.files: diff --git a/esmvalcore/cmor/_fixes/cesm/cesm2.py b/esmvalcore/cmor/_fixes/cesm/cesm2.py index a76e5a1a48..5b92121555 100644 --- a/esmvalcore/cmor/_fixes/cesm/cesm2.py +++ b/esmvalcore/cmor/_fixes/cesm/cesm2.py @@ -78,6 +78,6 @@ def _fix_time(self, cube): # Fix time coordinate time_coord = cube.coord('time') - if time_coord.bounds is not None: - time_coord.points = time_coord.bounds.mean(axis=-1) + if time_coord.has_bounds(): + time_coord.points = time_coord.core_bounds().mean(axis=-1) self.fix_regular_time(cube, coord=time_coord) diff --git a/esmvalcore/cmor/_fixes/cmip5/hadgem2_cc.py b/esmvalcore/cmor/_fixes/cmip5/hadgem2_cc.py index 33e4347a65..3531f78f06 100644 --- a/esmvalcore/cmor/_fixes/cmip5/hadgem2_cc.py +++ b/esmvalcore/cmor/_fixes/cmip5/hadgem2_cc.py @@ -1,34 +1,9 @@ - """Fix HadGEM2_CC.""" -import numpy as np - from ..fix import Fix +from .hadgem2_es import AllVars as BaseAllVars -class AllVars(Fix): - """Fix common errors for all vars.""" - - def fix_metadata(self, cubes): - """ - Fix latitude. - - Parameters - ---------- - cube: iris.cube.CubeList - - Returns - ------- - iris.cube.CubeList - - """ - for cube in cubes: - lats = cube.coords('latitude') - if lats: - lat = cube.coord('latitude') - lat.points = np.clip(lat.points, -90., 90.) - lat.bounds = np.clip(lat.bounds, -90., 90.) - - return cubes +AllVars = BaseAllVars class O2(Fix): diff --git a/esmvalcore/cmor/_fixes/cmip5/hadgem2_es.py b/esmvalcore/cmor/_fixes/cmip5/hadgem2_es.py index b2cf33c8d2..f7360dae2b 100644 --- a/esmvalcore/cmor/_fixes/cmip5/hadgem2_es.py +++ b/esmvalcore/cmor/_fixes/cmip5/hadgem2_es.py @@ -25,10 +25,10 @@ def fix_metadata(self, cubes): lats = cube.coords('latitude') if lats: lat = cube.coord('latitude') - lat.points = np.clip(lat.points, -90., 90.) + lat.points = np.clip(lat.core_points(), -90., 90.) if not lat.has_bounds(): lat.guess_bounds() - lat.bounds = np.clip(lat.bounds, -90., 90.) + lat.bounds = np.clip(lat.core_bounds(), -90., 90.) return cubes diff --git a/esmvalcore/cmor/_fixes/cmip5/miroc_esm.py b/esmvalcore/cmor/_fixes/cmip5/miroc_esm.py index 0721e62409..fc94bc45cc 100644 --- a/esmvalcore/cmor/_fixes/cmip5/miroc_esm.py +++ b/esmvalcore/cmor/_fixes/cmip5/miroc_esm.py @@ -100,7 +100,7 @@ def fix_metadata(self, cubes): calendar='gregorian') if cube.coord('time').units != expected_time_units: continue - if cube.coord('time').bounds is None: + if not cube.coord('time').has_bounds(): continue # Only apply fix if there is a year < 1 in the first element diff --git a/esmvalcore/cmor/_fixes/cmip6/access_esm1_5.py b/esmvalcore/cmor/_fixes/cmip6/access_esm1_5.py index da884b5e3a..7b0497e493 100644 --- a/esmvalcore/cmor/_fixes/cmip6/access_esm1_5.py +++ b/esmvalcore/cmor/_fixes/cmip6/access_esm1_5.py @@ -102,9 +102,9 @@ def fix_metadata(self, cubes): """ cube = self.get_cube_from_list(cubes) cube.coord('air_pressure').points = \ - np.round(cube.coord('air_pressure').points, 0) + np.round(cube.coord('air_pressure').core_points(), 0) cube.coord('air_pressure').bounds = \ - np.round(cube.coord('air_pressure').bounds, 0) + np.round(cube.coord('air_pressure').core_bounds(), 0) return cubes diff --git a/esmvalcore/cmor/_fixes/cmip6/cesm2.py b/esmvalcore/cmor/_fixes/cmip6/cesm2.py index 134f68b29c..6ded187fbb 100644 --- a/esmvalcore/cmor/_fixes/cmip6/cesm2.py +++ b/esmvalcore/cmor/_fixes/cmip6/cesm2.py @@ -142,16 +142,12 @@ def fix_metadata(self, cubes): """ for cube in cubes: - latitude = cube.coord('latitude') - if latitude.bounds is None: - latitude.guess_bounds() - latitude.bounds = latitude.bounds.astype(np.float64) - latitude.bounds = np.round(latitude.bounds, 4) - longitude = cube.coord('longitude') - if longitude.bounds is None: - longitude.guess_bounds() - longitude.bounds = longitude.bounds.astype(np.float64) - longitude.bounds = np.round(longitude.bounds, 4) + for coord_name in ['latitude', 'longitude']: + coord = cube.coord(coord_name) + if not coord.has_bounds(): + coord.guess_bounds() + coord.bounds = np.round(coord.core_bounds().astype(np.float64), + 4) return cubes diff --git a/esmvalcore/cmor/_fixes/cmip6/ciesm.py b/esmvalcore/cmor/_fixes/cmip6/ciesm.py index c169db6ed4..c344c1b611 100644 --- a/esmvalcore/cmor/_fixes/cmip6/ciesm.py +++ b/esmvalcore/cmor/_fixes/cmip6/ciesm.py @@ -20,9 +20,32 @@ def fix_data(self, cube): ------- iris.cube.Cube """ - metadata = cube.metadata - cube *= 100 - cube.metadata = metadata + if cube.core_data().max() <= 1.0: + cube.units = '1' + cube.convert_units('%') + return cube + + +class Clt(Fix): + """Fixes for clt.""" + + def fix_data(self, cube): + """Fix data. + + Fixes discrepancy between declared units and real units. + + Parameters + ---------- + cube: iris.cube.Cube + Input cube. + + Returns + ------- + iris.cube.Cube + """ + if cube.core_data().max() <= 1.0: + cube.units = '1' + cube.convert_units('%') return cube diff --git a/esmvalcore/cmor/_fixes/cmip6/ec_earth3.py b/esmvalcore/cmor/_fixes/cmip6/ec_earth3.py index 27ce37961d..49bb6a66a2 100644 --- a/esmvalcore/cmor/_fixes/cmip6/ec_earth3.py +++ b/esmvalcore/cmor/_fixes/cmip6/ec_earth3.py @@ -68,7 +68,7 @@ def fix_metadata(self, cubes): for cube in cubes: latitude = cube.coord('latitude') - latitude.points = np.round(latitude.points, 8) - latitude.bounds = np.round(latitude.bounds, 8) + latitude.points = np.round(latitude.core_points(), 8) + latitude.bounds = np.round(latitude.core_bounds(), 8) return cubes diff --git a/esmvalcore/cmor/_fixes/cmip6/ec_earth3_veg.py b/esmvalcore/cmor/_fixes/cmip6/ec_earth3_veg.py index edcf5624d2..73bf689725 100644 --- a/esmvalcore/cmor/_fixes/cmip6/ec_earth3_veg.py +++ b/esmvalcore/cmor/_fixes/cmip6/ec_earth3_veg.py @@ -72,7 +72,7 @@ def fix_metadata(self, cubes): for cube in cubes: latitude = cube.coord('latitude') - latitude.points = np.round(latitude.points, 8) - latitude.bounds = np.round(latitude.bounds, 8) + latitude.points = np.round(latitude.core_points(), 8) + latitude.bounds = np.round(latitude.core_bounds(), 8) return cubes diff --git a/esmvalcore/cmor/_fixes/cmip6/fio_esm_2_0.py b/esmvalcore/cmor/_fixes/cmip6/fio_esm_2_0.py index 748e181e07..d6462ad922 100644 --- a/esmvalcore/cmor/_fixes/cmip6/fio_esm_2_0.py +++ b/esmvalcore/cmor/_fixes/cmip6/fio_esm_2_0.py @@ -70,3 +70,26 @@ def fix_metadata(self, cubes): longitude.bounds = None longitude.guess_bounds() return cubes + + +class Clt(Fix): + """Fixes for clt.""" + + def fix_data(self, cube): + """Fix data. + + Fixes discrepancy between declared units and real units. + + Parameters + ---------- + cube: iris.cube.Cube + Input cube. + + Returns + ------- + iris.cube.Cube + """ + if cube.core_data().max() <= 1.0: + cube.units = '1' + cube.convert_units('%') + return cube diff --git a/esmvalcore/cmor/_fixes/cmip6/gfdl_cm4.py b/esmvalcore/cmor/_fixes/cmip6/gfdl_cm4.py index fdc349d08a..274a495bc7 100644 --- a/esmvalcore/cmor/_fixes/cmip6/gfdl_cm4.py +++ b/esmvalcore/cmor/_fixes/cmip6/gfdl_cm4.py @@ -62,36 +62,30 @@ def fix_metadata(self, cubes): """ cube = self.get_cube_from_list(cubes) try: - cube.coord('height') + cube.coord('height').attributes.pop('description') except iris.exceptions.CoordinateNotFoundError: add_scalar_height_coord(cube, 2.0) return cubes -class Uas(Fix): - """Fixes for uas.""" +class Tasmin(Tas): + """Fixes for tasmin.""" - def fix_metadata(self, cubes): - """ - Add height (10m) coordinate. - Parameters - ---------- - cubes : iris.cube.CubeList - Input cubes. +class Tasmax(Tas): + """Fixes for tasmax.""" - Returns - ------- - iris.cube.CubeList - """ - cube = self.get_cube_from_list(cubes) - add_scalar_height_coord(cube, 10.0) - return cubes +class SfcWind(Tas): + """Fixes for sfcWind.""" -class Vas(Fix): - """Fixes for vas.""" +class Hurs(Tas): + """Fixes for hurs.""" + + +class Uas(Fix): + """Fixes for uas.""" def fix_metadata(self, cubes): """ @@ -112,6 +106,10 @@ def fix_metadata(self, cubes): return cubes +class Vas(Uas): + """Fixes for vas.""" + + Omon = BaseOmon diff --git a/esmvalcore/cmor/_fixes/cmip6/giss_e2_1_g.py b/esmvalcore/cmor/_fixes/cmip6/giss_e2_1_g.py index 5d7033fbdf..f4fb27d35c 100644 --- a/esmvalcore/cmor/_fixes/cmip6/giss_e2_1_g.py +++ b/esmvalcore/cmor/_fixes/cmip6/giss_e2_1_g.py @@ -1,6 +1,4 @@ """Fixes for GISS-E2-1-G model.""" -import dask.array as da - from ..common import ClFixHybridPressureCoord from ..fix import Fix @@ -22,7 +20,7 @@ def fix_metadata(self, cubes): """ for cube in cubes: if (cube.units == 'degC' - and da.any(cube.core_data().ravel()[:1000] > 100.)): + and cube.core_data().ravel()[:1000].max() > 100.): cube.units = 'K' cube.convert_units(self.vardef.units) return cubes diff --git a/esmvalcore/cmor/_fixes/cmip6/kiost_esm.py b/esmvalcore/cmor/_fixes/cmip6/kiost_esm.py index 77ab4805e5..e06922a0af 100644 --- a/esmvalcore/cmor/_fixes/cmip6/kiost_esm.py +++ b/esmvalcore/cmor/_fixes/cmip6/kiost_esm.py @@ -35,6 +35,14 @@ class Huss(Tas): """Fixes for huss.""" +class Tasmin(Tas): + """Fixes for tasmin.""" + + +class Tasmax(Tas): + """Fixes for tasmax.""" + + class SfcWind(Fix): """Fixes for sfcWind.""" diff --git a/esmvalcore/cmor/_fixes/cmip6/miroc6.py b/esmvalcore/cmor/_fixes/cmip6/miroc6.py index b440d81c64..cf1d40ca86 100644 --- a/esmvalcore/cmor/_fixes/cmip6/miroc6.py +++ b/esmvalcore/cmor/_fixes/cmip6/miroc6.py @@ -28,18 +28,12 @@ def fix_metadata(self, cubes): iris.cube.CubeList """ for cube in cubes: - latitude = cube.coord('latitude') - if latitude.bounds is None: - latitude.guess_bounds() - latitude.points = latitude.points.astype(np.float32).astype( - np.float64) - latitude.bounds = latitude.bounds.astype(np.float32).astype( - np.float64) - longitude = cube.coord('longitude') - if longitude.bounds is None: - longitude.guess_bounds() - longitude.points = longitude.points.astype(np.float32).astype( - np.float64) - longitude.bounds = longitude.bounds.astype(np.float32).astype( - np.float64) + for coord_name in ['latitude', 'longitude']: + coord = cube.coord(coord_name) + coord.points = coord.core_points().astype(np.float32).astype( + np.float64) + if not coord.has_bounds(): + coord.guess_bounds() + coord.bounds = coord.core_bounds().astype(np.float32).astype( + np.float64) return cubes diff --git a/esmvalcore/cmor/_fixes/cmip6/mpi_esm1_2_hr.py b/esmvalcore/cmor/_fixes/cmip6/mpi_esm1_2_hr.py index b1bdc08727..c8a9ec58bb 100644 --- a/esmvalcore/cmor/_fixes/cmip6/mpi_esm1_2_hr.py +++ b/esmvalcore/cmor/_fixes/cmip6/mpi_esm1_2_hr.py @@ -47,6 +47,10 @@ def fix_metadata(self, cubes): return cubes +class Tasmax(Tas): + """Fixes for tasmax.""" + + class Ta(Fix): """Fixes for ta.""" @@ -100,3 +104,15 @@ def fix_metadata(self, cubes): add_scalar_height_coord(cube, height=10.0) return cubes + + +class SfcWindmax(SfcWind): + """Fixes for sfcWindmax.""" + + +class Uas(SfcWind): + """Fixes for uas.""" + + +class Vas(SfcWind): + """Fixes for vas.""" diff --git a/esmvalcore/cmor/_fixes/cmip6/mpi_esm1_2_xr.py b/esmvalcore/cmor/_fixes/cmip6/mpi_esm1_2_xr.py index c3ae86a6a6..201ffcca63 100644 --- a/esmvalcore/cmor/_fixes/cmip6/mpi_esm1_2_xr.py +++ b/esmvalcore/cmor/_fixes/cmip6/mpi_esm1_2_xr.py @@ -9,6 +9,14 @@ class Tas(BaseTas): """Fixes for tas.""" +class Tasmax(BaseTas): + """Fixes for tasmax.""" + + +class Tasmin(BaseTas): + """Fixes for tasmin.""" + + class Ta(BaseFix): """Fixes for ta.""" @@ -27,3 +35,15 @@ class Ua(BaseFix): class SfcWind(BaseSfcWind): """Fixes for sfcWind.""" + + +class SfcWindmax(BaseSfcWind): + """Fixes for sfcWindmax.""" + + +class Uas(BaseSfcWind): + """Fixes for uas.""" + + +class Vas(BaseSfcWind): + """Fixes for vas.""" diff --git a/esmvalcore/cmor/_fixes/common.py b/esmvalcore/cmor/_fixes/common.py index 6557e65853..d68bbac538 100644 --- a/esmvalcore/cmor/_fixes/common.py +++ b/esmvalcore/cmor/_fixes/common.py @@ -83,7 +83,7 @@ def fix_metadata(self, cubes): # This was originally done by iris, but it has to be repeated since # a has bounds now ap_coord = cube.coord(var_name='ap') - if ap_coord.bounds is None: + if not ap_coord.has_bounds(): cube.remove_coord(ap_coord) a_coord = cube.coord(var_name='a') p0_coord = cube.coord(var_name='p0') diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py index 9ccbfd4ed3..73c5e53ef0 100644 --- a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -100,7 +100,7 @@ def fix_metadata(self, cubes): if coord.dtype in ['>f8', '>f4']: coord.points = coord.core_points().astype( np.float64, casting='same_kind') - if coord.bounds is not None: + if coord.has_bounds(): coord.bounds = coord.core_bounds().astype( np.float64, casting='same_kind') return cubes @@ -161,6 +161,10 @@ def _fix_geographical_coords(self, cube, domain): units=Unit(domain[aux_coord].units), bounds=bounds, ) + if aux_coord == 'lon' and new_coord.points.min() < 0.: + lon_inds = (new_coord.points < 0.) & (old_coord.points > 0.) + old_coord.points[lon_inds] = old_coord.points[lon_inds] - 360. + self._check_grid_differences(old_coord, new_coord) aux_coord_dims = (cube.coord(var_name='rlat').cube_dims(cube) + cube.coord(var_name='rlon').cube_dims(cube)) diff --git a/esmvalcore/cmor/_fixes/emac/emac.py b/esmvalcore/cmor/_fixes/emac/emac.py index 27a7d3c74d..38ae862916 100644 --- a/esmvalcore/cmor/_fixes/emac/emac.py +++ b/esmvalcore/cmor/_fixes/emac/emac.py @@ -174,7 +174,7 @@ def _fix_alevel(cube, cubes): for coord in (ap_coord, b_coord, ps_coord): coord.points = coord.core_points().astype( float, casting='same_kind') - if coord.bounds is not None: + if coord.has_bounds(): coord.bounds = coord.core_bounds().astype( float, casting='same_kind') diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py index a6156a231c..def33fccc5 100644 --- a/esmvalcore/cmor/_fixes/fix.py +++ b/esmvalcore/cmor/_fixes/fix.py @@ -9,6 +9,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Optional +import dask import numpy as np from cf_units import Unit from iris.coords import Coord, CoordExtent @@ -22,10 +23,10 @@ _get_new_generic_level_coord, _get_simplified_calendar, _get_single_cube, - _is_unstructured_grid, ) from esmvalcore.cmor.fixes import get_time_bounds from esmvalcore.cmor.table import get_var_info +from esmvalcore.iris_helpers import has_unstructured_grid if TYPE_CHECKING: from esmvalcore.cmor.table import CoordinateInfo, VariableInfo @@ -675,18 +676,17 @@ def _fix_longitude_0_360( if not cube_coord.standard_name == 'longitude': return (cube, cube_coord) - # Only apply fixes when values are outside of valid range [0, 360] - inside_0_360 = all([ - cube_coord.core_points().min() >= 0.0, - cube_coord.core_points().max() <= 360.0, - ]) - if inside_0_360: + points = cube_coord.core_points() + min_, max_ = dask.compute(points.min(), points.max()) + + # Do not apply fixes when values are inside of valid range [0, 360] + if min_ >= 0.0 and max_ <= 360.0: return (cube, cube_coord) # Cannot fix longitudes outside [-360, 720] - if np.any(cube_coord.core_points() < -360.0): + if min_ < -360.0: return (cube, cube_coord) - if np.any(cube_coord.core_points() > 720.0): + if max_ > 720.0: return (cube, cube_coord) # cube.intersection only works for cells with 0 or 2 bounds @@ -727,7 +727,7 @@ def _fix_coord_bounds( return # Skip guessing bounds for unstructured grids - if _is_unstructured_grid(cube) and cube_coord.standard_name in ( + if has_unstructured_grid(cube) and cube_coord.standard_name in ( 'latitude', 'longitude'): self._debug_msg( cube, @@ -763,7 +763,7 @@ def _fix_coord_direction( return (cube, cube_coord) if cube_coord.dtype.kind == 'U': return (cube, cube_coord) - if _is_unstructured_grid(cube) and cube_coord.standard_name in ( + if has_unstructured_grid(cube) and cube_coord.standard_name in ( 'latitude', 'longitude' ): return (cube, cube_coord) diff --git a/esmvalcore/cmor/_fixes/icon/_base_fixes.py b/esmvalcore/cmor/_fixes/icon/_base_fixes.py index 4fc67f490e..858cb41124 100644 --- a/esmvalcore/cmor/_fixes/icon/_base_fixes.py +++ b/esmvalcore/cmor/_fixes/icon/_base_fixes.py @@ -450,6 +450,6 @@ def _load_cubes(path): @staticmethod def _set_range_in_0_360(lon_coord): """Convert longitude coordinate to [0, 360].""" - lon_coord.points = (lon_coord.points + 360.0) % 360.0 - if lon_coord.bounds is not None: - lon_coord.bounds = (lon_coord.bounds + 360.0) % 360.0 + lon_coord.points = (lon_coord.core_points() + 360.0) % 360.0 + if lon_coord.has_bounds(): + lon_coord.bounds = (lon_coord.core_bounds() + 360.0) % 360.0 diff --git a/esmvalcore/cmor/_fixes/icon/icon.py b/esmvalcore/cmor/_fixes/icon/icon.py index 214cb6e99f..2e1dfd3bb3 100644 --- a/esmvalcore/cmor/_fixes/icon/icon.py +++ b/esmvalcore/cmor/_fixes/icon/icon.py @@ -492,9 +492,8 @@ def _fix_invalid_time_units(time_coord): # it to correct units. Note: we also round to next second, otherwise # this results in times that are off by 1s (e.g., 13:59:59 instead of # 14:00:00). - new_datetimes = (year_month_day + day_float).round( - 'S' - ).dt.to_pydatetime() + rounded_datetimes = (year_month_day + day_float).round('s') + new_datetimes = np.array(rounded_datetimes.dt.to_pydatetime()) new_dt_points = date2num(np.array(new_datetimes), new_t_units) # Modify time coordinate in place diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py index d76fd21701..6c67494aaa 100644 --- a/esmvalcore/cmor/_fixes/native6/era5.py +++ b/esmvalcore/cmor/_fixes/native6/era5.py @@ -7,9 +7,9 @@ from esmvalcore.iris_helpers import date2num +from ...table import CMOR_TABLES from ..fix import Fix from ..shared import add_scalar_height_coord -from ...table import CMOR_TABLES logger = logging.getLogger(__name__) @@ -382,7 +382,7 @@ def _fix_coordinates(self, cube): coord.var_name = coord_def.out_name coord.long_name = coord_def.long_name coord.points = coord.core_points().astype('float64') - if (coord.bounds is None and len(coord.points) > 1 + if (not coord.has_bounds() and len(coord.core_points()) > 1 and coord_def.must_have_bounds == "yes"): coord.guess_bounds() diff --git a/esmvalcore/cmor/_fixes/native6/mswep.py b/esmvalcore/cmor/_fixes/native6/mswep.py index 9441b3a12d..7ed50fcdcb 100644 --- a/esmvalcore/cmor/_fixes/native6/mswep.py +++ b/esmvalcore/cmor/_fixes/native6/mswep.py @@ -113,7 +113,7 @@ def _fix_bounds(self, cube): coord = cube.coord(axis=coord_def.axis) - if coord.bounds is None: + if not coord.has_bounds(): coord.guess_bounds() def _fix_names(self, cube): diff --git a/esmvalcore/cmor/_fixes/shared.py b/esmvalcore/cmor/_fixes/shared.py index e9ae61499f..b50deb422b 100644 --- a/esmvalcore/cmor/_fixes/shared.py +++ b/esmvalcore/cmor/_fixes/shared.py @@ -359,7 +359,7 @@ def fix_bounds(cube, cubes, coord_var_names): """ for coord_var_name in coord_var_names: coord = cube.coord(var_name=coord_var_name) - if coord.bounds is not None: + if coord.has_bounds(): continue bounds_cube = get_bounds_cube(cubes, coord_var_name) cube.coord(var_name=coord_var_name).bounds = bounds_cube.core_data() @@ -398,10 +398,9 @@ def round_coordinates(cubes, decimals=5, coord_names=None): else: coords = [cube.coord(c) for c in coord_names if cube.coords(c)] for coord in coords: - coord.points = da.round(da.asarray(coord.core_points()), decimals) - if coord.bounds is not None: - coord.bounds = da.round(da.asarray(coord.core_bounds()), - decimals) + coord.points = np.round(coord.core_points(), decimals) + if coord.has_bounds(): + coord.bounds = np.round(coord.core_bounds(), decimals) return cubes diff --git a/esmvalcore/cmor/_utils.py b/esmvalcore/cmor/_utils.py index 4b083f19de..be837d9c10 100644 --- a/esmvalcore/cmor/_utils.py +++ b/esmvalcore/cmor/_utils.py @@ -7,7 +7,6 @@ from iris.coords import Coord from iris.cube import Cube -from iris.exceptions import CoordinateNotFoundError from esmvalcore.cmor.table import CMOR_TABLES, CoordinateInfo, VariableInfo @@ -173,19 +172,6 @@ def _get_simplified_calendar(calendar: str) -> str: return calendar_aliases.get(calendar, calendar) -def _is_unstructured_grid(cube: Cube) -> bool: - """Check if cube uses unstructured grid.""" - try: - lat = cube.coord('latitude') - lon = cube.coord('longitude') - except CoordinateNotFoundError: - pass - else: - if lat.ndim == 1 and (cube.coord_dims(lat) == cube.coord_dims(lon)): - return True - return False - - def _get_single_cube( cube_list: Sequence[Cube], short_name: str, diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index 43214168b8..09f7a6331d 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -3,17 +3,20 @@ import logging import warnings +from collections import namedtuple from collections.abc import Callable from enum import IntEnum from functools import cached_property from typing import Optional import cf_units +import dask import iris.coord_categorisation import iris.coords import iris.exceptions import iris.util import numpy as np +from iris.coords import Coord from iris.cube import Cube from esmvalcore.cmor._fixes.fix import GenericFix @@ -22,10 +25,10 @@ _get_generic_lev_coord_names, _get_new_generic_level_coord, _get_simplified_calendar, - _is_unstructured_grid, ) -from esmvalcore.cmor.table import get_var_info +from esmvalcore.cmor.table import CoordinateInfo, get_var_info from esmvalcore.exceptions import ESMValCoreDeprecationWarning +from esmvalcore.iris_helpers import has_unstructured_grid class CheckLevels(IntEnum): @@ -137,7 +140,7 @@ def __init__(self, @cached_property def _unstructured_grid(self) -> bool: """Cube uses unstructured grid.""" - return _is_unstructured_grid(self._cube) + return has_unstructured_grid(self._cube) def check_metadata(self, logger: Optional[logging.Logger] = None) -> Cube: """Check the cube metadata. @@ -500,6 +503,7 @@ def _check_alternative_dim_names(self, key): def _check_coords(self): """Check coordinates.""" + coords = [] for coordinate in self._cmor_var.coordinates.values(): # Cannot check generic_level coords with no CMOR information if coordinate.generic_level and not coordinate.out_name: @@ -513,6 +517,36 @@ def _check_coords(self): continue self._check_coord(coordinate, coord, var_name) + coords.append((coordinate, coord)) + + self._check_coord_ranges(coords) + + def _check_coord_ranges(self, coords: list[tuple[CoordinateInfo, Coord]]): + """Check coordinate value are inside valid ranges.""" + Limit = namedtuple('Limit', ['name', 'type', 'limit', 'value']) + + limits = [] + for coord_info, coord in coords: + points = coord.core_points() + for limit_type in 'min', 'max': + valid = getattr(coord_info, f'valid_{limit_type}') + if valid != "": + limit = Limit( + name=coord_info.out_name, + type=limit_type, + limit=float(valid), + value=getattr(points, limit_type)(), + ) + limits.append(limit) + + limits = dask.compute(*limits) + for limit in limits: + if limit.type == 'min' and limit.value < limit.limit: + self.report_critical(self._vals_msg, limit.name, + '< valid_min =', limit.limit) + if limit.type == 'max' and limit.value > limit.limit: + self.report_critical(self._vals_msg, limit.name, + '> valid_max =', limit.limit) def _check_coords_data(self): """Check coordinate data.""" @@ -593,24 +627,7 @@ def _check_coord_monotonicity_and_direction(self, cmor, coord, var_name): def _check_coord_points(self, coord_info, coord, var_name): """Check coordinate points: values, bounds and monotonicity.""" - # Check requested coordinate values exist in coord.points self._check_requested_values(coord, coord_info, var_name) - - # Check coordinate value ranges - if coord_info.valid_min: - valid_min = float(coord_info.valid_min) - if np.any(coord.core_points() < valid_min): - self.report_critical(self._vals_msg, var_name, - '< {} ='.format('valid_min'), - valid_min) - - if coord_info.valid_max: - valid_max = float(coord_info.valid_max) - if np.any(coord.core_points() > valid_max): - self.report_critical(self._vals_msg, var_name, - '> {} ='.format('valid_max'), - valid_max) - self._check_coord_bounds(coord_info, coord, var_name) self._check_coord_monotonicity_and_direction(coord_info, coord, var_name) diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index ebce5431ae..a0c685654a 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -36,7 +36,7 @@ ) -def _update_cmor_facets(facets, override=False): +def _update_cmor_facets(facets): """Update `facets` with information from CMOR table.""" project = facets['project'] mip = facets['mip'] @@ -53,7 +53,7 @@ def _update_cmor_facets(facets, override=False): f"'{short_name}' with mip '{mip}'") facets['original_short_name'] = table_entry.short_name for key in _CMOR_KEYS: - if key not in facets or override: + if key not in facets: value = getattr(table_entry, key, None) if value is not None: facets[key] = value @@ -262,7 +262,7 @@ def get_variable( self, table_name: str, short_name: str, - derived: Optional[bool] = False, + derived: bool = False, ) -> VariableInfo | None: """Search and return the variable information. @@ -296,11 +296,11 @@ def get_variable( pass # If that didn't work, look in all tables (i.e., other MIPs) if - # cmor_strict=False - var_info = self._look_in_all_tables(alt_names_list) + # cmor_strict=False or derived=True + var_info = self._look_in_all_tables(derived, alt_names_list) - # If that didn' work either, look in default table if cmor_strict=False - # or derived=True + # If that didn't work either, look in default table if + # cmor_strict=False or derived=True if not var_info: var_info = self._look_in_default(derived, alt_names_list, table_name) @@ -324,10 +324,10 @@ def _look_in_default(self, derived, alt_names_list, table_name): break return var_info - def _look_in_all_tables(self, alt_names_list): + def _look_in_all_tables(self, derived, alt_names_list): """Look for variable in all tables.""" var_info = None - if not self.strict: + if (not self.strict or derived): for alt_names in alt_names_list: var_info = self._look_all_tables(alt_names) if var_info: @@ -988,51 +988,52 @@ class CustomInfo(CMIP5Info): Parameters ---------- - cmor_tables_path: str or None + cmor_tables_path: Full path to the table or name for the table if it is present in - ESMValTool repository - """ - - def __init__(self, cmor_tables_path=None): - cwd = os.path.dirname(os.path.realpath(__file__)) - default_cmor_folder = os.path.join(cwd, 'tables', 'custom') + ESMValTool repository. If ``None``, use default tables from + `esmvalcore/cmor/tables/custom`. - # Get custom location of CMOR tables if possible - if cmor_tables_path is None: - self._cmor_folder = default_cmor_folder - else: - self._cmor_folder = self._get_cmor_path(cmor_tables_path) - if not os.path.isdir(self._cmor_folder): - raise ValueError(f"Custom CMOR tables path {self._cmor_folder} is " - f"not a directory") + """ + def __init__(self, cmor_tables_path: Optional[str | Path] = None) -> None: + """Initialize class member.""" + self.coords = {} self.tables = {} - self.var_to_freq = {} + self.var_to_freq: dict[str, dict] = {} table = TableInfo() table.name = 'custom' self.tables[table.name] = table - # Try to read coordinates from custom location, use default location if - # not possible - coordinates_file = os.path.join( - self._cmor_folder, - 'CMOR_coordinates.dat', - ) - if os.path.isfile(coordinates_file): - self._coordinates_file = coordinates_file + # First, read default custom tables from repository + self._cmor_folder = self._get_cmor_path('custom') + self._read_table_dir(self._cmor_folder) + + # Second, if given, update default tables with user-defined custom + # tables + if cmor_tables_path is not None: + self._user_table_folder = self._get_cmor_path(cmor_tables_path) + if not os.path.isdir(self._user_table_folder): + raise ValueError( + f"Custom CMOR tables path {self._user_table_folder} is " + f"not a directory" + ) + self._read_table_dir(self._user_table_folder) else: - self._coordinates_file = os.path.join( - default_cmor_folder, - 'CMOR_coordinates.dat', - ) + self._user_table_folder = None - self.coords = {} - self._read_table_file(self._coordinates_file, self.tables['custom']) - for dat_file in glob.glob(os.path.join(self._cmor_folder, '*.dat')): - if dat_file == self._coordinates_file: + def _read_table_dir(self, table_dir: str) -> None: + """Read CMOR tables from directory.""" + # If present, read coordinates + coordinates_file = os.path.join(table_dir, 'CMOR_coordinates.dat') + if os.path.isfile(coordinates_file): + self._read_table_file(coordinates_file) + + # Read other variables + for dat_file in glob.glob(os.path.join(table_dir, '*.dat')): + if dat_file == coordinates_file: continue try: - self._read_table_file(dat_file, self.tables['custom']) + self._read_table_file(dat_file) except Exception: msg = f"Exception raised when loading {dat_file}" # Logger may not be ready at this stage @@ -1042,29 +1043,40 @@ def __init__(self, cmor_tables_path=None): print(msg) raise - def get_variable(self, table, short_name, derived=False): + def get_variable( + self, + table: str, + short_name: str, + derived: bool = False + ) -> VariableInfo | None: """Search and return the variable info. Parameters ---------- - table: str - Table name - short_name: str - Variable's short name - derived: bool, optional + table: + Table name. Ignored for custom tables. + short_name: + Variable's short name. + derived: Variable is derived. Info retrieval for derived variables always - look on the default tables if variable is not find in the - requested table + looks on the default tables if variable is not found in the + requested table. Ignored for custom tables. Returns ------- - VariableInfo - Return the VariableInfo object for the requested variable if - found, returns None if not + VariableInfo | None + `VariableInfo` object for the requested variable if found, returns + None if not. + """ return self.tables['custom'].get(short_name, None) - def _read_table_file(self, table_file, table=None): + def _read_table_file( + self, + table_file: str, + _: Optional[TableInfo] = None, + ) -> None: + """Read a single table file.""" with open(table_file, 'r', encoding='utf-8') as self._current_table: self._read_line() while True: @@ -1079,7 +1091,9 @@ def _read_table_file(self, table_file, table=None): self.coords[value] = self._read_coordinate(value) continue elif key == 'variable_entry': - table[value] = self._read_variable(value, '') + self.tables['custom'][value] = self._read_variable( + value, '' + ) continue if not self._read_line(): return diff --git a/esmvalcore/cmor/tables/cmip6/Tables/CMIP6_Amon.json b/esmvalcore/cmor/tables/cmip6/Tables/CMIP6_Amon.json index 8a6425dcc4..456fd921e2 100644 --- a/esmvalcore/cmor/tables/cmip6/Tables/CMIP6_Amon.json +++ b/esmvalcore/cmor/tables/cmip6/Tables/CMIP6_Amon.json @@ -1238,6 +1238,7 @@ "ok_min_mean_abs": "", "ok_max_mean_abs": "" }, + "ts": { "frequency": "mon", "modeling_realm": "atmos", @@ -1256,6 +1257,208 @@ "ok_min_mean_abs": "", "ok_max_mean_abs": "" }, + + "tsDay": { + "frequency": "mon", + "modeling_realm": "atmos", + "standard_name": "surface_temperature", + "units": "K", + "cell_methods": "area: time: mean", + "cell_measures": "area: areacella", + "long_name": "Surface Temperature Daytime Overpass", + "comment": "Temperature of the lower boundary of the atmosphere", + "dimensions": "longitude latitude time", + "out_name": "ts", + "type": "real", + "positive": "", + "valid_min": "", + "valid_max": "", + "ok_min_mean_abs": "", + "ok_max_mean_abs": "" + }, + + "tsNight": { + "frequency": "mon", + "modeling_realm": "atmos", + "standard_name": "surface_temperature", + "units": "K", + "cell_methods": "area: time: mean", + "cell_measures": "area: areacella", + "long_name": "Surface Temperature Nighttime Overpass", + "comment": "Temperature of the lower boundary of the atmosphere", + "dimensions": "longitude latitude time", + "out_name": "ts", + "type": "real", + "positive": "", + "valid_min": "", + "valid_max": "", + "ok_min_mean_abs": "", + "ok_max_mean_abs": "" + }, + "tsLocalSfcErrDay": { + "frequency": "mon", + "modeling_realm": "atmos", + "standard_name": "surface_temperature", + "units": "K", + "cell_methods": "area: time: mean", + "cell_measures": "area: areacella", + "long_name": "Surface Temperature Local Surface Error Day", + "comment": "Temperature of the lower boundary of the atmosphere", + "dimensions": "longitude latitude time", + "out_name": "ts", + "type": "real", + "positive": "", + "valid_min": "", + "valid_max": "", + "ok_min_mean_abs": "", + "ok_max_mean_abs": "" + }, + "tsLocalSfcErrNight": { + "frequency": "mon", + "modeling_realm": "atmos", + "standard_name": "surface_temperature", + "units": "K", + "cell_methods": "area: time: mean", + "cell_measures": "area: areacella", + "long_name": "Surface Temperature Local Surface Error Night", + "comment": "Temperature of the lower boundary of the atmosphere", + "dimensions": "longitude latitude time", + "out_name": "ts", + "type": "real", + "positive": "", + "valid_min": "", + "valid_max": "", + "ok_min_mean_abs": "", + "ok_max_mean_abs": "" + }, + "tsLocalAtmErrDay": { + "frequency": "mon", + "modeling_realm": "atmos", + "standard_name": "surface_temperature", + "units": "K", + "cell_methods": "area: time: mean", + "cell_measures": "area: areacella", + "long_name": "Surface Temperature Local Atmosphere Error Day", + "comment": "Temperature of the lower boundary of the atmosphere", + "dimensions": "longitude latitude time", + "out_name": "ts", + "type": "real", + "positive": "", + "valid_min": "", + "valid_max": "", + "ok_min_mean_abs": "", + "ok_max_mean_abs": "" + }, + "tsLocalAtmErrNight": { + "frequency": "mon", + "modeling_realm": "atmos", + "standard_name": "surface_temperature", + "units": "K", + "cell_methods": "area: time: mean", + "cell_measures": "area: areacella", + "long_name": "Surface Temperature Local Atmosphere Error Night", + "comment": "Temperature of the lower boundary of the atmosphere", + "dimensions": "longitude latitude time", + "out_name": "ts", + "type": "real", + "positive": "", + "valid_min": "", + "valid_max": "", + "ok_min_mean_abs": "", + "ok_max_mean_abs": "" + }, + "tsLSSysErrDay": { + "frequency": "mon", + "modeling_realm": "atmos", + "standard_name": "surface_temperature", + "units": "K", + "cell_methods": "area: time: mean", + "cell_measures": "area: areacella", + "long_name": "Surface Temperature Large Scale Systematic Error Day", + "comment": "Temperature of the lower boundary of the atmosphere", + "dimensions": "longitude latitude time", + "out_name": "ts", + "type": "real", + "positive": "", + "valid_min": "", + "valid_max": "", + "ok_min_mean_abs": "", + "ok_max_mean_abs": "" + }, + "tsLSSysErrNight": { + "frequency": "mon", + "modeling_realm": "atmos", + "standard_name": "surface_temperature", + "units": "K", + "cell_methods": "area: time: mean", + "cell_measures": "area: areacella", + "long_name": "Surface Temperature Large Scale Systematic Error Night", + "comment": "Temperature of the lower boundary of the atmosphere", + "dimensions": "longitude latitude time", + "out_name": "ts", + "type": "real", + "positive": "", + "valid_min": "", + "valid_max": "", + "ok_min_mean_abs": "", + "ok_max_mean_abs": "" + }, + + "tsUnCorErrDay": { + "frequency": "mon", + "modeling_realm": "atmos", + "standard_name": "surface_temperature", + "units": "K", + "cell_methods": "area: time: mean", + "cell_measures": "area: areacella", + "long_name": "Surface Temperature Uncorrelated Error Day", + "comment": "Temperature of the lower boundary of the atmosphere", + "dimensions": "longitude latitude time", + "out_name": "ts", + "type": "real", + "positive": "", + "valid_min": "", + "valid_max": "", + "ok_min_mean_abs": "", + "ok_max_mean_abs": "" + }, + "tsUnCorErrNight": { + "frequency": "mon", + "modeling_realm": "atmos", + "standard_name": "surface_temperature", + "units": "K", + "cell_methods": "area: time: mean", + "cell_measures": "area: areacella", + "long_name": "Surface Temperature Uncorrelated Error Large Night", + "comment": "Temperature of the lower boundary of the atmosphere", + "dimensions": "longitude latitude time", + "out_name": "ts", + "type": "real", + "positive": "", + "valid_min": "", + "valid_max": "", + "ok_min_mean_abs": "", + "ok_max_mean_abs": "" + }, + "tsStdeer": { + "frequency": "mon", + "modeling_realm": "atmos", + "standard_name": "surface_temperature", + "units": "K", + "cell_methods": "area: time: mean", + "cell_measures": "area: areacella", + "long_name": "Surface Temperature Large Standard Error", + "comment": "Temperature of the lower boundary of the atmosphere", + "dimensions": "longitude latitude time", + "out_name": "ts", + "type": "real", + "positive": "", + "valid_min": "", + "valid_max": "", + "ok_min_mean_abs": "", + "ok_max_mean_abs": "" + }, + "ua": { "frequency": "mon", "modeling_realm": "atmos", diff --git a/esmvalcore/cmor/tables/cmip6/Tables/CMIP6_Lmon.json b/esmvalcore/cmor/tables/cmip6/Tables/CMIP6_Lmon.json index 7b00f8e66c..f58eb22d5f 100644 --- a/esmvalcore/cmor/tables/cmip6/Tables/CMIP6_Lmon.json +++ b/esmvalcore/cmor/tables/cmip6/Tables/CMIP6_Lmon.json @@ -599,7 +599,7 @@ "cell_measures": "area: areacella", "long_name": "Moisture in Upper Portion of Soil Column", "comment": "The mass of water in all phases in the upper 10cm of the soil layer.", - "dimensions": "longitude latitude time sdepth1", + "dimensions": "longitude latitude time", "out_name": "mrsos", "type": "real", "positive": "", diff --git a/esmvalcore/cmor/tables/custom/CMOR_LSTuncertTest.dat b/esmvalcore/cmor/tables/custom/CMOR_LSTuncertTest.dat new file mode 100644 index 0000000000..85b8680b7d --- /dev/null +++ b/esmvalcore/cmor/tables/custom/CMOR_LSTuncertTest.dat @@ -0,0 +1,23 @@ +SOURCE: CMIP5 (adapted from mrso) +!============ +variable_entry: LSTuncertTest +!============ +modeling_realm: atmos +!---------------------------------- +! Variable attributes: +!---------------------------------- +standard_name: +units: K +cell_methods: time: mean +cell_measures: +long_name: uncertainty from locally correlated errors on surface scales +!---------------------------------- +! Additional variable information: +!---------------------------------- +dimensions: longitude latitude time +out_name: LSTuncertTest +type: real +valid_min: +valid_max: +!---------------------------------- +! diff --git a/esmvalcore/cmor/tables/custom/CMOR_tcw.dat b/esmvalcore/cmor/tables/custom/CMOR_tcw.dat new file mode 100644 index 0000000000..bd809b495c --- /dev/null +++ b/esmvalcore/cmor/tables/custom/CMOR_tcw.dat @@ -0,0 +1,21 @@ +SOURCE: custom +!============ +variable_entry: tcw +!============ +modeling_realm: atmos +!---------------------------------- +! Variable attributes: +!---------------------------------- +standard_name: +units: kg kg-1 +cell_methods: time: mean +cell_measures: area: areacella +long_name: Mass Fraction of Cloud Total Water (liquid + ice) +!---------------------------------- +! Additional variable information: +!---------------------------------- +dimensions: longitude latitude plevs time +out_name: tcw +type: real +!---------------------------------- +! diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsDay.dat b/esmvalcore/cmor/tables/custom/CMOR_tsDay.dat new file mode 100644 index 0000000000..685a78668d --- /dev/null +++ b/esmvalcore/cmor/tables/custom/CMOR_tsDay.dat @@ -0,0 +1,24 @@ +SOURCE: CMIP5 +!============ +variable_entry: tsDay +!============ +modeling_realm: atmos +!---------------------------------- +! Variable attributes: +!---------------------------------- +standard_name: +var_name: tsDay +units: K +cell_methods: time: mean +cell_measures: +long_name: surface temperture daytime +comment: +!---------------------------------- +! Additional variable information: +!---------------------------------- +dimensions: time latitude longitude +type: real +valid_min: +valid_max: +!---------------------------------- +! diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLSSysErrDay.dat b/esmvalcore/cmor/tables/custom/CMOR_tsLSSysErrDay.dat new file mode 100644 index 0000000000..70c94668ab --- /dev/null +++ b/esmvalcore/cmor/tables/custom/CMOR_tsLSSysErrDay.dat @@ -0,0 +1,24 @@ +SOURCE: CMIP5 +!============ +variable_entry: tsLSSysErrDay +!============ +modeling_realm: atmos +!---------------------------------- +! Variable attributes: +!---------------------------------- +standard_name: +var_name: tsLSSysErrDay +units: K +cell_methods: time: mean +cell_measures: +long_name: uncertainty from large-scale systematic errors daytime +comment: +!---------------------------------- +! Additional variable information: +!---------------------------------- +dimensions: time +type: real +valid_min: +valid_max: +!---------------------------------- +! diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLSSysErrNight.dat b/esmvalcore/cmor/tables/custom/CMOR_tsLSSysErrNight.dat new file mode 100644 index 0000000000..c827e46530 --- /dev/null +++ b/esmvalcore/cmor/tables/custom/CMOR_tsLSSysErrNight.dat @@ -0,0 +1,24 @@ +SOURCE: CMIP5 +!============ +variable_entry: tsLSSysErrNight +!============ +modeling_realm: atmos +!---------------------------------- +! Variable attributes: +!---------------------------------- +standard_name: +var_name: tsLSSysErrNight +units: K +cell_methods: time: mean +cell_measures: +long_name: uncertainty from large-scale systematic errors night time +comment: +!---------------------------------- +! Additional variable information: +!---------------------------------- +dimensions: time +type: real +valid_min: +valid_max: +!---------------------------------- +! diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLocalAtmErrDay.dat b/esmvalcore/cmor/tables/custom/CMOR_tsLocalAtmErrDay.dat new file mode 100644 index 0000000000..7ddae0e566 --- /dev/null +++ b/esmvalcore/cmor/tables/custom/CMOR_tsLocalAtmErrDay.dat @@ -0,0 +1,24 @@ +SOURCE: CMIP5 +!============ +variable_entry: tsLocalAtmErrDay +!============ +modeling_realm: atmos +!---------------------------------- +! Variable attributes: +!---------------------------------- +standard_name: +var_name: tsLocalAtmErrDay +units: K +cell_methods: time: mean +cell_measures: +long_name: uncertainty from locally correlated errors on atmospheric scales daytime +comment: +!---------------------------------- +! Additional variable information: +!---------------------------------- +dimensions: time latitude longitude +type: real +valid_min: +valid_max: +!---------------------------------- +! diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLocalAtmErrNight.dat b/esmvalcore/cmor/tables/custom/CMOR_tsLocalAtmErrNight.dat new file mode 100644 index 0000000000..4182010e3f --- /dev/null +++ b/esmvalcore/cmor/tables/custom/CMOR_tsLocalAtmErrNight.dat @@ -0,0 +1,24 @@ +SOURCE: CMIP5 +!============ +variable_entry: tsLocalAtmErrNight +!============ +modeling_realm: atmos +!---------------------------------- +! Variable attributes: +!---------------------------------- +standard_name: +var_name: tsLocalAtmErrNight +units: K +cell_methods: time: mean +cell_measures: +long_name: uncertainty from locally correlated errors on atmospheric scales night time +comment: +!---------------------------------- +! Additional variable information: +!---------------------------------- +dimensions: time latitude longitude +type: real +valid_min: +valid_max: +!---------------------------------- +! diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLocalSfcErrDay.dat b/esmvalcore/cmor/tables/custom/CMOR_tsLocalSfcErrDay.dat new file mode 100644 index 0000000000..d0ad25074e --- /dev/null +++ b/esmvalcore/cmor/tables/custom/CMOR_tsLocalSfcErrDay.dat @@ -0,0 +1,24 @@ +SOURCE: CMIP5 +!============ +variable_entry: tsLocalSfcErrDay +!============ +modeling_realm: atmos +!---------------------------------- +! Variable attributes: +!---------------------------------- +standard_name: +var_name: tsLocalSfcErrDay +units: K +cell_methods: time: mean +cell_measures: +long_name: uncertainty from locally correlated errors on surface scales daytime +comment: +!---------------------------------- +! Additional variable information: +!---------------------------------- +dimensions: time latitude longitude +type: real +valid_min: +valid_max: +!---------------------------------- +! diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLocalSfcErrNight.dat b/esmvalcore/cmor/tables/custom/CMOR_tsLocalSfcErrNight.dat new file mode 100644 index 0000000000..2ea411f1ee --- /dev/null +++ b/esmvalcore/cmor/tables/custom/CMOR_tsLocalSfcErrNight.dat @@ -0,0 +1,24 @@ +SOURCE: CMIP5 +!============ +variable_entry: tsLocalSfcErrNight +!============ +modeling_realm: atmos +!---------------------------------- +! Variable attributes: +!---------------------------------- +standard_name: +var_name: tsLocalSfcErrNight +units: K +cell_methods: time: mean +cell_measures: +long_name: uncertainty from locally correlated errors on surface scales night time +comment: +!---------------------------------- +! Additional variable information: +!---------------------------------- +dimensions: time latitude longitude +type: real +valid_min: +valid_max: +!---------------------------------- +! diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsNight.dat b/esmvalcore/cmor/tables/custom/CMOR_tsNight.dat new file mode 100644 index 0000000000..c222ae6145 --- /dev/null +++ b/esmvalcore/cmor/tables/custom/CMOR_tsNight.dat @@ -0,0 +1,24 @@ +SOURCE: CMIP5 +!============ +variable_entry: tsNight +!============ +modeling_realm: atmos +!---------------------------------- +! Variable attributes: +!---------------------------------- +standard_name: +var_name: tsNight +units: K +cell_methods: time: mean +cell_measures: +long_name: surface temperature nighttime +comment: +!---------------------------------- +! Additional variable information: +!---------------------------------- +dimensions: time latitude longitude +type: real +valid_min: +valid_max: +!---------------------------------- +! diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsUnCorErrDay.dat b/esmvalcore/cmor/tables/custom/CMOR_tsUnCorErrDay.dat new file mode 100644 index 0000000000..f3ac84ebcc --- /dev/null +++ b/esmvalcore/cmor/tables/custom/CMOR_tsUnCorErrDay.dat @@ -0,0 +1,24 @@ +SOURCE: CMIP5 +!============ +variable_entry: tsUnCorErrDay +!============ +modeling_realm: atmos +!---------------------------------- +! Variable attributes: +!---------------------------------- +standard_name: +var_name: tsUnCorErrDay +units: K +cell_methods: time: mean +cell_measures: +long_name: uncertainty from uncorrelated errors daytime +comment: +!---------------------------------- +! Additional variable information: +!---------------------------------- +dimensions: time latitude longitude +type: real +valid_min: +valid_max: +!---------------------------------- +! diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsUnCorErrNight.dat b/esmvalcore/cmor/tables/custom/CMOR_tsUnCorErrNight.dat new file mode 100644 index 0000000000..36649ec227 --- /dev/null +++ b/esmvalcore/cmor/tables/custom/CMOR_tsUnCorErrNight.dat @@ -0,0 +1,24 @@ +SOURCE: CMIP5 +!============ +variable_entry: tsUnCorErrNight +!============ +modeling_realm: atmos +!---------------------------------- +! Variable attributes: +!---------------------------------- +standard_name: +var_name: tsUnCorErrNight +units: K +cell_methods: time: mean +cell_measures: +long_name: uncertainty from uncorrelated errors night time +comment: +!---------------------------------- +! Additional variable information: +!---------------------------------- +dimensions: time latitude longitude +type: real +valid_min: +valid_max: +!---------------------------------- +! diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml index 745626339b..9d5f1bc62f 100644 --- a/esmvalcore/config-developer.yml +++ b/esmvalcore/config-developer.yml @@ -36,6 +36,7 @@ CMIP6: ESGF: '{project}/{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{version}' ETHZ: '{exp}/{mip}/{short_name}/{dataset}/{ensemble}/{grid}/' SYNDA: '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{version}' + NCI: '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{version}' input_file: '{short_name}_{mip}_{dataset}_{exp}_{ensemble}_{grid}*.nc' output_file: '{project}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}_{grid}' cmor_type: 'CMIP6' @@ -45,6 +46,7 @@ CMIP5: input_dir: default: '/' BADC: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{version}/{short_name}' + NCI: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{version}/{short_name}' BSC: '{type}/{project}/{exp}/{dataset.lower}' CP4CDS: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{short_name}/latest/' DKRZ: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{version}/{short_name}' @@ -64,6 +66,7 @@ CMIP3: DKRZ: '{exp}/{modeling_realm}/{frequency}/{short_name}/{dataset}/{ensemble}' ESGF: '{project.lower}/{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{ensemble}/{short_name}/{version}' IPSL: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{ensemble}/{short_name}/{version}/{short_name}' + NCI: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{ensemble}/{short_name}/{latestversion}' input_file: '{short_name}_*.nc' output_file: '{project}_{institute}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}' cmor_type: 'CMIP3' diff --git a/esmvalcore/config-user.yml b/esmvalcore/config-user.yml index ab9c8f3b4d..6df4d97a0c 100644 --- a/esmvalcore/config-user.yml +++ b/esmvalcore/config-user.yml @@ -247,3 +247,23 @@ drs: # OBS: default # obs4MIPs: default # ana4mips: BADC + +# Site-specific entries: NCI +# Uncomment the lines below to locate data at NCI. +#rootpath: +# CMIP6: [/g/data/oi10/replicas/CMIP6, /g/data/fs38/publications/CMIP6, /g/data/xp65/public/apps/esmvaltool/replicas/CMIP6] +# CMIP5: [/g/data/r87/DRSv3/CMIP5, /g/data/al33/replicas/CMIP5/combined, /g/data/rr3/publications/CMIP5/output1, /g/data/xp65/public/apps/esmvaltool/replicas/cmip5/output1] +# CMIP3: /g/data/r87/DRSv3/CMIP3 +# OBS: /g/data/ct11/access-nri/replicas/esmvaltool/obsdata-v2 +# OBS6: /g/data/ct11/access-nri/replicas/esmvaltool/obsdata-v2 +# obs4MIPs: /g/data/ct11/access-nri/replicas/esmvaltool/obsdata-v2 +# ana4mips: /g/data/ct11/access-nri/replicas/esmvaltool/obsdata-v2 +# native6: /g/data/xp65/public/apps/esmvaltool/native6 +# +#drs: +# CMIP6: NCI +# CMIP5: NCI +# CMIP3: NCI +# CORDEX: ESGF +# obs4MIPs: default +# ana4mips: default diff --git a/esmvalcore/config/__init__.py b/esmvalcore/config/__init__.py index 74bcc486df..7c2b8e379c 100644 --- a/esmvalcore/config/__init__.py +++ b/esmvalcore/config/__init__.py @@ -4,8 +4,10 @@ ESMValCore configuration. - By default this will be loaded from the file - ``~/.esmvaltool/config-user.yml``. + By default, this will be loaded from the file + ``~/.esmvaltool/config-user.yml``. If used within the ``esmvaltool`` + program, this will respect the ``--config_file`` argument. + """ from ._config_object import CFG, Config, Session diff --git a/esmvalcore/config/_config_object.py b/esmvalcore/config/_config_object.py index 0b9c596177..92bd409a0b 100644 --- a/esmvalcore/config/_config_object.py +++ b/esmvalcore/config/_config_object.py @@ -1,15 +1,18 @@ """Importable config object.""" +from __future__ import annotations import os +import sys from datetime import datetime from pathlib import Path from types import MappingProxyType -from typing import Optional, Union +from typing import Optional import yaml import esmvalcore from esmvalcore.cmor.check import CheckLevels +from esmvalcore.exceptions import InvalidConfigParameter from ._config_validators import ( _deprecated_options_defaults, @@ -27,7 +30,9 @@ class Config(ValidatedConfig): Do not instantiate this class directly, but use :obj:`esmvalcore.config.CFG` instead. + """ + _DEFAULT_USER_CONFIG_DIR = Path.home() / '.esmvaltool' _validate = _validators _deprecate = _deprecators @@ -38,49 +43,65 @@ class Config(ValidatedConfig): ) @classmethod - def _load_user_config(cls, - filename: Union[os.PathLike, str], - raise_exception: bool = True): + def _load_user_config( + cls, + filename: Optional[os.PathLike | str] = None, + raise_exception: bool = True, + ): """Load user configuration from the given file. The config is cleared and updated in-place. Parameters ---------- - filename: pathlike - Name of the config file, must be yaml format + filename: + Name of the user configuration file (must be YAML format). If + `None`, use the rules given in `Config._get_config_user_path` to + determine the path. raise_exception : bool - Raise an exception if `filename` can not be found (default). - Otherwise, silently pass and use the default configuration. This - setting is necessary for the case where - `.esmvaltool/config-user.yml` has not been defined (i.e. first - start). + If ``True``, raise an exception if `filename` cannot be found. If + ``False``, silently pass and use the default configuration. This + setting is necessary during the loading of this module when no + configuration file is given (relevant if used within a script or + notebook). """ new = cls() + new.update(CFG_DEFAULT) + + config_user_path = cls._get_config_user_path(filename) try: - mapping = _read_config_file(filename) - mapping['config_file'] = filename - except IOError: + mapping = cls._read_config_file(config_user_path) + mapping['config_file'] = config_user_path + except FileNotFoundError: if raise_exception: raise mapping = {} - new.update(CFG_DEFAULT) - new.update(mapping) - new.check_missing() + try: + new.update(mapping) + new.check_missing() + except InvalidConfigParameter as exc: + raise InvalidConfigParameter( + f"Failed to parse user configuration file {config_user_path}: " + f"{str(exc)}" + ) from exc return new @classmethod - def _load_default_config(cls, filename: Union[os.PathLike, str]): + def _load_default_config(cls): """Load the default configuration.""" new = cls() - mapping = _read_config_file(filename) + package_config_user_path = Path( + esmvalcore.__file__ + ).parent / 'config-user.yml' + mapping = cls._read_config_file(package_config_user_path) + # Add defaults that are not available in esmvalcore/config-user.yml mapping['check_level'] = CheckLevels.DEFAULT - mapping['config_file'] = filename + mapping['config_file'] = package_config_user_path mapping['diagnostics'] = None mapping['extra_facets_dir'] = tuple() mapping['max_datasets'] = None @@ -93,29 +114,142 @@ def _load_default_config(cls, filename: Union[os.PathLike, str]): return new + @staticmethod + def _read_config_file(config_user_path: Path) -> dict: + """Read configuration file and store settings in a dictionary.""" + if not config_user_path.is_file(): + raise FileNotFoundError( + f"Config file '{config_user_path}' does not exist" + ) + + with open(config_user_path, 'r', encoding='utf-8') as file: + cfg = yaml.safe_load(file) + + return cfg + + @staticmethod + def _get_config_user_path( + filename: Optional[os.PathLike | str] = None + ) -> Path: + """Get path to user configuration file. + + `filename` can be given as absolute or relative path. In the latter + case, search in the current working directory and `~/.esmvaltool` (in + that order). + + If `filename` is not given, try to get user configuration file from the + following locations (sorted by descending priority): + + 1. Internal `_ESMVALTOOL_USER_CONFIG_FILE_` environment variable + (this ensures that any subprocess spawned by the esmvaltool program + will use the correct user configuration file). + 2. Command line arguments `--config-file` or `--config_file` (both + variants are allowed by the fire module), but only if script name is + `esmvaltool`. + 3. `config-user.yml` within default ESMValTool configuration directory + `~/.esmvaltool`. + + Note + ---- + This will NOT check if the returned file actually exists to allow + loading the module without any configuration file (this is relevant if + the module is used within a script or notebook). To check if the file + actually exists, use the method `load_from_file` (this is done when + using the `esmvaltool` CLI). + + If used within the esmvaltool program, set the + _ESMVALTOOL_USER_CONFIG_FILE_ at the end of this method to make sure + that subsequent calls of this method (also in suprocesses) use the + correct user configuration file. + + """ + # (1) Try to get user configuration file from `filename` argument + config_user = filename + + # (2) Try to get user configuration file from internal + # _ESMVALTOOL_USER_CONFIG_FILE_ environment variable + if ( + config_user is None and + '_ESMVALTOOL_USER_CONFIG_FILE_' in os.environ + ): + config_user = os.environ['_ESMVALTOOL_USER_CONFIG_FILE_'] + + # (3) Try to get user configuration file from CLI arguments + if config_user is None: + config_user = Config._get_config_path_from_cli() + + # (4) Default location + if config_user is None: + config_user = Config._DEFAULT_USER_CONFIG_DIR / 'config-user.yml' + + config_user = Path(config_user).expanduser() + + # Also search path relative to ~/.esmvaltool if necessary + if not (config_user.is_file() or config_user.is_absolute()): + config_user = Config._DEFAULT_USER_CONFIG_DIR / config_user + config_user = config_user.absolute() + + # If used within the esmvaltool program, make sure that subsequent + # calls of this method (also in suprocesses) use the correct user + # configuration file + if Path(sys.argv[0]).name == 'esmvaltool': + os.environ['_ESMVALTOOL_USER_CONFIG_FILE_'] = str(config_user) + + return config_user + + @staticmethod + def _get_config_path_from_cli() -> None | str: + """Try to get configuration path from CLI arguments. + + The hack of directly parsing the CLI arguments here (instead of using + the fire or argparser module) ensures that the correct user + configuration file is used. This will always work, regardless of when + this module has been imported in the code. + + Note + ---- + This only works if the script name is `esmvaltool`. Does not check if + file exists. + + """ + if Path(sys.argv[0]).name != 'esmvaltool': + return None + + for arg in sys.argv: + for opt in ('--config-file', '--config_file'): + if opt in arg: + # Parse '--config-file=/file.yml' or + # '--config_file=/file.yml' + partition = arg.partition('=') + if partition[2]: + return partition[2] + + # Parse '--config-file /file.yml' or + # '--config_file /file.yml' + config_idx = sys.argv.index(opt) + if config_idx == len(sys.argv) - 1: # no file given + return None + return sys.argv[config_idx + 1] + + return None + def load_from_file( self, - filename: Optional[Union[os.PathLike, str]] = None, + filename: Optional[os.PathLike | str] = None, ) -> None: """Load user configuration from the given file.""" - if filename is None: - filename = USER_CONFIG - path = Path(filename).expanduser() - if not path.exists(): - try_path = USER_CONFIG_DIR / filename - if try_path.exists(): - path = try_path - else: - raise FileNotFoundError(f'Cannot find: `{filename}`' - f'locally or in `{try_path}`') - self.clear() - self.update(Config._load_user_config(path)) + self.update(Config._load_user_config(filename)) def reload(self): """Reload the config file.""" - filename = self.get('config_file', DEFAULT_CONFIG) - self.load_from_file(filename) + if 'config_file' not in self: + raise ValueError( + "Cannot reload configuration, option 'config_file' is " + "missing; make sure to only use the `CFG` object from the " + "`esmvalcore.config` module" + ) + self.load_from_file(self['config_file']) def start_session(self, name: str): """Start a new session from this configuration object. @@ -161,7 +295,7 @@ class Session(ValidatedConfig): def __init__(self, config: dict, name: str = 'session'): super().__init__(config) - self.session_name: Union[str, None] = None + self.session_name: str | None = None self.set_session_name(name) def set_session_name(self, name: str = 'session'): @@ -201,7 +335,7 @@ def run_dir(self): @property def config_dir(self): """Return user config directory.""" - return USER_CONFIG_DIR + return Path(self['config_file']).parent @property def main_log(self): @@ -219,24 +353,6 @@ def _fixed_file_dir(self): return self.session_dir / self._relative_fixed_file_dir -def _read_config_file(config_file): - """Read config user file and store settings in a dictionary.""" - config_file = Path(config_file) - if not config_file.exists(): - raise IOError(f'Config file `{config_file}` does not exist.') - - with open(config_file, 'r', encoding='utf-8') as file: - cfg = yaml.safe_load(file) - - return cfg - - -DEFAULT_CONFIG_DIR = Path(esmvalcore.__file__).parent -DEFAULT_CONFIG = DEFAULT_CONFIG_DIR / 'config-user.yml' - -USER_CONFIG_DIR = Path.home() / '.esmvaltool' -USER_CONFIG = USER_CONFIG_DIR / 'config-user.yml' - -# initialize placeholders -CFG_DEFAULT = MappingProxyType(Config._load_default_config(DEFAULT_CONFIG)) -CFG = Config._load_user_config(USER_CONFIG, raise_exception=False) +# Initialize configuration objects +CFG_DEFAULT = MappingProxyType(Config._load_default_config()) +CFG = Config._load_user_config(raise_exception=False) diff --git a/esmvalcore/config/_esgf_pyclient.py b/esmvalcore/config/_esgf_pyclient.py index 1d473068e6..1dfcd38ede 100644 --- a/esmvalcore/config/_esgf_pyclient.py +++ b/esmvalcore/config/_esgf_pyclient.py @@ -12,7 +12,6 @@ import logging import os import stat -import textwrap from functools import lru_cache from pathlib import Path from types import ModuleType @@ -30,59 +29,6 @@ CONFIG_FILE = Path.home() / '.esmvaltool' / 'esgf-pyclient.yml' -INSTRUCTIONS = textwrap.dedent(""" -ESGF credentials missing, only data that is accessible without -logging in will be available. - -See https://esgf.github.io/esgf-user-support/user_guide.html -for instructions on how to create an account if you do not have -one yet. - -Next, configure your system so esmvaltool can use your -credentials. This can be done using the keyring package, or -you can just enter them in {cfg_file}. - -keyring -======= -First install the keyring package (requires a supported -backend, see https://pypi.org/project/keyring/): -$ pip install keyring - -Next, set your username and password by running the commands: -$ keyring set ESGF hostname -$ keyring set ESGF username -$ keyring set ESGF password - -To check that you entered your credentials correctly, run: -$ keyring get ESGF hostname -$ keyring get ESGF username -$ keyring get ESGF password - -configuration file -================== -You can store the hostname, username, and password or your OpenID -account in a plain text in the file {cfg_file} like this: - -logon: - hostname: "your-hostname" - username: "your-username" - password: "your-password" - -or your can configure an interactive log in: - -logon: - interactive: true - -Note that storing your password in plain text in the configuration -file is less secure. On shared systems, make sure the permissions -of the file are set so only you can read it, i.e. - -$ ls -l {cfg_file} - -shows permissions -rw-------. - -""".format(cfg_file=CONFIG_FILE)) - def get_keyring_credentials(): """Load credentials from keyring.""" @@ -172,14 +118,6 @@ def load_esgf_pyclient_config(): cfg['search_connection']['cache'] = cache_file Path(cache_file).parent.mkdir(parents=True, exist_ok=True) - missing_credentials = [] - for key in ['hostname', 'username', 'password']: - if key not in cfg['logon']: - missing_credentials.append(key) - - if missing_credentials and not cfg['logon'].get('interactive'): - logger.warning(INSTRUCTIONS) - return cfg diff --git a/esmvalcore/iris_helpers.py b/esmvalcore/iris_helpers.py index e5bc3dbeea..9b5fddbfe1 100644 --- a/esmvalcore/iris_helpers.py +++ b/esmvalcore/iris_helpers.py @@ -1,13 +1,16 @@ """Auxiliary functions for :mod:`iris`.""" -from typing import Dict, List, Sequence +from __future__ import annotations + +from typing import Dict, Iterable, List, Literal, Sequence import dask.array as da import iris import iris.cube import iris.util import numpy as np +from iris.coords import Coord from iris.cube import Cube -from iris.exceptions import CoordinateMultiDimError +from iris.exceptions import CoordinateMultiDimError, CoordinateNotFoundError from esmvalcore.typing import NetCDFAttr @@ -157,3 +160,161 @@ def merge_cube_attributes( # Step 3: modify the cubes in-place for cube in cubes: cube.attributes = final_attributes + + +def _rechunk( + array: da.core.Array, + complete_dims: list[int], + remaining_dims: int | Literal['auto'], +) -> da.core.Array: + """Rechunk a given array so that it is not chunked along given dims.""" + new_chunks: list[str | int] = [remaining_dims] * array.ndim + for dim in complete_dims: + new_chunks[dim] = -1 + return array.rechunk(new_chunks) + + +def _rechunk_dim_metadata( + cube: Cube, + complete_dims: Iterable[int], + remaining_dims: int | Literal['auto'] = 'auto', +) -> None: + """Rechunk dimensional metadata of a cube (in-place).""" + # Non-dimensional coords that span complete_dims + # Note: dimensional coords are always realized (i.e., numpy arrays), so no + # chunking is necessary + for coord in cube.coords(dim_coords=False): + dims = cube.coord_dims(coord) + complete_dims_ = [dims.index(d) for d in complete_dims if d in dims] + if complete_dims_: + if coord.has_lazy_points(): + coord.points = _rechunk( + coord.lazy_points(), complete_dims_, remaining_dims + ) + if coord.has_bounds() and coord.has_lazy_bounds(): + coord.bounds = _rechunk( + coord.lazy_bounds(), complete_dims_, remaining_dims + ) + + # Rechunk cell measures that span complete_dims + for measure in cube.cell_measures(): + dims = cube.cell_measure_dims(measure) + complete_dims_ = [dims.index(d) for d in complete_dims if d in dims] + if complete_dims_ and measure.has_lazy_data(): + measure.data = _rechunk( + measure.lazy_data(), complete_dims_, remaining_dims + ) + + # Rechunk ancillary variables that span complete_dims + for anc_var in cube.ancillary_variables(): + dims = cube.ancillary_variable_dims(anc_var) + complete_dims_ = [dims.index(d) for d in complete_dims if d in dims] + if complete_dims_ and anc_var.has_lazy_data(): + anc_var.data = _rechunk( + anc_var.lazy_data(), complete_dims_, remaining_dims + ) + + +def rechunk_cube( + cube: Cube, + complete_coords: Iterable[Coord | str], + remaining_dims: int | Literal['auto'] = 'auto', +) -> Cube: + """Rechunk cube so that it is not chunked along given dimensions. + + This will rechunk the cube's data, but also all non-dimensional + coordinates, cell measures, and ancillary variables that span at least one + of the given dimensions. + + Note + ---- + This will only rechunk `dask` arrays. `numpy` arrays are not changed. + + Parameters + ---------- + cube: + Input cube. + complete_coords: + (Names of) coordinates along which the output cubes should not be + chunked. The given coordinates must span exactly 1 dimension. + remaining_dims: + Chunksize of the remaining dimensions. + + Returns + ------- + Cube + Rechunked cube. This will always be a copy of the input cube. + + """ + cube = cube.copy() # do not modify input cube + + # Make sure that complete_coords span exactly 1 dimension + complete_dims = [] + for coord in complete_coords: + coord = cube.coord(coord) + dims = cube.coord_dims(coord) + if len(dims) != 1: + raise CoordinateMultiDimError( + f"Complete coordinates must be 1D coordinates, got " + f"{len(dims):d}D coordinate '{coord.name()}'" + ) + complete_dims.append(dims[0]) + + # Rechunk data + if cube.has_lazy_data(): + cube.data = _rechunk(cube.lazy_data(), complete_dims, remaining_dims) + + # Rechunk dimensional metadata + _rechunk_dim_metadata(cube, complete_dims, remaining_dims=remaining_dims) + + return cube + + +def has_irregular_grid(cube: Cube) -> bool: + """Check if a cube has an irregular grid. + + Parameters + ---------- + cube: + Cube to be checked. + + Returns + ------- + bool + ``True`` if input cube has an irregular grid, else ``False``. + + """ + try: + lat = cube.coord('latitude') + lon = cube.coord('longitude') + except CoordinateNotFoundError: + return False + if lat.ndim == 2 and lon.ndim == 2: + return True + return False + + +def has_unstructured_grid(cube: Cube) -> bool: + """Check if a cube has an unstructured grid. + + Parameters + ---------- + cube: + Cube to be checked. + + Returns + ------- + bool + ``True`` if input cube has an unstructured grid, else ``False``. + + """ + try: + lat = cube.coord('latitude') + lon = cube.coord('longitude') + except CoordinateNotFoundError: + return False + if lat.ndim != 1 or lon.ndim != 1: + return False + if cube.coord_dims(lat) != cube.coord_dims(lon): + return False + return True diff --git a/esmvalcore/local.py b/esmvalcore/local.py index 83f4299bf4..5cb499dbbb 100644 --- a/esmvalcore/local.py +++ b/esmvalcore/local.py @@ -63,8 +63,7 @@ def _get_from_pattern(pattern, date_range_pattern, stem, group): def _get_start_end_date( - file: str | Path | LocalFile | ESGFFile -) -> tuple[str, str]: + file: str | Path | LocalFile | ESGFFile) -> tuple[str, str]: """Get the start and end dates as a string from a file name. Examples of allowed dates: 1980, 198001, 1980-01, 19801231, 1980-12-31, @@ -92,7 +91,6 @@ def _get_start_end_date( ------ ValueError Start or end date cannot be determined. - """ if hasattr(file, 'name'): # Path, LocalFile, ESGFFile stem = Path(file.name).stem @@ -114,9 +112,8 @@ def _get_start_end_date( # Dates can either be delimited by '-', '_', or '_cat_' (the latter for # CMIP3) - date_range_pattern = ( - datetime_pattern + r"[-_](?:cat_)?" + end_datetime_pattern - ) + date_range_pattern = (datetime_pattern + r"[-_](?:cat_)?" + + end_datetime_pattern) # Find dates using the regex start_date, end_date = _get_from_pattern(datetime_pattern, @@ -124,8 +121,8 @@ def _get_start_end_date( 'datetime') # As final resort, try to get the dates from the file contents - if ((start_date is None or end_date is None) and - isinstance(file, (str, Path)) and Path(file).exists()): + if ((start_date is None or end_date is None) + and isinstance(file, (str, Path)) and Path(file).exists()): logger.debug("Must load file %s for daterange ", file) cubes = iris.load(file) @@ -145,8 +142,7 @@ def _get_start_end_date( if start_date is None or end_date is None: raise ValueError( f"File {file} datetimes do not match a recognized pattern and " - f"time coordinate can not be read from the file" - ) + f"time coordinate can not be read from the file") # Remove potential '-' characters from datetimes start_date = start_date.replace('-', '') @@ -156,12 +152,10 @@ def _get_start_end_date( def _get_start_end_year( - file: str | Path | LocalFile | ESGFFile -) -> tuple[int, int]: + file: str | Path | LocalFile | ESGFFile) -> tuple[int, int]: """Get the start and end year as int from a file name. See :func:`_get_start_end_date`. - """ (start_date, end_date) = _get_start_end_date(file) return (int(start_date[:4]), int(end_date[:4])) @@ -224,8 +218,8 @@ def _parse_period(timerange): start_date = None end_date = None time_format = None - datetime_format = ( - isodate.DATE_BAS_COMPLETE + 'T' + isodate.TIME_BAS_COMPLETE) + datetime_format = (isodate.DATE_BAS_COMPLETE + 'T' + + isodate.TIME_BAS_COMPLETE) if timerange.split('/')[0].startswith('P'): try: end_date = isodate.parse_datetime(timerange.split('/')[1]) @@ -246,13 +240,13 @@ def _parse_period(timerange): end_date = start_date + delta if time_format == datetime_format: - start_date = str(isodate.datetime_isoformat( - start_date, format=datetime_format)) - end_date = str(isodate.datetime_isoformat( - end_date, format=datetime_format)) - elif time_format == isodate.DATE_BAS_COMPLETE: start_date = str( - isodate.date_isoformat(start_date, format=time_format)) + isodate.datetime_isoformat(start_date, format=datetime_format)) + end_date = str( + isodate.datetime_isoformat(end_date, format=datetime_format)) + elif time_format == isodate.DATE_BAS_COMPLETE: + start_date = str(isodate.date_isoformat(start_date, + format=time_format)) end_date = str(isodate.date_isoformat(end_date, format=time_format)) if start_date is None and end_date is None: @@ -289,11 +283,11 @@ def _truncate_dates(date, file_date): def _select_files(filenames, timerange): """Select files containing data between a given timerange. - If the timerange is given as a period, the file selection - occurs taking only the years into account. + If the timerange is given as a period, the file selection occurs + taking only the years into account. - Otherwise, the file selection occurs taking into account - the time resolution of the file. + Otherwise, the file selection occurs taking into account the time + resolution of the file. """ if '*' in timerange: # TODO: support * combined with a period @@ -414,7 +408,8 @@ def _get_rootpath(project): key, ', '.join(str(p) for p in nonexistent)) _ROOTPATH_WARNED.add((key, nonexistent)) return rootpath[key] - raise KeyError('default rootpath must be specified in config-user file') + raise KeyError(f'The "{project}" option is missing from the "rootpath" ' + 'section in the config-user.yml file.') def _get_globs(variable): @@ -495,8 +490,7 @@ def _get_multiproduct_filename(attributes: dict, preproc_dir: Path) -> Path: filename_segments = list(dict.fromkeys(filename_segments)) # Add period and extension - filename_segments.append( - f"{attributes['timerange'].replace('/', '-')}.nc") + filename_segments.append(f"{attributes['timerange'].replace('/', '-')}.nc") outfile = Path( preproc_dir, @@ -517,7 +511,8 @@ def _path2facets(path: Path, drs: str) -> dict[str, str]: start, end = -len(keys) - 1, -1 values = path.parts[start:end] facets = { - key: values[idx] for idx, key in enumerate(keys) if "{" not in key + key: values[idx] + for idx, key in enumerate(keys) if "{" not in key } if len(facets) != len(keys): @@ -532,8 +527,7 @@ def _path2facets(path: Path, drs: str) -> dict[str, str]: def _filter_versions_called_latest( - files: list['LocalFile'], -) -> list['LocalFile']: + files: list['LocalFile']) -> list['LocalFile']: """Filter out versions called 'latest' if they are duplicates. On compute clusters it is usual to have a symbolic link to the diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 4a5027bce5..2144e5edce 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -70,6 +70,7 @@ extract_season, extract_time, hourly_statistics, + local_solar_time, monthly_statistics, regrid_time, resample_hours, @@ -148,8 +149,6 @@ 'extract_volume', 'extract_trajectory', 'extract_transect', - # 'average_zone': average_zone, - # 'cross_section': cross_section, 'detrend', 'extract_named_regions', 'axis_statistics', @@ -157,8 +156,7 @@ 'area_statistics', 'volume_statistics', # Time operations - # 'annual_cycle': annual_cycle, - # 'diurnal_cycle': diurnal_cycle, + 'local_solar_time', 'amplitude', 'zonal_statistics', 'meridional_statistics', diff --git a/esmvalcore/preprocessor/_area.py b/esmvalcore/preprocessor/_area.py index ae78526a2d..7d7bbe0c91 100644 --- a/esmvalcore/preprocessor/_area.py +++ b/esmvalcore/preprocessor/_area.py @@ -8,7 +8,7 @@ import logging import warnings from pathlib import Path -from typing import TYPE_CHECKING, Iterable, Optional +from typing import TYPE_CHECKING, Iterable, Literal, Optional import fiona import iris @@ -20,8 +20,13 @@ from iris.cube import Cube, CubeList from iris.exceptions import CoordinateMultiDimError, CoordinateNotFoundError -from ._shared import get_iris_aggregator, guess_bounds, update_weights_kwargs -from ._supplementary_vars import ( +from esmvalcore.preprocessor._shared import ( + get_iris_aggregator, + get_normalized_cube, + guess_bounds, + update_weights_kwargs, +) +from esmvalcore.preprocessor._supplementary_vars import ( add_ancillary_variable, add_cell_measure, register_supplementaries, @@ -84,7 +89,6 @@ def extract_region( latitude=(start_latitude, end_latitude), ignore_bounds=True, ) - region_subset = region_subset.intersection(longitude=(0., 360.)) else: region_subset = _extract_irregular_region( cube, @@ -188,6 +192,7 @@ def _extract_irregular_region(cube, start_longitude, end_longitude, def zonal_statistics( cube: Cube, operator: str, + normalize: Optional[Literal['subtract', 'divide']] = None, **operator_kwargs ) -> Cube: """Compute zonal statistics. @@ -200,6 +205,11 @@ def zonal_statistics( The operation. Used to determine the :class:`iris.analysis.Aggregator` object used to calculate the statistics. Allowed options are given in :ref:`this table `. + normalize: + If given, do not return the statistics cube itself, but rather, the + input cube, normalized with the statistics cube. Can either be + `subtract` (statistics cube is subtracted from the input cube) or + `divide` (input cube is divided by the statistics cube). **operator_kwargs: Optional keyword arguments for the :class:`iris.analysis.Aggregator` object defined by `operator`. @@ -207,7 +217,8 @@ def zonal_statistics( Returns ------- iris.cube.Cube - Zonal statistics cube. + Zonal statistics cube or input cube normalized by statistics cube (see + `normalize`). Raises ------ @@ -221,14 +232,17 @@ def zonal_statistics( "Zonal statistics on irregular grids not yet implemented" ) (agg, agg_kwargs) = get_iris_aggregator(operator, **operator_kwargs) - cube = cube.collapsed('longitude', agg, **agg_kwargs) - cube.data = cube.core_data().astype(np.float32, casting='same_kind') - return cube + result = cube.collapsed('longitude', agg, **agg_kwargs) + if normalize is not None: + result = get_normalized_cube(cube, result, normalize) + result.data = result.core_data().astype(np.float32, casting='same_kind') + return result def meridional_statistics( cube: Cube, operator: str, + normalize: Optional[Literal['subtract', 'divide']] = None, **operator_kwargs, ) -> Cube: """Compute meridional statistics. @@ -241,6 +255,11 @@ def meridional_statistics( The operation. Used to determine the :class:`iris.analysis.Aggregator` object used to calculate the statistics. Allowed options are given in :ref:`this table `. + normalize: + If given, do not return the statistics cube itself, but rather, the + input cube, normalized with the statistics cube. Can either be + `subtract` (statistics cube is subtracted from the input cube) or + `divide` (input cube is divided by the statistics cube). **operator_kwargs: Optional keyword arguments for the :class:`iris.analysis.Aggregator` object defined by `operator`. @@ -262,9 +281,11 @@ def meridional_statistics( "Meridional statistics on irregular grids not yet implemented" ) (agg, agg_kwargs) = get_iris_aggregator(operator, **operator_kwargs) - cube = cube.collapsed('latitude', agg, **agg_kwargs) - cube.data = cube.core_data().astype(np.float32, casting='same_kind') - return cube + result = cube.collapsed('latitude', agg, **agg_kwargs) + if normalize is not None: + result = get_normalized_cube(cube, result, normalize) + result.data = result.core_data().astype(np.float32, casting='same_kind') + return result def compute_area_weights(cube): @@ -349,6 +370,7 @@ def _try_adding_calculated_cell_area(cube: Cube) -> None: def area_statistics( cube: Cube, operator: str, + normalize: Optional[Literal['subtract', 'divide']] = None, **operator_kwargs, ) -> Cube: """Apply a statistical operator in the horizontal plane. @@ -371,6 +393,11 @@ def area_statistics( The operation. Used to determine the :class:`iris.analysis.Aggregator` object used to calculate the statistics. Allowed options are given in :ref:`this table `. + normalize: + If given, do not return the statistics cube itself, but rather, the + input cube, normalized with the statistics cube. Can either be + `subtract` (statistics cube is subtracted from the input cube) or + `divide` (input cube is divided by the statistics cube). **operator_kwargs: Optional keyword arguments for the :class:`iris.analysis.Aggregator` object defined by `operator`. @@ -397,6 +424,8 @@ def area_statistics( ) result = cube.collapsed(['latitude', 'longitude'], agg, **agg_kwargs) + if normalize is not None: + result = get_normalized_cube(cube, result, normalize) # Make sure to preserve dtype new_dtype = result.dtype diff --git a/esmvalcore/preprocessor/_bias.py b/esmvalcore/preprocessor/_bias.py index 816baa8635..a8e0f4f2c0 100644 --- a/esmvalcore/preprocessor/_bias.py +++ b/esmvalcore/preprocessor/_bias.py @@ -1,36 +1,62 @@ """Preprocessor functions to calculate biases from data.""" +from __future__ import annotations + import logging +from collections.abc import Iterable +from typing import TYPE_CHECKING, Literal, Optional import dask.array as da -import iris.cube +from iris.cube import Cube, CubeList from ._io import concatenate +if TYPE_CHECKING: + from esmvalcore.preprocessor import PreprocessorFile + logger = logging.getLogger(__name__) -def bias(products, bias_type='absolute', denominator_mask_threshold=1e-3, - keep_reference_dataset=False): - """Calculate biases. +BiasType = Literal['absolute', 'relative'] + + +def bias( + products: set[PreprocessorFile] | Iterable[Cube], + ref_cube: Optional[Cube] = None, + bias_type: BiasType = 'absolute', + denominator_mask_threshold: float = 1e-3, + keep_reference_dataset: bool = False, +) -> set[PreprocessorFile] | CubeList: + """Calculate biases relative to a reference dataset. + + The reference dataset needs to be broadcastable to all input `products`. + This supports `iris' rich broadcasting abilities + `__. To ensure this, the preprocessors + :func:`esmvalcore.preprocessor.regrid` and/or + :func:`esmvalcore.preprocessor.regrid_time` might be helpful. Notes ----- - This preprocessor requires a reference dataset. For this, exactly one input - dataset needs to have the facet ``reference_for_bias: true`` defined in the - recipe. In addition, all input datasets need to have identical dimensional - coordinates. This can for example be ensured with the preprocessors - :func:`esmvalcore.preprocessor.regrid` and/or - :func:`esmvalcore.preprocessor.regrid_time`. + The reference dataset can be specified with the `ref_cube` argument. If + `ref_cube` is ``None``, exactly one input dataset in the `products` set + needs to have the facet ``reference_for_bias: true`` defined in the recipe. + Please do **not** specify the option `ref_cube` when using this + preprocessor function in a recipe. Parameters ---------- - products: set of esmvalcore.preprocessor.PreprocessorFile - Input datasets. Exactly one datasets needs the facet - ``reference_for_bias: true``. - bias_type: str, optional (default: 'absolute') + products: + Input datasets/cubes for which the bias is calculated relative to a + reference dataset/cube. + ref_cube: + Cube which is used as reference for the bias calculation. If ``None``, + `products` needs to be a :obj:`set` of + `~esmvalcore.preprocessor.PreprocessorFile` objects and exactly one + dataset in `products` needs the facet ``reference_for_bias: true``. + bias_type: Bias type that is calculated. Must be one of ``'absolute'`` (dataset - ref) or ``'relative'`` ((dataset - ref) / ref). - denominator_mask_threshold: float, optional (default: 1e-3) + denominator_mask_threshold: Threshold to mask values close to zero in the denominator (i.e., the reference dataset) during the calculation of relative biases. All values in the reference dataset with absolute value less than the given @@ -40,79 +66,123 @@ def bias(products, bias_type='absolute', denominator_mask_threshold=1e-3, :math:`< 10^{-6}` kg m :math:`^{-2}` s :math:`^{-1}`) it is absolutely essential to change the default value in order to get reasonable results. - keep_reference_dataset: bool, optional (default: False) + keep_reference_dataset: If ``True``, keep the reference dataset in the output. If ``False``, - drop the reference dataset. + drop the reference dataset. Ignored if `ref_cube` is given. Returns ------- - set of esmvalcore.preprocessor.PreprocessorFile - Output datasets. + set[PreprocessorFile] | CubeList + Output datasets/cubes. Will be a :obj:`set` of + :class:`~esmvalcore.preprocessor.PreprocessorFile` objects if + `products` is also one, a :class:`~iris.cube.CubeList` otherwise. Raises ------ ValueError Not exactly one input datasets contains the facet - ``reference_for_bias: true``; ``bias_type`` is not one of + ``reference_for_bias: true`` if ``ref_cube=None``; ``ref_cube=None`` + and the input products are given as iterable of + :class:`~iris.cube.Cube` objects; ``bias_type`` is not one of ``'absolute'`` or ``'relative'``. """ - # Get reference product - reference_product = [] - for product in products: - if product.attributes.get('reference_for_bias', False): - reference_product.append(product) - if len(reference_product) != 1: - raise ValueError( - f"Expected exactly 1 dataset with 'reference_for_bias: true', " - f"found {len(reference_product):d}") - reference_product = reference_product[0] + ref_product = None + all_cubes_given = all(isinstance(p, Cube) for p in products) - # Extract reference cube - # Note: For technical reasons, product objects contain the member - # ``cubes``, which is a list of cubes. However, this is expected to be a - # list with exactly one element due to the call of concatenate earlier in - # the preprocessing chain of ESMValTool. To make sure that this - # preprocessor can also be used outside the ESMValTool preprocessing chain, - # an additional concatenate call is added here. - ref_cube = concatenate(reference_product.cubes) + # Get reference cube if not explicitly given + if ref_cube is None: + if all_cubes_given: + raise ValueError( + "A list of Cubes is given to this preprocessor; please " + "specify a `ref_cube`" + ) + (ref_cube, ref_product) = _get_ref(products, 'reference_for_bias') + else: + ref_product = None + + # Mask reference cube appropriately for relative biases if bias_type == 'relative': ref_cube = ref_cube.copy() - ref_cube.data = da.ma.masked_inside(ref_cube.core_data(), - -denominator_mask_threshold, - denominator_mask_threshold) - - # Iterate over all input datasets and calculate bias + ref_cube.data = da.ma.masked_inside( + ref_cube.core_data(), + -denominator_mask_threshold, + denominator_mask_threshold, + ) + + # If input is an Iterable of Cube objects, calculate bias for each element + if all_cubes_given: + cubes = [_calculate_bias(c, ref_cube, bias_type) for c in products] + return CubeList(cubes) + + # Otherwise, iterate over all input products, calculate bias and adapt + # metadata and provenance information accordingly output_products = set() for product in products: - if product == reference_product: + if product == ref_product: continue cube = concatenate(product.cubes) - cube_metadata = cube.metadata # Calculate bias - if bias_type == 'absolute': - cube = cube - ref_cube - new_units = str(cube.units) - elif bias_type == 'relative': - cube = (cube - ref_cube) / ref_cube - new_units = '1' - else: - raise ValueError( - f"Expected one of ['absolute', 'relative'] for bias_type, got " - f"'{bias_type}'") + cube = _calculate_bias(cube, ref_cube, bias_type) - # Adapt cube metadata and provenance information - cube.metadata = cube_metadata - cube.units = new_units - product.attributes['units'] = new_units - product.wasderivedfrom(reference_product) + # Adapt metadata and provenance information + product.attributes['units'] = str(cube.units) + if ref_product is not None: + product.wasderivedfrom(ref_product) - product.cubes = iris.cube.CubeList([cube]) + product.cubes = CubeList([cube]) output_products.add(product) # Add reference dataset to output if desired - if keep_reference_dataset: - output_products.add(reference_product) + if keep_reference_dataset and ref_product is not None: + output_products.add(ref_product) return output_products + + +def _get_ref(products, ref_tag: str) -> tuple[Cube, PreprocessorFile]: + """Get reference cube and product.""" + ref_products = [] + for product in products: + if product.attributes.get(ref_tag, False): + ref_products.append(product) + if len(ref_products) != 1: + raise ValueError( + f"Expected exactly 1 dataset with '{ref_tag}: true', found " + f"{len(ref_products):d}" + ) + ref_product = ref_products[0] + + # Extract reference cube + # Note: For technical reasons, product objects contain the member + # ``cubes``, which is a list of cubes. However, this is expected to be a + # list with exactly one element due to the call of concatenate earlier in + # the preprocessing chain of ESMValTool. To make sure that this + # preprocessor can also be used outside the ESMValTool preprocessing chain, + # an additional concatenate call is added here. + ref_cube = concatenate(ref_product.cubes) + + return (ref_cube, ref_product) + + +def _calculate_bias(cube: Cube, ref_cube: Cube, bias_type: BiasType) -> Cube: + """Calculate bias for a single cube relative to a reference cube.""" + cube_metadata = cube.metadata + + if bias_type == 'absolute': + cube = cube - ref_cube + new_units = cube.units + elif bias_type == 'relative': + cube = (cube - ref_cube) / ref_cube + new_units = '1' + else: + raise ValueError( + f"Expected one of ['absolute', 'relative'] for bias_type, got " + f"'{bias_type}'" + ) + + cube.metadata = cube_metadata + cube.units = new_units + + return cube diff --git a/esmvalcore/preprocessor/_derive/sm.py b/esmvalcore/preprocessor/_derive/sm.py index 2e22df7c7c..e2da2857a6 100644 --- a/esmvalcore/preprocessor/_derive/sm.py +++ b/esmvalcore/preprocessor/_derive/sm.py @@ -29,7 +29,7 @@ def calculate(cubes): """ mrsos_cube = cubes.extract_cube(NameConstraint(var_name='mrsos')) - depth = mrsos_cube.coord('depth').bounds.astype(np.float32) + depth = mrsos_cube.coord('depth').core_bounds().astype(np.float32) layer_thickness = depth[..., 1] - depth[..., 0] sm_cube = mrsos_cube / layer_thickness / 998.2 diff --git a/esmvalcore/preprocessor/_io.py b/esmvalcore/preprocessor/_io.py index f1586a9f17..d53955b132 100644 --- a/esmvalcore/preprocessor/_io.py +++ b/esmvalcore/preprocessor/_io.py @@ -6,7 +6,7 @@ import os from itertools import groupby from pathlib import Path -from typing import Optional, NamedTuple +from typing import NamedTuple, Optional from warnings import catch_warnings, filterwarnings import cftime diff --git a/esmvalcore/preprocessor/_mask.py b/esmvalcore/preprocessor/_mask.py index 741d45686f..c0e8348d77 100644 --- a/esmvalcore/preprocessor/_mask.py +++ b/esmvalcore/preprocessor/_mask.py @@ -23,7 +23,7 @@ def _get_fx_mask(fx_data, fx_option, mask_type): """Build a percentage-thresholded mask from an fx file.""" - inmask = np.zeros_like(fx_data, bool) + inmask = da.zeros_like(fx_data, bool) if mask_type == 'sftlf': if fx_option == 'land': # Mask land out @@ -52,11 +52,10 @@ def _get_fx_mask(fx_data, fx_option, mask_type): def _apply_fx_mask(fx_mask, var_data): """Apply the fx data extracted mask on the actual processed data.""" # Apply mask across - if np.ma.is_masked(var_data): - fx_mask |= var_data.mask - - # Build the new masked data - var_data = np.ma.array(var_data, mask=fx_mask, fill_value=1e+20) + old_mask = da.ma.getmaskarray(var_data) + mask = old_mask | fx_mask + var_data = da.ma.masked_array(var_data, mask=mask) + # maybe fill_value=1e+20 return var_data @@ -104,6 +103,7 @@ def mask_landsea(cube, mask_out): """ # Dict to store the Natural Earth masks cwd = os.path.dirname(__file__) + # ne_10m_land is fast; ne_10m_ocean is very slow shapefiles = { 'land': os.path.join(cwd, 'ne_masks/ne_10m_land.shp'), @@ -125,7 +125,7 @@ def mask_landsea(cube, mask_out): fx_cube_data = da.broadcast_to(fx_cube.core_data(), cube.shape) landsea_mask = _get_fx_mask(fx_cube_data, mask_out, fx_cube.var_name) - cube.data = _apply_fx_mask(landsea_mask, cube.data) + cube.data = _apply_fx_mask(landsea_mask, cube.core_data()) logger.debug("Applying land-sea mask: %s", fx_cube.var_name) else: if cube.coord('longitude').points.ndim < 2: @@ -186,7 +186,7 @@ def mask_landseaice(cube, mask_out): if fx_cube: fx_cube_data = da.broadcast_to(fx_cube.core_data(), cube.shape) landice_mask = _get_fx_mask(fx_cube_data, mask_out, fx_cube.var_name) - cube.data = _apply_fx_mask(landice_mask, cube.data) + cube.data = _apply_fx_mask(landice_mask, cube.core_data()) logger.debug("Applying landsea-ice mask: sftgif") else: msg = "Landsea-ice mask could not be found. Stopping. " @@ -281,13 +281,10 @@ def _mask_with_shp(cube, shapefilename, region_indices=None): if region_indices: regions = [regions[idx] for idx in region_indices] - # Create a mask for the data - mask = np.zeros(cube.shape, dtype=bool) - # Create a set of x,y points from the cube # 1D regular grids if cube.coord('longitude').points.ndim < 2: - x_p, y_p = np.meshgrid( + x_p, y_p = da.meshgrid( cube.coord(axis='X').points, cube.coord(axis='Y').points) # 2D irregular grids; spit an error for now @@ -298,27 +295,30 @@ def _mask_with_shp(cube, shapefilename, region_indices=None): raise ValueError(msg) # Wrap around longitude coordinate to match data - x_p_180 = np.where(x_p >= 180., x_p - 360., x_p) + x_p_180 = da.where(x_p >= 180., x_p - 360., x_p) + # the NE mask has no points at x = -180 and y = +/-90 # so we will fool it and apply the mask at (-179, -89, 89) instead - x_p_180 = np.where(x_p_180 == -180., x_p_180 + 1., x_p_180) - y_p_0 = np.where(y_p == -90., y_p + 1., y_p) - y_p_90 = np.where(y_p_0 == 90., y_p_0 - 1., y_p_0) + x_p_180 = da.where(x_p_180 == -180., x_p_180 + 1., x_p_180) + + y_p_0 = da.where(y_p == -90., y_p + 1., y_p) + y_p_90 = da.where(y_p_0 == 90., y_p_0 - 1., y_p_0) + mask = None for region in regions: # Build mask with vectorization - if cube.ndim == 2: + if mask is None: mask = shp_vect.contains(region, x_p_180, y_p_90) - elif cube.ndim == 3: - mask[:] = shp_vect.contains(region, x_p_180, y_p_90) - elif cube.ndim == 4: - mask[:, :] = shp_vect.contains(region, x_p_180, y_p_90) - - # Then apply the mask - if isinstance(cube.data, np.ma.MaskedArray): - cube.data.mask |= mask else: - cube.data = np.ma.masked_array(cube.data, mask) + mask |= shp_vect.contains(region, x_p_180, y_p_90) + + mask = da.array(mask) + iris.util.broadcast_to_shape(mask, cube.shape, cube.coord_dims('latitude') + + cube.coord_dims('longitude')) + + old_mask = da.ma.getmaskarray(cube.core_data()) + mask = old_mask | mask + cube.data = da.ma.masked_array(cube.core_data(), mask=mask) return cube @@ -364,6 +364,7 @@ def count_spells(data, threshold, axis, spell_length): data_hits = np.ones_like(data, dtype=bool) else: data_hits = data > float(threshold) + # Make an array with data values "windowed" along the time axis. ############################################################### # WARNING: default step is = window size i.e. no overlapping @@ -374,10 +375,13 @@ def count_spells(data, threshold, axis, spell_length): window=spell_length, step=spell_length, axis=axis) + # Find the windows "full of True-s" (along the added 'window axis'). full_windows = np.all(hit_windows, axis=axis + 1) + # Count points fulfilling the condition (along the time axis). spell_point_counts = np.sum(full_windows, axis=axis, dtype=int) + return spell_point_counts diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 7a36f140c1..b74c6950cc 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -680,7 +680,7 @@ def _get_stat_identifier(statistic: str | dict) -> str: def multi_model_statistics( products: set[PreprocessorFile] | Iterable[Cube], span: str, - statistics: list[str], + statistics: list[str | dict], output_products=None, groupby: Optional[tuple] = None, keep_input_datasets: bool = True, @@ -699,10 +699,11 @@ def multi_model_statistics( There are two options to combine time coordinates of different lengths, see the ``span`` argument. - Uses the statistical operators in :py:mod:`iris.analysis`, including - ``mean``, ``median``, ``min``, ``max``, and ``std``. Percentiles are also - supported and can be specified like ``pXX.YY`` (for percentile ``XX.YY``; - decimal part optional). + Desired statistics need to be given as a list, e.g., ``statistics: ['mean', + 'median']``. For some statistics like percentiles, it is also possible to + pass additional keyword arguments, for example ``statistics: [{'operator': + 'mean', 'percent': 20}]``. A full list of supported statistics is available + in the section on :ref:`stat_preprocs`. This function can handle cubes with differing metadata: @@ -755,8 +756,11 @@ def multi_model_statistics( missing data. This option is ignored if input cubes do not have time dimensions. statistics: - Statistical metrics to be computed, e.g. [``mean``, ``max``]. Allowed - options are given in :ref:`this table `. + Statistical operations to be computed, e.g., ``['mean', 'median']``. + For some statistics like percentiles, it is also possible to pass + additional keyword arguments, e.g., ``[{'operator': 'mean', 'percent': + 20}]``. All supported options are are given in :ref:`this table + `. output_products: dict For internal use only. A dict with statistics names as keys and preprocessorfiles as values. If products are passed as input, the @@ -822,26 +826,28 @@ def multi_model_statistics( def ensemble_statistics( products: set[PreprocessorFile] | Iterable[Cube], - statistics: list[str], + statistics: list[str | dict], output_products, span: str = 'overlap', ignore_scalar_coords: bool = False, ) -> dict | set: - """Entry point for ensemble statistics. + """Compute ensemble statistics. - An ensemble grouping is performed on the input products. - The statistics are then computed calling - the :func:`esmvalcore.preprocessor.multi_model_statistics` module, - taking the grouped products as an input. + An ensemble grouping is performed on the input products (using the + `ensemble` facet of input datasets). The statistics are then computed + calling :func:`esmvalcore.preprocessor.multi_model_statistics` with + appropriate groups. Parameters ---------- products: Cubes (or products) over which the statistics will be computed. statistics: - Statistical metrics to be computed, e.g. [``mean``, ``max``]. Choose - from the operators listed in the iris.analysis package. Percentiles can - be specified like ``pXX.YY``. + Statistical operations to be computed, e.g., ``['mean', 'median']``. + For some statistics like percentiles, it is also possible to pass + additional keyword arguments, e.g., ``[{'operator': 'mean', 'percent': + 20}]``. All supported options are are given in :ref:`this table + `. output_products: dict For internal use only. A dict with statistics names as keys and preprocessorfiles as values. If products are passed as input, the diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index 1008cf67ad..8b3c483154 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -1,30 +1,50 @@ """Horizontal and vertical regridding module.""" +from __future__ import annotations +import functools import importlib import inspect import logging import os import re import ssl +import warnings from copy import deepcopy from decimal import Decimal from pathlib import Path -from typing import Dict +from typing import TYPE_CHECKING, Any import dask.array as da import iris import numpy as np import stratify from geopy.geocoders import Nominatim -from iris.analysis import AreaWeighted, Linear, Nearest, UnstructuredNearest +from iris.analysis import AreaWeighted, Linear, Nearest +from iris.cube import Cube from iris.util import broadcast_to_shape -from ..cmor._fixes.shared import add_altitude_from_plev, add_plev_from_altitude -from ..cmor.table import CMOR_TABLES -from ._other import get_array_module -from ._regrid_esmpy import ESMF_REGRID_METHODS -from ._regrid_esmpy import regrid as esmpy_regrid -from ._supplementary_vars import add_ancillary_variable, add_cell_measure +from esmvalcore.cmor._fixes.shared import ( + add_altitude_from_plev, + add_plev_from_altitude, +) +from esmvalcore.cmor.table import CMOR_TABLES +from esmvalcore.exceptions import ESMValCoreDeprecationWarning +from esmvalcore.iris_helpers import has_irregular_grid, has_unstructured_grid +from esmvalcore.preprocessor._other import get_array_module +from esmvalcore.preprocessor._supplementary_vars import ( + add_ancillary_variable, + add_cell_measure, +) +from esmvalcore.preprocessor.regrid_schemes import ( + ESMPyAreaWeighted, + ESMPyLinear, + ESMPyNearest, + GenericFuncScheme, + UnstructuredNearest, +) + +if TYPE_CHECKING: + from esmvalcore.dataset import Dataset logger = logging.getLogger(__name__) @@ -48,22 +68,29 @@ _LON_MAX = 360.0 _LON_RANGE = _LON_MAX - _LON_MIN -# A cached stock of standard horizontal target grids. -_CACHE: Dict[str, iris.cube.Cube] = {} - # Supported point interpolation schemes. POINT_INTERPOLATION_SCHEMES = { 'linear': Linear(extrapolation_mode='mask'), 'nearest': Nearest(extrapolation_mode='mask'), } -# Supported horizontal regridding schemes. -HORIZONTAL_SCHEMES = { +# Supported horizontal regridding schemes for regular grids +HORIZONTAL_SCHEMES_REGULAR = { + 'area_weighted': AreaWeighted(), 'linear': Linear(extrapolation_mode='mask'), - 'linear_extrapolate': Linear(extrapolation_mode='extrapolate'), 'nearest': Nearest(extrapolation_mode='mask'), - 'area_weighted': AreaWeighted(), - 'unstructured_nearest': UnstructuredNearest(), +} + +# Supported horizontal regridding schemes for irregular grids +HORIZONTAL_SCHEMES_IRREGULAR = { + 'area_weighted': ESMPyAreaWeighted(), + 'linear': ESMPyLinear(), + 'nearest': ESMPyNearest(), +} + +# Supported horizontal regridding schemes for unstructured grids +HORIZONTAL_SCHEMES_UNSTRUCTURED = { + 'nearest': UnstructuredNearest(), } # Supported vertical interpolation schemes. @@ -132,7 +159,7 @@ def _generate_cube_from_dimcoords(latdata, londata, circular: bool = False): Returns ------- - :class:`~iris.cube.Cube` + iris.cube.Cube """ lats = iris.coords.DimCoord(latdata, standard_name='latitude', @@ -155,11 +182,12 @@ def _generate_cube_from_dimcoords(latdata, londata, circular: bool = False): shape = (latdata.size, londata.size) dummy = np.empty(shape, dtype=np.dtype('int8')) coords_spec = [(lats, 0), (lons, 1)] - cube = iris.cube.Cube(dummy, dim_coords_and_dims=coords_spec) + cube = Cube(dummy, dim_coords_and_dims=coords_spec) return cube +@functools.lru_cache def _global_stock_cube(spec, lat_offset=True, lon_offset=True): """Create a stock cube. @@ -185,7 +213,7 @@ def _global_stock_cube(spec, lat_offset=True, lon_offset=True): Returns ------- - :class:`~iris.cube.Cube` + iris.cube.Cube """ dlon, dlat = parse_cell_spec(spec) mid_dlon, mid_dlat = dlon / 2, dlat / 2 @@ -280,7 +308,7 @@ def _regional_stock_cube(spec: dict): Returns ------- - :class:`~iris.cube.Cube`. + iris.cube.Cube """ latdata, londata = _spec_to_latlonvals(**spec) @@ -300,19 +328,6 @@ def add_bounds_from_step(coord, step): return cube -def _attempt_irregular_regridding(cube, scheme): - """Check if irregular regridding with ESMF should be used.""" - if isinstance(scheme, str) and scheme in ESMF_REGRID_METHODS: - try: - lat_dim = cube.coord('latitude').ndim - lon_dim = cube.coord('longitude').ndim - if lat_dim == lon_dim == 2: - return True - except iris.exceptions.CoordinateNotFoundError: - pass - return False - - def extract_location(cube, location, scheme): """Extract a point using a location name, with interpolation. @@ -415,7 +430,7 @@ def extract_point(cube, latitude, longitude, scheme): Returns ------- - :py:class:`~iris.cube.Cube` + iris.cube.Cube Returns a cube with the extracted point(s), and with adjusted latitude and longitude coordinates (see above). If desired point outside values for at least one coordinate, this cube will have fully @@ -470,19 +485,180 @@ def is_dataset(dataset): return hasattr(dataset, 'facets') -def regrid(cube, target_grid, scheme, lat_offset=True, lon_offset=True): +def _get_target_grid_cube( + cube: Cube, + target_grid: Cube | Dataset | Path | str | dict, + lat_offset: bool = True, + lon_offset: bool = True, +) -> Cube: + """Get target grid cube.""" + if is_dataset(target_grid): + target_grid = target_grid.copy() # type: ignore + target_grid.supplementaries.clear() # type: ignore + target_grid.files = [target_grid.files[0]] # type: ignore + target_grid_cube = target_grid.load() # type: ignore + elif isinstance(target_grid, (str, Path)) and os.path.isfile(target_grid): + target_grid_cube = iris.load_cube(target_grid) + elif isinstance(target_grid, str): + # Generate a target grid from the provided cell-specification, + # and cache the resulting stock cube for later use. + target_grid_cube = _global_stock_cube( + target_grid, lat_offset, lon_offset + ) + # Align the target grid coordinate system to the source + # coordinate system. + src_cs = cube.coord_system() + xcoord = target_grid_cube.coord(axis='x', dim_coords=True) + ycoord = target_grid_cube.coord(axis='y', dim_coords=True) + xcoord.coord_system = src_cs + ycoord.coord_system = src_cs + elif isinstance(target_grid, dict): + # Generate a target grid from the provided specification, + target_grid_cube = _regional_stock_cube(target_grid) + else: + target_grid_cube = target_grid + + if not isinstance(target_grid_cube, Cube): + raise ValueError(f'Expecting a cube, got {target_grid}.') + + return target_grid_cube + + +def _attempt_irregular_regridding(cube: Cube, scheme: str) -> bool: + """Check if irregular regridding with ESMF should be used.""" + if not has_irregular_grid(cube): + return False + if scheme not in HORIZONTAL_SCHEMES_IRREGULAR: + raise ValueError( + f"Regridding scheme '{scheme}' does not support irregular data, " + f"expected one of {list(HORIZONTAL_SCHEMES_IRREGULAR)}" + ) + return True + + +def _attempt_unstructured_regridding(cube: Cube, scheme: str) -> bool: + """Check if unstructured regridding should be used.""" + if not has_unstructured_grid(cube): + return False + if scheme not in HORIZONTAL_SCHEMES_UNSTRUCTURED: + raise ValueError( + f"Regridding scheme '{scheme}' does not support unstructured " + f"data, expected one of {list(HORIZONTAL_SCHEMES_UNSTRUCTURED)}" + ) + return True + + +def _load_scheme(src_cube: Cube, scheme: str | dict): + """Return scheme that can be used in :meth:`iris.cube.Cube.regrid`.""" + loaded_scheme: Any = None + + # Deprecations + if scheme == 'unstructured_nearest': + msg = ( + "The regridding scheme `unstructured_nearest` has been deprecated " + "in ESMValCore version 2.11.0 and is scheduled for removal in " + "version 2.13.0. Please use the scheme `nearest` instead. This is " + "an exact replacement for data on unstructured grids. Since " + "version 2.11.0, ESMValCore is able to determine the most " + "suitable regridding scheme based on the input data." + ) + warnings.warn(msg, ESMValCoreDeprecationWarning) + scheme = 'nearest' + + if scheme == 'linear_extrapolate': + msg = ( + "The regridding scheme `linear_extrapolate` has been deprecated " + "in ESMValCore version 2.11.0 and is scheduled for removal in " + "version 2.13.0. Please use a generic scheme with `reference: " + "iris.analysis:Linear` and `extrapolation_mode: extrapolate` " + "instead (see https://docs.esmvaltool.org/projects/ESMValCore/en/" + "latest/recipe/preprocessor.html#generic-regridding-schemes)." + "This is an exact replacement." + ) + warnings.warn(msg, ESMValCoreDeprecationWarning) + scheme = 'linear' + loaded_scheme = Linear(extrapolation_mode='extrapolate') + logger.debug("Loaded regridding scheme %s", loaded_scheme) + return loaded_scheme + + # Scheme is a dict -> assume this describes a generic regridding scheme + if isinstance(scheme, dict): + loaded_scheme = _load_generic_scheme(scheme) + + # Scheme is a str -> load appropriate regridding scheme depending on the + # type of input data + elif _attempt_irregular_regridding(src_cube, scheme): + loaded_scheme = HORIZONTAL_SCHEMES_IRREGULAR[scheme] + elif _attempt_unstructured_regridding(src_cube, scheme): + loaded_scheme = HORIZONTAL_SCHEMES_UNSTRUCTURED[scheme] + else: + loaded_scheme = HORIZONTAL_SCHEMES_REGULAR.get(scheme) + + if loaded_scheme is None: + raise ValueError( + f"Got invalid regridding scheme string '{scheme}', expected one " + f"of {list(HORIZONTAL_SCHEMES_REGULAR)}" + ) + + logger.debug("Loaded regridding scheme %s", loaded_scheme) + + return loaded_scheme + + +def _load_generic_scheme(scheme: dict): + """Load generic regridding scheme.""" + scheme = dict(scheme) # do not overwrite original scheme + + try: + object_ref = scheme.pop("reference") + except KeyError as key_err: + raise ValueError( + "No reference specified for generic regridding." + ) from key_err + module_name, separator, scheme_name = object_ref.partition(":") + try: + obj: Any = importlib.import_module(module_name) + except ImportError as import_err: + raise ValueError( + f"Could not import specified generic regridding module " + f"'{module_name}'. Please double check spelling and that the " + f"required module is installed." + ) from import_err + if separator: + for attr in scheme_name.split('.'): + obj = getattr(obj, attr) + + # If `obj` is a function that requires `src_cube` and `grid_cube`, use + # GenericFuncScheme + scheme_args = inspect.getfullargspec(obj).args + if 'src_cube' in scheme_args and 'grid_cube' in scheme_args: + loaded_scheme = GenericFuncScheme(obj, **scheme) + else: + loaded_scheme = obj(**scheme) + + return loaded_scheme + + +def regrid( + cube: Cube, + target_grid: Cube | Dataset | Path | str | dict, + scheme: str | dict, + lat_offset: bool = True, + lon_offset: bool = True, +) -> Cube: """Perform horizontal regridding. - Note that the target grid can be a cube (:py:class:`~iris.cube.Cube`), - path to a cube (``str``), a grid spec (``str``) in the form - of `MxN`, or a ``dict`` specifying the target grid. + Note that the target grid can be a :class:`~iris.cube.Cube`, a + :class:`~esmvalcore.dataset.Dataset`, a path to a cube + (:class:`~pathlib.Path` or :obj:`str`), a grid spec (:obj:`str`) in the + form of `MxN`, or a :obj:`dict` specifying the target grid. - For the latter, the ``target_grid`` should be a ``dict`` with the + For the latter, the `target_grid` should be a :obj:`dict` with the following keys: - ``start_longitude``: longitude at the center of the first grid cell. - ``end_longitude``: longitude at the center of the last grid cell. - - ``step_longitude``: constant longitude distance between grid cell \ + - ``step_longitude``: constant longitude distance between grid cell centers. - ``start_latitude``: latitude at the center of the first grid cell. - ``end_latitude``: longitude at the center of the last grid cell. @@ -490,39 +666,40 @@ def regrid(cube, target_grid, scheme, lat_offset=True, lon_offset=True): Parameters ---------- - cube : :py:class:`~iris.cube.Cube` + cube: The source cube to be regridded. - target_grid : Cube or str or dict + target_grid: The (location of a) cube that specifies the target or reference grid for the regridding operation. + Alternatively, a :class:`~esmvalcore.dataset.Dataset` can be provided. Alternatively, a string cell specification may be provided, of the form ``MxN``, which specifies the extent of the cell, longitude by latitude (degrees) for a global, regular target grid. Alternatively, a dictionary with a regional target grid may be specified (see above). - scheme : str or dict - The regridding scheme to perform. If both source and target grid are - structured (regular or irregular), can be one of the built-in schemes - ``linear``, ``linear_extrapolate``, ``nearest``, ``area_weighted``, - ``unstructured_nearest``. - Alternatively, a `dict` that specifies generic regridding (see below). - lat_offset : bool - Offset the grid centers of the latitude coordinate w.r.t. the - pole by half a grid step. This argument is ignored if ``target_grid`` - is a cube or file. - lon_offset : bool + scheme: + The regridding scheme to perform. If the source grid is structured + (regular or irregular), can be one of the built-in schemes ``linear``, + ``nearest``, ``area_weighted``. If the source grid is unstructured, can + be one of the built-in schemes ``nearest``. Alternatively, a `dict` + that specifies generic regridding can be given (see below). + lat_offset: + Offset the grid centers of the latitude coordinate w.r.t. the pole by + half a grid step. This argument is ignored if `target_grid` is a cube + or file. + lon_offset: Offset the grid centers of the longitude coordinate w.r.t. Greenwich - meridian by half a grid step. - This argument is ignored if ``target_grid`` is a cube or file. + meridian by half a grid step. This argument is ignored if + `target_grid` is a cube or file. Returns ------- - :py:class:`~iris.cube.Cube` + iris.cube.Cube Regridded cube. See Also -------- - extract_levels : Perform vertical regridding. + extract_levels: Perform vertical regridding. Notes ----- @@ -563,105 +740,34 @@ def regrid(cube, target_grid, scheme, lat_offset=True, lon_offset=True): reference: esmf_regrid.schemes:ESMFAreaWeighted """ - if is_dataset(target_grid): - target_grid = target_grid.copy() - target_grid.supplementaries.clear() - target_grid.files = [target_grid.files[0]] - target_grid = target_grid.load() - elif isinstance(target_grid, (str, Path)) and os.path.isfile(target_grid): - target_grid = iris.load_cube(target_grid) - elif isinstance(target_grid, str): - # Generate a target grid from the provided cell-specification, - # and cache the resulting stock cube for later use. - target_grid = _CACHE.setdefault( - target_grid, - _global_stock_cube(target_grid, lat_offset, lon_offset), - ) - # Align the target grid coordinate system to the source - # coordinate system. - src_cs = cube.coord_system() - xcoord = target_grid.coord(axis='x', dim_coords=True) - ycoord = target_grid.coord(axis='y', dim_coords=True) - xcoord.coord_system = src_cs - ycoord.coord_system = src_cs - elif isinstance(target_grid, dict): - # Generate a target grid from the provided specification, - target_grid = _regional_stock_cube(target_grid) - - if not isinstance(target_grid, iris.cube.Cube): - raise ValueError(f'Expecting a cube, got {target_grid}.') - - if isinstance(scheme, dict): - scheme = dict(scheme) # do not overwrite original scheme - try: - object_ref = scheme.pop("reference") - except KeyError as key_err: - raise ValueError( - "No reference specified for generic regridding.") from key_err - module_name, separator, scheme_name = object_ref.partition(":") - try: - obj = importlib.import_module(module_name) - except ImportError as import_err: - raise ValueError( - "Could not import specified generic regridding module. " - "Please double check spelling and that the required module is " - "installed.") from import_err - if separator: - for attr in scheme_name.split('.'): - obj = getattr(obj, attr) - - scheme_args = inspect.getfullargspec(obj).args - # Add source and target cubes as arguments if required - if 'src_cube' in scheme_args: - scheme['src_cube'] = cube - if 'grid_cube' in scheme_args: - scheme['grid_cube'] = target_grid - - loaded_scheme = obj(**scheme) - else: - loaded_scheme = HORIZONTAL_SCHEMES.get(scheme.lower()) - - if loaded_scheme is None: - emsg = 'Unknown regridding scheme, got {!r}.' - raise ValueError(emsg.format(scheme)) - - # Unstructured regridding requires x2 2d spatial coordinates, - # so ensure to purge any 1d native spatial dimension coordinates - # for the regridder. - if scheme == 'unstructured_nearest': - for axis in ['x', 'y']: - coords = cube.coords(axis=axis, dim_coords=True) - if coords: - [coord] = coords - cube.remove_coord(coord) + # Load target grid and select appropriate scheme + target_grid_cube = _get_target_grid_cube( + cube, target_grid, lat_offset=lat_offset, lon_offset=lon_offset, + ) # Horizontal grids from source and target (almost) match # -> Return source cube with target coordinates - if _horizontal_grid_is_close(cube, target_grid): + if _horizontal_grid_is_close(cube, target_grid_cube): for coord in ['latitude', 'longitude']: - cube.coord(coord).points = target_grid.coord(coord).points - cube.coord(coord).bounds = target_grid.coord(coord).bounds + cube.coord(coord).points = ( + target_grid_cube.coord(coord).core_points() + ) + cube.coord(coord).bounds = ( + target_grid_cube.coord(coord).core_bounds() + ) return cube - # Horizontal grids from source and target do not match - # -> Regrid - if _attempt_irregular_regridding(cube, scheme): - cube = esmpy_regrid(cube, target_grid, scheme) - elif isinstance(loaded_scheme, iris.cube.Cube): - # Return regridded cube in cases in which the - # scheme is a function f(src_cube, grid_cube) -> Cube - cube = loaded_scheme - else: - cube = _rechunk(cube, target_grid) - cube = cube.regrid(target_grid, loaded_scheme) + # Load scheme, rechunk and regrid + if isinstance(scheme, str): + scheme = scheme.lower() + loaded_scheme = _load_scheme(cube, scheme) + cube = _rechunk(cube, target_grid_cube) + cube = cube.regrid(target_grid_cube, loaded_scheme) return cube -def _rechunk( - cube: iris.cube.Cube, - target_grid: iris.cube.Cube, -) -> iris.cube.Cube: +def _rechunk(cube: Cube, target_grid: Cube) -> Cube: """Re-chunk cube with optimal chunk sizes for target grid.""" if not cube.has_lazy_data() or cube.ndim < 3: # Only rechunk lazy multidimensional data @@ -698,29 +804,30 @@ def _rechunk( return cube -def _horizontal_grid_is_close(cube1, cube2): +def _horizontal_grid_is_close(cube1: Cube, cube2: Cube) -> bool: """Check if two cubes have the same horizontal grid definition. The result of the function is a boolean answer, if both cubes have the same horizontal grid definition. The function checks both longitude and latitude, based on extent and resolution. + Note + ---- + The current implementation checks if the bounds and the grid shapes are the + same. Exits on first difference. + Parameters ---------- - cube1 : cube + cube1: The first of the cubes to be checked. - cube2 : cube + cube2: The second of the cubes to be checked. Returns ------- bool + ``True`` if grids are close; ``False`` if not. - .. note:: - - The current implementation checks if the bounds and the - grid shapes are the same. - Exits on first difference. """ # Go through the 2 expected horizontal coordinates longitude and latitude. for coord in ['latitude', 'longitude']: @@ -779,7 +886,7 @@ def _create_cube(src_cube, data, src_levels, levels): # Construct the resultant cube with the interpolated data # and the source cube metadata. kwargs = deepcopy(src_cube.metadata)._asdict() - result = iris.cube.Cube(data, **kwargs) + result = Cube(data, **kwargs) # Add the appropriate coordinates to the cube, excluding # any coordinates that span the z-dimension of interpolation. @@ -1117,7 +1224,7 @@ def extract_coordinate_points(cube, definition, scheme): Returns ------- - :py:class:`~iris.cube.Cube` + iris.cube.Cube Returns a cube with the extracted point(s), and with adjusted latitude and longitude coordinates (see above). If desired point outside values for at least one coordinate, this cube will have fully diff --git a/esmvalcore/preprocessor/_regrid_esmpy.py b/esmvalcore/preprocessor/_regrid_esmpy.py index c7df8bbb36..7da6dd63c1 100755 --- a/esmvalcore/preprocessor/_regrid_esmpy.py +++ b/esmvalcore/preprocessor/_regrid_esmpy.py @@ -11,6 +11,7 @@ raise exc import iris import numpy as np +from iris.cube import Cube from ._mapping import get_empty_data, map_slices, ref_to_dims_index @@ -39,6 +40,149 @@ # } +class ESMPyRegridder: + """General ESMPy regridder. + + Parameters + ---------- + src_cube: + Cube defining the source grid. + tgt_cube: + Cube defining the target grid. + method: + Regridding algorithm. Must be one of `linear`, `area_weighted`, + `nearest`. + mask_threshold: + Threshold used to regrid mask of input cube. + + """ + + def __init__( + self, + src_cube: Cube, + tgt_cube: Cube, + method: str = 'linear', + mask_threshold: float = 0.99, + ): + """Initialize class instance.""" + self.src_cube = src_cube + self.tgt_cube = tgt_cube + self.method = method + self.mask_threshold = mask_threshold + + def __call__(self, cube: Cube) -> Cube: + """Perform regridding. + + Parameters + ---------- + cube: + Cube to be regridded. + + Returns + ------- + Cube + Regridded cube. + + """ + src_rep, dst_rep = get_grid_representants(cube, self.tgt_cube) + regridder = build_regridder( + src_rep, dst_rep, self.method, mask_threshold=self.mask_threshold + ) + result = map_slices(cube, regridder, src_rep, dst_rep) + return result + + +class _ESMPyScheme: + """General irregular regridding scheme. + + This class can be used in :meth:`iris.cube.Cube.regrid`. + + Note + ---- + See `ESMPy `__ for more details on + this. + + Parameters + ---------- + mask_threshold: + Threshold used to regrid mask of source cube. + + """ + + _METHOD = '' + + def __init__(self, mask_threshold: float = 0.99): + """Initialize class instance.""" + self.mask_threshold = mask_threshold + + def __repr__(self) -> str: + """Return string representation of class.""" + return ( + f'{self.__class__.__name__}(mask_threshold={self.mask_threshold})' + ) + + def regridder(self, src_cube: Cube, tgt_cube: Cube) -> ESMPyRegridder: + """Get regridder. + + Parameters + ---------- + src_cube: + Cube defining the source grid. + tgt_cube: + Cube defining the target grid. + + Returns + ------- + ESMPyRegridder + Regridder instance. + + """ + return ESMPyRegridder( + src_cube, + tgt_cube, + method=self._METHOD, + mask_threshold=self.mask_threshold, + ) + + +class ESMPyAreaWeighted(_ESMPyScheme): + """ESMPy area-weighted regridding scheme. + + This class can be used in :meth:`iris.cube.Cube.regrid`. + + Does not support lazy regridding. + + """ + + _METHOD = 'area_weighted' + + +class ESMPyLinear(_ESMPyScheme): + """ESMPy bilinear regridding scheme. + + This class can be used in :meth:`iris.cube.Cube.regrid`. + + Does not support lazy regridding. + + """ + + _METHOD = 'linear' + + +class ESMPyNearest(_ESMPyScheme): + """ESMPy nearest-neighbor regridding scheme. + + This class can be used in :meth:`iris.cube.Cube.regrid`. + + Does not support lazy regridding. + + """ + + _METHOD = 'nearest' + + def cf_2d_bounds_to_esmpy_corners(bounds, circular): """Convert cf style 2d bounds to normal (esmpy style) corners.""" no_lat_points, no_lon_points = bounds.shape[:2] @@ -121,7 +265,7 @@ def is_lon_circular(lon): else: raise NotImplementedError('AuxCoord longitude is higher ' 'dimensional than 2d. Giving up.') - circular = np.alltrue(abs(seam) % 360. < 1.e-3) + circular = np.all(abs(seam) % 360. < 1.e-3) else: raise ValueError('longitude is neither DimCoord nor AuxCoord. ' 'Giving up.') @@ -318,37 +462,3 @@ def get_grid_representants(src, dst): aux_coords_and_dims=aux_coords_and_dims, ) return src_rep, dst_rep - - -def regrid(src, dst, method='linear'): - """ - Regrid src_cube to the grid defined by dst_cube. - - Regrid the data in src_cube onto the grid defined by dst_cube. - - Parameters - ---------- - src: :class:`iris.cube.Cube` - Source data. Must have latitude and longitude coords. - These can be 1d or 2d and should have bounds. - dst: :class:`iris.cube.Cube` - Defines the target grid. - method: - Selects the regridding method. - Can be 'linear', 'area_weighted', - or 'nearest'. See ESMPy_. - - Returns - ------- - :class:`iris.cube.Cube`: - The regridded cube. - - - .. _ESMPy: http://www.earthsystemmodeling.org/ - esmf_releases/non_public/ESMF_7_0_0/esmpy_doc/html/ - RegridMethod.html#ESMF.api.constants.RegridMethod - """ - src_rep, dst_rep = get_grid_representants(src, dst) - regridder = build_regridder(src_rep, dst_rep, method) - res = map_slices(src, regridder, src_rep, dst_rep) - return res diff --git a/esmvalcore/preprocessor/_regrid_unstructured.py b/esmvalcore/preprocessor/_regrid_unstructured.py new file mode 100644 index 0000000000..d23464cf42 --- /dev/null +++ b/esmvalcore/preprocessor/_regrid_unstructured.py @@ -0,0 +1,49 @@ +"""Unstructured grid regridding.""" +from __future__ import annotations + +import logging + +from iris.analysis import UnstructuredNearest as IrisUnstructuredNearest +from iris.analysis.trajectory import UnstructuredNearestNeigbourRegridder +from iris.cube import Cube + +logger = logging.getLogger(__name__) + + +class UnstructuredNearest(IrisUnstructuredNearest): + """Unstructured nearest-neighbor regridding scheme. + + This class is a wrapper around :class:`iris.analysis.UnstructuredNearest` + that removes any additional X or Y coordinates prior to regridding if + necessary. It can be used in :meth:`iris.cube.Cube.regrid`. + + """ + + def regridder( + self, + src_cube: Cube, + tgt_cube: Cube, + ) -> UnstructuredNearestNeigbourRegridder: + """Get regridder. + + Parameters + ---------- + src_cube: + Cube defining the source grid. + tgt_cube: + Cube defining the target grid. + + Returns + ------- + UnstructuredNearestNeigbourRegridder + Regridder instance. + + """ + # Unstructured nearest-neighbor regridding requires exactly one X and + # one Y coordinate (latitude and longitude). Remove any X or Y + # dimensional coordinates if necessary. + for axis in ['x', 'y']: + if src_cube.coords(axis=axis, dim_coords=True): + coord = src_cube.coord(axis=axis, dim_coords=True) + src_cube.remove_coord(coord) + return super().regridder(src_cube, tgt_cube) diff --git a/esmvalcore/preprocessor/_shared.py b/esmvalcore/preprocessor/_shared.py index 09e98d6936..56bfc42560 100644 --- a/esmvalcore/preprocessor/_shared.py +++ b/esmvalcore/preprocessor/_shared.py @@ -7,8 +7,9 @@ import re import warnings from collections.abc import Callable -from typing import Any, Optional +from typing import Any, Literal, Optional +import dask.array as da import iris.analysis import numpy as np from iris.coords import DimCoord @@ -181,3 +182,59 @@ def update_weights_kwargs( else: kwargs.pop('weights', None) return kwargs + + +def get_normalized_cube( + cube: Cube, + statistics_cube: Cube, + normalize: Literal['subtract', 'divide'], +) -> Cube: + """Get cube normalized with statistics cube. + + Parameters + ---------- + cube: + Input cube that will be normalized. + statistics_cube: + Cube that is used to normalize the input cube. Needs to be + broadcastable to the input cube's shape according to iris' rich + broadcasting rules enabled by the use of named dimensions (see also + https://scitools-iris.readthedocs.io/en/latest/userguide/cube_maths. + html#calculating-a-cube-anomaly). This is usually ensure by using + :meth:`iris.cube.Cube.collapsed` to calculate the statistics cube. + normalize: + Normalization operation. Can either be `subtract` (statistics cube is + subtracted from the input cube) or `divide` (input cube is divided by + the statistics cube). + + Returns + ------- + Cube + Input cube normalized with statistics cube. + + """ + if normalize == 'subtract': + normalized_cube = cube - statistics_cube + + elif normalize == 'divide': + normalized_cube = cube / statistics_cube + + # Iris sometimes masks zero-divisions, sometimes not + # (https://github.com/SciTools/iris/issues/5523). Make sure to + # consistently mask them here. + normalized_cube.data = da.ma.masked_invalid( + normalized_cube.core_data() + ) + + else: + raise ValueError( + f"Expected 'subtract' or 'divide' for `normalize`, got " + f"'{normalize}'" + ) + + # Keep old metadata except for units + new_units = normalized_cube.units + normalized_cube.metadata = cube.metadata + normalized_cube.units = new_units + + return normalized_cube diff --git a/esmvalcore/preprocessor/_time.py b/esmvalcore/preprocessor/_time.py index 7d5e7ed603..ca4e95ce56 100644 --- a/esmvalcore/preprocessor/_time.py +++ b/esmvalcore/preprocessor/_time.py @@ -9,6 +9,7 @@ import datetime import logging import warnings +from functools import partial from typing import Iterable, Optional from warnings import filterwarnings @@ -16,18 +17,23 @@ import dask.config import iris import iris.coord_categorisation -import iris.exceptions import iris.util import isodate import numpy as np -from iris.coords import AuxCoord +from cf_units import Unit +from iris.coords import AuxCoord, Coord, DimCoord from iris.cube import Cube, CubeList +from iris.exceptions import CoordinateMultiDimError, CoordinateNotFoundError from iris.time import PartialDateTime +from iris.util import broadcast_to_shape +from numpy.typing import DTypeLike from esmvalcore.cmor.fixes import get_next_month, get_time_bounds -from esmvalcore.iris_helpers import date2num - -from ._shared import get_iris_aggregator, update_weights_kwargs +from esmvalcore.iris_helpers import date2num, rechunk_cube +from esmvalcore.preprocessor._shared import ( + get_iris_aggregator, + update_weights_kwargs, +) logger = logging.getLogger(__name__) @@ -1150,7 +1156,7 @@ def timeseries_filter( """ try: cube.coord('time') - except iris.exceptions.CoordinateNotFoundError: + except CoordinateNotFoundError: logger.error("Cube %s does not have time coordinate", cube) raise @@ -1292,3 +1298,426 @@ def resample_time( raise ValueError( f"Time coordinate {dates} does not contain {requested} for {cube}") return cube + + +def _lin_pad(array: np.ndarray, delta: float, pad_with: int) -> np.ndarray: + """Linearly pad an array on both sides with constant difference.""" + end_values = (array[0] - pad_with * delta, array[-1] + pad_with * delta) + new_array = np.pad(array, pad_with, 'linear_ramp', end_values=end_values) + return new_array + + +def _guess_time_bounds(time_coord: DimCoord) -> None: + """Guess bounds of time coordinate in-place.""" + if time_coord.has_bounds(): + return + try: + time_coord.guess_bounds() + except ValueError: # coordinate has only 1 point + point = time_coord.points[0] + time_coord.bounds = [[point - 0.5, point + 0.5]] + + +def _get_lst_offset(lon_coord: Coord) -> np.ndarray: + """Get offsets to shift UTC time to local solar time (LST). + + Note + ---- + This function expects longitude in degrees. Can be in [0, 360] or [-180, + 180] format. + + """ + # Make sure that longitude is in degrees and shift it to [-180, 180] first + # (do NOT overwrite input coordinate) + lon_coord = lon_coord.copy() + lon_coord.convert_units('degrees') + shifted_lon = (lon_coord.points + 180.0) % 360 - 180.0 + return 12.0 * (shifted_lon / 180.0) + + +def _get_lsts(time_coord: DimCoord, lon_coord: Coord) -> np.ndarray: + """Get array of binned local solar times (LSTs) of shape (lon, time). + + Note + ---- + LSTs outside of the time bins given be the time coordinate bounds are put + into a bin below/above the time coordinate. + + """ + # Pad time coordinate with 1 time step at both sides for the bins for LSTs + # outside of the time coordinate + dtime = np.abs( + time_coord.bounds[0, 1] - time_coord.bounds[0, 0] + ) + new_points = _lin_pad(time_coord.points, dtime, 1) + bnds = time_coord.bounds + new_bounds = np.stack( + (_lin_pad(bnds[:, 0], dtime, 1), _lin_pad(bnds[:, 1], dtime, 1)), + axis=-1, + ) + time_coord = time_coord.copy(new_points, bounds=new_bounds) + + n_time = time_coord.shape[0] + n_lon = lon_coord.shape[0] + + # Calculate LST + time_array = np.broadcast_to(time_coord.points, (n_lon, n_time)) + time_offsets = _get_lst_offset(lon_coord).reshape(-1, 1) + exact_lst_array = time_array + time_offsets # (lon, time) + + # Put LST into bins given be the time coordinate bounds + bins = np.concatenate(([time_coord.bounds[0, 0]], time_coord.bounds[:, 1])) + idx = np.digitize(exact_lst_array, bins) - 1 # (lon, time); idx for time + idx[idx < 0] = 0 # values outside the time coordinate + idx[idx >= n_time] = - 1 # values outside the time coordinate + lst_array = time_coord.points[idx] # (lon, time) + + # Remove time steps again that have been added previously + lst_array = lst_array[:, 1:-1] + + return lst_array + + +def _get_time_index_and_mask( + time_coord: DimCoord, + lon_coord: Coord, +) -> tuple[np.ndarray, np.ndarray]: + """Get advanced index and mask for time dimension of shape (time, lon). + + Note + ---- + The mask considers the fact that not all values for all local solar times + (LSTs) are given. E.g., for hourly data with first time point 01:00:00 + UTC, LST in Berlin is already 02:00:00 (assuming no daylight saving time). + Thus, for 01:00:00 LST on this day, there is no value for Berlin. + + """ + # Make sure that time coordinate has bounds (these are necessary for the + # binning) and uses 'hours' as reference units + time_coord.convert_units( + Unit('hours since 1850-01-01', calendar=time_coord.units.calendar) + ) + _guess_time_bounds(time_coord) + + lsts = _get_lsts(time_coord, lon_coord) # (lon, time) + n_time = time_coord.points.shape[0] + + # We use np.searchsorted to calculate the indices necessary to put the UTC + # times into their corresponding (binned) LSTs. These incides are 2D since + # they depend on time and longitude. + searchsorted_l = partial(np.searchsorted, side='left') + _get_indices_l = np.vectorize(searchsorted_l, signature='(i),(i)->(i)') + time_index_l = _get_indices_l(lsts, time_coord.points) # (lon, time) + + # To calculate the mask, we need to detect which LSTs are outside of the + # time coordinate. Unfortunately, searchsorted can only detect outliers on + # one side of the array. This side is determined by the `side` keyword + # argument. To consistently detect outliers on both sides, we use + # searchsorted again, this time with `side='right'` (the default is + # 'left'). Indices that are the same in both arrays need to be masked, as + # these are the ones outside of the time coordinate. All others will + # change. + searchsorted_r = partial(np.searchsorted, side='right') + _get_indices_r = np.vectorize(searchsorted_r, signature='(i),(i)->(i)') + time_index_r = _get_indices_r(lsts, time_coord.points) # (lon, time) + mask = time_index_l == time_index_r # (lon, time) + + # The index is given by the left indices (these are identical to the right + # indices minus 1) + time_index_l[time_index_l < 0] = 0 # will be masked + time_index_l[time_index_l >= n_time] = -1 # will be masked + + return (time_index_l.T, mask.T) # (time, lon) + + +def _transform_to_lst_eager( + data: np.ndarray, + *, + time_index: np.ndarray, + mask: np.ndarray, + time_dim: int, + lon_dim: int, + **__, +) -> np.ndarray: + """Transform array with UTC coord to local solar time (LST) coord. + + Note + ---- + This function is the eager version of `_transform_to_lst_lazy`. + + `data` needs to be at least 2D. `time_dim` and `lon_dim` correspond to the + dimensions that describe time and longitude dimensions in `data`, + respectively. + + `time_index` is an `advanced index + `__ + for the time dimension of `data` with shape (time, lon). It is used to + reorder the data along the time axis based on the longitude axis. + + `mask` is 2D with shape (time, lon) that will be applied to the final data. + + """ + # Apart from the time index, all other dimensions will stay the same; this + # is ensured with np.ogrid + idx = np.ogrid[tuple(slice(0, d) for d in data.shape)] + time_index = broadcast_to_shape( + time_index, data.shape, (time_dim, lon_dim) + ) + idx[time_dim] = time_index + new_data = data[tuple(idx)] + + # Apply properly broadcasted mask + mask = broadcast_to_shape(mask, new_data.shape, (time_dim, lon_dim)) + new_mask = mask | np.ma.getmaskarray(new_data) + + return np.ma.masked_array(new_data, mask=new_mask) + + +def _transform_to_lst_lazy( + data: da.core.Array, + *, + time_index: np.ndarray, + mask: np.ndarray, + time_dim: int, + lon_dim: int, + output_dtypes: DTypeLike, +) -> da.core.Array: + """Transform array with UTC coord to local solar time (LST) coord. + + Note + ---- + This function is the lazy version of `_transform_to_lst_eager` using + dask's :func:`dask.array.apply_gufunc`. + + `data` needs to be at least 2D. `time_dim` and `lon_dim` correspond to the + dimensions that describe time and longitude dimensions in `data`, + respectively. + + `time_index` is an `advanced index + `__ + for the time dimension of `data` with shape (time, lon). It is used to + reorder the data along the time axis based on the longitude axis. + + `mask` is 2D with shape (time, lon) that will be applied to the final data. + + """ + _transform_chunk_to_lst = partial( + _transform_to_lst_eager, + time_index=time_index, + mask=mask, + time_dim=-2, # this is ensured by da.apply_gufunc + lon_dim=-1, # this is ensured by da.apply_gufunc + ) + new_data = da.apply_gufunc( + _transform_chunk_to_lst, + '(t,y)->(t,y)', + data, + axes=[(time_dim, lon_dim), (time_dim, lon_dim)], + output_dtypes=output_dtypes, + ) + return new_data + + +def _transform_arr_to_lst( + data: np.ndarray | da.core.Array, + *, + time_index: np.ndarray, + mask: np.ndarray, + time_dim: int, + lon_dim: int, + output_dtypes: DTypeLike, +) -> np.ndarray | da.core.Array: + """Transform array with UTC coord to local solar time (LST) coord. + + Note + ---- + This function either calls `_transform_to_lst_eager` or + `_transform_to_lst_lazy` depending on the type of input data. + + """ + if isinstance(data, np.ndarray): + func = _transform_to_lst_eager # type: ignore + else: + func = _transform_to_lst_lazy # type: ignore + new_data = func( + data, # type: ignore + time_index=time_index, + mask=mask, + time_dim=time_dim, + lon_dim=lon_dim, + output_dtypes=output_dtypes, + ) + return new_data + + +def _transform_cube_to_lst(cube: Cube) -> Cube: + """Transform cube to local solar time (LST) coordinate (lazy; in-place).""" + # Rechunk cube properly (it must not be chunked along time and longitude + # dimension); this also creates a new cube so the original input cube is + # not overwritten + complete_coords = [ + cube.coord('time', dim_coords=True), cube.coord('longitude'), + ] + cube = rechunk_cube(cube, complete_coords) + + time_coord = cube.coord('time', dim_coords=True) + lon_coord = cube.coord('longitude') + time_dim = cube.coord_dims(time_coord)[0] + lon_dim = cube.coord_dims(lon_coord)[0] + + # Transform cube data + (time_index, mask) = _get_time_index_and_mask(time_coord, lon_coord) + _transform_arr = partial( + _transform_arr_to_lst, + time_index=time_index, + mask=mask, + ) + cube.data = _transform_arr( + cube.core_data(), + time_dim=time_dim, + lon_dim=lon_dim, + output_dtypes=cube.dtype, + ) + + # Transform aux coords that span time and longitude dimensions + for coord in cube.coords(dim_coords=False): + dims = cube.coord_dims(coord) + if time_dim in dims and lon_dim in dims: + time_dim_ = dims.index(time_dim) + lon_dim_ = dims.index(lon_dim) + coord.points = _transform_arr( + coord.core_points(), + time_dim=time_dim_, + lon_dim=lon_dim_, + output_dtypes=coord.dtype, + ) + if coord.has_bounds(): + coord.bounds = _transform_arr( + coord.core_bounds(), + time_dim=time_dim_, + lon_dim=lon_dim_, + output_dtypes=coord.bounds_dtype, + ) + + # Transform cell measures that span time and longitude dimensions + for cell_measure in cube.cell_measures(): + dims = cube.cell_measure_dims(cell_measure) + if time_dim in dims and lon_dim in dims: + time_dim_ = dims.index(time_dim) + lon_dim_ = dims.index(lon_dim) + cell_measure.data = _transform_arr( + cell_measure.core_data(), + time_dim=time_dim_, + lon_dim=lon_dim_, + output_dtypes=cell_measure.dtype, + ) + + # Transform ancillary variables that span time and longitude dimensions + for anc_var in cube.ancillary_variables(): + dims = cube.ancillary_variable_dims(anc_var) + if time_dim in dims and lon_dim in dims: + time_dim_ = dims.index(time_dim) + lon_dim_ = dims.index(lon_dim) + anc_var.data = _transform_arr( + anc_var.core_data(), + time_dim=time_dim_, + lon_dim=lon_dim_, + output_dtypes=anc_var.dtype, + ) + + return cube + + +def _check_cube_coords(cube): + if not cube.coords('time', dim_coords=True): + raise CoordinateNotFoundError( + f"Input cube {cube.summary(shorten=True)} needs a dimensional " + f"coordinate `time`" + ) + time_coord = cube.coord('time', dim_coords=True) + # The following works since DimCoords are always 1D and monotonic + if time_coord.points[0] > time_coord.points[-1]: + raise ValueError("`time` coordinate must be monotonically increasing") + + if not cube.coords('longitude'): + raise CoordinateNotFoundError( + f"Input cube {cube.summary(shorten=True)} needs a coordinate " + f"`longitude`" + ) + lon_ndim = len(cube.coord_dims('longitude')) + if lon_ndim != 1: + raise CoordinateMultiDimError( + f"Input cube {cube.summary(shorten=True)} needs a 1D coordinate " + f"`longitude`, got {lon_ndim:d}D" + ) + + +def local_solar_time(cube: Cube) -> Cube: + """Convert UTC time coordinate to local solar time (LST). + + This preprocessor transforms input data with a UTC-based time coordinate to + a `local solar time (LST) `__ + coordinate. In LST, 12:00 noon is defined as the moment when the sun + reaches its highest point in the sky. Thus, LST is mainly determined by + longitude of a location. LST is particularly suited to analyze diurnal + cycles across larger regions of the globe, which would be phase-shifted + against each other when using UTC time. + + To transform data from UTC to LST, this function shifts data along the time + axis based on the longitude. In addition to the `cube`'s data, this + function also considers auxiliary coordinates, cell measures and ancillary + variables that span both the time and longitude dimension. + + Note + ---- + This preprocessor preserves the temporal frequency of the input data. For + example, hourly input data will be transformed into hourly output data. For + this, a location's exact LST will be put into corresponding bins defined + by the bounds of the input time coordinate (in this example, the bin size + is 1 hour). If time bounds are not given or cannot be approximated (only + one time step is given), a bin size of 1 hour is assumed. + + LST is approximated as `UTC_time + 12*longitude/180`, where `longitude` is + assumed to be in [-180, 180] (this function automatically calculates the + correct format for the longitude). This is only an approximation since the + exact LST also depends on the day of year due to the eccentricity of + Earth's orbit (see `equation of time + `__). However, since the + corresponding error is ~15 min at most, this is ignored here, as most + climate model data has a courser temporal resolution and the time scale for + diurnal evolution of meteorological phenomena is usually in the order of + hours, not minutes. + + Parameters + ---------- + cube: + Input cube. Needs a 1D monotonically increasing dimensional coordinate + `time` (assumed to refer to UTC time) and a 1D coordinate `longitude`. + + Returns + ------- + Cube + Transformed cube of same shape as input cube with an LST coordinate + instead of UTC time. + + Raises + ------ + iris.exceptions.CoordinateNotFoundError + Input cube does not contain valid `time` and/or `longitude` coordinate. + iris.exceptions.CoordinateMultiDimError + Input cube has multidimensional `longitude` coordinate. + ValueError + `time` coordinate of input cube is not monotonically increasing. + + """ + # Make sure that cube has valid time and longitude coordinates + _check_cube_coords(cube) + + # Transform cube data and all dimensional metadata that spans time AND + # longitude dimensions + cube = _transform_cube_to_lst(cube) + + # Adapt metadata of time coordinate + cube.coord('time', dim_coords=True).long_name = 'Local Solar Time' + + return cube diff --git a/esmvalcore/preprocessor/_units.py b/esmvalcore/preprocessor/_units.py index 4f435b7f86..8c96f78ae1 100644 --- a/esmvalcore/preprocessor/_units.py +++ b/esmvalcore/preprocessor/_units.py @@ -2,11 +2,14 @@ Allows for unit conversions. """ +from __future__ import annotations + import logging -from cf_units import Unit +import dask.array as da import iris import numpy as np +from cf_units import Unit logger = logging.getLogger(__name__) @@ -21,6 +24,10 @@ ('precipitation_flux', 'kg m-2 s-1'), ('lwe_precipitation_rate', 'mm s-1'), ], + [ + ('equivalent_thickness_at_stp_of_atmosphere_ozone_content', 'm'), + ('equivalent_thickness_at_stp_of_atmosphere_ozone_content', '1e5 DU'), + ], ] @@ -41,7 +48,7 @@ def _try_special_conversions(cube, units): if (cube.standard_name == std_name and cube.units.is_convertible(special_units)): for (target_std_name, target_units) in special_case: - if target_std_name == std_name: + if target_units == special_units: continue # Step 2: find suitable target name and units @@ -79,8 +86,10 @@ def convert_units(cube, units): Currently, the following special conversions are supported: - * ``precipitation_flux`` (``kg m-2 s-1``) -- - ``lwe_precipitation_rate`` (``mm day-1``) + * ``precipitation_flux`` (``kg m-2 s-1``) -- + ``lwe_precipitation_rate`` (``mm day-1``) + * ``equivalent_thickness_at_stp_of_atmosphere_ozone_content`` (``m``) -- + ``equivalent_thickness_at_stp_of_atmosphere_ozone_content`` (``DU``) Names in the list correspond to ``standard_names`` of the input data. Conversions are allowed from each quantity to any other quantity given in a @@ -114,7 +123,10 @@ def convert_units(cube, units): return cube -def accumulate_coordinate(cube, coordinate): +def accumulate_coordinate( + cube: iris.cube.Cube, + coordinate: str | iris.coords.DimCoord | iris.coords.AuxCoord +) -> iris.cube.Cube: """Weight data using the bounds from a given coordinate. The resulting cube will then have units given by @@ -122,10 +134,10 @@ def accumulate_coordinate(cube, coordinate): Parameters ---------- - cube : iris.cube.Cube + cube: Data cube for the flux - coordinate: str + coordinate: Name of the coordinate that will be used as weights. Returns @@ -152,8 +164,9 @@ def accumulate_coordinate(cube, coordinate): raise NotImplementedError( f'Multidimensional coordinate {coord} not supported.') + array_module = da if coord.has_lazy_bounds() else np factor = iris.coords.AuxCoord( - np.diff(coord.bounds)[..., -1], + array_module.diff(coord.core_bounds())[..., -1], var_name=coord.var_name, long_name=coord.long_name, units=coord.units, diff --git a/esmvalcore/preprocessor/_volume.py b/esmvalcore/preprocessor/_volume.py index e0c376585e..5f64609193 100644 --- a/esmvalcore/preprocessor/_volume.py +++ b/esmvalcore/preprocessor/_volume.py @@ -7,7 +7,7 @@ import logging import warnings -from typing import Iterable, Sequence +from typing import Iterable, Literal, Optional, Sequence import dask.array as da import iris @@ -17,7 +17,11 @@ from iris.exceptions import CoordinateMultiDimError from ._area import compute_area_weights -from ._shared import get_iris_aggregator, update_weights_kwargs +from ._shared import ( + get_iris_aggregator, + get_normalized_cube, + update_weights_kwargs, +) from ._supplementary_vars import register_supplementaries logger = logging.getLogger(__name__) @@ -124,7 +128,7 @@ def calculate_volume(cube: Cube) -> da.core.Array: z_dim = cube.coord_dims(depth)[0] # Calculate Z-direction thickness - thickness = depth.bounds[..., 1] - depth.bounds[..., 0] + thickness = depth.core_bounds()[..., 1] - depth.core_bounds()[..., 0] # Try to calculate grid cell area try: @@ -182,6 +186,7 @@ def _try_adding_calculated_ocean_volume(cube: Cube) -> None: def volume_statistics( cube: Cube, operator: str, + normalize: Optional[Literal['subtract', 'divide']] = None, **operator_kwargs, ) -> Cube: """Apply a statistical operation over a volume. @@ -201,6 +206,11 @@ def volume_statistics( The operation. Used to determine the :class:`iris.analysis.Aggregator` object used to calculate the statistics. Currently, only `mean` is allowed. + normalize: + If given, do not return the statistics cube itself, but rather, the + input cube, normalized with the statistics cube. Can either be + `subtract` (statistics cube is subtracted from the input cube) or + `divide` (input cube is divided by the statistics cube). **operator_kwargs: Optional keyword arguments for the :class:`iris.analysis.Aggregator` object defined by `operator`. @@ -232,6 +242,8 @@ def volume_statistics( agg, **agg_kwargs, ) + if normalize is not None: + result = get_normalized_cube(cube, result, normalize) # Make sure input cube has not been modified if not has_cell_measure and cube.cell_measures('ocean_volume'): @@ -244,6 +256,7 @@ def axis_statistics( cube: Cube, axis: str, operator: str, + normalize: Optional[Literal['subtract', 'divide']] = None, **operator_kwargs, ) -> Cube: """Perform statistics along a given axis. @@ -267,6 +280,11 @@ def axis_statistics( The operation. Used to determine the :class:`iris.analysis.Aggregator` object used to calculate the statistics. Allowed options are given in :ref:`this table `. + normalize: + If given, do not return the statistics cube itself, but rather, the + input cube, normalized with the statistics cube. Can either be + `subtract` (statistics cube is subtracted from the input cube) or + `divide` (input cube is divided by the statistics cube). **operator_kwargs: Optional keyword arguments for the :class:`iris.analysis.Aggregator` object defined by `operator`. @@ -320,6 +338,9 @@ def axis_statistics( ) result = cube.collapsed(coord, agg, **agg_kwargs) + if normalize is not None: + result = get_normalized_cube(cube, result, normalize) + # Make sure input and output cubes do not have auxiliary coordinate if cube.coords('_axis_statistics_weights_'): cube.remove_coord('_axis_statistics_weights_') diff --git a/esmvalcore/preprocessor/regrid_schemes.py b/esmvalcore/preprocessor/regrid_schemes.py new file mode 100644 index 0000000000..13dee49a0d --- /dev/null +++ b/esmvalcore/preprocessor/regrid_schemes.py @@ -0,0 +1,121 @@ +"""Regridding schemes.""" +from __future__ import annotations + +import logging +from collections.abc import Callable + +from iris.cube import Cube + +from esmvalcore.preprocessor._regrid_esmpy import ( + ESMPyAreaWeighted, + ESMPyLinear, + ESMPyNearest, + ESMPyRegridder, +) +from esmvalcore.preprocessor._regrid_unstructured import UnstructuredNearest + +logger = logging.getLogger(__name__) + + +__all__ = [ + 'ESMPyAreaWeighted', + 'ESMPyLinear', + 'ESMPyNearest', + 'ESMPyRegridder', + 'GenericFuncScheme', + 'GenericRegridder', + 'UnstructuredNearest', +] + + +class GenericRegridder: + r"""Generic function regridder. + + Parameters + ---------- + src_cube: + Cube defining the source grid. + tgt_cube: + Cube defining the target grid. + func: + Generic regridding function with signature f(src_cube: Cube, grid_cube: + Cube, \*\*kwargs) -> Cube. + **kwargs: + Keyword arguments for the generic regridding function. + + """ + + def __init__( + self, + src_cube: Cube, + tgt_cube: Cube, + func: Callable, + **kwargs, + ): + """Initialize class instance.""" + self.src_cube = src_cube + self.tgt_cube = tgt_cube + self.func = func + self.kwargs = kwargs + + def __call__(self, cube: Cube) -> Cube: + """Perform regridding. + + Parameters + ---------- + cube: + Cube to be regridded. + + Returns + ------- + Cube + Regridded cube. + + """ + return self.func(cube, self.tgt_cube, **self.kwargs) + + +class GenericFuncScheme: + r"""Regridding with a generic function. + + This class can be used in :meth:`iris.cube.Cube.regrid`. + + Does support lazy regridding if `func` does. + + Parameters + ---------- + func: + Generic regridding function with signature f(src_cube: Cube, grid_cube: + Cube, \*\*kwargs) -> Cube. + **kwargs: + Keyword arguments for the generic regridding function. + + """ + + def __init__(self, func: Callable, **kwargs): + """Initialize class instance.""" + self.func = func + self.kwargs = kwargs + + def __repr__(self) -> str: + """Return string representation of class.""" + kwargs = ', '.join(f"{k}={v}" for (k, v) in self.kwargs.items()) + return f'GenericFuncScheme({self.func.__name__}, {kwargs})' + + def regridder(self, src_cube: Cube, tgt_cube: Cube) -> GenericRegridder: + """Get regridder. + + Parameters + ---------- + src_cube: + Cube defining the source grid. + tgt_cube: + Cube defining the target grid. + + Returns + ------- + GenericRegridder + Regridder instance. + + """ + return GenericRegridder(src_cube, tgt_cube, self.func, **self.kwargs) diff --git a/setup.py b/setup.py index 84379ea008..1fe1f9278d 100755 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ 'netCDF4', 'numpy!=1.24.3', 'packaging', - 'pandas', + 'pandas!=2.2.0', # github.com/ESMValGroup/ESMValCore/pull/2305 'pillow', 'prov', 'psutil', @@ -58,7 +58,7 @@ 'scipy>=1.6', # See the following issue for info on the iris pin below: # https://github.com/ESMValGroup/ESMValTool/issues/3239#issuecomment-1613298587 - 'scitools-iris>=3.4.0', + 'scitools-iris>=3.6.1', 'shapely>=2.0.0', 'stratify>=0.3', 'yamale', @@ -71,7 +71,7 @@ 'pytest-env', 'pytest-html!=2.1.0', 'pytest-metadata>=1.5.1', - 'pytest-mypy', + 'pytest-mypy>=0.10.3', # gh issue/2314 'pytest-mock', 'pytest-xdist', 'ESMValTool_sample_data==0.0.3', diff --git a/tests/integration/cmor/_fixes/cmip5/test_hadgem2_es.py b/tests/integration/cmor/_fixes/cmip5/test_hadgem2_es.py index c0ab2bb8fd..17d928e574 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_hadgem2_es.py +++ b/tests/integration/cmor/_fixes/cmip5/test_hadgem2_es.py @@ -1,10 +1,16 @@ """Test HADGEM2-ES fixes.""" import unittest +import dask.array as da +import iris.coords +import iris.cube +import numpy as np + from esmvalcore.cmor._fixes.cmip5.hadgem2_es import O2, AllVars, Cl from esmvalcore.cmor._fixes.common import ClFixHybridHeightCoord from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix +from tests import assert_array_equal class TestAllVars(unittest.TestCase): @@ -16,6 +22,30 @@ def test_get(self): Fix.get_fixes('CMIP5', 'HADGEM2-ES', 'Amon', 'tas'), [AllVars(None), GenericFix(None)]) + @staticmethod + def test_clip_latitude(): + cube = iris.cube.Cube( + da.arange(2, dtype=np.float32), + aux_coords_and_dims=[ + ( + iris.coords.AuxCoord( + da.asarray([90., 91.]), + bounds=da.asarray([[89.5, 90.5], [90.5, 91.5]]), + standard_name='latitude', + ), + 0, + ), + ], + ) + fix = AllVars(None) + cubes = fix.fix_metadata([cube]) + assert len(cubes) == 1 + coord = cubes[0].coord('latitude') + assert coord.has_lazy_points() + assert coord.has_lazy_bounds() + assert_array_equal(coord.points, np.array([90., 90])) + assert_array_equal(coord.bounds, np.array([[89.5, 90.], [90., 90.]])) + class TestO2(unittest.TestCase): """Test o2 fixes.""" diff --git a/tests/integration/cmor/_fixes/cmip6/test_ciesm.py b/tests/integration/cmor/_fixes/cmip6/test_ciesm.py index 1cf1aa3aad..53fc57723b 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_ciesm.py +++ b/tests/integration/cmor/_fixes/cmip6/test_ciesm.py @@ -1,5 +1,6 @@ """Tests for the fixes of CIESM.""" import iris.cube +import numpy as np import pytest from esmvalcore.cmor._fixes.cmip6.ciesm import Cl @@ -38,8 +39,17 @@ def test_cl_fix_data(cl_cube): assert out_cube.data == [100.0] +def test_clt_fix(): + """Test `Clt.fix_data`.""" + cube = iris.cube.Cube(0.5) + fix = Fix.get_fixes('CMIP6', 'CIESM', 'Amon', 'clt')[0] + out_cube = fix.fix_data(cube) + np.testing.assert_allclose(out_cube.data, 50.0) + assert out_cube.units == '%' + + def test_pr_fix(): - """Test `Pr.fix_metadata`.""" + """Test `Pr.fix_data`.""" cube = iris.cube.Cube( [2.82e-08], var_name='pr', diff --git a/tests/integration/cmor/_fixes/cmip6/test_fio_esm_2_0.py b/tests/integration/cmor/_fixes/cmip6/test_fio_esm_2_0.py index 4ac1d05d4c..2bf811e455 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_fio_esm_2_0.py +++ b/tests/integration/cmor/_fixes/cmip6/test_fio_esm_2_0.py @@ -11,6 +11,15 @@ from esmvalcore.cmor.table import get_var_info +def test_clt_fix(): + """Test `Clt.fix_data`.""" + cube = iris.cube.Cube(0.5) + fix = Fix.get_fixes('CMIP6', 'FIO-ESM-2-0', 'Amon', 'clt')[0] + out_cube = fix.fix_data(cube) + np.testing.assert_allclose(out_cube.data, 50.0) + assert out_cube.units == '%' + + def test_get_tas_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'FIO-ESM-2-0', 'Amon', 'tas') diff --git a/tests/integration/cmor/_fixes/cmip6/test_gfdl_cm4.py b/tests/integration/cmor/_fixes/cmip6/test_gfdl_cm4.py index f28310c6c2..46b4424f52 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_gfdl_cm4.py +++ b/tests/integration/cmor/_fixes/cmip6/test_gfdl_cm4.py @@ -10,11 +10,14 @@ Fgco2, Omon, Siconc, + Tas, + Uas, ) from esmvalcore.cmor._fixes.common import SiconcFixScalarCoord from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix from esmvalcore.cmor.table import get_var_info +from cf_units import Unit def test_get_cl_fix(): @@ -116,3 +119,103 @@ def test_get_siconc_fix(): def test_siconc_fix(): """Test fix for ``siconc``.""" assert Siconc is SiconcFixScalarCoord + + +@pytest.fixture +def tas_cubes(): + correct_lat_coord = iris.coords.DimCoord([0.0], + var_name='lat', + standard_name='latitude') + wrong_lat_coord = iris.coords.DimCoord([0.0], + var_name='latitudeCoord', + standard_name='latitude') + correct_lon_coord = iris.coords.DimCoord([0.0], + var_name='lon', + standard_name='longitude') + wrong_lon_coord = iris.coords.DimCoord([0.0], + var_name='longitudeCoord', + standard_name='longitude') + correct_cube = iris.cube.Cube([[2.0]], + var_name='tas', + dim_coords_and_dims=[(correct_lat_coord, 0), + (correct_lon_coord, 1)]) + wrong_cube = iris.cube.Cube([[2.0]], + var_name='ta', + dim_coords_and_dims=[(wrong_lat_coord, 0), + (wrong_lon_coord, 1)]) + scalar_cube = iris.cube.Cube(0.0, var_name='ps') + return iris.cube.CubeList([correct_cube, wrong_cube, scalar_cube]) + + +def test_get_tas_fix(): + fixes = Fix.get_fixes('CMIP6', 'GFDL-CM4', 'day', 'tas') + assert Tas(None) in fixes + + +def test_tas_fix_metadata(tas_cubes): + for cube in tas_cubes: + with pytest.raises(iris.exceptions.CoordinateNotFoundError): + cube.coord('height') + height_coord = iris.coords.AuxCoord(2.0, + var_name='height', + standard_name='height', + long_name='height', + units=Unit('m'), + attributes={'positive': 'up'}) + vardef = get_var_info('CMIP6', 'day', 'tas') + fix = Tas(vardef) + + out_cubes = fix.fix_metadata(tas_cubes) + assert out_cubes[0].var_name == 'tas' + coord = out_cubes[0].coord('height') + assert coord == height_coord + + +@pytest.fixture +def uas_cubes(): + correct_lat_coord = iris.coords.DimCoord([0.0], + var_name='lat', + standard_name='latitude') + wrong_lat_coord = iris.coords.DimCoord([0.0], + var_name='latitudeCoord', + standard_name='latitude') + correct_lon_coord = iris.coords.DimCoord([0.0], + var_name='lon', + standard_name='longitude') + wrong_lon_coord = iris.coords.DimCoord([0.0], + var_name='longitudeCoord', + standard_name='longitude') + correct_cube = iris.cube.Cube([[10.0]], + var_name='uas', + dim_coords_and_dims=[(correct_lat_coord, 0), + (correct_lon_coord, 1)]) + wrong_cube = iris.cube.Cube([[10.0]], + var_name='ua', + dim_coords_and_dims=[(wrong_lat_coord, 0), + (wrong_lon_coord, 1)]) + scalar_cube = iris.cube.Cube(0.0, var_name='ps') + return iris.cube.CubeList([correct_cube, wrong_cube, scalar_cube]) + + +def test_get_uas_fix(): + fixes = Fix.get_fixes('CMIP6', 'GFDL-CM4', 'day', 'uas') + assert Uas(None) in fixes + + +def test_uas_fix_metadata(uas_cubes): + for cube in uas_cubes: + with pytest.raises(iris.exceptions.CoordinateNotFoundError): + cube.coord('height') + height_coord = iris.coords.AuxCoord(10.0, + var_name='height', + standard_name='height', + long_name='height', + units=Unit('m'), + attributes={'positive': 'up'}) + vardef = get_var_info('CMIP6', 'day', 'uas') + fix = Uas(vardef) + + out_cubes = fix.fix_metadata(uas_cubes) + assert out_cubes[0].var_name == 'uas' + coord = out_cubes[0].coord('height') + assert coord == height_coord diff --git a/tests/integration/cmor/_fixes/cmip6/test_mpi_esm1_2_hr.py b/tests/integration/cmor/_fixes/cmip6/test_mpi_esm1_2_hr.py index 777434d51c..3a9dac1fb6 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_mpi_esm1_2_hr.py +++ b/tests/integration/cmor/_fixes/cmip6/test_mpi_esm1_2_hr.py @@ -1,9 +1,19 @@ """Test fixes for MPI-ESM1-2-HR.""" import iris - -from esmvalcore.cmor._fixes.cmip6.mpi_esm1_2_hr import AllVars, Cl, Cli, Clw +import pytest +from cf_units import Unit + +from esmvalcore.cmor._fixes.cmip6.mpi_esm1_2_hr import ( + AllVars, + Cl, + Cli, + Clw, + SfcWind, + Tas, +) from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor.table import get_var_info def test_get_allvars_fix(): @@ -42,6 +52,58 @@ def test_allvars_r2i1p1f1(): assert fixed_cubes[1].coord('latitude').points[0] == -86.49036676628 +@pytest.fixture +def sfcwind_cubes(): + correct_lat_coord = iris.coords.DimCoord([0.0], + var_name='lat', + standard_name='latitude') + wrong_lat_coord = iris.coords.DimCoord([0.0], + var_name='latitudeCoord', + standard_name='latitude') + correct_lon_coord = iris.coords.DimCoord([0.0], + var_name='lon', + standard_name='longitude') + wrong_lon_coord = iris.coords.DimCoord([0.0], + var_name='longitudeCoord', + standard_name='longitude') + correct_cube = iris.cube.Cube([[10.0]], var_name='sfcWind', + dim_coords_and_dims=[(correct_lat_coord, 0), + (correct_lon_coord, 1)] + ) + wrong_cube = iris.cube.Cube([[10.0]], + var_name='ta', + dim_coords_and_dims=[(wrong_lat_coord, 0), + (wrong_lon_coord, 1)]) + scalar_cube = iris.cube.Cube(0.0, var_name='ps') + return iris.cube.CubeList([correct_cube, wrong_cube, scalar_cube]) + + +@pytest.fixture +def tas_cubes(): + correct_lat_coord = iris.coords.DimCoord([0.0], + var_name='lat', + standard_name='latitude') + wrong_lat_coord = iris.coords.DimCoord([0.0], + var_name='latitudeCoord', + standard_name='latitude') + correct_lon_coord = iris.coords.DimCoord([0.0], + var_name='lon', + standard_name='longitude') + wrong_lon_coord = iris.coords.DimCoord([0.0], + var_name='longitudeCoord', + standard_name='longitude') + correct_cube = iris.cube.Cube([[10.0]], + var_name='tas', + dim_coords_and_dims=[(correct_lat_coord, 0), + (correct_lon_coord, 1)]) + wrong_cube = iris.cube.Cube([[10.0]], + var_name='ta', + dim_coords_and_dims=[(wrong_lat_coord, 0), + (wrong_lon_coord, 1)]) + scalar_cube = iris.cube.Cube(0.0, var_name='ps') + return iris.cube.CubeList([correct_cube, wrong_cube, scalar_cube]) + + def test_get_cl_fix(): """Test getting of fix.""" fixes = Fix.get_fixes('CMIP6', 'MPI-ESM1-2-HR', 'Amon', 'cl') @@ -73,3 +135,51 @@ def test_get_clw_fix(): def test_clw_fix(): """Test fix for ``clw``.""" assert Clw is ClFixHybridPressureCoord + + +def test_get_sfcwind_fix(): + fixes = Fix.get_fixes('CMIP6', 'MPI_ESM1_2_HR', 'day', 'sfcWind') + assert SfcWind(None) in fixes + + +def test_sfcwind_fix_metadata(sfcwind_cubes): + for cube in sfcwind_cubes: + with pytest.raises(iris.exceptions.CoordinateNotFoundError): + cube.coord('height') + height_coord = iris.coords.AuxCoord(10.0, + var_name='height', + standard_name='height', + long_name='height', + units=Unit('m'), + attributes={'positive': 'up'}) + vardef = get_var_info('CMIP6', 'day', 'sfcWind') + fix = SfcWind(vardef) + + out_cubes = fix.fix_metadata(sfcwind_cubes) + assert out_cubes[0].var_name == 'sfcWind' + coord = out_cubes[0].coord('height') + assert coord == height_coord + + +def test_get_tas_fix(): + fixes = Fix.get_fixes('CMIP6', 'MPI_ESM1_2_HR', 'day', 'tas') + assert Tas(None) in fixes + + +def test_tas_fix_metadata(tas_cubes): + for cube in tas_cubes: + with pytest.raises(iris.exceptions.CoordinateNotFoundError): + cube.coord('height') + height_coord = iris.coords.AuxCoord(2.0, + var_name='height', + standard_name='height', + long_name='height', + units=Unit('m'), + attributes={'positive': 'up'}) + vardef = get_var_info('CMIP6', 'day', 'tas') + fix = Tas(vardef) + + out_cubes = fix.fix_metadata(tas_cubes) + assert out_cubes[0].var_name == 'tas' + coord = out_cubes[0].coord('height') + assert coord == height_coord diff --git a/tests/integration/cmor/test_read_cmor_tables.py b/tests/integration/cmor/test_read_cmor_tables.py index 32a472b316..a77b9b2946 100644 --- a/tests/integration/cmor/test_read_cmor_tables.py +++ b/tests/integration/cmor/test_read_cmor_tables.py @@ -1,4 +1,5 @@ from pathlib import Path +from textwrap import dedent import pytest import yaml @@ -7,17 +8,6 @@ from esmvalcore.cmor.table import __file__ as root from esmvalcore.cmor.table import read_cmor_tables -CUSTOM_CFG_DEVELOPER = { - 'custom': {'cmor_path': Path(root).parent / 'tables' / 'custom'}, - 'CMIP6': { - 'cmor_strict': True, - 'input_dir': {'default': '/'}, - 'input_file': '*.nc', - 'output_file': 'out.nc', - 'cmor_type': 'CMIP6', - }, -} - def test_read_cmor_tables_raiser(): """Test func raiser.""" @@ -52,12 +42,108 @@ def test_read_cmor_tables(): assert Path(table._cmor_folder) == table_path / 'obs4mips' / 'Tables' assert table.strict is False + project = 'custom' + table = CMOR_TABLES[project] + assert Path(table._cmor_folder) == table_path / 'custom' + assert table._user_table_folder is None + assert table.coords + assert table.tables['custom'] + + +CMOR_NEWVAR_ENTRY = dedent( + """ + !============ + variable_entry: newvarfortesting + !============ + modeling_realm: atmos + !---------------------------------- + ! Variable attributes: + !---------------------------------- + standard_name: + units: kg s m A + cell_methods: time: mean + cell_measures: area: areacella + long_name: Custom Variable for Testing + !---------------------------------- + ! Additional variable information: + !---------------------------------- + dimensions: longitude latitude time + type: real + positive: up + !---------------------------------- + ! + """ +) +CMOR_NETCRE_ENTRY = dedent( + """ + !============ + variable_entry: netcre + !============ + modeling_realm: atmos + !---------------------------------- + ! Variable attributes: + !---------------------------------- + standard_name: air_temperature ! for testing + units: K ! for testing + cell_methods: time: mean + cell_measures: area: areacella + long_name: This is New ! for testing + !---------------------------------- + ! Additional variable information: + !---------------------------------- + dimensions: longitude latitude time + type: real + positive: up + !---------------------------------- + ! + """ +) +CMOR_NEWCOORD_ENTRY = dedent( + """ + !============ + axis_entry: newcoordfortesting + !============ + !---------------------------------- + ! Axis attributes: + !---------------------------------- + standard_name: + units: kg + axis: Y ! X, Y, Z, T (default: undeclared) + long_name: Custom Coordinate for Testing + !---------------------------------- + ! Additional axis information: + !---------------------------------- + out_name: newcoordfortesting + valid_min: -90.0 + valid_max: 90.0 + stored_direction: increasing + type: double + must_have_bounds: yes + !---------------------------------- + ! + """ +) + def test_read_custom_cmor_tables(tmp_path): """Test reading of custom CMOR tables.""" + (tmp_path / 'CMOR_newvarfortesting.dat').write_text(CMOR_NEWVAR_ENTRY) + (tmp_path / 'CMOR_netcre.dat').write_text(CMOR_NETCRE_ENTRY) + (tmp_path / 'CMOR_coordinates.dat').write_text(CMOR_NEWCOORD_ENTRY) + + custom_cfg_developer = { + 'custom': {'cmor_path': str(tmp_path)}, + 'CMIP6': { + 'cmor_strict': True, + 'input_dir': {'default': '/'}, + 'input_file': '*.nc', + 'output_file': 'out.nc', + 'cmor_type': 'CMIP6', + }, + } cfg_file = tmp_path / 'config-developer.yml' with cfg_file.open('w', encoding='utf-8') as file: - yaml.safe_dump(CUSTOM_CFG_DEVELOPER, file) + yaml.safe_dump(custom_cfg_developer, file) read_cmor_tables(cfg_file) @@ -66,10 +152,23 @@ def test_read_custom_cmor_tables(tmp_path): assert 'custom' in CMOR_TABLES custom_table = CMOR_TABLES['custom'] - assert (Path(custom_table._cmor_folder) == - Path(root).parent / 'tables' / 'custom') - assert (Path(custom_table._coordinates_file) == - Path(root).parent / 'tables' / 'custom' / 'CMOR_coordinates.dat') + assert ( + custom_table._cmor_folder == + str(Path(root).parent / 'tables' / 'custom') + ) + assert custom_table._user_table_folder == str(tmp_path) + + # Make sure that default tables have been read + assert 'alb' in custom_table.tables['custom'] + assert 'latitude' in custom_table.coords + + # Make sure that custom tables have been read + assert 'newvarfortesting' in custom_table.tables['custom'] + assert 'newcoordfortesting' in custom_table.coords + netcre = custom_table.get_variable('custom', 'netcre') + assert netcre.standard_name == 'air_temperature' + assert netcre.units == 'K' + assert netcre.long_name == 'This is New' cmip6_table = CMOR_TABLES['CMIP6'] assert cmip6_table.default is custom_table diff --git a/tests/integration/cmor/test_table.py b/tests/integration/cmor/test_table.py index 8b8d63e57c..385869bc5f 100644 --- a/tests/integration/cmor/test_table.py +++ b/tests/integration/cmor/test_table.py @@ -105,6 +105,11 @@ def test_get_variable_from_alt_names(self): var = self.variables_info.get_variable('SImon', 'sic') self.assertEqual(var.short_name, 'siconc') + def test_get_variable_derived(self): + """Test that derived variable are looked up from other MIP tables.""" + var = self.variables_info.get_variable('3hr', 'sfcWind', derived=True) + self.assertEqual(var.short_name, 'sfcWind') + def test_get_variable_from_custom(self): """Get a variable from default.""" self.variables_info.strict = False @@ -411,32 +416,23 @@ def test_custom_tables_default_location(self): 'tables', 'custom', ) - expected_coordinate_file = os.path.join( - os.path.dirname(esmvalcore.cmor.__file__), - 'tables', - 'custom', - 'CMOR_coordinates.dat', - ) self.assertEqual(custom_info._cmor_folder, expected_cmor_folder) - self.assertEqual(custom_info._coordinates_file, - expected_coordinate_file) + self.assertTrue(custom_info.tables['custom']) + self.assertTrue(custom_info.coords) def test_custom_tables_location(self): """Test constructor with custom tables location.""" cmor_path = os.path.dirname(os.path.realpath(esmvalcore.cmor.__file__)) + default_cmor_tables_path = os.path.join(cmor_path, 'tables', 'custom') cmor_tables_path = os.path.join(cmor_path, 'tables', 'cmip5') cmor_tables_path = os.path.abspath(cmor_tables_path) + custom_info = CustomInfo(cmor_tables_path) - self.assertEqual(custom_info._cmor_folder, cmor_tables_path) - expected_coordinate_file = os.path.join( - os.path.dirname(esmvalcore.cmor.__file__), - 'tables', - 'custom', - 'CMOR_coordinates.dat', - ) - self.assertEqual(custom_info._coordinates_file, - expected_coordinate_file) + self.assertEqual(custom_info._cmor_folder, default_cmor_tables_path) + self.assertEqual(custom_info._user_table_folder, cmor_tables_path) + self.assertTrue(custom_info.tables['custom']) + self.assertTrue(custom_info.coords) def test_custom_tables_invalid_location(self): """Test constructor with invalid custom tables location.""" diff --git a/tests/integration/data_finder.yml b/tests/integration/data_finder.yml index 3724154d64..40e0c3e821 100644 --- a/tests/integration/data_finder.yml +++ b/tests/integration/data_finder.yml @@ -452,6 +452,84 @@ get_input_filelist: - ta_Amon_HadGEM2-ES_historical_r1i1p1*.nc found_files: - historical/Amon/ta/HadGEM2-ES/r1i1p1/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc + + - drs: NCI + variable: + <<: *variable + timerange: '1980/2002' + available_files: + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110329/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_193412-195911.nc + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110329/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_195912-198411.nc + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110329/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_193412-195911.nc + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_195912-198411.nc + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc + dirs: + - INPE/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/*/ta + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/*/ta + file_patterns: + - ta_Amon_HadGEM2-ES_historical_r1i1p1*.nc + found_files: + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_195912-198411.nc + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc + + - drs: NCI + variable: + <<: *variable + timerange: '2000/2005' + version: v20110329 + available_files: + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110329/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc + dirs: + - INPE/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110329/ta + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110329/ta + file_patterns: + - ta_Amon_HadGEM2-ES_historical_r1i1p1*.nc + found_files: + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110329/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc + + - drs: NCI + variable: + <<: *variable + ensemble: '*' + timerange: '2000/2005' + available_files: + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110329/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r2i1p1/v20110329/ta/ta_Amon_HadGEM2-ES_historical_r2i1p1_198412-200511.nc + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r2i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r2i1p1_198412-200511.nc + dirs: + - INPE/HadGEM2-ES/historical/mon/atmos/Amon/*/*/ta + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/*/*/ta + file_patterns: + - ta_Amon_HadGEM2-ES_historical_**.nc + found_files: + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r2i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r2i1p1_198412-200511.nc + + - drs: NCI + variable: + <<: *variable + timerange: '1980/2002' + available_files: + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110329/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_193412-195911.nc + - MOHC/HadGEM2-ES/historical/mon/atmo1/Amon/r1i1p1/v20110329/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_195912-198411.nc + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110329/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_193412-195911.nc + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_195912-198411.nc + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc + available_symlinks: + - link_name: MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/latest + target: v20120928 + dirs: + - INPE/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/*/ta + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/*/ta + file_patterns: + - ta_Amon_HadGEM2-ES_historical_r1i1p1*.nc + found_files: + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_195912-198411.nc + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc # Test other projects @@ -515,6 +593,38 @@ get_input_filelist: found_files: - historical/atmos/mon/ta/HADGEM1/r1i1p1/ta_HADGEM1_195001-199912.nc - historical/atmos/mon/ta/HADGEM1/r1i1p1/ta_HADGEM1_200001-200112.nc + + - drs: NCI + variable: + variable_group: test + short_name: ta + original_short_name: ta + dataset: HadGEM3-GC31-LL + activity: CMIP + project: CMIP6 + institute: [MOHC, NERC] + frequency: mon + modeling_realm: [atmos] + mip: Amon + exp: historical + grid: gn + ensemble: r1i1p1f1 + timerange: '1999/2000' + diagnostic: test_diag + preprocessor: test_preproc + available_files: + - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Amon/tas/gn/v20200101/tas_Amon_HadGEM3-GC31-LL_historical_r1i1p1f1_gn_190001-194912.nc + - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Amon/ta/gn/v20200101/ta_Amon_HadGEM3-GC31-LL_historical_r1i1p1f1_gn_190001-194912.nc + - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Amon/ta/gn/v20200101/ta_Amon_HadGEM3-GC31-LL_historical_r1i1p1f1_gn_195001-199912.nc + - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Amon/ta/gn/v20200101/ta_Amon_HadGEM3-GC31-LL_historical_r1i1p1f1_gn_200001-201412.nc + dirs: + - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Amon/ta/gn/*/ + - CMIP/NERC/HadGEM3-GC31-LL/historical/r1i1p1f1/Amon/ta/gn/*/ + file_patterns: + - ta_Amon_HadGEM3-GC31-LL_historical_r1i1p1f1_gn*.nc + found_files: + - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Amon/ta/gn/v20200101/ta_Amon_HadGEM3-GC31-LL_historical_r1i1p1f1_gn_195001-199912.nc + - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Amon/ta/gn/v20200101/ta_Amon_HadGEM3-GC31-LL_historical_r1i1p1f1_gn_200001-201412.nc - drs: default variable: diff --git a/tests/integration/preprocessor/_derive/test_sispeed.py b/tests/integration/preprocessor/_derive/test_sispeed.py index 3bb8188ac1..9daee38954 100644 --- a/tests/integration/preprocessor/_derive/test_sispeed.py +++ b/tests/integration/preprocessor/_derive/test_sispeed.py @@ -4,9 +4,8 @@ from unittest import mock import numpy as np - -from iris.cube import Cube, CubeList from iris.coords import AuxCoord +from iris.cube import Cube, CubeList from esmvalcore.preprocessor._derive.sispeed import DerivedVariable @@ -23,7 +22,9 @@ def get_cube(name, lat=((0.5, 1.5), (2.5, 3.5)), lon=((0.5, 1.5), (2.5, 3.5))): @mock.patch( - 'esmvalcore.preprocessor._regrid.esmpy_regrid', autospec=True) + 'esmvalcore.preprocessor._regrid_esmpy.ESMPyRegridder.__call__', + autospec=True, +) def test_sispeed_calculation(mock_regrid): """Test calculation of `sispeed.""" siu = get_cube('sea_ice_x_velocity') @@ -37,7 +38,9 @@ def test_sispeed_calculation(mock_regrid): @mock.patch( - 'esmvalcore.preprocessor._regrid.esmpy_regrid', autospec=True) + 'esmvalcore.preprocessor._regrid_esmpy.ESMPyRegridder.__call__', + autospec=True, +) def test_sispeed_calculation_coord_differ(mock_regrid): """Test calculation of `sispeed.""" siu = get_cube('sea_ice_x_velocity') diff --git a/tests/integration/preprocessor/_regrid/test_regrid.py b/tests/integration/preprocessor/_regrid/test_regrid.py index de7a20749d..c774433a42 100644 --- a/tests/integration/preprocessor/_regrid/test_regrid.py +++ b/tests/integration/preprocessor/_regrid/test_regrid.py @@ -10,6 +10,7 @@ from numpy import ma from esmvalcore.dataset import Dataset +from esmvalcore.exceptions import ESMValCoreDeprecationWarning from esmvalcore.preprocessor import regrid from tests import assert_array_equal from tests.unit.preprocessor._regrid import _make_cube @@ -54,42 +55,56 @@ def setUp(self): units='degrees_north', coord_system=self.cs) coords_spec = [(lats, 0), (lons, 1)] - self.grid_for_unstructured_nearest = iris.cube.Cube( + self.tgt_grid_for_unstructured = iris.cube.Cube( data, dim_coords_and_dims=coords_spec) - # Replace 1d spatial coords with 2d spatial coords. lons = self.cube.coord('longitude') lats = self.cube.coord('latitude') x, y = np.meshgrid(lons.points, lats.points) lats = iris.coords.AuxCoord( - y, + y.ravel(), standard_name=lats.metadata.standard_name, long_name=lats.metadata.long_name, var_name=lats.metadata.var_name, units=lats.metadata.units, attributes=lats.metadata.attributes, coord_system=lats.metadata.coord_system, - climatological=lats.metadata.climatological) + climatological=lats.metadata.climatological, + ) lons = iris.coords.AuxCoord( - x, + x.ravel(), standard_name=lons.metadata.standard_name, long_name=lons.metadata.long_name, var_name=lons.metadata.var_name, units=lons.metadata.units, attributes=lons.metadata.attributes, coord_system=lons.metadata.coord_system, - climatological=lons.metadata.climatological) - - self.unstructured_grid_cube = self.cube.copy() - self.unstructured_grid_cube.remove_coord('longitude') - self.unstructured_grid_cube.remove_coord('latitude') - self.unstructured_grid_cube.remove_coord('Pressure Slice') - self.unstructured_grid_cube.add_aux_coord(lons, (1, 2)) - self.unstructured_grid_cube.add_aux_coord(lats, (1, 2)) - self.unstructured_grid_cube.data = np.ma.masked_less( - self.cube.data.astype(np.float32), 3.5 + climatological=lons.metadata.climatological, + ) + + unstructured_data = np.ma.masked_less( + self.cube.data.reshape(3, 4).astype(np.float32), 3.5 + ) + + self.unstructured_grid_cube = iris.cube.Cube( + unstructured_data, + dim_coords_and_dims=[(self.cube.coord('air_pressure'), 0)], + aux_coords_and_dims=[(lats, 1), (lons, 1)], + ) + self.unstructured_grid_cube.metadata = self.cube.metadata + + # Setup irregular cube and grid + lons_2d = iris.coords.AuxCoord( + [[0, 1]], standard_name='longitude', units='degrees_east' + ) + lats_2d = iris.coords.AuxCoord( + [[0, 1]], standard_name='latitude', units='degrees_north' + ) + self.irregular_grid = iris.cube.Cube( + [[1, 1]], + aux_coords_and_dims=[(lats_2d, (0, 1)), (lons_2d, (0, 1))], ) def test_regrid__linear(self): @@ -155,7 +170,7 @@ def test_regrid__linear_do_not_preserve_dtype(self): assert np.issubdtype(self.cube.dtype, np.integer) assert np.issubdtype(result.dtype, np.floating) - def test_regrid__linear_extrapolate(self): + def test_regrid__linear_with_extrapolation(self): data = np.empty((3, 3)) lons = iris.coords.DimCoord([0, 1.5, 3], standard_name='longitude', @@ -169,13 +184,17 @@ def test_regrid__linear_extrapolate(self): coord_system=self.cs) coords_spec = [(lats, 0), (lons, 1)] grid = iris.cube.Cube(data, dim_coords_and_dims=coords_spec) - result = regrid(self.cube, grid, 'linear_extrapolate') + scheme = { + 'reference': 'iris.analysis:Linear', + 'extrapolation_mode': 'extrapolate', + } + result = regrid(self.cube, grid, scheme) expected = [[[-3., -1.5, 0.], [0., 1.5, 3.], [3., 4.5, 6.]], [[1., 2.5, 4.], [4., 5.5, 7.], [7., 8.5, 10.]], [[5., 6.5, 8.], [8., 9.5, 11.], [11., 12.5, 14.]]] assert_array_equal(result.data, expected) - def test_regrid__linear_extrapolate_with_mask(self): + def test_regrid__linear_with_mask(self): data = np.empty((3, 3)) grid = iris.cube.Cube(data) lons = iris.coords.DimCoord([0, 1.5, 3], @@ -273,11 +292,11 @@ def test_regrid__esmf_area_weighted(self): expected = np.array([[[1.499886]], [[5.499886]], [[9.499886]]]) np.testing.assert_array_almost_equal(result.data, expected, decimal=6) - def test_regrid__unstructured_nearest_float(self): - """Test unstructured_nearest regridding with cube of floats.""" + def test_regrid_nearest_unstructured_grid_float(self): + """Test `nearest` regridding with unstructured cube of floats.""" result = regrid(self.unstructured_grid_cube, - self.grid_for_unstructured_nearest, - 'unstructured_nearest') + self.tgt_grid_for_unstructured, + 'nearest') expected = np.ma.array([[[3.0]], [[7.0]], [[11.0]]], mask=[[[True]], [[False]], [[False]]]) np.testing.assert_array_equal(result.data.mask, expected.mask) @@ -289,11 +308,74 @@ def test_regrid__unstructured_nearest_float(self): assert self.unstructured_grid_cube.dtype == np.float32 assert result.dtype == np.float32 - def test_regrid__unstructured_nearest_int(self): - """Test unstructured_nearest regridding with cube of ints.""" - self.unstructured_grid_cube.data = np.ones((3, 2, 2), dtype=int) + def test_regrid_nearest_unstructured_grid_int(self): + """Test `nearest` regridding with unstructured cube of ints.""" + self.unstructured_grid_cube.data = np.ones((3, 4), dtype=int) result = regrid(self.unstructured_grid_cube, - self.grid_for_unstructured_nearest, - 'unstructured_nearest') + self.tgt_grid_for_unstructured, + 'nearest') expected = np.array([[[1]], [[1]], [[1]]]) np.testing.assert_array_equal(result.data, expected) + + def test_invalid_scheme_for_unstructured_grid(self): + """Test invalid scheme for unstructured cube.""" + msg = ( + "Regridding scheme 'invalid' does not support unstructured data, " + ) + with pytest.raises(ValueError, match=msg): + regrid( + self.unstructured_grid_cube, + self.tgt_grid_for_unstructured, + 'invalid', + ) + + def test_invalid_scheme_for_irregular_grid(self): + """Test invalid scheme for irregular cube.""" + msg = ( + "Regridding scheme 'invalid' does not support irregular data, " + ) + with pytest.raises(ValueError, match=msg): + regrid( + self.irregular_grid, + self.tgt_grid_for_unstructured, + 'invalid', + ) + + def test_deprecate_unstrucured_nearest(self): + """Test deprecation of `unstructured_nearest` regridding scheme.""" + with pytest.warns(ESMValCoreDeprecationWarning): + result = regrid( + self.unstructured_grid_cube, + self.tgt_grid_for_unstructured, + 'unstructured_nearest', + ) + expected = np.ma.array( + [[[3.0]], [[7.0]], [[11.0]]], + mask=[[[True]], [[False]], [[False]]], + ) + np.testing.assert_array_equal(result.data.mask, expected.mask) + np.testing.assert_array_almost_equal(result.data, expected, decimal=6) + + def test_deprecate_linear_extrapolate(self): + """Test deprecation of `linear_extrapolate` regridding scheme.""" + data = np.empty((3, 3)) + lons = iris.coords.DimCoord([0, 1.5, 3], + standard_name='longitude', + bounds=[[0, 1], [1, 2], [2, 3]], + units='degrees_east', + coord_system=self.cs) + lats = iris.coords.DimCoord([0, 1.5, 3], + standard_name='latitude', + bounds=[[0, 1], [1, 2], [2, 3]], + units='degrees_north', + coord_system=self.cs) + coords_spec = [(lats, 0), (lons, 1)] + grid = iris.cube.Cube(data, dim_coords_and_dims=coords_spec) + + with pytest.warns(ESMValCoreDeprecationWarning): + result = regrid(self.cube, grid, 'linear_extrapolate') + + expected = [[[-3., -1.5, 0.], [0., 1.5, 3.], [3., 4.5, 6.]], + [[1., 2.5, 4.], [4., 5.5, 7.], [7., 8.5, 10.]], + [[5., 6.5, 8.], [8., 9.5, 11.], [11., 12.5, 14.]]] + assert_array_equal(result.data, expected) diff --git a/tests/integration/preprocessor/_regrid/test_regrid_schemes.py b/tests/integration/preprocessor/_regrid/test_regrid_schemes.py new file mode 100644 index 0000000000..b4dd039f44 --- /dev/null +++ b/tests/integration/preprocessor/_regrid/test_regrid_schemes.py @@ -0,0 +1,55 @@ +"""Integration tests for regrid schemes.""" +import numpy as np +import pytest +from iris.cube import Cube + +from esmvalcore.preprocessor.regrid_schemes import ( + GenericFuncScheme, + GenericRegridder, +) + + +def set_data_to_const(cube, _, const=1.0): + """Dummy function to test ``GenericFuncScheme``.""" + cube = cube.copy(np.full(cube.shape, const)) + return cube + + +@pytest.fixture +def generic_func_scheme(): + """Generic function scheme.""" + return GenericFuncScheme(set_data_to_const, const=2) + + +def test_generic_func_scheme_init(generic_func_scheme): + """Test ``GenericFuncScheme``.""" + assert generic_func_scheme.func == set_data_to_const + assert generic_func_scheme.kwargs == {'const': 2} + + +def test_generic_func_scheme_repr(generic_func_scheme): + """Test ``GenericFuncScheme``.""" + repr = generic_func_scheme.__repr__() + assert repr == 'GenericFuncScheme(set_data_to_const, const=2)' + + +def test_generic_func_scheme_regridder(generic_func_scheme, mocker): + """Test ``GenericFuncScheme``.""" + regridder = generic_func_scheme.regridder( + mocker.sentinel.src_cube, + mocker.sentinel.tgt_cube, + ) + assert isinstance(regridder, GenericRegridder) + assert regridder.src_cube == mocker.sentinel.src_cube + assert regridder.tgt_cube == mocker.sentinel.tgt_cube + assert regridder.func == set_data_to_const + assert regridder.kwargs == {'const': 2} + + +def test_generic_func_scheme_regrid(generic_func_scheme, mocker): + """Test ``GenericFuncScheme``.""" + cube = Cube([0.0, 0.0], var_name='x') + + result = cube.regrid(mocker.sentinel.tgt_grid, generic_func_scheme) + + assert result == Cube([2, 2], var_name='x') diff --git a/tests/integration/preprocessor/_regrid/test_regrid_unstructured.py b/tests/integration/preprocessor/_regrid/test_regrid_unstructured.py new file mode 100644 index 0000000000..98f454bda7 --- /dev/null +++ b/tests/integration/preprocessor/_regrid/test_regrid_unstructured.py @@ -0,0 +1,99 @@ +""" Integration tests for unstructured regridding.""" + +import numpy as np +import pytest +from iris.coords import AuxCoord, DimCoord +from iris.cube import Cube + +from esmvalcore.preprocessor._regrid import _global_stock_cube +from esmvalcore.preprocessor._regrid_unstructured import UnstructuredNearest + + +@pytest.fixture +def unstructured_grid_cube(): + """Sample cube with unstructured grid.""" + time = DimCoord( + [0.0, 1.0], standard_name='time', units='days since 1950-01-01' + ) + lat = AuxCoord( + [-50.0, -50.0, 20.0, 20.0], + standard_name='latitude', + units='degrees_north', + ) + lon = AuxCoord( + [70.0, 250.0, 250.0, 70.0], + standard_name='longitude', + units='degrees_east', + ) + cube = Cube( + np.array([[0.0, 1.0, 2.0, 3.0], [0.0, 0.0, 0.0, 0.0]]), + standard_name='air_temperature', + units='K', + dim_coords_and_dims=[(time, 0)], + aux_coords_and_dims=[(lat, 1), (lon, 1)], + ) + return cube + + +@pytest.fixture +def target_grid(): + """Sample cube with regular grid.""" + return _global_stock_cube('120x60') + + +class TestUnstructuredNearest: + """Test ``UnstructuredNearest``.""" + + def test_regridding(self, unstructured_grid_cube, target_grid): + """Test regridding.""" + src_cube = unstructured_grid_cube.copy() + + result = src_cube.regrid(target_grid, UnstructuredNearest()) + + assert src_cube == unstructured_grid_cube + assert result.shape == (2, 3, 3) + assert result.coord('time') == src_cube.coord('time') + assert result.coord('latitude') == target_grid.coord('latitude') + assert result.coord('longitude') == target_grid.coord('longitude') + np.testing.assert_allclose( + result.data, + [[[0.0, 1.0, 1.0], + [3.0, 2.0, 2.0], + [3.0, 2.0, 2.0]], + [[0.0, 0.0, 0.0], + [0.0, 0.0, 0.0], + [0.0, 0.0, 0.0]]], + ) + + def test_regridding_with_dim_coord( + self, + unstructured_grid_cube, + target_grid, + ): + """Test regridding.""" + src_cube = unstructured_grid_cube.copy() + dim_coord = DimCoord( + [0, 1, 2, 3], + var_name='x', + standard_name='grid_latitude', + ) + src_cube.add_dim_coord(dim_coord, 1) + assert src_cube != unstructured_grid_cube + + result = src_cube.regrid(target_grid, UnstructuredNearest()) + + assert src_cube == unstructured_grid_cube + assert not src_cube.coords('grid_latitude') + assert result.shape == (2, 3, 3) + assert result.coord('time') == src_cube.coord('time') + assert result.coord('latitude') == target_grid.coord('latitude') + assert result.coord('longitude') == target_grid.coord('longitude') + np.testing.assert_allclose( + result.data, + [[[0.0, 1.0, 1.0], + [3.0, 2.0, 2.0], + [3.0, 2.0, 2.0]], + [[0.0, 0.0, 0.0], + [0.0, 0.0, 0.0], + [0.0, 0.0, 0.0]]], + ) diff --git a/tests/integration/preprocessor/_time/__init__.py b/tests/integration/preprocessor/_time/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/preprocessor/_time/test_time.py b/tests/integration/preprocessor/_time/test_time.py new file mode 100644 index 0000000000..f1da8d45e7 --- /dev/null +++ b/tests/integration/preprocessor/_time/test_time.py @@ -0,0 +1,561 @@ +"""Unit tests for the :func:`esmvalcore.preprocessor._time` module.""" + +import dask.array as da +import numpy as np +import pytest +from cf_units import Unit +from iris.coords import ( + AncillaryVariable, + AuxCoord, + CellMeasure, + CellMethod, + DimCoord, +) +from iris.cube import Cube +from iris.exceptions import CoordinateMultiDimError, CoordinateNotFoundError + +from esmvalcore.preprocessor._time import climate_statistics, local_solar_time +from tests import assert_array_equal + + +@pytest.fixture +def easy_2d_cube(): + """Create easy 2D cube to test statistical operators.""" + time = DimCoord( + [2.0, 3.0], + bounds=[[-0.5, 2.5], [2.5, 3.5]], + standard_name='time', + units='days since 2000-01-01', + ) + lat = DimCoord( + [0.0, 1.0], standard_name='latitude', units='degrees' + ) + cube = Cube( + np.arange(4, dtype=np.float32).reshape(2, 2), + standard_name='air_temperature', + units='K', + dim_coords_and_dims=[(time, 0), (lat, 1)], + ) + return cube + + +@pytest.mark.parametrize( + 'operator,kwargs,expected_data,expected_units', + [ + ('gmean', {}, [0.0, 1.7320509], 'K'), + ('hmean', {}, [0.0, 1.5], 'K'), + ('max', {}, [2.0, 3.0], 'K'), + ('mean', {}, [0.5, 1.5], 'K'), + ('mean', {'weights': False}, [1.0, 2.0], 'K'), + ('median', {}, [1.0, 2.0], 'K'), + ('min', {}, [0.0, 1.0], 'K'), + ('peak', {}, [2.0, 3.0], 'K'), + ('percentile', {'percent': 0.0}, [0.0, 1.0], 'K'), + ('rms', {}, [1.0, 1.7320509], 'K'), + ('rms', {'weights': False}, [1.414214, 2.236068], 'K'), + ('std_dev', {}, [1.414214, 1.414214], 'K'), + ('std_dev', {'ddof': 0}, [1.0, 1.0], 'K'), + ('sum', {}, [2.0, 6.0], 'K day'), + ('sum', {'weights': False}, [2.0, 4.0], 'K'), + ('variance', {}, [2.0, 2.0], 'K2'), + ('variance', {'ddof': 0}, [1.0, 1.0], 'K2'), + ('wpercentile', {'percent': 50.0}, [0.5, 1.5], 'K'), + ] +) +def test_statistical_operators( + operator, kwargs, expected_data, expected_units, easy_2d_cube +): + """Test ``climate_statistics`` with different operators.""" + res = climate_statistics(easy_2d_cube, operator, **kwargs) + + assert res.var_name == easy_2d_cube.var_name + assert res.long_name == easy_2d_cube.long_name + assert res.standard_name == easy_2d_cube.standard_name + assert res.attributes == easy_2d_cube.attributes + assert res.units == expected_units + assert res.coord('latitude') == easy_2d_cube.coord('latitude') + assert res.coord('time').shape == (1, ) + np.testing.assert_allclose(res.data, expected_data, atol=1e-6, rtol=1e-6) + + +@pytest.fixture +def realistic_4d_cube(): + """Create realistic 4D cube.""" + time = DimCoord( + [11.0, 12.0], + standard_name='time', + units=Unit('hours since 1851-01-01', calendar='360_day'), + ) + plev = DimCoord([50000], standard_name='air_pressure', units='Pa') + lat = DimCoord([0.0, 1.0], standard_name='latitude', units='degrees') + lon = DimCoord( + [0.0, 20.0, 345.0], standard_name='longitude', units='degrees' + ) + + aux_2d_data = np.arange(2 * 3).reshape(2, 3) + aux_2d_bounds = np.stack( + (aux_2d_data - 1, aux_2d_data, aux_2d_data + 1), axis=-1 + ) + aux_2d = AuxCoord(aux_2d_data, var_name='aux_2d') + aux_2d_with_bnds = AuxCoord( + aux_2d_data, bounds=aux_2d_bounds, var_name='aux_2d_with_bnds' + ) + aux_time = AuxCoord(['Jan', 'Jan'], var_name='aux_time') + aux_lon = AuxCoord([0, 1, 2], var_name='aux_lon') + + cell_area = CellMeasure( + np.arange(2 * 2 * 3).reshape(2, 2, 3) + 10, + standard_name='cell_area', + units='m2', + measure='area', + ) + type_var = AncillaryVariable( + [['sea', 'land', 'lake'], ['lake', 'sea', 'land']], + var_name='type', + units='no_unit', + ) + + cube = Cube( + np.ma.masked_inside( + np.arange(2 * 1 * 2 * 3).reshape(2, 1, 2, 3), 1, 3 + ), + var_name='ta', + standard_name='air_temperature', + long_name='Air Temperature', + units='K', + cell_methods=[CellMethod('mean', 'time')], + dim_coords_and_dims=[(time, 0), (plev, 1), (lat, 2), (lon, 3)], + aux_coords_and_dims=[ + (aux_2d, (0, 3)), + (aux_2d_with_bnds, (0, 3)), + (aux_time, 0), + (aux_lon, 3), + ], + cell_measures_and_dims=[(cell_area, (0, 2, 3))], + ancillary_variables_and_dims=[(type_var, (0, 3))], + attributes={'test': 1}, + ) + return cube + + +def test_local_solar_time_regular(realistic_4d_cube): + """Test ``local_solar_time``.""" + input_cube = realistic_4d_cube.copy() + + result = local_solar_time(input_cube) + + assert input_cube == realistic_4d_cube + + assert result.metadata == input_cube.metadata + assert result.shape == input_cube.shape + assert result.coord('time') != input_cube.coord('time') + assert result.coord('air_pressure') == input_cube.coord('air_pressure') + assert result.coord('latitude') == input_cube.coord('latitude') + assert result.coord('longitude') == input_cube.coord('longitude') + + assert result.coord('time').standard_name == 'time' + assert result.coord('time').var_name is None + assert result.coord('time').long_name == 'Local Solar Time' + assert result.coord('time').units == Unit( + 'hours since 1850-01-01', calendar='360_day' + ) + assert result.coord('time').attributes == {} + np.testing.assert_allclose( + result.coord('time').points, [8651.0, 8652.0] + ) + np.testing.assert_allclose( + result.coord('time').bounds, [[8650.5, 8651.5], [8651.5, 8652.5]] + ) + + assert result.coord('aux_time') == input_cube.coord('aux_time') + assert result.coord('aux_lon') == input_cube.coord('aux_lon') + assert ( + result.coord('aux_2d').metadata == input_cube.coord('aux_2d').metadata + ) + assert not result.coord('aux_2d').has_lazy_points() + assert_array_equal( + result.coord('aux_2d').points, + np.ma.masked_equal([[0, 99, 5], [3, 1, 99]], 99), + ) + assert not result.coord('aux_2d').has_bounds() + assert ( + result.coord('aux_2d_with_bnds').metadata == + input_cube.coord('aux_2d_with_bnds').metadata + ) + assert not result.coord('aux_2d_with_bnds').has_lazy_points() + assert_array_equal( + result.coord('aux_2d_with_bnds').points, + np.ma.masked_equal([[0, 99, 5], [3, 1, 99]], 99), + ) + assert not result.coord('aux_2d_with_bnds').has_lazy_bounds() + assert_array_equal( + result.coord('aux_2d_with_bnds').bounds, + np.ma.masked_equal( + [ + [[-1, 0, 1], [99, 99, 99], [4, 5, 6]], + [[2, 3, 4], [0, 1, 2], [99, 99, 99]], + ], + 99, + ), + ) + + assert ( + result.cell_measure('cell_area').metadata == + input_cube.cell_measure('cell_area').metadata + ) + assert not result.cell_measure('cell_area').has_lazy_data() + assert_array_equal( + result.cell_measure('cell_area').data, + np.ma.masked_equal( + [ + [[10, 99, 18], [13, 99, 21]], + [[16, 11, 99], [19, 14, 99]], + ], + 99, + ), + ) + assert ( + result.ancillary_variable('type').metadata == + input_cube.ancillary_variable('type').metadata + ) + assert not result.ancillary_variable('type').has_lazy_data() + assert_array_equal( + result.ancillary_variable('type').data, + np.ma.masked_equal( + [['sea', 'miss', 'land'], ['lake', 'land', 'miss']], 'miss' + ), + ) + + assert not result.has_lazy_data() + assert_array_equal( + result.data, + np.ma.masked_equal( + [ + [[[0, 99, 8], [99, 99, 11]]], + [[[6, 99, 99], [9, 4, 99]]], + ], + 99, + ), + ) + + +def test_local_solar_time_1_time_step(realistic_4d_cube): + """Test ``local_solar_time``.""" + input_cube = realistic_4d_cube[[0]] + + result = local_solar_time(input_cube) + + assert input_cube == realistic_4d_cube[[0]] + + assert result.metadata == input_cube.metadata + assert result.shape == input_cube.shape + assert result.coord('time') != input_cube.coord('time') + assert result.coord('air_pressure') == input_cube.coord('air_pressure') + assert result.coord('latitude') == input_cube.coord('latitude') + assert result.coord('longitude') == input_cube.coord('longitude') + + assert result.coord('time').standard_name == 'time' + assert result.coord('time').var_name is None + assert result.coord('time').long_name == 'Local Solar Time' + assert result.coord('time').units == Unit( + 'hours since 1850-01-01', calendar='360_day' + ) + assert result.coord('time').attributes == {} + np.testing.assert_allclose(result.coord('time').points, [8651.0]) + np.testing.assert_allclose(result.coord('time').bounds, [[8650.5, 8651.5]]) + + assert result.coord('aux_time') == input_cube.coord('aux_time') + assert result.coord('aux_lon') == input_cube.coord('aux_lon') + assert ( + result.coord('aux_2d').metadata == input_cube.coord('aux_2d').metadata + ) + assert not result.coord('aux_2d').has_lazy_points() + assert_array_equal( + result.coord('aux_2d').points, np.ma.masked_equal([[0, 99, 99]], 99) + ) + assert not result.coord('aux_2d').has_bounds() + assert ( + result.coord('aux_2d_with_bnds').metadata == + input_cube.coord('aux_2d_with_bnds').metadata + ) + assert not result.coord('aux_2d_with_bnds').has_lazy_points() + assert_array_equal( + result.coord('aux_2d_with_bnds').points, + np.ma.masked_equal([[0, 99, 99]], 99), + ) + assert not result.coord('aux_2d_with_bnds').has_lazy_bounds() + assert_array_equal( + result.coord('aux_2d_with_bnds').bounds, + np.ma.masked_equal([[[-1, 0, 1], [99, 99, 99], [99, 99, 99]]], 99), + ) + + assert ( + result.cell_measure('cell_area').metadata == + input_cube.cell_measure('cell_area').metadata + ) + assert not result.cell_measure('cell_area').has_lazy_data() + assert_array_equal( + result.cell_measure('cell_area').data, + np.ma.masked_equal([[[10, 99, 99], [13, 99, 99]]], 99), + ) + assert ( + result.ancillary_variable('type').metadata == + input_cube.ancillary_variable('type').metadata + ) + assert not result.ancillary_variable('type').has_lazy_data() + assert_array_equal( + result.ancillary_variable('type').data, + np.ma.masked_equal([['sea', 'miss', 'miss']], 'miss'), + ) + + assert not result.has_lazy_data() + assert_array_equal( + result.data, + np.ma.masked_equal([[[[0, 99, 99], [99, 99, 99]]]], 99), + ) + + +@pytest.fixture +def realistic_unstructured_cube(): + """Create realistic unstructured cube.""" + time = DimCoord( + [0.0, 6.0, 12.0, 18.0, 24.0], + bounds=[ + [-3.0, 3.0], [3.0, 9.0], [9.0, 15.0], [15.0, 21.0], [21.0, 27.0] + ], + var_name='time', + standard_name='time', + long_name='time', + units=Unit('hours since 1851-01-01'), + ) + + lat = AuxCoord( + [0.0, 0.0, 0.0, 0.0], + var_name='lat', + standard_name='latitude', + long_name='latitude', + units='degrees_north', + ) + lon = AuxCoord( + [0.0, 80 * np.pi / 180.0, -120 * np.pi / 180.0, 160 * np.pi / 180.0], + var_name='lon', + standard_name='longitude', + long_name='longitude', + units='rad', + ) + aux_2d_data = da.ma.masked_inside(da.arange(4 * 5).reshape(4, 5), 3, 10) + aux_2d_bounds = da.stack((aux_2d_data - 1, aux_2d_data + 1), axis=-1) + aux_2d = AuxCoord(aux_2d_data, var_name='aux_2d') + aux_2d_with_bnds = AuxCoord( + aux_2d_data, bounds=aux_2d_bounds, var_name='aux_2d_with_bnds' + ) + aux_0d = AuxCoord([0], var_name='aux_0d') + + cell_measure_2d = CellMeasure( + da.ma.masked_inside(da.arange(4 * 5).reshape(4, 5), 3, 10), + var_name='cell_measure', + ) + anc_var_2d = AncillaryVariable( + da.ma.masked_inside(da.arange(4 * 5).reshape(4, 5), 3, 10), + var_name='anc_var', + ) + + cube = Cube( + da.arange(4 * 5).reshape(4, 5), + var_name='ta', + standard_name='air_temperature', + long_name='Air Temperature', + units='K', + dim_coords_and_dims=[(time, 1)], + aux_coords_and_dims=[ + (lat, 0), + (lon, 0), + (aux_2d, (0, 1)), + (aux_2d_with_bnds, (0, 1)), + (aux_0d, ()), + ], + cell_measures_and_dims=[(cell_measure_2d, (0, 1))], + ancillary_variables_and_dims=[(anc_var_2d, (0, 1))], + ) + return cube + + +def test_local_solar_time_unstructured(realistic_unstructured_cube): + """Test ``local_solar_time``.""" + input_cube = realistic_unstructured_cube.copy() + + result = local_solar_time(input_cube) + + assert input_cube == realistic_unstructured_cube + + assert result.metadata == input_cube.metadata + assert result.shape == input_cube.shape + assert result.coord('time') != input_cube.coord('time') + assert result.coord('latitude') == input_cube.coord('latitude') + assert result.coord('longitude') == input_cube.coord('longitude') + + assert result.coord('time').standard_name == 'time' + assert result.coord('time').var_name == 'time' + assert result.coord('time').long_name == 'Local Solar Time' + assert result.coord('time').units == 'hours since 1850-01-01' + assert result.coord('time').attributes == {} + np.testing.assert_allclose( + result.coord('time').points, [8760.0, 8766.0, 8772.0, 8778.0, 8784.0] + ) + np.testing.assert_allclose( + result.coord('time').bounds, + [ + [8757.0, 8763.0], + [8763.0, 8769.0], + [8769.0, 8775.0], + [8775.0, 8781.0], + [8781.0, 8787.0], + ], + ) + + assert result.coord('aux_0d') == input_cube.coord('aux_0d') + assert ( + result.coord('aux_2d').metadata == input_cube.coord('aux_2d').metadata + ) + assert result.coord('aux_2d').has_lazy_points() + assert_array_equal( + result.coord('aux_2d').points, + np.ma.masked_equal( + [ + [0, 1, 2, 99, 99], + [99, 99, 99, 99, 99], + [11, 12, 13, 14, 99], + [99, 99, 15, 16, 17], + ], + 99, + ), + ) + assert not result.coord('aux_2d').has_bounds() + assert ( + result.coord('aux_2d_with_bnds').metadata == + input_cube.coord('aux_2d_with_bnds').metadata + ) + assert result.coord('aux_2d_with_bnds').has_lazy_points() + assert_array_equal( + result.coord('aux_2d_with_bnds').points, + np.ma.masked_equal( + [ + [0, 1, 2, 99, 99], + [99, 99, 99, 99, 99], + [11, 12, 13, 14, 99], + [99, 99, 15, 16, 17], + ], + 99, + ), + ) + assert result.coord('aux_2d_with_bnds').has_lazy_bounds() + assert_array_equal( + result.coord('aux_2d_with_bnds').bounds, + np.ma.masked_equal( + [ + [[-1, 1], [0, 2], [1, 3], [99, 99], [99, 99]], + [[99, 99], [99, 99], [99, 99], [99, 99], [99, 99]], + [[10, 12], [11, 13], [12, 14], [13, 15], [99, 99]], + [[99, 99], [99, 99], [14, 16], [15, 17], [16, 18]], + ], + 99, + ), + ) + + assert ( + result.cell_measure('cell_measure').metadata == + input_cube.cell_measure('cell_measure').metadata + ) + assert result.cell_measure('cell_measure').has_lazy_data() + assert_array_equal( + result.cell_measure('cell_measure').data, + np.ma.masked_equal( + [ + [0, 1, 2, 99, 99], + [99, 99, 99, 99, 99], + [11, 12, 13, 14, 99], + [99, 99, 15, 16, 17], + ], + 99, + ), + ) + assert ( + result.ancillary_variable('anc_var').metadata == + input_cube.ancillary_variable('anc_var').metadata + ) + assert result.ancillary_variable('anc_var').has_lazy_data() + assert_array_equal( + result.ancillary_variable('anc_var').data, + np.ma.masked_equal( + [ + [0, 1, 2, 99, 99], + [99, 99, 99, 99, 99], + [11, 12, 13, 14, 99], + [99, 99, 15, 16, 17], + ], + 99, + ), + ) + + assert result.has_lazy_data() + assert_array_equal( + result.data, + np.ma.masked_equal( + [ + [0, 1, 2, 3, 4], + [99, 5, 6, 7, 8], + [11, 12, 13, 14, 99], + [99, 99, 15, 16, 17], + ], + 99, + ), + ) + + +def test_local_solar_time_no_time_fail(realistic_4d_cube): + """Test ``local_solar_time``.""" + realistic_4d_cube.remove_coord('time') + msg = 'needs a dimensional coordinate `time`' + with pytest.raises(CoordinateNotFoundError, match=msg): + local_solar_time(realistic_4d_cube) + + +def test_local_solar_time_scalar_time_fail(realistic_4d_cube): + """Test ``local_solar_time``.""" + input_cube = realistic_4d_cube[0] + msg = 'needs a dimensional coordinate `time`' + with pytest.raises(CoordinateNotFoundError, match=msg): + local_solar_time(input_cube) + + +def test_local_solar_time_time_decreasing_fail(realistic_4d_cube): + """Test ``local_solar_time``.""" + input_cube = realistic_4d_cube[::-1] + msg = '`time` coordinate must be monotonically increasing' + with pytest.raises(ValueError, match=msg): + local_solar_time(input_cube) + + +def test_local_solar_time_no_lon_fail(realistic_4d_cube): + """Test ``local_solar_time``.""" + realistic_4d_cube.remove_coord('longitude') + msg = 'needs a coordinate `longitude`' + with pytest.raises(CoordinateNotFoundError, match=msg): + local_solar_time(realistic_4d_cube) + + +def test_local_solar_time_scalar_lon_fail(realistic_4d_cube): + """Test ``local_solar_time``.""" + input_cube = realistic_4d_cube[..., 0] + msg = 'needs a 1D coordinate `longitude`, got 0D' + with pytest.raises(CoordinateMultiDimError, match=msg): + local_solar_time(input_cube) + + +def test_local_solar_time_2d_lon_fail(easy_2d_cube): + """Test ``local_solar_time``.""" + lon_coord = AuxCoord(easy_2d_cube.data, standard_name='longitude') + easy_2d_cube.add_aux_coord(lon_coord, (0, 1)) + msg = 'needs a 1D coordinate `longitude`, got 2D' + with pytest.raises(CoordinateMultiDimError, match=msg): + local_solar_time(easy_2d_cube) diff --git a/tests/integration/recipe/test_recipe.py b/tests/integration/recipe/test_recipe.py index 8b7bfedb86..38e015078f 100644 --- a/tests/integration/recipe/test_recipe.py +++ b/tests/integration/recipe/test_recipe.py @@ -165,6 +165,80 @@ def get_recipe(tempdir: Path, content: str, session: Session): return recipe +def test_recipe_missing_scripts(tmp_path, session): + content = dedent(""" + datasets: + - dataset: bcc-csm1-1 + + diagnostics: + diagnostic_name: + variables: + ta: + project: CMIP5 + mip: Amon + exp: historical + ensemble: r1i1p1 + timerange: 1999/2002 + """) + exc_message = ("Missing scripts section in diagnostic 'diagnostic_name'.") + with pytest.raises(RecipeError) as exc: + get_recipe(tmp_path, content, session) + assert str(exc.value) == exc_message + + +def test_recipe_duplicate_var_script_name(tmp_path, session): + content = dedent(""" + datasets: + - dataset: bcc-csm1-1 + + diagnostics: + diagnostic_name: + variables: + ta: + project: CMIP5 + mip: Amon + exp: historical + ensemble: r1i1p1 + start_year: 1999 + end_year: 2002 + scripts: + ta: + script: tmp_path / 'diagnostic.py' + """) + exc_message = ("Invalid script name 'ta' encountered in diagnostic " + "'diagnostic_name': scripts cannot have the same " + "name as variables.") + with pytest.raises(RecipeError) as exc: + get_recipe(tmp_path, content, session) + assert str(exc.value) == exc_message + + +def test_recipe_no_script(tmp_path, session): + content = dedent(""" + datasets: + - dataset: bcc-csm1-1 + + diagnostics: + diagnostic_name: + variables: + ta: + project: CMIP5 + mip: Amon + exp: historical + ensemble: r1i1p1 + start_year: 1999 + end_year: 2002 + scripts: + script_name: + argument: 1 + """) + exc_message = ("No script defined for script 'script_name' in " + "diagnostic 'diagnostic_name'.") + with pytest.raises(RecipeError) as exc: + get_recipe(tmp_path, content, session) + assert str(exc.value) == exc_message + + def test_recipe_no_datasets(tmp_path, session): content = dedent(""" diagnostics: @@ -181,7 +255,56 @@ def test_recipe_no_datasets(tmp_path, session): """) exc_message = ("You have not specified any dataset " "or additional_dataset groups for variable " - "ta in diagnostic diagnostic_name.") + "'ta' in diagnostic 'diagnostic_name'.") + with pytest.raises(RecipeError) as exc: + get_recipe(tmp_path, content, session) + assert str(exc.value) == exc_message + + +def test_recipe_duplicated_datasets(tmp_path, session): + content = dedent(""" + datasets: + - dataset: bcc-csm1-1 + - dataset: bcc-csm1-1 + + diagnostics: + diagnostic_name: + variables: + ta: + project: CMIP5 + mip: Amon + exp: historical + ensemble: r1i1p1 + timerange: 1999/2002 + scripts: null + """) + exc_message = ("Duplicate dataset\n{'dataset': 'bcc-csm1-1'}\n" + "for variable 'ta' in diagnostic 'diagnostic_name'.") + with pytest.raises(RecipeError) as exc: + get_recipe(tmp_path, content, session) + assert str(exc.value) == exc_message + + +def test_recipe_var_missing_args(tmp_path, session): + content = dedent(""" + datasets: + - dataset: bcc-csm1-1 + + diagnostics: + diagnostic_name: + variables: + ta: + project: CMIP5 + exp: historical + ensemble: r1i1p1 + timerange: 1999/2002 + scripts: null + """) + exc_message = ("Missing keys {'mip'} in\n{'dataset': 'bcc-csm1-1'," + "\n 'ensemble': 'r1i1p1',\n 'exp': 'historical',\n" + " 'project': 'CMIP5',\n 'short_name': 'ta',\n " + "'timerange': '1999/2002'}\nfor variable 'ta' " + "in diagnostic 'diagnostic_name'.") with pytest.raises(RecipeError) as exc: get_recipe(tmp_path, content, session) assert str(exc.value) == exc_message @@ -2767,3 +2890,253 @@ def test_invalid_stat_preproc(tmp_path, patched_datafinder, session): msg = "Unknown preprocessor function" with pytest.raises(ValueError, match=msg): get_recipe(tmp_path, content, session) + + +def test_bias_no_ref(tmp_path, patched_datafinder, session): + content = dedent(""" + preprocessors: + test_bias: + bias: + bias_type: relative + denominator_mask_threshold: 5 + + diagnostics: + diagnostic_name: + variables: + ta: + preprocessor: test_bias + project: CMIP6 + mip: Amon + exp: historical + timerange: '20000101/20001231' + ensemble: r1i1p1f1 + grid: gn + additional_datasets: + - {dataset: CanESM5} + - {dataset: CESM2} + + scripts: null + """) + msg = ( + "Expected exactly 1 dataset with 'reference_for_bias: true' in " + "products" + ) + with pytest.raises(RecipeError) as exc: + get_recipe(tmp_path, content, session) + assert str(exc.value) == INITIALIZATION_ERROR_MSG + assert msg in exc.value.failed_tasks[0].message + assert "found 0" in exc.value.failed_tasks[0].message + + +def test_bias_two_refs(tmp_path, patched_datafinder, session): + content = dedent(""" + preprocessors: + test_bias: + bias: + bias_type: relative + denominator_mask_threshold: 5 + + diagnostics: + diagnostic_name: + variables: + ta: + preprocessor: test_bias + project: CMIP6 + mip: Amon + exp: historical + timerange: '20000101/20001231' + ensemble: r1i1p1f1 + grid: gn + additional_datasets: + - {dataset: CanESM5, reference_for_bias: true} + - {dataset: CESM2, reference_for_bias: true} + + scripts: null + """) + msg = ( + "Expected exactly 1 dataset with 'reference_for_bias: true' in " + "products" + ) + with pytest.raises(RecipeError) as exc: + get_recipe(tmp_path, content, session) + assert str(exc.value) == INITIALIZATION_ERROR_MSG + assert msg in exc.value.failed_tasks[0].message + assert "found 2" in exc.value.failed_tasks[0].message + + +def test_inlvaid_bias_type(tmp_path, patched_datafinder, session): + content = dedent(""" + preprocessors: + test_bias: + bias: + bias_type: INVALID + denominator_mask_threshold: 5 + + diagnostics: + diagnostic_name: + variables: + ta: + preprocessor: test_bias + project: CMIP6 + mip: Amon + exp: historical + timerange: '20000101/20001231' + ensemble: r1i1p1f1 + grid: gn + additional_datasets: + - {dataset: CanESM5} + - {dataset: CESM2, reference_for_bias: true} + + scripts: null + """) + msg = ( + "Expected one of ('absolute', 'relative') for `bias_type`, got " + "'INVALID'" + ) + with pytest.raises(RecipeError) as exc: + get_recipe(tmp_path, content, session) + assert str(exc.value) == INITIALIZATION_ERROR_MSG + assert exc.value.failed_tasks[0].message == msg + + +def test_invalid_builtin_regridding_scheme( + tmp_path, patched_datafinder, session +): + content = dedent(""" + preprocessors: + test: + regrid: + scheme: INVALID + target_grid: 2x2 + diagnostics: + diagnostic_name: + variables: + tas: + mip: Amon + preprocessor: test + timerange: '2000/2010' + additional_datasets: + - {project: CMIP5, dataset: CanESM2, exp: amip, + ensemble: r1i1p1} + scripts: null + """) + msg = ( + "Got invalid built-in regridding scheme 'INVALID', expected one of " + ) + with pytest.raises(RecipeError) as rec_err_exp: + get_recipe(tmp_path, content, session) + assert str(rec_err_exp.value) == INITIALIZATION_ERROR_MSG + assert msg in str(rec_err_exp.value.failed_tasks[0].message) + + +def test_generic_regridding_scheme_no_ref( + tmp_path, patched_datafinder, session +): + content = dedent(""" + preprocessors: + test: + regrid: + scheme: + no_reference: given + target_grid: 2x2 + diagnostics: + diagnostic_name: + variables: + tas: + mip: Amon + preprocessor: test + timerange: '2000/2010' + additional_datasets: + - {project: CMIP5, dataset: CanESM2, exp: amip, + ensemble: r1i1p1} + scripts: null + """) + msg = ( + "Failed to load generic regridding scheme: No reference specified for " + "generic regridding. See " + ) + with pytest.raises(RecipeError) as rec_err_exp: + get_recipe(tmp_path, content, session) + assert str(rec_err_exp.value) == INITIALIZATION_ERROR_MSG + assert msg in str(rec_err_exp.value.failed_tasks[0].message) + + +def test_invalid_generic_regridding_scheme( + tmp_path, patched_datafinder, session +): + content = dedent(""" + preprocessors: + test: + regrid: + scheme: + reference: invalid.module:and.function + target_grid: 2x2 + diagnostics: + diagnostic_name: + variables: + tas: + mip: Amon + preprocessor: test + timerange: '2000/2010' + additional_datasets: + - {project: CMIP5, dataset: CanESM2, exp: amip, + ensemble: r1i1p1} + scripts: null + """) + msg = ( + "Failed to load generic regridding scheme: Could not import specified " + "generic regridding module 'invalid.module'. Please double check " + "spelling and that the required module is installed. " + ) + with pytest.raises(RecipeError) as rec_err_exp: + get_recipe(tmp_path, content, session) + assert str(rec_err_exp.value) == INITIALIZATION_ERROR_MSG + assert msg in str(rec_err_exp.value.failed_tasks[0].message) + + +def test_deprecated_linear_extrapolate_scheme( + tmp_path, patched_datafinder, session +): + content = dedent(""" + preprocessors: + test: + regrid: + scheme: linear_extrapolate + target_grid: 2x2 + diagnostics: + diagnostic_name: + variables: + tas: + mip: Amon + preprocessor: test + timerange: '2000/2010' + additional_datasets: + - {project: CMIP5, dataset: CanESM2, exp: amip, + ensemble: r1i1p1} + scripts: null + """) + get_recipe(tmp_path, content, session) + + +def test_deprecated_unstructured_nearest_scheme( + tmp_path, patched_datafinder, session +): + content = dedent(""" + preprocessors: + test: + regrid: + scheme: unstructured_nearest + target_grid: 2x2 + diagnostics: + diagnostic_name: + variables: + tas: + mip: Amon + preprocessor: test + timerange: '2000/2010' + additional_datasets: + - {project: CMIP5, dataset: CanESM2, exp: amip, + ensemble: r1i1p1} + scripts: null + """) + get_recipe(tmp_path, content, session) diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index 6372a1c041..c840027796 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -4,6 +4,7 @@ import pytest import yaml +import esmvalcore from esmvalcore.cmor.check import CheckLevels from esmvalcore.config import CFG, _config from esmvalcore.config._config import ( @@ -193,6 +194,7 @@ def test_load_default_config(monkeypatch, default_config): 'preproc_dir', 'run_dir', 'work_dir', + 'config_dir', } # Check that only allowed keys are in it assert set(default_cfg) == set(cfg) @@ -210,6 +212,7 @@ def test_load_default_config(monkeypatch, default_config): for path in ('preproc', 'work', 'run'): assert getattr(cfg, path + '_dir') == cfg.session_dir / path assert cfg.plot_dir == cfg.session_dir / 'plots' + assert cfg.config_dir == Path(esmvalcore.__file__).parent # Check that projects were configured assert project_cfg diff --git a/tests/unit/config/test_config_object.py b/tests/unit/config/test_config_object.py index 9e916ff950..16084effc4 100644 --- a/tests/unit/config/test_config_object.py +++ b/tests/unit/config/test_config_object.py @@ -1,10 +1,26 @@ +import contextlib +import os +import sys from collections.abc import MutableMapping +from copy import deepcopy from pathlib import Path import pytest -from esmvalcore.config import Config, Session, _config_object +import esmvalcore +import esmvalcore.config._config_object +from esmvalcore.config import Config, Session from esmvalcore.exceptions import InvalidConfigParameter +from tests.integration.test_main import arguments + + +@contextlib.contextmanager +def environment(**kwargs): + """Temporary environment variables.""" + backup = deepcopy(os.environ) + os.environ = kwargs + yield + os.environ = backup def test_config_class(): @@ -57,25 +73,66 @@ def test_config_init(): def test_load_from_file(monkeypatch): - default_config_user_file = Path.home() / '.esmvaltool' / 'config-user.yml' - assert _config_object.USER_CONFIG == default_config_user_file - monkeypatch.setattr( - _config_object, - 'USER_CONFIG', - _config_object.DEFAULT_CONFIG, - ) + default_config_file = Path(esmvalcore.__file__).parent / 'config-user.yml' config = Config() assert not config - config.load_from_file() + config.load_from_file(default_config_file) assert config +def test_load_from_file_filenotfound(monkeypatch): + """Test `Config.load_from_file`.""" + config = Config() + assert not config + + expected_path = Path.home() / '.esmvaltool' / 'not_existent_file.yml' + msg = f"Config file '{expected_path}' does not exist" + with pytest.raises(FileNotFoundError, match=msg): + config.load_from_file('not_existent_file.yml') + + +def test_load_from_file_invalidconfigparameter(monkeypatch, tmp_path): + """Test `Config.load_from_file`.""" + monkeypatch.chdir(tmp_path) + cfg_path = tmp_path / 'test.yml' + cfg_path.write_text('invalid_param: 42') + + config = Config() + assert not config + + msg = ( + f"Failed to parse user configuration file {cfg_path}: `invalid_param` " + f"is not a valid config parameter." + ) + with pytest.raises(InvalidConfigParameter, match=msg): + config.load_from_file(cfg_path) + + def test_config_key_error(): config = Config() with pytest.raises(KeyError): config['invalid_key'] +def test_reload(): + """Test `Config.reload`.""" + cfg_path = Path(esmvalcore.__file__).parent / 'config-user.yml' + config = Config(config_file=cfg_path) + config.reload() + assert config['config_file'] == cfg_path + + +def test_reload_fail(): + """Test `Config.reload`.""" + config = Config() + msg = ( + "Cannot reload configuration, option 'config_file' is missing; make " + "sure to only use the `CFG` object from the `esmvalcore.config` module" + ) + with pytest.raises(ValueError, match=msg): + config.reload() + + def test_session(): config = Config({'output_dir': 'config'}) @@ -90,3 +147,174 @@ def test_session_key_error(): session = Session({}) with pytest.raises(KeyError): session['invalid_key'] + + +TEST_GET_CFG_PATH = [ + (None, None, None, '~/.esmvaltool/config-user.yml', False), + ( + None, + None, + ('any_other_module', '--config_file=cli.yml'), + '~/.esmvaltool/config-user.yml', + False, + ), + ( + None, + None, + ('esmvaltool', 'run', '--max-parallel-tasks=4'), + '~/.esmvaltool/config-user.yml', + True, + ), + ( + None, + None, + ('esmvaltool', '--config_file'), + '~/.esmvaltool/config-user.yml', + True, + ), + ( + None, + None, + ('esmvaltool', 'run', '--config_file=/cli.yml'), + '/cli.yml', + True, + ), + ( + None, + None, + ('esmvaltool', 'run', '--config_file=/cli.yml'), + '/cli.yml', + True, + ), + ( + None, + None, + ('esmvaltool', 'run', '--config-file', '/cli.yml'), + '/cli.yml', + True, + ), + ( + None, + None, + ('esmvaltool', 'run', '--config-file=/cli.yml'), + '/cli.yml', + True, + ), + ( + None, + None, + ('esmvaltool', 'run', '--config-file=relative_cli.yml'), + '~/.esmvaltool/relative_cli.yml', + True, + ), + ( + None, + None, + ('esmvaltool', 'run', '--config-file=existing_cfg.yml'), + 'existing_cfg.yml', + True, + ), + ( + None, + {'_ESMVALTOOL_USER_CONFIG_FILE_': '/env.yml'}, + ('esmvaltool', 'run', '--config-file=/cli.yml'), + '/env.yml', + True, + ), + ( + None, + {'_ESMVALTOOL_USER_CONFIG_FILE_': '/env.yml'}, + None, + '/env.yml', + True, + ), + ( + None, + {'_ESMVALTOOL_USER_CONFIG_FILE_': 'existing_cfg.yml'}, + ('esmvaltool', 'run', '--config-file=/cli.yml'), + 'existing_cfg.yml', + True, + ), + ( + '/filename.yml', + {'_ESMVALTOOL_USER_CONFIG_FILE_': '/env.yml'}, + ('esmvaltool', 'run', '--config-file=/cli.yml'), + '/filename.yml', + True, + ), + ( + '/filename.yml', + None, + ('esmvaltool', 'run', '--config-file=/cli.yml'), + '/filename.yml', + True, + ), + ('/filename.yml', None, None, '/filename.yml', False), + ( + 'filename.yml', + None, + None, + '~/.esmvaltool/filename.yml', + False, + ), + ( + 'existing_cfg.yml', + {'_ESMVALTOOL_USER_CONFIG_FILE_': '/env.yml'}, + ('esmvaltool', 'run', '--config-file=/cli.yml'), + 'existing_cfg.yml', + True, + ), +] + + +@pytest.mark.parametrize( + 'filename,env,cli_args,output,env_var_set', TEST_GET_CFG_PATH +) +def test_get_config_user_path( + filename, env, cli_args, output, env_var_set, monkeypatch, tmp_path +): + """Test `Config._get_config_user_path`.""" + # Create empty test file + monkeypatch.chdir(tmp_path) + (tmp_path / 'existing_cfg.yml').write_text('') + + if env is None: + env = {} + if cli_args is None: + cli_args = sys.argv + + if output == 'existing_cfg.yml': + output = tmp_path / 'existing_cfg.yml' + else: + output = Path(output).expanduser() + + with environment(**env), arguments(*cli_args): + config_path = Config._get_config_user_path(filename) + if env_var_set: + assert os.environ['_ESMVALTOOL_USER_CONFIG_FILE_'] == str(output) + else: + assert '_ESMVALTOOL_USER_CONFIG_FILE_' not in os.environ + assert isinstance(config_path, Path) + assert config_path == output + + +def test_load_user_config_filenotfound(): + """Test `Config._load_user_config`.""" + expected_path = Path.home() / '.esmvaltool' / 'not_existent_file.yml' + msg = f"Config file '{expected_path}' does not exist" + with pytest.raises(FileNotFoundError, match=msg): + Config._load_user_config('not_existent_file.yml') + + +def test_load_user_config_invalidconfigparameter(monkeypatch, tmp_path): + """Test `Config._load_user_config`.""" + monkeypatch.chdir(tmp_path) + cfg_path = tmp_path / 'test.yml' + cfg_path.write_text('invalid_param: 42') + + msg = ( + f"Failed to parse user configuration file {cfg_path}: `invalid_param` " + f"is not a valid config parameter." + ) + with pytest.raises(InvalidConfigParameter, match=msg): + Config._load_user_config(cfg_path) diff --git a/tests/unit/local/test_get_rootpath.py b/tests/unit/local/test_get_rootpath.py new file mode 100644 index 0000000000..e300db1999 --- /dev/null +++ b/tests/unit/local/test_get_rootpath.py @@ -0,0 +1,28 @@ +"""Tests for ``_get_rootpath`` in ``esmvalcore.local``.""" +from unittest import mock + +import pytest + +from esmvalcore import local + + +@mock.patch("os.path.exists") +def test_get_rootpath_exists(mexists): + mexists.return_value = True + cfg = {"rootpath": {"CMIP5": ["/path1"], "CMIP6": ["/path2"]}} + project = "CMIP5" + with mock.patch.dict(local.CFG, cfg): + output = local._get_rootpath(project) + # 'output' is a list containing a PosixPath: + assert str(output[0]) == cfg["rootpath"][project][0] + + +@mock.patch("os.path.exists") +def test_get_rootpath_does_not_exist(mexists): + mexists.return_value = False + cfg = {"rootpath": {"CMIP5": ["path1"], "CMIP6": ["path2"]}} + project = "OBS" + with mock.patch.dict(local.CFG, cfg): + msg = rf"The \"{project}\" option is missing.*" + with pytest.raises(KeyError, match=msg): + local._get_rootpath(project) diff --git a/tests/unit/preprocessor/_area/test_area.py b/tests/unit/preprocessor/_area/test_area.py index a74ac8ba06..8835014ff2 100644 --- a/tests/unit/preprocessor/_area/test_area.py +++ b/tests/unit/preprocessor/_area/test_area.py @@ -188,6 +188,41 @@ def test_area_statistics_rms(self): self.assert_array_equal(result.data, expected) self.assertEqual(result.units, 'kg m-2 s-1') + def test_area_statistics_subtract_mean(self): + """Test for area average of a 2D field.""" + input_data = self.grid.copy() + self.assertFalse(input_data.cell_measures('cell_area')) + + result = area_statistics(input_data, 'mean', normalize='subtract') + + self.assertEqual(input_data, self.grid) + self.assertEqual(result.shape, input_data.shape) + expected = np.ma.zeros((2, 5, 5), dtype=np.float32) + self.assert_array_equal(result.data, expected) + self.assertFalse(result.cell_measures('cell_area')) + self.assertEqual(result.metadata, self.grid.metadata) + for coord in self.grid.coords(): + self.assertEqual(result.coord(coord.name()), coord) + + def test_area_statistics_cell_measure_subtract_mean(self): + """Test for area average of a 2D field. + + The area measure is pre-loaded in the cube. + """ + self._add_cell_measure_to_grid() + input_data = self.grid.copy() + + result = area_statistics(input_data, 'mean', normalize='subtract') + + self.assertEqual(input_data, self.grid) + self.assertEqual(result.shape, input_data.shape) + expected = np.ma.zeros((2, 5, 5), dtype=np.float32) + self.assert_array_equal(result.data, expected) + self.assertFalse(result.cell_measures('cell_area')) + self.assertEqual(result.metadata, self.grid.metadata) + for coord in self.grid.coords(): + self.assertEqual(result.coord(coord.name()), coord) + def test_extract_region(self): """Test for extracting a region from a 2D field.""" result = extract_region(self.grid, 1.5, 2.5, 1.5, 2.5) @@ -770,7 +805,7 @@ def test_crop_cube_with_ne_file(ne_ocean_shapefile): result = _crop_cube(cube, *geometries.bounds, cmor_coords=False) result = (result.coord("latitude").points[-1], result.coord("longitude").points[-1]) - expected = (89., 359.) + expected = (89., 179.) np.testing.assert_allclose(result, expected) @@ -1270,6 +1305,39 @@ def test_zonal_statistics(make_testcube): assert res.dtype == np.float32 +def test_zonal_statistics_divide_by_min(make_testcube): + """Test ``zonal_statistics``.""" + make_testcube.data = np.ones(make_testcube.shape, dtype=np.float32) + make_testcube.data[0, 0] = 0.0 + make_testcube.data[1, 0] = -1.0 + make_testcube.data[2, 0] = -0.5 + make_testcube.units = 'K' + input_data = make_testcube.copy() + + res = zonal_statistics(input_data, 'min', normalize='divide') + + assert input_data == make_testcube + assert res.shape == input_data.shape + expected = np.ma.masked_invalid( + [ + [np.nan, np.nan, np.nan, np.nan, np.nan], + [1.0, -1.0, -1.0, -1.0, -1.0], + [1.0, -2.0, -2.0, -2.0, -2.0], + [1.0, 1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 1.0], + ], + ) + tests.assert_array_equal(res.data, expected) + assert res.dtype == np.float32 + + assert res.standard_name == input_data.standard_name + assert res.var_name == input_data.var_name + assert res.long_name == input_data.long_name + assert res.cell_methods == input_data.cell_methods + assert res.attributes == input_data.attributes + assert res.units == '1' + + def test_zonal_statistics_2d_lon_fail(irreg_extract_shape_cube): """Test ``zonal_statistics``.""" with pytest.raises(ValueError): @@ -1286,11 +1354,50 @@ def test_meridional_statistics(make_testcube): assert res.dtype == np.float32 +def test_meridional_statistics_divide_by_max(make_testcube): + """Test ``meridional_statistics``.""" + make_testcube.data = np.ones(make_testcube.shape, dtype=np.float32) + make_testcube.data[0, 0] = 0.25 + make_testcube.data[0, 1] = 2.0 + make_testcube.units = 'K' + input_data = make_testcube.copy() + + res = meridional_statistics(input_data, 'max', normalize='divide') + + assert input_data == make_testcube + assert res.shape == input_data.shape + expected = np.ma.masked_invalid( + [ + [0.25, 1.0, 1.0, 1.0, 1.0], + [1.0, 0.5, 1.0, 1.0, 1.0], + [1.0, 0.5, 1.0, 1.0, 1.0], + [1.0, 0.5, 1.0, 1.0, 1.0], + [1.0, 0.5, 1.0, 1.0, 1.0], + ], + ) + tests.assert_array_equal(res.data, expected) + assert res.dtype == np.float32 + + assert res.standard_name == input_data.standard_name + assert res.var_name == input_data.var_name + assert res.long_name == input_data.long_name + assert res.cell_methods == input_data.cell_methods + assert res.attributes == input_data.attributes + assert res.units == '1' + + def test_meridional_statistics_2d_lon_fail(irreg_extract_shape_cube): """Test ``meridional_statistics``.""" with pytest.raises(ValueError): meridional_statistics(irreg_extract_shape_cube, 'sum') +def test_meridional_statistics_invalid_norm_fail(make_testcube): + """Test ``meridional_statistics``.""" + msg = "Expected 'subtract' or 'divide' for `normalize`" + with pytest.raises(ValueError, match=msg): + meridional_statistics(make_testcube, 'sum', normalize='x') + + if __name__ == '__main__': unittest.main() diff --git a/tests/unit/preprocessor/_bias/test_bias.py b/tests/unit/preprocessor/_bias/test_bias.py index 42ab9f95a4..9adab3872e 100644 --- a/tests/unit/preprocessor/_bias/test_bias.py +++ b/tests/unit/preprocessor/_bias/test_bias.py @@ -1,10 +1,10 @@ """Unit tests for :mod:`esmvalcore.preprocessor._bias`.""" import iris -import iris.cube import numpy as np import pytest from cf_units import Unit +from iris.cube import Cube, CubeList from esmvalcore.preprocessor._bias import bias from tests import PreprocessorFile @@ -42,8 +42,8 @@ def get_3d_cube(data, **cube_kwargs): var_name='lon', long_name='longitude', units='degrees_east') coord_specs = [(times, 0), (lats, 1), (lons, 2)] - cube = iris.cube.Cube(data.astype('float32'), - dim_coords_and_dims=coord_specs, **cube_kwargs) + cube = Cube(data.astype('float32'), + dim_coords_and_dims=coord_specs, **cube_kwargs) return cube @@ -53,7 +53,7 @@ def regular_cubes(): cube_data = np.arange(8.0).reshape(2, 2, 2) cube = get_3d_cube(cube_data, standard_name='air_temperature', var_name='tas', units='K') - return iris.cube.CubeList([cube]) + return CubeList([cube]) @pytest.fixture @@ -63,48 +63,18 @@ def ref_cubes(): cube_data[1, 1, 1] = 4.0 cube = get_3d_cube(cube_data, standard_name='air_temperature', var_name='tas', units='K') - return iris.cube.CubeList([cube]) + return CubeList([cube]) -def test_no_reference_for_bias(regular_cubes, ref_cubes): - """Test fail when no reference_for_bias is given.""" - products = { - PreprocessorFile(regular_cubes, 'A', {}), - PreprocessorFile(regular_cubes, 'B', {}), - PreprocessorFile(ref_cubes, 'REF', {}), - } - msg = "Expected exactly 1 dataset with 'reference_for_bias: true', found 0" - with pytest.raises(ValueError, match=msg): - bias(products) +TEST_BIAS = [ + ('absolute', [[[-2.0, -1.0], [0.0, 1.0]], [[2.0, 3.0], [4.0, 3.0]]], 'K'), + ('relative', [[[-1.0, -0.5], [0.0, 0.5]], [[1.0, 1.5], [2.0, 0.75]]], '1'), +] -def test_two_reference_for_bias(regular_cubes, ref_cubes): - """Test fail when two reference_for_bias is given.""" - products = { - PreprocessorFile(regular_cubes, 'A', {'reference_for_bias': False}), - PreprocessorFile(ref_cubes, 'REF1', {'reference_for_bias': True}), - PreprocessorFile(ref_cubes, 'REF2', {'reference_for_bias': True}), - } - msg = "Expected exactly 1 dataset with 'reference_for_bias: true', found 2" - with pytest.raises(ValueError, match=msg): - bias(products) - - -def test_invalid_bias_type(regular_cubes, ref_cubes): - """Test fail when invalid bias_type is given.""" - products = { - PreprocessorFile(regular_cubes, 'A', {}), - PreprocessorFile(regular_cubes, 'B', {}), - PreprocessorFile(ref_cubes, 'REF', {'reference_for_bias': True}), - } - msg = (r"Expected one of \['absolute', 'relative'\] for bias_type, got " - r"'invalid_bias_type'") - with pytest.raises(ValueError, match=msg): - bias(products, 'invalid_bias_type') - - -def test_absolute_bias(regular_cubes, ref_cubes): - """Test calculation of absolute bias.""" +@pytest.mark.parametrize('bias_type,data,units', TEST_BIAS) +def test_bias_products(regular_cubes, ref_cubes, bias_type, data, units): + """Test calculation of bias with products.""" ref_product = PreprocessorFile(ref_cubes, 'REF', {'reference_for_bias': True}) products = { @@ -112,23 +82,21 @@ def test_absolute_bias(regular_cubes, ref_cubes): PreprocessorFile(regular_cubes, 'B', {'dataset': 'b'}), ref_product, } - out_products = bias(products) + out_products = bias(products, bias_type=bias_type) + + assert isinstance(out_products, set) out_dict = products_set_to_dict(out_products) assert len(out_dict) == 2 product_a = out_dict['A'] assert product_a.filename == 'A' - assert product_a.attributes == {'units': 'K', 'dataset': 'a'} + assert product_a.attributes == {'units': units, 'dataset': 'a'} assert len(product_a.cubes) == 1 out_cube = product_a.cubes[0] - expected_data = [[[-2.0, -1.0], - [0.0, 1.0]], - [[2.0, 3.0], - [4.0, 3.0]]] - assert_array_equal(out_cube.data, expected_data) + assert_array_equal(out_cube.data, data) assert out_cube.var_name == 'tas' assert out_cube.standard_name == 'air_temperature' - assert out_cube.units == 'K' + assert out_cube.units == units assert out_cube.dim_coords == regular_cubes[0].dim_coords assert out_cube.aux_coords == regular_cubes[0].aux_coords product_a.wasderivedfrom.assert_called_once() @@ -136,82 +104,76 @@ def test_absolute_bias(regular_cubes, ref_cubes): product_b = out_dict['B'] assert product_b.filename == 'B' - assert product_b.attributes == {'units': 'K', 'dataset': 'b'} + assert product_b.attributes == {'units': units, 'dataset': 'b'} assert len(product_b.cubes) == 1 out_cube = product_b.cubes[0] - expected_data = [[[-2.0, -1.0], - [0.0, 1.0]], - [[2.0, 3.0], - [4.0, 3.0]]] - assert_array_equal(out_cube.data, expected_data) + assert_array_equal(out_cube.data, data) assert out_cube.var_name == 'tas' assert out_cube.standard_name == 'air_temperature' - assert out_cube.units == 'K' + assert out_cube.units == units assert out_cube.dim_coords == regular_cubes[0].dim_coords assert out_cube.aux_coords == regular_cubes[0].aux_coords product_b.wasderivedfrom.assert_called_once() assert product_b.mock_ancestors == {ref_product} -def test_relative_bias(regular_cubes, ref_cubes): - """Test calculation of relative bias.""" - ref_product = PreprocessorFile(ref_cubes, 'REF', - {'reference_for_bias': True}) - products = { - PreprocessorFile(regular_cubes, 'A', {'dataset': 'a'}), - PreprocessorFile(regular_cubes, 'B', {'dataset': 'b'}), - ref_product, - } - out_products = bias(products, 'relative') - out_dict = products_set_to_dict(out_products) - assert len(out_dict) == 2 +@pytest.mark.parametrize('bias_type,data,units', TEST_BIAS) +def test_bias_cubes(regular_cubes, ref_cubes, bias_type, data, units): + """Test calculation of bias with cubes.""" + ref_cube = ref_cubes[0] + out_cubes = bias(regular_cubes, ref_cube, bias_type=bias_type) - product_a = out_dict['A'] - assert product_a.filename == 'A' - assert product_a.attributes == {'units': '1', 'dataset': 'a'} - assert len(product_a.cubes) == 1 - out_cube = product_a.cubes[0] - expected_data = [[[-1.0, -0.5], - [0.0, 0.5]], - [[1.0, 1.5], - [2.0, 0.75]]] - assert_array_equal(out_cube.data, expected_data) + assert isinstance(out_cubes, CubeList) + assert len(out_cubes) == 1 + out_cube = out_cubes[0] + + assert_array_equal(out_cube.data, data) assert out_cube.var_name == 'tas' assert out_cube.standard_name == 'air_temperature' - assert out_cube.units == '1' + assert out_cube.units == units assert out_cube.dim_coords == regular_cubes[0].dim_coords assert out_cube.aux_coords == regular_cubes[0].aux_coords - product_a.wasderivedfrom.assert_called_once() - assert product_a.mock_ancestors == {ref_product} - product_b = out_dict['B'] - assert product_b.filename == 'B' - assert product_b.attributes == {'units': '1', 'dataset': 'b'} - assert len(product_b.cubes) == 1 - out_cube = product_b.cubes[0] - expected_data = [[[-1.0, -0.5], - [0.0, 0.5]], - [[1.0, 1.5], - [2.0, 0.75]]] - assert_array_equal(out_cube.data, expected_data) + +TEST_BIAS_BROADCASTABLE = [ + ('absolute', [[[-2.0, -1.0], [0.0, 1.0]], [[2.0, 3.0], [4.0, 5.0]]], 'K'), + ('relative', [[[-1.0, -0.5], [0.0, 0.5]], [[1.0, 1.5], [2.0, 2.5]]], '1'), +] + + +@pytest.mark.parametrize('bias_type,data,units', TEST_BIAS_BROADCASTABLE) +def test_bias_cubes_broadcastable( + regular_cubes, ref_cubes, bias_type, data, units +): + """Test calculation of bias with cubes.""" + ref_cube = ref_cubes[0][0] # only select one time step + out_cubes = bias(regular_cubes, ref_cube, bias_type=bias_type) + + assert isinstance(out_cubes, CubeList) + assert len(out_cubes) == 1 + out_cube = out_cubes[0] + + assert_array_equal(out_cube.data, data) assert out_cube.var_name == 'tas' assert out_cube.standard_name == 'air_temperature' - assert out_cube.units == '1' + assert out_cube.units == units assert out_cube.dim_coords == regular_cubes[0].dim_coords assert out_cube.aux_coords == regular_cubes[0].aux_coords - product_b.wasderivedfrom.assert_called_once() - assert product_b.mock_ancestors == {ref_product} -def test_denominator_mask_threshold(regular_cubes, ref_cubes): - """Test denominator_mask_threshold argument.""" +def test_denominator_mask_threshold_products(regular_cubes, ref_cubes): + """Test denominator_mask_threshold argument with products.""" ref_product = PreprocessorFile(ref_cubes, 'REF', {'reference_for_bias': True}) products = { PreprocessorFile(regular_cubes, 'A', {'dataset': 'a'}), ref_product, } - out_products = bias(products, 'relative', denominator_mask_threshold=3.0) + out_products = bias( + products, bias_type='relative', denominator_mask_threshold=3.0 + ) + + assert isinstance(out_products, set) out_dict = products_set_to_dict(out_products) assert len(out_dict) == 1 @@ -234,13 +196,44 @@ def test_denominator_mask_threshold(regular_cubes, ref_cubes): assert product_a.mock_ancestors == {ref_product} -def test_keep_reference_dataset(regular_cubes, ref_cubes): +def test_denominator_mask_threshold_cubes(regular_cubes, ref_cubes): + """Test denominator_mask_threshold argument with cubes.""" + ref_cube = ref_cubes[0] + out_cubes = bias( + regular_cubes, + ref_cube, + bias_type='relative', + denominator_mask_threshold=3.0, + ) + + assert isinstance(out_cubes, CubeList) + assert len(out_cubes) == 1 + out_cube = out_cubes[0] + + expected_data = np.ma.masked_equal([[[42.0, 42.0], + [42.0, 42.0]], + [[42.0, 42.0], + [42.0, 0.75]]], 42.0) + assert_array_equal(out_cube.data, expected_data) + assert out_cube.var_name == 'tas' + assert out_cube.standard_name == 'air_temperature' + assert out_cube.units == '1' + assert out_cube.dim_coords == regular_cubes[0].dim_coords + assert out_cube.aux_coords == regular_cubes[0].aux_coords + + +@pytest.mark.parametrize('bias_type', ['absolute', 'relative']) +def test_keep_reference_dataset(regular_cubes, ref_cubes, bias_type): """Test denominator_mask_threshold argument.""" products = { PreprocessorFile(regular_cubes, 'A', {'dataset': 'a'}), PreprocessorFile(ref_cubes, 'REF', {'reference_for_bias': True}) } - out_products = bias(products, keep_reference_dataset=True) + out_products = bias( + products, bias_type=bias_type, keep_reference_dataset=True + ) + + assert isinstance(out_products, set) out_dict = products_set_to_dict(out_products) assert len(out_dict) == 2 @@ -249,13 +242,92 @@ def test_keep_reference_dataset(regular_cubes, ref_cubes): assert product_ref.attributes == {'reference_for_bias': True} assert len(product_ref.cubes) == 1 out_cube = product_ref.cubes[0] - expected_data = [[[2.0, 2.0], - [2.0, 2.0]], - [[2.0, 2.0], - [2.0, 4.0]]] + expected_data = [[[2.0, 2.0], [2.0, 2.0]], [[2.0, 2.0], [2.0, 4.0]]] assert_array_equal(out_cube.data, expected_data) assert out_cube.var_name == 'tas' assert out_cube.standard_name == 'air_temperature' assert out_cube.units == 'K' assert out_cube.dim_coords == ref_cubes[0].dim_coords assert out_cube.aux_coords == ref_cubes[0].aux_coords + + +@pytest.mark.parametrize('bias_type,data,units', TEST_BIAS) +@pytest.mark.parametrize('keep_ref', [True, False]) +def test_bias_products_and_ref_cube( + regular_cubes, ref_cubes, keep_ref, bias_type, data, units +): + """Test calculation of bias with products and ref_cube given.""" + ref_cube = ref_cubes[0] + products = set([PreprocessorFile(regular_cubes, 'A', {'dataset': 'a'})]) + + out_products = bias( + products, + ref_cube=ref_cube, + bias_type=bias_type, + keep_reference_dataset=keep_ref, + ) + + assert isinstance(out_products, set) + out_dict = products_set_to_dict(out_products) + assert len(out_dict) == 1 + + product_a = out_dict['A'] + assert product_a.filename == 'A' + assert product_a.attributes == {'units': units, 'dataset': 'a'} + assert len(product_a.cubes) == 1 + out_cube = product_a.cubes[0] + assert_array_equal(out_cube.data, data) + assert out_cube.var_name == 'tas' + assert out_cube.standard_name == 'air_temperature' + assert out_cube.units == units + assert out_cube.dim_coords == regular_cubes[0].dim_coords + assert out_cube.aux_coords == regular_cubes[0].aux_coords + product_a.wasderivedfrom.assert_not_called() + assert product_a.mock_ancestors == set() + + +def test_no_reference_for_bias(regular_cubes, ref_cubes): + """Test fail when no reference_for_bias is given.""" + products = { + PreprocessorFile(regular_cubes, 'A', {}), + PreprocessorFile(regular_cubes, 'B', {}), + PreprocessorFile(ref_cubes, 'REF', {}), + } + msg = "Expected exactly 1 dataset with 'reference_for_bias: true', found 0" + with pytest.raises(ValueError, match=msg): + bias(products) + + +def test_two_references_for_bias(regular_cubes, ref_cubes): + """Test fail when two reference_for_bias products is given.""" + products = { + PreprocessorFile(regular_cubes, 'A', {'reference_for_bias': False}), + PreprocessorFile(ref_cubes, 'REF1', {'reference_for_bias': True}), + PreprocessorFile(ref_cubes, 'REF2', {'reference_for_bias': True}), + } + msg = "Expected exactly 1 dataset with 'reference_for_bias: true', found 2" + with pytest.raises(ValueError, match=msg): + bias(products) + + +def test_invalid_bias_type(regular_cubes, ref_cubes): + """Test fail when invalid bias_type is given.""" + products = { + PreprocessorFile(regular_cubes, 'A', {}), + PreprocessorFile(regular_cubes, 'B', {}), + PreprocessorFile(ref_cubes, 'REF', {'reference_for_bias': True}), + } + msg = (r"Expected one of \['absolute', 'relative'\] for bias_type, got " + r"'invalid_bias_type'") + with pytest.raises(ValueError, match=msg): + bias(products, bias_type='invalid_bias_type') + + +def test_ref_cube_non_cubes(regular_cubes): + """Test ref_cube=None with with cubes.""" + msg = ( + "A list of Cubes is given to this preprocessor; please specify a " + "`ref_cube`" + ) + with pytest.raises(ValueError, match=msg): + bias(regular_cubes) diff --git a/tests/unit/preprocessor/_derive/test_sm.py b/tests/unit/preprocessor/_derive/test_sm.py new file mode 100644 index 0000000000..7417b8f190 --- /dev/null +++ b/tests/unit/preprocessor/_derive/test_sm.py @@ -0,0 +1,32 @@ +import dask.array as da +import iris.coords +import iris.cube +import numpy as np + +from esmvalcore.preprocessor._derive.sm import DerivedVariable +from tests import assert_array_equal + + +def test_sm(): + + points = da.arange(0, 4, 2).astype(np.float32) + bounds = da.asarray([[-1., 1.], [1., 3]]) + + depth = iris.coords.AuxCoord( + points, + bounds=bounds, + standard_name='depth', + ) + cube = iris.cube.Cube( + da.asarray([0, 998.2]), + var_name='mrsos', + aux_coords_and_dims=[ + (depth, 0), + ], + ) + + result = DerivedVariable.calculate(iris.cube.CubeList([cube])) + assert result.has_lazy_data() + assert result.coord('depth').has_lazy_points() + assert result.coord('depth').has_lazy_bounds() + assert_array_equal(result.data, np.array([0, 0.5])) diff --git a/tests/unit/preprocessor/_mask/test_mask.py b/tests/unit/preprocessor/_mask/test_mask.py index a6b28e2cae..a3131c6052 100644 --- a/tests/unit/preprocessor/_mask/test_mask.py +++ b/tests/unit/preprocessor/_mask/test_mask.py @@ -49,6 +49,7 @@ def test_apply_fx_mask_on_nonmasked_data(self): dummy_fx_mask = np.ma.array((True, False, True)) app_mask = _apply_fx_mask(dummy_fx_mask, self.time_cube.data[0:3].astype('float64')) + app_mask = app_mask.compute() fixed_mask = np.ma.array(self.time_cube.data[0:3].astype('float64'), mask=dummy_fx_mask) self.assert_array_equal(fixed_mask, app_mask) @@ -59,6 +60,7 @@ def test_apply_fx_mask_on_masked_data(self): masked_data = np.ma.array(self.time_cube.data[0:3].astype('float64'), mask=np.ma.array((False, True, False))) app_mask = _apply_fx_mask(dummy_fx_mask, masked_data) + app_mask = app_mask.compute() fixed_mask = np.ma.array(self.time_cube.data[0:3].astype('float64'), mask=dummy_fx_mask) self.assert_array_equal(fixed_mask, app_mask) diff --git a/tests/unit/preprocessor/_regrid/test__stock_cube.py b/tests/unit/preprocessor/_regrid/test__stock_cube.py index 7946d18dc4..8142b66ed8 100644 --- a/tests/unit/preprocessor/_regrid/test__stock_cube.py +++ b/tests/unit/preprocessor/_regrid/test__stock_cube.py @@ -11,9 +11,15 @@ import numpy as np import tests -from esmvalcore.preprocessor._regrid import (_LAT_MAX, _LAT_MIN, _LAT_RANGE, - _LON_MAX, _LON_MIN, _LON_RANGE) -from esmvalcore.preprocessor._regrid import _global_stock_cube +from esmvalcore.preprocessor._regrid import ( + _LAT_MAX, + _LAT_MIN, + _LAT_RANGE, + _LON_MAX, + _LON_MIN, + _LON_RANGE, + _global_stock_cube, +) class Test(tests.Test): @@ -71,12 +77,18 @@ def _check(self, dx, dy, lat_off=True, lon_off=True): def setUp(self): self.Cube = mock.sentinel.Cube - self.mock_Cube = self.patch('iris.cube.Cube', return_value=self.Cube) + self.mock_Cube = self.patch( + 'esmvalcore.preprocessor._regrid.Cube', return_value=self.Cube + ) self.mock_coord = mock.Mock(spec=iris.coords.DimCoord) self.mock_DimCoord = self.patch( 'iris.coords.DimCoord', return_value=self.mock_coord) self.mocks = [self.mock_Cube, self.mock_coord, self.mock_DimCoord] + def tearDown(self) -> None: + _global_stock_cube.cache_clear() + return super().tearDown() + def test_invalid_cell_spec__alpha(self): emsg = 'Invalid MxN cell specification' with self.assertRaisesRegex(ValueError, emsg): diff --git a/tests/unit/preprocessor/_regrid/test_extract_point.py b/tests/unit/preprocessor/_regrid/test_extract_point.py index 5c19530ff7..304e071b36 100644 --- a/tests/unit/preprocessor/_regrid/test_extract_point.py +++ b/tests/unit/preprocessor/_regrid/test_extract_point.py @@ -7,7 +7,7 @@ import unittest from unittest import mock -import iris +from iris.tests.stock import lat_lon_cube import tests from esmvalcore.preprocessor import extract_point @@ -15,24 +15,13 @@ class Test(tests.Test): + def setUp(self): - self.coord_system = mock.Mock(return_value=None) - self.coord = mock.sentinel.coord - self.coords = mock.Mock(return_value=[self.coord]) - self.remove_coord = mock.Mock() - self.point_cube = mock.sentinel.point_cube - self.interpolate = mock.Mock(return_value=self.point_cube) - self.src_cube = mock.Mock( - spec=iris.cube.Cube, - coord_system=self.coord_system, - coords=self.coords, - remove_coord=self.remove_coord, - interpolate=self.interpolate) - self.schemes = ['linear', 'nearest'] - - self.mocks = [ - self.coord_system, self.coords, self.interpolate, self.src_cube - ] + # Use an Iris test cube with coordinates that have a coordinate + # system, see the following issue for more details: + # https://github.com/ESMValGroup/ESMValCore/issues/2177. + self.src_cube = lat_lon_cube() + self.schemes = ["linear", "nearest"] def test_invalid_scheme__unknown(self): dummy = mock.sentinel.dummy @@ -41,20 +30,30 @@ def test_invalid_scheme__unknown(self): extract_point(dummy, dummy, dummy, 'non-existent') def test_interpolation_schemes(self): - self.assertEqual( - set(POINT_INTERPOLATION_SCHEMES.keys()), set(self.schemes)) + self.assertEqual(set(POINT_INTERPOLATION_SCHEMES.keys()), + set(self.schemes)) def test_extract_point_interpolation_schemes(self): - dummy = mock.sentinel.dummy + latitude = -90. + longitude = 0. for scheme in self.schemes: - result = extract_point(self.src_cube, dummy, dummy, scheme) - self.assertEqual(result, self.point_cube) + result = extract_point(self.src_cube, latitude, longitude, scheme) + self._assert_coords(result, latitude, longitude) def test_extract_point(self): - dummy = mock.sentinel.dummy - scheme = 'linear' - result = extract_point(self.src_cube, dummy, dummy, scheme) - self.assertEqual(result, self.point_cube) + latitude = 90. + longitude = -180. + for scheme in self.schemes: + result = extract_point(self.src_cube, latitude, longitude, scheme) + self._assert_coords(result, latitude, longitude) + + def _assert_coords(self, cube, ref_lat, ref_lon): + lat_points = cube.coord("latitude").points + lon_points = cube.coord("longitude").points + self.assertEqual(len(lat_points), 1) + self.assertEqual(len(lon_points), 1) + self.assertEqual(lat_points[0], ref_lat) + self.assertEqual(lon_points[0], ref_lon) if __name__ == '__main__': diff --git a/tests/unit/preprocessor/_regrid/test_extract_regional_grid.py b/tests/unit/preprocessor/_regrid/test_extract_regional_grid.py index d22727b399..c3f4dfe79e 100644 --- a/tests/unit/preprocessor/_regrid/test_extract_regional_grid.py +++ b/tests/unit/preprocessor/_regrid/test_extract_regional_grid.py @@ -11,6 +11,18 @@ regrid, ) + +@pytest.fixture(autouse=True) +def clear_lru_cache(): + """Make sure to clear LRU cache for ``_global_stock_cube``. + + See https://github.com/ESMValGroup/ESMValCore/issues/2320. + + """ + yield # execute tests + _global_stock_cube.cache_clear() + + SPEC_KEYS = ('start_longitude', 'end_longitude', 'step_longitude', 'start_latitude', 'end_latitude', 'step_latitude') PASSING_SPECS = tuple( diff --git a/tests/unit/preprocessor/_regrid/test_regrid.py b/tests/unit/preprocessor/_regrid/test_regrid.py index b6334d0340..046da17c85 100644 --- a/tests/unit/preprocessor/_regrid/test_regrid.py +++ b/tests/unit/preprocessor/_regrid/test_regrid.py @@ -13,8 +13,7 @@ import tests from esmvalcore.preprocessor import regrid from esmvalcore.preprocessor._regrid import ( - _CACHE, - HORIZONTAL_SCHEMES, + HORIZONTAL_SCHEMES_REGULAR, _horizontal_grid_is_close, _rechunk, ) @@ -23,12 +22,10 @@ class Test(tests.Test): def _check(self, tgt_grid, scheme, spec=False): - expected_scheme = HORIZONTAL_SCHEMES[scheme] + expected_scheme = HORIZONTAL_SCHEMES_REGULAR[scheme] if spec: spec = tgt_grid - self.assertIn(spec, _CACHE) - self.assertEqual(_CACHE[spec], self.tgt_grid) self.coord_system.asset_called_once() expected_calls = [ mock.call(axis='x', dim_coords=True), @@ -37,14 +34,6 @@ def _check(self, tgt_grid, scheme, spec=False): self.assertEqual(self.tgt_grid_coord.mock_calls, expected_calls) self.regrid.assert_called_once_with(self.tgt_grid, expected_scheme) else: - if scheme == 'unstructured_nearest': - expected_calls = [ - mock.call(axis='x', dim_coords=True), - mock.call(axis='y', dim_coords=True) - ] - self.assertEqual(self.coords.mock_calls, expected_calls) - expected_calls = [mock.call(self.coord), mock.call(self.coord)] - self.assertEqual(self.remove_coord.mock_calls, expected_calls) self.regrid.assert_called_once_with(tgt_grid, expected_scheme) # Reset the mocks to enable multiple calls per test-case. @@ -73,10 +62,7 @@ def setUp(self): self.tgt_grid_coord = mock.Mock() self.tgt_grid = mock.Mock(spec=iris.cube.Cube, coord=self.tgt_grid_coord) - self.regrid_schemes = [ - 'linear', 'linear_extrapolate', 'nearest', 'area_weighted', - 'unstructured_nearest' - ] + self.regrid_schemes = ['linear', 'nearest', 'area_weighted'] def _mock_horizontal_grid_is_close(src, tgt): return False @@ -107,12 +93,12 @@ def test_invalid_tgt_grid__unknown(self): regrid(self.src_cube, dummy, scheme) def test_invalid_scheme__unknown(self): - emsg = 'Unknown regridding scheme' + emsg = "Got invalid regridding scheme string 'wibble'" with self.assertRaisesRegex(ValueError, emsg): regrid(self.src_cube, self.src_cube, 'wibble') def test_horizontal_schemes(self): - self.assertEqual(set(HORIZONTAL_SCHEMES.keys()), + self.assertEqual(set(HORIZONTAL_SCHEMES_REGULAR.keys()), set(self.regrid_schemes)) def test_regrid__horizontal_schemes(self): @@ -123,11 +109,6 @@ def test_regrid__horizontal_schemes(self): self._check(self.tgt_grid, scheme) def test_regrid__cell_specification(self): - # Clear cache before and after the test to avoid poisoning - # the cache with Mocked cubes - # https://github.com/ESMValGroup/ESMValCore/issues/953 - _CACHE.clear() - specs = ['1x1', '2x2', '3x3', '4x4', '5x5'] scheme = 'linear' for spec in specs: @@ -135,9 +116,6 @@ def test_regrid__cell_specification(self): self.assertEqual(result, self.regridded_cube) self.assertEqual(result.data, mock.sentinel.data) self._check(spec, scheme, spec=True) - self.assertEqual(set(_CACHE.keys()), set(specs)) - - _CACHE.clear() def test_regrid_generic_missing_reference(self): emsg = "No reference specified for generic regridding." diff --git a/tests/unit/preprocessor/_regrid_esmpy/test_regrid_esmpy.py b/tests/unit/preprocessor/_regrid_esmpy/test_regrid_esmpy.py index 107ab2a5c5..611966ab75 100644 --- a/tests/unit/preprocessor/_regrid_esmpy/test_regrid_esmpy.py +++ b/tests/unit/preprocessor/_regrid_esmpy/test_regrid_esmpy.py @@ -10,6 +10,9 @@ import tests from esmvalcore.preprocessor._regrid_esmpy import ( + ESMPyAreaWeighted, + ESMPyLinear, + ESMPyNearest, build_regridder, build_regridder_2d, coords_iris_to_esmpy, @@ -19,7 +22,6 @@ get_grid_representants, get_representant, is_lon_circular, - regrid, ) @@ -664,13 +666,62 @@ def test_get_grid_representants_2d_src(self, mock_cube, @mock.patch('esmvalcore.preprocessor._regrid_esmpy.build_regridder') @mock.patch('esmvalcore.preprocessor._regrid_esmpy.get_grid_representants', mock.Mock(side_effect=identity)) - def test_regrid(self, mock_build_regridder, mock_map_slices): + def test_regrid_nearest(self, mock_build_regridder, mock_map_slices): """Test full regrid method.""" mock_build_regridder.return_value = mock.sentinel.regridder mock_map_slices.return_value = mock.sentinel.regridded - regrid(self.cube_3d, self.cube) - mock_build_regridder.assert_called_once_with(self.cube_3d, self.cube, - 'linear') + regridder = ESMPyNearest().regridder(self.cube_3d, self.cube) + regridder(self.cube_3d) + mock_build_regridder.assert_called_once_with( + self.cube_3d, self.cube, 'nearest', mask_threshold=0.99 + ) + mock_map_slices.assert_called_once_with(self.cube_3d, + mock.sentinel.regridder, + self.cube_3d, self.cube) + + @mock.patch('esmvalcore.preprocessor._regrid_esmpy.map_slices') + @mock.patch('esmvalcore.preprocessor._regrid_esmpy.build_regridder') + @mock.patch('esmvalcore.preprocessor._regrid_esmpy.get_grid_representants', + mock.Mock(side_effect=identity)) + def test_regrid_linear(self, mock_build_regridder, mock_map_slices): + """Test full regrid method.""" + mock_build_regridder.return_value = mock.sentinel.regridder + mock_map_slices.return_value = mock.sentinel.regridded + regridder = ESMPyLinear().regridder(self.cube_3d, self.cube) + regridder(self.cube_3d) + mock_build_regridder.assert_called_once_with( + self.cube_3d, self.cube, 'linear', mask_threshold=0.99 + ) + mock_map_slices.assert_called_once_with(self.cube_3d, + mock.sentinel.regridder, + self.cube_3d, self.cube) + + @mock.patch('esmvalcore.preprocessor._regrid_esmpy.map_slices') + @mock.patch('esmvalcore.preprocessor._regrid_esmpy.build_regridder') + @mock.patch('esmvalcore.preprocessor._regrid_esmpy.get_grid_representants', + mock.Mock(side_effect=identity)) + def test_regrid_area_weighted(self, mock_build_regridder, mock_map_slices): + """Test full regrid method.""" + mock_build_regridder.return_value = mock.sentinel.regridder + mock_map_slices.return_value = mock.sentinel.regridded + regridder = ESMPyAreaWeighted().regridder(self.cube_3d, self.cube) + regridder(self.cube_3d) + mock_build_regridder.assert_called_once_with( + self.cube_3d, self.cube, 'area_weighted', mask_threshold=0.99 + ) mock_map_slices.assert_called_once_with(self.cube_3d, mock.sentinel.regridder, self.cube_3d, self.cube) + + +@pytest.mark.parametrize( + 'scheme,output', + [ + (ESMPyAreaWeighted(), 'ESMPyAreaWeighted(mask_threshold=0.99)'), + (ESMPyLinear(), 'ESMPyLinear(mask_threshold=0.99)'), + (ESMPyNearest(), 'ESMPyNearest(mask_threshold=0.99)'), + ] +) +def test_scheme_repr(scheme, output): + """Test ``_ESMPyScheme.__repr__``.""" + assert scheme.__repr__() == output diff --git a/tests/unit/preprocessor/_time/test_time.py b/tests/unit/preprocessor/_time/test_time.py index f9d207ed49..a89c0ba760 100644 --- a/tests/unit/preprocessor/_time/test_time.py +++ b/tests/unit/preprocessor/_time/test_time.py @@ -1518,7 +1518,7 @@ def test_timeseries_filter_timecoord(self): filter_stats='sum') def test_timeseries_filter_implemented(self): - """Test a not implemnted filter.""" + """Test a not implemented filter.""" with self.assertRaises(NotImplementedError): timeseries_filter(self.cube, 7, @@ -2160,65 +2160,5 @@ def test_resample_fails_scalar(self): resample_time(cube, day=16) -@pytest.fixture -def easy_2d_cube(): - """Create easy 2D cube to test statistical operators.""" - time = iris.coords.DimCoord( - [2.0, 3.0], - bounds=[[-0.5, 2.5], [2.5, 3.5]], - standard_name='time', - units='days since 2000-01-01', - ) - lat = iris.coords.DimCoord( - [0.0, 1.0], standard_name='latitude', units='degrees' - ) - cube = Cube( - np.arange(4, dtype=np.float32).reshape(2, 2), - standard_name='air_temperature', - units='K', - dim_coords_and_dims=[(time, 0), (lat, 1)], - ) - return cube - - -@pytest.mark.parametrize( - 'operator,kwargs,expected_data,expected_units', - [ - ('gmean', {}, [0.0, 1.7320509], 'K'), - ('hmean', {}, [0.0, 1.5], 'K'), - ('max', {}, [2.0, 3.0], 'K'), - ('mean', {}, [0.5, 1.5], 'K'), - ('mean', {'weights': False}, [1.0, 2.0], 'K'), - ('median', {}, [1.0, 2.0], 'K'), - ('min', {}, [0.0, 1.0], 'K'), - ('peak', {}, [2.0, 3.0], 'K'), - ('percentile', {'percent': 0.0}, [0.0, 1.0], 'K'), - ('rms', {}, [1.0, 1.7320509], 'K'), - ('rms', {'weights': False}, [1.414214, 2.236068], 'K'), - ('std_dev', {}, [1.414214, 1.414214], 'K'), - ('std_dev', {'ddof': 0}, [1.0, 1.0], 'K'), - ('sum', {}, [2.0, 6.0], 'K day'), - ('sum', {'weights': False}, [2.0, 4.0], 'K'), - ('variance', {}, [2.0, 2.0], 'K2'), - ('variance', {'ddof': 0}, [1.0, 1.0], 'K2'), - ('wpercentile', {'percent': 50.0}, [0.5, 1.5], 'K'), - ] -) -def test_statistical_operators( - operator, kwargs, expected_data, expected_units, easy_2d_cube -): - """Test ``climate_statistics`` with different operators.""" - res = climate_statistics(easy_2d_cube, operator, **kwargs) - - assert res.var_name == easy_2d_cube.var_name - assert res.long_name == easy_2d_cube.long_name - assert res.standard_name == easy_2d_cube.standard_name - assert res.attributes == easy_2d_cube.attributes - assert res.units == expected_units - assert res.coord('latitude') == easy_2d_cube.coord('latitude') - assert res.coord('time').shape == (1, ) - np.testing.assert_allclose(res.data, expected_data, atol=1e-6, rtol=1e-6) - - if __name__ == '__main__': unittest.main() diff --git a/tests/unit/preprocessor/_units/test_convert_units.py b/tests/unit/preprocessor/_units/test_convert_units.py index 02e67143f2..c1a2a2453b 100644 --- a/tests/unit/preprocessor/_units/test_convert_units.py +++ b/tests/unit/preprocessor/_units/test_convert_units.py @@ -44,6 +44,40 @@ def test_convert_compatible_units(self): self.assertEqual(result.units, expected_units) self.assert_array_equal(result.data, expected_data) + def test_convert_ozone_content_m_to_du(self): + """Test special conversion of ozone_content.""" + self.arr.standard_name = ( + 'equivalent_thickness_at_stp_of_atmosphere_ozone_content' + ) + self.arr.units = 'm' + result = convert_units(self.arr, 'DU') + self.assertEqual( + result.standard_name, + 'equivalent_thickness_at_stp_of_atmosphere_ozone_content', + ) + self.assertEqual(result.units, 'DU') + np.testing.assert_allclose( + result.data, + [[0.0, 1e5], [2e5, 3e5]], + ) + + def test_convert_ozone_content_du_to_m(self): + """Test special conversion of ozone_content.""" + self.arr.standard_name = ( + 'equivalent_thickness_at_stp_of_atmosphere_ozone_content' + ) + self.arr.units = 'DU' + result = convert_units(self.arr, 'mm') + self.assertEqual( + result.standard_name, + 'equivalent_thickness_at_stp_of_atmosphere_ozone_content', + ) + self.assertEqual(result.units, 'mm') + np.testing.assert_allclose( + result.data, + [[0.0, 1e-2], [2e-2, 3e-2]], + ) + def test_convert_precipitation_flux(self): """Test special conversion of precipitation_flux.""" self.arr.standard_name = 'precipitation_flux' diff --git a/tests/unit/preprocessor/_volume/test_volume.py b/tests/unit/preprocessor/_volume/test_volume.py index 2e35f98d3b..f0b730561b 100644 --- a/tests/unit/preprocessor/_volume/test_volume.py +++ b/tests/unit/preprocessor/_volume/test_volume.py @@ -173,6 +173,34 @@ def test_axis_statistics_sum(self): self.assert_array_equal(result.data, expected) self.assertEqual(result.units, 'kg m-2') + def test_axis_statistics_subtract_mean(self): + """Test axis statistics with operator mean.""" + data = np.ma.arange(1, 25).reshape(2, 3, 2, 2) + self.grid_4d.data = data + result = axis_statistics( + self.grid_4d, 'z', 'mean', normalize='subtract' + ) + bounds = self.grid_4d.coord(axis='z').bounds + weights = (bounds[:, 1] - bounds[:, 0]) + expected = ( + data - np.average(data, axis=1, weights=weights, keepdims=True) + ) + self.assert_array_equal(result.data, expected) + self.assertEqual(result.units, 'kg m-3') + self.assertFalse(self.grid_4d.coords('_axis_statistics_weights_')) + self.assertFalse(result.coords('_axis_statistics_weights_')) + + def test_axis_statistics_divide_min(self): + """Test axis statistics with operator sum.""" + data = np.ma.arange(1, 25).reshape(2, 3, 2, 2) + self.grid_4d.data = data + result = axis_statistics(self.grid_4d, 'z', 'min', normalize='divide') + expected = data / np.min(data, axis=1, keepdims=True) + self.assert_array_equal(result.data, expected) + self.assertEqual(result.units, '1') + self.assertFalse(self.grid_4d.coords('_axis_statistics_weights_')) + self.assertFalse(result.coords('_axis_statistics_weights_')) + def test_wrong_axis_statistics_fail(self): """Test raises error when axis is not found in cube.""" with self.assertRaises(ValueError) as err: @@ -360,6 +388,36 @@ def test_volume_statistics_masked_timestep(self): self.assert_array_equal(result.data, expected) self.assertEqual(result.units, 'kg m-3') + def test_volume_statistics_subtract_mean(self): + """Test to take the volume weighted average of a (2,3,2,2) cube.""" + self.assertFalse(self.grid_4d.cell_measures('ocean_volume')) + + result = volume_statistics(self.grid_4d, 'mean', normalize='subtract') + + expected = np.ma.zeros((2, 3, 2, 2)) + self.assert_array_equal(result.data, expected) + self.assertEqual(result.units, 'kg m-3') + self.assertFalse(self.grid_4d.cell_measures('ocean_volume')) + self.assertFalse(result.cell_measures('ocean_volume')) + + def test_volume_statistics_cell_measure_divide_mean(self): + """Test to take the volume weighted average of a (2,3,2,2) cube. + + The volume measure is pre-loaded in the cube. + """ + grid_volume = calculate_volume(self.grid_4d) + measure = iris.coords.CellMeasure(grid_volume, + standard_name='ocean_volume', + units='m3', + measure='volume') + self.grid_4d.add_cell_measure(measure, range(0, measure.ndim)) + + result = volume_statistics(self.grid_4d, 'mean', normalize='divide') + + expected = np.ma.ones((2, 3, 2, 2)) + self.assert_array_equal(result.data, expected) + self.assertEqual(result.units, '1') + def test_volume_statistics_weights(self): """Test to take the volume weighted average of a (2,3,2,2) cube. diff --git a/tests/unit/recipe/test_to_datasets.py b/tests/unit/recipe/test_to_datasets.py index a2fe0e76d9..ac68dd2f41 100644 --- a/tests/unit/recipe/test_to_datasets.py +++ b/tests/unit/recipe/test_to_datasets.py @@ -277,6 +277,30 @@ def test_merge_supplementaries_missing_short_name_fails(session): Dataset.from_recipe(recipe_txt, session) +def test_get_input_datasets_derive(session): + dataset = Dataset( + dataset='ERA5', + project='native6', + mip='E1hr', + short_name='rlus', + alias='ERA5', + derive=True, + force_derivation=True, + frequency='1hr', + recipe_dataset_index=0, + tier='3', + type='reanaly', + version='v1', + ) + rlds, rlns = to_datasets._get_input_datasets(dataset) + assert rlds['short_name'] == 'rlds' + assert rlds['long_name'] == 'Surface Downwelling Longwave Radiation' + assert rlds['frequency'] == '1hr' + assert rlns['short_name'] == 'rlns' + assert rlns['long_name'] == 'Surface Net downward Longwave Radiation' + assert rlns['frequency'] == '1hr' + + def test_max_years(session): recipe_txt = textwrap.dedent(""" diagnostics: @@ -300,6 +324,7 @@ def test_max_years(session): @pytest.mark.parametrize('found_files', [True, False]) def test_dataset_from_files_fails(monkeypatch, found_files): + def from_files(_): file = LocalFile('/path/to/file') file.facets = {'facets1': 'value1'} @@ -323,6 +348,7 @@ def from_files(_): def test_fix_cmip5_fx_ensemble(monkeypatch): + def find_files(self): if self.facets['ensemble'] == 'r0i0p0': self._files = ['file1.nc'] @@ -343,6 +369,7 @@ def find_files(self): def test_get_supplementary_short_names(monkeypatch): + def _update_cmor_facets(facets): facets['modeling_realm'] = 'atmos' diff --git a/tests/unit/test_iris_helpers.py b/tests/unit/test_iris_helpers.py index 07e81608e8..0b742ffe30 100644 --- a/tests/unit/test_iris_helpers.py +++ b/tests/unit/test_iris_helpers.py @@ -4,6 +4,7 @@ from itertools import permutations from unittest import mock +import dask.array as da import numpy as np import pytest from cf_units import Unit @@ -20,7 +21,10 @@ from esmvalcore.iris_helpers import ( add_leading_dim_to_cube, date2num, + has_irregular_grid, + has_unstructured_grid, merge_cube_attributes, + rechunk_cube, ) @@ -220,3 +224,272 @@ def test_merge_cube_attributes_1_cube(): merge_cube_attributes(cubes) assert len(cubes) == 1 assert_attribues_equal(cubes[0].attributes, expected_attributes) + + +@pytest.fixture +def cube_3d(): + """3D sample cube.""" + # DimCoords + x = DimCoord([0, 1, 2], var_name='x') + y = DimCoord([0, 1, 2], var_name='y') + z = DimCoord([0, 1, 2, 3], var_name='z') + + # AuxCoords + aux_x = AuxCoord( + da.ones(3, chunks=1), + bounds=da.ones((3, 3), chunks=(1, 1)), + var_name='aux_x', + ) + aux_z = AuxCoord(da.ones(4, chunks=1), var_name='aux_z') + aux_xy = AuxCoord(da.ones((3, 3), chunks=(1, 1)), var_name='xy') + aux_xz = AuxCoord(da.ones((3, 4), chunks=(1, 1)), var_name='xz') + aux_yz = AuxCoord(da.ones((3, 4), chunks=(1, 1)), var_name='yz') + aux_xyz = AuxCoord( + da.ones((3, 3, 4), chunks=(1, 1, 1)), + bounds=da.ones((3, 3, 4, 3), chunks=(1, 1, 1, 1)), + var_name='xyz', + ) + aux_coords_and_dims = [ + (aux_x, 0), + (aux_z, 2), + (aux_xy, (0, 1)), + (aux_xz, (0, 2)), + (aux_yz, (1, 2)), + (aux_xyz, (0, 1, 2)), + ] + + # CellMeasures and AncillaryVariables + cell_measure = CellMeasure( + da.ones((3, 4), chunks=(1, 1)), var_name='cell_measure' + ) + anc_var = AncillaryVariable( + da.ones((3, 4), chunks=(1, 1)), var_name='anc_var' + ) + + return Cube( + da.ones((3, 3, 4), chunks=(1, 1, 1)), + var_name='cube', + dim_coords_and_dims=[(x, 0), (y, 1), (z, 2)], + aux_coords_and_dims=aux_coords_and_dims, + cell_measures_and_dims=[(cell_measure, (1, 2))], + ancillary_variables_and_dims=[(anc_var, (0, 2))], + ) + + +def test_rechunk_cube_fully_lazy(cube_3d): + """Test ``rechunk_cube``.""" + input_cube = cube_3d.copy() + + x_coord = input_cube.coord('x') + result = rechunk_cube(input_cube, [x_coord, 'y'], remaining_dims=2) + + assert input_cube == cube_3d + assert result == cube_3d + assert result.core_data().chunksize == (3, 3, 2) + assert result.coord('aux_x').core_points().chunksize == (3,) + assert result.coord('aux_z').core_points().chunksize == (1,) + assert result.coord('xy').core_points().chunksize == (3, 3) + assert result.coord('xz').core_points().chunksize == (3, 2) + assert result.coord('yz').core_points().chunksize == (3, 2) + assert result.coord('xyz').core_points().chunksize == (3, 3, 2) + assert result.coord('aux_x').core_bounds().chunksize == (3, 2) + assert result.coord('aux_z').core_bounds() is None + assert result.coord('xy').core_bounds() is None + assert result.coord('xz').core_bounds() is None + assert result.coord('yz').core_bounds() is None + assert result.coord('xyz').core_bounds().chunksize == (3, 3, 2, 2) + assert result.cell_measure('cell_measure').core_data().chunksize == (3, 2) + assert result.ancillary_variable('anc_var').core_data().chunksize == (3, 2) + + +def test_rechunk_cube_partly_lazy(cube_3d): + """Test ``rechunk_cube``.""" + input_cube = cube_3d.copy() + + # Realize some arrays + input_cube.data + input_cube.coord('xyz').points + input_cube.coord('xyz').bounds + input_cube.cell_measure('cell_measure').data + + result = rechunk_cube(input_cube, ['x', 'y'], remaining_dims=2) + + assert input_cube == cube_3d + assert result == cube_3d + assert not result.has_lazy_data() + assert result.coord('aux_x').core_points().chunksize == (3,) + assert result.coord('aux_z').core_points().chunksize == (1,) + assert result.coord('xy').core_points().chunksize == (3, 3) + assert result.coord('xz').core_points().chunksize == (3, 2) + assert result.coord('yz').core_points().chunksize == (3, 2) + assert not result.coord('xyz').has_lazy_points() + assert result.coord('aux_x').core_bounds().chunksize == (3, 2) + assert result.coord('aux_z').core_bounds() is None + assert result.coord('xy').core_bounds() is None + assert result.coord('xz').core_bounds() is None + assert result.coord('yz').core_bounds() is None + assert not result.coord('xyz').has_lazy_bounds() + assert not result.cell_measure('cell_measure').has_lazy_data() + assert result.ancillary_variable('anc_var').core_data().chunksize == (3, 2) + + +def test_rechunk_cube_invalid_coord_fail(cube_3d): + """Test ``rechunk_cube``.""" + msg = ( + "Complete coordinates must be 1D coordinates, got 2D coordinate 'xy'" + ) + with pytest.raises(CoordinateMultiDimError, match=msg): + rechunk_cube(cube_3d, ['xy']) + + +@pytest.fixture +def lat_coord_1d(): + """1D latitude coordinate.""" + return DimCoord([0, 1], standard_name='latitude') + + +@pytest.fixture +def lon_coord_1d(): + """1D longitude coordinate.""" + return DimCoord([0, 1], standard_name='longitude') + + +@pytest.fixture +def lat_coord_2d(): + """2D latitude coordinate.""" + return AuxCoord([[0, 1]], standard_name='latitude') + + +@pytest.fixture +def lon_coord_2d(): + """2D longitude coordinate.""" + return AuxCoord([[0, 1]], standard_name='longitude') + + +def test_has_irregular_grid_no_lat_lon(): + """Test `has_irregular_grid`.""" + cube = Cube(0) + assert has_irregular_grid(cube) is False + + +def test_has_irregular_grid_no_lat(lon_coord_2d): + """Test `has_irregular_grid`.""" + cube = Cube([[0, 1]], aux_coords_and_dims=[(lon_coord_2d, (0, 1))]) + assert has_irregular_grid(cube) is False + + +def test_has_irregular_grid_no_lon(lat_coord_2d): + """Test `has_irregular_grid`.""" + cube = Cube([[0, 1]], aux_coords_and_dims=[(lat_coord_2d, (0, 1))]) + assert has_irregular_grid(cube) is False + + +def test_has_irregular_grid_1d_lon(lat_coord_2d, lon_coord_1d): + """Test `has_irregular_grid`.""" + cube = Cube( + [[0, 1]], + dim_coords_and_dims=[(lon_coord_1d, 1)], + aux_coords_and_dims=[(lat_coord_2d, (0, 1))], + ) + assert has_irregular_grid(cube) is False + + +def test_has_irregular_grid_1d_lat(lat_coord_1d, lon_coord_2d): + """Test `has_irregular_grid`.""" + cube = Cube( + [[0, 1]], + dim_coords_and_dims=[(lat_coord_1d, 1)], + aux_coords_and_dims=[(lon_coord_2d, (0, 1))], + ) + assert has_irregular_grid(cube) is False + + +def test_has_irregular_grid_1d_lat_lon(lat_coord_1d, lon_coord_1d): + """Test `has_irregular_grid`.""" + cube = Cube( + [0, 1], aux_coords_and_dims=[(lat_coord_1d, 0), (lon_coord_1d, 0)] + ) + assert has_irregular_grid(cube) is False + + +def test_has_irregular_grid_regular_grid(lat_coord_1d, lon_coord_1d): + """Test `has_irregular_grid`.""" + cube = Cube( + [[0, 1], [2, 3]], + dim_coords_and_dims=[(lat_coord_1d, 0), (lon_coord_1d, 1)], + ) + assert has_irregular_grid(cube) is False + + +def test_has_irregular_grid_true(lat_coord_2d, lon_coord_2d): + """Test `has_irregular_grid`.""" + cube = Cube( + [[0, 1]], + aux_coords_and_dims=[(lat_coord_2d, (0, 1)), (lon_coord_2d, (0, 1))], + ) + assert has_irregular_grid(cube) is True + + +def test_has_unstructured_grid_no_lat_lon(): + """Test `has_unstructured_grid`.""" + cube = Cube(0) + assert has_unstructured_grid(cube) is False + + +def test_has_unstructured_grid_no_lat(lon_coord_1d): + """Test `has_unstructured_grid`.""" + cube = Cube([0, 1], dim_coords_and_dims=[(lon_coord_1d, 0)]) + assert has_unstructured_grid(cube) is False + + +def test_has_unstructured_grid_no_lon(lat_coord_1d): + """Test `has_unstructured_grid`.""" + cube = Cube([0, 1], dim_coords_and_dims=[(lat_coord_1d, 0)]) + assert has_unstructured_grid(cube) is False + + +def test_has_unstructured_grid_2d_lat(lat_coord_2d, lon_coord_1d): + """Test `has_unstructured_grid`.""" + cube = Cube( + [[0, 1]], + dim_coords_and_dims=[(lon_coord_1d, 1)], + aux_coords_and_dims=[(lat_coord_2d, (0, 1))], + ) + assert has_unstructured_grid(cube) is False + + +def test_has_unstructured_grid_2d_lon(lat_coord_1d, lon_coord_2d): + """Test `has_unstructured_grid`.""" + cube = Cube( + [[0, 1]], + dim_coords_and_dims=[(lat_coord_1d, 1)], + aux_coords_and_dims=[(lon_coord_2d, (0, 1))], + ) + assert has_unstructured_grid(cube) is False + + +def test_has_unstructured_grid_2d_lat_lon(lat_coord_2d, lon_coord_2d): + """Test `has_unstructured_grid`.""" + cube = Cube( + [[0, 1]], + aux_coords_and_dims=[(lat_coord_2d, (0, 1)), (lon_coord_2d, (0, 1))], + ) + assert has_unstructured_grid(cube) is False + + +def test_has_unstructured_grid_regular_grid(lat_coord_1d, lon_coord_1d): + """Test `has_unstructured_grid`.""" + cube = Cube( + [[0, 1], [2, 3]], + dim_coords_and_dims=[(lat_coord_1d, 0), (lon_coord_1d, 1)], + ) + assert has_unstructured_grid(cube) is False + + +def test_has_unstructured_grid_true(lat_coord_1d, lon_coord_1d): + """Test `has_unstructured_grid`.""" + cube = Cube( + [[0, 1], [2, 3]], + aux_coords_and_dims=[(lat_coord_1d, 0), (lon_coord_1d, 0)], + ) + assert has_unstructured_grid(cube) is True