Add intake-esgf, update 2025-06 (#150)

# Overview

This PR fixes Ouranosinc/pavics-sdi#358 by
adding `intake-esgf` to the docker conda recipe. It also updates a
handful of core dependencies.

> [!NOTE]
> The `esgf-dap.ipynb` notebook was not included in the Jenkins buildlog
but was independently tested in a secondary build and was passing.

## Changes

- Jupyter env changes:
  - Added `intake-esgf`
    - Ouranosinc/pavics-sdi#358
  - Updated `ravenpy` and other core libraries
    - CSHS-CWRA/RavenPy#524
  - Relevant changes (alphabetical order):
```diff
>   - globus-sdk=3.61.0=pyhd8ed1ab_0
>   - intake-esgf=2025.6.6=pyhd8ed1ab_0

<   - ravenpy=0.18.1=pyhd8ed1ab_0
>   - ravenpy=0.19.0=pyhd8ed1ab_0

<   - xclim=0.56.0=pyhd8ed1ab_0
>   - xclim=0.57.0=pyhd8ed1ab_0

<   - xscen=0.12.1=pyhd8ed1ab_0
>   - xscen=0.12.3=pyhd8ed1ab_0

<   - xsdba=0.4.0=pyhd8ed1ab_0
>   - xsdba=0.5.0=pyhd8ed1ab_0
```

## Testing Checklist

- [x] Deployed as "alpha" image in production for bokeh visualization
performance regression testing.
- [x] Manually tested notebook
https://github.com/Ouranosinc/PAVICS-landing/blob/master/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-5Visualization.ipynb
for bokeh visualization performance.
- [x] Committed the Jenkins build log to this Pull Request:
https://github.com/Ouranosinc/PAVICS-e2e-workflow-tests/blob/807da3100f5964f18855f84cd6c30f26516fcd77/docker/saved_buildout/jenkins-buildlogs-default.txt

## Related Issue / Discussion

- Matching notebook fixes:
  - pavics-sdi:
    - Ouranosinc/pavics-sdi#358
    - Ouranosinc/pavics-sdi#359
      - These need to be merged together
        - This was performed 
- PAVICS-landing: Ouranosinc/PAVICS-landing#111
  - RavenPy: CSHS-CWRA/RavenPy#524

- Deployment to PAVICS:
bird-house/birdhouse-deploy#567


- Previous release:
#147

## Additional Information

Full diff of the conda env export:

release-py311-250423...807da31

Full new conda env export:

https://github.com/Ouranosinc/PAVICS-e2e-workflow-tests/blob/807da3100f5964f18855f84cd6c30f26516fcd77/docker/saved_buildout/conda-env-export.yml

DockerHub build log:

https://github.com/Ouranosinc/PAVICS-e2e-workflow-tests/blob/807da3100f5964f18855f84cd6c30f26516fcd77/docker/saved_buildout/docker-buildlogs.txt

Author: Zeitsperre

Jenkinsfile

@@ -10,7 +10,7 @@ pipeline {
     // https://jenkins.io/doc/book/pipeline/syntax/
     agent {
         docker {
-            image "pavics/workflow-tests:py311-250423"
+            image "pavics/workflow-tests:py311-250423-update250730"
             label 'linux && docker'
         }
     }
@@ -61,7 +61,7 @@ Requires 'weaver' component to be active on the target 'PAVICS_HOST' server
 (see https://github.com/bird-house/birdhouse-deploy/tree/master/birdhouse/components#weaver).''')
         booleanParam(name: 'TEST_FINCH_REPO', defaultValue: true,
                      description: 'Check the box to test finch repo.')
-        string(name: 'FINCH_BRANCH', defaultValue: 'master',
+        string(name: 'FINCH_BRANCH', defaultValue: 'main',
                description: 'FINCH_REPO branch to test against.', trim: true)
         string(name: 'FINCH_REPO', defaultValue: 'bird-house/finch',
                description: 'https://github.com/bird-house/finch repo or fork to test against.', trim: true)
@@ -79,7 +79,7 @@ Requires 'weaver' component to be active on the target 'PAVICS_HOST' server
                description: 'https://github.com/Ouranosinc/raven repo or fork to test against.', trim: true)
         booleanParam(name: 'TEST_RAVENPY_REPO', defaultValue: true,
                      description: 'Check the box to test RavenPy repo.')
-        string(name: 'RAVENPY_BRANCH', defaultValue: 'master',
+        string(name: 'RAVENPY_BRANCH', defaultValue: 'main',
                description: 'RAVENPY_REPO branch to test against.', trim: true)
         string(name: 'RAVENPY_REPO', defaultValue: 'CSHS-CWRA/RavenPy',
                description: 'https://github.com/CSHS-CWRA/RavenPy repo or fork to test against.', trim: true)

binder/Dockerfile

@@ -1,4 +1,4 @@
-FROM pavics/workflow-tests:py311-250423
+FROM pavics/workflow-tests:py311-250423-update250730
 
 USER root
 

default_build_params

@@ -29,7 +29,7 @@ else
 fi
 
 if [ -z "$FINCH_BRANCH" ]; then
-    FINCH_BRANCH=master
+    FINCH_BRANCH=main
     echo "FINCH_BRANCH not set, default to '$FINCH_BRANCH'"
 else
     echo "FINCH_BRANCH has been set to '$FINCH_BRANCH'"
@@ -92,7 +92,7 @@ else
 fi
 
 if [ -z "$RAVENPY_BRANCH" ]; then
-    RAVENPY_BRANCH=master
+    RAVENPY_BRANCH=main
     echo "RAVENPY_BRANCH not set, default to '$RAVENPY_BRANCH'"
 else
     echo "RAVENPY_BRANCH has been set to '$RAVENPY_BRANCH'"

docker/Dockerfile.testing

@@ -1,6 +1,6 @@
 # For testing quickly without having to do a full rebuild.
 
-FROM pavics/workflow-tests:py311-250415
+FROM pavics/workflow-tests:py311-250423
 
 # ENV ESMFMKFILE="/opt/conda/envs/birdy/lib/esmf.mk"
 # ENV PROJ_DATA="/opt/conda/envs/birdy/share/proj"
@@ -9,11 +9,24 @@ USER root
 
 # Use 'update' for existing and 'install' for new package.
 # Keep same channel ordering to not revert anything.
+# Be sure to specify the environment name with '-n environment'.
+RUN umask 0000 && \
+    conda config --prepend channels nodefaults || true && \
+    conda config --remove channels defaults || true && \
+    mamba install --force-reinstall -c conda-forge -c cdat -c bokeh -c fortiers -n birdy intake-esgf==2025.6.6 numpy==2.2.6 && \
+    mamba update -c conda-forge -c cdat -c bokeh -c fortiers -n birdy numpy==2.2.6 ravenpy==0.19.0 xclim==0.57.0 xscen==0.12.3 xsdba==0.5.0 && \
+    mamba clean --all --yes
+
+RUN umask 0000 && \
+    curl -sSL https://raw.githubusercontent.com/CSHS-CWRA/RavenPy/refs/heads/fix-notebooks/src/ravenpy/testing/registry.txt \
+    -o /opt/conda/envs/birdy/lib/python3.11/site-packages/ravenpy/testing/registry.txt && \
+    chmod 777 /opt/conda/envs/birdy/lib/python3.11/site-packages/ravenpy/testing/registry.txt
+
 # RUN umask 0000 && \
 #     conda config --prepend channels nodefaults || true && \
 #     conda config --remove channels defaults || true && \
 #     pip uninstall -y hsclient && \
-#     mamba install --force-reinstall -c conda-forge -c cdat -c bokeh -c fortiers ravenpy==0.18.1 sparse==0.15.5 && \
+#     mamba install --force-reinstall -c conda-forge -c cdat -c bokeh -c fortiers -n birdy ravenpy==0.18.1 sparse==0.15.5 && \
 #     mamba clean --all --yes
 
 # RUN umask 0000 && \

docker/environment.yml

@@ -21,19 +21,19 @@ dependencies:
   ## CORE LIBRARIES
 
   # clisops: Data reduction operations on xarray climate data
-  - clisops >= 0.16.1
+  - clisops >= 0.16.2
   # figanos: Outils pour produire des graphiques informatifs sur les impacts des changements climatiques.
   - figanos >= 0.5.0
   # RavenPy: A Python wrapper for configuring and running the hydrologic modelling framework Raven.
-  - ravenpy >= 0.18.1
+  - ravenpy >= 0.19.0
   # xclim: Climate indices computation package based on Xarray.
-  - xclim >= 0.56.0
+  - xclim >= 0.57.0
   # Hydrological analysis library built with xarray.
   # - xhydro  # 2025-04: Currently deactivated due to older dependency pins
   # xscen: A climate change scenario-building analysis framework, built with xclim/xarray.
-  - xscen >= 0.12.1
+  - xscen >= 0.12.3
   # xsdba: Statistical correction and bias adjustment tools for xarray.
-  - xsdba >= 0.4.0
+  - xsdba >= 0.5.0
 
   ## COMMON LIBRARIES
   # Only add pins here as needed
@@ -57,7 +57,7 @@ dependencies:
   # netcdf4: Python interface to the netCDF C library
   # - netcdf4  # managed by core libraries
   # numpy: fundamental package for scientific computing with Python
-  - numpy >= 1.24
+  - numpy >= 1.25
   # pandas: data structures and data analysis tools for Python
   # - pandas  # managed by core libraries
   # pydantic: Data validation and settings management using Python type annotations
@@ -153,6 +153,8 @@ dependencies:
   # https://intake.readthedocs.io/en/latest/
   - intake
   # https://intake.readthedocs.io/en/latest/plugin-directory.html
+  # intake-esgf: Plugin for building and loading intake catalogues for ESGF (Earth System Grid Federation) data sets.
+  - intake-esgf >= 2025.6.6
   # intake-esm: Plugin for building and loading intake catalogues for earth system data sets
   # holdings, such as CMIP (Coupled Model Intercomparison Project) and CESM Large Ensemble datasets.
   - intake-esm >= 2025.2.3

docker/saved_buildout/conda-env-export.yml

@@ -186,6 +186,7 @@ dependencies:
   - giflib=5.2.2=hd590300_0
   - gitdb=4.0.12=pyhd8ed1ab_0
   - gitpython=3.1.44=pyhff2d567_0
+  - globus-sdk=3.61.0=pyhd8ed1ab_0
   - glog=0.7.1=hbabe93e_0
   - google-api-core=2.24.2=pyhd8ed1ab_0
   - google-auth=2.39.0=pyhd8ed1ab_0
@@ -221,6 +222,7 @@ dependencies:
   - importlib_resources=6.5.2=pyhd8ed1ab_0
   - iniconfig=2.0.0=pyhd8ed1ab_1
   - intake=2.0.8=pyhd8ed1ab_0
+  - intake-esgf=2025.6.6=pyhd8ed1ab_0
   - intake-esm=2025.2.3=pyhd8ed1ab_0
   - intake-geopandas=0.4.0=pyhd8ed1ab_1
   - intake-thredds=2025.2.18=pyhd8ed1ab_0
@@ -544,7 +546,7 @@ dependencies:
   - rasterio=1.4.3=py311he01b476_1
   - rav1e=0.6.6=he8a937b_2
   - raven-hydro=0.4.0=py311h81cb690_0
-  - ravenpy=0.18.1=pyhd8ed1ab_0
+  - ravenpy=0.19.0=pyhd8ed1ab_0
   - rdma-core=57.0=h5888daf_0
   - re2=2024.07.02=h9925aae_3
   - readline=8.2=h8c095d6_2
@@ -662,7 +664,7 @@ dependencies:
   - xcb-util-keysyms=0.4.1=hb711507_0
   - xcb-util-renderutil=0.3.10=hb711507_0
   - xcb-util-wm=0.4.2=hb711507_0
-  - xclim=0.56.0=pyhd8ed1ab_0
+  - xclim=0.57.0=pyhd8ed1ab_0
   - xerces-c=3.2.5=h988505b_2
   - xesmf=0.8.9=pyhd8ed1ab_0
   - xeus=5.2.1=hf4f7e68_0
@@ -692,9 +694,9 @@ dependencies:
   - xorg-libxtst=1.2.5=hb9d3cd8_3
   - xorg-libxxf86vm=1.1.6=hb9d3cd8_0
   - xrft=1.0.1=pyhd8ed1ab_1
-  - xscen=0.12.1=pyhd8ed1ab_0
+  - xscen=0.12.3=pyhd8ed1ab_0
   - xsdata=25.4=pyhd8ed1ab_0
-  - xsdba=0.4.0=pyhd8ed1ab_0
+  - xsdba=0.5.0=pyhd8ed1ab_0
   - xskillscore=0.0.26=pyhd8ed1ab_1
   - xxhash=0.8.3=hb9d3cd8_0
   - xyzservices=2025.1.0=pyhd8ed1ab_0