stricter URL detection for netcdf/dap

Author: ianhi

doc/whats-new.rst

@@ -24,9 +24,11 @@ Deprecations
 
 Bug fixes
 ~~~~~~~~~
-- ``netcdf`` and ``pydap`` engines no longer incorrectly claim to read all remote URLs preventing
-  the ``zarr`` backend from reading remote zarr stores without an explicit ``engine=`` argument.
-  (:pull:`10804`). By `Ian Hunt-Isaak <https://github.com/ianhi>`_.
+- ``netcdf4`` and ``pydap`` backends now use stricter URL detection to avoid incorrectly claiming
+  remote URLs. The ``pydap`` backend now only claims URLs with explicit DAP protocol indicators
+  (``dap2://`` or ``dap4://`` schemes, or ``/dap2/`` or ``/dap4/`` in the URL path). This prevents
+  both backends from claiming remote Zarr stores and other non-DAP URLs without an explicit
+  ``engine=`` argument. (:pull:`10804`). By `Ian Hunt-Isaak <https://github.com/ianhi>`_.
 
 Documentation
 ~~~~~~~~~~~~~

xarray/backends/h5netcdf_.py

@@ -462,13 +462,19 @@ class H5netcdfBackendEntrypoint(BackendEntrypoint):
     supports_groups = True
 
     def guess_can_open(self, filename_or_obj: T_PathFileOrDataStore) -> bool:
+        from xarray.core.utils import is_remote_uri, strip_uri_params
+
         filename_or_obj = _normalize_filename_or_obj(filename_or_obj)
         magic_number = try_read_magic_number_from_file_or_path(filename_or_obj)
         if magic_number is not None:
             return magic_number.startswith(b"\211HDF\r\n\032\n")
 
         if isinstance(filename_or_obj, str | os.PathLike):
-            _, ext = os.path.splitext(filename_or_obj)
+            path = str(filename_or_obj)
+            # For remote URIs, strip query parameters and fragments before checking extension
+            if isinstance(filename_or_obj, str) and is_remote_uri(path):
+                path = strip_uri_params(path)
+            _, ext = os.path.splitext(path)
             return ext in {".nc", ".nc4", ".cdf"}
 
         return False

xarray/backends/netCDF4_.py

@@ -701,26 +701,36 @@ class NetCDF4BackendEntrypoint(BackendEntrypoint):
     supports_groups = True
 
     def guess_can_open(self, filename_or_obj: T_PathFileOrDataStore) -> bool:
+        # Helper to check if magic number is netCDF or HDF5
+        def _is_netcdf_magic(magic: bytes) -> bool:
+            return magic.startswith((b"CDF", b"\211HDF\r\n\032\n"))
+
+        # Helper to check if extension is netCDF
+        def _has_netcdf_ext(path: str | os.PathLike, is_remote: bool = False) -> bool:
+            from xarray.core.utils import strip_uri_params
+
+            path = str(path).rstrip("/")
+            # For remote URIs, strip query parameters and fragments
+            if is_remote:
+                path = strip_uri_params(path)
+            _, ext = os.path.splitext(path)
+            return ext in {".nc", ".nc4", ".cdf"}
+
         if isinstance(filename_or_obj, str) and is_remote_uri(filename_or_obj):
-            # For remote URIs, check file extension to avoid claiming non-netCDF URLs
-            # (e.g., remote Zarr stores)
-            _, ext = os.path.splitext(filename_or_obj.rstrip("/"))
-            # Accept remote URIs with netCDF extensions or no extension
-            # (OPeNDAP endpoints often have no extension)
-            return ext in {".nc", ".nc4", ".cdf", ""}
-
-        magic_number = (
-            bytes(filename_or_obj[:8])
-            if isinstance(filename_or_obj, bytes | memoryview)
-            else try_read_magic_number_from_path(filename_or_obj)
-        )
-        if magic_number is not None:
-            # netcdf 3 or HDF5
-            return magic_number.startswith((b"CDF", b"\211HDF\r\n\032\n"))
+            # For remote URIs, check extension (accounting for query params/fragments)
+            # Remote netcdf-c can handle both regular URLs and DAP URLs
+            return _has_netcdf_ext(filename_or_obj, is_remote=True)
 
         if isinstance(filename_or_obj, str | os.PathLike):
-            _, ext = os.path.splitext(filename_or_obj)
-            return ext in {".nc", ".nc4", ".cdf"}
+            # For local paths, check magic number first, then extension
+            magic_number = try_read_magic_number_from_path(filename_or_obj)
+            if magic_number is not None:
+                return _is_netcdf_magic(magic_number)
+            # No magic number available, fallback to extension
+            return _has_netcdf_ext(filename_or_obj)
+
+        if isinstance(filename_or_obj, bytes | memoryview):
+            return _is_netcdf_magic(bytes(filename_or_obj[:8]))
 
         return False
 

xarray/backends/pydap_.py

@@ -210,14 +210,23 @@ class PydapBackendEntrypoint(BackendEntrypoint):
     url = "https://docs.xarray.dev/en/stable/generated/xarray.backends.PydapBackendEntrypoint.html"
 
     def guess_can_open(self, filename_or_obj: T_PathFileOrDataStore) -> bool:
-        if not (isinstance(filename_or_obj, str) and is_remote_uri(filename_or_obj)):
+        if not isinstance(filename_or_obj, str):
             return False
 
-        # Check file extension to avoid claiming non-OPeNDAP URLs (e.g., remote Zarr stores)
-        _, ext = os.path.splitext(filename_or_obj.rstrip("/"))
-        # Pydap handles OPeNDAP endpoints, which typically have no extension or .nc/.nc4
-        # Reject URLs with non-OPeNDAP extensions like .zarr
-        return ext not in {".zarr", ".zip", ".tar", ".gz"}
+        # Check for explicit DAP protocol indicators:
+        # 1. DAP scheme: dap2:// or dap4:// (case-insensitive, may not be recognized by is_remote_uri)
+        # 2. Remote URI with /dap2/ or /dap4/ in URL path (case-insensitive)
+        # Note: We intentionally do NOT check for .dap suffix as that would match
+        # file extensions like .dap which trigger downloads of binary data
+        url_lower = filename_or_obj.lower()
+        if url_lower.startswith(("dap2://", "dap4://")):
+            return True
+
+        # For standard remote URIs, check for DAP indicators in path
+        if is_remote_uri(filename_or_obj):
+            return "/dap2/" in url_lower or "/dap4/" in url_lower
+
+        return False
 
     def open_dataset(
         self,

xarray/core/utils.py

@@ -728,6 +728,40 @@ def is_remote_uri(path: str) -> bool:
     return bool(re.search(r"^[a-z][a-z0-9]*(\://|\:\:)", path))
 
 
+def strip_uri_params(uri: str) -> str:
+    """Strip query parameters and fragments from a URI.
+
+    This is useful for extracting the file extension from URLs that
+    contain query parameters (e.g., OPeNDAP constraint expressions).
+
+    Parameters
+    ----------
+    uri : str
+        The URI to strip
+
+    Returns
+    -------
+    str
+        The URI without query parameters (?) or fragments (#)
+
+    Examples
+    --------
+    >>> strip_uri_params("http://example.com/file.nc?var=temp&time=0")
+    'http://example.com/file.nc'
+    >>> strip_uri_params("http://example.com/file.nc#section")
+    'http://example.com/file.nc'
+    >>> strip_uri_params("/local/path/file.nc")
+    '/local/path/file.nc'
+    """
+    from urllib.parse import urlsplit, urlunsplit
+
+    # Use urlsplit to properly parse the URI
+    # This handles both absolute URLs and relative paths
+    parsed = urlsplit(uri)
+    # Reconstruct without query and fragment using urlunsplit
+    return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, "", ""))
+
+
 def read_magic_number_from_file(filename_or_obj, count=8) -> bytes:
     # check byte header to determine file type
     if not isinstance(filename_or_obj, io.IOBase):

xarray/tests/test_backends.py

@@ -7159,7 +7159,10 @@ def test_netcdf4_entrypoint(tmp_path: Path) -> None:
     _check_guess_can_open_and_open(entrypoint, path, engine="netcdf4", expected=ds)
     _check_guess_can_open_and_open(entrypoint, str(path), engine="netcdf4", expected=ds)
 
-    assert entrypoint.guess_can_open("http://something/remote")
+    # Remote URLs without extensions are no longer claimed (stricter detection)
+    assert not entrypoint.guess_can_open("http://something/remote")
+    # Remote URLs with netCDF extensions are claimed
+    assert entrypoint.guess_can_open("http://something/remote.nc")
     assert entrypoint.guess_can_open("something-local.nc")
     assert entrypoint.guess_can_open("something-local.nc4")
     assert entrypoint.guess_can_open("something-local.cdf")
@@ -7202,6 +7205,10 @@ def test_scipy_entrypoint(tmp_path: Path) -> None:
     assert entrypoint.guess_can_open("something-local.nc.gz")
     assert not entrypoint.guess_can_open("not-found-and-no-extension")
     assert not entrypoint.guess_can_open(b"not-a-netcdf-file")
+    # Should not claim .gz files that aren't netCDF
+    assert not entrypoint.guess_can_open("something.zarr.gz")
+    assert not entrypoint.guess_can_open("something.tar.gz")
+    assert not entrypoint.guess_can_open("something.txt.gz")
 
 
 @requires_h5netcdf
@@ -7252,75 +7259,73 @@ def test_zarr_entrypoint(tmp_path: Path) -> None:
     assert not entrypoint.guess_can_open("something.zarr.txt")
 
 
+@requires_h5netcdf
 @requires_netCDF4
 @requires_pydap
 @requires_zarr
 def test_remote_url_backend_auto_detection() -> None:
     """
-    Test that remote URLs are correctly claimed by appropriate backends.
+    Test that remote URLs are correctly selected by the backend resolution system.
 
-    This tests the fix for issue where netCDF4 and pydap backends were
+    This tests the fix for issue where netCDF4, h5netcdf, and pydap backends were
     claiming ALL remote URLs, preventing remote Zarr stores from being
     auto-detected.
 
     See: https://github.com/pydata/xarray/issues/10801
     """
-    from xarray.backends.netCDF4_ import NetCDF4BackendEntrypoint
-    from xarray.backends.pydap_ import PydapBackendEntrypoint
-    from xarray.backends.zarr import ZarrBackendEntrypoint
-
-    netcdf4_entrypoint = NetCDF4BackendEntrypoint()
-    pydap_entrypoint = PydapBackendEntrypoint()
-    zarr_entrypoint = ZarrBackendEntrypoint()
-
-    # Remote Zarr URLs should be claimed by Zarr backend, not netCDF4/pydap
-    remote_zarr_urls = [
-        "https://example.com/store.zarr",
-        "http://example.com/data.zarr/",
-        "s3://bucket/path/to/data.zarr",
+    from xarray.backends.plugins import guess_engine
+
+    # Test cases: (url, expected_backend)
+    test_cases = [
+        # Remote Zarr URLs
+        ("https://example.com/store.zarr", "zarr"),
+        ("http://example.com/data.zarr/", "zarr"),
+        ("s3://bucket/path/to/data.zarr", "zarr"),
+        # Remote netCDF URLs (non-DAP) - h5netcdf wins (first in order)
+        ("https://example.com/file.nc", "h5netcdf"),
+        ("http://example.com/data.nc4", "h5netcdf"),
+        ("https://example.com/test.cdf", "h5netcdf"),
+        ("https://example.com/data.nc?var=temperature&time=0", "h5netcdf"),
+        # DAP URLs with query parameters - h5netcdf wins (has .nc4 ext, first in order)
+        (
+            "http://test.opendap.org/opendap/dap4/StaggeredGrid.nc4?dap4.ce=/time[0:1:0]",
+            "h5netcdf",
+        ),
+        # DAP URLs without extensions - pydap wins
+        ("dap2://opendap.earthdata.nasa.gov/collections/dataset", "pydap"),
+        ("dap4://opendap.earthdata.nasa.gov/collections/dataset", "pydap"),
+        ("DAP2://example.com/dataset", "pydap"),  # uppercase scheme
+        ("DAP4://example.com/dataset", "pydap"),  # uppercase scheme
+        ("https://example.com/services/DAP2/dataset", "pydap"),  # uppercase in path
+        # DAP URLs with .nc extensions - h5netcdf wins (first in order)
+        ("http://test.opendap.org/opendap/dap4/StaggeredGrid.nc4", "h5netcdf"),
+        ("https://example.com/DAP4/data.nc", "h5netcdf"),
+        ("http://example.com/data/Dap4/file.nc", "h5netcdf"),
+        ("s3://bucket/path/to/data.nc", "h5netcdf"),
     ]
 
-    for url in remote_zarr_urls:
-        assert zarr_entrypoint.guess_can_open(url), f"Zarr should claim {url}"
-        assert not netcdf4_entrypoint.guess_can_open(url), (
-            f"NetCDF4 should not claim {url}"
+    for url, expected_backend in test_cases:
+        engine = guess_engine(url)
+        assert engine == expected_backend, (
+            f"URL {url!r} should select {expected_backend!r} but got {engine!r}"
         )
-        assert not pydap_entrypoint.guess_can_open(url), f"Pydap should not claim {url}"
 
-    # Remote netCDF URLs with extensions should be claimed by netCDF4, not Zarr
-    remote_netcdf_urls_with_ext = [
-        "https://example.com/file.nc",
-        "http://example.com/data.nc4",
-        "https://example.com/test.cdf",
+    # URLs that should raise ValueError (no backend can open them)
+    invalid_urls = [
+        "http://test.opendap.org/opendap/data/nc/coads_climatology.nc.dap",  # .dap suffix
+        "https://example.com/data.dap",  # .dap suffix
+        "http://opendap.example.com/data",  # no extension, no DAP indicators
+        "https://test.opendap.org/dataset",  # no extension, no DAP indicators
     ]
 
-    for url in remote_netcdf_urls_with_ext:
-        assert not zarr_entrypoint.guess_can_open(url), f"Zarr should not claim {url}"
-        assert netcdf4_entrypoint.guess_can_open(url), f"NetCDF4 should claim {url}"
-
-    # OPeNDAP endpoints (no extension) should be claimed by both netCDF4 and pydap
-    opendap_urls = [
-        "http://opendap.example.com/data",
-        "https://test.opendap.org/dataset",
-    ]
-
-    for url in opendap_urls:
-        assert not zarr_entrypoint.guess_can_open(url), f"Zarr should not claim {url}"
-        assert netcdf4_entrypoint.guess_can_open(url), f"NetCDF4 should claim {url}"
-        assert pydap_entrypoint.guess_can_open(url), f"Pydap should claim {url}"
-
-    # Other file types should not be claimed
-    other_urls = [
-        "https://example.com/data.zip",
-        "https://example.com/data.tar.gz",
-    ]
-
-    for url in other_urls:
-        assert not zarr_entrypoint.guess_can_open(url), f"Zarr should not claim {url}"
-        assert not netcdf4_entrypoint.guess_can_open(url), (
-            f"NetCDF4 should not claim {url}"
-        )
-        assert not pydap_entrypoint.guess_can_open(url), f"Pydap should not claim {url}"
+    for url in invalid_urls:
+        try:
+            engine = guess_engine(url)
+            raise AssertionError(
+                f"URL {url!r} should not be claimed by any backend, but {engine!r} claimed it"
+            )
+        except ValueError:
+            pass  # Expected
 
 
 @requires_netCDF4