feat: Add consistent URL representation for all storage paths (#1326)

Implements unified URL handling for all storage backends including local files:

- Add URL_PROTOCOLS tuple including file://
- Add is_url() to check if path is a URL
- Add normalize_to_url() to convert local paths to file:// URLs
- Add parse_url() to parse any URL into protocol and path
- Add StorageBackend.get_url() to return full URLs for any backend
- Add comprehensive unit tests for URL functions

This enables consistent internal representation across all storage types,
aligning with fsspec's unified approach to filesystems.

Closes #1326

Co-Authored-By: Claude Opus 4.5 <[email protected]>

Author: dimitri-yatsenko

src/datajoint/objectref.py

@@ -128,32 +128,6 @@ def to_json(self) -> dict:
             data["item_count"] = self.item_count
         return data
 
-    def to_dict(self) -> dict:
-        """
-        Return the raw JSON metadata as a dictionary.
-
-        This is useful for inspecting the stored metadata without triggering
-        any storage backend operations. The returned dict matches the JSON
-        structure stored in the database.
-
-        Returns
-        -------
-        dict
-            Dict containing the object metadata:
-
-            - path: Relative storage path within the store
-            - url: Full URI (e.g., 's3://bucket/path') (optional)
-            - store: Store name (optional, None for default store)
-            - size: File/folder size in bytes (or None)
-            - hash: Content hash (or None)
-            - ext: File extension (or None)
-            - is_dir: True if folder
-            - timestamp: Upload timestamp
-            - mime_type: MIME type (files only, optional)
-            - item_count: Number of files (folders only, optional)
-        """
-        return self.to_json()
-
     def _ensure_backend(self):
         """Ensure storage backend is available for I/O operations."""
         if self._backend is None:

src/datajoint/storage.py

@@ -24,13 +24,13 @@
 # Characters safe for use in filenames and URLs
 TOKEN_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
 
-# Supported remote URL protocols for copy insert
-REMOTE_PROTOCOLS = ("s3://", "gs://", "gcs://", "az://", "abfs://", "http://", "https://")
+# Supported URL protocols
+URL_PROTOCOLS = ("file://", "s3://", "gs://", "gcs://", "az://", "abfs://", "http://", "https://")
 
 
-def is_remote_url(path: str) -> bool:
+def is_url(path: str) -> bool:
     """
-    Check if a path is a remote URL.
+    Check if a path is a URL.
 
     Parameters
     ----------
@@ -40,19 +40,57 @@ def is_remote_url(path: str) -> bool:
     Returns
     -------
     bool
-        True if path starts with a supported remote protocol.
+        True if path starts with a supported URL protocol.
     """
-    return path.lower().startswith(REMOTE_PROTOCOLS)
+    return path.lower().startswith(URL_PROTOCOLS)
 
 
-def parse_remote_url(url: str) -> tuple[str, str]:
+def normalize_to_url(path: str) -> str:
     """
-    Parse a remote URL into protocol and path.
+    Normalize a path to URL form.
+
+    Converts local filesystem paths to file:// URLs. URLs are returned unchanged.
+
+    Parameters
+    ----------
+    path : str
+        Path string (local path or URL).
+
+    Returns
+    -------
+    str
+        URL form of the path.
+
+    Examples
+    --------
+    >>> normalize_to_url("/data/file.dat")
+    'file:///data/file.dat'
+    >>> normalize_to_url("s3://bucket/key")
+    's3://bucket/key'
+    >>> normalize_to_url("file:///already/url")
+    'file:///already/url'
+    """
+    if is_url(path):
+        return path
+    # Convert local path to file:// URL
+    # Ensure absolute path and proper format
+    abs_path = str(Path(path).resolve())
+    # Handle Windows paths (C:\...) vs Unix paths (/...)
+    if abs_path.startswith("/"):
+        return f"file://{abs_path}"
+    else:
+        # Windows: file:///C:/path
+        return f"file:///{abs_path.replace(chr(92), '/')}"
+
+
+def parse_url(url: str) -> tuple[str, str]:
+    """
+    Parse a URL into protocol and path.
 
     Parameters
     ----------
     url : str
-        Remote URL (e.g., ``'s3://bucket/path/file.dat'``).
+        URL (e.g., ``'s3://bucket/path/file.dat'`` or ``'file:///path/to/file'``).
 
     Returns
     -------
@@ -63,11 +101,19 @@ def parse_remote_url(url: str) -> tuple[str, str]:
     ------
     DataJointError
         If URL protocol is not supported.
+
+    Examples
+    --------
+    >>> parse_url("s3://bucket/key/file.dat")
+    ('s3', 'bucket/key/file.dat')
+    >>> parse_url("file:///data/file.dat")
+    ('file', '/data/file.dat')
     """
     url_lower = url.lower()
 
     # Map URL schemes to fsspec protocols
     protocol_map = {
+        "file://": "file",
         "s3://": "s3",
         "gs://": "gcs",
         "gcs://": "gcs",
@@ -82,7 +128,7 @@ def parse_remote_url(url: str) -> tuple[str, str]:
             path = url[len(prefix) :]
             return protocol, path
 
-    raise errors.DataJointError(f"Unsupported remote URL protocol: {url}")
+    raise errors.DataJointError(f"Unsupported URL protocol: {url}")
 
 
 def generate_token(length: int = 8) -> str:
@@ -356,6 +402,53 @@ def _full_path(self, path: str | PurePosixPath) -> str:
                 return str(Path(location) / path)
             return path
 
+    def get_url(self, path: str | PurePosixPath) -> str:
+        """
+        Get the full URL for a path in storage.
+
+        Returns a consistent URL representation for any storage backend,
+        including file:// URLs for local filesystem.
+
+        Parameters
+        ----------
+        path : str or PurePosixPath
+            Relative path within the storage location.
+
+        Returns
+        -------
+        str
+            Full URL (e.g., 's3://bucket/path' or 'file:///data/path').
+
+        Examples
+        --------
+        >>> backend = StorageBackend({"protocol": "file", "location": "/data"})
+        >>> backend.get_url("schema/table/file.dat")
+        'file:///data/schema/table/file.dat'
+
+        >>> backend = StorageBackend({"protocol": "s3", "bucket": "mybucket", ...})
+        >>> backend.get_url("schema/table/file.dat")
+        's3://mybucket/schema/table/file.dat'
+        """
+        full_path = self._full_path(path)
+
+        if self.protocol == "file":
+            # Ensure absolute path for file:// URL
+            abs_path = str(Path(full_path).resolve())
+            if abs_path.startswith("/"):
+                return f"file://{abs_path}"
+            else:
+                # Windows path
+                return f"file:///{abs_path.replace(chr(92), '/')}"
+        elif self.protocol == "s3":
+            return f"s3://{full_path}"
+        elif self.protocol == "gcs":
+            return f"gs://{full_path}"
+        elif self.protocol == "azure":
+            return f"az://{full_path}"
+        else:
+            # Fallback: use protocol prefix
+            return f"{self.protocol}://{full_path}"
+
     def put_file(self, local_path: str | Path, remote_path: str | PurePosixPath, metadata: dict | None = None) -> None:
         """
         Upload a file from local filesystem to storage.
@@ -672,7 +765,7 @@ def copy_from_url(self, source_url: str, dest_path: str | PurePosixPath) -> int:
         int
             Size of copied file in bytes.
         """
-        protocol, source_path = parse_remote_url(source_url)
+        protocol, source_path = parse_url(source_url)
         full_dest = self._full_path(dest_path)
 
         logger.debug(f"copy_from_url: {protocol}://{source_path} -> {self.protocol}:{full_dest}")
@@ -772,8 +865,8 @@ def source_is_directory(self, source: str) -> bool:
         bool
             True if source is a directory.
         """
-        if is_remote_url(source):
-            protocol, path = parse_remote_url(source)
+        if is_url(source):
+            protocol, path = parse_url(source)
             source_fs = fsspec.filesystem(protocol)
             return source_fs.isdir(path)
         else:
@@ -793,8 +886,8 @@ def source_exists(self, source: str) -> bool:
         bool
             True if source exists.
         """
-        if is_remote_url(source):
-            protocol, path = parse_remote_url(source)
+        if is_url(source):
+            protocol, path = parse_url(source)
             source_fs = fsspec.filesystem(protocol)
             return source_fs.exists(path)
         else:
@@ -815,8 +908,8 @@ def get_source_size(self, source: str) -> int | None:
             Size in bytes, or None if directory or cannot determine.
         """
         try:
-            if is_remote_url(source):
-                protocol, path = parse_remote_url(source)
+            if is_url(source):
+                protocol, path = parse_url(source)
                 source_fs = fsspec.filesystem(protocol)
                 if source_fs.isdir(path):
                     return None

tests/unit/test_storage_urls.py

@@ -0,0 +1,121 @@
+"""Unit tests for storage URL functions."""
+
+import pytest
+
+from datajoint.storage import (
+    URL_PROTOCOLS,
+    is_url,
+    normalize_to_url,
+    parse_url,
+)
+
+
+class TestURLProtocols:
+    """Test URL protocol constants."""
+
+    def test_url_protocols_includes_file(self):
+        """URL_PROTOCOLS should include file://."""
+        assert "file://" in URL_PROTOCOLS
+
+    def test_url_protocols_includes_s3(self):
+        """URL_PROTOCOLS should include s3://."""
+        assert "s3://" in URL_PROTOCOLS
+
+    def test_url_protocols_includes_cloud_providers(self):
+        """URL_PROTOCOLS should include major cloud providers."""
+        assert "gs://" in URL_PROTOCOLS
+        assert "az://" in URL_PROTOCOLS
+
+
+class TestIsUrl:
+    """Test is_url function."""
+
+    def test_s3_url(self):
+        assert is_url("s3://bucket/key")
+
+    def test_gs_url(self):
+        assert is_url("gs://bucket/key")
+
+    def test_file_url(self):
+        assert is_url("file:///path/to/file")
+
+    def test_http_url(self):
+        assert is_url("http://example.com/file")
+
+    def test_https_url(self):
+        assert is_url("https://example.com/file")
+
+    def test_local_path_not_url(self):
+        assert not is_url("/path/to/file")
+
+    def test_relative_path_not_url(self):
+        assert not is_url("relative/path/file.dat")
+
+    def test_case_insensitive(self):
+        assert is_url("S3://bucket/key")
+        assert is_url("FILE:///path")
+
+
+class TestNormalizeToUrl:
+    """Test normalize_to_url function."""
+
+    def test_local_path_to_file_url(self):
+        url = normalize_to_url("/data/file.dat")
+        assert url.startswith("file://")
+        assert "data/file.dat" in url
+
+    def test_s3_url_unchanged(self):
+        url = "s3://bucket/key/file.dat"
+        assert normalize_to_url(url) == url
+
+    def test_file_url_unchanged(self):
+        url = "file:///data/file.dat"
+        assert normalize_to_url(url) == url
+
+    def test_relative_path_becomes_absolute(self):
+        url = normalize_to_url("relative/path.dat")
+        assert url.startswith("file://")
+        # Should be absolute (contain full path)
+        assert "/" in url[7:]  # After "file://"
+
+
+class TestParseUrl:
+    """Test parse_url function."""
+
+    def test_parse_s3(self):
+        protocol, path = parse_url("s3://bucket/key/file.dat")
+        assert protocol == "s3"
+        assert path == "bucket/key/file.dat"
+
+    def test_parse_gs(self):
+        protocol, path = parse_url("gs://bucket/key")
+        assert protocol == "gcs"
+        assert path == "bucket/key"
+
+    def test_parse_gcs(self):
+        protocol, path = parse_url("gcs://bucket/key")
+        assert protocol == "gcs"
+        assert path == "bucket/key"
+
+    def test_parse_file(self):
+        protocol, path = parse_url("file:///data/file.dat")
+        assert protocol == "file"
+        assert path == "/data/file.dat"
+
+    def test_parse_http(self):
+        protocol, path = parse_url("http://example.com/file")
+        assert protocol == "http"
+        assert path == "example.com/file"
+
+    def test_parse_https(self):
+        protocol, path = parse_url("https://example.com/file")
+        assert protocol == "https"
+        assert path == "example.com/file"
+
+    def test_unsupported_protocol_raises(self):
+        with pytest.raises(Exception, match="Unsupported URL protocol"):
+            parse_url("ftp://example.com/file")
+
+    def test_local_path_raises(self):
+        with pytest.raises(Exception, match="Unsupported URL protocol"):
+            parse_url("/local/path")