WIP: Patch open, os and os.path builtins (#322)

* WIP

* Working implementation

* more WIP

* Implement glob

* WIP docs and tests

* tests and docs

* tests, docs, compatibility

* copilot review

* update history

* code review changes

* pin pytest-rerunfailures

Author: pjbull

HISTORY.md

@@ -1,9 +1,11 @@
 # cloudpathlib Changelog
 
-## UNRELEASED
+## v0.22.0 (2025-08-29)
 
 - Fixed issue with GS credentials, using default auth enables a wider set of authentication methods in GS (Issue [#390](https://github.com/drivendataorg/cloudpathlib/issues/390), PR [#514](https://github.com/drivendataorg/cloudpathlib/pull/514), thanks @ljyanesm) 
 - Added support for http(s) urls with `HttpClient`, `HttpPath`, `HttpsClient`, and `HttpsPath`. (Issue [#455](https://github.com/drivendataorg/cloudpathlib/issues/455), PR [#468](https://github.com/drivendataorg/cloudpathlib/pull/468))
+- Added experimental support for patching the builtins `open`, `os`, `os.path`, and `glob` to work with `CloudPath` objects. It is off by default; see the new "Compatibility" section in the docs for more information. (Issue [#128](https://github.com/drivendataorg/cloudpathlib/issues/128), PR [#322](https://github.com/drivendataorg/cloudpathlib/pull/322))
+- Added support for `CloudPath(*parts)` to create a `CloudPath` object from a list of parts (to match `pathlib.Path`). **This is a potentially breaking change for users that relied on the second arg being the `client` instead of making it an explicit kwarg.** (PR [#322](https://github.com/drivendataorg/cloudpathlib/pull/322))
 
 ## v0.21.1 (2025-05-14)
 

cloudpathlib/__init__.py

@@ -1,9 +1,11 @@
+import os
 import sys
 
 from .anypath import AnyPath
 from .azure.azblobclient import AzureBlobClient
 from .azure.azblobpath import AzureBlobPath
 from .cloudpath import CloudPath, implementation_registry
+from .patches import patch_open, patch_os_functions, patch_glob, patch_all_builtins
 from .gs.gsclient import GSClient
 from .gs.gspath import GSPath
 from .http.httpclient import HttpClient, HttpsClient
@@ -33,6 +35,23 @@
     "HttpsClient",
     "HttpPath",
     "HttpsPath",
+    "patch_open",
+    "patch_glob",
+    "patch_os_functions",
+    "patch_all_builtins",
     "S3Client",
     "S3Path",
 ]
+
+
+if bool(os.environ.get("CLOUDPATHLIB_PATCH_OPEN", "")):
+    patch_open()
+
+if bool(os.environ.get("CLOUDPATHLIB_PATCH_OS", "")):
+    patch_os_functions()
+
+if bool(os.environ.get("CLOUDPATHLIB_PATCH_GLOB", "")):
+    patch_glob()
+
+if bool(os.environ.get("CLOUDPATHLIB_PATCH_ALL", "")):
+    patch_all_builtins()

cloudpathlib/client.py

@@ -109,8 +109,8 @@ def set_as_default_client(self) -> None:
         instances for this cloud without a client specified."""
         self.__class__._default_client = self
 
-    def CloudPath(self, cloud_path: Union[str, BoundedCloudPath]) -> BoundedCloudPath:
-        return self._cloud_meta.path_class(cloud_path=cloud_path, client=self)  # type: ignore
+    def CloudPath(self, cloud_path: Union[str, BoundedCloudPath], *parts: str) -> BoundedCloudPath:
+        return self._cloud_meta.path_class(cloud_path, *parts, client=self)  # type: ignore
 
     def clear_cache(self):
         """Clears the contents of the cache folder.

cloudpathlib/cloudpath.py

@@ -81,6 +81,7 @@ def _make_selector(pattern_parts, _flavour, case_sensitive=True):  # noqa: F811
 from .exceptions import (
     ClientMismatchError,
     CloudPathFileExistsError,
+    CloudPathFileNotFoundError,
     CloudPathIsADirectoryError,
     CloudPathNotADirectoryError,
     CloudPathNotExistsError,
@@ -235,13 +236,21 @@ class CloudPath(metaclass=CloudPathMeta):
     def __init__(
         self,
         cloud_path: Union[str, Self, "CloudPath"],
+        *parts: str,
         client: Optional["Client"] = None,
     ) -> None:
         # handle if local file gets opened. must be set at the top of the method in case any code
         # below raises an exception, this prevents __del__ from raising an AttributeError
         self._handle: Optional[IO] = None
         self._client: Optional["Client"] = None
 
+        if parts:
+            # ensure first part ends in "/"; (sometimes it is just prefix, sometimes a longer path)
+            if not str(cloud_path).endswith("/"):
+                cloud_path = str(cloud_path) + "/"
+
+            cloud_path = str(cloud_path) + "/".join(p.strip("/") for p in parts)
+
         self.is_valid_cloudpath(cloud_path, raise_on_error=True)
         self._cloud_meta.validate_completeness()
 
@@ -673,11 +682,18 @@ def open(
         force_overwrite_to_cloud: Optional[bool] = None,  # extra kwarg not in pathlib
     ) -> "IO[Any]":
         # if trying to call open on a directory that exists
-        if self.exists() and not self.is_file():
+        exists_on_cloud = self.exists()
+
+        if exists_on_cloud and not self.is_file():
             raise CloudPathIsADirectoryError(
                 f"Cannot open directory, only files. Tried to open ({self})"
             )
 
+        if not exists_on_cloud and any(m in mode for m in ("r", "a")):
+            raise CloudPathFileNotFoundError(
+                f"File opened for read or append, but it does not exist on cloud: {self}"
+            )
+
         if mode == "x" and self.exists():
             raise CloudPathFileExistsError(f"Cannot open existing file ({self}) for creation.")
 
@@ -1247,7 +1263,7 @@ def _local(self) -> Path:
         """Cached local version of the file."""
         return self.client._local_cache_dir / self._no_prefix
 
-    def _new_cloudpath(self, path: Union[str, os.PathLike]) -> Self:
+    def _new_cloudpath(self, path: Union[str, os.PathLike], *parts: str) -> Self:
         """Use the scheme, client, cache dir of this cloudpath to instantiate
         a new cloudpath of the same type with the path passed.
 
@@ -1263,7 +1279,7 @@ def _new_cloudpath(self, path: Union[str, os.PathLike]) -> Self:
         if not path.startswith(self.anchor):
             path = f"{self.anchor}{path}"
 
-        return self.client.CloudPath(path)
+        return self.client.CloudPath(path, *parts)
 
     def _refresh_cache(self, force_overwrite_from_cloud: Optional[bool] = None) -> None:
         try:

cloudpathlib/exceptions.py

@@ -24,6 +24,10 @@ class CloudPathNotExistsError(CloudPathException):
     pass
 
 
+class CloudPathFileNotFoundError(CloudPathException, FileNotFoundError):
+    pass
+
+
 class CloudPathIsADirectoryError(CloudPathException, IsADirectoryError):
     pass
 
@@ -77,3 +81,7 @@ class OverwriteNewerCloudError(CloudPathException):
 
 class OverwriteNewerLocalError(CloudPathException):
     pass
+
+
+class InvalidGlobArgumentsError(CloudPathException):
+    pass

cloudpathlib/http/httpclient.py

@@ -79,6 +79,12 @@ def _get_metadata(self, cloud_path: HttpPath) -> dict:
                 "content_type": response.headers.get("Content-Type", None),
             }
 
+    def _is_file_or_dir(self, cloud_path: HttpPath) -> Optional[str]:
+        if self.dir_matcher(cloud_path.as_url()):
+            return "dir"
+        else:
+            return "file"
+
     def _download_file(self, cloud_path: HttpPath, local_path: Union[str, os.PathLike]) -> Path:
         local_path = Path(local_path)
         with self.opener.open(cloud_path.as_url()) as response:

cloudpathlib/http/httppath.py

@@ -21,9 +21,10 @@ class HttpPath(CloudPath):
     def __init__(
         self,
         cloud_path: Union[str, "HttpPath"],
+        *parts: str,
         client: Optional["HttpClient"] = None,
     ) -> None:
-        super().__init__(cloud_path, client)
+        super().__init__(cloud_path, *parts, client=client)
 
         self._path = (
             PurePosixPath(self._url.path)

cloudpathlib/local/localclient.py

@@ -118,6 +118,14 @@ def _is_file(self, cloud_path: "LocalPath", follow_symlinks=True) -> bool:
 
         return self._cloud_path_to_local(cloud_path).is_file(**kwargs)
 
+    def _is_file_or_dir(self, cloud_path: "LocalPath") -> Optional[str]:
+        if self._is_dir(cloud_path):
+            return "dir"
+        elif self._is_file(cloud_path):
+            return "file"
+        else:
+            raise FileNotFoundError(f"Path could not be identified as file or dir: {cloud_path}")
+
     def _list_dir(
         self, cloud_path: "LocalPath", recursive=False
     ) -> Iterable[Tuple["LocalPath", bool]]: