improve test

Author: fangchenli

pandas/io/common.py

@@ -1378,11 +1378,6 @@ def iterdir(
                 yield resolved_path
             return
 
-        if not resolved_path.is_dir():
-            raise NotADirectoryError(
-                f"Path {path!r} is neither a file nor a directory."
-            )
-
         for entry in resolved_path.iterdir():
             if entry.is_file():
                 if _match_file(
@@ -1393,7 +1388,7 @@ def iterdir(
                     yield entry
         return
 
-    # Remote paths (e.g., s3)
+    # Remote paths
     fsspec = import_optional_dependency("fsspec", extra=scheme)
     fs = fsspec.filesystem(scheme)
     path_without_scheme = fsspec.core.strip_protocol(path_str)
@@ -1405,13 +1400,10 @@ def iterdir(
         ):
             yield PurePosixPath(path_without_scheme)
         return
-    if not fs.isdir(path_without_scheme):
-        raise NotADirectoryError(f"Path {path!r} is neither a file nor a directory.")
 
-    files = fs.ls(path_without_scheme, detail=True)
-    for f in files:
-        if f["type"] == "file":
-            path_obj = PurePosixPath(f["name"])
+    for file in fs.ls(path_without_scheme, detail=True):
+        if fs.isfile(file):
+            path_obj = PurePosixPath(file["name"])
             if _match_file(
                 path_obj,
                 extensions,

pandas/tests/io/conftest.py

@@ -226,18 +226,18 @@ def compression_ext(request):
 
 
 @pytest.fixture
-def directory_with_dummy_csv(tmp_path):
+def local_csv_directory(tmp_path):
     """
     Fixture to create a directory with dummy CSV files for testing.
     """
     for i in range(3):
-        file_path = tmp_path / f"file_{i}.csv"
+        file_path = tmp_path / f"{i}.csv"
         file_path.touch()
     return tmp_path
 
 
 @pytest.fixture
-def mock_remote_csv_directory(monkeypatch):
+def remote_csv_directory(monkeypatch):
     _ = pytest.importorskip("fsspec", reason="fsspec is required for remote tests")
 
     from fsspec.implementations.memory import MemoryFileSystem

pandas/tests/io/test_common.py

@@ -697,14 +697,13 @@ def test_pyarrow_read_csv_datetime_dtype():
     tm.assert_frame_equal(expect, result)
 
 
-def test_iterdir_local(directory_with_dummy_csv):
-    for file in icom.iterdir(directory_with_dummy_csv):
+def test_iterdir_local(local_csv_directory):
+    for file in icom.iterdir(local_csv_directory):
         assert file.is_file()
-        assert file.name.startswith("file_")
         assert file.suffix == ".csv"
 
 
-def test_mock_remote_csv_directory_contents(mock_remote_csv_directory):
+def test_remote_csv_directory(remote_csv_directory):
     import fsspec
     from fsspec.implementations.memory import MemoryFileSystem
 
@@ -726,10 +725,13 @@ def test_mock_remote_csv_directory_contents(mock_remote_csv_directory):
     assert nested_files[0]["name"] == "/remote-bucket/nested/ignored.csv"
 
 
-def test_iterdir_remote(mock_remote_csv_directory):
+def test_iterdir_remote(remote_csv_directory):
     import fsspec
 
     fs = fsspec.filesystem("s3")
-    for file in icom.iterdir(mock_remote_csv_directory):
-        assert fs.isfile(file)
+    for file in icom.iterdir(remote_csv_directory):
+        assert fs.exists(file)
         assert file.suffix == ".csv"
+
+        # for fsspec<2024.5.0, fs.isfle(PurePosixPath) returns False
+        assert fs.isfile(str(file))