format

Author: fangchenli

pandas/io/common.py

@@ -1295,8 +1295,8 @@ def _infer_protocol(path: str) -> str:
     if is_platform_windows() and re.match(r"^[a-zA-Z]:[\\/]", path):
         return "file"
 
-    parsed = parse_url(path)
-    if parsed.scheme in _VALID_URLS:
+    if is_fsspec_url(path):
+        parsed = parse_url(path)
         return parsed.scheme
     return "file"
 
@@ -1396,19 +1396,19 @@ def iterdir(
     # Remote paths (e.g., s3)
     fsspec = import_optional_dependency("fsspec", extra=scheme)
     fs = fsspec.filesystem(scheme)
-    if fs.isfile(path):
-        path_obj = PurePosixPath(path)
+    path_without_scheme = fsspec.core.strip_protocol(path_str)
+    if fs.isfile(path_without_scheme):
         if _match_file(
-            path_obj,
+            path_without_scheme,
             extensions,
             glob,
         ):
-            yield path_obj
+            yield PurePosixPath(path_without_scheme)
         return
-    if not fs.isdir(path):
+    if not fs.isdir(path_without_scheme):
         raise NotADirectoryError(f"Path {path!r} is neither a file nor a directory.")
 
-    files = fs.ls(path, detail=True)
+    files = fs.ls(path_without_scheme, detail=True)
     for f in files:
         if f["type"] == "file":
             path_obj = PurePosixPath(f["name"])

pandas/tests/io/conftest.py

@@ -234,3 +234,21 @@ def directory_with_dummy_csv(tmp_path):
         file_path = tmp_path / f"file_{i}.csv"
         file_path.touch()
     return tmp_path
+
+
+@pytest.fixture
+def mock_remote_csv_directory(monkeypatch):
+    _ = pytest.importorskip("fsspec", reason="fsspec is required for remote tests")
+
+    from fsspec.implementations.memory import MemoryFileSystem
+
+    fs = MemoryFileSystem()
+    fs.store.clear()
+
+    dir_name = "remote-bucket"
+    fs.pipe(f"{dir_name}/a.csv", b"a,b,c\n1,2,3\n")
+    fs.pipe(f"{dir_name}/b.csv", b"a,b,c\n4,5,6\n")
+    fs.pipe(f"{dir_name}/nested/ignored.csv", b"x,y,z\n")
+
+    monkeypatch.setattr("fsspec.filesystem", lambda _: fs)
+    return f"s3://{dir_name}"

pandas/tests/io/test_common.py

@@ -697,8 +697,39 @@ def test_pyarrow_read_csv_datetime_dtype():
     tm.assert_frame_equal(expect, result)
 
 
-def test_iterdir(directory_with_dummy_csv):
+def test_iterdir_local(directory_with_dummy_csv):
     for file in icom.iterdir(directory_with_dummy_csv):
         assert file.is_file()
         assert file.name.startswith("file_")
         assert file.suffix == ".csv"
+
+
+def test_mock_remote_csv_directory_contents(mock_remote_csv_directory):
+    import fsspec
+    from fsspec.implementations.memory import MemoryFileSystem
+
+    fs = fsspec.filesystem("s3")
+    assert isinstance(fs, MemoryFileSystem)
+
+    assert fs.exists("remote-bucket")
+    assert fs.isdir("remote-bucket")
+
+    files = fs.ls("remote-bucket", detail=True)
+
+    file_names = sorted(f["name"] for f in files if f["type"] == "file")
+    assert file_names == ["/remote-bucket/a.csv", "/remote-bucket/b.csv"]
+
+    dir_names = [f["name"] for f in files if f["type"] == "directory"]
+    assert "/remote-bucket/nested" in dir_names
+
+    nested_files = fs.ls("remote-bucket/nested", detail=True)
+    assert nested_files[0]["name"] == "/remote-bucket/nested/ignored.csv"
+
+
+def test_iterdir_remote(mock_remote_csv_directory):
+    import fsspec
+
+    fs = fsspec.filesystem("s3")
+    for file in icom.iterdir(mock_remote_csv_directory):
+        assert fs.isfile(file)
+        assert file.suffix == ".csv"