cleanup

Author: fangchenli

pandas/tests/io/conftest.py

@@ -68,6 +68,15 @@ def moto_s3_resource(moto_server):
     return s3
 
 
+@pytest.fixture
+def s3so(moto_server):
+    return {
+        "client_kwargs": {
+            "endpoint_url": moto_server,
+        }
+    }
+
+
 @pytest.fixture
 def s3_bucket_public(moto_s3_resource):
     """

pandas/tests/io/excel/test_readers.py

@@ -934,12 +934,7 @@ def test_read_from_http_url(self, httpserver, read_ext):
 
     @td.skip_if_not_us_locale
     @pytest.mark.single_cpu
-    def test_read_from_s3_url(self, read_ext, s3_bucket_public):
-        s3so = {
-            "client_kwargs": {
-                "endpoint_url": s3_bucket_public.meta.client.meta.endpoint_url,
-            }
-        }
+    def test_read_from_s3_url(self, read_ext, s3_bucket_public, s3so):
         with open("test1" + read_ext, "rb") as f:
             s3_bucket_public.put_object(Key="test1" + read_ext, Body=f)
 
@@ -950,13 +945,8 @@ def test_read_from_s3_url(self, read_ext, s3_bucket_public):
         tm.assert_frame_equal(url_table, local_table)
 
     @pytest.mark.single_cpu
-    def test_read_from_s3_object(self, read_ext, s3_bucket_public):
+    def test_read_from_s3_object(self, read_ext, s3_bucket_public, s3so):
         # GH 38788
-        s3so = {
-            "client_kwargs": {
-                "endpoint_url": s3_bucket_public.meta.client.meta.endpoint_url,
-            }
-        }
         with open("test1" + read_ext, "rb") as f:
             s3_bucket_public.put_object(Key="test1" + read_ext, Body=f)
 

pandas/tests/io/excel/test_style.py

@@ -323,11 +323,6 @@ def test_styler_to_s3(s3_bucket_public, s3so):
     mock_bucket_name = s3_bucket_public.name
     target_file = f"{uuid.uuid4()}.xlsx"
     df = DataFrame({"x": [1, 2, 3], "y": [2, 4, 6]})
-    s3so = {
-        "client_kwargs": {
-            "endpoint_url": s3_bucket_public.meta.client.meta.endpoint_url
-        }
-    }
     styler = df.style.set_sticky(axis="index")
     styler.to_excel(f"s3://{mock_bucket_name}/{target_file}", storage_options=s3so)
     timeout = 5

pandas/tests/io/json/test_compression.py

@@ -43,7 +43,7 @@ def test_read_zipped_json(datapath):
 @td.skip_if_not_us_locale
 @pytest.mark.single_cpu
 @pytest.mark.network
-def test_with_s3_url(compression, s3_bucket_public):
+def test_with_s3_url(compression, s3_bucket_public, s3so):
     # Bucket created in tests/io/conftest.py
     df = pd.read_json(StringIO('{"a": [1, 2, 3], "b": [4, 5, 6]}'))
 
@@ -53,11 +53,6 @@ def test_with_s3_url(compression, s3_bucket_public):
         with open(path, "rb") as f:
             s3_bucket_public.put_object(Key=key, Body=f)
 
-    s3so = {
-        "client_kwargs": {
-            "endpoint_url": s3_bucket_public.meta.client.meta.endpoint_url
-        }
-    }
     roundtripped_df = pd.read_json(
         f"s3://{s3_bucket_public.name}/{key}",
         compression=compression,

pandas/tests/io/json/test_pandas.py

@@ -1412,15 +1412,8 @@ def test_read_inline_jsonl(self):
     @pytest.mark.single_cpu
     @pytest.mark.network
     @td.skip_if_not_us_locale
-    def test_read_s3_jsonl(self, s3_bucket_public_with_data):
+    def test_read_s3_jsonl(self, s3_bucket_public_with_data, s3so):
         # GH17200
-
-        s3so = {
-            "client_kwargs": {
-                "endpoint_url": s3_bucket_public_with_data.meta.client.meta.endpoint_url
-            }
-        }
-
         result = read_json(
             f"s3n://{s3_bucket_public_with_data.name}/items.jsonl",
             lines=True,
@@ -2018,16 +2011,11 @@ def test_json_multiindex(self):
 
     @pytest.mark.single_cpu
     @pytest.mark.network
-    def test_to_s3(self, s3_bucket_public):
+    def test_to_s3(self, s3_bucket_public, s3so):
         # GH 28375
         mock_bucket_name = s3_bucket_public.name
         target_file = f"{uuid.uuid4()}.json"
         df = DataFrame({"x": [1, 2, 3], "y": [2, 4, 6]})
-        s3so = {
-            "client_kwargs": {
-                "endpoint_url": s3_bucket_public.meta.client.meta.endpoint_url
-            }
-        }
         df.to_json(f"s3://{mock_bucket_name}/{target_file}", storage_options=s3so)
         timeout = 5
         while True:

pandas/tests/io/parser/test_network.py

@@ -89,16 +89,18 @@ class TestS3:
     @pytest.mark.parametrize("nrows", [None, 10])
     @pytest.mark.parametrize("engine", ["c", "python"])
     def test_parse_public_s3_bucket(
-        self, s3_bucket_public_with_data, tips_df, suffix, compression, nrows, engine
+        self,
+        s3_bucket_public_with_data,
+        s3so,
+        tips_df,
+        suffix,
+        compression,
+        nrows,
+        engine,
     ):
         # more of an integration test due to the not-public contents portion
         # can probably mock this though.
         pytest.importorskip("s3fs")
-        s3so = {
-            "client_kwargs": {
-                "endpoint_url": s3_bucket_public_with_data.meta.client.meta.endpoint_url
-            }
-        }
         df = read_csv(
             f"s3://{s3_bucket_public_with_data.name}/tips.csv{suffix}",
             nrows=nrows,
@@ -108,32 +110,19 @@ def test_parse_public_s3_bucket(
         )
         tm.assert_frame_equal(df, tips_df.iloc[:nrows])
 
-    def test_parse_private_s3_bucket(self, s3_bucket_private_with_data, tips_df):
+    def test_parse_private_s3_bucket(self, s3_bucket_private_with_data, s3so, tips_df):
         # Read public file from bucket with not-public contents
         pytest.importorskip("s3fs")
-
-        s3so = {
-            "client_kwargs": {
-                "endpoint_url": (
-                    s3_bucket_private_with_data.meta.client.meta.endpoint_url
-                )
-            }
-        }
         df = read_csv(
             f"s3://{s3_bucket_private_with_data.name}/tips.csv", storage_options=s3so
         )
         tm.assert_frame_equal(df, tips_df)
 
     @pytest.mark.parametrize("scheme", ["s3n", "s3a"])
     def test_parse_public_bucket_s3n_s3a(
-        self, s3_bucket_public_with_data, tips_df, scheme
+        self, s3_bucket_public_with_data, s3so, tips_df, scheme
     ):
         nrows = 10
-        s3so = {
-            "client_kwargs": {
-                "endpoint_url": s3_bucket_public_with_data.meta.client.meta.endpoint_url
-            }
-        }
         df = read_csv(
             f"{scheme}://{s3_bucket_public_with_data.name}/tips.csv",
             nrows=nrows,
@@ -151,15 +140,10 @@ def test_parse_public_bucket_s3n_s3a(
     )
     @pytest.mark.parametrize("engine", ["c", "python"])
     def test_parse_public_s3_bucket_chunked(
-        self, s3_bucket_public_with_data, tips_df, suffix, compression, engine
+        self, s3_bucket_public_with_data, s3so, tips_df, suffix, compression, engine
     ):
         # Read with a chunksize
         chunksize = 5
-        s3so = {
-            "client_kwargs": {
-                "endpoint_url": s3_bucket_public_with_data.meta.client.meta.endpoint_url
-            }
-        }
         with read_csv(
             f"s3://{s3_bucket_public_with_data.name}/tips.csv{suffix}",
             chunksize=chunksize,
@@ -178,12 +162,9 @@ def test_parse_public_s3_bucket_chunked(
                 tm.assert_frame_equal(true_df, df)
 
     @pytest.mark.parametrize("suffix", ["", ".gz", ".bz2"])
-    def test_infer_s3_compression(self, s3_bucket_public_with_data, tips_df, suffix):
-        s3so = {
-            "client_kwargs": {
-                "endpoint_url": s3_bucket_public_with_data.meta.client.meta.endpoint_url
-            }
-        }
+    def test_infer_s3_compression(
+        self, s3_bucket_public_with_data, s3so, tips_df, suffix
+    ):
         df = read_csv(
             f"s3://{s3_bucket_public_with_data.name}/tips.csv{suffix}",
             engine="python",
@@ -192,25 +173,13 @@ def test_infer_s3_compression(self, s3_bucket_public_with_data, tips_df, suffix)
         )
         tm.assert_frame_equal(df, tips_df)
 
-    def test_read_s3_fails(self, s3_bucket_public_with_data):
-        s3so = {
-            "client_kwargs": {
-                "endpoint_url": s3_bucket_public_with_data.meta.client.meta.endpoint_url
-            }
-        }
+    def test_read_s3_fails(self, s3_bucket_public_with_data, s3so):
         msg = "The specified bucket does not exist"
         with pytest.raises(OSError, match=msg):
             read_csv("s3://nyqpug/asdf.csv", storage_options=s3so)
 
-    def test_read_s3_fails_private(self, s3_bucket_private_with_data):
+    def test_read_s3_fails_private(self, s3_bucket_private_with_data, s3so):
         s3_url = f"{s3_bucket_private_with_data.name}/file.csv"
-        s3so = {
-            "client_kwargs": {
-                "endpoint_url": (
-                    s3_bucket_private_with_data.meta.client.meta.endpoint_url
-                )
-            }
-        }
         msg = rf"{s3_url}"
         # Receive a permission error when trying to read a private bucket.
         # It's irrelevant here that this isn't actually a table.
@@ -237,14 +206,9 @@ def test_read_csv_handles_boto_s3_object(
         tm.assert_frame_equal(result, expected)
 
     @pytest.mark.single_cpu
-    def test_read_csv_chunked_download(self, s3_bucket_public, caplog):
+    def test_read_csv_chunked_download(self, s3_bucket_public, s3so, caplog):
         # 8 MB, S3FS uses 5MB chunks
         df = DataFrame(np.zeros((100000, 4)), columns=list("abcd"))
-        s3so = {
-            "client_kwargs": {
-                "endpoint_url": s3_bucket_public.meta.client.meta.endpoint_url
-            }
-        }
         with BytesIO(df.to_csv().encode("utf-8")) as buf:
             s3_bucket_public.put_object(Key="large-file.csv", Body=buf)
             uri = f"{s3_bucket_public.name}/large-file.csv"
@@ -260,26 +224,18 @@ def test_read_csv_chunked_download(self, s3_bucket_public, caplog):
                         # Less than 8 MB
                         assert int(match.group("stop")) < 8000000
 
-    def test_read_s3_with_hash_in_key(self, s3_bucket_public_with_data, tips_df):
+    def test_read_s3_with_hash_in_key(self, s3_bucket_public_with_data, s3so, tips_df):
         # GH 25945
-        s3so = {
-            "client_kwargs": {
-                "endpoint_url": s3_bucket_public_with_data.meta.client.meta.endpoint_url
-            }
-        }
         result = read_csv(
             f"s3://{s3_bucket_public_with_data.name}/tips#1.csv", storage_options=s3so
         )
         tm.assert_frame_equal(tips_df, result)
 
-    def test_read_feather_s3_file_path(self, s3_bucket_public_with_data, feather_file):
+    def test_read_feather_s3_file_path(
+        self, s3_bucket_public_with_data, s3so, feather_file
+    ):
         # GH 29055
         pytest.importorskip("pyarrow")
-        s3so = {
-            "client_kwargs": {
-                "endpoint_url": s3_bucket_public_with_data.meta.client.meta.endpoint_url
-            }
-        }
         expected = read_feather(feather_file)
         res = read_feather(
             f"s3://{s3_bucket_public_with_data.name}/simple_dataset.feather",

pandas/tests/io/test_fsspec.py

@@ -231,14 +231,8 @@ def test_fastparquet_options(fsspectest):
 
 @pytest.mark.single_cpu
 @pytest.mark.parametrize("compression_suffix", ["", ".gz", ".bz2"])
-def test_from_s3_csv(s3_bucket_public_with_data, tips_file, compression_suffix):
+def test_from_s3_csv(s3_bucket_public_with_data, s3so, tips_file, compression_suffix):
     pytest.importorskip("s3fs")
-
-    s3so = {
-        "client_kwargs": {
-            "endpoint_url": s3_bucket_public_with_data.meta.client.meta.endpoint_url
-        }
-    }
     df_from_s3 = read_csv(
         f"s3://{s3_bucket_public_with_data.name}/tips.csv{compression_suffix}",
         storage_options=s3so,
@@ -249,14 +243,8 @@ def test_from_s3_csv(s3_bucket_public_with_data, tips_file, compression_suffix):
 
 @pytest.mark.single_cpu
 @pytest.mark.parametrize("protocol", ["s3", "s3a", "s3n"])
-def test_s3_protocols(s3_bucket_public_with_data, tips_file, protocol):
+def test_s3_protocols(s3_bucket_public_with_data, s3so, tips_file, protocol):
     pytest.importorskip("s3fs")
-
-    s3so = {
-        "client_kwargs": {
-            "endpoint_url": s3_bucket_public_with_data.meta.client.meta.endpoint_url
-        }
-    }
     df_from_s3 = read_csv(
         f"{protocol}://{s3_bucket_public_with_data.name}/tips.csv",
         storage_options=s3so,
@@ -267,16 +255,11 @@ def test_s3_protocols(s3_bucket_public_with_data, tips_file, protocol):
 
 @pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string) fastparquet")
 @pytest.mark.single_cpu
-def test_s3_parquet(s3_bucket_public, df1):
+def test_s3_parquet(s3_bucket_public, s3so, df1):
     pytest.importorskip("fastparquet")
     pytest.importorskip("s3fs")
 
     fn = f"s3://{s3_bucket_public.name}/test.parquet"
-    s3so = {
-        "client_kwargs": {
-            "endpoint_url": s3_bucket_public.meta.client.meta.endpoint_url
-        }
-    }
     df1.to_parquet(
         fn, index=False, engine="fastparquet", compression=None, storage_options=s3so
     )

pandas/tests/io/test_parquet.py

@@ -809,13 +809,8 @@ def test_categorical(self, pa):
         check_round_trip(df, pa)
 
     @pytest.mark.single_cpu
-    def test_s3_roundtrip_explicit_fs(self, df_compat, s3_bucket_public, pa):
+    def test_s3_roundtrip_explicit_fs(self, df_compat, s3_bucket_public, s3so, pa):
         s3fs = pytest.importorskip("s3fs")
-        s3so = {
-            "client_kwargs": {
-                "endpoint_url": s3_bucket_public.meta.client.meta.endpoint_url,
-            }
-        }
         s3 = s3fs.S3FileSystem(**s3so)
         kw = {"filesystem": s3}
         check_round_trip(
@@ -827,13 +822,8 @@ def test_s3_roundtrip_explicit_fs(self, df_compat, s3_bucket_public, pa):
         )
 
     @pytest.mark.single_cpu
-    def test_s3_roundtrip(self, df_compat, s3_bucket_public, pa):
+    def test_s3_roundtrip(self, df_compat, s3_bucket_public, s3so, pa):
         # GH #19134
-        s3so = {
-            "client_kwargs": {
-                "endpoint_url": s3_bucket_public.meta.client.meta.endpoint_url,
-            }
-        }
         s3so = {"storage_options": s3so}
         check_round_trip(
             df_compat,
@@ -1316,13 +1306,8 @@ def test_filter_row_groups(self, fp):
         assert len(result) == 1
 
     @pytest.mark.single_cpu
-    def test_s3_roundtrip(self, df_compat, s3_bucket_public, fp):
+    def test_s3_roundtrip(self, df_compat, s3_bucket_public, s3so, fp):
         # GH #19134
-        s3so = {
-            "client_kwargs": {
-                "endpoint_url": s3_bucket_public.meta.client.meta.endpoint_url,
-            }
-        }
         check_round_trip(
             df_compat,
             fp,