Merge remote-tracking branch 'upstream/main' into read_iceberg

Author: datapythonista

.github/workflows/unit-tests.yml

@@ -27,7 +27,7 @@ jobs:
     strategy:
       matrix:
         platform: [ubuntu-22.04, ubuntu-24.04-arm]
-        env_file: [actions-310.yaml, actions-311.yaml, actions-312.yaml]
+        env_file: [actions-310.yaml, actions-311.yaml, actions-312.yaml, actions-313.yaml]
         # Prevent the include jobs from overriding other jobs
         pattern: [""]
         pandas_future_infer_string: ["0"]
@@ -188,7 +188,7 @@ jobs:
       matrix:
         # Note: Don't use macOS latest since macos 14 appears to be arm64 only
         os: [macos-13, macos-14, windows-latest]
-        env_file: [actions-310.yaml, actions-311.yaml, actions-312.yaml]
+        env_file: [actions-310.yaml, actions-311.yaml, actions-312.yaml, actions-313.yaml]
       fail-fast: false
     runs-on: ${{ matrix.os }}
     name: ${{ format('{0} {1}', matrix.os, matrix.env_file) }}
@@ -316,7 +316,7 @@ jobs:
     #    To freeze this file, uncomment out the ``if: false`` condition, and migrate the jobs
     #    to the corresponding posix/windows-macos/sdist etc. workflows.
     # Feel free to modify this comment as necessary.
-    # if: false # Uncomment this to freeze the workflow, comment it to unfreeze
+    if: false
     defaults:
       run:
         shell: bash -eou pipefail {0}

ci/deps/actions-313.yaml

@@ -0,0 +1,63 @@
+name: pandas-dev-313
+channels:
+  - conda-forge
+dependencies:
+  - python=3.13
+
+  # build dependencies
+  - versioneer
+  - cython>=0.29.33
+  - meson=1.2.1
+  - meson-python=0.13.1
+
+  # test dependencies
+  - pytest>=7.3.2
+  - pytest-cov
+  - pytest-xdist>=3.4.0
+  - pytest-localserver>=0.8.1
+  - pytest-qt>=4.4.0
+  - boto3
+
+  # required dependencies
+  - python-dateutil
+  - numpy
+
+  # optional dependencies
+  - beautifulsoup4>=4.12.3
+  - blosc>=1.21.3
+  - bottleneck>=1.3.6
+  - fastparquet>=2024.2.0
+  - fsspec>=2024.2.0
+  - html5lib>=1.1
+  - hypothesis>=6.84.0
+  - gcsfs>=2024.2.0
+  - jinja2>=3.1.3
+  - lxml>=4.9.2
+  - matplotlib>=3.8.3
+  - numba>=0.59.0
+  - numexpr>=2.9.0
+  - odfpy>=1.4.1
+  - qtpy>=2.3.0
+  - pyqt>=5.15.9
+  - openpyxl>=3.1.2
+  - psycopg2>=2.9.6
+  - pyarrow>=10.0.1
+  - pymysql>=1.1.0
+  - pyreadstat>=1.2.6
+  - pytables>=3.8.0
+  - python-calamine>=0.1.7
+  - pytz>=2023.4
+  - pyxlsb>=1.0.10
+  - s3fs>=2024.2.0
+  - scipy>=1.12.0
+  - sqlalchemy>=2.0.0
+  - tabulate>=0.9.0
+  - xarray>=2024.1.1, <=2024.9.0
+  - xlrd>=2.0.1
+  - xlsxwriter>=3.2.0
+  - zstandard>=0.22.0
+
+  - pip:
+    - adbc-driver-postgresql>=0.10.0
+    - adbc-driver-sqlite>=0.8.0
+    - tzdata>=2022.7

doc/source/_static/css/getting_started.css

@@ -249,13 +249,15 @@ ul.task-bullet > li > p:first-child {
 
 .tutorial-card .card-header {
   --bs-card-cap-color: var(--pst-color-text-base);
+  color: var(--pst-color-text-base);
   cursor: pointer;
   background-color: var(--pst-color-surface);
   border: 1px solid var(--pst-color-border)
 }
 
 .tutorial-card .card-body {
   background-color: var(--pst-color-on-background);
+  color: var(--pst-color-text-base);
 }
 
 .tutorial-card .badge {

doc/source/reference/arrays.rst

@@ -664,6 +664,7 @@ Data type introspection
     api.types.is_datetime64_dtype
     api.types.is_datetime64_ns_dtype
     api.types.is_datetime64tz_dtype
+    api.types.is_dtype_equal
     api.types.is_extension_array_dtype
     api.types.is_float_dtype
     api.types.is_int64_dtype

pandas/core/dtypes/common.py

@@ -655,24 +655,38 @@ def is_dtype_equal(source, target) -> bool:
 
     Parameters
     ----------
-    source : The first dtype to compare
-    target : The second dtype to compare
+    source : type or str
+        The first dtype to compare.
+    target : type or str
+        The second dtype to compare.
 
     Returns
     -------
     boolean
         Whether or not the two dtypes are equal.
 
+    See Also
+    --------
+    api.types.is_categorical_dtype : Check whether the provided array or dtype
+                                            is of the Categorical dtype.
+    api.types.is_string_dtype : Check whether the provided array or dtype
+                                       is of the string dtype.
+    api.types.is_object_dtype : Check whether an array-like or dtype is of the
+                                       object dtype.
+
     Examples
     --------
+    >>> from pandas.api.types import is_dtype_equal
     >>> is_dtype_equal(int, float)
     False
     >>> is_dtype_equal("int", int)
     True
     >>> is_dtype_equal(object, "category")
     False
+    >>> from pandas.core.dtypes.dtypes import CategoricalDtype
     >>> is_dtype_equal(CategoricalDtype(), "category")
     True
+    >>> from pandas.core.dtypes.dtypes import DatetimeTZDtype
     >>> is_dtype_equal(DatetimeTZDtype(tz="UTC"), "datetime64")
     False
     """

pandas/plotting/_matplotlib/style.py

@@ -22,8 +22,6 @@
 
 from pandas.core.dtypes.common import is_list_like
 
-import pandas.core.common as com
-
 if TYPE_CHECKING:
     from matplotlib.colors import Colormap
 
@@ -251,31 +249,17 @@ def _is_floats_color(color: Color | Collection[Color]) -> bool:
 def _get_colors_from_color_type(color_type: str, num_colors: int) -> list[Color]:
     """Get colors from user input color type."""
     if color_type == "default":
-        return _get_default_colors(num_colors)
+        prop_cycle = mpl.rcParams["axes.prop_cycle"]
+        return [
+            c["color"]
+            for c in itertools.islice(prop_cycle, min(num_colors, len(prop_cycle)))
+        ]
     elif color_type == "random":
-        return _get_random_colors(num_colors)
+        return np.random.default_rng(num_colors).random((num_colors, 3)).tolist()
     else:
         raise ValueError("color_type must be either 'default' or 'random'")
 
 
-def _get_default_colors(num_colors: int) -> list[Color]:
-    """Get `num_colors` of default colors from matplotlib rc params."""
-    colors = [c["color"] for c in mpl.rcParams["axes.prop_cycle"]]
-    return colors[0:num_colors]
-
-
-def _get_random_colors(num_colors: int) -> list[Color]:
-    """Get `num_colors` of random colors."""
-    return [_random_color(num) for num in range(num_colors)]
-
-
-def _random_color(column: int) -> list[float]:
-    """Get a random color represented as a list of length 3"""
-    # GH17525 use common._random_state to avoid resetting the seed
-    rs = com.random_state(column)
-    return rs.rand(3).tolist()
-
-
 def _is_single_string_color(color: Color) -> bool:
     """Check if `color` is a single string color.
 

pandas/tests/io/test_sql.py

@@ -2498,10 +2498,8 @@ def test_sqlalchemy_integer_overload_mapping(conn, request, integer):
             sql.SQLTable("test_type", db, frame=df)
 
 
-@pytest.mark.parametrize("conn", all_connectable)
-def test_database_uri_string(conn, request, test_frame1):
+def test_database_uri_string(request, test_frame1):
     pytest.importorskip("sqlalchemy")
-    conn = request.getfixturevalue(conn)
     # Test read_sql and .to_sql method with a database URI (GH10654)
     # db_uri = 'sqlite:///:memory:' # raises
     # sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) near
@@ -2520,10 +2518,8 @@ def test_database_uri_string(conn, request, test_frame1):
 
 
 @td.skip_if_installed("pg8000")
-@pytest.mark.parametrize("conn", all_connectable)
-def test_pg8000_sqlalchemy_passthrough_error(conn, request):
+def test_pg8000_sqlalchemy_passthrough_error(request):
     pytest.importorskip("sqlalchemy")
-    conn = request.getfixturevalue(conn)
     # using driver that will not be installed on CI to trigger error
     # in sqlalchemy.create_engine -> test passing of this error to user
     db_uri = "postgresql+pg8000://user:pass@host/dbname"
@@ -2731,25 +2727,26 @@ def test_delete_rows_is_atomic(conn_name, request):
     replacing_df = DataFrame({"a": [5, 6, 7], "b": [8, 8, 8]}, dtype="int32")
 
     conn = request.getfixturevalue(conn_name)
-    pandasSQL = pandasSQL_builder(conn)
+    with pandasSQL_builder(conn) as pandasSQL:
+        with pandasSQL.run_transaction() as cur:
+            cur.execute(table_stmt)
 
-    with pandasSQL.run_transaction() as cur:
-        cur.execute(table_stmt)
+        with pandasSQL.run_transaction():
+            pandasSQL.to_sql(original_df, table_name, if_exists="append", index=False)
 
-    with pandasSQL.run_transaction():
-        pandasSQL.to_sql(original_df, table_name, if_exists="append", index=False)
+        # inserting duplicated values in a UNIQUE constraint column
+        with pytest.raises(pd.errors.DatabaseError):
+            with pandasSQL.run_transaction():
+                pandasSQL.to_sql(
+                    replacing_df, table_name, if_exists="delete_rows", index=False
+                )
 
-    # inserting duplicated values in a UNIQUE constraint column
-    with pytest.raises(pd.errors.DatabaseError):
+        # failed "delete_rows" is rolled back preserving original data
         with pandasSQL.run_transaction():
-            pandasSQL.to_sql(
-                replacing_df, table_name, if_exists="delete_rows", index=False
+            result_df = pandasSQL.read_query(
+                f"SELECT * FROM {table_name}", dtype="int32"
             )
-
-    # failed "delete_rows" is rolled back preserving original data
-    with pandasSQL.run_transaction():
-        result_df = pandasSQL.read_query(f"SELECT * FROM {table_name}", dtype="int32")
-        tm.assert_frame_equal(result_df, original_df)
+            tm.assert_frame_equal(result_df, original_df)
 
 
 @pytest.mark.parametrize("conn", all_connectable)
@@ -2759,10 +2756,10 @@ def test_roundtrip(conn, request, test_frame1):
 
     conn_name = conn
     conn = request.getfixturevalue(conn)
-    pandasSQL = pandasSQL_builder(conn)
-    with pandasSQL.run_transaction():
-        assert pandasSQL.to_sql(test_frame1, "test_frame_roundtrip") == 4
-        result = pandasSQL.read_query("SELECT * FROM test_frame_roundtrip")
+    with pandasSQL_builder(conn) as pandasSQL:
+        with pandasSQL.run_transaction():
+            assert pandasSQL.to_sql(test_frame1, "test_frame_roundtrip") == 4
+            result = pandasSQL.read_query("SELECT * FROM test_frame_roundtrip")
 
     if "adbc" in conn_name:
         result = result.rename(columns={"__index_level_0__": "level_0"})
@@ -3577,13 +3574,6 @@ def test_options_get_engine():
         assert isinstance(get_engine("sqlalchemy"), SQLAlchemyEngine)
 
 
-def test_get_engine_auto_error_message():
-    # Expect different error messages from get_engine(engine="auto")
-    # if engines aren't installed vs. are installed but bad version
-    pass
-    # TODO(GH#36893) fill this in when we add more engines
-
-
 @pytest.mark.parametrize("conn", all_connectable)
 @pytest.mark.parametrize("func", ["read_sql", "read_sql_query"])
 def test_read_sql_dtype_backend(