Merge branch 'main' into searchsorted_branch

Author: GSAUC3

.github/workflows/unit-tests.yml

@@ -241,12 +241,14 @@ jobs:
           fi
       - name: Build environment and Run Tests
         # https://github.com/numpy/numpy/issues/24703#issuecomment-1722379388
+        # Note: Pinned to Cython 3.0.10 to avoid numerical instability in 32-bit environments
+        # https://github.com/pandas-dev/pandas/pull/61423
         run: |
           /opt/python/cp313-cp313/bin/python -m venv ~/virtualenvs/pandas-dev
           . ~/virtualenvs/pandas-dev/bin/activate
           python -m pip install --no-cache-dir -U pip wheel setuptools meson[ninja]==1.2.1 meson-python==0.13.1
           python -m pip install numpy -Csetup-args="-Dallow-noblas=true"
-          python -m pip install --no-cache-dir versioneer[toml] cython python-dateutil pytest>=7.3.2 pytest-xdist>=3.4.0 hypothesis>=6.84.0
+          python -m pip install --no-cache-dir versioneer[toml] cython==3.0.10 python-dateutil pytest>=7.3.2 pytest-xdist>=3.4.0 hypothesis>=6.84.0
           python -m pip install --no-cache-dir --no-build-isolation -e . -Csetup-args="--werror"
           python -m pip list --no-cache-dir
           PANDAS_CI=1 python -m pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml

Dockerfile

@@ -1,16 +1,20 @@
 FROM python:3.10.8
 WORKDIR /home/pandas
 
-RUN apt-get update && apt-get -y upgrade
-RUN apt-get install -y build-essential bash-completion
+RUN apt-get update && \
+    apt-get --no-install-recommends -y upgrade && \
+    apt-get --no-install-recommends -y install \
+    build-essential \
+    bash-completion \
+    # hdf5 needed for pytables installation
+    libhdf5-dev \
+    # libgles2-mesa needed for pytest-qt
+    libgles2-mesa-dev && \
+    rm -rf /var/lib/apt/lists/*
 
-# hdf5 needed for pytables installation
-# libgles2-mesa needed for pytest-qt
-RUN apt-get install -y libhdf5-dev libgles2-mesa-dev
-
-RUN python -m pip install --upgrade pip
 COPY requirements-dev.txt /tmp
-RUN python -m pip install -r /tmp/requirements-dev.txt
+RUN python -m pip install --no-cache-dir --upgrade pip && \
+    python -m pip install --no-cache-dir -r /tmp/requirements-dev.txt
 RUN git config --global --add safe.directory /home/pandas
 
 ENV SHELL="/bin/bash"

ci/deps/actions-310-minimum_versions.yaml

@@ -28,10 +28,10 @@ dependencies:
   - beautifulsoup4=4.12.3
   - bottleneck=1.3.6
   - fastparquet=2024.2.0
-  - fsspec=2024.2.0
+  - fsspec=2023.12.2
   - html5lib=1.1
   - hypothesis=6.84.0
-  - gcsfs=2024.2.0
+  - gcsfs=2023.12.2
   - jinja2=3.1.3
   - lxml=4.9.2
   - matplotlib=3.8.3
@@ -42,14 +42,15 @@ dependencies:
   - openpyxl=3.1.2
   - psycopg2=2.9.6
   - pyarrow=10.0.1
+  - pyiceberg=0.7.1
   - pymysql=1.1.0
   - pyqt=5.15.9
   - pyreadstat=1.2.6
   - pytables=3.8.0
   - python-calamine=0.1.7
   - pytz=2023.4
   - pyxlsb=1.0.10
-  - s3fs=2024.2.0
+  - s3fs=2023.12.2
   - scipy=1.12.0
   - sqlalchemy=2.0.0
   - tabulate=0.9.0

ci/deps/actions-310.yaml

@@ -26,10 +26,10 @@ dependencies:
   - beautifulsoup4>=4.12.3
   - bottleneck>=1.3.6
   - fastparquet>=2024.2.0
-  - fsspec>=2024.2.0
+  - fsspec>=2023.12.2
   - html5lib>=1.1
   - hypothesis>=6.84.0
-  - gcsfs>=2024.2.0
+  - gcsfs>=2023.12.2
   - jinja2>=3.1.3
   - lxml>=4.9.2
   - matplotlib>=3.8.3
@@ -40,14 +40,15 @@ dependencies:
   - openpyxl>=3.1.2
   - psycopg2>=2.9.6
   - pyarrow>=10.0.1
+  - pyiceberg>=0.7.1
   - pymysql>=1.1.0
   - pyqt>=5.15.9
   - pyreadstat>=1.2.6
   - pytables>=3.8.0
   - python-calamine>=0.1.7
   - pytz>=2023.4
   - pyxlsb>=1.0.10
-  - s3fs>=2024.2.0
+  - s3fs>=2023.12.2
   - scipy>=1.12.0
   - sqlalchemy>=2.0.0
   - tabulate>=0.9.0

ci/deps/actions-311-downstream_compat.yaml

@@ -27,10 +27,10 @@ dependencies:
   - beautifulsoup4>=4.12.3
   - bottleneck>=1.3.6
   - fastparquet>=2024.2.0
-  - fsspec>=2024.2.0
+  - fsspec>=2023.12.2
   - html5lib>=1.1
   - hypothesis>=6.84.0
-  - gcsfs>=2024.2.0
+  - gcsfs>=2023.12.2
   - jinja2>=3.1.3
   - lxml>=4.9.2
   - matplotlib>=3.8.3
@@ -41,14 +41,15 @@ dependencies:
   - openpyxl>=3.1.2
   - psycopg2>=2.9.6
   - pyarrow>=10.0.1
+  - pyiceberg>=0.7.1
   - pymysql>=1.1.0
   - pyqt>=5.15.9
   - pyreadstat>=1.2.6
   - pytables>=3.8.0
   - python-calamine>=0.1.7
   - pytz>=2023.4
   - pyxlsb>=1.0.10
-  - s3fs>=2024.2.0
+  - s3fs>=2023.12.2
   - scipy>=1.12.0
   - sqlalchemy>=2.0.0
   - tabulate>=0.9.0

ci/deps/actions-311.yaml

@@ -26,10 +26,10 @@ dependencies:
   - beautifulsoup4>=4.12.3
   - bottleneck>=1.3.6
   - fastparquet>=2024.2.0
-  - fsspec>=2024.2.0
+  - fsspec>=2023.12.2
   - html5lib>=1.1
   - hypothesis>=6.84.0
-  - gcsfs>=2024.2.0
+  - gcsfs>=2023.12.2
   - jinja2>=3.1.3
   - lxml>=4.9.2
   - matplotlib>=3.8.3
@@ -41,13 +41,14 @@ dependencies:
   - openpyxl>=3.1.2
   - psycopg2>=2.9.6
   - pyarrow>=10.0.1
+  - pyiceberg>=0.7.1
   - pymysql>=1.1.0
   - pyreadstat>=1.2.6
   - pytables>=3.8.0
   - python-calamine>=0.1.7
   - pytz>=2023.4
   - pyxlsb>=1.0.10
-  - s3fs>=2024.2.0
+  - s3fs>=2023.12.2
   - scipy>=1.12.0
   - sqlalchemy>=2.0.0
   - tabulate>=0.9.0

ci/deps/actions-312.yaml

@@ -26,10 +26,10 @@ dependencies:
   - beautifulsoup4>=4.12.3
   - bottleneck>=1.3.6
   - fastparquet>=2024.2.0
-  - fsspec>=2024.2.0
+  - fsspec>=2023.12.2
   - html5lib>=1.1
   - hypothesis>=6.84.0
-  - gcsfs>=2024.2.0
+  - gcsfs>=2023.12.2
   - jinja2>=3.1.3
   - lxml>=4.9.2
   - matplotlib>=3.8.3
@@ -41,13 +41,14 @@ dependencies:
   - openpyxl>=3.1.2
   - psycopg2>=2.9.6
   - pyarrow>=10.0.1
+  - pyiceberg>=0.7.1
   - pymysql>=1.1.0
   - pyreadstat>=1.2.6
   - pytables>=3.8.0
   - python-calamine>=0.1.7
   - pytz>=2023.4
   - pyxlsb>=1.0.10
-  - s3fs>=2024.2.0
+  - s3fs>=2023.12.2
   - scipy>=1.12.0
   - sqlalchemy>=2.0.0
   - tabulate>=0.9.0

ci/deps/actions-313.yaml

@@ -27,10 +27,10 @@ dependencies:
   - blosc>=1.21.3
   - bottleneck>=1.3.6
   - fastparquet>=2024.2.0
-  - fsspec>=2024.2.0
+  - fsspec>=2023.12.2
   - html5lib>=1.1
   - hypothesis>=6.84.0
-  - gcsfs>=2024.2.0
+  - gcsfs>=2023.12.2
   - jinja2>=3.1.3
   - lxml>=4.9.2
   - matplotlib>=3.8.3
@@ -48,7 +48,7 @@ dependencies:
   - python-calamine>=0.1.7
   - pytz>=2023.4
   - pyxlsb>=1.0.10
-  - s3fs>=2024.2.0
+  - s3fs>=2023.12.2
   - scipy>=1.12.0
   - sqlalchemy>=2.0.0
   - tabulate>=0.9.0

doc/source/getting_started/install.rst

@@ -299,7 +299,7 @@ Dependency                                                         Minimum Versi
 Other data sources
 ^^^^^^^^^^^^^^^^^^
 
-Installable with ``pip install "pandas[hdf5, parquet, feather, spss, excel]"``
+Installable with ``pip install "pandas[hdf5, parquet, iceberg, feather, spss, excel]"``
 
 ====================================================== ================== ================ ==========================================================
 Dependency                                             Minimum Version    pip extra        Notes
@@ -308,6 +308,7 @@ Dependency                                             Minimum Version    pip ex
 `zlib <https://github.com/madler/zlib>`__                                 hdf5             Compression for HDF5
 `fastparquet <https://github.com/dask/fastparquet>`__  2024.2.0           -                Parquet reading / writing (pyarrow is default)
 `pyarrow <https://github.com/apache/arrow>`__          10.0.1             parquet, feather Parquet, ORC, and feather reading / writing
+`PyIceberg <https://py.iceberg.apache.org/>`__         0.7.1              iceberg          Apache Iceberg reading
 `pyreadstat <https://github.com/Roche/pyreadstat>`__   1.2.6              spss             SPSS files (.sav) reading
 `odfpy <https://github.com/eea/odfpy>`__               1.4.1              excel            Open document format (.odf, .ods, .odt) reading / writing
 ====================================================== ================== ================ ==========================================================
@@ -328,10 +329,10 @@ Installable with ``pip install "pandas[fss, aws, gcp]"``
 ============================================ ================== =============== ==========================================================
 Dependency                                   Minimum Version    pip extra       Notes
 ============================================ ================== =============== ==========================================================
-`fsspec <https://github.com/fsspec>`__       2024.2.0           fss, gcp, aws   Handling files aside from simple local and HTTP (required
+`fsspec <https://github.com/fsspec>`__       2023.12.2          fss, gcp, aws   Handling files aside from simple local and HTTP (required
                                                                                 dependency of s3fs, gcsfs).
-`gcsfs <https://github.com/fsspec/gcsfs>`__  2024.2.0           gcp             Google Cloud Storage access
-`s3fs <https://github.com/fsspec/s3fs>`__    2024.2.0           aws             Amazon S3 access
+`gcsfs <https://github.com/fsspec/gcsfs>`__  2023.12.2          gcp             Google Cloud Storage access
+`s3fs <https://github.com/fsspec/s3fs>`__    2023.12.2          aws             Amazon S3 access
 ============================================ ================== =============== ==========================================================
 
 Clipboard

doc/source/reference/io.rst

@@ -156,6 +156,15 @@ Parquet
    read_parquet
    DataFrame.to_parquet
 
+Iceberg
+~~~~~~~
+.. autosummary::
+    :toctree: api/
+
+    read_iceberg
+
+.. warning:: ``read_iceberg`` is experimental and may change without warning.
+
 ORC
 ~~~
 .. autosummary::