Merge pull request #1877 from rstudio/fix/pandas-3-compatibility

Add support for pandas 3.0 class names

Author: t-kalinowski

NAMESPACE

@@ -44,6 +44,8 @@ S3method(as.matrix,numpy.ndarray)
 S3method(as.raw,python.builtin.bytes)
 S3method(as.vector,numpy.ndarray)
 S3method(dim,numpy.ndarray)
+S3method(dim,pandas.DataFrame)
+S3method(dim,pandas.Series)
 S3method(dim,pandas.core.frame.DataFrame)
 S3method(dim,pandas.core.series.Series)
 S3method(dim,scipy.sparse._base._spbase)
@@ -54,6 +56,8 @@ S3method(format,python.builtin.module)
 S3method(format,python.builtin.object)
 S3method(format,python.builtin.traceback)
 S3method(length,numpy.ndarray)
+S3method(length,pandas.DataFrame)
+S3method(length,pandas.Series)
 S3method(length,pandas.core.frame.DataFrame)
 S3method(length,pandas.core.series.Series)
 S3method(length,python.builtin.dict)
@@ -88,8 +92,11 @@ S3method(py_to_r,datatable.Frame)
 S3method(py_to_r,datetime.date)
 S3method(py_to_r,datetime.datetime)
 S3method(py_to_r,default)
+S3method(py_to_r,pandas.DataFrame)
+S3method(py_to_r,pandas.Series)
 S3method(py_to_r,pandas._libs.missing.C_NAType)
 S3method(py_to_r,pandas._libs.missing.NAType)
+S3method(py_to_r,pandas.arrays.NumpyExtensionArray)
 S3method(py_to_r,pandas.core.arrays.categorical.Categorical)
 S3method(py_to_r,pandas.core.categorical.Categorical)
 S3method(py_to_r,pandas.core.frame.DataFrame)
@@ -117,6 +124,8 @@ S3method(r_to_py,sparseMatrix)
 S3method(str,py_config)
 S3method(str,python.builtin.module)
 S3method(str,python.builtin.object)
+S3method(summary,pandas.DataFrame)
+S3method(summary,pandas.Series)
 S3method(summary,pandas.core.frame.DataFrame)
 S3method(summary,pandas.core.series.Series)
 S3method(summary,python.builtin.object)

NEWS.md

@@ -1,5 +1,7 @@
 # reticulate (development version)
 
+- Added support for pandas 3.0 (#1874, #1875).
+
 - Fix spurious `Error: ignoring SIGPIPE signal` on Unix when embedded Python writes to a closed pipe (#1868).
 
 - Fixed Positron UI interactions, such as restart/create new session with reticulate (#1871, #1869).

R/conversion.R

@@ -186,6 +186,9 @@ py_to_r.pandas.core.series.Series <- function(x) {
   values
 }
 
+#' @export
+py_to_r.pandas.Series <- py_to_r.pandas.core.series.Series
+
 #' @export
 py_to_r.pandas.core.categorical.Categorical <- function(x) {
   local_conversion_scope(x, FALSE)
@@ -199,6 +202,11 @@ py_to_r.pandas.core.categorical.Categorical <- function(x) {
 py_to_r.pandas.core.arrays.categorical.Categorical <-
   py_to_r.pandas.core.categorical.Categorical
 
+#' @export
+py_to_r.pandas.arrays.NumpyExtensionArray <- function(x) {
+  py_to_r(x$to_numpy())
+}
+
 #' @export
 py_to_r.pandas._libs.missing.NAType <- function(x) {
   NA
@@ -221,6 +229,9 @@ summary.pandas.core.series.Series <- function(object, ...) {
     object$describe()
 }
 
+#' @export
+summary.pandas.Series <- summary.pandas.core.series.Series
+
 #' @export
 length.pandas.core.series.Series <- function(x) {
   if (py_is_null_xptr(x) || !py_available())
@@ -230,11 +241,17 @@ length.pandas.core.series.Series <- function(x) {
   }
 }
 
+#' @export
+length.pandas.Series <- length.pandas.core.series.Series
+
 #' @export
 dim.pandas.core.series.Series <- function(x) {
   NULL
 }
 
+#' @export
+dim.pandas.Series <- dim.pandas.core.series.Series
+
 #' @export
 r_to_py.data.frame <- function(x, convert = FALSE) {
 
@@ -313,10 +330,14 @@ py_to_r.pandas.core.frame.DataFrame <- function(x) {
   attr(df, "pandas.index") <- index
 
   if (inherits(index, c("pandas.core.indexes.base.Index",
-                        "pandas.indexes.base.Index"))) {
+                        "pandas.indexes.base.Index",
+                        "pandas.Index"
+                      ))) {
 
     if (inherits(index, c("pandas.core.indexes.range.RangeIndex",
-                          "pandas.indexes.range.RangeIndex")) &&
+                          "pandas.indexes.range.RangeIndex",
+                          "pandas.RangeIndex"
+                        )) &&
         np$issubdtype(index$dtype, np$number))
     {
       # check for a range index from 0 -> n. in such a case, we don't need
@@ -347,7 +368,9 @@ py_to_r.pandas.core.frame.DataFrame <- function(x) {
     }
 
     else if (inherits(index, c("pandas.core.indexes.datetimes.DatetimeIndex",
-                               "pandas.tseries.index.DatetimeIndex"))) {
+                               "pandas.tseries.index.DatetimeIndex",
+                               "pandas.DatetimeIndex"
+                              ))) {
 
       converted <- tryCatch(py_to_r(index$values), error = identity)
 
@@ -380,9 +403,15 @@ py_to_r.pandas.core.frame.DataFrame <- function(x) {
 
 }
 
+#' @export
+py_to_r.pandas.DataFrame <- py_to_r.pandas.core.frame.DataFrame
+
 #' @export
 summary.pandas.core.frame.DataFrame <- summary.pandas.core.series.Series
 
+#' @export
+summary.pandas.DataFrame <- summary.pandas.core.frame.DataFrame
+
 #' @export
 length.pandas.core.frame.DataFrame <- function(x) {
   if (py_is_null_xptr(x) || !py_available())
@@ -392,6 +421,9 @@ length.pandas.core.frame.DataFrame <- function(x) {
   }
 }
 
+#' @export
+length.pandas.DataFrame <- length.pandas.core.frame.DataFrame
+
 #' @export
 dim.pandas.core.frame.DataFrame <- function(x) {
   if (py_is_null_xptr(x) || !py_available())
@@ -400,6 +432,10 @@ dim.pandas.core.frame.DataFrame <- function(x) {
     py_object_shape(x)
 }
 
+#' @export
+dim.pandas.DataFrame <- dim.pandas.core.frame.DataFrame
+
+
 # Scipy sparse matrices
 #' @importFrom Matrix Matrix
 

R/knitr-engine.R

@@ -737,7 +737,7 @@ eng_python_autoprint <- function(captured, options) {
     .engine_context$pending_plots$push(included_path)
     return("")
 
-  } else if (inherits(value, "pandas.core.frame.DataFrame")) {
+  } else if (inherits(value, c("pandas.core.frame.DataFrame", "pandas.DataFrame"))) {
 
     return(captured)
 

src/python.cpp

@@ -4382,9 +4382,8 @@ PyObject* r_to_py_pandas_nullable_series (const RObject& column, const bool conv
   // strings are not built using np array + mask. Instead they take a
   // np array with OBJECT type, with None's in the place of NA's
   if (TYPEOF(column) == STRSXP) {
-    PyObjectPtr args(PyTuple_New(2));
+    PyObjectPtr args(PyTuple_New(1));
     PyTuple_SetItem(args, 0, (PyObject*)r_to_py_numpy(column, convert));
-    PyTuple_SetItem(args, 1, Py_False);
 
     PyObject* pd_col(PyObject_Call(constructor, args, NULL));
 

tests/testthat/test-python-pandas.R

@@ -1,5 +1,10 @@
 context("pandas")
 
+pandas_version <- function() {
+  pd <- import("pandas", convert = TRUE)
+  numeric_version(pd$`__version__`)
+}
+
 test_that("Simple Pandas data frames can be roundtripped", {
   skip_if_no_pandas()
 
@@ -293,12 +298,12 @@ test_that("can cast from pandas nullable types", {
 
 test_that("NA in string columns don't prevent simplification", {
   skip_if_no_pandas()
-
+  
   pd <- import("pandas", convert = FALSE)
   np <- import("numpy", convert = FALSE)
 
   x <- pd$Series(list("a", pd$`NA`, NULL, np$nan))
-  expect_equal(py_to_r(x$dtype$name), "object")
+  expect_equal(py_to_r(x$dtype$name), if (pandas_version() < "3") "object" else "str")
 
   r <- py_to_r(x)