Merge branch 'main' into jkew/emit-hybrid-metrics

Author: sfc-gh-jkew

modin/core/storage_formats/pandas/query_compiler_caster.py

@@ -222,7 +222,9 @@ def get_backend(self) -> str:
         pass
 
     @abstractmethod
-    def set_backend(self, backend: str, inplace: bool) -> Optional[Self]:
+    def set_backend(
+        self, backend: str, inplace: bool, *, switch_operation: Optional[str] = None
+    ) -> Optional[Self]:
         """
         Set the backend of this object.
 
@@ -231,9 +233,13 @@ def set_backend(self, backend: str, inplace: bool) -> Optional[Self]:
         backend : str
             The new backend.
 
-        inplace : bool, default False
+        inplace : bool, default: False
             Whether to update the object in place.
 
+        switch_operation : Optional[str], default: None
+            The name of the operation that triggered the set_backend call.
+            Internal argument used for displaying progress bar information.
+
         Returns
         -------
         Optional[Self]
@@ -463,7 +469,11 @@ def cast_to_qc(arg: Any) -> Any:
             and arg.get_backend() != result_backend
         ):
             return arg
-        arg.set_backend(result_backend, inplace=True)
+        arg.set_backend(
+            result_backend,
+            inplace=True,
+            switch_operation=f"{class_of_wrapped_fn}.{function_name}",
+        )
         return arg
 
     return result_backend, cast_to_qc
@@ -538,7 +548,8 @@ def _maybe_switch_backend_post_op(
                 class_of_wrapped_fn=class_of_wrapped_fn,
                 function_name=function_name,
                 arguments=arguments,
-            )
+            ),
+            switch_operation=f"{class_of_wrapped_fn}.{function_name}",
         )
     return result
 
@@ -923,7 +934,9 @@ def cast_to_qc(arg):
                     and arg.get_backend() != result_backend
                 ):
                     return arg
-                cast = arg.set_backend(result_backend)
+                cast = arg.set_backend(
+                    result_backend, switch_operation=f"{class_of_wrapped_fn}.{name}"
+                )
                 inplace_update_trackers.append(
                     InplaceUpdateTracker(
                         input_castable=arg,

modin/pandas/base.py

@@ -4496,7 +4496,9 @@ def _set_backend_pinned(
             return self
 
     @doc(SET_BACKEND_DOC, class_name=__qualname__)
-    def set_backend(self, backend: str, inplace: bool = False) -> Optional[Self]:
+    def set_backend(
+        self, backend: str, inplace: bool = False, *, switch_operation: str = None
+    ) -> Optional[Self]:
         # TODO(https://github.com/modin-project/modin/issues/7467): refactor
         # to avoid this cyclic import in most places we do I/O, e.g. in
         # modin/pandas/io.py
@@ -4506,16 +4508,24 @@ def set_backend(self, backend: str, inplace: bool = False) -> Optional[Self]:
 
         progress_split_count = 2
         progress_iter = iter(range(progress_split_count))
-        if Backend.normalize(backend) != self.get_backend():
+        self_backend = self.get_backend()
+        normalized_backend = Backend.normalize(backend)
+        if normalized_backend != self_backend:
             try:
                 from tqdm.auto import trange
 
-                progress_iter = iter(
-                    trange(
-                        progress_split_count,
-                        desc=f"Transferring data from {self.get_backend()} to {Backend.normalize(backend)} ...",
+                max_rows, max_cols = self._query_compiler._max_shape()
+                if switch_operation is None:
+                    desc = (
+                        f"Transferring data from {self_backend} to {normalized_backend}"
+                        + f" with max estimated shape {max_rows}x{max_cols}"
                     )
-                )
+                else:
+                    desc = (
+                        f"Transferring data from {self_backend} to {normalized_backend} for"
+                        + f" '{switch_operation}' with max estimated shape {max_rows}x{max_cols}"
+                    )
+                progress_iter = iter(trange(progress_split_count, desc=desc))
             except ImportError:
                 # Iterate over blank range(2) if tqdm is not installed
                 pass

modin/pandas/dataframe.py

@@ -3360,8 +3360,16 @@ def __reduce__(self):
     # Persistance support methods - END
 
     @doc(SET_BACKEND_DOC, class_name=__qualname__)
-    def set_backend(self, backend: str, inplace: bool = False) -> Optional[Self]:
-        return super().set_backend(backend=backend, inplace=inplace)
+    def set_backend(
+        self,
+        backend: str,
+        inplace: bool = False,
+        *,
+        switch_operation: Optional[str] = None,
+    ) -> Optional[Self]:
+        return super().set_backend(
+            backend=backend, inplace=inplace, switch_operation=switch_operation
+        )
 
     move_to = set_backend
 

modin/pandas/groupby.py

@@ -166,13 +166,19 @@ def _get_query_compiler(self) -> Optional[BaseQueryCompiler]:
         return None
 
     @disable_logging
-    @_inherit_docstrings(QueryCompilerCaster._get_query_compiler)
+    @_inherit_docstrings(QueryCompilerCaster.get_backend)
     def get_backend(self) -> str:
         return self._df.get_backend()
 
     @disable_logging
-    @_inherit_docstrings(QueryCompilerCaster._get_query_compiler)
-    def set_backend(self, backend: str, inplace: bool = False) -> Optional[Self]:
+    @_inherit_docstrings(QueryCompilerCaster.set_backend)
+    def set_backend(
+        self,
+        backend: str,
+        inplace: bool = False,
+        *,
+        switch_operation: Optional[str] = None,
+    ) -> Optional[Self]:
         # TODO(https://github.com/modin-project/modin/issues/7544): implement
         # this method to support automatic pre-operation backend switch for
         # groupby methods.

modin/pandas/indexing.py

@@ -32,7 +32,7 @@
 from __future__ import annotations
 
 import itertools
-from typing import TYPE_CHECKING, Union
+from typing import TYPE_CHECKING, Optional, Union
 
 import numpy as np
 import pandas
@@ -53,6 +53,8 @@
 from .utils import is_scalar
 
 if TYPE_CHECKING:
+    from typing_extensions import Self
+
     from modin.core.storage_formats import BaseQueryCompiler
 
 
@@ -333,8 +335,12 @@ def _set_backend_pinned(self, pinned: bool, inplace: bool = False):
 
     @disable_logging
     @_inherit_docstrings(QueryCompilerCaster.set_backend)
-    def set_backend(self, backend, inplace: bool = False):
-        result = type(self)(self.df.set_backend(backend))
+    def set_backend(
+        self, backend, inplace: bool = False, *, switch_operation: Optional[str] = None
+    ) -> Optional[Self]:
+        result = type(self)(
+            self.df.set_backend(backend, switch_operation=switch_operation)
+        )
         if inplace:
             result._copy_into(self)
             return None

modin/pandas/series.py

@@ -2872,8 +2872,16 @@ def __reduce__(self):
     # Persistance support methods - END
 
     @doc(SET_BACKEND_DOC, class_name=__qualname__)
-    def set_backend(self, backend: str, inplace: bool = False) -> Optional[Self]:
-        return super().set_backend(backend=backend, inplace=inplace)
+    def set_backend(
+        self,
+        backend: str,
+        inplace: bool = False,
+        *,
+        switch_operation: Optional[str] = None,
+    ) -> Optional[Self]:
+        return super().set_backend(
+            backend=backend, inplace=inplace, switch_operation=switch_operation
+        )
 
     move_to = set_backend
 

modin/pandas/utils.py

@@ -64,6 +64,9 @@
     The name of the backend to set.
 inplace : bool, default: False
     Whether to modify this ``{class_name}`` in place.
+switch_operation : Optional[str], default: None
+    The name of the operation that triggered the set_backend call.
+    Internal argument used for displaying progress bar information.
 
 Returns
 -------