WIP - many questions; few answers

Author: sfc-gh-jkew

modin/core/storage_formats/pandas/query_compiler.py

@@ -2717,6 +2717,8 @@ def fillna(self, **kwargs):
         full_axis = method is not None or limit is not None
         new_dtypes = None
         if isinstance(value, BaseQueryCompiler):
+            # This code assumes that the operation occurs with the same query compiler
+            assert isinstance(value, PandasQueryCompiler)
             if squeeze_self:
                 # Self is a Series type object
                 if full_axis:

modin/core/storage_formats/pandas/query_compiler_caster.py

@@ -98,6 +98,7 @@ def _normalize_class_name(class_of_wrapped_fn: Optional[str]) -> str:
     "_query_compiler",
     "_get_query_compiler",
     "_copy_into",
+    "_update_inplace",
     "is_backend_pinned",
     "_set_backend_pinned",
     "pin_backend",
@@ -121,6 +122,7 @@ def _normalize_class_name(class_of_wrapped_fn: Optional[str]) -> str:
     "_set_backend_pinned",
     "pin_backend",
     "unpin_backend",
+    "_update_inplace",
 }
 
 
@@ -1126,11 +1128,13 @@ def cast_to_qc(arg):
                         switch_operation=f"{_normalize_class_name(class_of_wrapped_fn)}.{name}",
                         inplace=True,
                     )
+                    assert arg.get_backend() == result_backend
                     cast = arg
                 else:
                     cast = arg.set_backend(
                         result_backend,
                         switch_operation=f"{_normalize_class_name(class_of_wrapped_fn)}.{name}",
+                        inplace=False,
                     )
                 inplace_update_trackers.append(
                     InplaceUpdateTracker(

modin/pandas/base.py

@@ -4585,9 +4585,12 @@ def set_backend(
             # Last call to next informs tqdm that the operation is done
             pass
         if inplace:
+            assert query_compiler.get_backend() == backend
             self._update_inplace(query_compiler)
             # Always unpin after an explicit set_backend operation
             self._pinned = False
+            assert self._query_compiler.get_backend() == backend
+            assert self.get_backend() == backend
             return None
         else:
             return self.__constructor__(query_compiler=query_compiler)

modin/pandas/series.py

@@ -2672,6 +2672,8 @@ def _update_inplace(self, new_query_compiler) -> None:
                 self._parent.loc[self.name] = self
             else:
                 self._parent[self.name] = self
+        super(Series, self)._update_inplace(new_query_compiler=new_query_compiler)
+        assert self.get_backend() == new_query_compiler.get_backend()
 
     def _create_or_update_from_compiler(
         self, new_query_compiler, inplace=False

modin/tests/pandas/native_df_interoperability/test_binary.py

@@ -86,6 +86,7 @@ def test_math_functions(other, axis, op, backend, df_mode_pair):
         backend,
         lambda df1, df2: getattr(df1, op)(other(df2, axis), axis=axis),
         df_mode_pair,
+        pin_backend = True,
     )
 
 

modin/tests/pandas/native_df_interoperability/test_indexing.py

@@ -182,6 +182,7 @@ def _test_loc_rows(df1, df2):
         _test_loc_rows,
         expected_exception=expected_exception,
         df_mode_pair=df_mode_pair,
+        pin_backend=True,
     )
 
 

modin/tests/pandas/native_df_interoperability/utils.py

@@ -37,27 +37,35 @@ def switch_to_native_execution():
 
 
 def create_test_df_in_defined_mode(
-    *args, post_fn=None, backend=None, native=None, **kwargs
+    *args, post_fn=None, backend=None, native=None, pin_backend=False, **kwargs
 ):
     assert not current_execution_is_native(), "already in native dataframe mode."
 
     if not isinstance(native, bool):
         raise ValueError("`native` should be True or False.")
 
     with switch_to_native_execution() if native else nullcontext():
-        return create_test_dfs(*args, post_fn=post_fn, backend=backend, **kwargs)
+        modin_df, pandas_df = create_test_dfs(*args, post_fn=post_fn, backend=backend, **kwargs)
+        # pin the backend for interop tests explicitly
+        if native and pin_backend:
+            modin_df = modin_df.move_to("Pandas").pin_backend()
+        return modin_df, pandas_df
 
 
 def create_test_series_in_defined_mode(
-    vals, sort=False, backend=None, native=None, **kwargs
+    vals, sort=False, backend=None, native=None, pin_backend=False, **kwargs
 ):
     assert not current_execution_is_native(), "already in native dataframe mode."
 
     if not isinstance(native, bool):
         raise ValueError("`native` should be True or False.")
 
     with switch_to_native_execution() if native else nullcontext():
-        return create_test_series(vals, sort=sort, backend=backend, **kwargs)
+        modin_ser, pandas_ser = create_test_series(vals, sort=sort, backend=backend, **kwargs)
+        # pin the backend for interop tests explicitly
+        if native and pin_backend:
+            modin_ser = modin_ser.move_to("Pandas").pin_backend()
+        return modin_ser, pandas_ser
 
 
 def eval_general_interop(
@@ -71,14 +79,15 @@ def eval_general_interop(
     check_kwargs_callable=True,
     md_extra_kwargs=None,
     comparator_kwargs=None,
+    pin_backend=False,
     **kwargs,
 ):
     df1_native, df2_native = df_mode_pair
     modin_df1, pandas_df1 = create_test_df_in_defined_mode(
-        data, backend=backend, native=df1_native
+        data, backend=backend, native=df1_native, pin_backend=pin_backend
     )
     modin_df2, pandas_df2 = create_test_df_in_defined_mode(
-        data, backend=backend, native=df2_native
+        data, backend=backend, native=df2_native, pin_backend=pin_backend
     )
     md_kwargs, pd_kwargs = {}, {}