Merge branch 'main' into jkew/emit-hybrid-metrics

Author: sfc-gh-jkew

modin/config/__init__.py

@@ -44,6 +44,8 @@
     MinPartitionSize,
     MinRowPartitionSize,
     ModinNumpy,
+    NativePandasMaxRows,
+    NativePandasTransferThreshold,
     NPartitions,
     PersistentPickle,
     ProgressBar,
@@ -85,6 +87,9 @@
     "LazyExecution",
     # Dask specific
     "DaskThreadsPerWorker",
+    # Native Pandas Specific
+    "NativePandasMaxRows",
+    "NativePandasTransferThreshold",
     # Partitioning
     "NPartitions",
     "MinPartitionSize",

modin/config/envvars.py

@@ -1305,6 +1305,25 @@ class DaskThreadsPerWorker(EnvironmentVariable, type=int):
     default = 1
 
 
+class NativePandasMaxRows(EnvironmentVariable, type=int):
+    """Maximum number of rows which can be processed using local, native, pandas."""
+
+    varname = "MODIN_NATIVE_MAX_ROWS"
+    default = 10_000_000
+
+
+class NativePandasTransferThreshold(EnvironmentVariable, type=int):
+    """
+    Targeted max number of dataframe rows which should be transferred between engines.
+
+    This is often the same value as MODIN_NATIVE_MAX_ROWS but it can be independently
+    set to change how transfer costs are considered.
+    """
+
+    varname = "MODIN_NATIVE_MAX_XFER_ROWS"
+    default = 10_000_000
+
+
 class DynamicPartitioning(EnvironmentVariable, type=bool):
     """
     Set to true to use Modin's dynamic-partitioning implementation where possible.

modin/core/storage_formats/base/query_compiler.py

@@ -356,13 +356,13 @@ def move_to_cost(
         """
         if isinstance(self, other_qc_type):
             return QCCoercionCost.COST_ZERO
-        if self._TRANSFER_THRESHOLD <= 0:
+        if self.__class__._transfer_threshold() <= 0:
             return QCCoercionCost.COST_ZERO
         cost = int(
             (
                 QCCoercionCost.COST_IMPOSSIBLE
                 * self._max_shape()[0]
-                / self._TRANSFER_THRESHOLD
+                / self.__class__._transfer_threshold()
             )
         )
         if cost > QCCoercionCost.COST_IMPOSSIBLE:
@@ -447,7 +447,7 @@ def stay_cost(
         return self._stay_cost_rows(
             self._max_shape()[0],
             self._OPERATION_PER_ROW_OVERHEAD,
-            self._MAX_SIZE_THIS_ENGINE_CAN_HANDLE,
+            self.__class__._engine_max_size(),
             self._OPERATION_INITIALIZATION_OVERHEAD,
         )
 
@@ -498,10 +498,20 @@ def move_to_me_cost(
         return cls._stay_cost_rows(
             row_count,
             cls._OPERATION_PER_ROW_OVERHEAD,
-            cls._MAX_SIZE_THIS_ENGINE_CAN_HANDLE,
+            cls._engine_max_size(),
             cls._OPERATION_INITIALIZATION_OVERHEAD,
         )
 
+    @classmethod
+    def _engine_max_size(cls) -> int:
+        """Maximum number of rows this engine can handle."""
+        return cls._MAX_SIZE_THIS_ENGINE_CAN_HANDLE
+
+    @classmethod
+    def _transfer_threshold(cls) -> int:
+        """Maximum number of rows this backend can handle before transferring data to another backend."""
+        return cls._TRANSFER_THRESHOLD
+
     @disable_logging
     def max_cost(self) -> int:
         """

modin/core/storage_formats/pandas/native_query_compiler.py

@@ -23,12 +23,11 @@
 import pandas
 from pandas.core.dtypes.common import is_scalar
 
+from modin.config.envvars import NativePandasMaxRows, NativePandasTransferThreshold
 from modin.core.dataframe.base.interchange.dataframe_protocol.dataframe import (
     ProtocolDataframe,
 )
-from modin.core.storage_formats.base.query_compiler import (
-    BaseQueryCompiler,
-)
+from modin.core.storage_formats.base.query_compiler import BaseQueryCompiler
 from modin.utils import _inherit_docstrings
 
 _NO_REPARTITION_ON_NATIVE_EXECUTION_EXCEPTION_MESSAGE = (
@@ -92,10 +91,8 @@ class NativeQueryCompiler(BaseQueryCompiler):
         The pandas frame to query with the compiled queries.
     """
 
-    _MAX_SIZE_THIS_ENGINE_CAN_HANDLE = 10_000_000
     _OPERATION_INITIALIZATION_OVERHEAD = 0
     _OPERATION_PER_ROW_OVERHEAD = 0
-    _TRANSFER_THRESHOLD = 10_000_000
 
     _modin_frame: pandas.DataFrame
     _should_warn_on_default_to_pandas: bool = False
@@ -212,8 +209,21 @@ def free(self):
     def finalize(self):
         return
 
-    # Dataframe interchange protocol
+    @classmethod
+    def _engine_max_size(cls):
+        # do not return the custom configuration for sub-classes
+        if cls == NativeQueryCompiler:
+            return NativePandasMaxRows.get()
+        return cls._MAX_SIZE_THIS_ENGINE_CAN_HANDLE
 
+    @classmethod
+    def _transfer_threshold(cls):
+        # do not return the custom configuration for sub-classes
+        if cls == NativeQueryCompiler:
+            return NativePandasTransferThreshold.get()
+        return cls._TRANSFER_THRESHOLD
+
+    # Dataframe interchange protocol
     def to_interchange_dataframe(
         self, nan_as_null: bool = False, allow_copy: bool = True
     ):

modin/core/storage_formats/pandas/query_compiler_caster.py

@@ -61,7 +61,6 @@
     "__delattr__",
     "__getattr__",
     "_getattribute__from_extension_impl",
-    "_getattr__from_extension_impl",
     "get_backend",
     "move_to",
     "_update_inplace",

modin/pandas/base.py

@@ -110,7 +110,6 @@
     "_query_compiler",
     "get_backend",
     "_getattribute__from_extension_impl",
-    "_getattr__from_extension_impl",
     "_get_query_compiler",
     "set_backend",
     "_pinned",
@@ -4376,29 +4375,6 @@ def default_handler(*args, **kwargs):
                 return default_handler
         return attr
 
-    @disable_logging
-    def __getattr__(self, item) -> Any:
-        """
-        Return attribute from this `BasePandasDataset`.
-
-        Parameters
-        ----------
-        item : str
-            Item to get.
-
-        Returns
-        -------
-        Any
-            The attribute from this `BasePandasDataset`.
-        """
-        # NOTE that to get an attribute, python calls __getattribute__() first and
-        # then falls back to __getattr__() if the former raises an AttributeError.
-        if item not in _EXTENSION_NO_LOOKUP:
-            extension = self._getattr__from_extension_impl(item, __class__._extensions)
-            if extension is not sentinel:
-                return extension
-        return object.__getattribute__(self, item)
-
     def __array_ufunc__(
         self, ufunc: np.ufunc, method: str, *inputs: Any, **kwargs: Any
     ) -> DataFrame | Series | Any:
@@ -4664,36 +4640,6 @@ def _getattribute__from_extension_impl(
             )
         return sentinel
 
-    @disable_logging
-    def _getattr__from_extension_impl(self, item, extensions: EXTENSION_DICT_TYPE):
-        """
-        __getattr__() an extension with the given name from the given set of extensions.
-
-        Implement __getattr__() for extensions. python falls back to
-        __getattr__() if __getattribute__() raises an AttributeError.
-
-        Parameters
-        ----------
-        item : str
-            The name of the attribute to get.
-        extensions : EXTENSION_DICT_TYPE
-            The set of extensions.
-
-        Returns
-        -------
-        Any
-            The attribute from the extension, or `sentinel` if the attribute is
-            not found.
-        """
-        extension = self._get_extension(item, extensions)
-        if extension is not sentinel:
-            # We need to implement callable extensions before we fall back
-            # to __getattr__(), because they need to dispatch to the
-            # appropriate backend.
-            assert not callable(extension)
-            return extension
-        return sentinel
-
     @disable_logging
     @_inherit_docstrings(QueryCompilerCaster._get_query_compiler)
     def _get_query_compiler(self):

modin/pandas/dataframe.py

@@ -2644,11 +2644,6 @@ def __getattr__(self, key) -> Any:
         """
         # NOTE that to get an attribute, python calls __getattribute__() first and
         # then falls back to __getattr__() if the former raises an AttributeError.
-
-        if key not in _EXTENSION_NO_LOOKUP:
-            extension = self._getattr__from_extension_impl(key, __class__._extensions)
-            if extension is not sentinel:
-                return extension
         try:
             return super().__getattr__(key)
         except AttributeError as err:

modin/pandas/series.py

@@ -387,10 +387,6 @@ def __getattr__(self, key: Hashable) -> Any:
         """
         # NOTE that to get an attribute, python calls __getattribute__() first and
         # then falls back to __getattr__() if the former raises an AttributeError.
-        if key not in _EXTENSION_NO_LOOKUP:
-            extension = self._getattr__from_extension_impl(key, __class__._extensions)
-            if extension is not sentinel:
-                return extension
         try:
             return super().__getattr__(key)
         except AttributeError as err:

modin/tests/pandas/extensions/test_base_extensions.py

@@ -217,13 +217,23 @@ def del_property(self):
         )
 
         modin_object = data_class({"a": [1, 2, 3], "b": [4, 5, 6]}).set_backend(backend)
-        assert hasattr(modin_object, public_property_name)
         setattr(modin_object, public_property_name, "value")
         assert getattr(modin_object, public_property_name) == "value"
         delattr(modin_object, public_property_name)
         # check that the deletion works.
         assert not hasattr(modin_object, private_property_name)
 
+    def test_get_property_that_raises_attribute_error_on_get_modin_issue_7562(
+        self, data_class
+    ):
+        def get_property(self):
+            raise AttributeError
+
+        register_base_accessor(name="extension_property")(property(fget=get_property))
+        modin_object = data_class()
+        with pytest.raises(AttributeError):
+            getattr(modin_object, "extension_property")
+
     def test_non_settable_extension_property(self, Backend1, data_class):
         modin_object = data_class([0])
         property_name = "property_name"

modin/tests/pandas/native_df_interoperability/test_compiler_caster.py

@@ -24,7 +24,13 @@
 
 import modin.pandas as pd
 from modin.config import context as config_context
-from modin.config.envvars import Backend, Engine, Execution
+from modin.config.envvars import (
+    Backend,
+    Engine,
+    Execution,
+    NativePandasMaxRows,
+    NativePandasTransferThreshold,
+)
 from modin.core.execution.dispatching.factories import factories
 from modin.core.execution.dispatching.factories.factories import BaseFactory
 from modin.core.io.io import BaseIO
@@ -1317,3 +1323,34 @@ def test_second_init_only_calls_from_pandas_once_github_issue_7559():
         ) as mock_from_pandas:
             pd.DataFrame([1])
             mock_from_pandas.assert_called_once()
+
+
+def test_native_config():
+    qc = NativeQueryCompiler(pandas.DataFrame([0, 1, 2]))
+
+    # Native Query Compiler gets a special configuration
+    assert qc._TRANSFER_THRESHOLD == 0
+    assert qc._transfer_threshold() == NativePandasTransferThreshold.get()
+    assert qc._MAX_SIZE_THIS_ENGINE_CAN_HANDLE == 1
+    assert qc._engine_max_size() == NativePandasMaxRows.get()
+
+    oldmax = qc._engine_max_size()
+    oldthresh = qc._transfer_threshold()
+
+    with config_context(NativePandasMaxRows=123, NativePandasTransferThreshold=321):
+        qc2 = NativeQueryCompiler(pandas.DataFrame([0, 1, 2]))
+        assert qc2._transfer_threshold() == 321
+        assert qc2._engine_max_size() == 123
+        assert qc._engine_max_size() == 123
+        assert qc._transfer_threshold() == 321
+
+        # sub class configuration is unchanged
+        class AQC(NativeQueryCompiler):
+            pass
+
+        subqc = AQC(pandas.DataFrame([0, 1, 2]))
+        assert subqc._TRANSFER_THRESHOLD == 0
+        assert subqc._MAX_SIZE_THIS_ENGINE_CAN_HANDLE == 1
+
+    assert qc._engine_max_size() == oldmax
+    assert qc._transfer_threshold() == oldthresh