Revert "[PERF] Get rid of MultiIndex conversion in IntervalIndex.intersection (pandas-dev#26225)"

This reverts commit 5d2d6b4.

Author: TomAugspurger

asv_bench/benchmarks/index_object.py

@@ -196,20 +196,11 @@ def setup(self, N):
         self.intv = IntervalIndex.from_arrays(left, right)
         self.intv._engine
 
-        self.left = IntervalIndex.from_breaks(np.arange(N))
-        self.right = IntervalIndex.from_breaks(np.arange(N - 3, 2 * N - 3))
-
     def time_monotonic_inc(self, N):
         self.intv.is_monotonic_increasing
 
     def time_is_unique(self, N):
         self.intv.is_unique
 
-    def time_intersection(self, N):
-        self.left.intersection(self.right)
-
-    def time_intersection_duplicate(self, N):
-        self.intv.intersection(self.right)
-
 
 from .pandas_vb_common import setup  # noqa: F401

doc/source/whatsnew/v0.25.0.rst

@@ -510,7 +510,6 @@ Performance Improvements
 - Improved performance of :meth:`read_csv` by much faster parsing of ``MM/YYYY`` and ``DD/MM/YYYY`` datetime formats (:issue:`25922`)
 - Improved performance of nanops for dtypes that cannot store NaNs. Speedup is particularly prominent for :meth:`Series.all` and :meth:`Series.any` (:issue:`25070`)
 - Improved performance of :meth:`Series.map` for dictionary mappers on categorical series by mapping the categories instead of mapping all values (:issue:`23785`)
-- Improved performance of :meth:`IntervalIndex.intersection` (:issue:`24813`)
 - Improved performance of :meth:`read_csv` by faster concatenating date columns without extra conversion to string for integer/float zero and float ``NaN``; by faster checking the string for the possibility of being a date (:issue:`25754`)
 - Improved performance of :attr:`IntervalIndex.is_unique` by removing conversion to ``MultiIndex`` (:issue:`24813`)
 

pandas/core/indexes/base.py

@@ -2440,7 +2440,9 @@ def _union(self, other, sort):
     def _wrap_setop_result(self, other, result):
         return self._constructor(result, name=get_op_result_name(self, other))
 
-    _index_shared_docs['intersection'] = """
+    # TODO: standardize return type of non-union setops type(self vs other)
+    def intersection(self, other, sort=False):
+        """
         Form the intersection of two Index objects.
 
         This returns a new Index with elements common to the index and `other`.
@@ -2474,10 +2476,6 @@ def _wrap_setop_result(self, other, result):
         >>> idx1.intersection(idx2)
         Int64Index([3, 4], dtype='int64')
         """
-
-    # TODO: standardize return type of non-union setops type(self vs other)
-    @Appender(_index_shared_docs['intersection'])
-    def intersection(self, other, sort=False):
         self._validate_sort_keyword(sort)
         self._assert_can_do_setop(other)
         other = ensure_index(other)

pandas/core/indexes/interval.py

@@ -97,42 +97,6 @@ def _new_IntervalIndex(cls, d):
     return cls.from_arrays(**d)
 
 
-class SetopCheck:
-    """
-    This is called to decorate the set operations of IntervalIndex
-    to perform the type check in advance.
-    """
-    def __init__(self, op_name):
-        self.op_name = op_name
-
-    def __call__(self, setop):
-        def func(intvidx_self, other, sort=False):
-            intvidx_self._assert_can_do_setop(other)
-            other = ensure_index(other)
-
-            if not isinstance(other, IntervalIndex):
-                result = getattr(intvidx_self.astype(object),
-                                 self.op_name)(other)
-                if self.op_name in ('difference',):
-                    result = result.astype(intvidx_self.dtype)
-                return result
-            elif intvidx_self.closed != other.closed:
-                msg = ('can only do set operations between two IntervalIndex '
-                       'objects that are closed on the same side')
-                raise ValueError(msg)
-
-            # GH 19016: ensure set op will not return a prohibited dtype
-            subtypes = [intvidx_self.dtype.subtype, other.dtype.subtype]
-            common_subtype = find_common_type(subtypes)
-            if is_object_dtype(common_subtype):
-                msg = ('can only do {op} between two IntervalIndex '
-                       'objects that have compatible dtypes')
-                raise TypeError(msg.format(op=self.op_name))
-
-            return setop(intvidx_self, other, sort)
-        return func
-
-
 @Appender(_interval_shared_docs['class'] % dict(
     klass="IntervalIndex",
     summary="Immutable index of intervals that are closed on the same side.",
@@ -1138,78 +1102,28 @@ def equals(self, other):
     def overlaps(self, other):
         return self._data.overlaps(other)
 
-    @Appender(_index_shared_docs['intersection'])
-    @SetopCheck(op_name='intersection')
-    def intersection(self, other, sort=False):
-        if self.left.is_unique and self.right.is_unique:
-            taken = self._intersection_unique(other)
-        else:
-            # duplicates
-            taken = self._intersection_non_unique(other)
-
-        if sort is None:
-            taken = taken.sort_values()
-
-        return taken
-
-    def _intersection_unique(self, other):
-        """
-        Used when the IntervalIndex does not have any common endpoint,
-        no mater left or right.
-        Return the intersection with another IntervalIndex.
-
-        Parameters
-        ----------
-        other : IntervalIndex
-
-        Returns
-        -------
-        taken : IntervalIndex
-        """
-        lindexer = self.left.get_indexer(other.left)
-        rindexer = self.right.get_indexer(other.right)
-
-        match = (lindexer == rindexer) & (lindexer != -1)
-        indexer = lindexer.take(match.nonzero()[0])
-
-        return self.take(indexer)
-
-    def _intersection_non_unique(self, other):
-        """
-        Used when the IntervalIndex does have some common endpoints,
-        on either sides.
-        Return the intersection with another IntervalIndex.
-
-        Parameters
-        ----------
-        other : IntervalIndex
-
-        Returns
-        -------
-        taken : IntervalIndex
-        """
-        mask = np.zeros(len(self), dtype=bool)
-
-        if self.hasnans and other.hasnans:
-            first_nan_loc = np.arange(len(self))[self.isna()][0]
-            mask[first_nan_loc] = True
-
-        lmiss = other.left.get_indexer_non_unique(self.left)[1]
-        lmatch = np.setdiff1d(np.arange(len(self)), lmiss)
-
-        for i in lmatch:
-            potential = other.left.get_loc(self.left[i])
-            if is_scalar(potential):
-                if self.right[i] == other.right[potential]:
-                    mask[i] = True
-            elif self.right[i] in other.right[potential]:
-                mask[i] = True
-
-        return self[mask]
-
     def _setop(op_name, sort=None):
-        @SetopCheck(op_name=op_name)
         def func(self, other, sort=sort):
+            self._assert_can_do_setop(other)
+            other = ensure_index(other)
+            if not isinstance(other, IntervalIndex):
+                result = getattr(self.astype(object), op_name)(other)
+                if op_name in ('difference',):
+                    result = result.astype(self.dtype)
+                return result
+            elif self.closed != other.closed:
+                msg = ('can only do set operations between two IntervalIndex '
+                       'objects that are closed on the same side')
+                raise ValueError(msg)
+
+            # GH 19016: ensure set op will not return a prohibited dtype
+            subtypes = [self.dtype.subtype, other.dtype.subtype]
+            common_subtype = find_common_type(subtypes)
+            if is_object_dtype(common_subtype):
+                msg = ('can only do {op} between two IntervalIndex '
+                       'objects that have compatible dtypes')
+                raise TypeError(msg.format(op=op_name))
+
             result = getattr(self._multiindex, op_name)(other._multiindex,
                                                         sort=sort)
             result_name = get_op_result_name(self, other)
@@ -1234,6 +1148,7 @@ def is_all_dates(self):
         return False
 
     union = _setop('union')
+    intersection = _setop('intersection', sort=False)
     difference = _setop('difference')
     symmetric_difference = _setop('symmetric_difference')
 

pandas/tests/indexes/interval/test_interval.py

@@ -795,6 +795,140 @@ def test_non_contiguous(self, closed):
 
         assert 1.5 not in index
 
+    @pytest.mark.parametrize("sort", [None, False])
+    def test_union(self, closed, sort):
+        index = self.create_index(closed=closed)
+        other = IntervalIndex.from_breaks(range(5, 13), closed=closed)
+
+        expected = IntervalIndex.from_breaks(range(13), closed=closed)
+        result = index[::-1].union(other, sort=sort)
+        if sort is None:
+            tm.assert_index_equal(result, expected)
+        assert tm.equalContents(result, expected)
+
+        result = other[::-1].union(index, sort=sort)
+        if sort is None:
+            tm.assert_index_equal(result, expected)
+        assert tm.equalContents(result, expected)
+
+        tm.assert_index_equal(index.union(index, sort=sort), index)
+        tm.assert_index_equal(index.union(index[:1], sort=sort), index)
+
+        # GH 19101: empty result, same dtype
+        index = IntervalIndex(np.array([], dtype='int64'), closed=closed)
+        result = index.union(index, sort=sort)
+        tm.assert_index_equal(result, index)
+
+        # GH 19101: empty result, different dtypes
+        other = IntervalIndex(np.array([], dtype='float64'), closed=closed)
+        result = index.union(other, sort=sort)
+        tm.assert_index_equal(result, index)
+
+    @pytest.mark.parametrize("sort", [None, False])
+    def test_intersection(self, closed, sort):
+        index = self.create_index(closed=closed)
+        other = IntervalIndex.from_breaks(range(5, 13), closed=closed)
+
+        expected = IntervalIndex.from_breaks(range(5, 11), closed=closed)
+        result = index[::-1].intersection(other, sort=sort)
+        if sort is None:
+            tm.assert_index_equal(result, expected)
+        assert tm.equalContents(result, expected)
+
+        result = other[::-1].intersection(index, sort=sort)
+        if sort is None:
+            tm.assert_index_equal(result, expected)
+        assert tm.equalContents(result, expected)
+
+        tm.assert_index_equal(index.intersection(index, sort=sort), index)
+
+        # GH 19101: empty result, same dtype
+        other = IntervalIndex.from_breaks(range(300, 314), closed=closed)
+        expected = IntervalIndex(np.array([], dtype='int64'), closed=closed)
+        result = index.intersection(other, sort=sort)
+        tm.assert_index_equal(result, expected)
+
+        # GH 19101: empty result, different dtypes
+        breaks = np.arange(300, 314, dtype='float64')
+        other = IntervalIndex.from_breaks(breaks, closed=closed)
+        result = index.intersection(other, sort=sort)
+        tm.assert_index_equal(result, expected)
+
+    @pytest.mark.parametrize("sort", [None, False])
+    def test_difference(self, closed, sort):
+        index = IntervalIndex.from_arrays([1, 0, 3, 2],
+                                          [1, 2, 3, 4],
+                                          closed=closed)
+        result = index.difference(index[:1], sort=sort)
+        expected = index[1:]
+        if sort is None:
+            expected = expected.sort_values()
+        tm.assert_index_equal(result, expected)
+
+        # GH 19101: empty result, same dtype
+        result = index.difference(index, sort=sort)
+        expected = IntervalIndex(np.array([], dtype='int64'), closed=closed)
+        tm.assert_index_equal(result, expected)
+
+        # GH 19101: empty result, different dtypes
+        other = IntervalIndex.from_arrays(index.left.astype('float64'),
+                                          index.right, closed=closed)
+        result = index.difference(other, sort=sort)
+        tm.assert_index_equal(result, expected)
+
+    @pytest.mark.parametrize("sort", [None, False])
+    def test_symmetric_difference(self, closed, sort):
+        index = self.create_index(closed=closed)
+        result = index[1:].symmetric_difference(index[:-1], sort=sort)
+        expected = IntervalIndex([index[0], index[-1]])
+        if sort is None:
+            tm.assert_index_equal(result, expected)
+        assert tm.equalContents(result, expected)
+
+        # GH 19101: empty result, same dtype
+        result = index.symmetric_difference(index, sort=sort)
+        expected = IntervalIndex(np.array([], dtype='int64'), closed=closed)
+        if sort is None:
+            tm.assert_index_equal(result, expected)
+        assert tm.equalContents(result, expected)
+
+        # GH 19101: empty result, different dtypes
+        other = IntervalIndex.from_arrays(index.left.astype('float64'),
+                                          index.right, closed=closed)
+        result = index.symmetric_difference(other, sort=sort)
+        tm.assert_index_equal(result, expected)
+
+    @pytest.mark.parametrize('op_name', [
+        'union', 'intersection', 'difference', 'symmetric_difference'])
+    @pytest.mark.parametrize("sort", [None, False])
+    def test_set_incompatible_types(self, closed, op_name, sort):
+        index = self.create_index(closed=closed)
+        set_op = getattr(index, op_name)
+
+        # TODO: standardize return type of non-union setops type(self vs other)
+        # non-IntervalIndex
+        if op_name == 'difference':
+            expected = index
+        else:
+            expected = getattr(index.astype('O'), op_name)(Index([1, 2, 3]))
+        result = set_op(Index([1, 2, 3]), sort=sort)
+        tm.assert_index_equal(result, expected)
+
+        # mixed closed
+        msg = ('can only do set operations between two IntervalIndex objects '
+               'that are closed on the same side')
+        for other_closed in {'right', 'left', 'both', 'neither'} - {closed}:
+            other = self.create_index(closed=other_closed)
+            with pytest.raises(ValueError, match=msg):
+                set_op(other, sort=sort)
+
+        # GH 19016: incompatible dtypes
+        other = interval_range(Timestamp('20180101'), periods=9, closed=closed)
+        msg = ('can only do {op} between two IntervalIndex objects that have '
+               'compatible dtypes').format(op=op_name)
+        with pytest.raises(TypeError, match=msg):
+            set_op(other, sort=sort)
+
     def test_isin(self, closed):
         index = self.create_index(closed=closed)