Optimistic backjumping with cutoff (#188)

Author: notatallshaw

pyproject.toml

@@ -98,7 +98,7 @@ exclude = [
 ]
 
 [tool.ruff.lint.mccabe]
-max-complexity = 12
+max-complexity = 20
 
 [tool.mypy]
 warn_unused_configs = true

src/resolvelib/resolvers/resolution.py

@@ -3,7 +3,7 @@
 import collections
 import itertools
 import operator
-from typing import TYPE_CHECKING, Collection, Generic, Iterable, Mapping
+from typing import TYPE_CHECKING, Generic
 
 from ..structs import (
     CT,
@@ -27,9 +27,13 @@
 )
 
 if TYPE_CHECKING:
+    from collections.abc import Collection, Iterable, Mapping
+
     from ..providers import AbstractProvider, Preference
     from ..reporters import BaseReporter
 
+_OPTIMISTIC_BACKJUMPING_RATIO: float = 0.1
+
 
 def _build_result(state: State[RT, CT, KT]) -> Result[RT, CT, KT]:
     mapping = state.mapping
@@ -77,6 +81,11 @@ def __init__(
         self._r = reporter
         self._states: list[State[RT, CT, KT]] = []
 
+        # Optimistic backjumping variables
+        self._optimistic_backjumping_ratio = _OPTIMISTIC_BACKJUMPING_RATIO
+        self._save_states: list[State[RT, CT, KT]] | None = None
+        self._optimistic_start_round: int | None = None
+
     @property
     def state(self) -> State[RT, CT, KT]:
         try:
@@ -274,6 +283,25 @@ def _patch_criteria(
             )
         return True
 
+    def _save_state(self) -> None:
+        """Save states for potential rollback if optimistic backjumping fails."""
+        if self._save_states is None:
+            self._save_states = [
+                State(
+                    mapping=s.mapping.copy(),
+                    criteria=s.criteria.copy(),
+                    backtrack_causes=s.backtrack_causes[:],
+                )
+                for s in self._states
+            ]
+
+    def _rollback_states(self) -> None:
+        """Rollback states and disable optimistic backjumping."""
+        self._optimistic_backjumping_ratio = 0.0
+        if self._save_states:
+            self._states = self._save_states
+            self._save_states = None
+
     def _backjump(self, causes: list[RequirementInformation[RT, CT]]) -> bool:
         """Perform backjumping.
 
@@ -324,13 +352,26 @@ def _backjump(self, causes: list[RequirementInformation[RT, CT]]) -> bool:
                 except (IndexError, KeyError):
                     raise ResolutionImpossible(causes) from None
 
-                # Only backjump if the current broken state is
-                # an incompatible dependency
-                if name not in incompatible_deps:
+                if (
+                    not self._optimistic_backjumping_ratio
+                    and name not in incompatible_deps
+                ):
+                    # For safe backjumping only backjump if the current dependency
+                    # is not the same as the incompatible dependency
                     break
 
+                # On the first time a non-safe backjump is done the state
+                # is saved so we can restore it later if the resolution fails
+                if (
+                    self._optimistic_backjumping_ratio
+                    and self._save_states is None
+                    and name not in incompatible_deps
+                ):
+                    self._save_state()
+
                 # If the current dependencies and the incompatible dependencies
-                # are overlapping then we have found a cause of the incompatibility
+                # are overlapping then we have likely found a cause of the
+                # incompatibility
                 current_dependencies = {
                     self._p.identify(d) for d in self._p.get_dependencies(candidate)
                 }
@@ -394,9 +435,32 @@ def resolve(self, requirements: Iterable[RT], max_rounds: int) -> State[RT, CT,
         # pinning the virtual "root" package in the graph.
         self._push_new_state()
 
+        # Variables for optimistic backjumping
+        optimistic_rounds_cutoff: int | None = None
+        optimistic_backjumping_start_round: int | None = None
+
         for round_index in range(max_rounds):
             self._r.starting_round(index=round_index)
 
+            # Handle if optimistic backjumping has been running for too long
+            if self._optimistic_backjumping_ratio and self._save_states is not None:
+                if optimistic_backjumping_start_round is None:
+                    optimistic_backjumping_start_round = round_index
+                    optimistic_rounds_cutoff = int(
+                        (max_rounds - round_index) * self._optimistic_backjumping_ratio
+                    )
+
+                    if optimistic_rounds_cutoff <= 0:
+                        self._rollback_states()
+                        continue
+                elif optimistic_rounds_cutoff is not None:
+                    if (
+                        round_index - optimistic_backjumping_start_round
+                        >= optimistic_rounds_cutoff
+                    ):
+                        self._rollback_states()
+                        continue
+
             unsatisfied_names = [
                 key
                 for key, criterion in self.state.criteria.items()
@@ -448,12 +512,29 @@ def resolve(self, requirements: Iterable[RT], max_rounds: int) -> State[RT, CT,
                 # Backjump if pinning fails. The backjump process puts us in
                 # an unpinned state, so we can work on it in the next round.
                 self._r.resolving_conflicts(causes=causes)
-                success = self._backjump(causes)
-                self.state.backtrack_causes[:] = causes
 
-                # Dead ends everywhere. Give up.
-                if not success:
-                    raise ResolutionImpossible(self.state.backtrack_causes)
+                try:
+                    success = self._backjump(causes)
+                except ResolutionImpossible:
+                    if self._optimistic_backjumping_ratio and self._save_states:
+                        failed_optimistic_backjumping = True
+                    else:
+                        raise
+                else:
+                    failed_optimistic_backjumping = bool(
+                        not success
+                        and self._optimistic_backjumping_ratio
+                        and self._save_states
+                    )
+
+                if failed_optimistic_backjumping and self._save_states:
+                    self._rollback_states()
+                else:
+                    self.state.backtrack_causes[:] = causes
+
+                    # Dead ends everywhere. Give up.
+                    if not success:
+                        raise ResolutionImpossible(self.state.backtrack_causes)
             else:
                 # discard as information sources any invalidated names
                 # (unsatisfied names that were previously satisfied)

tests/functional/python/inputs/case/backjump-test-3.json

@@ -0,0 +1,18 @@
+{
+	"index": "backjump-test-3",
+	"requested": [
+		"a==1",
+		"b",
+		"c"
+	],
+	"resolved": {
+		"a": "1",
+		"b": "1",
+		"c": "2",
+		"d": "1"
+	},
+	"unvisited": {
+		"c": ["1"]
+	},
+	"needs_optimistic": true
+}

tests/functional/python/inputs/case/backjump-test-4.json

@@ -0,0 +1,24 @@
+{
+	"index": "backjump-test-4",
+	"requested": [
+		"a==1",
+		"b",
+		"c",
+		"e",
+		"f"
+	],
+	"resolved": {
+		"a": "1",
+		"b": "1",
+		"c": "2",
+		"d": "1",
+		"e": "2",
+		"f": "2"
+	},
+	"unvisited": {
+		"c": ["1"],
+		"e": ["1"],
+		"f": ["1"]
+	},
+	"needs_optimistic": true
+}

tests/functional/python/inputs/index/backjump-test-3.json

@@ -0,0 +1,43 @@
+{
+    "a": {
+        "1": {
+            "dependencies": []
+        },
+        "2": {
+            "dependencies": []
+        }
+    },
+    "b": {
+        "2": {
+            "dependencies": [
+                "d==2"
+            ]
+        },
+        "1": {
+            "dependencies": [
+                "d==1"
+            ]
+        }
+    },
+    "c": {
+        "1": {
+            "dependencies": []
+        },
+        "2": {
+            "dependencies": []
+        }
+    },
+    "d": {
+        "2": {
+            "dependencies": [
+                "a==2"
+            ]
+        },
+        "1": {
+            "dependencies": [
+                "a==1"
+            ]
+        }
+    },
+	"needs_optimistic": true
+}

tests/functional/python/inputs/index/backjump-test-4.json

@@ -0,0 +1,58 @@
+{
+    "a": {
+        "1": {
+            "dependencies": []
+        },
+        "2": {
+            "dependencies": []
+        }
+    },
+    "b": {
+        "2": {
+            "dependencies": [
+                "d==2"
+            ]
+        },
+        "1": {
+            "dependencies": [
+                "d==1"
+            ]
+        }
+    },
+    "c": {
+        "1": {
+            "dependencies": []
+        },
+        "2": {
+            "dependencies": []
+        }
+    },
+    "d": {
+        "2": {
+            "dependencies": [
+                "a==2"
+            ]
+        },
+        "1": {
+            "dependencies": [
+                "a==1"
+            ]
+        }
+    },
+    "e": {
+        "1": {
+            "dependencies": []
+        },
+        "2": {
+            "dependencies": []
+        }
+    },
+    "f": {
+        "1": {
+            "dependencies": []
+        },
+        "2": {
+            "dependencies": []
+        }
+    }
+}

tests/functional/python/test_resolvers_python.py

@@ -6,11 +6,11 @@
 
 import packaging.markers
 import packaging.requirements
-import packaging.specifiers
 import packaging.utils
 import packaging.version
 import pytest
 
+import resolvelib.resolvers.resolution
 from resolvelib import AbstractProvider, ResolutionImpossible, Resolver
 
 Candidate = collections.namedtuple("Candidate", "name version extras")
@@ -65,6 +65,11 @@ def __init__(self, filename):
         else:
             self.expected_unvisited = None
 
+        if "needs_optimistic" in case_data:
+            self.needs_optimistic = case_data["needs_optimistic"]
+        else:
+            self.needs_optimistic = False
+
     def identify(self, requirement_or_candidate):
         name = packaging.utils.canonicalize_name(requirement_or_candidate.name)
         if requirement_or_candidate.extras:
@@ -219,3 +224,38 @@ def test_resolver(provider, reporter):
             assert not unexpected_versions, (
                 f"Unexpcted versions visited {name}: {', '.join(unexpected_versions)}"
             )
+
+
+def test_no_optimistic_backtracking_resolver(provider, reporter, monkeypatch):
+    """
+    Tests the resolver works with optimistic backtracking disabled for all
+    cases, except for skipping candidates that is known to require optimistic
+    backtracking.
+    """
+    monkeypatch.setattr(
+        resolvelib.resolvers.resolution, "_OPTIMISTIC_BACKJUMPING_RATIO", 0.0
+    )
+    resolver = Resolver(provider, reporter)
+
+    if provider.expected_confliction:
+        with pytest.raises(ResolutionImpossible) as ctx:
+            result = resolver.resolve(provider.root_requirements)
+            print(_format_resolution(result))  # Provide some debugging hints.
+        assert _format_confliction(ctx.value) == provider.expected_confliction
+    else:
+        resolution = resolver.resolve(provider.root_requirements)
+        assert _format_resolution(resolution) == provider.expected_resolution
+
+    if provider.expected_unvisited and not provider.needs_optimistic:
+        visited_versions = defaultdict(set)
+        for visited_candidate in reporter.visited:
+            visited_versions[visited_candidate.name].add(str(visited_candidate.version))
+
+        for name, versions in provider.expected_unvisited.items():
+            if name not in visited_versions:
+                continue
+
+            unexpected_versions = set(versions).intersection(visited_versions[name])
+            assert not unexpected_versions, (
+                f"Unexpcted versions visited {name}: {', '.join(unexpected_versions)}"
+            )