Fix infinite os.execv when unsuitable dependencies are specified in PYTHONPATH (#151)

Remove incompatible Python tools from environment

Whenever a Python tool is about to be installed with `pip`
in the virtualenv, get rid of previous installation of this tool
in the environment if any (sys.path and PYTHONPATH environment variable)
so that it doesn't interference with pip installation.
The change also detects the conflicts encountered by pip to ensure
that unsuitable dependencies of the package are also excluded from
the environment.

This change solves an issue where an unsuitable version of black was provided
via PYTHONPATH, and pip was not able to uninstall it because provided from
outside. The required version of black also needed a newer version of `pathspec`
provided in PYTHONPATH. Therefore, despite the `pip install proper-version-of-black`,
`pip` considered that the conditions were not met, and the loop was going on forever.

Author: tristan0x

cpp/lib.py

@@ -28,6 +28,34 @@
 DEFAULT_RE_EXTRACT_VERSION = "([0-9]+\\.[0-9]+(\\.[0-9]+)?[ab]?)"
 
 
+def forget_python_pkg(package):
+    """
+    exclude from PYTHONPATH environment variable
+    and sys.path trace of the specified package
+
+    Args:
+        package: name of the Python package to exclude
+    """
+    import pkg_resources
+
+    try:
+        dist = pkg_resources.get_distribution(package)
+    except pkg_resources.DistributionNotFound:
+        return
+    PYTHONPATH = os.environ.get("PYTHONPATH")
+    if PYTHONPATH is not None and dist.location in PYTHONPATH:
+        logging.debug(
+            "Remove incompatible version of %s in PYTHONPATH: %s",
+            package,
+            dist.location,
+        )
+        os.environ["PYTHONPATH"] = PYTHONPATH.replace(dist.location, "")
+        try:
+            sys.path.remove(dist.location)
+        except ValueError:
+            pass
+
+
 @functools.lru_cache()
 def source_dir():
     """
@@ -370,6 +398,9 @@ def is_requirement_met(self, requirement) -> bool:
             return True
         except ImportError:
             self._install_requirement("setuptools", restart=True)
+        except pkg_resources.ContextualVersionConflict as conflict:
+            forget_python_pkg(conflict.req.name)
+            return False
         except (pkg_resources.VersionConflict, pkg_resources.DistributionNotFound):
             return False
 
@@ -708,6 +739,7 @@ def configure(self):
                 self._path, self._version = self.find_tool_in_path()
             except FileNotFoundError as e:
                 if self.requirement:
+                    forget_python_pkg(self.requirement.name)
                     BBPProject.virtualenv().ensure_requirement(self.requirement)
                     self._path, self._version = self.find_tool_in_path(
                         [BBPProject.virtualenv().bin_dir]