Plumb honoring editable builds as needed.

Author: jsirois

pex/artifact_url.py

@@ -129,6 +129,10 @@ def from_hashing_fingerprint(cls, fingerprint):
     algorithm = attr.ib()  # type: str
     hash = attr.ib()  # type: str
 
+    def __str__(self):
+        # type: () -> str
+        return "{algorithm}:{hash}".format(algorithm=self.algorithm, hash=self.hash)
+
 
 # These ranks prefer the highest digest size and then use alphabetic order for a tie-break.
 RANKED_ALGORITHMS = tuple(
@@ -268,3 +272,7 @@ def is_wheel(self):
     def fingerprint(self):
         # type: () -> Optional[Fingerprint]
         return self.fingerprints[0] if self.fingerprints else None
+
+    def __str__(self):
+        # type: () -> str
+        return self.raw_url

pex/bin/pex.py

@@ -42,6 +42,7 @@
 from pex.pex_bootstrapper import ensure_venv
 from pex.pex_builder import Check, PEXBuilder
 from pex.pex_info import PexInfo
+from pex.requirements import as_parsed_requirement
 from pex.resolve import (
     project,
     requirement_options,
@@ -862,7 +863,7 @@ def configure_clp():
     )
 
     configure_clp_sources(parser)
-    requirement_options.register(parser, include_short_editable_switch=False)
+    requirement_options.register(parser, include_editable_requirements=False)
     dependency_configuration.register(parser)
 
     parser.add_argument(
@@ -1056,7 +1057,7 @@ def build_pex(
     group_requirements = project.get_group_requirements(options)
     if group_requirements:
         requirements = OrderedSet(requirement_configuration.requirements)
-        requirements.update(str(req) for req in group_requirements)
+        requirements.update(as_parsed_requirement(req) for req in group_requirements)
         requirement_configuration = attr.evolve(
             requirement_configuration, requirements=requirements
         )
@@ -1086,7 +1087,7 @@ def build_pex(
             project_dependencies.update(built_project.iter_requirements())
 
         requirements = OrderedSet(requirement_configuration.requirements)
-        requirements.update(str(req) for req in project_dependencies)
+        requirements.update(as_parsed_requirement(req) for req in project_dependencies)
         requirement_configuration = attr.evolve(
             requirement_configuration, requirements=requirements
         )
@@ -1096,7 +1097,7 @@ def build_pex(
             " ".join(
                 itertools.chain.from_iterable(
                     (
-                        requirement_configuration.requirements or (),
+                        map(str, requirement_configuration.requirements or ()),
                         requirement_configuration.requirement_files or (),
                     )
                 )
@@ -1290,7 +1291,10 @@ def main(args=None):
         with global_environment(options) as env:
             try:
                 resolver_configuration = resolver_options.configure(
-                    options, use_system_time=options.use_system_time
+                    options,
+                    use_system_time=options.use_system_time,
+                    # N.B.: Since a PEX freezes in whls, there is no point to using editable whls.
+                    honor_editable=False,
                 )
             except resolver_options.InvalidConfigurationError as e:
                 die(str(e))

pex/build_system/pep_517.py

@@ -15,6 +15,7 @@
 from pex.dist_metadata import DistMetadata, Distribution, MetadataType
 from pex.jobs import Job, SpawnedJob
 from pex.pip.version import PipVersion, PipVersionValue
+from pex.requirements import as_parsed_requirement, parse_requirement_string
 from pex.resolve.resolvers import Resolver
 from pex.result import Error, try_
 from pex.targets import Target, Targets
@@ -24,6 +25,8 @@
 if TYPE_CHECKING:
     from typing import Any, Dict, Iterable, List, Mapping, Optional, Set, Union
 
+    from pex.requirements import ParsedRequirement
+
 _DEFAULT_BUILD_SYSTEMS = {}  # type: Dict[PipVersionValue, BuildSystem]
 
 
@@ -41,22 +44,26 @@ def _default_build_system(
             "Building {build_backend} build_backend PEX".format(build_backend=DEFAULT_BUILD_BACKEND)
         ):
             extra_env = {}  # type: Dict[str, str]
-            resolved_reqs = set()  # type: Set[str]
+            resolved_reqs = set()  # type: Set[ParsedRequirement]
             resolved_dists = []  # type: List[Distribution]
             if selected_pip_version is PipVersion.VENDORED:
-                requires = ["setuptools", str(selected_pip_version.wheel_requirement)]
+                setuptools_req = parse_requirement_string("setuptools")
+                requires = [
+                    setuptools_req,
+                    as_parsed_requirement(selected_pip_version.wheel_requirement),
+                ]
                 resolved_dists.extend(
                     Distribution.load(dist_location)
                     for dist_location in third_party.expose_installed_wheels(
                         ["setuptools"], interpreter=target.get_interpreter()
                     )
                 )
-                resolved_reqs.add("setuptools")
+                resolved_reqs.add(setuptools_req)
                 extra_env.update(__PEX_UNVENDORED__="setuptools")
             else:
                 requires = [
-                    str(selected_pip_version.setuptools_requirement),
-                    str(selected_pip_version.wheel_requirement),
+                    as_parsed_requirement(selected_pip_version.setuptools_requirement),
+                    as_parsed_requirement(selected_pip_version.wheel_requirement),
                 ]
             unresolved = [
                 requirement for requirement in requires if requirement not in resolved_reqs
@@ -71,7 +78,7 @@ def _default_build_system(
             build_system = try_(
                 BuildSystem.create(
                     interpreter=target.get_interpreter(),
-                    requires=requires,
+                    requires=map(str, requires),
                     resolved=resolved_dists,
                     build_backend=DEFAULT_BUILD_BACKEND,
                     backend_path=(),

pex/build_system/pep_518.py

@@ -14,6 +14,7 @@
 from pex.pex import PEX
 from pex.pex_bootstrapper import VenvPex, ensure_venv
 from pex.pex_builder import PEXBuilder
+from pex.requirements import parse_requirement_strings
 from pex.resolve.resolvers import Resolver
 from pex.result import Error
 from pex.targets import LocalInterpreter, Target, Targets
@@ -184,7 +185,7 @@ def load_build_system(
     ):
         result = resolver.resolve_requirements(
             targets=Targets.from_target(LocalInterpreter.create(target.get_interpreter())),
-            requirements=build_system_table.requires,
+            requirements=tuple(parse_requirement_strings(build_system_table.requires)),
         )
         return BuildSystem.create(
             interpreter=target.get_interpreter(),

pex/cli/commands/lock.py

@@ -41,7 +41,7 @@
 from pex.pep_440 import Version
 from pex.pep_503 import ProjectName
 from pex.pip.version import PipVersionValue
-from pex.requirements import LocalProjectRequirement
+from pex.requirements import LocalProjectRequirement, as_parsed_requirement
 from pex.resolve import project, requirement_options, resolver_options, target_options
 from pex.resolve.config import finalize as finalize_resolve_config
 from pex.resolve.configured_resolver import ConfiguredResolver
@@ -1179,15 +1179,15 @@ def _merge_project_requirements(
             return requirement_configuration
 
         requirements = OrderedSet(requirement_configuration.requirements)
-        requirements.update(str(req) for req in group_requirements)
+        requirements.update(as_parsed_requirement(req) for req in group_requirements)
         if projects:
             with TRACER.timed(
                 "Collecting requirements from {count} local {projects}".format(
                     count=len(projects), projects=pluralize(projects, "project")
                 )
             ):
                 requirements.update(
-                    str(req)
+                    as_parsed_requirement(req)
                     for req in projects.collect_requirements(
                         resolver=ConfiguredResolver(pip_configuration),
                         interpreter=targets.interpreter,
@@ -1357,7 +1357,7 @@ def _export(self, requirement_configuration=RequirementConfiguration()):
                             requirement_configuration
                             if requirement_configuration.has_requirements
                             else RequirementConfiguration(
-                                requirements=map(str, lock_file.requirements)
+                                requirements=map(as_parsed_requirement, lock_file.requirements)
                             )
                         ),
                     )
@@ -2212,10 +2212,7 @@ def _update(self):
     def _sync(self):
         # type: () -> Result
 
-        resolver_configuration = cast(
-            LockRepositoryConfiguration,
-            resolver_options.configure(self.options, use_system_time=False),
-        )
+        resolver_configuration = resolver_options.configure(self.options, use_system_time=False)
         production_assert(isinstance(resolver_configuration, LockRepositoryConfiguration))
         pip_configuration = resolver_configuration.pip_configuration
         locking_configuration = try_(self._locking_configuration(pip_configuration))

pex/cli/commands/run.py

@@ -40,7 +40,12 @@
 from pex.orderedset import OrderedSet
 from pex.os import safe_execv
 from pex.pip.version import PipVersionValue
-from pex.requirements import LocalProjectRequirement, ParseError, parse_requirement_string
+from pex.requirements import (
+    LocalProjectRequirement,
+    ParseError,
+    as_parsed_requirement,
+    parse_requirement_string,
+)
 from pex.resolve import lock_resolver, requirement_options, resolver_options, target_options
 from pex.resolve.configured_resolver import ConfiguredResolver
 from pex.resolve.locked_resolve import FileArtifact, UnFingerprintedLocalProjectArtifact
@@ -54,6 +59,7 @@
 from pex.targets import LocalInterpreter, Targets
 from pex.tracer import TRACER
 from pex.typing import TYPE_CHECKING
+from pex.util import CacheHelper
 from pex.variables import ENV, venv_dir
 from pex.venv import installer
 from pex.venv.installer import Provenance
@@ -115,7 +121,7 @@ class RunSpec(object):
     entry_point = attr.ib()  # type: str
     is_script = attr.ib()  # type: bool
     pip_configuration = attr.ib()  # type: PipConfiguration
-    run_requirement = attr.ib(default=None)  # type: Optional[Requirement]
+    run_requirement = attr.ib(default=None)  # type: Optional[ParsedRequirement]
     all_requirements = attr.ib(default=RequirementConfiguration())  # type: RequirementConfiguration
     args = attr.ib(default=())  # type: Tuple[str, ...]
     locks = attr.ib(default=())  # type: Tuple[str, ...]
@@ -263,8 +269,8 @@ def resolve_run_spec(
     ):
         # type: (...) -> RunSpec
 
-        requirement = None  # type: Optional[Requirement]
-        extra_requirements = OrderedSet()  # type: OrderedSet[Requirement]
+        requirement = None  # type: Optional[ParsedRequirement]
+        extra_requirements = OrderedSet()  # type: OrderedSet[ParsedRequirement]
         local_projects = OrderedSet()  # type: OrderedSet[LocalProjectRequirement]
 
         entry_point = None  # type: Optional[str]
@@ -289,7 +295,7 @@ def resolve_run_spec(
         if isinstance(self.requirement, LocalProjectRequirement):
             local_projects.add(self.requirement)
         elif self.requirement:
-            requirement = self.requirement.requirement
+            requirement = self.requirement
 
         local_project_to_dist = dict(
             _create_dists(
@@ -301,8 +307,13 @@ def resolve_run_spec(
         )
         for local_project, dist in local_project_to_dist.items():
             project_name = DistMetadata.load(dist).project_name
-            local_project_req = Requirement.local(
-                project_name=project_name, path=dist, editable=local_project.editable
+            local_project_req = as_parsed_requirement(
+                Requirement.local(
+                    project_name=project_name,
+                    path=dist,
+                    editable=local_project.editable,
+                    fragment_parameters=[("sha256", CacheHelper.hash(dist, hasher=hashlib.sha256))],
+                )
             )
             if self.entry_point == local_project:
                 entry_point = project_name.raw
@@ -326,10 +337,10 @@ def resolve_run_spec(
             locks.append("pylock.{name}.toml".format(name=name))
             locks.append("pylock.toml")
 
-        requirements = OrderedSet()  # type: OrderedSet[str]
+        requirements = OrderedSet()  # type: OrderedSet[ParsedRequirement]
         if requirement:
-            requirements.add(str(requirement))
-        requirements.update(map(str, extra_requirements))
+            requirements.add(requirement)
+        requirements.update(extra_requirements)
         if self.extra_requirements.requirements:
             requirements.update(self.extra_requirements.requirements)
 
@@ -424,7 +435,7 @@ def resolve(
         refresh=False,  # type: bool
         locked_choice=LockedChoice.AUTO,  # type: LockedChoice.Value
     ):
-        # type: (...) -> Union[Tuple[str, Iterable[Requirement], LocalInterpreter], Error]
+        # type: (...) -> Union[Tuple[str, Iterable[ParsedRequirement], LocalInterpreter], Error]
 
         script = try_(
             self._resolve_script(pip_configuration, refresh=refresh, locked_choice=locked_choice)
@@ -438,14 +449,11 @@ def resolve(
         except Unsatisfiable as e:
             return Error(str(e))
 
-        requirements = OrderedSet()  # type: OrderedSet[Requirement]
+        requirements = OrderedSet()  # type: OrderedSet[ParsedRequirement]
         if script_metadata_application.target_does_not_apply(target):
             target = try_(_resolve_local_interpreter(target_configuration, script))
         if script_metadata_application.requirement_configuration.requirements:
-            requirements.update(
-                Requirement.parse(req)
-                for req in script_metadata_application.requirement_configuration.requirements
-            )
+            requirements.update(script_metadata_application.requirement_configuration.requirements)
 
         return script, requirements, target
 
@@ -626,7 +634,7 @@ def _resolve_pylock(self, run_spec):
 
         downloaded = pip_resolver.download(
             targets=targets,
-            requirements=[str(requirement)],
+            requirements=[requirement],
             constraint_files=run_spec.all_requirements.constraint_files,
             allow_prereleases=pip_configuration.allow_prereleases,
             transitive=False,
@@ -658,7 +666,11 @@ def _resolve_pylock(self, run_spec):
             index=-1,
             project_name=dist_metadata.project_name,
             artifact=FileArtifact(
-                url=ArtifactURL.parse("file://{path}".format(path=distribution.path)),
+                url=ArtifactURL.parse(
+                    "file://{path}#sha256={hash}".format(
+                        path=distribution.path, hash=distribution.fingerprint
+                    )
+                ),
                 verified=True,
                 fingerprint=Fingerprint(algorithm="sha256", hash=distribution.fingerprint),
                 filename=os.path.basename(distribution.path),

pex/cli/commands/scie.py

@@ -263,7 +263,12 @@ def _create(self):
         if not scie_options:
             return Error("You must specify `--style {eager,lazy}`.")
 
-        resolver_configuration = resolver_options.configure(self.options)
+        resolver_configuration = resolver_options.configure(
+            self.options,
+            # N.B.: Since a scie freezes in whls via freezing in a PEX, there is no point to using
+            # editable whls.
+            honor_editable=False,
+        )
         if os.path.exists(self.options.pex[0]):
             pex_file, pex_info = try_(_extract_pex_info(self.options.pex[0]))
             if self.options.dest_dir and os.path.realpath(

pex/dist_metadata.py

@@ -902,10 +902,22 @@ def local(
         project_name,  # type: ProjectName
         path,  # type: Text
         editable=False,  # type: bool
+        fragment_parameters=(),  # type: Iterable[Tuple[str, str]]
     ):
         # type: (...) -> Requirement
         return cls.parse(
-            "{project_name} @ file://{path}".format(project_name=project_name, path=path),
+            "{project_name} @ file://{path}{fragment_parameters}".format(
+                project_name=project_name,
+                path=path,
+                fragment_parameters="#{params}".format(
+                    params="&".join(
+                        "{key}={value}".format(key=key, value=value)
+                        for key, value in fragment_parameters
+                    )
+                )
+                if fragment_parameters
+                else "",
+            ),
             editable=editable,
         )