Backend: add support for path rewrite (#1956)

Author: vincent-hatakeyama

backend/src/hatchling/builders/constants.py

@@ -40,4 +40,4 @@ class BuildEnvVars:
     CLEAN_HOOKS_AFTER = "HATCH_BUILD_CLEAN_HOOKS_AFTER"
 
 
-EDITABLES_REQUIREMENT = "editables~=0.3"
+EDITABLES_REQUIREMENT = "editables~=0.4"

backend/src/hatchling/builders/wheel.py

@@ -531,6 +531,7 @@ def build_editable_detection(self, directory: str, **build_data: Any) -> str:
             RecordFile() as records,
         ):
             exposed_packages = {}
+            exposed_subpackages = {}
             for included_file in self.recurse_selected_project_files():
                 if not included_file.path.endswith(".py"):
                     continue
@@ -541,7 +542,10 @@ def build_editable_detection(self, directory: str, **build_data: Any) -> str:
 
                 # Root file
                 if len(path_parts) == 1:  # no cov
-                    exposed_packages[os.path.splitext(relative_path)[0]] = os.path.join(self.root, relative_path)
+                    if distribution_path in relative_path:
+                        exposed_packages[os.path.splitext(relative_path)[0]] = os.path.join(self.root, relative_path)
+                    else:
+                        exposed_subpackages[distribution_path] = os.path.join(self.root, relative_path)
                     continue
 
                 # Root package
@@ -550,27 +554,33 @@ def build_editable_detection(self, directory: str, **build_data: Any) -> str:
                     exposed_packages[root_module] = os.path.join(self.root, root_module)
                 else:
                     distribution_module = distribution_path.split(os.sep)[0]
-                    try:
+                    if distribution_path in relative_path:
                         exposed_packages[distribution_module] = os.path.join(
                             self.root,
                             f"{relative_path[: relative_path.index(distribution_path)]}{distribution_module}",
                         )
-                    except ValueError:
-                        message = (
-                            "Dev mode installations are unsupported when any path rewrite in the `sources` option "
-                            "changes a prefix rather than removes it, see: "
-                            "https://github.com/pfmoore/editables/issues/20"
-                        )
-                        raise ValueError(message) from None
+                    else:
+                        exposed_subpackages[distribution_path] = os.path.join(self.root, relative_path)
 
             editable_project = EditableProject(self.metadata.core.name, self.root)
 
             if self.config.dev_mode_exact:
                 for module, relative_path in exposed_packages.items():
                     editable_project.map(module, relative_path)
+                for module, relative_path in exposed_subpackages.items():
+                    editable_project.map(module, relative_path)
             else:
                 for relative_path in exposed_packages.values():
                     editable_project.add_to_path(os.path.dirname(relative_path))
+                exposed_subpackages_dirnames = {}
+                for module, relative_path in exposed_subpackages.items():
+                    exposed_subpackages_dirnames[os.path.dirname(module)] = os.path.dirname(relative_path)
+                for module, relative_path in exposed_subpackages_dirnames.items():
+                    # Do not add packages if parent already is included
+                    if os.path.dirname(module) not in exposed_subpackages_dirnames or exposed_subpackages_dirnames[
+                        os.path.dirname(module)
+                    ] != os.path.dirname(relative_path):
+                        editable_project.add_to_subpackage(module, relative_path)
 
             for raw_filename, content in sorted(editable_project.files()):
                 filename = raw_filename

tests/backend/builders/test_wheel.py

@@ -3484,7 +3484,8 @@ def test_no_strict_naming(self, hatch, helpers, temp_dir, config_file):
         )
         helpers.assert_files(extraction_directory, expected_files)
 
-    def test_editable_sources_rewrite_error(self, hatch, temp_dir):
+    @fixed_pathlib_resolution
+    def test_editable_sources_rewrite(self, hatch, helpers, temp_dir):
         project_name = "My.App"
 
         with temp_dir.as_cwd():
@@ -3516,18 +3517,38 @@ def test_editable_sources_rewrite_error(self, hatch, temp_dir):
         build_path = project_path / "dist"
         build_path.mkdir()
 
-        with (
-            project_path.as_cwd(),
-            pytest.raises(
-                ValueError,
-                match=(
-                    "Dev mode installations are unsupported when any path rewrite in the `sources` option "
-                    "changes a prefix rather than removes it, see: "
-                    "https://github.com/pfmoore/editables/issues/20"
-                ),
-            ),
-        ):
-            list(builder.build(directory=str(build_path)))
+        with project_path.as_cwd():
+            artifacts = list(builder.build(directory=str(build_path)))
+
+        assert len(artifacts) == 1
+        expected_artifact = artifacts[0]
+
+        build_artifacts = list(build_path.iterdir())
+        assert len(build_artifacts) == 1
+        assert expected_artifact == str(build_artifacts[0])
+        assert expected_artifact == str(build_path / f"{builder.project_id}-{get_python_versions_tag()}-none-any.whl")
+
+        extraction_directory = temp_dir / "_archive"
+        extraction_directory.mkdir()
+
+        with zipfile.ZipFile(str(expected_artifact), "r") as zip_archive:
+            zip_archive.extractall(str(extraction_directory))
+
+        metadata_directory = f"{builder.project_id}.dist-info"
+        expected_files = helpers.get_template_files(
+            "wheel.standard_editable_sources_rewrite",
+            project_name,
+            metadata_directory=metadata_directory,
+            namespace=["namespace", "plugins"],
+            source_path=project_path / "src" / "my_app",
+        )
+        helpers.assert_files(extraction_directory, expected_files)
+
+        # Inspect the archive rather than the extracted files because on Windows they lose their metadata
+        # https://stackoverflow.com/q/9813243
+        with zipfile.ZipFile(str(expected_artifact), "r") as zip_archive:
+            zip_info = zip_archive.getinfo(f"{metadata_directory}/WHEEL")
+            assert zip_info.date_time == (2020, 2, 2, 0, 0, 0)
 
     @pytest.mark.skipif(
         sys.platform != "darwin" or sys.version_info < (3, 8),

tests/helpers/templates/wheel/standard_editable_exact.py

@@ -45,7 +45,7 @@ def get_files(**kwargs):
 Name: {kwargs["project_name"]}
 Version: 0.0.1
 License-File: LICENSE.txt
-Requires-Dist: editables~=0.3
+Requires-Dist: editables~=0.4
 """,
         ),
     ))

tests/helpers/templates/wheel/standard_editable_exact_extra_dependencies.py

@@ -46,7 +46,7 @@ def get_files(**kwargs):
 Version: 0.0.1
 License-File: LICENSE.txt
 Requires-Dist: binary
-Requires-Dist: editables~=0.3
+Requires-Dist: editables~=0.4
 """,
         ),
     ))

tests/helpers/templates/wheel/standard_editable_exact_force_include.py

@@ -46,7 +46,7 @@ def get_files(**kwargs):
 Name: {kwargs["project_name"]}
 Version: 0.0.1
 License-File: LICENSE.txt
-Requires-Dist: editables~=0.3
+Requires-Dist: editables~=0.4
 """,
         ),
     ))

tests/helpers/templates/wheel/standard_editable_sources_rewrite.py

@@ -0,0 +1,56 @@
+from hatch.template import File
+from hatch.utils.fs import Path
+from hatchling.__about__ import __version__
+from hatchling.metadata.spec import DEFAULT_METADATA_VERSION
+
+from ..new.default import get_files as get_template_files
+from .utils import update_record_file_contents
+
+
+def get_files(**kwargs):
+    metadata_directory = kwargs.get("metadata_directory", "")
+    namespace_package = kwargs["namespace"]
+
+    files = []
+    for f in get_template_files(**kwargs):
+        if str(f.path) == "LICENSE.txt":
+            files.append(File(Path(metadata_directory, "licenses", f.path), f.contents))
+
+        if f.path.parts[0] != kwargs["package_name"]:
+            continue
+
+        f.path = Path(namespace_package, f.path)
+        files.append(f)
+
+    pth_file_name = f"_{kwargs['package_name']}.pth"
+    files.extend((
+        File(Path(pth_file_name), ""),
+        File(
+            Path(*namespace_package, kwargs["package_name"], "__init__.py"), f"__path__ = ['{kwargs['source_path']}']"
+        ),
+        File(
+            Path(metadata_directory, "WHEEL"),
+            f"""\
+Wheel-Version: 1.0
+Generator: hatchling {__version__}
+Root-Is-Purelib: true
+Tag: py2-none-any
+Tag: py3-none-any
+""",
+        ),
+        File(
+            Path(metadata_directory, "METADATA"),
+            f"""\
+Metadata-Version: {DEFAULT_METADATA_VERSION}
+Name: {kwargs["project_name"]}
+Version: 0.0.1
+License-File: LICENSE.txt
+""",
+        ),
+    ))
+
+    record_file = File(Path(metadata_directory, "RECORD"), "")
+    update_record_file_contents(record_file, files, generated_files={pth_file_name})
+    files.append(record_file)
+
+    return files