diff --git a/tests/test_cli.py b/tests/test_cli.py index dd299553..99dc2aa2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -691,3 +691,90 @@ def fake_env_name_to_prefix( # Optionally, verify that our fake function printed the expected message. output = capsys.readouterr().out assert "Fake create called with" in output + + +def test_local_dependency_with_extras( + tmp_path: Path, + capsys: pytest.CaptureFixture, +) -> None: + """Test that local dependencies with extras are properly installed.""" + # Set up the directory structure + package1_dir = tmp_path / "package1" + my_package_dir = tmp_path / "my_package" + my_package2_dir = tmp_path / "my_package2" + + package1_dir.mkdir() + my_package_dir.mkdir() + my_package2_dir.mkdir() + + # Create requirements.yaml for package1 + (package1_dir / "requirements.yaml").write_text( + """ + dependencies: + - common-dep + local_dependencies: + - ../my_package[my-extra] + """, + ) + + # Create requirements.yaml for my_package + (my_package_dir / "requirements.yaml").write_text( + """ + dependencies: + - my-package-dep + optional_dependencies: + my-extra: + - ../my_package2 + """, + ) + + # Make my_package pip installable + (my_package_dir / "setup.py").write_text( + """ + from setuptools import setup + setup(name="my_package", version="0.1.0") + """, + ) + + # Create requirements.yaml for my_package2 + (my_package2_dir / "requirements.yaml").write_text( + """ + dependencies: + - my-package2-dep + """, + ) + + # Make my_package2 pip installable + (my_package2_dir / "setup.py").write_text( + """ + from setuptools import setup + setup(name="my_package2", version="0.1.0") + """, + ) + + # Run the unidep install command + _install_command( + package1_dir / "requirements.yaml", + conda_executable="micromamba", + conda_env_name=None, + conda_env_prefix=None, + conda_lock_file=None, + dry_run=True, # Just print, don't execute + editable=False, + verbose=True, + ) + + # Check the output + captured = capsys.readouterr().out + + # Expect the dependencies to be installed + assert "common-dep" in captured + assert "my-package-dep" in captured + assert "my-package2-dep" in captured + + # We expect both my_package and my_package2 to be installed + assert "../my_package" in captured + assert ( + f"pip install --no-deps --verbose {my_package_dir.resolve()} {my_package2_dir.resolve()}" + in captured + ) diff --git a/tests/test_parse_yaml_local_dependencies.py b/tests/test_parse_yaml_local_dependencies.py index eca12e60..8c6a8347 100644 --- a/tests/test_parse_yaml_local_dependencies.py +++ b/tests/test_parse_yaml_local_dependencies.py @@ -17,6 +17,7 @@ parse_requirements, resolve_conflicts, ) +from unidep._dependencies_parsing import _get_local_deps_from_optional_section from .helpers import maybe_as_toml @@ -531,3 +532,248 @@ def test_parse_local_dependencies_without_local_deps_themselves( r2.write_text("") with pytest.raises(RuntimeError, match="is not pip installable"): parse_local_dependencies(r1, verbose=True, raise_if_missing=True) + + +def test_local_dependency_with_extras(tmp_path: Path) -> None: + """Test that local dependencies with extras are properly installed.""" + # Set up the directory structure + package1_dir = tmp_path / "package1" + my_package_dir = tmp_path / "my_package" + my_package2_dir = tmp_path / "my_package2" + + package1_dir.mkdir() + my_package_dir.mkdir() + my_package2_dir.mkdir() + + # Create requirements.yaml for package1 + (package1_dir / "requirements.yaml").write_text( + """ + dependencies: + - common-dep + local_dependencies: + - ../my_package[my-extra] + """, + ) + + # Create requirements.yaml for my_package + (my_package_dir / "requirements.yaml").write_text( + """ + dependencies: + - my-package-dep + optional_dependencies: + my-extra: + - ../my_package2 + """, + ) + + # Make my_package pip installable + (my_package_dir / "setup.py").write_text( + """ + from setuptools import setup + setup(name="my_package", version="0.1.0") + """, + ) + + # Create requirements.yaml for my_package2 + (my_package2_dir / "requirements.yaml").write_text( + """ + dependencies: + - my-package2-dep + """, + ) + + # Make my_package2 pip installable + (my_package2_dir / "setup.py").write_text( + """ + from setuptools import setup + setup(name="my_package2", version="0.1.0") + """, + ) + local_dependencies = parse_local_dependencies( + package1_dir / "requirements.yaml", + verbose=True, + ) + assert local_dependencies == { + package1_dir.absolute(): [ + my_package_dir.absolute(), + my_package2_dir.absolute(), + ], + } + + +def test_nested_extras_in_local_dependencies( + tmp_path: Path, + capsys: pytest.CaptureFixture, +) -> None: + """Test local dependencies with nested extras chains. + + main_package + -> lib_package[extra1,another-extra] + -> utility_package[extra2] (from extra1) + -> base_package (from extra2) + """ + # Create a complex dependency structure: + # main_package -> lib_package[extra1] -> utility_package[extra2] -> base_package + + main_dir = tmp_path / "main_package" + lib_dir = tmp_path / "lib_package" + utility_dir = tmp_path / "utility_package" + base_dir = tmp_path / "base_package" + + for dir_path in [main_dir, lib_dir, utility_dir, base_dir]: + dir_path.mkdir() + # Make all packages pip-installable + (dir_path / "setup.py").write_text( + f""" + from setuptools import setup + setup(name="{dir_path.name}", version="0.1.0") + """, + ) + + # Main package depends on lib_package with extra1 + (main_dir / "requirements.yaml").write_text( + """ + dependencies: + - main-dependency + local_dependencies: + - ../lib_package[extra1,another-extra] + """, + ) + + # Lib package has optional dependency on utility_package with extra2 + (lib_dir / "requirements.yaml").write_text( + """ + dependencies: + - lib-dependency + optional_dependencies: + extra1: + - lib-extra1-dependency + - ../utility_package[extra2] + another-extra: + - another-extra-dependency + """, + ) + + # Utility package has optional dependency on base_package + (utility_dir / "requirements.yaml").write_text( + """ + dependencies: + - utility-dependency + optional_dependencies: + extra2: + - utility-extra2-dependency + - ../base_package + other-extra: + - not-included-dependency + """, + ) + + # Base package has standard dependencies + (base_dir / "requirements.yaml").write_text( + """ + dependencies: + - base-dependency + """, + ) + + # Parse dependencies with verbose output to capture logs + local_dependencies = parse_local_dependencies( + main_dir / "requirements.yaml", + verbose=True, + ) + + # Capture and print the output to help with debugging + output = capsys.readouterr().out + print(output) + + # Check that all packages are correctly included in dependencies + assert local_dependencies == { + main_dir.absolute(): sorted( + [ + lib_dir.absolute(), + utility_dir.absolute(), + base_dir.absolute(), + ], + ), + } + + # Verify that extras were processed correctly through verbose output + assert "Processing `../lib_package[extra1,another-extra]`" in output + + yaml = YAML(typ="safe") + extras = ["extra1"] + + # Test the function directly to verify non-empty nested extras + deps_from_extras = _get_local_deps_from_optional_section( + req_path=lib_dir / "requirements.yaml", + extras_list=extras, + yaml=yaml, + verbose=True, + ) + + # We expect a tuple with utility_package path and ["extra2"] as nested extras + assert len(deps_from_extras) == 1 + path, extra, nested_extras = deps_from_extras[0] + assert extra == "../utility_package[extra2]" + assert path.name == "utility_package" + assert nested_extras == ["extra2"] + + # Also test with "*" to ensure it handles all extras + all_extras_deps = _get_local_deps_from_optional_section( + req_path=lib_dir / "requirements.yaml", + extras_list=["*"], + yaml=yaml, + verbose=True, + ) + + # Should include dependencies from both extra1 and another-extra + assert len(all_extras_deps) == 1 # Only one is a path + assert all_extras_deps[0][0].name == "utility_package" + + +def test_wildcard_extras_processing(tmp_path: Path) -> None: + """Test handling of wildcard extras.""" + package_dir = tmp_path / "package" + package_dir.mkdir() + + # Create a requirements file with multiple extras + (package_dir / "requirements.yaml").write_text( + """ + dependencies: + - main-dep + optional_dependencies: + extra1: + - ../dep1 + extra2: + - ../dep2 + extra3: + - not-a-path + """, + ) + + yaml = YAML(typ="safe") + + # Test with wildcard + wildcard_deps = _get_local_deps_from_optional_section( + req_path=package_dir / "requirements.yaml", + extras_list=["*"], + yaml=yaml, + verbose=True, + ) + + # Should find both path dependencies from all extras + assert len(wildcard_deps) == 2 + paths = {dep[0].name for dep in wildcard_deps} + assert paths == {"dep1", "dep2"} + + # Test with specific extras + specific_deps = _get_local_deps_from_optional_section( + req_path=package_dir / "requirements.yaml", + extras_list=["extra1"], + yaml=yaml, + verbose=True, + ) + + # Should only find the dependency from extra1 + assert len(specific_deps) == 1 + assert specific_deps[0][0].name == "dep1" diff --git a/unidep/_dependencies_parsing.py b/unidep/_dependencies_parsing.py index 54158119..fdacdbf0 100644 --- a/unidep/_dependencies_parsing.py +++ b/unidep/_dependencies_parsing.py @@ -574,7 +574,13 @@ def _add_dependencies( parse_yaml_requirements = parse_requirements -def _extract_local_dependencies( # noqa: PLR0912 +def _resolve_local_path(dep_path: str, parent_path: Path) -> tuple[Path, list[str]]: + local_path, dep_extras = split_path_and_extras(dep_path) + abs_path = (parent_path / local_path).resolve() + return abs_path, dep_extras + + +def _extract_local_dependencies( path: Path, base_path: Path, processed: set[Path], @@ -596,82 +602,179 @@ def _extract_local_dependencies( # noqa: PLR0912 path_with_extras=PathWithExtras(path, extras), verbose=verbose, ) + # Handle "local_dependencies" (or old name "includes", changed in 0.42.0) for local_dependency in _get_local_dependencies(data): assert not os.path.isabs(local_dependency) # noqa: PTH117 - local_path, extras = split_path_and_extras(local_dependency) - abs_local = (path.parent / local_path).resolve() - if abs_local.suffix in (".whl", ".zip"): - if verbose: - print(f"🔗 Adding `{local_dependency}` from `local_dependencies`") - dependencies[str(base_path)].add(str(abs_local)) - continue - if not abs_local.exists(): - if raise_if_missing: - msg = f"File `{abs_local}` not found." - raise FileNotFoundError(msg) - continue - - try: - requirements_path = parse_folder_or_filename(abs_local).path - except FileNotFoundError: - # Means that this is a local package that is not managed by unidep. - if is_pip_installable(abs_local): - dependencies[str(base_path)].add(str(abs_local)) - if warn_non_managed: - # We do not need to emit this warning when `pip install` is called - warn( - f"⚠️ Installing a local dependency (`{abs_local.name}`) which" - " is not managed by unidep, this will skip all of its" - " dependencies, i.e., it will call `pip install` with" - " `--no-deps`. To properly manage this dependency," - " add a `requirements.yaml` or `pyproject.toml` file with" - " `[tool.unidep]` in its directory.", - ) - elif _is_empty_folder(abs_local): - msg = ( - f"`{local_dependency}` in `local_dependencies` is not pip" - " installable because it is an empty folder. Is it perhaps" - " an uninitialized Git submodule? If so, initialize it with" - " `git submodule update --init --recursive`. Otherwise," - " remove it from `local_dependencies`." - ) - raise RuntimeError(msg) from None - elif _is_empty_git_submodule(abs_local): - # Extra check for empty Git submodules (common problem folks run into) - msg = ( - f"`{local_dependency}` in `local_dependencies` is not installable" - " by pip because it is an empty Git submodule. Either remove it" - " from `local_dependencies` or fetch the submodule with" - " `git submodule update --init --recursive`." - ) - raise RuntimeError(msg) from None - else: - msg = ( - f"`{local_dependency}` in `local_dependencies` is not pip" - " installable nor is it managed by unidep. Remove it" - " from `local_dependencies`." - ) - raise RuntimeError(msg) from None - continue - - project_path = str(requirements_path.parent) - if project_path == str(base_path): - continue - if not check_pip_installable or is_pip_installable(requirements_path.parent): - dependencies[str(base_path)].add(project_path) + abs_local, extras = _resolve_local_path(local_dependency, path.parent) if verbose: - print(f"🔗 Adding `{requirements_path}` from `local_dependencies`") - _extract_local_dependencies( - requirements_path, - base_path, - processed, - dependencies, + print(f"🔗 Processing `{local_dependency}` from `local_dependencies`") + _add_dependency( + dep_path=abs_local, + dep_unresolved=local_dependency, + base_path=base_path, + dependencies=dependencies, + yaml=yaml, + processed=processed, + with_extras=extras, check_pip_installable=check_pip_installable, verbose=verbose, + raise_if_missing=raise_if_missing, + warn_non_managed=warn_non_managed, ) +def _add_dependency( # noqa: PLR0912 + dep_path: Path, + dep_unresolved: str, # only for printing + base_path: Path, + dependencies: dict[str, set[str]], + yaml: YAML, + processed: set[Path], + *, + with_extras: list[str] | None = None, + check_pip_installable: bool = True, + verbose: bool = False, + raise_if_missing: bool = True, + warn_non_managed: bool = True, +) -> None: + if dep_path.suffix in (".whl", ".zip"): + if verbose: + print(f"🔗 Adding `{dep_unresolved}` from `local_dependencies`") + dependencies[str(base_path)].add(str(dep_path)) + return + + if not dep_path.exists(): + if raise_if_missing: + msg = f"File `{dep_path}` not found." + raise FileNotFoundError(msg) + return + + try: + requirements_path = parse_folder_or_filename(dep_path).path + except FileNotFoundError: + # Means that this is a local package that is not managed by unidep. + if is_pip_installable(dep_path): + dependencies[str(base_path)].add(str(dep_path)) + if warn_non_managed: + # We do not need to emit this warning when `pip install` is called + warn( + f"⚠️ Installing a local dependency (`{dep_path.name}`) which" + " is not managed by unidep, this will skip all of its" + " dependencies, i.e., it will call `pip install` with" + " `--no-deps`. To properly manage this dependency," + " add a `requirements.yaml` or `pyproject.toml` file with" + " `[tool.unidep]` in its directory.", + ) + elif _is_empty_folder(dep_path): + msg = ( + f"`{dep_unresolved}` in `local_dependencies` is not pip" + " installable because it is an empty folder. Is it perhaps" + " an uninitialized Git submodule? If so, initialize it with" + " `git submodule update --init --recursive`. Otherwise," + " remove it from `local_dependencies`." + ) + raise RuntimeError(msg) from None + elif _is_empty_git_submodule(dep_path): + # Extra check for empty Git submodules (common problem folks run into) + msg = ( + f"`{dep_unresolved}` in `local_dependencies` is not installable" + " by pip because it is an empty Git submodule. Either remove it" + " from `local_dependencies` or fetch the submodule with" + " `git submodule update --init --recursive`." + ) + raise RuntimeError(msg) from None + else: + msg = ( + f"`{dep_unresolved}` in `local_dependencies` is not pip" + " installable nor is it managed by unidep. Remove it" + " from `local_dependencies`." + ) + raise RuntimeError(msg) from None + return + + # It's a valid requirements file + project_path = str(requirements_path.parent) + if project_path == str(base_path): + return + if not check_pip_installable or is_pip_installable(project_path): + dependencies[str(base_path)].add(project_path) + + # Process extras if specified + if with_extras: + local_deps_from_extras = _get_local_deps_from_optional_section( + req_path=requirements_path, + extras_list=with_extras, + yaml=yaml, + verbose=verbose, + ) + + for extra_path, extra_unresolved, nested_extras in local_deps_from_extras: + if verbose: + print(f"🔗 Processing `{extra_path}` from optional dependencies") + _add_dependency( + dep_path=extra_path, + dep_unresolved=extra_unresolved, + base_path=base_path, + dependencies=dependencies, + yaml=yaml, + processed=processed, + with_extras=nested_extras, + check_pip_installable=check_pip_installable, + verbose=verbose, + warn_non_managed=False, # Avoid excessive warnings + ) + + # Continue recursive processing + if verbose: + print(f"🔗 Processing dependencies in `{requirements_path}`") + _extract_local_dependencies( + requirements_path, + base_path, + processed, + dependencies, + check_pip_installable=check_pip_installable, + verbose=verbose, + raise_if_missing=raise_if_missing, + warn_non_managed=warn_non_managed, + ) + + +def _get_local_deps_from_optional_section( + req_path: Path, + extras_list: list[str], + yaml: YAML, + verbose: bool, # noqa: FBT001 +) -> list[tuple[Path, str, list[str]]]: + """Extract local dependencies from optional dependency sections. + + Returns a list of tuples (dependency_path, dependency_unresolved, nested_extras) + """ + result = [] + dep_data = _load(req_path, yaml) + opt_deps = dep_data.get("optional_dependencies", {}) + + # Determine which sections to process + sections_to_process = set() + for extra in extras_list: + if extra == "*": + sections_to_process.update(opt_deps) + elif extra in opt_deps: + sections_to_process.add(extra) + + # Process each section + for section in sections_to_process: + for dep in opt_deps[section]: + if isinstance(dep, str) and _str_is_path_like(dep): + abs_path, nested_extras = _resolve_local_path(dep, req_path.parent) + if verbose: + msg = f"🔗 Found local dependency `{abs_path}` in optional section `{section}`" # noqa: E501 + print(msg) + result.append((abs_path, dep, nested_extras)) + + return result + + def parse_local_dependencies( *paths: Path, check_pip_installable: bool = True, @@ -687,7 +790,6 @@ def parse_local_dependencies( name of the project folder => list of `Path`s of local dependencies folders. """ # noqa: E501 dependencies: dict[str, set[str]] = defaultdict(set) - for p in paths: if verbose: print(f"🔗 Analyzing dependencies in `{p}`")