Merge pull request #17 from IBM/SetupTools

Setup tools

Author: gabe-l-hart

import_tracker/__init__.py

@@ -4,6 +4,7 @@
 """
 
 # Local
+from . import setup_tools
 from .import_tracker import (
     BEST_EFFORT,
     LAZY,

import_tracker/import_tracker.py

@@ -6,7 +6,7 @@
 # Standard
 from contextlib import contextmanager
 from types import ModuleType
-from typing import Dict, List, Optional
+from typing import Dict, Iterable, List, Optional
 import copy
 import importlib
 import inspect
@@ -128,26 +128,7 @@ def get_required_imports(name: str) -> List[str]:
 
 def get_required_packages(name: str) -> List[str]:
     """Get the set of installable packages required by this names module"""
-    # Lazily create the global mapping
-    global _module_to_pkg
-    if _module_to_pkg is None:
-        _module_to_pkg = _map_modules_to_package_names()
-
-    # Get all required imports
-    required_modules = get_required_imports(name)
-
-    # Merge the required packages for each
-    required_pkgs = set()
-    for mod in required_modules:
-        # If there is a known mapping, use it
-        if mod in _module_to_pkg:
-            required_pkgs.update(_module_to_pkg[mod])
-
-        # Otherwise, assume that the name of the module is itself the name of
-        # the package
-        else:
-            required_pkgs.add(mod)
-    return sorted(list(required_pkgs))
+    return _get_required_packages_for_imports(get_required_imports(name))
 
 
 def get_tracked_modules(prefix: str = "") -> List[str]:
@@ -299,7 +280,7 @@ def _map_modules_to_package_names():
                         ),
                     ):
                         modules_to_package_names.setdefault(modname, set()).add(
-                            package_name
+                            _standardize_package_name(package_name)
                         )
 
     return modules_to_package_names
@@ -335,3 +316,31 @@ def _load_static_tracker():
         with open(static_tracker, "r") as handle:
             global _module_dep_mapping
             _module_dep_mapping.update(json.load(handle))
+
+
+def _standardize_package_name(raw_package_name):
+    """Helper to convert the arbitrary ways packages can be represented to a
+    common (matchable) representation
+    """
+    return raw_package_name.strip().lower().replace("-", "_")
+
+
+def _get_required_packages_for_imports(imports: Iterable[str]) -> List[str]:
+    """Get the set of installable packages required by this list of imports"""
+    # Lazily create the global mapping
+    global _module_to_pkg
+    if _module_to_pkg is None:
+        _module_to_pkg = _map_modules_to_package_names()
+
+    # Merge the required packages for each
+    required_pkgs = set()
+    for mod in imports:
+        # If there is a known mapping, use it
+        if mod in _module_to_pkg:
+            required_pkgs.update(_module_to_pkg[mod])
+
+        # Otherwise, assume that the name of the module is itself the name of
+        # the package
+        else:
+            required_pkgs.add(mod)
+    return sorted(list(required_pkgs))

import_tracker/setup_tools.py

@@ -0,0 +1,128 @@
+"""
+This module holds tools for libraries to use when definint requirements and
+extras_require sets in a setup.py
+"""
+
+# Standard
+from functools import reduce
+from typing import Dict, List, Tuple
+import importlib
+import logging
+import re
+import sys
+
+# Local
+from .import_tracker import (
+    _get_required_packages_for_imports,
+    _standardize_package_name,
+    get_required_packages,
+    get_tracked_modules,
+)
+
+# Regex for parsing requirements
+req_split_expr = re.compile(r"[=><!~\[]")
+
+# Shared logger
+log = logging.getLogger("SETUP")
+
+
+def _map_requirements(declared_dependencies, dependency_set):
+    """Given the declared dependencies from requirements.txt and the given
+    programmatic dependency set, return the subset of declared dependencies that
+    matches the dependency set
+    """
+    return sorted(
+        [
+            declared_dependencies[dep.replace("-", "_")]
+            for dep in dependency_set
+            if dep.replace("-", "_") in declared_dependencies
+        ]
+    )
+
+
+def parse_requirements(
+    requirements_file: str,
+    library_name: str,
+) -> Tuple[List[str], Dict[str, List[str]]]:
+    """This helper uses the lists of required modules and parameters for the
+    given library to produce requirements and the extras_require dict.
+
+    Args:
+        requirements_file:  str
+            Path to the requirements file for this library
+        library_name:  str
+            The name of the library being setup
+
+    Returns:
+        requirements:  List[str]
+            The list of requirements to pass to setup()
+        extras_require:  Dict[str, List[str]]
+            The extras_require dict to pass to setup()
+    """
+
+    # Import the library. This is used at build time, so it's safe to do so.
+    importlib.import_module(library_name)
+
+    # Load all requirements from the requirements file
+    with open(requirements_file, "r") as handle:
+        requirements = {
+            _standardize_package_name(req_split_expr.split(line, 1)[0]): line.strip()
+            for line in handle.readlines()
+            if line.strip() and not line.startswith("#")
+        }
+    this_pkg = sys.modules[__name__].__name__.split(".")[0]
+    assert (
+        this_pkg in requirements or this_pkg.replace("_", "-") in requirements
+    ), f"No requirement for {this_pkg} found"
+    log.debug("Requirements: %s", requirements)
+
+    # Get the raw import sets for each tracked module
+    import_sets = {
+        tracked_module.split(".")[-1]: set(get_required_packages(tracked_module))
+        for tracked_module in get_tracked_modules(library_name)
+    }
+    log.debug("Import sets: %s", import_sets)
+
+    # Determine the common requirements from the intersection of all import sets
+    common_imports = None
+    for import_set in import_sets.values():
+        if common_imports is None:
+            common_imports = import_set
+        else:
+            common_imports = common_imports.intersection(import_set)
+    common_imports.add(_get_required_packages_for_imports([this_pkg])[0])
+    log.debug("Common imports: %s", common_imports)
+
+    # Compute the sets of unique requirements for each tracked module
+    extras_require_sets = {
+        set_name: import_set - common_imports
+        for set_name, import_set in import_sets.items()
+    }
+    log.debug("Extras require sets: %s", extras_require_sets)
+
+    # Add any listed requirements in that don't show up in any tracked module.
+    # These requirements may be needed by an untracked portion of the library or
+    # they may be runtime imports.
+    all_tracked_requirements = reduce(
+        lambda acc_set, req_set: acc_set.union(req_set),
+        extras_require_sets.values(),
+        common_imports,
+    )
+    missing_reqs = (
+        set(_get_required_packages_for_imports(requirements.keys()))
+        - all_tracked_requirements
+    )
+    log.debug(
+        "Adding missing requirements %s to common_imports",
+        sorted(list(missing_reqs)),
+    )
+    common_imports = common_imports.union(missing_reqs)
+
+    # Map all dependencies through those listed in requirements.txt
+    standardized_requirements = {
+        key.replace("-", "_"): val for key, val in requirements.items()
+    }
+    return _map_requirements(standardized_requirements, common_imports), {
+        set_name: _map_requirements(standardized_requirements, import_set)
+        for set_name, import_set in extras_require_sets.items()
+    }

test/helpers.py

@@ -14,6 +14,23 @@
 import import_tracker
 
 
+@pytest.fixture(autouse=True)
+def configure_logging():
+    """Fixture that configures logging from the env. It is auto-used, so if
+    imported, it will automatically configure for each test.
+
+    NOTE: The import of alog is inside the function since alog is used as a
+        sample package for lazy importing in some tests
+    """
+    # First Party
+    import alog
+
+    alog.configure(
+        default_level=os.environ.get("LOG_LEVEL", "info"),
+        filters=os.environ.get("LOG_FILTERS", ""),
+    )
+
+
 @pytest.fixture(autouse=True)
 def reset_sys_modules():
     """This fixture will reset the sys.modules dict to only the keys held before

test/test_import_tracker.py

@@ -138,13 +138,13 @@ def test_get_required_packages_static_tracker(mode):
         import sample_lib
 
         assert set(import_tracker.get_required_packages("sample_lib.submod2")) == {
-            "alchemy-logging",
+            "alchemy_logging",
         }
         assert set(
             import_tracker.get_required_packages("sample_lib.nested.submod3")
         ) == {
-            "alchemy-logging",
-            "PyYAML",
+            "alchemy_logging",
+            "pyyaml",
         }
 
 

test/test_setup_tools.py

@@ -0,0 +1,81 @@
+"""
+Tests for setup tools
+"""
+
+# Standard
+import os
+import tempfile
+
+# Third Party
+import pytest
+
+# Local
+from .helpers import configure_logging
+from import_tracker.setup_tools import parse_requirements
+
+sample_lib_requirements = [
+    "alchemy-logging>=1.0.3",
+    "PyYaml >= 6.0",
+    "conditional_deps",
+    "import-tracker",
+]
+
+
+def test_parse_requirements_happy():
+    """Make sure that parse_requirements correctly parses requirements for a
+    library with multiple tracked modules
+    """
+    with tempfile.NamedTemporaryFile("w") as requirements_file:
+        # Make a requirements file that looks normal
+        requirements_file.write("\n".join(sample_lib_requirements))
+        requirements_file.flush()
+
+        # Parse the reqs for "sample_lib"
+        requirements, extras_require = parse_requirements(
+            requirements_file.name,
+            "sample_lib",
+        )
+
+        # Make sure the right parsing happened
+        assert requirements == ["import-tracker"]
+        assert extras_require == {
+            "submod3": sorted(["PyYaml >= 6.0", "alchemy-logging>=1.0.3"]),
+            "submod1": sorted(["conditional_deps"]),
+            "submod2": sorted(["alchemy-logging>=1.0.3"]),
+        }
+
+
+def test_parse_requirements_add_untracked_reqs():
+    """Make sure that packages in the requirements.txt which don't show up in
+    any of the tracked modules are added to the common requirements
+    """
+    with tempfile.NamedTemporaryFile("w") as requirements_file:
+        # Make a requirements file with an extra entry
+        extra_req = "something-ElSe[extras]~=1.2.3"
+        requirements_file.write("\n".join(sample_lib_requirements + [extra_req]))
+        requirements_file.flush()
+
+        # Parse the reqs for "sample_lib"
+        requirements, _ = parse_requirements(
+            requirements_file.name,
+            "sample_lib",
+        )
+
+        # Make sure the extra requirement was added
+        extra_req in requirements
+
+
+def test_parse_requirements_missing_import_tracker():
+    """Make sure that parse_requirements does require import_tracker (or
+    import-tracker) in the requirements list
+    """
+    with tempfile.NamedTemporaryFile("w") as requirements_file:
+        # Make a requirements file that is missing import_tracker
+        requirements_file.write(
+            "\n".join(set(sample_lib_requirements) - {"import-tracker"})
+        )
+        requirements_file.flush()
+
+        # Make sure the assertion is tripped
+        with pytest.raises(AssertionError):
+            parse_requirements(requirements_file.name, "sample_lib")