Support complex recursive dependency category specification (#646)

When computing the dependency graph, the existing API exposes a
parameter for indicating which direct dependency categories to collect
as well as what indirect (recursive) dependency categories to collect.

This change allows the caller to specify different recursive
dependency categories depending on which category included the
dependency in the graph to begin with.

Author: cottsay

colcon_core/package_decorator.py

@@ -46,8 +46,9 @@ def add_recursive_dependencies(
 
     :param set decorators: The known packages to consider
     :param Iterable[str] direct_categories: The names of the direct categories
-    :param Iterable[str] recursive_categories: The names of the recursive
-    categories
+    :param Iterable[str]|Mapping[str, Iterable[str]] recursive_categories:
+      The names of the recursive categories, optionally mapped from the
+      immediate upstream category which included the dependency
     """
     descriptors = [decorator.descriptor for decorator in decorators]
     for decorator in decorators:

colcon_core/package_descriptor.py

@@ -2,6 +2,7 @@
 # Licensed under the Apache License, Version 2.0
 
 from collections import defaultdict
+from collections.abc import Mapping
 from copy import deepcopy
 import os
 from pathlib import Path
@@ -105,12 +106,15 @@ def get_recursive_dependencies(
           consider
         :param Iterable[str] direct_categories: The names of the direct
           categories
-        :param Iterable[str] recursive_categories: The names of the recursive
-          categories
+        :param Iterable[str]|Mapping[str, Iterable[str]] recursive_categories:
+          The names of the recursive categories, optionally mapped from the
+          immediate upstream category which included the dependency
         :returns: The dependencies
         :rtype: set[DependencyDescriptor]
         :raises AssertionError: if a package lists itself as a dependency
         """
+        if not isinstance(recursive_categories, Mapping):
+            recursive_categories = defaultdict(lambda: recursive_categories)
         # the following variable only exists for faster access within the loop
         descriptors_by_name = defaultdict(set)
         for d in descriptors:
@@ -132,11 +136,17 @@ def get_recursive_dependencies(
                 descs = descriptors_by_name[dep]
                 if not descs:
                     continue
+                categories = set()
+                for category in dep.metadata['categories']:
+                    cats = recursive_categories.get(category)
+                    if cats is None:
+                        categories = None
+                        break
+                    categories.update(cats)
                 # recursing into the same function of the dependency descriptor
                 # queue recursive dependencies
                 for d in descs:
-                    queue |= d.get_dependencies(
-                        categories=recursive_categories)
+                    queue |= d.get_dependencies(categories=categories)
                 # add the depth
                 dep.metadata['depth'] = depth
                 # add dependency to result set

colcon_core/package_selection/__init__.py

@@ -138,8 +138,9 @@ def get_packages(
     :param additional_argument_names: A list of additional arguments to
       consider
     :param Iterable[str] direct_categories: The names of the direct categories
-    :param Iterable[str] recursive_categories: The names of the recursive
-      categories
+    :param Iterable[str]|Mapping[str, Iterable[str]] recursive_categories:
+      The names of the recursive categories, optionally mapped from the
+      immediate upstream category which included the dependency
     :rtype: list
     :raises RuntimeError: if the returned set of packages contains duplicates
       package names

test/test_package_descriptor.py

@@ -1,6 +1,7 @@
 # Copyright 2016-2018 Dirk Thomas
 # Licensed under the Apache License, Version 2.0
 
+from collections import defaultdict
 import os
 from pathlib import Path
 
@@ -49,7 +50,8 @@ def test_get_dependencies():
     assert "'self'" in str(e.value)
 
 
-def test_get_recursive_dependencies():
+@pytest.fixture
+def recursive_dependencies():
     d = PackageDescriptor('/some/path')
     d.name = 'A'
     d.dependencies['build'].add('B')
@@ -70,6 +72,7 @@ def test_get_recursive_dependencies():
     d3.dependencies['build'].add('h')
     d3.dependencies['test'].add('G')
     d3.dependencies['test'].add('I')
+    d3.dependencies['test'].add('J')
 
     d4 = PackageDescriptor('/more/path')
     d4.name = 'G'
@@ -80,10 +83,35 @@ def test_get_recursive_dependencies():
     # circular dependencies should be ignored
     d5.dependencies['run'].add('A')
 
-    rec_deps = d.get_recursive_dependencies(
-        {d, d1, d2, d3, d4, d5},
+    d6 = PackageDescriptor('/paths/galore')
+    d6.name = 'J'
+
+    return d, {d, d1, d2, d3, d4, d5, d6}
+
+
+def test_get_recursive_dependencies(recursive_dependencies):
+    desc, all_descs = recursive_dependencies
+    rec_deps = desc.get_recursive_dependencies(
+        all_descs,
         direct_categories=('build', 'run'),
         recursive_categories=('run', 'test'))
+    assert rec_deps == {
+        # direct dependencies
+        'B',
+        # recursive dependencies
+        'F', 'G', 'I', 'J',
+    }
+
+
+def test_get_recursive_dependencies_map(recursive_dependencies):
+    recursive_categories = defaultdict(lambda: ('run', 'test'))
+    recursive_categories['run'] = ('run',)
+
+    desc, all_descs = recursive_dependencies
+    rec_deps = desc.get_recursive_dependencies(
+        all_descs,
+        direct_categories=('build', 'run'),
+        recursive_categories=recursive_categories)
     assert rec_deps == {
         # direct dependencies
         'B',