graph: auto rebuild/validation of reverse deps

This commit introduces the graph modules with class to handle a graph of
reverse dependencies between packages in a Rift project.

When a package is built, especially when its version has changed, it is
important to verify that it does not break build and tests of other
packages that depends on it (aka. reverse dependencies).

With this new feature, as soon as dependency_tracking option is enabled
in project configuration, the graph of reverse dependencies is
automatically created with rift build and validate actions, unless
--skip-deps option is set on command line. Dependencies are either found
in packages info.yaml metadata file or by parsing subpackages (including
their provides) and build requirements out of RPM spec files as a
fallback.

Rift then solves the reverse dependencies of the packages provided by
users with build and validate actions to determine the packages that
should be rebuilt or validated consequently.

Unit tests are added to validate the code introduced for this feature.

Author: rezib

lib/rift/Config.py

@@ -86,6 +86,7 @@ class RiftDeprecatedConfWarning(FutureWarning):
 _DEFAULT_SYNC_METHOD = 'dnf'
 _DEFAULT_SYNC_INCLUDE = []
 _DEFAULT_SYNC_EXCLUDE = []
+_DEFAULT_DEPENDENCY_TRACKING = False
 
 class Config():
     """
@@ -301,6 +302,10 @@ class Config():
             'values': ['9p', 'virtiofs'],
         },
         'sync_output': {},
+        'dependency_tracking': {
+            'check': 'bool',
+            'default': _DEFAULT_DEPENDENCY_TRACKING,
+        },
         # XXX?: 'mock.name' ?
         # XXX?: 'mock.template' ?
     }

lib/rift/Controller.py

@@ -52,6 +52,7 @@
 from rift.Mock import Mock
 from rift.Package import Package, Test
 from rift.Repository import LocalRepository, ProjectArchRepositories
+from rift.graph import PackagesDependencyGraph
 from rift.RPM import RPM, Spec, RPMLINT_CONFIG_V1, RPMLINT_CONFIG_V2
 from rift.TempDir import TempDir
 from rift.TestResults import TestCase, TestResults
@@ -140,6 +141,8 @@ def make_parser():
     subprs.add_argument('-s', '--sign', action='store_true',
                         help='sign built packages with GPG key '
                              '(implies -p, --publish)')
+    subprs.add_argument('-S', '--skip-deps', action='store_true',
+                        help='Skip automatic rebuild of reverse dependencies')
     subprs.add_argument('--junit', metavar='FILENAME',
                         help='write junit result file')
     subprs.add_argument('--dont-update-repo', dest='updaterepo', action='store_false',
@@ -178,6 +181,8 @@ def make_parser():
                         help='write junit result file')
     subprs.add_argument('-p', '--publish', action='store_true',
                         help='publish build RPMS to repository')
+    subprs.add_argument('-S', '--skip-deps', action='store_true',
+                        help='Skip automatic validation of reverse dependencies')
 
     # Validate diff
     subprs = subparsers.add_parser('validdiff')
@@ -443,15 +448,19 @@ def __init__(self, pkg, config=None):
         Test.__init__(self, cmd, "basic_install")
         self.local = False
 
-def build_pkg(config, args, pkg, arch):
+def build_pkg(config, args, pkg, arch, staging):
     """
     Build a package for a specific architecture
       - config: rift configuration
+      - args: command line arguments
       - pkg: package to build
-      - repo: rpm repositories to use
-      - suppl_repos: optional additional repositories
+      - arch: CPU architecture
+      - staging: temporary staging rpm repositories to hold dependencies when
+        testing builds of reserve dependencies recursively.
     """
-    repos = ProjectArchRepositories(config, arch)
+    repos = ProjectArchRepositories(config, arch,
+                                    extra=staging.consumables[arch]
+                                    if staging is not None else None)
     if args.publish and not repos.can_publish():
         raise RiftError("Cannot publish if 'working_repo' is undefined")
 
@@ -468,6 +477,14 @@ def build_pkg(config, args, pkg, arch):
         logging.info('Built: %s', rpm.filepath)
     message("RPMS successfully built")
 
+    # If defined, publish in staging repository
+    if staging:
+        message("Publishing RPMS in staging repository...")
+        mock.publish(staging)
+
+        message("Updating staging repository...")
+        staging.update()
+
     # Publish
     if args.publish:
         message("Publishing RPMS...")
@@ -576,7 +593,11 @@ def validate_pkgs(config, args, pkgs, arch):
         - launch tests
     """
 
-    repos = ProjectArchRepositories(config, arch)
+    # Create staging repository for all packages and add it to the project
+    # supplementary repositories.
+    (staging, stagedir) = create_staging_repo(config)
+    repos = ProjectArchRepositories(config, arch,
+                                    extra=staging.consumables[arch])
 
     if args.publish and not repos.can_publish():
         raise RiftError("Cannot publish if 'working_repo' is undefined")
@@ -615,10 +636,12 @@ def validate_pkgs(config, args, pkgs, arch):
         message('Validate specfile...')
         spec.check(pkg)
 
-        (staging, stagedir) = create_staging_repo(config)
-
         message('Preparing Mock environment...')
         mock = Mock(config, arch, config.get('version'))
+
+        for repo in repos.all:
+            logging.debug("Mock with repo %s: %s", repo.name, repo.url)
+
         mock.init(repos.all)
 
         try:
@@ -665,7 +688,8 @@ def validate_pkgs(config, args, pkgs, arch):
             message("Keep environment, VM is running. Use: rift vm connect")
         else:
             mock.clean()
-            stagedir.delete()
+
+    stagedir.delete()
 
     banner(f"All packages checked on architecture {arch}")
 
@@ -747,7 +771,7 @@ def action_vm(args, config):
         ret = vm_build(vm, args, config)
     return ret
 
-def build_pkgs(config, args, pkgs, arch):
+def build_pkgs(config, args, pkgs, arch, staging):
     """
     Build a list of packages on a given architecture and return results.
     """
@@ -776,7 +800,7 @@ def build_pkgs(config, args, pkgs, arch):
         now = time.time()
         try:
             pkg.load()
-            build_pkg(config, args, pkg, arch)
+            build_pkg(config, args, pkg, arch, staging)
         except RiftError as ex:
             logging.error("Build failure: %s", str(ex))
             results.add_failure(case, time.time() - now, err=str(ex))
@@ -800,17 +824,30 @@ def action_build(args, config):
     results = TestResults('build')
 
     staff, modules = staff_modules(config)
+    pkgs = get_packages_to_build(config, staff, modules, args)
+    logging.info(
+        "Ordered list of packages to build: %s",
+        str([pkg.name for pkg in pkgs])
+    )
 
     # Build all packages for all project supported architectures
     for arch in config.get('arch'):
 
-        pkgs = Package.list(config, staff, modules, args.packages)
-        results.extend(build_pkgs(config, args, pkgs, arch))
+        # Create temporary staging repository to hold dependencies unless
+        # dependency tracking is disabled in project configuration or user set
+        # --skip-deps argument.
+        staging = stagedir = None
+        if config.get('dependency_tracking') and not args.skip_deps:
+            (staging, stagedir) = create_staging_repo(config)
+
+        results.extend(build_pkgs(config, args, pkgs, arch, staging))
 
         if getattr(args, 'junit', False):
             logging.info('Writing test results in %s', args.junit)
             results.junit(args.junit)
 
+        if stagedir:
+            stagedir.delete()
         banner(f"All packages processed for architecture {arch}")
 
     banner('All architectures processed')
@@ -866,13 +903,18 @@ def action_validate(args, config):
 
     staff, modules = staff_modules(config)
     results = TestResults('validate')
+    pkgs = get_packages_to_build(config, staff, modules, args)
+    logging.info(
+        "Ordered list of packages to validate: %s",
+        str([pkg.name for pkg in pkgs])
+    )
     # Validate packages on all project supported architectures
     for arch in config.get('arch'):
         results.extend(
             validate_pkgs(
                 config,
                 args,
-                Package.list(config, staff, modules, args.packages),
+                pkgs,
                 arch
             )
         )
@@ -1047,6 +1089,54 @@ def get_packages_from_patch(patch, config, modules, staff):
 
     return updated, removed
 
+def get_packages_to_build(config, staff, modules, args):
+    """
+    Return ordered list of Packages to build. If dependency_tracking is disabled
+    in project configuration or --skip-deps arguments is set, only the list of
+    packages in arguments is selected. Else, this function builds a dependency
+    graph of all packages in the project to determine the list of packages that
+    reverse depends on the list of packages in arguments, recursively.
+    """
+    if not config.get('dependency_tracking') or args.skip_deps:
+        return list(Package.list(config, staff, modules, args.packages))
+
+    # Build dependency graph with all projects packages.
+    graph = PackagesDependencyGraph.from_project(config, staff, modules)
+
+    result = []
+
+    def result_position(new_build_requirements):
+        """
+        Return the first index in result of packages in provided build
+        requirements list.
+        """
+        for build_requirement in new_build_requirements:
+            for index, package in enumerate(result):
+                if build_requirement.package == package:
+                    return index
+        return -1
+
+    for pkg in Package.list(config, staff, modules, args.packages):
+        required_builds = graph.solve(pkg)
+        for index, required_build in enumerate(required_builds):
+            if required_build.package in result:
+                continue
+            # Search the position in result before all its own reverse
+            # dependencies.
+            position = result_position(required_builds[index+1:])
+            logging.info(
+                "Package %s must be built: %s",
+                required_build.package.name,
+                required_build.reasons,
+            )
+            # No position constraint in result, just append the package at the
+            # end. Else insert at the right position.
+            if position == -1:
+                result.append(required_build.package)
+            else:
+                result.insert(position, required_build.package)
+    return result
+
 def create_staging_repo(config):
     """
     Create and return staging temporary repository with a 2-tuple containing

lib/rift/Package.py

@@ -69,6 +69,7 @@ def __init__(self, name, config, staff, modules):
         self.origin = None
         self.ignore_rpms = None
         self.rpmnames = None
+        self.depends = None
 
         # Static paths
         pkgdir = os.path.join(self._config.get('packages_dir'), self.name)
@@ -164,6 +165,13 @@ def load(self, infopath=None):
         else:
             self.ignore_rpms = data.get('ignore_rpms', [])
 
+        depends = data.get('depends')
+        if depends is not None:
+            if isinstance(depends, str):
+                self.depends = [depends]
+            else:
+                self.depends = depends
+
         self.check_info()
 
         if os.path.exists(self.sourcesdir):

lib/rift/RPM.py

@@ -40,6 +40,7 @@
 import shutil
 from subprocess import Popen, PIPE, STDOUT, run, CalledProcessError
 import time
+import itertools
 
 import rpm
 
@@ -213,6 +214,7 @@ def __init__(self, filepath=None, config=None):
         self.filepath = filepath
         self.srpmname = None
         self.pkgnames = []
+        self.provides = []
         self.sources = []
         self.basename = None
         self.version = None
@@ -265,6 +267,12 @@ def load(self):
         except ValueError as exp:
             raise RiftError(f"{self.filepath}: {exp}") from exp
         self.pkgnames = [_header_values(pkg.header['name']) for pkg in spec.packages]
+        # Global unique list of provides. Here dict.fromkeys() is used to remove
+        # duplicates as an alternative to set() for the sake of preserving order.
+        self.provides = list(dict.fromkeys(
+            itertools.chain(
+                *[_header_values(pkg.header['provides'])
+                  for pkg in spec.packages])))
         hdr = spec.sourceHeader
         self.srpmname = hdr.sprintf('%{NAME}-%{VERSION}-%{RELEASE}.src.rpm')
         self.basename = hdr.sprintf('%{NAME}')

lib/rift/Repository.py

@@ -263,7 +263,8 @@ class ProjectArchRepositories:
     """
     Manipulate repositories defined in a project for a particular architecture.
     """
-    def __init__(self, config, arch):
+    def __init__(self, config, arch, extra=None):
+
         self.working = None
         self.arch = arch
         if config.get('working_repo'):
@@ -275,6 +276,8 @@ def __init__(self, config, arch):
             )
             self.working.create()
         self.supplementaries = []
+        if extra:
+            self.supplementaries.append(extra)
         repos = config.get('repos', arch=arch)
         if repos:
             for name, data in repos.items():