add possibility to exted behaviour of fmf by plugins with one for references

_config.yml

@@ -0,0 +1 @@
+theme: jekyll-theme-merlot

examples/plugin_resolver/.fmf/version

@@ -0,0 +1 @@
+1

examples/plugin_resolver/a/b/c/main.fmf

@@ -0,0 +1,6 @@
+/leaf1:
+  x: 4
+  fourth: True
+
+/leaf2:
+  x: 5

examples/plugin_resolver/a/b/main.fmf

@@ -0,0 +1,3 @@
+/c:
+   x: 3
+   third: True

examples/plugin_resolver/a/main.fmf

@@ -0,0 +1,3 @@
+/b/c:
+   x: 2
+   second: True

examples/plugin_resolver/main.fmf

@@ -0,0 +1,19 @@
+/a/b/c:
+   x: 1
+   first: True
+
+/d/dddd:
+   y: 1
+   filtered: asd
+   tags+: ["a", "b"]
+/inherited@dddd:
+  hallo: world
+  tags+: ["c"]
+
+/inherited/tree@a:
+  inside: value
+
+/referencedget@protocols:
+  inside: value
+  tags+: ["ref_inherited"]
+

fmf/base.py

@@ -39,6 +39,9 @@ def construct_yaml_str(self, node):
 
 class Tree(object):
     """ Metadata Tree """
+    _plugin_name_prefix = "plugin"
+    _plugin_option = None
+
     def __init__(self, data, name=None, parent=None):
         """
         Initialize metadata tree from directory path or data dictionary
@@ -76,6 +79,11 @@ def __init__(self, data, name=None, parent=None):
             self.update(data)
         else:
             self.grow(data)
+        # call all plugin functions if any for whole tree after it is constructed
+        for item in dir(self):
+            if item.startswith(self._plugin_name_prefix):
+                log.debug("Calling plugin method for tree: {}".format(item))
+                getattr(self, item)()
         log.debug("New tree '{0}' created.".format(self))
 
     def __unicode__(self):
@@ -213,6 +221,7 @@ def child(self, name, data, source=None):
         # Save source file
         if source is not None:
             self.children[name].sources.append(source)
+        return self.children[name]
 
     def grow(self, path):
         """
@@ -290,6 +299,29 @@ def find(self, name):
                 return node
         return None
 
+    def search(self, name, whole=False):
+        """ Search node with given name based on regexp, basic method (find) uses equality"""
+        for node in self.climb(whole=whole):
+            if re.search(name, node.name):
+                return node
+        return None
+
+    def merge_tree(self, reference):
+        self.merge(parent=reference)
+        for name, child in reference.children.items():
+            self.children[name] = self.deepcopy(name, parent=self, reference=child)
+        self.inherit()
+
+    def deepcopy(self, name, parent=None, reference=None):
+        if not reference:
+            reference = self
+        output = Tree(data=copy.deepcopy(reference.original_data), name=name, parent=parent)
+        for name, child in reference.children.items():
+            new_child = self.deepcopy(name=name, parent=output, reference=child)
+            output.children[name] = new_child
+        return output
+
+
     def prune(self, whole=False, keys=[], names=[], filters=[]):
         """ Filter tree nodes based on given criteria """
         for node in self.climb(whole):

fmf/cli.py

@@ -24,6 +24,7 @@
 import sys
 import shlex
 import argparse
+import importlib
 
 import fmf
 import fmf.utils as utils
@@ -35,6 +36,7 @@
 
 class Parser(object):
     """ Command line options parser """
+    TreeClass = fmf.Tree
 
     def __init__(self, arguments=None, path=None):
         """ Prepare the parser. """
@@ -110,6 +112,9 @@ def options_utils(self):
         group.add_argument(
             "--debug", action="store_true",
             help="Turn on debugging output, do not catch exceptions")
+        group.add_argument(
+            "--plugin", action="store",default="",
+            help="Enable selected plugin")
 
     def command_ls(self):
         """ List names """
@@ -150,10 +155,27 @@ def command_init(self):
     def show(self, brief=False):
         """ Show metadata for each path given """
         output = []
+        if self.options.plugin:
+            plugin = self.options.plugin.split(":", 1)
+            if "." not in plugin[0]:
+                plugin_name = "fmf.plugins." + plugin[0]
+            else:
+                plugin_name = plugin[0]
+            if len(plugin)>1:
+                plugin_option = plugin[1]
+            else:
+                plugin_option = None
+            utils.info("Using plugin: {}".format(plugin_name))
+            try:
+                module = importlib.import_module(plugin_name)
+            except (NameError, ImportError):
+                raise utils.GeneralError("Unable to find python module plugin: {}".format(plugin_name))
+            self.TreeClass = module.Tree
+            self.TreeClass._plugin_option = plugin_option
         for path in self.options.paths or ["."]:
             if self.options.verbose:
                 utils.info("Checking {0} for metadata.".format(path))
-            tree = fmf.Tree(path)
+            tree = self.TreeClass(path)
             for node in tree.prune(
                     self.options.whole, self.options.keys, self.options.names,
                     self.options.filters):
@@ -193,3 +215,6 @@ def main(arguments=None, path=None):
     """ Parse options, do what is requested """
     parser = Parser(arguments, path)
     return parser.output
+
+if __name__ == "__main__":
+    main()

fmf/plugins/reference.py

@@ -0,0 +1,89 @@
+import logging
+import re
+
+"""
+Module handling FMF stored metadata for classes and resolve references by special tag prefix "@"
+"""
+
+from fmf import Tree as TreeOrigin
+
+logger = logging.getLogger(__name__)
+
+
+class Tree(TreeOrigin):
+    """
+    FMF Extension. Allows to use references via @ to another items -> usefull for rulesets
+    """
+
+
+    def __remove_append_items(self, whole=False):
+        """
+        internal method, delete all append items (ends with +)
+        :param whole: pass thru 'whole' param to climb
+        :return: None
+        """
+        for node in self.climb(whole=whole):
+            for key in sorted(node.data.keys()):
+                if key.endswith('+'):
+                    del node.data[key]
+
+    def pluginReferenceResolver(self, datatrees=None, whole=False):
+        """
+        Reference name resolver (eg. /a/b/c/d@.x.y or /a/b/c/@y will search data in .x.y or y nodes)
+        there are used regular expressions (re.search) to match names
+        it uses simple references schema, do not use references to another references,
+        avoid usind / in reference because actual solution creates also these tree items.
+        datatree contains for example data like (original check data)
+        /dockerfile/maintainer_check:
+          class: SomeClass
+          tags: [dockerfile]
+        and reference could be like (ruleset)
+        /default/check1@maintainer_check:
+           tags+: [required]
+        will produce output (output ruleset tree):
+        /default/check1@maintainer_check:
+          class: SomeClass
+          tags: [dockerfile, required]
+        :param whole: 'whole' param of original climb method, in colin this is not used anyhow now
+                      iterate over all items not only leaves if True
+        :param datatrees: list of original trees with testcases to contain parent nodes
+        :return: None
+        """
+        if datatrees is None:
+            if self._plugin_option:
+                datatrees = [TreeOrigin(path) for path in self._plugin_option.split(":")]
+            else:
+                datatrees = [self]
+        if not isinstance(datatrees, list):
+            raise ValueError("datatrees argument has to be list of fmf trees")
+        reference_nodes = self.prune(whole=whole, names=["@"])
+        for node in reference_nodes:
+            ref_item_name = node.name.rsplit("@", 1)[1]
+            #if "/" in ref_item_name:
+            #    logger.debug("SKIP inter merging: %s", ref_item_name)
+            #    continue
+            node.data = node.original_data
+            # match item what does not contain @ before name, otherwise it
+            # match same item
+            reference_node = None
+            for datatree in datatrees:
+                reference_node = datatree.search("[^@]%s" % ref_item_name, whole=True)
+                if reference_node is not None:
+                    break
+            if not reference_node:
+                raise ValueError("Unable to find reference for node: %s via name search: %s" %
+                                 (node.name, ref_item_name))
+            if not reference_node.children:
+                logger.debug("MERGING: %s from %s (root %s)",
+                             node.name,
+                             reference_node.name,
+                             reference_node.root)
+                node.merge(parent=reference_node)
+            else:
+                logger.debug("MERGING TREE: %s from %s (root %s)",
+                             node.name,
+                             reference_node.name,
+                             reference_node.root)
+                node.merge_tree(reference=reference_node)
+
+        self.__remove_append_items(whole=whole)