add there important improvement of file nadling

Author: jscotkahat

fmf/base.py

@@ -19,7 +19,7 @@
 import fmf.utils as utils
 
 from io import open
-from fmf.utils import log, dict_to_yaml
+from fmf.utils import log, dict_to_yaml, FileSorting
 from fmf.constants import SUFFIX, IGNORED_DIRECTORIES, MAIN
 from fmf.plugin_loader import get_suffixes, get_plugin_for_file
 from pprint import pformat as pretty
@@ -452,15 +452,11 @@ def grow(self, path):
         except StopIteration:
             log.debug("Skipping '{0}' (not accessible).".format(path))
             return
-        # Investigate main.fmf as the first file (for correct inheritance)
-        filenames = sorted(
-            [filename for filename in filenames if any(filter(filename.endswith, get_suffixes()))])
-        try:
-            filenames.insert(0, filenames.pop(filenames.index(MAIN)))
-        except ValueError:
-            pass
+
+        filenames_sorted = sorted(
+            [FileSorting(filename) for filename in filenames if any(filter(filename.endswith, get_suffixes()))])
         # Check every metadata file and load data (ignore hidden)
-        for filename in filenames:
+        for filename in [filename.value for filename in filenames_sorted]:
             if filename.startswith("."):
                 continue
             fullpath = os.path.abspath(os.path.join(dirpath, filename))
@@ -474,16 +470,13 @@ def grow(self, path):
                     raise (utils.FileError("Failed to parse '{0}'.\n{1}".format(
                         fullpath, error)))
             else:
+                data = None
                 plugin = get_plugin_for_file(fullpath)
                 log.debug("Used plugin {}".format(plugin))
-                if plugin.file_patters and any(filter(lambda x: re.search(x, fullpath), plugin.file_patters)):
-                    log.info("Matched patters {}".format(plugin.file_patters))
-                    data = plugin().get_data(fullpath)
-                    # ignore results of output if there is None
-                    if data is None:
-                        continue
-                else:
-                    log.debug("Does not match patters {}".format(plugin.file_patters))
+                if plugin:
+                    data = plugin().read(fullpath)
+                # ignore results of output if there is None
+                if data is None:
                     continue
             log.data(pretty(data))
             # Handle main.fmf as data for self
@@ -734,7 +727,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):
             with open(source, "w", encoding='utf-8') as file:
                 file.write(dict_to_yaml(full_data))
         else:
-            plugin().put_data(source, hierarchy, node_data, append, modified, deleted)
+            plugin().write(source, hierarchy, node_data, append, modified, deleted)
 
     def __getitem__(self, key):
         """

fmf/plugin_loader.py

@@ -1,9 +1,13 @@
 from functools import lru_cache
 import inspect
+import copy
+import yaml
+
 from fmf.constants import PLUGIN_ENV, SUFFIX
 from fmf.utils import log
 import importlib
 import os
+import re
 
 
 class Plugin:
@@ -14,19 +18,40 @@ class Plugin:
     extensions = list()
     file_patters = list()
 
-    def get_data(self, filename):
+    def read(self, filename):
         """
         return python dictionary representation of metadata inside file (FMF structure)
         """
         raise NotImplementedError("Define own impementation")
 
-    def put_data(
+    @staticmethod
+    def __define_undefined(hierarchy, modified, append):
+        output = dict()
+        current = output
+        for key in hierarchy:
+            if key not in current or current[key] is None:
+                current[key] = dict()
+            current = current[key]
+        for k,v in modified.items():
+            current[k] = v
+        for k,v in append.items():
+            current[k] = v
+        return output
+
+    def write(
             self, filename, hierarchy, data, append_dict, modified_dict,
             deleted_items):
         """
-        Write data in dictionary representation back to file
+        Write data in dictionary representation back to file, if not defined, create new fmf file with same name.
+        When created, nodes will not use plugin method anyway
         """
-        raise NotImplementedError("Define own impementation")
+        path = os.path.dirname(filename)
+        basename = os.path.basename(filename)
+        current_extension = list(filter(lambda x: basename.endswith(x), self.extensions))[0]
+        without_extension = basename[0:-len(list(current_extension))]
+        fmf_file = os.path.join(path, without_extension + ".fmf")
+        with open(fmf_file, "w") as fd:
+            yaml.safe_dump(self.__define_undefined(hierarchy, modified_dict, append_dict), stream=fd)
 
 
 @lru_cache(maxsize=1)
@@ -61,6 +86,6 @@ def get_suffixes():
 def get_plugin_for_file(filename):
     extension = "." + filename.rsplit(".", 1)[1]
     for item in enabled_plugins():
-        if extension in item.extensions:
+        if extension in item.extensions and any(filter(lambda x: re.search(x, filename), item.file_patters)):
             log.debug("File {} parsed by by plugin {}".format(filename, item))
             return item

fmf/plugins/bash.py

@@ -21,6 +21,6 @@ def update_data(filename, pattern="^#.*:FMF:"):
                     out[identifier] = value
         return out
 
-    def get_data(self, file_name):
+    def read(self, file_name):
         log.info("Processing Item: {}".format(file_name))
         return self.update_data(file_name)

fmf/plugins/pytest/plugin.py

@@ -234,7 +234,7 @@ def update_data(store_dict, func, config):
         define_undefined(store_dict, keys, config, filename, cls, test)
         return store_dict
 
-    def get_data(self, file_name):
+    def read(self, file_name):
         def call_collect(queue, file_name):
             """
             have to call in separate process, to avoid problems with pytest multiple collectitons
@@ -268,7 +268,7 @@ def import_test_module(filename):
         loader.exec_module(module)
         return module
 
-    def put_data(
+    def write(
             self, filename, hierarchy, data, append_dict, modified_dict,
             deleted_items):
         module = self.import_test_module(filename)

fmf/utils.py

@@ -25,6 +25,7 @@
     from io import StringIO
 
 import fmf.base
+from fmf.constants import MAIN
 
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 #  Constants
@@ -856,3 +857,39 @@ def representer(self, data): return self.represent_mapping(
         return output.getvalue().decode('utf-8')
     except AttributeError:
         return output.getvalue()
+
+
+class FileSorting:
+    def __init__(self, value):
+        self._value = value
+
+    @property
+    def value(self):
+        return self._value
+
+    @property
+    def splitted(self):
+        splitted = self._value.rsplit(".", 1)
+        basename = splitted[0]
+        suffix = splitted[1] if len(splitted) > 1 else ''
+        return basename, suffix
+
+    @property
+    def basename(self):
+        return self.splitted[0]
+
+    @property
+    def suffix(self):
+        return self.splitted[1]
+
+    def __lt__(self, other):
+        # Investigate main.fmf as the first file (for correct inheritance)
+        if self.value == MAIN:
+            return True
+        # if there are same filenames and other endswith fmf, it has to be last one
+        elif self.basename == other.basename and other.suffix == "fmf":
+            return True
+        elif self.basename == other.basename and self.suffix == "fmf":
+            return False
+        else:
+            return self.value < other.value

tests/tests_plugin/test_basic.py

@@ -14,6 +14,7 @@ def test_fail():
     assert False
 
 
+@TMT.summary("Some summary")
 @pytest.mark.skip
 def test_skip():
     assert True

tests/tests_plugin/test_rewrite.fmf

@@ -0,0 +1,4 @@
+/test_pass:
+  added_fmf_file: added
+  summary: Rewrite
+  tag+: [tier2]

tests/tests_plugin/test_rewrite.py

@@ -0,0 +1,7 @@
+from fmf.plugins.pytest import TMT
+
+
+@TMT.tag("Tier1")
+@TMT.summary("Rewritten")
+def test_pass():
+    assert True

tests/unit/test_plugin.py

@@ -35,8 +35,13 @@ def setUp(self):
         os.environ[PLUGIN_ENV] = "fmf.plugins.pytest"
         self.plugin_tree = Tree(self.tempdir)
 
+    def tearDown(self):
+        enabled_plugins.cache_clear()
+        #rmtree(self.tempdir)
+
     def test_basic(self):
         item = self.plugin_tree.find("/test_basic/test_skip")
+
         self.assertFalse(item.data.get("enabled"))
         self.assertIn("Jan", item.data["author"])
         self.assertIn(
@@ -63,6 +68,28 @@ def test_modify(self):
         self.assertIn("tier2", item.data["tag"])
         self.assertNotIn("tier", item.data)
 
+    def test_rewrite(self):
+        item = self.plugin_tree.find("/test_rewrite/test_pass")
+        self.assertNotIn("duration", item.data)
+        self.assertIn("Tier1", item.data["tag"])
+        self.assertIn("tier2", item.data["tag"])
+        self.assertEqual("added", item.data["added_fmf_file"])
+        self.assertEqual("Rewrite", item.data["summary"])
+
+    def test_rewrite_modify(self):
+        self.test_rewrite()
+        item = self.plugin_tree.find("/test_rewrite/test_pass")
+        with item as data:
+            data["tag+"] += ["tier3"]
+            data["extra_id"] = 1234
+
+        self.plugin_tree = Tree(self.tempdir)
+        item = self.plugin_tree.find("/test_rewrite/test_pass")
+        self.test_rewrite()
+        self.assertEqual(1234, item.data["extra_id"])
+        self.assertIn("tier3", item.data["tag"])
+
+
 
 class Bash(Base):
     """ Verify reading data done via plugins """
@@ -73,8 +100,24 @@ def setUp(self):
             os.path.join(PLUGIN_PATH, "bash.py"))
         self.plugin_tree = Tree(self.tempdir)
 
-    def test_pytest_plugin(self):
+    def test_read(self):
         item = self.plugin_tree.find("/runtest")
         self.assertIn("tier1", item.data["tag"])
         self.assertIn("./runtest.sh", item.data["test"])
         self.assertIn("Jan", item.data["author"])
+
+    def test_modify(self):
+        self.assertNotIn("runtest.fmf", os.listdir(self.tempdir))
+        item = self.plugin_tree.find("/runtest")
+        self.assertIn("tier1", item.data["tag"])
+        self.assertIn("./runtest.sh", item.data["test"])
+        with item as data:
+            data["tier"] = 0
+            data["duration"] = "10m"
+        self.plugin_tree = Tree(self.tempdir)
+        item = self.plugin_tree.find("/runtest")
+        self.assertIn("runtest.fmf", os.listdir(self.tempdir))
+        self.assertEqual("10m", item.data["duration"])
+        self.assertEqual(0, item.data["tier"])
+        self.assertIn("tier1", item.data["tag"])
+        self.assertIn("./runtest.sh", item.data["test"])