Merge pull request #96 from bigbZik/issue-info

Adding issue info to test result in RP

Author: Dzmitry Humianiuk

README.rst

@@ -68,6 +68,8 @@ The following parameters are optional:
 - :code:`rp_hierarchy_class = True` - Enables hierarchy for class, default `True`. Doesn't support 'xdist' plugin.
 - :code:`rp_hierarchy_parametrize = True` - Enables hierarchy parametrized tests, default `False`. Doesn't support 'xdist' plugin.
 - :code:`rp_hierarchy_dirs_level = 0` - Directory starting hierarchy level (from pytest.ini level) (default `0`)
+- :code:`rp_issue_marks = 'xfail' 'issue'` - Pytest marks that could be used to get issue information (id, type, reason)
+- :code:`rp_issue_system_url = https://bugzilla.olympus.f5net.com/show_bug.cgi?id=` - URL to get issue description (issue id from pytest mark will be added to this URL)
 - :code:`rp_verify_ssl = True` - Verify SSL when connecting to the server
 
 If you like to override the above parameters from command line, or from CI environment based on your build, then pass
@@ -158,6 +160,27 @@ To run test with Report Portal you must provide '--reportportal' flag:
     py.test ./tests --reportportal
 
 
+Test issue info
+~~~~~~~~~
+
+Some pytest marks could be used to specify information about skipped or failed test result.
+List of this marks should be specified in pytest ini file (see :code:`rp_issue_marks`).
+
+The following mark fields are used to get information about test issue:
+
+- :code:`issue_id` - issue id (or list) in tracking system. This id will be added as comment to test fail result. If URL is specified in pytest ini file (see :code:`rp_issue_system_url`), id will added as link to tracking system.
+- :code:`reason` - some comment that will be added to test fail description.
+- :code:`issue_type` - short name of RP issue type that should be assigned to failed or skipped test.
+
+Example:
+
+.. code-block:: python
+
+    @pytest.mark.issue(issue_id="111111", reason="Some bug", issue_type="PB")
+    def test():
+        assert False
+
+
 Troubleshooting
 ~~~~~~~~~
 

pytest_reportportal/listener.py

@@ -41,7 +41,7 @@ def pytest_runtest_protocol(self, item):
         self.PyTestService.finish_pytest_item(item, self.result or 'SKIPPED', self.issue or None)
 
     @pytest.hookimpl(hookwrapper=True)
-    def pytest_runtest_makereport(self):
+    def pytest_runtest_makereport(self, item):
         report = (yield).get_result()
 
         if report.longrepr:
@@ -58,17 +58,61 @@ def pytest_runtest_makereport(self):
                 # This happens for example when a fixture fails to run
                 # causing the test to error
                 self.result = 'FAILED'
+                self._add_issue_info(item, report)
             elif report.skipped:
                 # This happens when a testcase is marked "skip".  It will
                 # show in reportportal as not requiring investigation.
                 self.result = 'SKIPPED'
-                self.issue['issue_type'] = 'NOT_ISSUE'
+                self._add_issue_info(item, report)
 
         if report.when == 'call':
             if report.passed:
                 item_result = 'PASSED'
             elif report.skipped:
                 item_result = 'SKIPPED'
+                self._add_issue_info(item, report)
             else:
                 item_result = 'FAILED'
+                self._add_issue_info(item, report)
             self.result = item_result
+
+
+    def _add_issue_info(self, item, report):
+
+        issue_type = None
+        comment = ""
+        url = item.session.config.getini('rp_issue_system_url')
+        issue_marks = item.session.config.getini('rp_issue_marks')
+
+        for mark_name in issue_marks:
+            try:
+                mark = item.get_closest_marker(mark_name)
+            except AttributeError:
+                # pytest < 3.6
+                mark = item.get_marker(mark_name)
+
+            if mark:
+                if "reason" in mark.kwargs:
+                    comment += "\n" if comment else ""
+                    comment += mark.kwargs["reason"]
+                if "issue_id" in mark.kwargs:
+                    issue_ids = mark.kwargs["issue_id"]
+                    if not isinstance(issue_ids, list):
+                        issue_ids = [issue_ids]
+                    comment += "\n" if comment else ""
+                    comment += "Issues:"
+
+                    for issue_id in issue_ids:
+                        comment += " [{}]({}{})".format(issue_id, url, issue_id) if url else " {}".format(issue_id)
+
+                if "issue_type" in mark.kwargs:
+                    issue_type = mark.kwargs["issue_type"]
+
+        if comment:
+            self.issue['comment'] = comment
+
+        if issue_type and self.PyTestService.issue_types and (issue_type in self.PyTestService.issue_types):
+            self.issue['issue_type'] = self.PyTestService.issue_types[issue_type]
+            # self.issue['ignoreAnalyzer'] = True ???
+        elif (report.when == 'setup') and report.skipped:
+            self.issue['issue_type'] = 'NOT_ISSUE'

pytest_reportportal/plugin.py

@@ -281,6 +281,17 @@ def pytest_addoption(parser):
         type='bool',
         help='Enables hierarchy for parametrized tests')
 
+    parser.addini(
+        'rp_issue_marks',
+        type='args',
+        default='',
+        help='Pytest marks to get issue information')
+
+    parser.addini(
+        'rp_issue_system_url',
+        default='',
+        help='URL to get issue description. Issue id from pytest mark will be added to this URL')
+
     parser.addini(
         'rp_verify_ssl',
         default=True,

pytest_reportportal/service.py

@@ -95,6 +95,8 @@ def init_service(self, endpoint, project, uuid, log_batch_size,
                 log_batch_size=log_batch_size,
                 verify_ssl=verify_ssl
             )
+            self.project_settiings = self.RP.rp_client.get_project_settings() if self.RP else None
+            self.issue_types = self.get_issue_types()
         else:
             log.debug('The pytest is already initialized')
         return self.RP
@@ -289,6 +291,16 @@ def _stop_if_necessary(self):
         except queue.Empty:
             pass
 
+    def get_issue_types(self):
+        issue_types = {}
+        if not self.project_settiings:
+            return issue_types
+
+        for item_type in ("AUTOMATION_BUG", "PRODUCT_BUG", "SYSTEM_ISSUE", "NO_DEFECT", "TO_INVESTIGATE"):
+            for item in self.project_settiings["subTypes"][item_type]:
+                issue_types[item["shortName"]] = item["locator"]
+
+        return issue_types
 
     @staticmethod
     def _add_item_hier_parts_dirs(item, hier_flag, dirs_level, report_parts, dirs_parts, rp_name=""):
@@ -408,13 +420,25 @@ def _get_item_tags(self, item):
         # Try to extract names of @pytest.mark.* decorators used for test item
         # and exclude those which present in rp_ignore_tags parameter
         def get_marker_value(item, keyword):
-            marker = item.keywords.get(keyword)
+            try:
+                marker = item.get_closest_marker(keyword)
+            except AttributeError:
+                # pytest < 3.6
+                marker = item.keywords.get(keyword)
+
             return "{}:{}".format(keyword, marker.args[0]) \
                 if marker and marker.args else keyword
 
-        tags = [get_marker_value(item, k) for k in item.keywords
-                if item.get_marker(k) is not None
-                and k not in self.ignored_tags]
+        try:
+            tags = [get_marker_value(item, k) for k in item.keywords
+                    if item.get_closest_marker(k) is not None
+                    and k not in self.ignored_tags]
+        except AttributeError:
+            # pytest < 3.6
+            tags = [get_marker_value(item, k) for k in item.keywords
+                    if item.get_marker(k) is not None
+                    and k not in self.ignored_tags]
+
         tags.extend(item.session.config.getini('rp_tests_tags'))
 
         return tags