Fill in step names in cucumber report.

According to [1], when the outline scenarios are
reported in 'expand' mode, all the outlines will
be expanded to scenarios with their test results.
Also, the step name parameters are filled in.

This is a different mode than without 'expand'
mode[2], where the scenario outlines are reported
so the step names are not filled in and there are
no test results. The "examples" are reported in
this mode.

Since pytest-bdd does report the results of each
step of an outline scenario as separate scenarios,
it looks more like the 'expand' mode, so I've
added the filling in of the parameters in the
step names to make it more compliant.

[1] https://relishapp.com/cucumber/cucumber/docs/formatters/json-output-formatter#scenario-outline-expanded
[2] https://relishapp.com/cucumber/cucumber/docs/formatters/json-output-formatter#scenario-outline

Author: Michiel Holtkamp

pytest_bdd/cucumber_json.py

@@ -6,6 +6,7 @@
 import time
 
 import py
+import re
 import six
 
 from .feature import force_unicode
@@ -27,12 +28,21 @@ def add_options(parser):
         help="create cucumber json style report file at given path.",
     )
 
+    group._addoption(
+        "--cucumberjson-expanded",
+        "--cucumber-json-expanded",
+        action="store_true",
+        dest="expand",
+        default=False,
+        help="expand scenario outlines into scenarios and fill in the step names",
+    )
+
 
 def configure(config):
     cucumber_json_path = config.option.cucumber_json_path
     # prevent opening json log on slave nodes (xdist)
     if cucumber_json_path and not hasattr(config, "slaveinput"):
-        config._bddcucumberjson = LogBDDCucumberJSON(cucumber_json_path)
+        config._bddcucumberjson = LogBDDCucumberJSON(cucumber_json_path, expand=config.option.expand)
         config.pluginmanager.register(config._bddcucumberjson)
 
 
@@ -47,10 +57,11 @@ class LogBDDCucumberJSON(object):
 
     """Logging plugin for cucumber like json output."""
 
-    def __init__(self, logfile):
+    def __init__(self, logfile, expand=False):
         logfile = os.path.expanduser(os.path.expandvars(logfile))
         self.logfile = os.path.normpath(os.path.abspath(logfile))
         self.features = {}
+        self.expand = expand
 
     def append(self, obj):
         self.features[-1].append(obj)
@@ -95,6 +106,23 @@ def _serialize_tags(self, item):
             for tag in item["tags"]
         ]
 
+    def _format_name(self, name, keys, values):
+        for param, value in zip(keys, values):
+            name = name.replace('<{}>'.format(param), value)
+        return name
+
+    def _format_step_name(self, report, step):
+        examples = report.scenario["examples"]
+        if len(examples) == 0:
+            return step["name"]
+
+        # we take the keys from the first "examples", but in each table, the keys should
+        # be the same anyway since all the variables need to be filled in.
+        keys, values = examples[0]["rows"]
+        row_index = examples[0]["row_index"]
+
+        return self._format_name(step["name"], keys, values[row_index])
+
     def pytest_runtest_logreport(self, report):
         try:
             scenario = report.scenario
@@ -112,9 +140,17 @@ def stepmap(step):
                 scenario['failed'] = True
                 error_message = True
 
+            if self.expand:
+                # XXX The format is already 'expanded' (scenario oultines -> scenarios),
+                # but the step names were not filled in with parameters. To be backwards
+                # compatible, do not fill in the step names unless explicitly asked for.
+                step_name = self._format_step_name(report, step)
+            else:
+                step_name = step["name"]
+
             return {
                 "keyword": step['keyword'],
-                "name": step['name'],
+                "name": step_name,
                 "line": step['line_number'],
                 "match": {
                     "location": "",

tests/feature/test_cucumber_json.py

@@ -44,6 +44,16 @@ def test_step_trace(testdir):
         Scenario: Failing
             Given a passing step
             And a failing step
+
+        @scenario-outline-passing-tag
+        Scenario: Passing outline
+            Given type <type> and value <value>
+
+            Examples: example1
+            | type    | value  |
+            | str     | hello  |
+            | int     | 42     |
+            | float   | 1.0    |
     """))
     testdir.makepyfile(textwrap.dedent("""
         import pytest
@@ -61,13 +71,21 @@ def some_other_passing_step():
         def a_failing_step():
             raise Exception('Error')
 
+        @given('type <type> and value <value>')
+        def type_type_and_value_value():
+            return 'pass'
+
         @scenario('test.feature', 'Passing')
         def test_passing():
             pass
 
         @scenario('test.feature', 'Failing')
         def test_failing():
             pass
+
+        @scenario('test.feature', 'Passing outline')
+        def test_passing_outline():
+            pass
     """))
     result, jsonobject = runandparse(testdir)
     assert result.ret
@@ -156,6 +174,84 @@ def test_failing():
                         }
                     ],
                     "type": "scenario"
+                },
+                {
+                    "description": "",
+                    "keyword": "Scenario",
+                    "tags": [
+                        {
+                            "line": 14,
+                            "name": "scenario-outline-passing-tag"
+                        }
+                    ],
+                    "steps": [
+                        {
+                            "line": 16,
+                            "match": {"location": ""},
+                            "result": {
+                                "status": "passed",
+                                "duration": equals_any(int)
+                            },
+                            "keyword": "Given",
+                            "name": "type <type> and value <value>"
+                        }
+                    ],
+                    "line": 15,
+                    "type": "scenario",
+                    "id": "test_passing_outline[str-hello]",
+                    "name": "Passing outline"
+                },
+                {
+                    "description": "",
+                    "keyword": "Scenario",
+                    "tags": [
+                        {
+                            "line": 14,
+                            "name": "scenario-outline-passing-tag"
+                        }
+                    ],
+                    "steps": [
+                        {
+                            "line": 16,
+                            "match": {"location": ""},
+                            "result": {
+                                "status": "passed",
+                                "duration": equals_any(int)
+                            },
+                            "keyword": "Given",
+                            "name": "type <type> and value <value>"
+                        }
+                    ],
+                    "line": 15,
+                    "type": "scenario",
+                    "id": "test_passing_outline[int-42]",
+                    "name": "Passing outline"
+                },
+                {
+                    "description": "",
+                    "keyword": "Scenario",
+                    "tags": [
+                        {
+                            "line": 14,
+                            "name": "scenario-outline-passing-tag"
+                        }
+                    ],
+                    "steps": [
+                        {
+                            "line": 16,
+                            "match": {"location": ""},
+                            "result": {
+                                "status": "passed",
+                                "duration": equals_any(int)
+                            },
+                            "keyword": "Given",
+                            "name": "type <type> and value <value>"
+                        }
+                    ],
+                    "line": 15,
+                    "type": "scenario",
+                    "id": "test_passing_outline[float-1.0]",
+                    "name": "Passing outline"
                 }
             ],
             "id": os.path.join("test_step_trace0", "test.feature"),
@@ -173,3 +269,39 @@ def test_failing():
     ]
 
     assert jsonobject == expected
+
+
+def test_step_trace_with_expand_option(testdir):
+    """Test step trace."""
+    testdir.makefile('.feature', test=textwrap.dedent("""
+    @feature-tag
+    Feature: One scenario outline, expanded to multiple scenarios
+
+        @scenario-outline-passing-tag
+        Scenario: Passing outline
+            Given type <type> and value <value>
+
+            Examples: example1
+            | type    | value  |
+            | str     | hello  |
+            | int     | 42     |
+            | float   | 1.0    |
+    """))
+    testdir.makepyfile(textwrap.dedent("""
+        import pytest
+        from pytest_bdd import given, scenario
+
+        @given('type <type> and value <value>')
+        def type_type_and_value_value():
+            return 'pass'
+
+        @scenario('test.feature', 'Passing outline')
+        def test_passing_outline():
+            pass
+    """))
+    result, jsonobject = runandparse(testdir, '--cucumber-json-expand')
+    assert result.ret == 0
+
+    assert jsonobject[0]["elements"][0]["steps"][0]["name"] == "type str and value hello"
+    assert jsonobject[0]["elements"][1]["steps"][0]["name"] == "type int and value 42"
+    assert jsonobject[0]["elements"][2]["steps"][0]["name"] == "type float and value 1.0"