Add option to run scenarios as *testing.T subtests (#419)

Author: vearutop

README.md

@@ -377,7 +377,54 @@ See implementation examples:
 
 ### Running Godog with go test
 
-You may integrate running **godog** in your **go test** command. You can run it using go [TestMain](https://golang.org/pkg/testing/#hdr-Main) func available since **go 1.4**. In this case it is not necessary to have **godog** command installed. See the following examples.
+You may integrate running **godog** in your **go test** command. 
+
+#### Subtests of *testing.T
+
+You can run test suite using go [Subtests](https://pkg.go.dev/testing#hdr-Subtests_and_Sub_benchmarks).
+In this case it is not necessary to have **godog** command installed. See the following example.
+
+```go
+package main_test
+
+import (
+	"testing"
+
+	"github.com/cucumber/godog"
+)
+
+func TestFeatures(t *testing.T) {
+  suite := godog.TestSuite{
+    ScenarioInitializer: func(s *godog.ScenarioContext) {
+      // Add step definitions here.
+    },
+    Options: &godog.Options{
+      Format:   "pretty",
+      Paths:    []string{"features"},
+      TestingT: t, // Testing instance that will run subtests.
+    },
+  }
+
+  if suite.Run() != 0 {
+    t.Fatal("non-zero status returned, failed to run feature tests")
+  }
+}
+```
+
+Then you can run suite.
+```
+go test -test.v -test.run ^TestFeatures$
+```
+
+Or a particular scenario.
+```
+go test -test.v -test.run ^TestFeatures$/^my_scenario$
+```
+
+#### TestMain
+
+You can run test suite using go [TestMain](https://golang.org/pkg/testing/#hdr-Main) func available since **go 1.4**. 
+In this case it is not necessary to have **godog** command installed. See the following examples.
 
 The following example binds **godog** flags with specified prefix `godog` in order to prevent flag collisions.
 

example_subtests_test.go

@@ -0,0 +1,45 @@
+package godog_test
+
+import (
+	"testing"
+
+	"github.com/cucumber/godog"
+)
+
+func ExampleTestSuite_Run_subtests() {
+	var t *testing.T // Comes from your test function, e.g. func TestFeatures(t *testing.T).
+
+	suite := godog.TestSuite{
+		ScenarioInitializer: func(s *godog.ScenarioContext) {
+			// Add step definitions here.
+		},
+		Options: &godog.Options{
+			Format:   "pretty",
+			Paths:    []string{"features"},
+			TestingT: t, // Testing instance that will run subtests.
+		},
+	}
+
+	if suite.Run() != 0 {
+		t.Fatal("non-zero status returned, failed to run feature tests")
+	}
+}
+
+func TestFeatures(t *testing.T) {
+	suite := godog.TestSuite{
+		ScenarioInitializer: func(s *godog.ScenarioContext) {
+			godog.InitializeScenario(s)
+
+			// Add step definitions here.
+		},
+		Options: &godog.Options{
+			Format:   "pretty",
+			Paths:    []string{"features"},
+			TestingT: t, // Testing instance that will run subtests.
+		},
+	}
+
+	if suite.Run() != 0 {
+		t.Fatal("non-zero status returned, failed to run feature tests")
+	}
+}

internal/flags/options.go

@@ -3,6 +3,7 @@ package flags
 import (
 	"context"
 	"io"
+	"testing"
 )
 
 // Options are suite run options
@@ -60,4 +61,7 @@ type Options struct {
 
 	// DefaultContext is used as initial context instead of context.Background().
 	DefaultContext context.Context
+
+	// TestingT runs scenarios as subtests.
+	TestingT *testing.T
 }

release-notes/v0.12.0.md

@@ -112,6 +112,13 @@ You can now use `string` instead of `*godog.DocString` in declaration.
 
 With the introduction of go1.16, go1.16 is now officially supported.
 
+### Running scenarios as subtests of *testing.T
+
+You can now assign an instance of `*testing.T` to `godog.Options.TestingT` so that scenarios will be invoked with 
+`t.Run` allowing granular control with standard Go tools. 
+
+[More info](https://github.com/cucumber/godog#running-godog-with-go-test).
+
 Non Backward Compatible Changes
 -------------------------------
 

run.go

@@ -12,6 +12,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"testing"
 
 	"github.com/cucumber/messages-go/v16"
 
@@ -38,6 +39,7 @@ type runner struct {
 	stopOnFailure, strict bool
 
 	defaultContext context.Context
+	testingT       *testing.T
 
 	features []*models.Feature
 
@@ -106,6 +108,7 @@ func (r *runner) concurrent(rate int) (failed bool) {
 					strict:         r.strict,
 					storage:        r.storage,
 					defaultContext: r.defaultContext,
+					testingT:       r.testingT,
 				}
 
 				if r.scenarioInitializer != nil {
@@ -236,6 +239,7 @@ func runWithOptions(suiteName string, runner runner, opt Options) int {
 	runner.stopOnFailure = opt.StopOnFailure
 	runner.strict = opt.Strict
 	runner.defaultContext = opt.DefaultContext
+	runner.testingT = opt.TestingT
 
 	// store chosen seed in environment, so it could be seen in formatter summary report
 	os.Setenv("GODOG_SEED", strconv.FormatInt(runner.randomSeed, 10))

run_test.go

@@ -432,6 +432,39 @@ func Test_AllFeaturesRun(t *testing.T) {
 	assert.Equal(t, expected, actualOutput)
 }
 
+func Test_AllFeaturesRunAsSubtests(t *testing.T) {
+	const concurrency = 100
+	const noRandomFlag = 0
+	const format = "progress"
+
+	const expected = `...................................................................... 70
+...................................................................... 140
+...................................................................... 210
+...................................................................... 280
+........................................                               320
+
+
+83 scenarios (83 passed)
+320 steps (320 passed)
+0s
+`
+
+	actualStatus, actualOutput := testRunWithOptions(
+		t,
+		Options{
+			Format:      format,
+			Concurrency: concurrency,
+			Paths:       []string{"features"},
+			Randomize:   noRandomFlag,
+			TestingT:    t,
+		},
+		InitializeScenario,
+	)
+
+	assert.Equal(t, exitSuccess, actualStatus)
+	assert.Equal(t, expected, actualOutput)
+}
+
 func Test_FormatterConcurrencyRun(t *testing.T) {
 	formatters := []string{
 		"progress",
@@ -484,17 +517,30 @@ func testRun(
 	randomSeed int64,
 	featurePaths []string,
 ) (int, string) {
-	output := new(bytes.Buffer)
+	t.Helper()
 
 	opts := Options{
 		Format:      format,
-		NoColors:    true,
 		Paths:       featurePaths,
 		Concurrency: concurrency,
 		Randomize:   randomSeed,
-		Output:      output,
 	}
 
+	return testRunWithOptions(t, opts, scenarioInitializer)
+}
+
+func testRunWithOptions(
+	t *testing.T,
+	opts Options,
+	scenarioInitializer func(*ScenarioContext),
+) (int, string) {
+	t.Helper()
+
+	output := new(bytes.Buffer)
+
+	opts.Output = output
+	opts.NoColors = true
+
 	status := TestSuite{
 		Name:                "succeed",
 		ScenarioInitializer: scenarioInitializer,

suite.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"reflect"
 	"strings"
+	"testing"
 
 	"github.com/cucumber/messages-go/v16"
 
@@ -54,6 +55,7 @@ type suite struct {
 	strict        bool
 
 	defaultContext context.Context
+	testingT       *testing.T
 
 	// suite event handlers
 	beforeScenarioHandlers []BeforeScenarioHook
@@ -442,7 +444,7 @@ func (s *suite) runPickle(pickle *messages.Pickle) (err error) {
 		return ErrUndefined
 	}
 
-	// Before scenario hooks are aclled in context of first evaluated step
+	// Before scenario hooks are called in context of first evaluated step
 	// so that error from handler can be added to step.
 
 	pr := models.PickleResult{PickleID: pickle.Id, StartedAt: utils.TimeNowFunc()}
@@ -451,7 +453,17 @@ func (s *suite) runPickle(pickle *messages.Pickle) (err error) {
 	s.fmt.Pickle(pickle)
 
 	// scenario
-	ctx, err = s.runSteps(ctx, pickle, pickle.Steps)
+	if s.testingT != nil {
+		// Running scenario as a subtest.
+		s.testingT.Run(pickle.Name, func(t *testing.T) {
+			ctx, err = s.runSteps(ctx, pickle, pickle.Steps)
+			if err != nil {
+				t.Error(err)
+			}
+		})
+	} else {
+		ctx, err = s.runSteps(ctx, pickle, pickle.Steps)
+	}
 
 	// After scenario handlers are called in context of last evaluated step
 	// so that error from handler can be added to step.