add test sandboxing tool (#5)

Author: eli-darkly

README.md

@@ -14,6 +14,8 @@ Subpackage `httphelpers` provides convenience wrappers for using `net/http` and
 
 Subpackage `ldservices` is specifically for testing LaunchDarkly SDK client components; it provides HTTP handlers that simulate the service endpoints used by the SDK.
 
+Subpackage `testbox` provides the ability to write tests-of-tests within the Go testing framework.
+
 ## Usage
 
 Import any of these packages in your test code:
@@ -23,10 +25,11 @@ import (
     "github.com/launchdarkly/go-test-helpers"
     "github.com/launchdarkly/go-test-helpers/httphelpers"
     "github.com/launchdarkly/go-test-helpers/ldservices"
+    "github.com/launchdarkly/go-test-helpers/testbox"
 )
 ```
 
-Breaking changes will only be made in a new major version. It is advisable to use a dependency manager to pin these dependencies to a major version branch (`v1`, etc.).
+Breaking changes will only be made in a new major version. It is advisable to use a dependency manager to pin these dependencies to a module version or a major version branch.
 
 ## Contributing
 

testbox/interface.go

@@ -0,0 +1,63 @@
+package testbox
+
+import "github.com/stretchr/testify/require"
+
+// TestingT is a subset of the testing.T interface that allows tests to run in either a real test
+// context, or a mock test scope that is decoupled from the regular testing framework (SandboxTest).
+//
+// This may be useful in a scenario where you have a contract test that verifies the behavior of some
+// unknown interface implementation. In order to verify that the contract test is reliable, you could
+// create implementations that either adhere to the contract or deliberately break it, run the contract
+// test against those, and verify that the test fails if and only if it should fail.
+//
+// The reason this cannot be done with the Go testing package alone is that the standard testing.T type
+// cannot be created from within test code; instances are always passed in from the test framework.
+// Therefore, the contract test would have to be run against the actual *testing.T instance that
+// belongs to the test-of-the-test, and if it failed in a situation when we actually wanted it to
+// to fail, that would be incorrectly reported as a failure of the test-of-the-test.
+//
+// To work around this limitation of the testing package, this package provides a TestingT interface
+// that has two implementations: real and mock. Test logic can then be written against this TestingT,
+// rather than *testing.T.
+//
+//     func RunContractTests(t *testing.T, impl InterfaceUnderTest) {
+//         runContractTests(testbox.RealTest(t))
+//     }
+//
+//     func runContractTests(abstractT testbox.TestingT, impl InterfaceUnderTest) {
+//         assert.True(abstractT, impl.SomeConditionThatShouldBeTrueForTheseInputs(someParams))
+//         abstractT.Run("subtest", func(abstractSubT helpers.TestingT) { ... }
+//     }
+//
+//     func TestContractTestFailureCondition(t *testing.T) {
+//         impl := createDeliberatelyBrokenImplementation()
+//         result := testbox.SandboxTest(func(abstractT testbox.TestingT) {
+//             runContractTests(abstractT, impl) })
+//         assert.True(t, result.Failed // we expect it to fail
+//         assert.Len(t, result.Failures, 1)
+//     }
+//
+// TestingT includes the same subsets of testing.T methods that are defined in the TestingT interfaces
+// of github.com/stretchr/testify/assert and github.com/stretchr/testify/require, so all assertions in
+// those packages will work. It also provides Run, Skip, and SkipNow. It does not support Parallel.
+type TestingT interface {
+	require.TestingT
+	// Run runs a subtest with a new TestingT that applies only to the scope of the subtest. It is
+	// equivalent to the same method in testing.T, except the subtest takes a parameter of type TestingT
+	// instead of *testing.T.
+	//
+	// If the subtest fails, the parent test also fails, but FailNow and SkipNow on the subtest do not
+	// cause the parent test to exit early.
+	Run(name string, action func(TestingT))
+
+	// Failed tells whether whether any assertions in the test have failed so far. It is equivalent to
+	// the same method in testing.T.
+	Failed() bool
+
+	// Skip marks the test as skipped and exits early, logging a message. It is equivalent to the same
+	// method in testing.T.
+	Skip(args ...interface{})
+
+	// SkipNow marks the test as skipped and exits early. It is equivalent to the same method in testing.T.
+	SkipNow()
+}

testbox/package_info.go

@@ -0,0 +1,3 @@
+// Package testbox provides the ability to run test logic that uses a subset of Go's testing.T
+// methods either inside or outside the regular testing environment.
+package testbox

testbox/real.go

@@ -0,0 +1,38 @@
+package testbox
+
+import "testing"
+
+type realTestingT struct {
+	t *testing.T
+}
+
+// RealTest provides an implementation of TestingT for running test logic in a regular test context.
+//
+// See TestingT for details.
+func RealTest(t *testing.T) TestingT {
+	return realTestingT{t}
+}
+
+func (r realTestingT) Errorf(format string, args ...interface{}) {
+	r.t.Errorf(format, args...) // COVERAGE: can't do this in test_sandbox_test; it'll cause a real failure
+}
+
+func (r realTestingT) Run(name string, action func(TestingT)) {
+	r.t.Run(name, func(tt *testing.T) { action(realTestingT{tt}) })
+}
+
+func (r realTestingT) FailNow() {
+	r.t.FailNow() // COVERAGE: can't do this in test_sandbox_test; it'll cause a real failure
+}
+
+func (r realTestingT) Failed() bool {
+	return r.t.Failed()
+}
+
+func (r realTestingT) Skip(args ...interface{}) {
+	r.t.Skip(args...)
+}
+
+func (r realTestingT) SkipNow() {
+	r.t.SkipNow()
+}

testbox/real_test.go

@@ -0,0 +1,81 @@
+package testbox
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRealTest(t *testing.T) {
+	// can't test failure cases, since then *this* test would fail
+
+	t.Run("success", func(t *testing.T) {
+		rt := RealTest(t)
+		assert.True(rt, true)
+
+		assert.False(t, rt.Failed())
+		assert.False(t, t.Failed())
+		assert.False(t, t.Skipped())
+	})
+
+	t.Run("subtest success", func(t *testing.T) {
+		ran := false
+
+		rt := RealTest(t)
+		rt.Run("sub", func(u TestingT) {
+			ran = true
+			assert.True(u, true)
+		})
+
+		assert.True(t, ran)
+
+		assert.False(t, rt.Failed())
+		assert.False(t, t.Failed())
+		assert.False(t, t.Skipped())
+	})
+
+	t.Run("skip", func(t *testing.T) { // this test will always be reported as skipped
+		rt := RealTest(t)
+		rt.Skip()
+
+		assert.True(t, false) // won't execute because we exited early on Skip
+	})
+
+	t.Run("subtest skip", func(t *testing.T) {
+		ran := false
+		continued := false
+
+		rt := RealTest(t)
+		rt.Run("sub", func(u TestingT) {
+			ran = true
+			u.Skip("let's skip this")
+			continued = true
+		})
+
+		assert.True(t, ran)
+		assert.False(t, continued)
+
+		assert.False(t, rt.Failed())
+		assert.False(t, t.Failed())
+		assert.False(t, t.Skipped())
+	})
+
+	t.Run("subtest SkipNow", func(t *testing.T) {
+		ran := false
+		continued := false
+
+		rt := RealTest(t)
+		rt.Run("sub", func(u TestingT) {
+			ran = true
+			u.SkipNow()
+			continued = true
+		})
+
+		assert.True(t, ran)
+		assert.False(t, continued)
+
+		assert.False(t, rt.Failed())
+		assert.False(t, t.Failed())
+		assert.False(t, t.Skipped())
+	})
+}

testbox/sandbox.go

@@ -0,0 +1,150 @@
+package testbox
+
+import (
+	"fmt"
+	"runtime"
+	"strings"
+	"sync"
+)
+
+// SandboxResult describes the aggregate test state produced by calling SandboxTest.
+type SandboxResult struct {
+	// True if any failures were reported during SandboxTest.
+	Failed bool
+
+	// True if the test run with SandboxTest called Skip or SkipNow. This is only true if
+	// the top-level TestingT was skipped, not any subtests.
+	Skipped bool
+
+	// All failures logged during SandboxTest, including subtests.
+	Failures []LogItem
+
+	// All tests that were skipped during SandboxTest, including subtests.
+	Skips []LogItem
+}
+
+type testState struct {
+	failed   bool
+	skipped  bool
+	failures []LogItem
+	skips    []LogItem
+}
+
+// TestPath identifies the level of test that failed or skipped. SandboxResult.Failures and
+// SandboxResult.Skips use this type to distinguish between the top-level test that was run with
+// SandboxTest and subtests that were run within that test with TestingT.Run(). A nil value means the
+// top-level test; a single string element is the name of a subtest run from the top level with
+// TestingT.Run(); nested subtests add an element for each level.
+type TestPath []string
+
+// LogItem describes either a failed assertion or a skip that happened during SandboxTest.
+type LogItem struct {
+	// Path identifies the level of test that failed or was skipped.
+	Path TestPath
+
+	// Message is the failure message or skip message, if any. It is the result of calling fmt.Sprintf
+	// or Sprintln on the arguments that were passed to TestingT.Errorf or TestingT.Skip. If a test
+	// failed without specifying a message, this is "".
+	Message string
+}
+
+type mockTestingT struct {
+	testState
+	path TestPath
+	lock sync.Mutex
+}
+
+// SandboxTest runs a test function against a TestingT instance that applies only to the scope of
+// that test. If the function makes a failed assertion, marks the test as skipped, or forces an early
+// exit with FailNow or SkipNow, this is reflected in the SandboxResult but does not affect the state
+// of the regular test framework (assuming that this code is even executing within a Go test; it does
+// not have to be).
+//
+// The reason this uses a callback function parameter, rather than simply having the SandboxResult
+// implement TestingT itself, is that the function must be run on a separate goroutine so that
+// the sandbox can intercept any early exits from FailNow or SkipNow.
+//
+// SandboxTest does not recover from panics.
+//
+// See TestingT for more details.
+func SandboxTest(action func(TestingT)) SandboxResult {
+	sub := new(mockTestingT)
+	sub.runSafely(action)
+	state := sub.getState()
+	return SandboxResult{
+		Failed:   state.failed,
+		Skipped:  state.skipped,
+		Failures: state.failures,
+		Skips:    state.skips,
+	}
+}
+
+func (m *mockTestingT) Errorf(format string, args ...interface{}) {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	m.failed = true
+	m.failures = append(m.failures, LogItem{Path: m.path, Message: fmt.Sprintf(format, args...)})
+}
+
+func (m *mockTestingT) Run(name string, action func(TestingT)) {
+	sub := &mockTestingT{path: append(m.path, name)}
+	sub.runSafely(action)
+	subState := sub.getState()
+
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	m.failed = m.failed || subState.failed
+	m.failures = append(m.failures, subState.failures...)
+	m.skips = append(m.skips, subState.skips...)
+}
+
+func (m *mockTestingT) FailNow() {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	m.testState.failed = true
+	runtime.Goexit()
+}
+
+func (m *mockTestingT) Failed() bool {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	return m.failed
+}
+
+func (m *mockTestingT) Skip(args ...interface{}) {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	m.skipped = true
+	m.skips = append(m.skips, LogItem{Path: m.path, Message: strings.TrimSuffix(fmt.Sprintln(args...), "\n")})
+	runtime.Goexit()
+}
+
+func (m *mockTestingT) SkipNow() {
+	m.Skip()
+}
+
+func (m *mockTestingT) getState() testState {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	ret := testState{failed: m.failed, skipped: m.skipped}
+	if len(m.failures) > 0 {
+		ret.failures = make([]LogItem, len(m.failures))
+		copy(ret.failures, m.failures)
+	}
+	if len(m.skips) > 0 {
+		ret.skips = make([]LogItem, len(m.skips))
+		copy(ret.skips, m.skips)
+	}
+	return ret
+}
+
+func (m *mockTestingT) runSafely(action func(TestingT)) {
+	exited := make(chan struct{}, 1)
+	go func() {
+		defer func() {
+			close(exited)
+		}()
+		action(m)
+	}()
+	<-exited
+}