Merge pull request #142 from commander-cli/execute-in-alphabetical-order

Preserve execution order alphabetically by test title

Author: SimonBaeumer

CHANGELOG.md

@@ -1,5 +1,6 @@
 # v2.3.0
 
+ - Preserve test execution order alphabetical order
  - Add property `skip`, adds the ability to skip test cases 
 
 # v2.2.0

README.md

@@ -1,6 +1,6 @@
 [![Build Status](https://travis-ci.org/commander-cli/commander.svg?branch=master)](https://travis-ci.org/commander-cli/commander)
 [![GoDoc](https://godoc.org/github.com/commander-cli/commander?status.svg)](https://godoc.org/github.com/commander-cli/commander)
-[![Go Report Card](https://goreportcard.com/badge/github.com/commander-cli/commander)](https://goreportcard.com/report/github.com/SimonBaeumer/commander)
+[![Go Report Card](https://goreportcard.com/badge/github.com/commander-cli/commander)](https://goreportcard.com/report/github.com/commander-cli/commander)
 [![Maintainability](https://api.codeclimate.com/v1/badges/cc848165784e0f809a51/maintainability)](https://codeclimate.com/github/commander-cli/commander/maintainability)
 [![Test Coverage](https://api.codeclimate.com/v1/badges/cc848165784e0f809a51/test_coverage)](https://codeclimate.com/github/commander-cli/commander/test_coverage)
 [![Github All Releases](https://img.shields.io/github/downloads/commander-cli/commander/total.svg)](https://github.com/commander-cli/commander/releases)

cmd/commander/commander.go

@@ -46,7 +46,22 @@ func createCliApp() *cli.App {
 func createTestCommand() cli.Command {
 	return cli.Command{
 		Name:      "test",
-		Usage:     "Execute the test suite, by default it will use the commander.yaml from your current directory",
+		Usage:     `Execute cli app tests
+
+By default it will use the commander.yaml from your current directory.
+Tests are always executed in alphabetical order.
+
+Examples:
+
+Filtering tests:
+test commander.yaml --filter="my test"
+
+Multiple filters:
+test commander.yaml --filter=filter1 --filter=filter2
+
+Regex filters:
+test commander.yaml --filter="^filter1$"
+`,
 		ArgsUsage: "[file] [--filter]",
 		Flags: []cli.Flag{
 			cli.BoolFlag{
@@ -65,14 +80,7 @@ func createTestCommand() cli.Command {
 			},
 			cli.StringSliceFlag{
 				Name: "filter",
-				Usage: `Filter tests by a given regex pattern. Tests are filtered by its title.
-
-Example:
-test commander.yaml --filter="my test"
-
-Apply multiple filters:
-test commander.yaml --filter=filter1 --filter=filter2
-`,
+				Usage: `Filter tests by a given regex pattern. Tests are filtered by its title.`,
 			},
 		},
 		Action: func(c *cli.Context) error {

commander_unix.yaml

@@ -111,4 +111,15 @@ tests:
         - ✓ [local] should be ignored
       not-contains:
         - executed at the beginning is ignored
-    exit-code: 0
+    exit-code: 0
+
+  it should be executed in alphabetical order:
+    command: ./commander test integration/unix/alphabetically_order.yaml
+    stdout:
+      contains:
+      - |-
+        ✓ [local] ---
+        ✓ [local] 123
+        ✓ [local] a
+        ✓ [local] b
+    exit-code: 0

examples/ordering.yaml

@@ -0,0 +1,18 @@
+# Tests are always executed in alphabetical order
+
+tests:
+  001 - test:
+    command: exit 0
+    exit-code: 0
+
+  002 - test:
+    command: exit 0
+    exit-code: 0
+
+  003 - test:
+    command: exit 0
+    exit-code: 0
+
+  004 - test:
+    command: exit 0
+    exit-code: 0

integration/unix/alphabetically_order.yaml

@@ -0,0 +1,13 @@
+tests:
+  a:
+    command: echo test
+    exit-code: 0
+  b:
+    command: echo test
+    exit-code: 0
+  123:
+    command: echo test
+    exit-code: 0
+  "---":
+    command: echo test
+    exit-code: 0

pkg/runtime/runtime.go

@@ -2,6 +2,7 @@ package runtime
 
 import (
 	"log"
+	"sort"
 	"time"
 )
 
@@ -131,6 +132,11 @@ type Result struct {
 
 // Start starts the given test suite and executes all tests
 func (r *Runtime) Start(tests []TestCase) Result {
+	// Sort tests alphabetically to preserve a reproducible execution order
+	sort.SliceStable(tests, func(i, j int) bool {
+		return tests[i].Title < tests[j].Title
+	})
+
 	result := Result{}
 	testCh := r.Runner.Run(tests)
 	start := time.Now()

pkg/runtime/runtime_test.go

@@ -47,6 +47,27 @@ func TestRuntime_WithRetries(t *testing.T) {
 	assert.Equal(t, 1, counter)
 }
 
+func Test_AlphabeticalOrder(t *testing.T) {
+	tests := []TestCase{
+		{Title: "bbb", Command: CommandUnderTest{Cmd: "exit 0;"}},
+		{Title: "aaa"},
+		{Title: "111"},
+		{Title: "_"},
+	}
+
+	got := []string{}
+	runtime := NewRuntime(&EventHandler{TestFinished: func(r TestResult) {
+		got = append(got, r.TestCase.Title)
+	}})
+
+	runtime.Start(tests)
+
+	assert.Equal(t, "111", got[0])
+	assert.Equal(t, "_", got[1])
+	assert.Equal(t, "aaa", got[2])
+	assert.Equal(t, "bbb", got[3])
+}
+
 func Test_RuntimeWithRetriesAndInterval(t *testing.T) {
 	s := getExampleTestCases()
 	s[0].Command.Retries = 3