main: match go test output

This commit makes the output of `tinygo test` similar to that of `go
test`. It changes the following things in the process:

  * Running multiple tests in a single command is now possible. They
    aren't paralellized yet.
  * Packages with no test files won't crash TinyGo, instead it logs it
    in the same way the Go toolchain does.

Author: aykevl

Makefile

@@ -182,25 +182,28 @@ tinygo:
 test: wasi-libc
 	CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test -v -buildmode exe -tags byollvm ./builder ./cgo ./compileopts ./compiler ./interp ./transform .
 
+TEST_PACKAGES = \
+	container/heap \
+	container/list \
+	container/ring \
+	crypto/des \
+	encoding \
+	encoding/ascii85 \
+	encoding/base32 \
+	encoding/hex \
+	hash/adler32 \
+	hash/fnv \
+	hash/crc64 \
+	math \
+	math/cmplx \
+	text/scanner \
+	unicode/utf8 \
+
 # Test known-working standard library packages.
-# TODO: do this in one command, parallelize, and only show failing tests (no
-# implied -v flag).
+# TODO: parallelize, and only show failing tests (no implied -v flag).
 .PHONY: tinygo-test
 tinygo-test:
-	$(TINYGO) test container/heap
-	$(TINYGO) test container/list
-	$(TINYGO) test container/ring
-	$(TINYGO) test crypto/des
-	$(TINYGO) test encoding/ascii85
-	$(TINYGO) test encoding/base32
-	$(TINYGO) test encoding/hex
-	$(TINYGO) test hash/adler32
-	$(TINYGO) test hash/fnv
-	$(TINYGO) test hash/crc64
-	$(TINYGO) test math
-	$(TINYGO) test math/cmplx
-	$(TINYGO) test text/scanner
-	$(TINYGO) test unicode/utf8
+	$(TINYGO) test $(TEST_PACKAGES)
 
 .PHONY: smoketest
 smoketest:

builder/build.go

@@ -39,6 +39,11 @@ type BuildResult struct {
 	// The directory of the main package. This is useful for testing as the test
 	// binary must be run in the directory of the tested package.
 	MainDir string
+
+	// ImportPath is the import path of the main package. This is useful for
+	// correctly printing test results: the import path isn't always the same as
+	// the path listed on the command line.
+	ImportPath string
 }
 
 // packageAction is the struct that is serialized to JSON and hashed, to work as
@@ -638,8 +643,9 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 		return fmt.Errorf("unknown output binary format: %s", outputBinaryFormat)
 	}
 	return action(BuildResult{
-		Binary:  tmppath,
-		MainDir: lprogram.MainPkg().Dir,
+		Binary:     tmppath,
+		MainDir:    lprogram.MainPkg().Dir,
+		ImportPath: lprogram.MainPkg().ImportPath,
 	})
 }
 

loader/errors.go

@@ -23,3 +23,13 @@ type Error struct {
 func (e Error) Error() string {
 	return e.Err.Error()
 }
+
+// Error returned when loading a *Program for a test binary but no test files
+// are present.
+type NoTestFilesError struct {
+	ImportPath string
+}
+
+func (e NoTestFilesError) Error() string {
+	return "no test files"
+}

loader/loader.go

@@ -210,6 +210,12 @@ func Load(config *compileopts.Config, inputPkgs []string, clangHeaders string, t
 		p.Packages[pkg.ImportPath] = pkg
 	}
 
+	if config.TestConfig.CompileTestBinary && !strings.HasSuffix(p.sorted[len(p.sorted)-1].ImportPath, ".test") {
+		// Trying to compile a test binary but there are no test files in this
+		// package.
+		return p, NoTestFilesError{p.sorted[len(p.sorted)-1].ImportPath}
+	}
+
 	return p, nil
 }
 

main.go

@@ -136,15 +136,17 @@ func Build(pkgName, outpath string, options *compileopts.Options) error {
 	})
 }
 
-// Test runs the tests in the given package.
-func Test(pkgName string, options *compileopts.Options, testCompileOnly bool, outpath string) error {
+// Test runs the tests in the given package. Returns whether the test passed and
+// possibly an error if the test failed to run.
+func Test(pkgName string, options *compileopts.Options, testCompileOnly bool, outpath string) (bool, error) {
 	options.TestConfig.CompileTestBinary = true
 	config, err := builder.NewConfig(options)
 	if err != nil {
-		return err
+		return false, err
 	}
 
-	return builder.Build(pkgName, outpath, config, func(result builder.BuildResult) error {
+	var passed bool
+	err = builder.Build(pkgName, outpath, config, func(result builder.BuildResult) error {
 		if testCompileOnly || outpath != "" {
 			// Write test binary to the specified file name.
 			if outpath == "" {
@@ -158,48 +160,78 @@ func Test(pkgName string, options *compileopts.Options, testCompileOnly bool, ou
 			// Do not run the test.
 			return nil
 		}
-		if len(config.Target.Emulator) == 0 {
-			// Run directly.
-			cmd := executeCommand(config.Options, result.Binary)
-			cmd.Stdout = os.Stdout
-			cmd.Stderr = os.Stderr
-			cmd.Dir = result.MainDir
-			err := cmd.Run()
-			if err != nil {
-				// Propagate the exit code
-				if err, ok := err.(*exec.ExitError); ok {
-					os.Exit(err.ExitCode())
-				}
-				return &commandError{"failed to run compiled binary", result.Binary, err}
-			}
-			return nil
+
+		// Run the test.
+		start := time.Now()
+		var err error
+		passed, err = runPackageTest(config, result)
+		if err != nil {
+			return err
+		}
+		duration := time.Since(start)
+
+		// Print the result.
+		importPath := strings.TrimSuffix(result.ImportPath, ".test")
+		if passed {
+			fmt.Printf("ok  \t%s\t%.3fs\n", importPath, duration.Seconds())
 		} else {
-			// Run in an emulator.
-			args := append(config.Target.Emulator[1:], result.Binary)
-			cmd := executeCommand(config.Options, config.Target.Emulator[0], args...)
-			buf := &bytes.Buffer{}
-			w := io.MultiWriter(os.Stdout, buf)
-			cmd.Stdout = w
-			cmd.Stderr = os.Stderr
-			err := cmd.Run()
-			if err != nil {
-				if err, ok := err.(*exec.ExitError); !ok || !err.Exited() {
-					// Workaround for QEMU which always exits with an error.
-					return &commandError{"failed to run emulator with", result.Binary, err}
-				}
+			fmt.Printf("FAIL\t%s\t%.3fs\n", importPath, duration.Seconds())
+		}
+		return nil
+	})
+	if err, ok := err.(loader.NoTestFilesError); ok {
+		fmt.Printf("?   \t%s\t[no test files]\n", err.ImportPath)
+		// Pretend the test passed - it at least didn't fail.
+		return true, nil
+	}
+	return passed, err
+}
+
+// runPackageTest runs a test binary that was previously built. The return
+// values are whether the test passed and any errors encountered while trying to
+// run the binary.
+func runPackageTest(config *compileopts.Config, result builder.BuildResult) (bool, error) {
+	if len(config.Target.Emulator) == 0 {
+		// Run directly.
+		cmd := executeCommand(config.Options, result.Binary)
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		cmd.Dir = result.MainDir
+		err := cmd.Run()
+		if err != nil {
+			if _, ok := err.(*exec.ExitError); ok {
+				// Binary exited with a non-zero exit code, which means the test
+				// failed.
+				return false, nil
 			}
-			testOutput := string(buf.Bytes())
-			if testOutput == "PASS\n" || strings.HasSuffix(testOutput, "\nPASS\n") {
-				// Test passed.
-				return nil
-			} else {
-				// Test failed, either by ending with the word "FAIL" or with a
-				// panic of some sort.
-				os.Exit(1)
-				return nil // unreachable
+			return false, &commandError{"failed to run compiled binary", result.Binary, err}
+		}
+		return true, nil
+	} else {
+		// Run in an emulator.
+		args := append(config.Target.Emulator[1:], result.Binary)
+		cmd := executeCommand(config.Options, config.Target.Emulator[0], args...)
+		buf := &bytes.Buffer{}
+		w := io.MultiWriter(os.Stdout, buf)
+		cmd.Stdout = w
+		cmd.Stderr = os.Stderr
+		err := cmd.Run()
+		if err != nil {
+			if err, ok := err.(*exec.ExitError); !ok || !err.Exited() {
+				// Workaround for QEMU which always exits with an error.
+				return false, &commandError{"failed to run emulator with", result.Binary, err}
 			}
 		}
-	})
+		testOutput := string(buf.Bytes())
+		if testOutput == "PASS\n" || strings.HasSuffix(testOutput, "\nPASS\n") {
+			// Test passed.
+			return true, nil
+		} else {
+			// Test failed, either by ending with the word "FAIL" or with a
+			// panic of some sort.
+			return false, nil
+		}
+	}
 }
 
 // Flash builds and flashes the built binary to the given serial port.
@@ -1072,16 +1104,26 @@ func main() {
 		err := Run(pkgName, options)
 		handleCompilerError(err)
 	case "test":
-		pkgName := "."
-		if flag.NArg() == 1 {
-			pkgName = filepath.ToSlash(flag.Arg(0))
-		} else if flag.NArg() > 1 {
-			fmt.Fprintln(os.Stderr, "test only accepts a single positional argument: package name, but multiple were specified")
-			usage()
+		var pkgNames []string
+		for i := 0; i < flag.NArg(); i++ {
+			pkgNames = append(pkgNames, filepath.ToSlash(flag.Arg(i)))
+		}
+		if len(pkgNames) == 0 {
+			pkgNames = []string{"."}
+		}
+		allTestsPassed := true
+		for _, pkgName := range pkgNames {
+			// TODO: parallelize building the test binaries
+			passed, err := Test(pkgName, options, *testCompileOnlyFlag, outpath)
+			handleCompilerError(err)
+			if !passed {
+				allTestsPassed = false
+			}
+		}
+		if !allTestsPassed {
+			fmt.Println("FAIL")
 			os.Exit(1)
 		}
-		err := Test(pkgName, options, *testCompileOnlyFlag, outpath)
-		handleCompilerError(err)
 	case "targets":
 		dir := filepath.Join(goenv.Get("TINYGOROOT"), "targets")
 		entries, err := ioutil.ReadDir(dir)

src/testing/testing.go

@@ -201,6 +201,10 @@ type M struct {
 
 // Run the test suite.
 func (m *M) Run() int {
+	if len(m.Tests) == 0 {
+		fmt.Fprintln(os.Stderr, "testing: warning: no tests to run")
+	}
+
 	failures := 0
 	for _, test := range m.Tests {
 		t := &T{
@@ -226,7 +230,6 @@ func (m *M) Run() int {
 	}
 
 	if failures > 0 {
-		fmt.Printf("exit status %d\n", failures)
 		fmt.Println("FAIL")
 	} else {
 		fmt.Println("PASS")