testscript: implement UpdateScripts parameter (#83)

This makes it possible for a test to automatically update the test scripts
when output changes. Does not cover `cmpenv`.

Author: rogpeppe

testscript/cmd.go

@@ -121,16 +121,24 @@ func (ts *TestScript) doCmdCmp(args []string, env bool) {
 	name1, name2 := args[0], args[1]
 	text1 := ts.ReadFile(name1)
 
-	data, err := ioutil.ReadFile(ts.MkAbs(name2))
+	absName2 := ts.MkAbs(name2)
+	data, err := ioutil.ReadFile(absName2)
 	ts.Check(err)
 	text2 := string(data)
 	if env {
 		text2 = ts.expand(text2)
 	}
-
 	if text1 == text2 {
 		return
 	}
+	if ts.params.UpdateScripts && !env && (args[0] == "stdout" || args[0] == "stderr") {
+		if scriptFile, ok := ts.scriptFiles[absName2]; ok {
+			ts.scriptUpdates[scriptFile] = text1
+			return
+		}
+		// The file being compared against isn't in the txtar archive, so don't
+		// update the script.
+	}
 
 	ts.Logf("[diff -%s +%s]\n%s\n", name1, name2, textutil.Diff(text1, text2))
 	ts.Fatalf("%s and %s differ", name1, name2)

testscript/testdata/testscript_update_script.txt

@@ -0,0 +1,17 @@
+unquote scripts/testscript.txt
+unquote testscript-new.txt
+testscript-update scripts
+cmp scripts/testscript.txt testscript-new.txt
+
+-- scripts/testscript.txt --
+>echo stdout right
+>cmp stdout expect
+>
+>-- expect --
+>wrong
+-- testscript-new.txt --
+>echo stdout right
+>cmp stdout expect
+>
+>-- expect --
+>right

testscript/testdata/testscript_update_script_not_in_archive.txt

@@ -0,0 +1,12 @@
+unquote scripts/testscript.txt
+cp scripts/testscript.txt unchanged
+! testscript-update scripts
+cmp scripts/testscript.txt unchanged
+
+-- scripts/testscript.txt --
+>echo stdout right
+>cp file expect
+>cmp stdout expect
+>
+>-- file --
+>wrong

testscript/testdata/testscript_update_script_quote.txt

@@ -0,0 +1,17 @@
+unquote scripts/testscript.txt
+unquote testscript-new.txt
+testscript-update scripts
+cmp scripts/testscript.txt testscript-new.txt
+
+-- scripts/testscript.txt --
+>echo stdout '-- lookalike --'
+>cmp stdout expect
+>
+>-- expect --
+>wrong
+-- testscript-new.txt --
+>echo stdout '-- lookalike --'
+>cmp stdout expect
+>
+>-- expect --
+>>-- lookalike --

testscript/testdata/testscript_update_script_stderr.txt

@@ -0,0 +1,17 @@
+unquote scripts/testscript.txt
+unquote testscript-new.txt
+testscript-update scripts
+cmp scripts/testscript.txt testscript-new.txt
+
+-- scripts/testscript.txt --
+>echo stderr right
+>cmp stderr expect
+>
+>-- expect --
+>wrong
+-- testscript-new.txt --
+>echo stderr right
+>cmp stderr expect
+>
+>-- expect --
+>right

testscript/testscript.go

@@ -103,6 +103,16 @@ type Params struct {
 	// (for example because the function invoked os.Exit), then the
 	// error will be ignored.
 	IgnoreMissedCoverage bool
+
+	// UpdateScripts specifies that if a `cmp` command fails and
+	// its first argument is `stdout` or `stderr` and its second argument
+	// refers to a file inside the testscript file, the command will succeed
+	// and the testscript file will be updated to reflect the actual output.
+	//
+	// The content will be quoted with txtar.Quote if needed;
+	// a manual change will be needed if it is not unquoted in the
+	// script.
+	UpdateScripts bool
 }
 
 // RunDir runs the tests in the given directory. All files in dir with a ".txt"
@@ -165,13 +175,15 @@ func RunT(t T, p Params) {
 		t.Run(name, func(t T) {
 			t.Parallel()
 			ts := &TestScript{
-				t:           t,
-				testTempDir: testTempDir,
-				name:        name,
-				file:        file,
-				params:      p,
-				ctxt:        context.Background(),
-				deferred:    func() {},
+				t:             t,
+				testTempDir:   testTempDir,
+				name:          name,
+				file:          file,
+				params:        p,
+				ctxt:          context.Background(),
+				deferred:      func() {},
+				scriptFiles:   make(map[string]string),
+				scriptUpdates: make(map[string]string),
 			}
 			defer func() {
 				if p.TestWork || *testWork {
@@ -191,27 +203,30 @@ func RunT(t T, p Params) {
 
 // A TestScript holds execution state for a single test script.
 type TestScript struct {
-	params      Params
-	t           T
-	testTempDir string
-	workdir     string                      // temporary work dir ($WORK)
-	log         bytes.Buffer                // test execution log (printed at end of test)
-	mark        int                         // offset of next log truncation
-	cd          string                      // current directory during test execution; initially $WORK/gopath/src
-	name        string                      // short name of test ("foo")
-	file        string                      // full file name ("testdata/script/foo.txt")
-	lineno      int                         // line number currently executing
-	line        string                      // line currently executing
-	env         []string                    // environment list (for os/exec)
-	envMap      map[string]string           // environment mapping (matches env; on Windows keys are lowercase)
-	values      map[interface{}]interface{} // values for custom commands
-	stdin       string                      // standard input to next 'go' command; set by 'stdin' command.
-	stdout      string                      // standard output from last 'go' command; for 'stdout' command
-	stderr      string                      // standard error from last 'go' command; for 'stderr' command
-	stopped     bool                        // test wants to stop early
-	start       time.Time                   // time phase started
-	background  []backgroundCmd             // backgrounded 'exec' and 'go' commands
-	deferred    func()                      // deferred cleanup actions.
+	params        Params
+	t             T
+	testTempDir   string
+	workdir       string                      // temporary work dir ($WORK)
+	log           bytes.Buffer                // test execution log (printed at end of test)
+	mark          int                         // offset of next log truncation
+	cd            string                      // current directory during test execution; initially $WORK/gopath/src
+	name          string                      // short name of test ("foo")
+	file          string                      // full file name ("testdata/script/foo.txt")
+	lineno        int                         // line number currently executing
+	line          string                      // line currently executing
+	env           []string                    // environment list (for os/exec)
+	envMap        map[string]string           // environment mapping (matches env; on Windows keys are lowercase)
+	values        map[interface{}]interface{} // values for custom commands
+	stdin         string                      // standard input to next 'go' command; set by 'stdin' command.
+	stdout        string                      // standard output from last 'go' command; for 'stdout' command
+	stderr        string                      // standard error from last 'go' command; for 'stderr' command
+	stopped       bool                        // test wants to stop early
+	start         time.Time                   // time phase started
+	background    []backgroundCmd             // backgrounded 'exec' and 'go' commands
+	deferred      func()                      // deferred cleanup actions.
+	archive       *txtar.Archive              // the testscript being run.
+	scriptFiles   map[string]string           // files stored in the txtar archive (absolute paths -> path in script)
+	scriptUpdates map[string]string           // updates to testscript files via UpdateScripts.
 
 	ctxt context.Context // per TestScript context
 }
@@ -256,8 +271,10 @@ func (ts *TestScript) setup() string {
 	// Unpack archive.
 	a, err := txtar.ParseFile(ts.file)
 	ts.Check(err)
+	ts.archive = a
 	for _, f := range a.Files {
 		name := ts.MkAbs(ts.expand(f.Name))
+		ts.scriptFiles[name] = f.Name
 		ts.Check(os.MkdirAll(filepath.Dir(name), 0777))
 		ts.Check(ioutil.WriteFile(name, f.Data, 0666))
 	}
@@ -327,6 +344,7 @@ func (ts *TestScript) run() {
 		fmt.Fprintf(&ts.log, "\n")
 		ts.mark = ts.log.Len()
 	}
+	defer ts.applyScriptUpdates()
 
 	// Run script.
 	// See testdata/script/README for documentation of script form.
@@ -434,6 +452,40 @@ Script:
 	}
 }
 
+func (ts *TestScript) applyScriptUpdates() {
+	if len(ts.scriptUpdates) == 0 {
+		return
+	}
+	for name, content := range ts.scriptUpdates {
+		found := false
+		for i := range ts.archive.Files {
+			f := &ts.archive.Files[i]
+			if f.Name != name {
+				continue
+			}
+			data := []byte(content)
+			if txtar.NeedsQuote(data) {
+				data1, err := txtar.Quote(data)
+				if err != nil {
+					ts.t.Fatal(fmt.Sprintf("cannot update script file %q: %v", f.Name, err))
+					continue
+				}
+				data = data1
+			}
+			f.Data = data
+			found = true
+		}
+		// Sanity check.
+		if !found {
+			panic("script update file not found")
+		}
+	}
+	if err := ioutil.WriteFile(ts.file, txtar.Format(ts.archive), 0666); err != nil {
+		ts.t.Fatal("cannot update script: ", err)
+	}
+	ts.Logf("%s updated", ts.file)
+}
+
 // condition reports whether the given condition is satisfied.
 func (ts *TestScript) condition(cond string) (bool, error) {
 	switch cond {

testscript/testscript_test.go

@@ -5,6 +5,7 @@
 package testscript
 
 import (
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -123,6 +124,35 @@ func TestScripts(t *testing.T) {
 					ts.Fatalf("reading %q; got %q want %q", args[0], got, want)
 				}
 			},
+			"testscript-update": func(ts *TestScript, neg bool, args []string) {
+				// Run testscript in testscript. Oooh! Meta!
+				if len(args) != 1 {
+					ts.Fatalf("testscript <dir>")
+				}
+				t := &fakeT{ts: ts}
+				func() {
+					defer func() {
+						if err := recover(); err != nil {
+							if err != errAbort {
+								panic(err)
+							}
+						}
+					}()
+					RunT(t, Params{
+						Dir:           ts.MkAbs(args[0]),
+						UpdateScripts: true,
+					})
+				}()
+				if neg {
+					if len(t.failMsgs) == 0 {
+						ts.Fatalf("testscript-update unexpectedly succeeded")
+					}
+					return
+				}
+				if len(t.failMsgs) > 0 {
+					ts.Fatalf("testscript-update unexpectedly failed with errors: %q", t.failMsgs)
+				}
+			},
 		},
 		Setup: func(env *Env) error {
 			infos, err := ioutil.ReadDir(env.WorkDir)
@@ -219,3 +249,37 @@ func waitFile(ts *TestScript, neg bool, args []string) {
 	}
 	ts.Fatalf("timed out waiting for %q to be created", path)
 }
+
+type fakeT struct {
+	ts       *TestScript
+	failMsgs []string
+}
+
+var errAbort = errors.New("abort test")
+
+func (t *fakeT) Skip(args ...interface{}) {
+	panic(errAbort)
+}
+
+func (t *fakeT) Fatal(args ...interface{}) {
+	t.failMsgs = append(t.failMsgs, fmt.Sprint(args...))
+	panic(errAbort)
+}
+
+func (t *fakeT) Parallel() {}
+
+func (t *fakeT) Log(args ...interface{}) {
+	t.ts.Logf("testscript: %v", fmt.Sprint(args...))
+}
+
+func (t *fakeT) FailNow() {
+	t.Fatal("failed")
+}
+
+func (t *fakeT) Run(name string, f func(T)) {
+	f(t)
+}
+
+func (t *fakeT) Verbose() bool {
+	return false
+}