testscript: add unquote command (#60)

We add an unquote command to the standard set of commands,
because it's not that uncommon a requirement.

We add a `-quote` flag to txtar-savedir that will automatically
quote files if needed, adding an unquote command to the
comment section.

Future work could add a way of quoting arbitrary data (e.g. binary
data) but this should do for now.

Author: rogpeppe

cmd/testscript/main_test.go

@@ -6,7 +6,6 @@ package main
 
 import (
 	"bytes"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -52,7 +51,6 @@ func TestScripts(t *testing.T) {
 			return nil
 		},
 		Cmds: map[string]func(ts *testscript.TestScript, neg bool, args []string){
-			"unquote":        unquote,
 			"dropgofrompath": dropgofrompath,
 			"setfilegoproxy": setfilegoproxy,
 		},
@@ -63,21 +61,6 @@ func TestScripts(t *testing.T) {
 	testscript.Run(t, p)
 }
 
-func unquote(ts *testscript.TestScript, neg bool, args []string) {
-	if neg {
-		ts.Fatalf("unsupported: ! unquote")
-	}
-	for _, arg := range args {
-		file := ts.MkAbs(arg)
-		data, err := ioutil.ReadFile(file)
-		ts.Check(err)
-		data = bytes.Replace(data, []byte("\n>"), []byte("\n"), -1)
-		data = bytes.TrimPrefix(data, []byte(">"))
-		err = ioutil.WriteFile(file, data, 0666)
-		ts.Check(err)
-	}
-}
-
 func dropgofrompath(ts *testscript.TestScript, neg bool, args []string) {
 	if neg {
 		ts.Fatalf("unsupported: ! dropgofrompath")

cmd/txtar-savedir/savedir.go

@@ -14,7 +14,8 @@
 package main
 
 import (
-	"flag"
+	"bytes"
+	stdflag "flag"
 	"fmt"
 	"io/ioutil"
 	"log"
@@ -26,20 +27,27 @@ import (
 	"github.com/rogpeppe/go-internal/txtar"
 )
 
+var flag = stdflag.NewFlagSet(os.Args[0], stdflag.ContinueOnError)
+
 func usage() {
 	fmt.Fprintf(os.Stderr, "usage: savedir dir >saved.txt\n")
-	os.Exit(2)
+	flag.PrintDefaults()
 }
 
+var quoteFlag = flag.Bool("quote", false, "quote files that contain txtar file markers instead of failing")
+
 func main() {
 	os.Exit(main1())
 }
 
 func main1() int {
 	flag.Usage = usage
-	flag.Parse()
+	if flag.Parse(os.Args[1:]) != nil {
+		return 2
+	}
 	if flag.NArg() != 1 {
 		usage()
+		return 2
 	}
 
 	log.SetPrefix("txtar-savedir: ")
@@ -71,7 +79,27 @@ func main1() int {
 			log.Printf("%s: ignoring invalid UTF-8 data", path)
 			return nil
 		}
-		a.Files = append(a.Files, txtar.File{Name: strings.TrimPrefix(path, dir+string(filepath.Separator)), Data: data})
+		if len(data) > 0 && !bytes.HasSuffix(data, []byte("\n")) {
+			log.Printf("%s: adding final newline", path)
+			data = append(data, '\n')
+		}
+		filename := strings.TrimPrefix(path, dir+string(filepath.Separator))
+		if txtar.NeedsQuote(data) {
+			if !*quoteFlag {
+				log.Printf("%s: ignoring file with txtar marker in", path)
+				return nil
+			}
+			data, err = txtar.Quote(data)
+			if err != nil {
+				log.Printf("%s: ignoring unquotable file: %v", path, err)
+				return nil
+			}
+			a.Comment = append(a.Comment, []byte("unquote "+filename+"\n")...)
+		}
+		a.Files = append(a.Files, txtar.File{
+			Name: filename,
+			Data: data,
+		})
 		return nil
 	})
 

cmd/txtar-savedir/testdata/needquote.txt

@@ -0,0 +1,14 @@
+unquote blah/withsep
+unquote expect
+txtar-savedir blah
+stderr 'txtar-savedir: blah.withsep: ignoring file with txtar marker in'
+cmp stdout expect
+
+-- blah/withsep --
+>-- separator --
+>foo
+-- blah/nosep --
+bar
+-- expect --
+>-- nosep --
+>bar

cmd/txtar-savedir/testdata/quote.txt

@@ -0,0 +1,14 @@
+unquote blah/withsep
+unquote expect
+txtar-savedir -quote blah
+! stderr .+
+cmp stdout expect
+
+-- blah/withsep --
+>-- separator --
+>foo
+-- expect --
+>unquote withsep
+>-- withsep --
+>>-- separator --
+>>foo

cmd/txtar-savedir/testdata/txtar-savedir-self.txt

@@ -2,9 +2,10 @@
 # define a txtar archive-generated file that is the golden output from txtar-savedir
 # So we can't actually test the output for now, instead we just rely on a simple
 # stdout check
+unquote expect
 txtar-savedir blah
 ! stderr .+
-stdout '^module example.com/blah$'
+cmp stdout expect
 
 -- blah/go.mod --
 module example.com/blah
@@ -17,3 +18,15 @@ import "fmt"
 func main() {
   fmt.Println("Hello, world!")
 }
+-- expect --
+>-- go.mod --
+>module example.com/blah
+>
+>-- main.go --
+>package main
+>
+>import "fmt"
+>
+>func main() {
+>  fmt.Println("Hello, world!")
+>}

testscript/cmd.go

@@ -15,6 +15,7 @@ import (
 	"strings"
 
 	"github.com/rogpeppe/go-internal/internal/textutil"
+	"github.com/rogpeppe/go-internal/txtar"
 )
 
 // scriptCmds are the script command implementations.
@@ -34,6 +35,7 @@ var scriptCmds = map[string]func(*TestScript, bool, []string){
 	"grep":    (*TestScript).cmdGrep,
 	"mkdir":   (*TestScript).cmdMkdir,
 	"rm":      (*TestScript).cmdRm,
+	"unquote": (*TestScript).cmdUnquote,
 	"skip":    (*TestScript).cmdSkip,
 	"stdin":   (*TestScript).cmdStdin,
 	"stderr":  (*TestScript).cmdStderr,
@@ -303,6 +305,22 @@ func (ts *TestScript) cmdMkdir(neg bool, args []string) {
 	}
 }
 
+// unquote unquotes files.
+func (ts *TestScript) cmdUnquote(neg bool, args []string) {
+	if neg {
+		ts.Fatalf("unsupported: ! unquote")
+	}
+	for _, arg := range args {
+		file := ts.MkAbs(arg)
+		data, err := ioutil.ReadFile(file)
+		ts.Check(err)
+		data, err = txtar.Unquote(data)
+		ts.Check(err)
+		err = ioutil.WriteFile(file, data, 0666)
+		ts.Check(err)
+	}
+}
+
 // rm removes files or directories.
 func (ts *TestScript) cmdRm(neg bool, args []string) {
 	if neg {

testscript/doc.go

@@ -166,6 +166,12 @@ The predefined commands are:
 - mkdir path...
   Create the listed directories, if they do not already exists.
 
+- unquote file...
+  Rewrite each file by replacing any leading ">" characters from
+  each line. This enables a file to contain substrings that look like
+  txtar file markers.
+  See also https://godoc.org/github.com/rogpeppe/go-internal/txtar#Unquote
+
 - rm file...
   Remove the listed files or directories.
 

txtar/archive.go

@@ -33,11 +33,13 @@ package txtar
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
+	"unicode/utf8"
 )
 
 // An Archive is a collection of files.
@@ -89,6 +91,55 @@ func Parse(data []byte) *Archive {
 	return a
 }
 
+// NeedsQuote reports whether the given data needs to
+// be quoted before it's included as a txtar file.
+func NeedsQuote(data []byte) bool {
+	_, _, after := findFileMarker(data)
+	return after != nil
+}
+
+// Quote quotes the data so that it can be safely stored in a txtar
+// file. This copes with files that contain lines that look like txtar
+// separators.
+//
+// The original data can be recovered with Unquote. It returns an error
+// if the data cannot be quoted (for example because it has no final
+// newline or it holds unprintable characters)
+func Quote(data []byte) ([]byte, error) {
+	if len(data) == 0 {
+		return nil, nil
+	}
+	if data[len(data)-1] != '\n' {
+		return nil, errors.New("data has no final newline")
+	}
+	if !utf8.Valid(data) {
+		return nil, fmt.Errorf("data contains non-UTF-8 characters")
+	}
+	var nd []byte
+	prev := byte('\n')
+	for _, b := range data {
+		if prev == '\n' {
+			nd = append(nd, '>')
+		}
+		nd = append(nd, b)
+		prev = b
+	}
+	return nd, nil
+}
+
+// Unquote unquotes data as quoted by Quote.
+func Unquote(data []byte) ([]byte, error) {
+	if len(data) == 0 {
+		return nil, nil
+	}
+	if data[0] != '>' || data[len(data)-1] != '\n' {
+		return nil, errors.New("data does not appear to be quoted")
+	}
+	data = bytes.Replace(data, []byte("\n>"), []byte("\n"), -1)
+	data = bytes.TrimPrefix(data, []byte(">"))
+	return data, nil
+}
+
 var (
 	newlineMarker = []byte("\n-- ")
 	marker        = []byte("-- ")

txtar/archive_test.go

@@ -104,3 +104,88 @@ func TestWrite(t *testing.T) {
 		t.Fatalf("expected %v; got %v", want, err)
 	}
 }
+
+var unquoteErrorTests = []struct {
+	testName    string
+	data        string
+	expectError string
+}{{
+	testName:    "no final newline",
+	data:        ">hello",
+	expectError: `data does not appear to be quoted`,
+}, {
+	testName:    "no initial >",
+	data:        "hello\n",
+	expectError: `data does not appear to be quoted`,
+}}
+
+func TestUnquote(t *testing.T) {
+	for _, test := range unquoteErrorTests {
+		t.Run(test.testName, func(t *testing.T) {
+			_, err := Unquote([]byte(test.data))
+			if err == nil {
+				t.Fatalf("unexpected success")
+			}
+			if err.Error() != test.expectError {
+				t.Fatalf("unexpected error; got %q want %q", err, test.expectError)
+			}
+		})
+	}
+}
+
+var quoteTests = []struct {
+	testName    string
+	data        string
+	expect      string
+	expectError string
+}{{
+	testName: "empty",
+	data:     "",
+	expect:   "",
+}, {
+	testName: "one line",
+	data:     "foo\n",
+	expect:   ">foo\n",
+}, {
+	testName: "several lines",
+	data:     "foo\nbar\n-- baz --\n",
+	expect:   ">foo\n>bar\n>-- baz --\n",
+}, {
+	testName:    "bad data",
+	data:        "foo\xff\n",
+	expectError: `data contains non-UTF-8 characters`,
+}, {
+	testName:    "no final newline",
+	data:        "foo",
+	expectError: `data has no final newline`,
+}}
+
+func TestQuote(t *testing.T) {
+	for _, test := range quoteTests {
+		t.Run(test.testName, func(t *testing.T) {
+			got, err := Quote([]byte(test.data))
+			if test.expectError != "" {
+				if err == nil {
+					t.Fatalf("unexpected success")
+				}
+				if err.Error() != test.expectError {
+					t.Fatalf("unexpected error; got %q want %q", err, test.expectError)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("quote error: %v", err)
+			}
+			if string(got) != test.expect {
+				t.Fatalf("unexpected result; got %q want %q", got, test.expect)
+			}
+			orig, err := Unquote(got)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if string(orig) != test.data {
+				t.Fatalf("round trip failed; got %q want %q", orig, test.data)
+			}
+		})
+	}
+}