feat: add raw string support to gengo comment tag arg parsing

Author: aaron-prindle

v2/comments.go

@@ -18,8 +18,10 @@ package gengo
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"strings"
+	"text/scanner"
 	"unicode"
 )
 
@@ -104,8 +106,10 @@ func ExtractSingleBoolCommentTag(marker string, key string, defaultVal bool, lin
 //   - 'marker' + "key=value"
 //   - 'marker' + "key()=value"
 //   - 'marker' + "key(arg)=value"
+//   - 'marker' + "key(`raw string`)=value"
 //
-// The arg is optional.  If not specified (either as "key=value" or as
+// The arg is optional.  It may be a Go identifier or a raw string literal
+// enclosed in back-ticks.  If not specified (either as "key=value" or as
 // "key()=value"), the resulting Tag will have an empty Args list.
 //
 // The value is optional.  If not specified, the resulting Tag will have "" as
@@ -169,12 +173,10 @@ func ExtractFunctionStyleCommentTags(marker string, tagNames []string, lines []s
 		if !strings.HasPrefix(line, marker) {
 			continue
 		}
-		line = stripTrailingComment(line)
-		kv := strings.SplitN(line[len(marker):], "=", 2)
-		key := kv[0]
-		val := ""
-		if len(kv) == 2 {
-			val = kv[1]
+		body := stripTrailingComment(line[len(marker):])
+		key, val, err := splitKeyValScanner(body)
+		if err != nil {
+			return nil, err
 		}
 
 		tag := Tag{}
@@ -260,31 +262,99 @@ func parseTagKey(input string, tagNames []string) (string, []string, error) {
 // '(', including the trailing ')'.
 //
 // At the moment this assumes that the entire string between the opening '('
-// and the trailing ')' is a single Go-style identifier token, but in the
-// future could be extended to have multiple arguments with actual syntax.  The
-// single token may consist only of letters and digits.  Whitespace is not
-// allowed.
+// and the trailing ')' is a single Go-style identifier token OR a raw string
+// literal. The single Go-style token may consist only of letters and digits
+// and whitespace is not allowed.
 func parseTagArgs(input string) ([]string, error) {
-	// This is really dumb, but should be extendable to a "real" parser if
-	// needed.
-	runes := []rune(input)
-	for i, r := range runes {
-		if unicode.IsLetter(r) || unicode.IsDigit(r) {
-			continue
-		}
-		if r == ',' {
-			return nil, fmt.Errorf("multiple arguments are not supported: %q", input)
+	s := initArgScanner(input)
+	var args []string
+	if s.Peek() != ')' {
+		// Arg found.
+		arg, err := parseArg(s)
+		if err != nil {
+			return nil, err
 		}
-		if r == ')' {
-			if i != len(runes)-1 {
-				return nil, fmt.Errorf("unexpected characters after ')': %q", string(runes[i:]))
+		args = append(args, arg)
+	}
+	// Expect one closing ')' after the arg.
+	if s.Scan() != ')' {
+		return nil, fmt.Errorf("no closing ')' found: %q", input)
+	}
+	// Expect no whitespace, etc. after the one ')'.
+	if s.Scan() != scanner.EOF {
+		pos := s.Pos().Offset - len(s.TokenText())
+		return nil, fmt.Errorf("unexpected characters after ')': %q", input[pos:])
+	}
+	return args, nil
+}
+
+type argScanner struct {
+	*scanner.Scanner
+	errs []error
+}
+
+func initArgScanner(input string) *argScanner {
+	s := &argScanner{Scanner: &scanner.Scanner{}}
+
+	s.Init(strings.NewReader(input))
+	s.Mode = scanner.ScanIdents | scanner.ScanRawStrings
+	s.Whitespace = 0
+
+	s.Error = func(_ *scanner.Scanner, msg string) {
+		s.errs = append(s.errs,
+			fmt.Errorf("error parsing %q at %v: %s", input, s.Position, msg))
+	}
+	return s
+}
+
+func (s *argScanner) unexpectedTokenError(expected string, token string) error {
+	s.Error(s.Scanner, fmt.Sprintf("expected %s but got (%q)", expected, token))
+	return errors.Join(s.errs...)
+}
+
+func parseArg(s *argScanner) (string, error) {
+	switch tok := s.Scan(); tok {
+	case scanner.RawString:
+		return s.TokenText(), nil
+	case scanner.Ident:
+		txt := s.TokenText()
+		for _, r := range txt {
+			if !unicode.IsLetter(r) && !unicode.IsDigit(r) {
+				return "", s.unexpectedTokenError("letter or digit", txt)
 			}
-			if i == 0 {
-				return nil, nil
+		}
+		return txt, nil
+	case ',':
+		return "", fmt.Errorf("multiple arguments are not supported")
+	default:
+		return "", s.unexpectedTokenError("Go-style identifier or raw string", s.TokenText())
+	}
+}
+
+// splitKeyValScanner parses a tag body of the form key[=val]. It parses left to
+// right and stops at the first "=" that is not inside a quoted or raw
+// string literal. Text before that point becomes the key (trimmed of spaces).
+// Text after becomes the val. If no "=" is found, the whole input
+// is returned as key and val is empty. The parsing understands Go-style identifiers,
+// and raw strings. Any other token or scanner error is
+// reported to the caller.
+func splitKeyValScanner(input string) (key, val string, err error) {
+	var s scanner.Scanner
+	s.Init(strings.NewReader(input))
+	s.Mode = scanner.ScanIdents | scanner.ScanRawStrings
+	for {
+		switch tok := s.Scan(); tok {
+		case scanner.EOF:
+			return strings.TrimSpace(input), "", nil
+		case '=':
+			// Split at the first top-level '='.  Everything before (trimmed) is the
+			// key, everything after (not trimmed) is the value.
+			start := s.Pos().Offset - len(s.TokenText())
+			key = strings.TrimSpace(input[:start])
+			if start+len(s.TokenText()) < len(input) {
+				val = input[start+len(s.TokenText()):]
 			}
-			return []string{string(runes[:i])}, nil
+			return key, val, nil
 		}
-		return nil, fmt.Errorf("unsupported character: %q", string(r))
 	}
-	return nil, fmt.Errorf("no closing ')' found: %q", input)
 }

v2/comments_test.go

@@ -217,6 +217,31 @@ func TestExtractExtendedCommentTags(t *testing.T) {
 				Tag{"pfx2Foo", nil, "val1"},
 				Tag{"pfx2Foo", mkstrs("arg"), "val2"}),
 		},
+	}, {
+		name: "raw arg with =, ), and space",
+		comments: []string{
+			"+rawEq(`a=b c=d )`)=xyz",
+		},
+		expect: map[string][]Tag{
+			"rawEq": mktags(Tag{"rawEq", mkstrs("`a=b c=d )`"), "xyz"}),
+		},
+	}, {
+		name: "raw arg no value",
+		comments: []string{
+			"+onlyRaw(`zzz`)",
+		},
+		expect: map[string][]Tag{
+			"onlyRaw": mktags(Tag{"onlyRaw", mkstrs("`zzz`"), ""}),
+		},
+	}, {
+		name: "raw string arg complex",
+		comments: []string{
+			"+rawTag(`[self.foo==10, ()), {}}, \"foo\", 'foo']`)=val",
+		},
+		expect: map[string][]Tag{
+			"rawTag": mktags(
+				Tag{"rawTag", mkstrs("`[self.foo==10, ()), {}}, \"foo\", 'foo']`"), "val"}),
+		},
 	}}
 
 	for _, tc := range cases {
@@ -246,6 +271,9 @@ func TestParseTagKey(t *testing.T) {
 		{"trailingSpace(arg) ", "", nil, true},
 		{"argWithDash(arg-name) ", "", nil, true},
 		{"argWithUnder(arg_name) ", "", nil, true},
+		{"withRaw(`a = b`)", "withRaw", mkss("`a = b`"), false},
+		{"badRaw(missing`)", "", nil, true},
+		{"badMix(arg,`raw`)", "", nil, true},
 	}
 	for _, tc := range cases {
 		key, args, err := parseTagKey(tc.input, nil)
@@ -331,6 +359,11 @@ func TestParseTagArgs(t *testing.T) {
 		{"noClosingParen", nil, true},
 		{"extraParen))", nil, true},
 		{"trailingSpace) ", nil, true},
+		{"`hasRawQuotes`)", mkss("`hasRawQuotes`"), false},
+		{"`raw with =`)", mkss("`raw with =`"), false},
+		{"`raw`   )", nil, true},
+		{"`raw`bad)", nil, true},
+		{"`first``second`)", nil, true},
 	}
 	for _, tc := range cases {
 		ret, err := parseTagArgs(tc.input)
@@ -355,3 +388,27 @@ func TestParseTagArgs(t *testing.T) {
 		}
 	}
 }
+
+func TestSplitKeyValScanner(t *testing.T) {
+	cases := []struct {
+		input string
+		key   string
+		val   string
+	}{
+		{`foo=bar`, "foo", "bar"},
+		{`foo   =   bar`, "foo", "   bar"},
+		{`keyWithRaw(` + "`a=b`" + `)=value`, "keyWithRaw(`a=b`)", "value"},
+		{`noValue`, "noValue", ""},
+		{`rawKey=` + "`x=y`", "rawKey", "`x=y`"},
+	}
+
+	for _, c := range cases {
+		k, v, err := splitKeyValScanner(c.input)
+		if err != nil {
+			t.Fatalf("[%q] unexpected err: %v", c.input, err)
+		}
+		if k != c.key || v != c.val {
+			t.Errorf("[%q] got (%q,%q) want (%q,%q)", c.input, k, v, c.key, c.val)
+		}
+	}
+}