parser,scanner: fix bugs with parsing and scanning sql comments

Fixes a bug in the parser where retained comments are mistakenly
added to multiple statements. This when parsing a multi-statement
sql containing strings with the "retainComments" bool set to true.
In this case, comments in a statement are propogated to subsequent
statements in the same parse.

Fixes a bug where `\n` characters are included in scanned singe-
line comments. Now, these wont be included in comments and will
instead be stripped out via the skipWhitespace function.

Epic: None
Release note: None

Author: kyle-a-wong

pkg/sql/parser/parse.go

@@ -201,6 +201,10 @@ func (p *Parser) parse(
 		return statements.Statement[tree.Statement]{}, err
 	}
 
+	// Once a statement has been parsed, reset the comments to ensure
+	// that the next statement does not pick up comments from the previous
+	// statement.
+	defer p.scanner.ResetComments()
 	return statements.Statement[tree.Statement]{
 		AST:             p.lexer.stmt,
 		SQL:             sql,

pkg/sql/parser/parse_internal_test.go

@@ -8,6 +8,8 @@ package parser
 import (
 	"reflect"
 	"testing"
+
+	"github.com/stretchr/testify/require"
 )
 
 func TestScanOneStmt(t *testing.T) {
@@ -135,3 +137,44 @@ func TestScanOneStmt(t *testing.T) {
 		}
 	}
 }
+
+func TestParseWithDept_retainComment(t *testing.T) {
+	testData := []struct {
+		in  string
+		exp [][]string
+	}{
+		{in: ``, exp: nil},
+		{in: `SELECT 1;`, exp: [][]string{nil}},
+		{in: `
+		-- comment
+		SELECT 1`, exp: [][]string{{`-- comment`}}},
+		{in: `/* comment */ SELECT 1`, exp: [][]string{{`/* comment */`}}},
+		{in: `SELECT 1 /* comment */`, exp: [][]string{{`/* comment */`}}},
+		{in: `SELECT 1;SELECT 2`, exp: [][]string{nil, nil}},
+		{in: `SELECT 1 /* block comment1 */ ;SELECT 2 /* block comment2 */`, exp: [][]string{{`/* block comment1 */`}, {`/* block comment2 */`}}},
+		{in: `SELECT 1 /* block comment */ ;SELECT 2 -- single-line-comment`, exp: [][]string{{`/* block comment */`}, {`-- single-line-comment`}}},
+		{in: `SELECT 1 /* block comment */ /* block comment */`, exp: [][]string{{`/* block comment */`, `/* block comment */`}}},
+		{in: `
+/* This is a select 1 */
+SELECT 1 -- in-line comment
+;
+-- This is a select 2
+SELECT 2; -- comment ignore, not part of statement
+/** Comment ignored, not part of statement */
+`, exp: [][]string{{`/* This is a select 1 */`, `-- in-line comment`}, {`-- This is a select 2`}}},
+	}
+	var p Parser
+	for _, d := range testData {
+		t.Run(d.in, func(t *testing.T) {
+			stmts, err := p.parseWithDepth(1, d.in, defaultNakedIntType, retainComments)
+			require.NoError(t, err)
+
+			var res [][]string
+			for i := range stmts {
+				res = append(res, stmts[i].Comments)
+			}
+
+			require.Equal(t, d.exp, res)
+		})
+	}
+}

pkg/sql/parser/scanner_test.go

@@ -135,14 +135,16 @@ func TestScanComment(t *testing.T) {
 		remainder string
 	}{
 		{`/* hello */world`, "", "world"},
+		{`/* hello */
+world`, "", "\nworld"},
 		{`/* hello */*`, "", "*"},
 		{`/* /* deeply /* nested */ comment */ */`, "", ""},
 		{`/* /* */* */`, "", ""},
 		{`/* /* /*/ */ */ */`, "", ""},
 		{`/* multi line
 comment */`, "", ""},
 		{`-- hello world
-foo`, "", "foo"},
+foo`, "", "\nfoo"},
 		{`/*`, "unterminated comment", ""},
 		{`/*/`, "unterminated comment", ""},
 		{`/* /* */`, "unterminated comment", ""},

pkg/sql/scanner/scan.go

@@ -100,6 +100,10 @@ func (s *Scanner) RetainComments() {
 	s.retainComments = true
 }
 
+func (s *Scanner) ResetComments() {
+	s.Comments = nil
+}
+
 // Cleanup is used to avoid holding on to memory unnecessarily (for the cases
 // where we reuse a Scanner).
 func (s *Scanner) Cleanup() {
@@ -596,8 +600,13 @@ func (s *Scanner) ScanComment(lval ScanSymType) (present, ok bool) {
 			return false, true
 		}
 		for {
-			switch s.next() {
-			case eof, '\n':
+			next := s.next()
+			switch next {
+			case eof:
+				return true, true
+			case '\n':
+				// Don't include the new-line character in in-line comments.
+				s.pos--
 				return true, true
 			}
 		}