scanner more safe whitespace skip (#8499)

Author: max-hoffman

go/cmd/dolt/commands/sql_statement_scanner.go

@@ -96,20 +96,8 @@ func (s *streamScanner) Scan() bool {
 	}
 
 	// discard leading whitespace
-	for {
-		if s.i >= s.fill {
-			if err := s.read(); err != nil {
-				s.err = err
-				return false
-			}
-		}
-		if !unicode.IsSpace(rune(s.buf[s.i])) {
-			break
-		}
-		if s.buf[s.i] == '\n' {
-			s.lineNum++
-		}
-		s.i++
+	if !s.skipWhitespace() {
+		return false
 	}
 	s.truncate()
 
@@ -143,6 +131,28 @@ func (s *streamScanner) Scan() bool {
 	}
 }
 
+func (s *streamScanner) skipWhitespace() bool {
+	for {
+		if s.i >= s.fill {
+			if err := s.read(); err != nil {
+				s.err = err
+				return false
+			}
+		}
+		if s.isEOF {
+			return true
+		}
+		if !unicode.IsSpace(rune(s.buf[s.i])) {
+			break
+		}
+		if s.buf[s.i] == '\n' {
+			s.lineNum++
+		}
+		s.i++
+	}
+	return true
+}
+
 func (s *streamScanner) truncate() {
 	// copy size should be 4k or less
 	s.state.start = s.i
@@ -213,13 +223,8 @@ func (s *streamScanner) isDelimiterExpr() (error, bool) {
 	if s.fill-s.i >= delimPrefixLen && bytes.EqualFold(s.buf[s.i:s.i+delimPrefixLen], delimPrefix) {
 		delimTokenIdx := s.i
 		s.i += delimPrefixLen
-		for ; !s.isEOF && unicode.IsSpace(rune(s.buf[s.i])); s.i++ {
-			if s.i >= s.fill {
-				if err := s.read(); err != nil {
-					s.err = err
-					return err, false
-				}
-			}
+		if !s.skipWhitespace() {
+			return nil, false
 		}
 		if s.isEOF {
 			// invalid delimiter

go/cmd/dolt/commands/sql_statement_scanner_test.go

@@ -195,6 +195,15 @@ insert into foo values (1,2,3)|`,
 			},
 			lineNums: []int{1, 2},
 		},
+		{
+			input: "DELIMITER" + strings.Repeat(" ", 4096) + `|
+insert into foo values (1,2,3)|`,
+			statements: []string{
+				"",
+				"insert into foo values (1,2,3)",
+			},
+			lineNums: []int{1, 2},
+		},
 	}
 
 	for _, tt := range testcases {