just increase buffer size instead

Author: James Cor

sql/plan/load_data.go

@@ -15,9 +15,10 @@
 package plan
 
 import (
-	"github.com/dolthub/vitess/go/vt/sqlparser"
+	"bytes"
 
 	"github.com/dolthub/go-mysql-server/sql"
+	"github.com/dolthub/vitess/go/vt/sqlparser"
 )
 
 type LoadData struct {
@@ -67,6 +68,25 @@ func (l *LoadData) IsReadOnly() bool {
 	return false
 }
 
+func (l *LoadData) SplitLines(data []byte, atEOF bool) (advance int, token []byte, err error) {
+	// Return Nothing if at end of file and no data passed.
+	if atEOF && len(data) == 0 {
+		return 0, nil, nil
+	}
+
+	// Find the index of the LINES TERMINATED BY delim.
+	if i := bytes.Index(data, []byte(l.LinesTerminatedBy)); i >= 0 {
+		return i + len(l.LinesTerminatedBy), data[0:i], nil
+	}
+
+	// If at end of file with data return the data.
+	if atEOF {
+		return len(data), data, nil
+	}
+
+	return
+}
+
 func (l *LoadData) WithChildren(children ...sql.Node) (sql.Node, error) {
 	if len(children) != 0 {
 		return nil, sql.ErrInvalidChildrenNumber.New(l, len(children), 0)

sql/rowexec/ddl.go

@@ -93,6 +93,10 @@ func (b *BaseBuilder) buildLoadData(ctx *sql.Context, n *plan.LoadData, row sql.
 		reader = file
 	}
 
+	scanner := bufio.NewScanner(reader)
+	scanner.Buffer(nil, int(types.LongTextBlobMax))
+	scanner.Split(n.SplitLines)
+
 	sch := n.Schema()
 	source := sch[0].Source // Schema will always have at least one column
 	colNames := n.ColNames
@@ -116,7 +120,7 @@ func (b *BaseBuilder) buildLoadData(ctx *sql.Context, n *plan.LoadData, row sql.
 	return &loadDataIter{
 		destSch:       n.DestSch,
 		reader:        reader,
-		scanner:       bufio.NewReader(reader),
+		scanner:       scanner,
 		colCount:      len(n.ColNames), // Needs to be the original column count
 		fieldToColMap: fieldToColMap,
 		setExprs:      n.SetExprs,

sql/rowexec/ddl_iters.go

@@ -16,8 +16,7 @@ package rowexec
 
 import (
 	"bufio"
-	"bytes"
-	"fmt"
+		"fmt"
 	"io"
 	"strings"
 	"sync"
@@ -39,10 +38,8 @@ import (
 )
 
 type loadDataIter struct {
-	scanner *bufio.Reader
+	scanner *bufio.Scanner
 	reader  io.ReadCloser
-	buffer  bytes.Buffer
-	hitEOF  bool
 
 	destSch       sql.Schema
 	colCount      int
@@ -64,38 +61,12 @@ type loadDataIter struct {
 var _ sql.RowIter = (*loadDataIter)(nil)
 var _ sql.Closer = (*loadDataIter)(nil)
 
-// readLine reads a line from the scanner. It does not include the delimiter.
-func (l *loadDataIter) readLine() ([]byte, error) {
-	// return EOF on the call after EOF
-	if l.hitEOF {
-		return nil, io.EOF
-	}
-	delim := []byte(l.linesTerminatedBy)
-	delimDelim := delim[len(delim)-1]
-	l.buffer.Reset()
-	var buf []byte
-	for {
-		chunk, err := l.scanner.ReadBytes(delimDelim)
-		if err != nil && err != io.EOF {
-			return nil, err
-		}
-		l.buffer.Write(chunk)
-		buf = l.buffer.Bytes()
-		if bytes.HasSuffix(buf, delim) {
-			return buf[:len(buf)-len(delim)], nil
-		}
-		if err == io.EOF {
-			l.hitEOF = true
-			return buf, nil
-		}
-	}
-}
-
 func (l *loadDataIter) Next(ctx *sql.Context) (returnRow sql.Row, returnErr error) {
 	// skip first ignoreNum lines
-	for ; l.ignoreNum > 0; l.ignoreNum-- {
-		_, err := l.readLine()
-		if err != nil {
+	var err error
+	for ; l.ignoreNum > 0 && l.scanner.Scan(); l.ignoreNum-- {
+		if err = l.scanner.Err(); err != nil {
+			l.reader.Close()
 			return nil, err
 		}
 	}
@@ -104,19 +75,20 @@ func (l *loadDataIter) Next(ctx *sql.Context) (returnRow sql.Row, returnErr erro
 	// until exprs != nil
 	var exprs []sql.Expression
 	for exprs == nil {
-		line, err := l.readLine()
-		if err != nil {
-			return nil, err
+		if keepGoing := l.scanner.Scan(); !keepGoing {
+			if err = l.scanner.Err(); err != nil {
+				return nil, err
+			}
+			return nil, io.EOF
 		}
-		exprs, err = l.parseFields(ctx, string(line))
+		exprs, err = l.parseFields(ctx, l.scanner.Text())
 		if err != nil {
 			return nil, err
 		}
 	}
 
 	row := make(sql.Row, len(exprs))
 	var secondPass []int
-	var err error
 	for i, expr := range exprs {
 		if expr != nil {
 			// Non-literal default values may reference other columns, so we need to evaluate them in a second pass.