LONG DATA handling

prerequisite for issue #33

Author: julienschmidt

README.md

@@ -30,6 +30,8 @@ A MySQL-Driver for Go's [database/sql](http://golang.org/pkg/database/sql) packa
   * Connections over TCP/IPv4, TCP/IPv6 or Unix Sockets
   * Automatic handling of broken connections
   * Automatic Connection-Pooling *(by database/sql package)*
+  * Supports queries larger than 16MB
+  * Intelligent `LONG DATA` handling in prepared statements
 
 ## Requirements
   * Go 1.0.3 or higher

connection.go

@@ -17,16 +17,18 @@ import (
 )
 
 type mysqlConn struct {
-	cfg          *config
-	flags        clientFlag
-	charset      byte
-	cipher       []byte
-	netConn      net.Conn
-	buf          *buffer
-	protocol     uint8
-	sequence     uint8
-	affectedRows uint64
-	insertId     uint64
+	cfg              *config
+	flags            clientFlag
+	charset          byte
+	cipher           []byte
+	netConn          net.Conn
+	buf              *buffer
+	protocol         uint8
+	sequence         uint8
+	affectedRows     uint64
+	insertId         uint64
+	maxPacketAllowed int
+	maxWriteSize     int
 }
 
 type config struct {
@@ -192,3 +194,31 @@ func (mc *mysqlConn) Query(query string, args []driver.Value) (driver.Rows, erro
 	// with args, must use prepared stmt
 	return nil, driver.ErrSkip
 }
+
+// Gets the value of the given MySQL System Variable
+func (mc *mysqlConn) getSystemVar(name string) (val []byte, err error) {
+	// Send command
+	err = mc.writeCommandPacketStr(comQuery, "SELECT @@"+name)
+	if err == nil {
+		// Read Result
+		var resLen int
+		resLen, err = mc.readResultSetHeaderPacket()
+		if err == nil {
+			rows := &mysqlRows{mc, false, nil, false}
+
+			if resLen > 0 {
+				// Columns
+				rows.columns, err = mc.readColumns(resLen)
+			}
+
+			dest := make([]driver.Value, resLen)
+			err = rows.readRow(dest)
+			if err == nil {
+				val = dest[0].([]byte)
+				err = mc.readUntilEOF()
+			}
+		}
+	}
+
+	return
+}

const.go

@@ -11,8 +11,8 @@ package mysql
 
 const (
 	minProtocolVersion byte = 10
-	//maxPacketSize      = 1<<24 - 1
-	timeFormat = "2006-01-02 15:04:05"
+	maxPacketSize           = 1<<24 - 1
+	timeFormat              = "2006-01-02 15:04:05"
 )
 
 // MySQL constants documentation:
@@ -47,10 +47,8 @@ const (
 	clientMultiResults
 )
 
-type commandType byte
-
 const (
-	comQuit commandType = iota + 1
+	comQuit byte = iota + 1
 	comInitDB
 	comQuery
 	comFieldList

driver.go

@@ -24,8 +24,11 @@ func (d *mysqlDriver) Open(dsn string) (driver.Conn, error) {
 	var err error
 
 	// New mysqlConn
-	mc := new(mysqlConn)
-	mc.cfg = parseDSN(dsn)
+	mc := &mysqlConn{
+		cfg:              parseDSN(dsn),
+		maxPacketAllowed: maxPacketSize,
+		maxWriteSize:     maxPacketSize - 1,
+	}
 
 	// Connect to Server
 	if _, ok := mc.cfg.params["timeout"]; ok { // with timeout
@@ -60,6 +63,16 @@ func (d *mysqlDriver) Open(dsn string) (driver.Conn, error) {
 		return nil, err
 	}
 
+	// Get max allowed packet size
+	maxap, err := mc.getSystemVar("max_allowed_packet")
+	if err != nil {
+		return nil, err
+	}
+	mc.maxPacketAllowed = stringToInt(maxap) - 1
+	if mc.maxPacketAllowed < maxPacketSize {
+		mc.maxWriteSize = mc.maxPacketAllowed
+	}
+
 	// Handle DSN Params
 	err = mc.handleParams()
 	if err != nil {

driver_test.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"strings"
 	"sync"
 	"testing"
 )
@@ -57,6 +58,9 @@ func getEnv() bool {
 func mustExec(t *testing.T, db *sql.DB, query string, args ...interface{}) (res sql.Result) {
 	res, err := db.Exec(query, args...)
 	if err != nil {
+		if len(query) > 300 {
+			query = "[query too large to print]"
+		}
 		t.Fatalf("Error on Exec %q: %v", query, err)
 	}
 	return
@@ -65,6 +69,9 @@ func mustExec(t *testing.T, db *sql.DB, query string, args ...interface{}) (res
 func mustQuery(t *testing.T, db *sql.DB, query string, args ...interface{}) (rows *sql.Rows) {
 	rows, err := db.Query(query, args...)
 	if err != nil {
+		if len(query) > 300 {
+			query = "[query too large to print]"
+		}
 		t.Fatalf("Error on Query %q: %v", query, err)
 	}
 	return
@@ -501,6 +508,76 @@ func TestNULL(t *testing.T) {
 	mustExec(t, db, "DROP TABLE IF EXISTS test")
 }
 
+func TestLongData(t *testing.T) {
+	if !getEnv() {
+		t.Logf("MySQL-Server not running on %s. Skipping TestLongData", netAddr)
+		return
+	}
+
+	db, err := sql.Open("mysql", dsn)
+	if err != nil {
+		t.Fatalf("Error connecting: %v", err)
+	}
+	defer db.Close()
+
+	var maxAllowedPacketSize int
+	err = db.QueryRow("select @@max_allowed_packet").Scan(&maxAllowedPacketSize)
+	if err != nil {
+		t.Fatal(err)
+	}
+	maxAllowedPacketSize--
+
+	// don't get too ambitious
+	if maxAllowedPacketSize > 1<<25 {
+		maxAllowedPacketSize = 1 << 25
+	}
+
+	mustExec(t, db, "DROP TABLE IF EXISTS test")
+	mustExec(t, db, "CREATE TABLE test (value LONGBLOB) CHARACTER SET utf8 COLLATE utf8_unicode_ci")
+
+	in := strings.Repeat(`0`, maxAllowedPacketSize+1)
+	var out string
+	var rows *sql.Rows
+
+	// Long text data
+	const nonDataQueryLen = 28 // length query w/o value
+	inS := in[:maxAllowedPacketSize-nonDataQueryLen]
+	mustExec(t, db, "INSERT INTO test VALUES('"+inS+"')")
+	rows = mustQuery(t, db, "SELECT value FROM test")
+	if rows.Next() {
+		rows.Scan(&out)
+		if inS != out {
+			t.Fatalf("LONGBLOB: length in: %d, length out: %d", len(inS), len(out))
+		}
+		if rows.Next() {
+			t.Error("LONGBLOB: unexpexted row")
+		}
+	} else {
+		t.Fatalf("LONGBLOB: no data")
+	}
+
+	// Empty table
+	mustExec(t, db, "TRUNCATE TABLE test")
+
+	// Long binary data
+	mustExec(t, db, "INSERT INTO test VALUES(?)", in)
+	rows = mustQuery(t, db, "SELECT value FROM test WHERE 1=?", 1)
+	if rows.Next() {
+		rows.Scan(&out)
+		if in != out {
+			t.Fatalf("LONGBLOB: length in: %d, length out: %d", len(in), len(out))
+		}
+		if rows.Next() {
+			t.Error("LONGBLOB: unexpexted row")
+		}
+		//t.Fatalf("%d %d %d", len(in)+nonDataQueryLen, len(out)+nonDataQueryLen, maxAllowedPacketSize)
+	} else {
+		t.Fatalf("LONGBLOB: no data")
+	}
+
+	mustExec(t, db, "DROP TABLE IF EXISTS test")
+}
+
 // Special cases
 
 func TestRowsClose(t *testing.T) {

errors.go

@@ -16,4 +16,5 @@ var (
 	errPktSync     = errors.New("Commands out of sync. You can't run this command now")
 	errPktSyncMul  = errors.New("Commands out of sync. Did you run multiple statements at once?")
 	errOldPassword = errors.New("It seems like you are using old_passwords, which is unsupported. See https://github.com/Go-SQL-Driver/MySQL/wiki/old_passwords")
+	errPktTooLarge = errors.New("Packet for query is too large. You can change this value on the server by adjusting the 'max_allowed_packet' variable.")
 )