sql read write support (#4976)

Co-authored-by: light.zhou <[email protected]>

Author: zhoushuguang

core/stores/sqlx/config.go

@@ -0,0 +1,29 @@
+package sqlx
+
+import "errors"
+
+var (
+	errEmptyDatasource = errors.New("empty datasource")
+	errEmptyDriverName = errors.New("empty driver name")
+)
+
+// SqlConf defines the configuration for sqlx.
+type SqlConf struct {
+	DataSource string
+	DriverName string   `json:",default=mysql"`
+	Replicas   []string `json:",optional"`
+	Policy     string   `json:",default=round-robin,options=round-robin|random"`
+}
+
+// Validate validates the SqlxConf.
+func (sc SqlConf) Validate() error {
+	if len(sc.DataSource) == 0 {
+		return errEmptyDatasource
+	}
+
+	if len(sc.DriverName) == 0 {
+		return errEmptyDriverName
+	}
+
+	return nil
+}

core/stores/sqlx/config_test.go

@@ -0,0 +1,29 @@
+package sqlx
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/zeromicro/go-zero/core/conf"
+)
+
+func TestValidate(t *testing.T) {
+	text := []byte(`DataSource: primary:password@tcp(127.0.0.1:3306)/primary_db
+`)
+
+	var sc SqlConf
+	err := conf.LoadFromYamlBytes(text, &sc)
+	assert.Nil(t, err)
+	assert.Equal(t, "mysql", sc.DriverName)
+	assert.Equal(t, policyRoundRobin, sc.Policy)
+	assert.Nil(t, sc.Validate())
+
+	sc = SqlConf{}
+	assert.Equal(t, errEmptyDatasource, sc.Validate())
+
+	sc.DataSource = "primary:password@tcp(127.0.0.1:3306)/primary_db"
+	assert.Equal(t, errEmptyDriverName, sc.Validate())
+
+	sc.DriverName = "mysql"
+	assert.Nil(t, sc.Validate())
+}

core/stores/sqlx/rwstrategy.go

@@ -0,0 +1,65 @@
+package sqlx
+
+import "context"
+
+const (
+	// policyRoundRobin round-robin policy for selecting replicas.
+	policyRoundRobin = "round-robin"
+	// policyRandom random policy for selecting replicas.
+	policyRandom = "random"
+
+	// readPrimaryMode indicates that the operation is a read,
+	// but should be performed on the primary database instance.
+	//
+	// This mode is used in scenarios where data freshness and consistency are critical,
+	// such as immediately after writes or where replication lag may cause stale reads.
+	readPrimaryMode readWriteMode = "read-primary"
+
+	// readReplicaMode indicates that the operation is a read from replicas.
+	// This is suitable for scenarios where eventual consistency is acceptable,
+	// and the goal is to offload traffic from the primary and improve read scalability.
+	readReplicaMode readWriteMode = "read-replica"
+
+	// writeMode indicates that the operation is a write operation (to primary).
+	writeMode readWriteMode = "write"
+
+	// notSpecifiedMode indicates that the read/write mode is not specified.
+	notSpecifiedMode readWriteMode = ""
+)
+
+type readWriteMode string
+
+var readWriteModeKey struct{}
+
+func (m readWriteMode) isValid() bool {
+	return m == readPrimaryMode || m == readReplicaMode || m == writeMode
+}
+
+func getReadWriteMode(ctx context.Context) readWriteMode {
+	if mode := ctx.Value(readWriteModeKey); mode != nil {
+		if v, ok := mode.(readWriteMode); ok && v.isValid() {
+			return v
+		}
+	}
+
+	return notSpecifiedMode
+}
+
+func useReplica(ctx context.Context) bool {
+	return getReadWriteMode(ctx) == readReplicaMode
+}
+
+// WithReadPrimaryMode sets the context to read-primary mode.
+func WithReadPrimaryMode(ctx context.Context) context.Context {
+	return context.WithValue(ctx, readWriteModeKey, readPrimaryMode)
+}
+
+// WithReadReplicaMode sets the context to read-replica mode.
+func WithReadReplicaMode(ctx context.Context) context.Context {
+	return context.WithValue(ctx, readWriteModeKey, readReplicaMode)
+}
+
+// WithWriteMode sets the context to write mode, indicating that the operation is a write operation.
+func WithWriteMode(ctx context.Context) context.Context {
+	return context.WithValue(ctx, readWriteModeKey, writeMode)
+}

core/stores/sqlx/rwstrategy_test.go

@@ -0,0 +1,133 @@
+package sqlx
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestIsValid(t *testing.T) {
+	testCases := []struct {
+		name     string
+		mode     readWriteMode
+		expected bool
+	}{
+		{
+			name:     "valid read-primary mode",
+			mode:     readPrimaryMode,
+			expected: true,
+		},
+		{
+			name:     "valid read-replica mode",
+			mode:     readReplicaMode,
+			expected: true,
+		},
+		{
+			name:     "valid write mode",
+			mode:     writeMode,
+			expected: true,
+		},
+		{
+			name:     "not specified mode (empty)",
+			mode:     notSpecifiedMode,
+			expected: false,
+		},
+		{
+			name:     "invalid custom string",
+			mode:     readWriteMode("delete"),
+			expected: false,
+		},
+		{
+			name:     "case sensitive check",
+			mode:     readWriteMode("READ"),
+			expected: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			actual := tc.mode.isValid()
+			assert.Equal(t, tc.expected, actual)
+		})
+	}
+}
+
+func TestWithReadMode(t *testing.T) {
+	ctx := context.Background()
+	readPrimaryCtx := WithReadPrimaryMode(ctx)
+
+	val := readPrimaryCtx.Value(readWriteModeKey)
+	assert.Equal(t, readPrimaryMode, val)
+
+	readReplicaCtx := WithReadReplicaMode(ctx)
+	val = readReplicaCtx.Value(readWriteModeKey)
+	assert.Equal(t, readReplicaMode, val)
+}
+
+func TestWithWriteMode(t *testing.T) {
+	ctx := context.Background()
+	writeCtx := WithWriteMode(ctx)
+
+	val := writeCtx.Value(readWriteModeKey)
+	assert.Equal(t, writeMode, val)
+}
+
+func TestGetReadWriteMode(t *testing.T) {
+	t.Run("valid read-primary mode", func(t *testing.T) {
+		ctx := context.WithValue(context.Background(), readWriteModeKey, readPrimaryMode)
+		assert.Equal(t, readPrimaryMode, getReadWriteMode(ctx))
+	})
+
+	t.Run("valid read-replica mode", func(t *testing.T) {
+		ctx := context.WithValue(context.Background(), readWriteModeKey, readReplicaMode)
+		assert.Equal(t, readReplicaMode, getReadWriteMode(ctx))
+	})
+
+	t.Run("valid write mode", func(t *testing.T) {
+		ctx := context.WithValue(context.Background(), readWriteModeKey, writeMode)
+		assert.Equal(t, writeMode, getReadWriteMode(ctx))
+	})
+
+	t.Run("invalid mode value (wrong type)", func(t *testing.T) {
+		ctx := context.WithValue(context.Background(), readWriteModeKey, "not-a-mode")
+		assert.Equal(t, notSpecifiedMode, getReadWriteMode(ctx))
+	})
+
+	t.Run("invalid mode value (wrong value)", func(t *testing.T) {
+		ctx := context.WithValue(context.Background(), readWriteModeKey, readWriteMode("delete"))
+		assert.Equal(t, notSpecifiedMode, getReadWriteMode(ctx))
+	})
+
+	t.Run("no mode set", func(t *testing.T) {
+		ctx := context.Background()
+		assert.Equal(t, notSpecifiedMode, getReadWriteMode(ctx))
+	})
+}
+
+func TestUuseReplica(t *testing.T) {
+	t.Run("context with read-replica mode", func(t *testing.T) {
+		ctx := context.WithValue(context.Background(), readWriteModeKey, readReplicaMode)
+		assert.True(t, useReplica(ctx))
+	})
+
+	t.Run("context with read-primary mode", func(t *testing.T) {
+		ctx := context.WithValue(context.Background(), readWriteModeKey, readPrimaryMode)
+		assert.False(t, useReplica(ctx))
+	})
+
+	t.Run("context with write mode", func(t *testing.T) {
+		ctx := context.WithValue(context.Background(), readWriteModeKey, writeMode)
+		assert.False(t, useReplica(ctx))
+	})
+
+	t.Run("context with invalid mode", func(t *testing.T) {
+		ctx := context.WithValue(context.Background(), readWriteModeKey, readWriteMode("invalid"))
+		assert.False(t, useReplica(ctx))
+	})
+
+	t.Run("context with no mode set", func(t *testing.T) {
+		ctx := context.Background()
+		assert.False(t, useReplica(ctx))
+	})
+}

core/stores/sqlx/sqlconn.go

@@ -4,6 +4,9 @@ import (
 	"context"
 	"database/sql"
 	"errors"
+	"fmt"
+	"math/rand"
+	"sync/atomic"
 
 	"github.com/zeromicro/go-zero/core/breaker"
 	"github.com/zeromicro/go-zero/core/errorx"
@@ -52,9 +55,10 @@ type (
 		beginTx  beginnable
 		brk      breaker.Breaker
 		accept   breaker.Acceptable
+		index    uint32
 	}
 
-	connProvider func() (*sql.DB, error)
+	connProvider func(ctx context.Context) (*sql.DB, error)
 
 	sessionConn interface {
 		Exec(query string, args ...any) (sql.Result, error)
@@ -64,10 +68,31 @@ type (
 	}
 )
 
+// NewConn returns a SqlConn with the given SqlConf.
+func NewConn(c SqlConf, opts ...SqlOption) SqlConn {
+	if err := c.Validate(); err != nil {
+		logx.Must(err)
+	}
+
+	conn := &commonSqlConn{
+		onError: func(ctx context.Context, err error) {
+			logInstanceError(ctx, c.DataSource, err)
+		},
+		beginTx: begin,
+		brk:     breaker.NewBreaker(),
+	}
+	for _, opt := range opts {
+		opt(conn)
+	}
+	conn.connProv = getConnProvider(conn, c.DriverName, c.DataSource, c.Policy, c.Replicas)
+
+	return conn
+}
+
 // NewSqlConn returns a SqlConn with given driver name and datasource.
 func NewSqlConn(driverName, datasource string, opts ...SqlOption) SqlConn {
 	conn := &commonSqlConn{
-		connProv: func() (*sql.DB, error) {
+		connProv: func(context.Context) (*sql.DB, error) {
 			return getSqlConn(driverName, datasource)
 		},
 		onError: func(ctx context.Context, err error) {
@@ -87,7 +112,7 @@ func NewSqlConn(driverName, datasource string, opts ...SqlOption) SqlConn {
 // Use it with caution; it's provided for other ORM to interact with.
 func NewSqlConnFromDB(db *sql.DB, opts ...SqlOption) SqlConn {
 	conn := &commonSqlConn{
-		connProv: func() (*sql.DB, error) {
+		connProv: func(ctx context.Context) (*sql.DB, error) {
 			return db, nil
 		},
 		onError: func(ctx context.Context, err error) {
@@ -123,7 +148,7 @@ func (db *commonSqlConn) ExecCtx(ctx context.Context, q string, args ...any) (
 
 	err = db.brk.DoWithAcceptableCtx(ctx, func() error {
 		var conn *sql.DB
-		conn, err = db.connProv()
+		conn, err = db.connProv(ctx)
 		if err != nil {
 			db.onError(ctx, err)
 			return err
@@ -151,7 +176,7 @@ func (db *commonSqlConn) PrepareCtx(ctx context.Context, query string) (stmt Stm
 
 	err = db.brk.DoWithAcceptableCtx(ctx, func() error {
 		var conn *sql.DB
-		conn, err = db.connProv()
+		conn, err = db.connProv(ctx)
 		if err != nil {
 			db.onError(ctx, err)
 			return err
@@ -242,7 +267,7 @@ func (db *commonSqlConn) QueryRowsPartialCtx(ctx context.Context, v any,
 }
 
 func (db *commonSqlConn) RawDB() (*sql.DB, error) {
-	return db.connProv()
+	return db.connProv(context.Background())
 }
 
 func (db *commonSqlConn) Transact(fn func(Session) error) error {
@@ -288,7 +313,7 @@ func (db *commonSqlConn) queryRows(ctx context.Context, scanner func(*sql.Rows)
 	q string, args ...any) (err error) {
 	var scanFailed bool
 	err = db.brk.DoWithAcceptableCtx(ctx, func() error {
-		conn, err := db.connProv()
+		conn, err := db.connProv(ctx)
 		if err != nil {
 			db.onError(ctx, err)
 			return err
@@ -311,6 +336,38 @@ func (db *commonSqlConn) queryRows(ctx context.Context, scanner func(*sql.Rows)
 	return
 }
 
+func getConnProvider(sc *commonSqlConn, driverName, datasource, policy string, replicas []string) connProvider {
+	return func(ctx context.Context) (*sql.DB, error) {
+		replicaCount := len(replicas)
+
+		if replicaCount == 0 || !useReplica(ctx) {
+			return getSqlConn(driverName, datasource)
+		}
+
+		var dsn string
+
+		if replicaCount == 1 {
+			dsn = replicas[0]
+		} else {
+			if len(policy) == 0 {
+				policy = policyRoundRobin
+			}
+
+			switch policy {
+			case policyRandom:
+				dsn = replicas[rand.Intn(replicaCount)]
+			case policyRoundRobin:
+				index := atomic.AddUint32(&sc.index, 1) - 1
+				dsn = replicas[index%uint32(replicaCount)]
+			default:
+				return nil, fmt.Errorf("unknown policy: %s", policy)
+			}
+		}
+
+		return getSqlConn(driverName, dsn)
+	}
+}
+
 // WithAcceptable returns a SqlOption that setting the acceptable function.
 // acceptable is the func to check if the error can be accepted.
 func WithAcceptable(acceptable func(err error) bool) SqlOption {