Skip to content

Commit 7320fda

Browse files
dm-2dm-2
authored
Merge pull request #1190 from wangzihuacool/add-rocksdb-as-transactional-engine
add rocksdb as transactional engine
2 parents 1a131d4 + 20af3af commit 7320fda

File tree

17 files changed

+109
-24
lines changed

17 files changed

+109
-24
lines changed

‎.github/workflows/replica-tests.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ jobs:
88
runs-on: ubuntu-20.04
99
strategy:
1010
matrix:
11-
version: [mysql-5.7.25,mysql-8.0.16]
11+
version: [mysql-5.7.25,mysql-8.0.16,PerconaServer-8.0.21]
1212

1313
steps:
1414
- uses: actions/checkout@v2

‎doc/command-line-flags.md

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,18 @@ Allows `gh-ost` to connect to the MySQL servers using encrypted connections, but
246246

247247
`--ssl-key=/path/to/ssl-key.key`: SSL private key file (in PEM format).
248248

249+
### storage-engine
250+
Default is `innodb`, and `rocksdb` support is currently experimental. InnoDB and RocksDB are both transactional engines, supporting both shared and exclusive row locks.
251+
252+
But RocksDB currently lacks a few features support compared to InnoDB:
253+
- Gap Locks
254+
- Foreign Key
255+
- Generated Columns
256+
- Spatial
257+
- Geometry
258+
259+
When `--storage-engine=rocksdb`, `gh-ost` will make some changes necessary (e.g. sets isolation level to `READ_COMMITTED`) to support RocksDB.
260+
249261
### test-on-replica
250262

251263
Issue the migration on a replica; do not modify data on master. Useful for validating, testing and benchmarking. See [`testing-on-replica`](testing-on-replica.md)

‎go/base/context.go

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,19 @@ func NewMigrationContext() *MigrationContext {
290290
}
291291
}
292292

293+
func (this *MigrationContext) SetConnectionConfig(storageEngine string) error {
294+
var transactionIsolation string
295+
switch storageEngine {
296+
case "rocksdb":
297+
transactionIsolation = "READ-COMMITTED"
298+
default:
299+
transactionIsolation = "REPEATABLE-READ"
300+
}
301+
this.InspectorConnectionConfig.TransactionIsolation = transactionIsolation
302+
this.ApplierConnectionConfig.TransactionIsolation = transactionIsolation
303+
return nil
304+
}
305+
293306
func getSafeTableName(baseName string, suffix string) string {
294307
name := fmt.Sprintf("_%s_%s", baseName, suffix)
295308
if len(name) <= mysql.MaxTableNameLength {
@@ -428,6 +441,10 @@ func (this *MigrationContext) IsTransactionalTable() bool {
428441
{
429442
return true
430443
}
444+
case "rocksdb":
445+
{
446+
return true
447+
}
431448
}
432449
return false
433450
}

‎go/cmd/gh-ost/main.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ func main() {
6868
flag.StringVar(&migrationContext.OriginalTableName, "table", "", "table name (mandatory)")
6969
flag.StringVar(&migrationContext.AlterStatement, "alter", "", "alter statement (mandatory)")
7070
flag.BoolVar(&migrationContext.AttemptInstantDDL, "attempt-instant-ddl", false, "Attempt to use instant DDL for this migration first")
71+
storageEngine := flag.String("storage-engine", "innodb", "Specify table storage engine (default: 'innodb'). When 'rocksdb': the session transaction isolation level is changed from REPEATABLE_READ to READ_COMMITTED.")
7172

7273
flag.BoolVar(&migrationContext.CountTableRows, "exact-rowcount", false, "actually count table rows as opposed to estimate them (results in more accurate progress estimation)")
7374
flag.BoolVar(&migrationContext.ConcurrentCountTableRows, "concurrent-rowcount", true, "(with --exact-rowcount), when true (default): count rows after row-copy begins, concurrently, and adjust row estimate later on; when false: first count rows, then start row copy")
@@ -182,6 +183,10 @@ func main() {
182183
migrationContext.Log.SetLevel(log.ERROR)
183184
}
184185

186+
if err := migrationContext.SetConnectionConfig(*storageEngine); err != nil {
187+
migrationContext.Log.Fatale(err)
188+
}
189+
185190
if migrationContext.AlterStatement == "" {
186191
log.Fatal("--alter must be provided and statement must not be empty")
187192
}
@@ -247,6 +252,9 @@ func main() {
247252
if *replicationLagQuery != "" {
248253
migrationContext.Log.Warning("--replication-lag-query is deprecated")
249254
}
255+
if *storageEngine == "rocksdb" {
256+
migrationContext.Log.Warning("RocksDB storage engine support is experimental")
257+
}
250258

251259
switch *cutOver {
252260
case "atomic", "default", "":

‎go/mysql/connection.go

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -18,18 +18,18 @@ import (
1818
)
1919

2020
const (
21-
transactionIsolation = "REPEATABLE-READ"
22-
TLS_CONFIG_KEY = "ghost"
21+
TLS_CONFIG_KEY = "ghost"
2322
)
2423

2524
// ConnectionConfig is the minimal configuration required to connect to a MySQL server
2625
type ConnectionConfig struct {
27-
Key InstanceKey
28-
User string
29-
Password string
30-
ImpliedKey *InstanceKey
31-
tlsConfig *tls.Config
32-
Timeout float64
26+
Key InstanceKey
27+
User string
28+
Password string
29+
ImpliedKey *InstanceKey
30+
tlsConfig *tls.Config
31+
Timeout float64
32+
TransactionIsolation string
3333
}
3434

3535
func NewConnectionConfig() *ConnectionConfig {
@@ -43,11 +43,12 @@ func NewConnectionConfig() *ConnectionConfig {
4343
// DuplicateCredentials creates a new connection config with given key and with same credentials as this config
4444
func (this *ConnectionConfig) DuplicateCredentials(key InstanceKey) *ConnectionConfig {
4545
config := &ConnectionConfig{
46-
Key: key,
47-
User: this.User,
48-
Password: this.Password,
49-
tlsConfig: this.tlsConfig,
50-
Timeout: this.Timeout,
46+
Key: key,
47+
User: this.User,
48+
Password: this.Password,
49+
tlsConfig: this.tlsConfig,
50+
Timeout: this.Timeout,
51+
TransactionIsolation: this.TransactionIsolation,
5152
}
5253
config.ImpliedKey = &config.Key
5354
return config
@@ -126,7 +127,7 @@ func (this *ConnectionConfig) GetDBUri(databaseName string) string {
126127
"charset=utf8mb4,utf8,latin1",
127128
"interpolateParams=true",
128129
fmt.Sprintf("tls=%s", tlsOption),
129-
fmt.Sprintf("transaction_isolation=%q", transactionIsolation),
130+
fmt.Sprintf("transaction_isolation=%q", this.TransactionIsolation),
130131
fmt.Sprintf("timeout=%fs", this.Timeout),
131132
fmt.Sprintf("readTimeout=%fs", this.Timeout),
132133
fmt.Sprintf("writeTimeout=%fs", this.Timeout),

‎go/mysql/connection_test.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@ import (
1313
test "github.com/openark/golib/tests"
1414
)
1515

16+
const (
17+
transactionIsolation = "REPEATABLE-READ"
18+
)
19+
1620
func init() {
1721
log.SetLevel(log.ERROR)
1822
}
@@ -25,6 +29,7 @@ func TestNewConnectionConfig(t *testing.T) {
2529
test.S(t).ExpectEquals(c.ImpliedKey.Port, 0)
2630
test.S(t).ExpectEquals(c.User, "")
2731
test.S(t).ExpectEquals(c.Password, "")
32+
test.S(t).ExpectEquals(c.TransactionIsolation, "")
2833
}
2934

3035
func TestDuplicateCredentials(t *testing.T) {
@@ -36,6 +41,7 @@ func TestDuplicateCredentials(t *testing.T) {
3641
InsecureSkipVerify: true,
3742
ServerName: "feathers",
3843
}
44+
c.TransactionIsolation = transactionIsolation
3945

4046
dup := c.DuplicateCredentials(InstanceKey{Hostname: "otherhost", Port: 3310})
4147
test.S(t).ExpectEquals(dup.Key.Hostname, "otherhost")
@@ -45,13 +51,15 @@ func TestDuplicateCredentials(t *testing.T) {
4551
test.S(t).ExpectEquals(dup.User, "gromit")
4652
test.S(t).ExpectEquals(dup.Password, "penguin")
4753
test.S(t).ExpectEquals(dup.tlsConfig, c.tlsConfig)
54+
test.S(t).ExpectEquals(dup.TransactionIsolation, c.TransactionIsolation)
4855
}
4956

5057
func TestDuplicate(t *testing.T) {
5158
c := NewConnectionConfig()
5259
c.Key = InstanceKey{Hostname: "myhost", Port: 3306}
5360
c.User = "gromit"
5461
c.Password = "penguin"
62+
c.TransactionIsolation = transactionIsolation
5563

5664
dup := c.Duplicate()
5765
test.S(t).ExpectEquals(dup.Key.Hostname, "myhost")
@@ -60,6 +68,7 @@ func TestDuplicate(t *testing.T) {
6068
test.S(t).ExpectEquals(dup.ImpliedKey.Port, 3306)
6169
test.S(t).ExpectEquals(dup.User, "gromit")
6270
test.S(t).ExpectEquals(dup.Password, "penguin")
71+
test.S(t).ExpectEquals(dup.TransactionIsolation, transactionIsolation)
6372
}
6473

6574
func TestGetDBUri(t *testing.T) {
@@ -68,6 +77,7 @@ func TestGetDBUri(t *testing.T) {
6877
c.User = "gromit"
6978
c.Password = "penguin"
7079
c.Timeout = 1.2345
80+
c.TransactionIsolation = transactionIsolation
7181

7282
uri := c.GetDBUri("test")
7383
test.S(t).ExpectEquals(uri, `gromit:penguin@tcp(myhost:3306)/test?autocommit=true&charset=utf8mb4,utf8,latin1&interpolateParams=true&tls=false&transaction_isolation="REPEATABLE-READ"&timeout=1.234500s&readTimeout=1.234500s&writeTimeout=1.234500s`)
@@ -80,6 +90,7 @@ func TestGetDBUriWithTLSSetup(t *testing.T) {
8090
c.Password = "penguin"
8191
c.Timeout = 1.2345
8292
c.tlsConfig = &tls.Config{}
93+
c.TransactionIsolation = transactionIsolation
8394

8495
uri := c.GetDBUri("test")
8596
test.S(t).ExpectEquals(uri, `gromit:penguin@tcp(myhost:3306)/test?autocommit=true&charset=utf8mb4,utf8,latin1&interpolateParams=true&tls=ghost&transaction_isolation="REPEATABLE-READ"&timeout=1.234500s&readTimeout=1.234500s&writeTimeout=1.234500s`)

‎localtests/discard-fk/ignore_versions

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Percona

‎localtests/fail-fk-parent/ignore_versions

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Percona

‎localtests/fail-fk/ignore_versions

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Percona

‎localtests/generated-columns-add/ignore_versions

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Percona

0 commit comments

Comments
 (0)