Skip to content

Commit 7f0254f

Browse files
timvaillancourttimvaillancourt
authored
Merge branch 'master' into update-go-sql-driver-2020-02
2 parents c9db407 + f334dbd commit 7f0254f

File tree

13 files changed

+356
-237
lines changed

13 files changed

+356
-237
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,4 @@
22
/bin/
33
/libexec/
44
/.vendor/
5+
.idea/

‎go/base/context.go

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919

2020
"github.com/github/gh-ost/go/mysql"
2121
"github.com/github/gh-ost/go/sql"
22+
"github.com/outbrain/golib/log"
2223

2324
"gopkg.in/gcfg.v1"
2425
gcfgscanner "gopkg.in/gcfg.v1/scanner"
@@ -119,6 +120,7 @@ type MigrationContext struct {
119120
ThrottleAdditionalFlagFile string
120121
throttleQuery string
121122
throttleHTTP string
123+
IgnoreHTTPErrors bool
122124
ThrottleCommandedByUser int64
123125
HibernateUntil int64
124126
maxLoad LoadMap
@@ -216,6 +218,25 @@ type MigrationContext struct {
216218
ForceTmpTableName string
217219

218220
recentBinlogCoordinates mysql.BinlogCoordinates
221+
222+
Log Logger
223+
}
224+
225+
type Logger interface {
226+
Debug(args ...interface{})
227+
Debugf(format string, args ...interface{})
228+
Info(args ...interface{})
229+
Infof(format string, args ...interface{})
230+
Warning(args ...interface{}) error
231+
Warningf(format string, args ...interface{}) error
232+
Error(args ...interface{}) error
233+
Errorf(format string, args ...interface{}) error
234+
Errore(err error) error
235+
Fatal(args ...interface{}) error
236+
Fatalf(format string, args ...interface{}) error
237+
Fatale(err error) error
238+
SetLevel(level log.LogLevel)
239+
SetPrintStackTrace(printStackTraceFlag bool)
219240
}
220241

221242
type ContextConfig struct {
@@ -250,6 +271,7 @@ func NewMigrationContext() *MigrationContext {
250271
pointOfInterestTimeMutex: &sync.Mutex{},
251272
ColumnRenameMap: make(map[string]string),
252273
PanicAbort: make(chan error),
274+
Log: NewDefaultLogger(),
253275
}
254276
}
255277

@@ -574,6 +596,13 @@ func (this *MigrationContext) SetThrottleHTTP(throttleHTTP string) {
574596
this.throttleHTTP = throttleHTTP
575597
}
576598

599+
func (this *MigrationContext) SetIgnoreHTTPErrors(ignoreHTTPErrors bool) {
600+
this.throttleHTTPMutex.Lock()
601+
defer this.throttleHTTPMutex.Unlock()
602+
603+
this.IgnoreHTTPErrors = ignoreHTTPErrors
604+
}
605+
577606
func (this *MigrationContext) GetMaxLoad() LoadMap {
578607
this.throttleMutex.Lock()
579608
defer this.throttleMutex.Unlock()

‎go/base/default_logger.go

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
package base
2+
3+
import (
4+
"github.com/outbrain/golib/log"
5+
)
6+
7+
type simpleLogger struct{}
8+
9+
func NewDefaultLogger() *simpleLogger {
10+
return &simpleLogger{}
11+
}
12+
13+
func (*simpleLogger) Debug(args ...interface{}) {
14+
log.Debug(args[0].(string), args[1:])
15+
return
16+
}
17+
18+
func (*simpleLogger) Debugf(format string, args ...interface{}) {
19+
log.Debugf(format, args...)
20+
return
21+
}
22+
23+
func (*simpleLogger) Info(args ...interface{}) {
24+
log.Info(args[0].(string), args[1:])
25+
return
26+
}
27+
28+
func (*simpleLogger) Infof(format string, args ...interface{}) {
29+
log.Infof(format, args...)
30+
return
31+
}
32+
33+
func (*simpleLogger) Warning(args ...interface{}) error {
34+
return log.Warning(args[0].(string), args[1:])
35+
}
36+
37+
func (*simpleLogger) Warningf(format string, args ...interface{}) error {
38+
return log.Warningf(format, args...)
39+
}
40+
41+
func (*simpleLogger) Error(args ...interface{}) error {
42+
return log.Error(args[0].(string), args[1:])
43+
}
44+
45+
func (*simpleLogger) Errorf(format string, args ...interface{}) error {
46+
return log.Errorf(format, args...)
47+
}
48+
49+
func (*simpleLogger) Errore(err error) error {
50+
return log.Errore(err)
51+
}
52+
53+
func (*simpleLogger) Fatal(args ...interface{}) error {
54+
return log.Fatal(args[0].(string), args[1:])
55+
}
56+
57+
func (*simpleLogger) Fatalf(format string, args ...interface{}) error {
58+
return log.Fatalf(format, args...)
59+
}
60+
61+
func (*simpleLogger) Fatale(err error) error {
62+
return log.Fatale(err)
63+
}
64+
65+
func (*simpleLogger) SetLevel(level log.LogLevel) {
66+
log.SetLevel(level)
67+
return
68+
}
69+
70+
func (*simpleLogger) SetPrintStackTrace(printStackTraceFlag bool) {
71+
log.SetPrintStackTrace(printStackTraceFlag)
72+
return
73+
}

‎go/base/utils.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ import (
1414

1515
gosql "database/sql"
1616
"github.com/github/gh-ost/go/mysql"
17-
"github.com/outbrain/golib/log"
1817
)
1918

2019
var (
@@ -86,7 +85,7 @@ func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig,
8685
}
8786

8887
if connectionConfig.Key.Port == port || (extraPort > 0 && connectionConfig.Key.Port == extraPort) {
89-
log.Infof("connection validated on %+v", connectionConfig.Key)
88+
migrationContext.Log.Infof("connection validated on %+v", connectionConfig.Key)
9089
return version, nil
9190
} else if extraPort == 0 {
9291
return "", fmt.Errorf("Unexpected database port reported: %+v", port)

‎go/binlog/gomysql_reader.go

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,13 @@ import (
1313
"github.com/github/gh-ost/go/mysql"
1414
"github.com/github/gh-ost/go/sql"
1515

16-
"github.com/outbrain/golib/log"
1716
gomysql "github.com/siddontang/go-mysql/mysql"
1817
"github.com/siddontang/go-mysql/replication"
1918
"golang.org/x/net/context"
2019
)
2120

2221
type GoMySQLReader struct {
22+
migrationContext *base.MigrationContext
2323
connectionConfig *mysql.ConnectionConfig
2424
binlogSyncer *replication.BinlogSyncer
2525
binlogStreamer *replication.BinlogStreamer
@@ -30,6 +30,7 @@ type GoMySQLReader struct {
3030

3131
func NewGoMySQLReader(migrationContext *base.MigrationContext) (binlogReader *GoMySQLReader, err error) {
3232
binlogReader = &GoMySQLReader{
33+
migrationContext: migrationContext,
3334
connectionConfig: migrationContext.InspectorConnectionConfig,
3435
currentCoordinates: mysql.BinlogCoordinates{},
3536
currentCoordinatesMutex: &sync.Mutex{},
@@ -57,11 +58,11 @@ func NewGoMySQLReader(migrationContext *base.MigrationContext) (binlogReader *Go
5758
// ConnectBinlogStreamer
5859
func (this *GoMySQLReader) ConnectBinlogStreamer(coordinates mysql.BinlogCoordinates) (err error) {
5960
if coordinates.IsEmpty() {
60-
return log.Errorf("Empty coordinates at ConnectBinlogStreamer()")
61+
return this.migrationContext.Log.Errorf("Empty coordinates at ConnectBinlogStreamer()")
6162
}
6263

6364
this.currentCoordinates = coordinates
64-
log.Infof("Connecting binlog streamer at %+v", this.currentCoordinates)
65+
this.migrationContext.Log.Infof("Connecting binlog streamer at %+v", this.currentCoordinates)
6566
// Start sync with specified binlog file and position
6667
this.binlogStreamer, err = this.binlogSyncer.StartSync(gomysql.Position{this.currentCoordinates.LogFile, uint32(this.currentCoordinates.LogPos)})
6768

@@ -78,7 +79,7 @@ func (this *GoMySQLReader) GetCurrentBinlogCoordinates() *mysql.BinlogCoordinate
7879
// StreamEvents
7980
func (this *GoMySQLReader) handleRowsEvent(ev *replication.BinlogEvent, rowsEvent *replication.RowsEvent, entriesChannel chan<- *BinlogEntry) error {
8081
if this.currentCoordinates.SmallerThanOrEquals(&this.LastAppliedRowsEventHint) {
81-
log.Debugf("Skipping handled query at %+v", this.currentCoordinates)
82+
this.migrationContext.Log.Debugf("Skipping handled query at %+v", this.currentCoordinates)
8283
return nil
8384
}
8485

@@ -147,14 +148,14 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha
147148
defer this.currentCoordinatesMutex.Unlock()
148149
this.currentCoordinates.LogFile = string(rotateEvent.NextLogName)
149150
}()
150-
log.Infof("rotate to next log from %s:%d to %s", this.currentCoordinates.LogFile, int64(ev.Header.LogPos), rotateEvent.NextLogName)
151+
this.migrationContext.Log.Infof("rotate to next log from %s:%d to %s", this.currentCoordinates.LogFile, int64(ev.Header.LogPos), rotateEvent.NextLogName)
151152
} else if rowsEvent, ok := ev.Event.(*replication.RowsEvent); ok {
152153
if err := this.handleRowsEvent(ev, rowsEvent, entriesChannel); err != nil {
153154
return err
154155
}
155156
}
156157
}
157-
log.Debugf("done streaming events")
158+
this.migrationContext.Log.Debugf("done streaming events")
158159

159160
return nil
160161
}

‎go/cmd/gh-ost/main.go

Lines changed: 34 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ func acceptSignals(migrationContext *base.MigrationContext) {
3131
for sig := range c {
3232
switch sig {
3333
case syscall.SIGHUP:
34-
log.Infof("Received SIGHUP. Reloading configuration")
34+
migrationContext.Log.Infof("Received SIGHUP. Reloading configuration")
3535
if err := migrationContext.ReadConfigFile(); err != nil {
3636
log.Errore(err)
3737
} else {
@@ -106,6 +106,7 @@ func main() {
106106
throttleControlReplicas := flag.String("throttle-control-replicas", "", "List of replicas on which to check for lag; comma delimited. Example: myhost1.com:3306,myhost2.com,myhost3.com:3307")
107107
throttleQuery := flag.String("throttle-query", "", "when given, issued (every second) to check if operation should throttle. Expecting to return zero for no-throttle, >0 for throttle. Query is issued on the migrated server. Make sure this query is lightweight")
108108
throttleHTTP := flag.String("throttle-http", "", "when given, gh-ost checks given URL via HEAD request; any response code other than 200 (OK) causes throttling; make sure it has low latency response")
109+
ignoreHTTPErrors := flag.Bool("ignore-http-errors", false, "ignore HTTP connection errors during throttle check")
109110
heartbeatIntervalMillis := flag.Int64("heartbeat-interval-millis", 100, "how frequently would gh-ost inject a heartbeat value")
110111
flag.StringVar(&migrationContext.ThrottleFlagFile, "throttle-flag-file", "", "operation pauses when this file exists; hint: use a file that is specific to the table being altered")
111112
flag.StringVar(&migrationContext.ThrottleAdditionalFlagFile, "throttle-additional-flag-file", "/tmp/gh-ost.throttle", "operation pauses when this file exists; hint: keep default, use for throttling multiple gh-ost operations")
@@ -156,69 +157,69 @@ func main() {
156157
return
157158
}
158159

159-
log.SetLevel(log.ERROR)
160+
migrationContext.Log.SetLevel(log.ERROR)
160161
if *verbose {
161-
log.SetLevel(log.INFO)
162+
migrationContext.Log.SetLevel(log.INFO)
162163
}
163164
if *debug {
164-
log.SetLevel(log.DEBUG)
165+
migrationContext.Log.SetLevel(log.DEBUG)
165166
}
166167
if *stack {
167-
log.SetPrintStackTrace(*stack)
168+
migrationContext.Log.SetPrintStackTrace(*stack)
168169
}
169170
if *quiet {
170171
// Override!!
171-
log.SetLevel(log.ERROR)
172+
migrationContext.Log.SetLevel(log.ERROR)
172173
}
173174

174175
if migrationContext.DatabaseName == "" {
175-
log.Fatalf("--database must be provided and database name must not be empty")
176+
migrationContext.Log.Fatalf("--database must be provided and database name must not be empty")
176177
}
177178
if migrationContext.OriginalTableName == "" {
178-
log.Fatalf("--table must be provided and table name must not be empty")
179+
migrationContext.Log.Fatalf("--table must be provided and table name must not be empty")
179180
}
180181
if migrationContext.AlterStatement == "" {
181-
log.Fatalf("--alter must be provided and statement must not be empty")
182+
migrationContext.Log.Fatalf("--alter must be provided and statement must not be empty")
182183
}
183184
migrationContext.Noop = !(*executeFlag)
184185
if migrationContext.AllowedRunningOnMaster && migrationContext.TestOnReplica {
185-
log.Fatalf("--allow-on-master and --test-on-replica are mutually exclusive")
186+
migrationContext.Log.Fatalf("--allow-on-master and --test-on-replica are mutually exclusive")
186187
}
187188
if migrationContext.AllowedRunningOnMaster && migrationContext.MigrateOnReplica {
188-
log.Fatalf("--allow-on-master and --migrate-on-replica are mutually exclusive")
189+
migrationContext.Log.Fatalf("--allow-on-master and --migrate-on-replica are mutually exclusive")
189190
}
190191
if migrationContext.MigrateOnReplica && migrationContext.TestOnReplica {
191-
log.Fatalf("--migrate-on-replica and --test-on-replica are mutually exclusive")
192+
migrationContext.Log.Fatalf("--migrate-on-replica and --test-on-replica are mutually exclusive")
192193
}
193194
if migrationContext.SwitchToRowBinlogFormat && migrationContext.AssumeRBR {
194-
log.Fatalf("--switch-to-rbr and --assume-rbr are mutually exclusive")
195+
migrationContext.Log.Fatalf("--switch-to-rbr and --assume-rbr are mutually exclusive")
195196
}
196197
if migrationContext.TestOnReplicaSkipReplicaStop {
197198
if !migrationContext.TestOnReplica {
198-
log.Fatalf("--test-on-replica-skip-replica-stop requires --test-on-replica to be enabled")
199+
migrationContext.Log.Fatalf("--test-on-replica-skip-replica-stop requires --test-on-replica to be enabled")
199200
}
200-
log.Warning("--test-on-replica-skip-replica-stop enabled. We will not stop replication before cut-over. Ensure you have a plugin that does this.")
201+
migrationContext.Log.Warning("--test-on-replica-skip-replica-stop enabled. We will not stop replication before cut-over. Ensure you have a plugin that does this.")
201202
}
202203
if migrationContext.CliMasterUser != "" && migrationContext.AssumeMasterHostname == "" {
203-
log.Fatalf("--master-user requires --assume-master-host")
204+
migrationContext.Log.Fatalf("--master-user requires --assume-master-host")
204205
}
205206
if migrationContext.CliMasterPassword != "" && migrationContext.AssumeMasterHostname == "" {
206-
log.Fatalf("--master-password requires --assume-master-host")
207+
migrationContext.Log.Fatalf("--master-password requires --assume-master-host")
207208
}
208209
if migrationContext.TLSCACertificate != "" && !migrationContext.UseTLS {
209-
log.Fatalf("--ssl-ca requires --ssl")
210+
migrationContext.Log.Fatalf("--ssl-ca requires --ssl")
210211
}
211212
if migrationContext.TLSCertificate != "" && !migrationContext.UseTLS {
212-
log.Fatalf("--ssl-cert requires --ssl")
213+
migrationContext.Log.Fatalf("--ssl-cert requires --ssl")
213214
}
214215
if migrationContext.TLSKey != "" && !migrationContext.UseTLS {
215-
log.Fatalf("--ssl-key requires --ssl")
216+
migrationContext.Log.Fatalf("--ssl-key requires --ssl")
216217
}
217218
if migrationContext.TLSAllowInsecure && !migrationContext.UseTLS {
218-
log.Fatalf("--ssl-allow-insecure requires --ssl")
219+
migrationContext.Log.Fatalf("--ssl-allow-insecure requires --ssl")
219220
}
220221
if *replicationLagQuery != "" {
221-
log.Warningf("--replication-lag-query is deprecated")
222+
migrationContext.Log.Warningf("--replication-lag-query is deprecated")
222223
}
223224

224225
switch *cutOver {
@@ -227,19 +228,19 @@ func main() {
227228
case "two-step":
228229
migrationContext.CutOverType = base.CutOverTwoStep
229230
default:
230-
log.Fatalf("Unknown cut-over: %s", *cutOver)
231+
migrationContext.Log.Fatalf("Unknown cut-over: %s", *cutOver)
231232
}
232233
if err := migrationContext.ReadConfigFile(); err != nil {
233-
log.Fatale(err)
234+
migrationContext.Log.Fatale(err)
234235
}
235236
if err := migrationContext.ReadThrottleControlReplicaKeys(*throttleControlReplicas); err != nil {
236-
log.Fatale(err)
237+
migrationContext.Log.Fatale(err)
237238
}
238239
if err := migrationContext.ReadMaxLoad(*maxLoad); err != nil {
239-
log.Fatale(err)
240+
migrationContext.Log.Fatale(err)
240241
}
241242
if err := migrationContext.ReadCriticalLoad(*criticalLoad); err != nil {
242-
log.Fatale(err)
243+
migrationContext.Log.Fatale(err)
243244
}
244245
if migrationContext.ServeSocketFile == "" {
245246
migrationContext.ServeSocketFile = fmt.Sprintf("/tmp/gh-ost.%s.%s.sock", migrationContext.DatabaseName, migrationContext.OriginalTableName)
@@ -248,7 +249,7 @@ func main() {
248249
fmt.Println("Password:")
249250
bytePassword, err := terminal.ReadPassword(int(syscall.Stdin))
250251
if err != nil {
251-
log.Fatale(err)
252+
migrationContext.Log.Fatale(err)
252253
}
253254
migrationContext.CliPassword = string(bytePassword)
254255
}
@@ -259,16 +260,17 @@ func main() {
259260
migrationContext.SetMaxLagMillisecondsThrottleThreshold(*maxLagMillis)
260261
migrationContext.SetThrottleQuery(*throttleQuery)
261262
migrationContext.SetThrottleHTTP(*throttleHTTP)
263+
migrationContext.SetIgnoreHTTPErrors(*ignoreHTTPErrors)
262264
migrationContext.SetDefaultNumRetries(*defaultRetries)
263265
migrationContext.ApplyCredentials()
264266
if err := migrationContext.SetupTLS(); err != nil {
265-
log.Fatale(err)
267+
migrationContext.Log.Fatale(err)
266268
}
267269
if err := migrationContext.SetCutOverLockTimeoutSeconds(*cutOverLockTimeoutSeconds); err != nil {
268-
log.Errore(err)
270+
migrationContext.Log.Errore(err)
269271
}
270272
if err := migrationContext.SetExponentialBackoffMaxInterval(*exponentialBackoffMaxInterval); err != nil {
271-
log.Errore(err)
273+
migrationContext.Log.Errore(err)
272274
}
273275

274276
log.Infof("starting gh-ost %+v", AppVersion)
@@ -278,7 +280,7 @@ func main() {
278280
err := migrator.Migrate()
279281
if err != nil {
280282
migrator.ExecOnFailureHook()
281-
log.Fatale(err)
283+
migrationContext.Log.Fatale(err)
282284
}
283285
fmt.Fprintf(os.Stdout, "# Done\n")
284286
}

0 commit comments

Comments
 (0)