Merge branch 'master' into connection-timeout

Author: timvaillancourt

.github/workflows/ci.yml

@@ -8,13 +8,18 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@master
+    - uses: actions/checkout@v2
 
-    - name: Set up Go 1.12
+    - name: Set up Go 1.14
       uses: actions/setup-go@v1
       with:
-        version: 1.12
-      id: go
+        go-version: 1.14
 
     - name: Build
       run: script/cibuild
+
+    - name: Upload gh-ost binary artifact
+      uses: actions/upload-artifact@v1
+      with:
+        name: gh-ost
+        path: bin/gh-ost

.github/workflows/replica-tests.yml

@@ -8,13 +8,12 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@master
+    - uses: actions/checkout@v2
 
-    - name: Set up Go 1.12
+    - name: Set up Go 1.14
       uses: actions/setup-go@v1
       with:
-        version: 1.12
-      id: go
+        go-version: 1.14
 
     - name: migration tests
       run:  script/cibuild-gh-ost-replica-tests

.gitignore

@@ -2,3 +2,4 @@
 /bin/
 /libexec/
 /.vendor/
+.idea/

Dockerfile.packaging

@@ -1,6 +1,6 @@
 #
 
-FROM golang:1.12.6
+FROM golang:1.14.7
 
 RUN apt-get update
 RUN apt-get install -y ruby ruby-dev rubygems build-essential

Dockerfile.test

@@ -1,4 +1,4 @@
-FROM golang:1.12.1
+FROM golang:1.14.7
 LABEL maintainer="[email protected]"
 
 RUN apt-get update

README.md

@@ -94,7 +94,7 @@ Please see [Coding gh-ost](doc/coding-ghost.md) for a guide to getting started d
 
 [Download latest release here](https://github.com/github/gh-ost/releases/latest)
 
-`gh-ost` is a Go project; it is built with Go `1.12` and above. To build on your own, use either:
+`gh-ost` is a Go project; it is built with Go `1.14` and above. To build on your own, use either:
 - [script/build](https://github.com/github/gh-ost/blob/master/script/build) - this is the same build script used by CI hence the authoritative; artifact is `./bin/gh-ost` binary.
 - [build.sh](https://github.com/github/gh-ost/blob/master/build.sh) for building `tar.gz` artifacts in `/tmp/gh-ost`
 
@@ -109,3 +109,4 @@ Generally speaking, `master` branch is stable, but only [releases](https://githu
 - [@shlomi-noach](https://github.com/shlomi-noach)
 - [@jessbreckenridge](https://github.com/jessbreckenridge)
 - [@gtowey](https://github.com/gtowey)
+- [@timvaillancourt](https://github.com/timvaillancourt)

build.sh

@@ -18,8 +18,8 @@ function build {
   GOOS=$3
   GOARCH=$4
 
-  if ! go version | egrep -q 'go(1\.1[234])' ; then
-    echo "go version must be 1.12 or above"
+  if ! go version | egrep -q 'go(1\.1[456])' ; then
+    echo "go version must be 1.14 or above"
     exit 1
   fi
 

go/base/context.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/github/gh-ost/go/mysql"
 	"github.com/github/gh-ost/go/sql"
+	"github.com/outbrain/golib/log"
 
 	"gopkg.in/gcfg.v1"
 	gcfgscanner "gopkg.in/gcfg.v1/scanner"
@@ -119,6 +120,7 @@ type MigrationContext struct {
 	ThrottleAdditionalFlagFile          string
 	throttleQuery                       string
 	throttleHTTP                        string
+	IgnoreHTTPErrors                    bool
 	ThrottleCommandedByUser             int64
 	HibernateUntil                      int64
 	maxLoad                             LoadMap
@@ -216,6 +218,25 @@ type MigrationContext struct {
 	ForceTmpTableName                string
 
 	recentBinlogCoordinates mysql.BinlogCoordinates
+
+	Log Logger
+}
+
+type Logger interface {
+	Debug(args ...interface{})
+	Debugf(format string, args ...interface{})
+	Info(args ...interface{})
+	Infof(format string, args ...interface{})
+	Warning(args ...interface{}) error
+	Warningf(format string, args ...interface{}) error
+	Error(args ...interface{}) error
+	Errorf(format string, args ...interface{}) error
+	Errore(err error) error
+	Fatal(args ...interface{}) error
+	Fatalf(format string, args ...interface{}) error
+	Fatale(err error) error
+	SetLevel(level log.LogLevel)
+	SetPrintStackTrace(printStackTraceFlag bool)
 }
 
 type ContextConfig struct {
@@ -250,6 +271,7 @@ func NewMigrationContext() *MigrationContext {
 		pointOfInterestTimeMutex:            &sync.Mutex{},
 		ColumnRenameMap:                     make(map[string]string),
 		PanicAbort:                          make(chan error),
+		Log:                                 NewDefaultLogger(),
 	}
 }
 
@@ -574,6 +596,13 @@ func (this *MigrationContext) SetThrottleHTTP(throttleHTTP string) {
 	this.throttleHTTP = throttleHTTP
 }
 
+func (this *MigrationContext) SetIgnoreHTTPErrors(ignoreHTTPErrors bool) {
+	this.throttleHTTPMutex.Lock()
+	defer this.throttleHTTPMutex.Unlock()
+
+	this.IgnoreHTTPErrors = ignoreHTTPErrors
+}
+
 func (this *MigrationContext) GetMaxLoad() LoadMap {
 	this.throttleMutex.Lock()
 	defer this.throttleMutex.Unlock()

go/base/default_logger.go

@@ -0,0 +1,73 @@
+package base
+
+import (
+	"github.com/outbrain/golib/log"
+)
+
+type simpleLogger struct{}
+
+func NewDefaultLogger() *simpleLogger {
+	return &simpleLogger{}
+}
+
+func (*simpleLogger) Debug(args ...interface{}) {
+	log.Debug(args[0].(string), args[1:])
+	return
+}
+
+func (*simpleLogger) Debugf(format string, args ...interface{}) {
+	log.Debugf(format, args...)
+	return
+}
+
+func (*simpleLogger) Info(args ...interface{}) {
+	log.Info(args[0].(string), args[1:])
+	return
+}
+
+func (*simpleLogger) Infof(format string, args ...interface{}) {
+	log.Infof(format, args...)
+	return
+}
+
+func (*simpleLogger) Warning(args ...interface{}) error {
+	return log.Warning(args[0].(string), args[1:])
+}
+
+func (*simpleLogger) Warningf(format string, args ...interface{}) error {
+	return log.Warningf(format, args...)
+}
+
+func (*simpleLogger) Error(args ...interface{}) error {
+	return log.Error(args[0].(string), args[1:])
+}
+
+func (*simpleLogger) Errorf(format string, args ...interface{}) error {
+	return log.Errorf(format, args...)
+}
+
+func (*simpleLogger) Errore(err error) error {
+	return log.Errore(err)
+}
+
+func (*simpleLogger) Fatal(args ...interface{}) error {
+	return log.Fatal(args[0].(string), args[1:])
+}
+
+func (*simpleLogger) Fatalf(format string, args ...interface{}) error {
+	return log.Fatalf(format, args...)
+}
+
+func (*simpleLogger) Fatale(err error) error {
+	return log.Fatale(err)
+}
+
+func (*simpleLogger) SetLevel(level log.LogLevel) {
+	log.SetLevel(level)
+	return
+}
+
+func (*simpleLogger) SetPrintStackTrace(printStackTraceFlag bool) {
+	log.SetPrintStackTrace(printStackTraceFlag)
+	return
+}

go/base/utils.go

@@ -14,7 +14,6 @@ import (
 
 	gosql "database/sql"
 	"github.com/github/gh-ost/go/mysql"
-	"github.com/outbrain/golib/log"
 )
 
 var (
@@ -86,7 +85,7 @@ func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig,
 	}
 
 	if connectionConfig.Key.Port == port || (extraPort > 0 && connectionConfig.Key.Port == extraPort) {
-		log.Infof("connection validated on %+v", connectionConfig.Key)
+		migrationContext.Log.Infof("connection validated on %+v", connectionConfig.Key)
 		return version, nil
 	} else if extraPort == 0 {
 		return "", fmt.Errorf("Unexpected database port reported: %+v", port)