Merge pull request #119 from github/row-copy-start-mark

ETA counting rows, fixed copy time on count

Author: Shlomi Noach

build.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 #
 #
-RELEASE_VERSION="1.0.6"
+RELEASE_VERSION="1.0.7"
 
 buildpath=/tmp/gh-ost
 target=gh-ost

go/base/context.go

@@ -111,6 +111,7 @@ type MigrationContext struct {
 	throttleReason            string
 	throttleMutex             *sync.Mutex
 	IsPostponingCutOver       int64
+	CountingRowsFlag          int64
 
 	OriginalTableColumns             *sql.ColumnList
 	OriginalTableUniqueKeys          [](*sql.UniqueKey)
@@ -261,11 +262,23 @@ func (this *MigrationContext) ElapsedTime() time.Duration {
 	return time.Now().Sub(this.StartTime)
 }
 
+// MarkRowCopyStartTime
+func (this *MigrationContext) MarkRowCopyStartTime() {
+	this.throttleMutex.Lock()
+	defer this.throttleMutex.Unlock()
+	this.RowCopyStartTime = time.Now()
+}
+
 // ElapsedRowCopyTime returns time since starting to copy chunks of rows
 func (this *MigrationContext) ElapsedRowCopyTime() time.Duration {
 	this.throttleMutex.Lock()
 	defer this.throttleMutex.Unlock()
 
+	if this.RowCopyStartTime.IsZero() {
+		// Row copy hasn't started yet
+		return 0
+	}
+
 	if this.RowCopyEndTime.IsZero() {
 		return time.Now().Sub(this.RowCopyStartTime)
 	}

go/logic/inspect.go

@@ -9,6 +9,7 @@ import (
 	gosql "database/sql"
 	"fmt"
 	"strings"
+	"sync/atomic"
 
 	"github.com/github/gh-ost/go/base"
 	"github.com/github/gh-ost/go/mysql"
@@ -364,13 +365,19 @@ func (this *Inspector) estimateTableRowsViaExplain() error {
 
 // CountTableRows counts exact number of rows on the original table
 func (this *Inspector) CountTableRows() error {
+	atomic.StoreInt64(&this.migrationContext.CountingRowsFlag, 1)
+	defer atomic.StoreInt64(&this.migrationContext.CountingRowsFlag, 0)
+
 	log.Infof("As instructed, I'm issuing a SELECT COUNT(*) on the table. This may take a while")
+
 	query := fmt.Sprintf(`select /* gh-ost */ count(*) as rows from %s.%s`, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
 	if err := this.db.QueryRow(query).Scan(&this.migrationContext.RowsEstimate); err != nil {
 		return err
 	}
 	this.migrationContext.UsedRowsEstimateMethod = base.CountRowsEstimate
+
 	log.Infof("Exact number of rows via COUNT: %d", this.migrationContext.RowsEstimate)
+
 	return nil
 }
 

go/logic/migrator.go

@@ -418,7 +418,7 @@ func (this *Migrator) Migrate() (err error) {
 	go this.initiateThrottler()
 	go this.executeWriteFuncs()
 	go this.iterateChunks()
-	this.migrationContext.RowCopyStartTime = time.Now()
+	this.migrationContext.MarkRowCopyStartTime()
 	go this.initiateStatus()
 
 	log.Debugf("Operating until row copy is complete")
@@ -951,7 +951,9 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) {
 
 	var etaSeconds float64 = math.MaxFloat64
 	eta := "N/A"
-	if atomic.LoadInt64(&this.migrationContext.IsPostponingCutOver) > 0 {
+	if atomic.LoadInt64(&this.migrationContext.CountingRowsFlag) > 0 {
+		eta = "counting rows"
+	} else if atomic.LoadInt64(&this.migrationContext.IsPostponingCutOver) > 0 {
 		eta = "postponing cut-over"
 	} else if isThrottled, throttleReason := this.migrationContext.IsThrottled(); isThrottled {
 		eta = fmt.Sprintf("throttled, %s", throttleReason)