Skip to content

Commit bb5c4f9

Browse files
BrandonRoehlBrandonRoehl
committed
Data streams
1 parent c7cef00 commit bb5c4f9

File tree

3 files changed

+103
-102
lines changed

3 files changed

+103
-102
lines changed

‎dump.go

Lines changed: 50 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ import (
77
"io"
88
"reflect"
99
"strings"
10-
"sync"
1110
"text/template"
1211
"time"
1312
)
@@ -27,8 +26,6 @@ type Data struct {
2726
headerTmpl *template.Template
2827
tableTmpl *template.Template
2928
footerTmpl *template.Template
30-
mux sync.Mutex
31-
wg sync.WaitGroup
3229
err error
3330
}
3431

@@ -69,7 +66,7 @@ const headerTmpl = `-- Go SQL Dump {{ .DumpVersion }}
6966

7067
const tableTmpl = `
7168
--
72-
-- Table structure for table {{ .Name }}
69+
-- Table structure for table {{ .NameEsc }}
7370
--
7471
7572
DROP TABLE IF EXISTS {{ .NameEsc }};
@@ -79,18 +76,16 @@ DROP TABLE IF EXISTS {{ .NameEsc }};
7976
/*!40101 SET character_set_client = @saved_cs_client */;
8077
8178
--
82-
-- Dumping data for table {{ .Name }}
79+
-- Dumping data for table {{ .NameEsc }}
8380
--
8481
8582
LOCK TABLES {{ .NameEsc }} WRITE;
86-
/*!40000 ALTER TABLE {{ .Name }} DISABLE KEYS */;
83+
/*!40000 ALTER TABLE {{ .NameEsc }} DISABLE KEYS */;
8784
{{- if .Next }}
88-
INSERT INTO {{ .Name }} VALUES {{ .RowValues }}
89-
{{- range .Next -}}
90-
, {{ .RowValues }}
91-
{{- end -}};
85+
INSERT INTO {{ .NameEsc }} VALUES {{ .RowValues }}
86+
{{- range $value := .Stream }},{{ $value }}{{ end -}};
9287
{{- end }}
93-
/*!40000 ALTER TABLE {{ .Name }} ENABLE KEYS */;
88+
/*!40000 ALTER TABLE {{ .NameEsc }} ENABLE KEYS */;
9489
UNLOCK TABLES;
9590
`
9691

@@ -132,13 +127,11 @@ func (data *Data) Dump() error {
132127
return err
133128
}
134129

135-
data.wg.Add(len(tables))
136130
for _, name := range tables {
137131
if err := data.dumpTable(name); err != nil {
138132
return err
139133
}
140134
}
141-
data.wg.Wait()
142135
if data.err != nil {
143136
return data.err
144137
}
@@ -160,24 +153,14 @@ func (data *Data) dumpTable(name string) error {
160153
return err
161154
}
162155

163-
go data.writeTable(table)
164-
return nil
156+
return data.writeTable(table)
165157
}
166158

167-
func (data *Data) writeTable(table *table) {
168-
// Keep a counter of how many tables have been written
169-
defer data.wg.Done()
170-
171-
// Force this method into serial
172-
data.mux.Lock()
173-
defer data.mux.Unlock()
174-
175-
if data.err != nil {
176-
return
177-
} else if err := data.tableTmpl.Execute(data.Out, table); err != nil {
178-
data.err = err
159+
func (data *Data) writeTable(table *table) error {
160+
if err := data.tableTmpl.Execute(data.Out, table); err != nil {
161+
return err
179162
}
180-
return
163+
return table.Err
181164
}
182165

183166
// MARK: get methods
@@ -255,11 +238,10 @@ func (table *table) NameEsc() string {
255238

256239
func (table *table) CreateSQL() (string, error) {
257240
var tableReturn, tableSQL sql.NullString
258-
err := table.data.Connection.QueryRow("SHOW CREATE TABLE "+table.NameEsc()).Scan(&tableReturn, &tableSQL)
259-
260-
if err != nil {
241+
if err := table.data.Connection.QueryRow("SHOW CREATE TABLE "+table.NameEsc()).Scan(&tableReturn, &tableSQL); err != nil {
261242
return "", err
262243
}
244+
263245
if tableReturn.String != table.Name {
264246
return "", errors.New("Returned table is not the same as requested table")
265247
}
@@ -325,6 +307,9 @@ func (table *table) Next() bool {
325307
if err := table.rows.Scan(table.values...); err != nil {
326308
table.Err = err
327309
return false
310+
} else if err := table.rows.Err(); err != nil {
311+
table.Err = err
312+
return false
328313
}
329314
} else {
330315
table.rows.Close()
@@ -334,37 +319,46 @@ func (table *table) Next() bool {
334319
return true
335320
}
336321

337-
func (table *table) RowValues() (string, error) {
322+
func (table *table) RowValues() string {
338323
dataStrings := make([]string, len(table.values))
339324

340325
for key, value := range table.values {
341-
if value == nil {
326+
switch s := value.(type) {
327+
case nil:
342328
dataStrings[key] = nullType
343-
} else {
344-
switch s := value.(type) {
345-
case *sql.NullString:
346-
if s.Valid {
347-
dataStrings[key] = "'" + sanitize(s.String) + "'"
348-
} else {
349-
dataStrings[key] = nullType
350-
}
351-
case *sql.NullInt64:
352-
if s.Valid {
353-
dataStrings[key] = fmt.Sprintf("%d", s.Int64)
354-
} else {
355-
dataStrings[key] = nullType
356-
}
357-
case *sql.RawBytes:
358-
if len(*s) == 0 {
359-
dataStrings[key] = nullType
360-
} else {
361-
dataStrings[key] = "_binary '" + sanitize(string(*s)) + "'"
362-
}
363-
default:
364-
dataStrings[key] = fmt.Sprint("'", value, "'")
329+
case *sql.NullString:
330+
if s.Valid {
331+
dataStrings[key] = "'" + sanitize(s.String) + "'"
332+
} else {
333+
dataStrings[key] = nullType
334+
}
335+
case *sql.NullInt64:
336+
if s.Valid {
337+
dataStrings[key] = fmt.Sprintf("%d", s.Int64)
338+
} else {
339+
dataStrings[key] = nullType
365340
}
341+
case *sql.RawBytes:
342+
if len(*s) == 0 {
343+
dataStrings[key] = nullType
344+
} else {
345+
dataStrings[key] = "_binary '" + sanitize(string(*s)) + "'"
346+
}
347+
default:
348+
dataStrings[key] = fmt.Sprint("'", value, "'")
366349
}
367350
}
368351

369-
return "(" + strings.Join(dataStrings, ",") + ")", table.rows.Err()
352+
return "(" + strings.Join(dataStrings, ",") + ")"
353+
}
354+
355+
func (table *table) Stream() <-chan string {
356+
valueOut := make(chan string, 1)
357+
go func(out chan string) {
358+
defer close(out)
359+
for table.Next() {
360+
out <- table.RowValues()
361+
}
362+
}(valueOut)
363+
return valueOut
370364
}

‎dump_test.go

Lines changed: 40 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
package mysqldump
22

33
import (
4+
"bytes"
45
"reflect"
6+
"strings"
57
"testing"
68

79
sqlmock "github.com/DATA-DOG/go-sqlmock"
@@ -153,9 +155,9 @@ func TestCreateTableRowValues(t *testing.T) {
153155
assert.NoError(t, err)
154156

155157
assert.True(t, table.Next())
156-
// TODO
157-
result, err := table.RowValues()
158-
assert.NoError(t, err)
158+
159+
result := table.RowValues()
160+
assert.NoError(t, table.Err)
159161

160162
// we make sure that all expectations were met
161163
assert.NoError(t, mock.ExpectationsWereMet(), "there were unfulfilled expections")
@@ -184,8 +186,8 @@ func TestCreateTableAllValuesWithNil(t *testing.T) {
184186

185187
results := make([]string, 0)
186188
for table.Next() {
187-
row, err := table.RowValues()
188-
assert.NoError(t, err)
189+
row := table.RowValues()
190+
assert.NoError(t, table.Err)
189191
results = append(results, row)
190192
}
191193

@@ -199,9 +201,7 @@ func TestCreateTableAllValuesWithNil(t *testing.T) {
199201

200202
func TestCreateTableOk(t *testing.T) {
201203
db, mock, err := sqlmock.New()
202-
if err != nil {
203-
t.Fatalf("an error '%s' was not expected when opening a stub database connection", err)
204-
}
204+
assert.NoError(t, err, "an error was not expected when opening a stub database connection")
205205

206206
defer db.Close()
207207

@@ -215,27 +215,43 @@ func TestCreateTableOk(t *testing.T) {
215215
mock.ExpectQuery("^SHOW CREATE TABLE `Test_Table`$").WillReturnRows(createTableRows)
216216
mock.ExpectQuery("^SELECT (.+) FROM `Test_Table`$").WillReturnRows(createTableValueRows)
217217

218+
var buf bytes.Buffer
218219
data := Data{
219220
Connection: db,
221+
Out: &buf,
220222
}
221223

222-
result, err := data.createTable("Test_Table")
223-
if err != nil {
224-
t.Errorf("error was not expected while updating stats: %s", err)
225-
}
224+
assert.NoError(t, data.getTemplates())
226225

227-
// we make sure that all expectations were met
228-
if err := mock.ExpectationsWereMet(); err != nil {
229-
t.Errorf("there were unfulfilled expections: %s", err)
230-
}
226+
table, err := data.createTable("Test_Table")
227+
assert.NoError(t, err)
231228

232-
expectedResult := &table{
233-
Name: "`Test_Table`",
234-
// SQL: "CREATE TABLE 'Test_Table' (`id` int(11) NOT NULL AUTO_INCREMENT,`s` char(60) DEFAULT NULL, PRIMARY KEY (`id`))ENGINE=InnoDB DEFAULT CHARSET=latin1",
235-
// Values: []string{"('1',NULL,'Test Name 1')", "('2','[email protected]','Test Name 2')"},
236-
}
229+
data.writeTable(table)
237230

238-
if !reflect.DeepEqual(result, expectedResult) {
239-
t.Fatalf("expected %#v, got %#v", expectedResult, result)
240-
}
231+
// we make sure that all expectations were met
232+
assert.NoError(t, mock.ExpectationsWereMet(), "there were unfulfilled expections")
233+
234+
expectedResult := `
235+
--
236+
-- Table structure for table ~Test_Table~
237+
--
238+
239+
DROP TABLE IF EXISTS ~Test_Table~;
240+
/*!40101 SET @saved_cs_client = @@character_set_client */;
241+
SET character_set_client = utf8mb4 ;
242+
CREATE TABLE 'Test_Table' (~id~ int(11) NOT NULL AUTO_INCREMENT,~s~ char(60) DEFAULT NULL, PRIMARY KEY (~id~))ENGINE=InnoDB DEFAULT CHARSET=latin1;
243+
/*!40101 SET character_set_client = @saved_cs_client */;
244+
245+
--
246+
-- Dumping data for table ~Test_Table~
247+
--
248+
249+
LOCK TABLES ~Test_Table~ WRITE;
250+
/*!40000 ALTER TABLE ~Test_Table~ DISABLE KEYS */;
251+
INSERT INTO ~Test_Table~ VALUES ('1',NULL,'Test Name 1'),('2','[email protected]','Test Name 2');
252+
/*!40000 ALTER TABLE ~Test_Table~ ENABLE KEYS */;
253+
UNLOCK TABLES;
254+
`
255+
result := strings.Replace(buf.String(), "`", "~", -1)
256+
assert.Equal(t, expectedResult, result)
241257
}

‎mysqldump_test.go

Lines changed: 13 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@ package mysqldump
33
import (
44
"bytes"
55
"os"
6-
"reflect"
76
"strings"
87
"testing"
98

109
sqlmock "github.com/DATA-DOG/go-sqlmock"
10+
"github.com/stretchr/testify/assert"
1111
)
1212

1313
func TestDumpOk(t *testing.T) {
@@ -16,10 +16,7 @@ func TestDumpOk(t *testing.T) {
1616
os.Remove(tmpFile)
1717

1818
db, mock, err := sqlmock.New()
19-
if err != nil {
20-
t.Fatalf("an error '%s' was not expected when opening a stub database connection", err)
21-
}
22-
19+
assert.NoError(t, err, "an error was not expected when opening a stub database connection")
2320
defer db.Close()
2421

2522
showTablesRows := sqlmock.NewRows([]string{"Tables_in_Testdb"}).
@@ -41,12 +38,9 @@ func TestDumpOk(t *testing.T) {
4138
mock.ExpectQuery("^SELECT (.+) FROM `Test_Table`$").WillReturnRows(createTableValueRows)
4239

4340
buf := new(bytes.Buffer)
44-
err = Dump(db, buf)
45-
if err != nil {
46-
t.Fatalf("an error '%s' was not expected when dumping a stub database connection", err)
47-
}
41+
assert.NoError(t, Dump(db, buf), "an error was not expected when dumping a stub database connection")
4842

49-
result := strings.Replace(strings.Split(buf.String(), "-- Dump completed")[0], "`", "\\", -1)
43+
result := strings.Replace(strings.Split(buf.String(), "-- Dump completed")[0], "`", "~", -1)
5044

5145
expected := `-- Go SQL Dump ` + version + `
5246
--
@@ -65,23 +59,23 @@ func TestDumpOk(t *testing.T) {
6559
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
6660
6761
--
68-
-- Table structure for table \Test_Table\
62+
-- Table structure for table ~Test_Table~
6963
--
7064
71-
DROP TABLE IF EXISTS \Test_Table\;
65+
DROP TABLE IF EXISTS ~Test_Table~;
7266
/*!40101 SET @saved_cs_client = @@character_set_client */;
7367
SET character_set_client = utf8mb4 ;
74-
CREATE TABLE 'Test_Table' (\id\ int(11) NOT NULL AUTO_INCREMENT,\email\ char(60) DEFAULT NULL, \name\ char(60), PRIMARY KEY (\id\))ENGINE=InnoDB DEFAULT CHARSET=latin1;
68+
CREATE TABLE 'Test_Table' (~id~ int(11) NOT NULL AUTO_INCREMENT,~email~ char(60) DEFAULT NULL, ~name~ char(60), PRIMARY KEY (~id~))ENGINE=InnoDB DEFAULT CHARSET=latin1;
7569
/*!40101 SET character_set_client = @saved_cs_client */;
7670
7771
--
78-
-- Dumping data for table \Test_Table\
72+
-- Dumping data for table ~Test_Table~
7973
--
8074
81-
LOCK TABLES \Test_Table\ WRITE;
82-
/*!40000 ALTER TABLE \Test_Table\ DISABLE KEYS */;
83-
INSERT INTO \Test_Table\ VALUES ('1',NULL,'Test Name 1'),('2','[email protected]','Test Name 2');
84-
/*!40000 ALTER TABLE \Test_Table\ ENABLE KEYS */;
75+
LOCK TABLES ~Test_Table~ WRITE;
76+
/*!40000 ALTER TABLE ~Test_Table~ DISABLE KEYS */;
77+
INSERT INTO ~Test_Table~ VALUES ('1',NULL,'Test Name 1'),('2','[email protected]','Test Name 2');
78+
/*!40000 ALTER TABLE ~Test_Table~ ENABLE KEYS */;
8579
UNLOCK TABLES;
8680
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
8781
@@ -94,8 +88,5 @@ UNLOCK TABLES;
9488
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
9589
9690
`
97-
98-
if !reflect.DeepEqual(result, expected) {
99-
t.Fatalf("expected \n%#v, got \n%#v", expected, result)
100-
}
91+
assert.Equal(t, expected, result)
10192
}

0 commit comments

Comments
 (0)