Skip to content

Commit c63196c

Browse files
BrandonRoehlBrandonRoehl
authored
Merge pull request jamf#11 from BrandonRoehl/develop
Don't backup virtual columns
2 parents 007f021 + 37739dc commit c63196c

File tree

5 files changed

+220
-72
lines changed

5 files changed

+220
-72
lines changed

‎dump.go

Lines changed: 101 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"fmt"
99
"io"
1010
"reflect"
11+
"strings"
1112
"text/template"
1213
"time"
1314
)
@@ -39,6 +40,7 @@ type table struct {
3940
Name string
4041
Err error
4142

43+
cols []string
4244
data *Data
4345
rows *sql.Rows
4446
values []interface{}
@@ -218,7 +220,7 @@ func (data *Data) writeTable(table *table) error {
218220

219221
// MARK: get methods
220222

221-
// getTemplates initilaizes the templates on data from the constants in this file
223+
// getTemplates initializes the templates on data from the constants in this file
222224
func (data *Data) getTemplates() (err error) {
223225
data.headerTmpl, err = template.New("mysqldumpHeader").Parse(headerTmpl)
224226
if err != nil {
@@ -300,49 +302,119 @@ func (table *table) CreateSQL() (string, error) {
300302
return tableSQL.String, nil
301303
}
302304

303-
func (table *table) Init() (err error) {
305+
func (table *table) initColumnData() error {
306+
colInfo, err := table.data.tx.Query("SHOW COLUMNS FROM " + table.NameEsc())
307+
if err != nil {
308+
return err
309+
}
310+
defer colInfo.Close()
311+
312+
cols, err := colInfo.Columns()
313+
if err != nil {
314+
return err
315+
}
316+
317+
fieldIndex, extraIndex := -1, -1
318+
for i, col := range cols {
319+
switch col {
320+
case "Field", "field":
321+
fieldIndex = i
322+
case "Extra", "extra":
323+
extraIndex = i
324+
}
325+
if fieldIndex >= 0 && extraIndex >= 0 {
326+
break
327+
}
328+
}
329+
if fieldIndex < 0 || extraIndex < 0 {
330+
return errors.New("database column information is malformed")
331+
}
332+
333+
info := make([]sql.NullString, len(cols))
334+
scans := make([]interface{}, len(cols))
335+
for i := range info {
336+
scans[i] = &info[i]
337+
}
338+
339+
var result []string
340+
for colInfo.Next() {
341+
// Read into the pointers to the info marker
342+
if err := colInfo.Scan(scans...); err != nil {
343+
return err
344+
}
345+
346+
// Ignore the virtual columns
347+
if !info[extraIndex].Valid || !strings.Contains(info[extraIndex].String, "VIRTUAL") {
348+
result = append(result, info[fieldIndex].String)
349+
}
350+
}
351+
table.cols = result
352+
return nil
353+
}
354+
355+
func (table *table) columnsList() string {
356+
return "`" + strings.Join(table.cols, "`, `") + "`"
357+
}
358+
359+
func (table *table) Init() error {
304360
if len(table.values) != 0 {
305361
return errors.New("can't init twice")
306362
}
307363

308-
table.rows, err = table.data.tx.Query("SELECT * FROM " + table.NameEsc())
309-
if err != nil {
364+
if err := table.initColumnData(); err != nil {
310365
return err
311366
}
312367

313-
columns, err := table.rows.Columns()
368+
if len(table.cols) == 0 {
369+
// No data to dump since this is a virtual table
370+
return nil
371+
}
372+
373+
var err error
374+
table.rows, err = table.data.tx.Query("SELECT " + table.columnsList() + " FROM " + table.NameEsc())
314375
if err != nil {
315376
return err
316377
}
317-
if len(columns) == 0 {
318-
return errors.New("No columns in table " + table.Name + ".")
319-
}
320378

321379
tt, err := table.rows.ColumnTypes()
322380
if err != nil {
323381
return err
324382
}
325383

326-
var t reflect.Type
327384
table.values = make([]interface{}, len(tt))
328385
for i, tp := range tt {
329-
st := tp.ScanType()
330-
if tp.DatabaseTypeName() == "BLOB" {
331-
t = reflect.TypeOf(sql.RawBytes{})
332-
} else if st != nil && (st.Kind() == reflect.Int ||
333-
st.Kind() == reflect.Int8 ||
334-
st.Kind() == reflect.Int16 ||
335-
st.Kind() == reflect.Int32 ||
336-
st.Kind() == reflect.Int64) {
337-
t = reflect.TypeOf(sql.NullInt64{})
338-
} else {
339-
t = reflect.TypeOf(sql.NullString{})
340-
}
341-
table.values[i] = reflect.New(t).Interface()
386+
table.values[i] = reflect.New(reflectColumnType(tp)).Interface()
342387
}
343388
return nil
344389
}
345390

391+
func reflectColumnType(tp *sql.ColumnType) reflect.Type {
392+
// reflect for scanable
393+
switch tp.ScanType().Kind() {
394+
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
395+
return reflect.TypeOf(sql.NullInt64{})
396+
case reflect.Float32, reflect.Float64:
397+
return reflect.TypeOf(sql.NullFloat64{})
398+
case reflect.String:
399+
return reflect.TypeOf(sql.NullString{})
400+
}
401+
402+
// determine by name
403+
switch tp.DatabaseTypeName() {
404+
case "BLOB", "BINARY":
405+
return reflect.TypeOf(sql.RawBytes{})
406+
case "VARCHAR", "TEXT", "DECIMAL":
407+
return reflect.TypeOf(sql.NullString{})
408+
case "BIGINT", "TINYINT", "INT":
409+
return reflect.TypeOf(sql.NullInt64{})
410+
case "DOUBLE":
411+
return reflect.TypeOf(sql.NullFloat64{})
412+
}
413+
414+
// unknown datatype
415+
return tp.ScanType()
416+
}
417+
346418
func (table *table) Next() bool {
347419
if table.rows == nil {
348420
if err := table.Init(); err != nil {
@@ -394,6 +466,12 @@ func (table *table) RowBuffer() *bytes.Buffer {
394466
} else {
395467
b.WriteString(nullType)
396468
}
469+
case *sql.NullFloat64:
470+
if s.Valid {
471+
fmt.Fprintf(&b, "%f", s.Float64)
472+
} else {
473+
b.WriteString(nullType)
474+
}
397475
case *sql.RawBytes:
398476
if len(*s) == 0 {
399477
b.WriteString(nullType)
@@ -425,7 +503,7 @@ func (table *table) Stream() <-chan string {
425503
}
426504

427505
if insert.Len() == 0 {
428-
fmt.Fprintf(&insert, "INSERT INTO %s VALUES ", table.NameEsc())
506+
fmt.Fprint(&insert, "INSERT INTO ", table.NameEsc(), " (", table.columnsList(), ") VALUES ")
429507
} else {
430508
insert.WriteString(",")
431509
}

‎dump_test.go

Lines changed: 59 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,19 @@ func getMockData() (data *Data, mock sqlmock.Sqlmock, err error) {
2626
return
2727
}
2828

29+
func c(name string, v interface{}) *sqlmock.Column {
30+
var t string
31+
switch reflect.ValueOf(v).Kind() {
32+
case reflect.String:
33+
t = "VARCHAR"
34+
case reflect.Int:
35+
t = "INT"
36+
case reflect.Bool:
37+
t = "BOOL"
38+
}
39+
return sqlmock.NewColumn(name).OfType(t, v).Nullable(true)
40+
}
41+
2942
func TestGetTablesOk(t *testing.T) {
3043
data, mock, err := getMockData()
3144
assert.NoError(t, err, "an error was not expected when opening a stub database connection")
@@ -134,16 +147,26 @@ func TestCreateTableSQLOk(t *testing.T) {
134147
}
135148
}
136149

150+
func mockTableSelect(mock sqlmock.Sqlmock, name string) {
151+
cols := sqlmock.NewRows([]string{"Field", "Extra"}).
152+
AddRow("id", "").
153+
AddRow("email", "").
154+
AddRow("name", "")
155+
156+
rows := sqlmock.NewRowsWithColumnDefinition(c("id", 0), c("email", ""), c("name", "")).
157+
AddRow(1, "[email protected]", "Test Name 1").
158+
AddRow(2, "[email protected]", "Test Name 2")
159+
160+
mock.ExpectQuery("^SHOW COLUMNS FROM `" + name + "`$").WillReturnRows(cols)
161+
mock.ExpectQuery("^SELECT (.+) FROM `" + name + "`$").WillReturnRows(rows)
162+
}
163+
137164
func TestCreateTableRowValues(t *testing.T) {
138165
data, mock, err := getMockData()
139166
assert.NoError(t, err, "an error was not expected when opening a stub database connection")
140167
defer data.Close()
141168

142-
rows := sqlmock.NewRows([]string{"id", "email", "name"}).
143-
AddRow(1, "[email protected]", "Test Name 1").
144-
AddRow(2, "[email protected]", "Test Name 2")
145-
146-
mock.ExpectQuery("^SELECT (.+) FROM `test`$").WillReturnRows(rows)
169+
mockTableSelect(mock, "test")
147170

148171
table := data.createTable("test")
149172

@@ -155,26 +178,22 @@ func TestCreateTableRowValues(t *testing.T) {
155178
// we make sure that all expectations were met
156179
assert.NoError(t, mock.ExpectationsWereMet(), "there were unfulfilled expections")
157180

158-
assert.EqualValues(t, "('1','[email protected]','Test Name 1')", result)
181+
assert.EqualValues(t, "(1,'[email protected]','Test Name 1')", result)
159182
}
160183

161184
func TestCreateTableValuesSteam(t *testing.T) {
162185
data, mock, err := getMockData()
163186
assert.NoError(t, err, "an error was not expected when opening a stub database connection")
164187
defer data.Close()
165188

166-
rows := sqlmock.NewRows([]string{"id", "email", "name"}).
167-
AddRow(1, "[email protected]", "Test Name 1").
168-
AddRow(2, "[email protected]", "Test Name 2")
169-
170-
mock.ExpectQuery("^SELECT (.+) FROM `test`$").WillReturnRows(rows)
189+
mockTableSelect(mock, "test")
171190

172191
data.MaxAllowedPacket = 4096
173192

174193
table := data.createTable("test")
175194

176195
s := table.Stream()
177-
assert.EqualValues(t, "INSERT INTO `test` VALUES ('1','[email protected]','Test Name 1'),('2','[email protected]','Test Name 2');", <-s)
196+
assert.EqualValues(t, "INSERT INTO `test` (`id`, `email`, `name`) VALUES (1,'[email protected]','Test Name 1'),(2,'[email protected]','Test Name 2');", <-s)
178197

179198
// we make sure that all expectations were met
180199
assert.NoError(t, mock.ExpectationsWereMet(), "there were unfulfilled expections")
@@ -185,19 +204,15 @@ func TestCreateTableValuesSteamSmallPackets(t *testing.T) {
185204
assert.NoError(t, err, "an error was not expected when opening a stub database connection")
186205
defer data.Close()
187206

188-
rows := sqlmock.NewRows([]string{"id", "email", "name"}).
189-
AddRow(1, "[email protected]", "Test Name 1").
190-
AddRow(2, "[email protected]", "Test Name 2")
191-
192-
mock.ExpectQuery("^SELECT (.+) FROM `test`$").WillReturnRows(rows)
207+
mockTableSelect(mock, "test")
193208

194209
data.MaxAllowedPacket = 64
195210

196211
table := data.createTable("test")
197212

198213
s := table.Stream()
199-
assert.EqualValues(t, "INSERT INTO `test` VALUES ('1','[email protected]','Test Name 1');", <-s)
200-
assert.EqualValues(t, "INSERT INTO `test` VALUES ('2','[email protected]','Test Name 2');", <-s)
214+
assert.EqualValues(t, "INSERT INTO `test` (`id`, `email`, `name`) VALUES (1,'[email protected]','Test Name 1');", <-s)
215+
assert.EqualValues(t, "INSERT INTO `test` (`id`, `email`, `name`) VALUES (2,'[email protected]','Test Name 2');", <-s)
201216

202217
// we make sure that all expectations were met
203218
assert.NoError(t, mock.ExpectationsWereMet(), "there were unfulfilled expections")
@@ -208,11 +223,17 @@ func TestCreateTableAllValuesWithNil(t *testing.T) {
208223
assert.NoError(t, err, "an error was not expected when opening a stub database connection")
209224
defer data.Close()
210225

211-
rows := sqlmock.NewRows([]string{"id", "email", "name"}).
226+
cols := sqlmock.NewRows([]string{"Field", "Extra"}).
227+
AddRow("id", "").
228+
AddRow("email", "").
229+
AddRow("name", "")
230+
231+
rows := sqlmock.NewRowsWithColumnDefinition(c("id", 0), c("email", ""), c("name", "")).
212232
AddRow(1, nil, "Test Name 1").
213233
AddRow(2, "[email protected]", "Test Name 2").
214234
AddRow(3, "", "Test Name 3")
215235

236+
mock.ExpectQuery("^SHOW COLUMNS FROM `test`$").WillReturnRows(cols)
216237
mock.ExpectQuery("^SELECT (.+) FROM `test`$").WillReturnRows(rows)
217238

218239
table := data.createTable("test")
@@ -227,7 +248,7 @@ func TestCreateTableAllValuesWithNil(t *testing.T) {
227248
// we make sure that all expectations were met
228249
assert.NoError(t, mock.ExpectationsWereMet(), "there were unfulfilled expections")
229250

230-
expectedResults := []string{"('1',NULL,'Test Name 1')", "('2','[email protected]','Test Name 2')", "('3','','Test Name 3')"}
251+
expectedResults := []string{"(1,NULL,'Test Name 1')", "(2,'[email protected]','Test Name 2')", "(3,'','Test Name 3')"}
231252

232253
assert.EqualValues(t, expectedResults, results)
233254
}
@@ -240,11 +261,17 @@ func TestCreateTableOk(t *testing.T) {
240261
createTableRows := sqlmock.NewRows([]string{"Table", "Create Table"}).
241262
AddRow("Test_Table", "CREATE TABLE 'Test_Table' (`id` int(11) NOT NULL AUTO_INCREMENT,`s` char(60) DEFAULT NULL, PRIMARY KEY (`id`))ENGINE=InnoDB DEFAULT CHARSET=latin1")
242263

243-
createTableValueRows := sqlmock.NewRows([]string{"id", "email", "name"}).
264+
createTableValueCols := sqlmock.NewRows([]string{"Field", "Extra"}).
265+
AddRow("id", "").
266+
AddRow("email", "").
267+
AddRow("name", "")
268+
269+
createTableValueRows := sqlmock.NewRowsWithColumnDefinition(c("id", 0), c("email", ""), c("name", "")).
244270
AddRow(1, nil, "Test Name 1").
245271
AddRow(2, "[email protected]", "Test Name 2")
246272

247273
mock.ExpectQuery("^SHOW CREATE TABLE `Test_Table`$").WillReturnRows(createTableRows)
274+
mock.ExpectQuery("^SHOW COLUMNS FROM `Test_Table`$").WillReturnRows(createTableValueCols)
248275
mock.ExpectQuery("^SELECT (.+) FROM `Test_Table`$").WillReturnRows(createTableValueRows)
249276

250277
var buf bytes.Buffer
@@ -277,7 +304,7 @@ CREATE TABLE 'Test_Table' (~id~ int(11) NOT NULL AUTO_INCREMENT,~s~ char(60) DEF
277304
278305
LOCK TABLES ~Test_Table~ WRITE;
279306
/*!40000 ALTER TABLE ~Test_Table~ DISABLE KEYS */;
280-
INSERT INTO ~Test_Table~ VALUES ('1',NULL,'Test Name 1'),('2','[email protected]','Test Name 2');
307+
INSERT INTO ~Test_Table~ (~id~, ~email~, ~name~) VALUES (1,NULL,'Test Name 1'),(2,'[email protected]','Test Name 2');
281308
/*!40000 ALTER TABLE ~Test_Table~ ENABLE KEYS */;
282309
UNLOCK TABLES;
283310
`
@@ -293,11 +320,17 @@ func TestCreateTableOkSmallPackets(t *testing.T) {
293320
createTableRows := sqlmock.NewRows([]string{"Table", "Create Table"}).
294321
AddRow("Test_Table", "CREATE TABLE 'Test_Table' (`id` int(11) NOT NULL AUTO_INCREMENT,`s` char(60) DEFAULT NULL, PRIMARY KEY (`id`))ENGINE=InnoDB DEFAULT CHARSET=latin1")
295322

296-
createTableValueRows := sqlmock.NewRows([]string{"id", "email", "name"}).
323+
createTableValueCols := sqlmock.NewRows([]string{"Field", "Extra"}).
324+
AddRow("id", "").
325+
AddRow("email", "").
326+
AddRow("name", "")
327+
328+
createTableValueRows := sqlmock.NewRowsWithColumnDefinition(c("id", 0), c("email", ""), c("name", "")).
297329
AddRow(1, nil, "Test Name 1").
298330
AddRow(2, "[email protected]", "Test Name 2")
299331

300332
mock.ExpectQuery("^SHOW CREATE TABLE `Test_Table`$").WillReturnRows(createTableRows)
333+
mock.ExpectQuery("^SHOW COLUMNS FROM `Test_Table`$").WillReturnRows(createTableValueCols)
301334
mock.ExpectQuery("^SELECT (.+) FROM `Test_Table`$").WillReturnRows(createTableValueRows)
302335

303336
var buf bytes.Buffer
@@ -330,8 +363,8 @@ CREATE TABLE 'Test_Table' (~id~ int(11) NOT NULL AUTO_INCREMENT,~s~ char(60) DEF
330363
331364
LOCK TABLES ~Test_Table~ WRITE;
332365
/*!40000 ALTER TABLE ~Test_Table~ DISABLE KEYS */;
333-
INSERT INTO ~Test_Table~ VALUES ('1',NULL,'Test Name 1');
334-
INSERT INTO ~Test_Table~ VALUES ('2','[email protected]','Test Name 2');
366+
INSERT INTO ~Test_Table~ (~id~, ~email~, ~name~) VALUES (1,NULL,'Test Name 1');
367+
INSERT INTO ~Test_Table~ (~id~, ~email~, ~name~) VALUES (2,'[email protected]','Test Name 2');
335368
/*!40000 ALTER TABLE ~Test_Table~ ENABLE KEYS */;
336369
UNLOCK TABLES;
337370
`

‎go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
module github.com/jamf/go-mysqldump
22

33
require (
4-
github.com/DATA-DOG/go-sqlmock v1.3.0
5-
github.com/stretchr/testify v1.4.0
4+
github.com/DATA-DOG/go-sqlmock v1.5.0
5+
github.com/stretchr/testify v1.7.0
66
)
77

88
go 1.13

0 commit comments

Comments
 (0)