Skip to content

Commit 1144c7f

Browse files
max-hoffmanmax-hoffman
authored
Fix panics on schema changes (#8505)
* Fix panics on schema changes * more time for stats auto-update test * debug output * more slep * more sleep * maybe null hash * revert test
1 parent 2fe3344 commit 1144c7f

File tree

6 files changed

+151
-19
lines changed

6 files changed

+151
-19
lines changed

‎go/libraries/doltcore/sqle/enginetest/stats_queries.go‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -536,6 +536,71 @@ var DoltStatsIOTests = []queries.ScriptTest{
536536
},
537537
},
538538
},
539+
{
540+
// https://github.com/dolthub/dolt/issues/8504
541+
Name: "alter index column type",
542+
SetUpScript: []string{
543+
"set @@PERSIST.dolt_stats_auto_refresh_interval = 0;",
544+
"set @@PERSIST.dolt_stats_auto_refresh_threshold = 0;",
545+
"CREATE table xy (x bigint primary key, y varchar(16))",
546+
"insert into xy values (0,'0'), (1,'1'), (2,'2')",
547+
"analyze table xy",
548+
},
549+
Assertions: []queries.ScriptTestAssertion{
550+
{
551+
Query: "select count(*) from dolt_statistics group by table_name, index_name",
552+
Expected: []sql.Row{{1}},
553+
},
554+
{
555+
Query: "alter table xy modify column x varchar(16);",
556+
},
557+
{
558+
Query: "insert into xy values ('3', '3')",
559+
},
560+
{
561+
Query: "call dolt_stats_restart()",
562+
},
563+
{
564+
Query: "select sleep(.2)",
565+
},
566+
{
567+
Query: "select count(*) from dolt_statistics group by table_name, index_name",
568+
Expected: []sql.Row{{1}},
569+
},
570+
},
571+
},
572+
{
573+
Name: "drop primary key",
574+
SetUpScript: []string{
575+
"set @@PERSIST.dolt_stats_auto_refresh_interval = 0;",
576+
"set @@PERSIST.dolt_stats_auto_refresh_threshold = 0;",
577+
"CREATE table xy (x bigint primary key, y varchar(16))",
578+
"insert into xy values (0,'0'), (1,'1'), (2,'2')",
579+
"analyze table xy",
580+
},
581+
Assertions: []queries.ScriptTestAssertion{
582+
{
583+
Query: "select count(*) from dolt_statistics group by table_name, index_name",
584+
Expected: []sql.Row{{1}},
585+
},
586+
{
587+
Query: "alter table xy drop primary key",
588+
},
589+
{
590+
Query: "insert into xy values ('3', '3')",
591+
},
592+
{
593+
Query: "call dolt_stats_restart()",
594+
},
595+
{
596+
Query: "select sleep(.2)",
597+
},
598+
{
599+
Query: "select count(*) from dolt_statistics group by table_name, index_name",
600+
Expected: []sql.Row{},
601+
},
602+
},
603+
},
539604
}
540605

541606
var StatBranchTests = []queries.ScriptTest{

‎go/libraries/doltcore/sqle/statsnoms/database.go‎

Lines changed: 39 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -118,13 +118,14 @@ func NewNomsStats(sourceDb, statsDb dsess.SqlDatabase) *NomsStatsDatabase {
118118
type dbStats map[sql.StatQualifier]*statspro.DoltStats
119119

120120
type NomsStatsDatabase struct {
121-
mu *sync.Mutex
122-
destDb dsess.SqlDatabase
123-
sourceDb dsess.SqlDatabase
124-
stats []dbStats
125-
branches []string
126-
latestTableRoots []map[string]hash.Hash
127-
dirty []*prolly.MutableMap
121+
mu *sync.Mutex
122+
destDb dsess.SqlDatabase
123+
sourceDb dsess.SqlDatabase
124+
stats []dbStats
125+
branches []string
126+
tableHashes []map[string]hash.Hash
127+
schemaHashes []map[string]hash.Hash
128+
dirty []*prolly.MutableMap
128129
}
129130

130131
var _ statspro.Database = (*NomsStatsDatabase)(nil)
@@ -158,7 +159,8 @@ func (n *NomsStatsDatabase) LoadBranchStats(ctx *sql.Context, branch string) err
158159
n.branches = append(n.branches, branch)
159160
n.stats = append(n.stats, doltStats)
160161
n.dirty = append(n.dirty, nil)
161-
n.latestTableRoots = append(n.latestTableRoots, make(map[string]hash.Hash))
162+
n.tableHashes = append(n.tableHashes, make(map[string]hash.Hash))
163+
n.schemaHashes = append(n.schemaHashes, make(map[string]hash.Hash))
162164
return nil
163165
}
164166

@@ -223,7 +225,8 @@ func (n *NomsStatsDatabase) SetStat(ctx context.Context, branch string, qual sql
223225
func (n *NomsStatsDatabase) trackBranch(ctx context.Context, branch string) error {
224226
n.branches = append(n.branches, branch)
225227
n.stats = append(n.stats, make(dbStats))
226-
n.latestTableRoots = append(n.latestTableRoots, make(map[string]hash.Hash))
228+
n.tableHashes = append(n.tableHashes, make(map[string]hash.Hash))
229+
n.schemaHashes = append(n.schemaHashes, make(map[string]hash.Hash))
227230

228231
kd, vd := schema.StatsTableDoltSchema.GetMapDescriptors()
229232
newMap, err := prolly.NewMapFromTuples(ctx, n.destDb.DbData().Ddb.NodeStore(), kd, vd)
@@ -268,7 +271,7 @@ func (n *NomsStatsDatabase) DeleteBranchStats(ctx *sql.Context, branch string, f
268271
n.branches = append(n.branches[:i], n.branches[i+1:]...)
269272
n.dirty = append(n.dirty[:i], n.dirty[i+1:]...)
270273
n.stats = append(n.stats[:i], n.stats[i+1:]...)
271-
n.latestTableRoots = append(n.latestTableRoots[:i], n.latestTableRoots[i+1:]...)
274+
n.tableHashes = append(n.tableHashes[:i], n.tableHashes[i+1:]...)
272275
}
273276
}
274277
if flush {
@@ -339,23 +342,45 @@ func (n *NomsStatsDatabase) Flush(ctx context.Context, branch string) error {
339342
return nil
340343
}
341344

342-
func (n *NomsStatsDatabase) GetLatestHash(branch, tableName string) hash.Hash {
345+
func (n *NomsStatsDatabase) GetTableHash(branch, tableName string) hash.Hash {
343346
n.mu.Lock()
344347
defer n.mu.Unlock()
345348
for i, b := range n.branches {
346349
if strings.EqualFold(branch, b) {
347-
return n.latestTableRoots[i][tableName]
350+
return n.tableHashes[i][tableName]
348351
}
349352
}
350353
return hash.Hash{}
351354
}
352355

353-
func (n *NomsStatsDatabase) SetLatestHash(branch, tableName string, h hash.Hash) {
356+
func (n *NomsStatsDatabase) SetTableHash(branch, tableName string, h hash.Hash) {
354357
n.mu.Lock()
355358
defer n.mu.Unlock()
356359
for i, b := range n.branches {
357360
if strings.EqualFold(branch, b) {
358-
n.latestTableRoots[i][tableName] = h
361+
n.tableHashes[i][tableName] = h
362+
break
363+
}
364+
}
365+
}
366+
367+
func (n *NomsStatsDatabase) GetSchemaHash(branch, tableName string) hash.Hash {
368+
n.mu.Lock()
369+
defer n.mu.Unlock()
370+
for i, b := range n.branches {
371+
if strings.EqualFold(branch, b) {
372+
return n.schemaHashes[i][tableName]
373+
}
374+
}
375+
return hash.Hash{}
376+
}
377+
378+
func (n *NomsStatsDatabase) SetSchemaHash(branch, tableName string, h hash.Hash) {
379+
n.mu.Lock()
380+
defer n.mu.Unlock()
381+
for i, b := range n.branches {
382+
if strings.EqualFold(branch, b) {
383+
n.schemaHashes[i][tableName] = h
359384
break
360385
}
361386
}

‎go/libraries/doltcore/sqle/statspro/analyze.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,26 @@ func (p *Provider) RefreshTableStatsWithBranch(ctx *sql.Context, table sql.Table
135135
p.setStatDb(dbName, statDb)
136136
}
137137

138+
schHash, err := dTab.GetSchemaHash(ctx)
139+
if err != nil {
140+
return err
141+
}
142+
143+
if oldSchHash := statDb.GetSchemaHash(branch, tableName); oldSchHash.IsEmpty() {
144+
statDb.SetSchemaHash(branch, tableName, schHash)
145+
} else if oldSchHash != schHash {
146+
ctx.GetLogger().Debugf("statistics refresh: detected table schema change: %s,%s/%s", dbName, table, branch)
147+
statDb.SetSchemaHash(branch, tableName, schHash)
148+
149+
stats, err := p.GetTableDoltStats(ctx, branch, dbName, tableName)
150+
if err != nil {
151+
return err
152+
}
153+
for _, stat := range stats {
154+
statDb.DeleteStats(ctx, branch, stat.Qualifier())
155+
}
156+
}
157+
138158
tablePrefix := fmt.Sprintf("%s.", tableName)
139159
var idxMetas []indexMeta
140160
for _, idx := range indexes {

‎go/libraries/doltcore/sqle/statspro/auto_refresh.go‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ func (p *Provider) checkRefresh(ctx *sql.Context, sqlDb sql.Database, dbName, br
149149
return err
150150
}
151151

152-
if statDb.GetLatestHash(branch, table) == tableHash {
152+
if statDb.GetTableHash(branch, table) == tableHash {
153153
// no data changes since last check
154154
tableExistsAndSkipped[table] = true
155155
ctx.GetLogger().Debugf("statistics refresh: table hash unchanged since last check: %s", tableHash)
@@ -158,6 +158,26 @@ func (p *Provider) checkRefresh(ctx *sql.Context, sqlDb sql.Database, dbName, br
158158
ctx.GetLogger().Debugf("statistics refresh: new table hash: %s", tableHash)
159159
}
160160

161+
schHash, err := dTab.GetSchemaHash(ctx)
162+
if err != nil {
163+
return err
164+
}
165+
166+
if oldSchHash := statDb.GetSchemaHash(branch, table); oldSchHash.IsEmpty() {
167+
statDb.SetSchemaHash(branch, table, schHash)
168+
} else if oldSchHash != schHash {
169+
ctx.GetLogger().Debugf("statistics refresh: detected table schema change: %s,%s/%s", dbName, table, branch)
170+
statDb.SetSchemaHash(branch, table, schHash)
171+
172+
stats, err := p.GetTableDoltStats(ctx, branch, dbName, table)
173+
if err != nil {
174+
return err
175+
}
176+
for _, stat := range stats {
177+
statDb.DeleteStats(ctx, branch, stat.Qualifier())
178+
}
179+
}
180+
161181
iat, ok := sqlTable.(sql.IndexAddressableTable)
162182
if !ok {
163183
return fmt.Errorf("table does not support indexes %s", table)
@@ -205,7 +225,7 @@ func (p *Provider) checkRefresh(ctx *sql.Context, sqlDb sql.Database, dbName, br
205225
// mark index for updating
206226
idxMetas = append(idxMetas, updateMeta)
207227
// update latest hash if we haven't already
208-
statDb.SetLatestHash(branch, table, tableHash)
228+
statDb.SetTableHash(branch, table, tableHash)
209229
}
210230
}
211231

‎go/libraries/doltcore/sqle/statspro/interface.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,10 @@ type Database interface {
5050
Flush(ctx context.Context, branch string) error
5151
// Close finalizes any file references.
5252
Close() error
53-
SetLatestHash(branch, tableName string, h hash.Hash)
54-
GetLatestHash(branch, tableName string) hash.Hash
53+
SetTableHash(branch, tableName string, h hash.Hash)
54+
GetTableHash(branch, tableName string) hash.Hash
55+
SetSchemaHash(branch, tableName string, h hash.Hash)
56+
GetSchemaHash(branch, tableName string) hash.Hash
5557
Branches() []string
5658
}
5759

‎integration-tests/bats/stats.bats‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ teardown() {
119119
[ "${lines[1]}" = "0" ]
120120

121121
start_sql_server
122-
dolt sql -q "insert into xy values (0,0), (1,1)"
122+
run dolt sql -q "insert into xy values (0,0), (1,1)"
123123
sleep 1
124124
stop_sql_server
125125

0 commit comments

Comments
 (0)