Post mortem: CONSENSUS FAILURE!!! after upgrade from v0.0.1-alpha-6 to v0.0.1-alpha-7 #33
Description
Timeline:
12:01:40: indexed block 736500
12:01:43: Service stop starts
12:01:46: Service fully stopped
Binary was replaced.
12:06:09: Service starting
12:06:41: Starting catchup on block 736501
12:06:56: indexed block 736501
Chain runs smoothly (o so it seems)
12:10:45: indexed block 736536
12:10:55: Validators start to get slashed: quasarvaloper15j65hlelv7tcrjtes9t6pmzqznxh0854f4zesx
12:11:01: slashed quasarvaloper1su60l57kqz8725mlfc7cml45d9z0j59zz55lqp and quasarvaloper1sztmmvmv5a3kpqzkcw7t2c43tfdfa67yplfmqn
12:11:01: CONSENSUS FAILURE
ERR CONSENSUS FAILURE!!! err="should never retrieve a jailed validator from the power store" module=consensus stack="goroutine 567 [running]:
runtime/debug.Stack()
runtime/debug/stack.go:24 +0x65
github.com/tendermint/tendermint/consensus.(*State).receiveRoutine.func2()
github.com/tendermint/tendermint@v0.34.21/consensus/state.go:727 +0x4c
panic({0x219b920, 0x2e72ae0})
runtime/panic.go:838 +0x207
github.com/cosmos/cosmos-sdk/x/staking/keeper.Keeper.ApplyAndReturnValidatorSetUpdates({{0x2e7d688, 0xc00150a560}, {0x2ea19e8, 0xc000fd4490}, {0x2e98020, 0xc0000f8120}, {0x7f1f2b147690, 0xc000f44dc0}, {0x2ea18a0, 0xc000f62390}, ...}, ...)
github.com/cosmos/cosmos-sdk@v0.45.9/x/staking/keeper/val_state_change.go:134 +0x167a
github.com/cosmos/cosmos-sdk/x/staking/keeper.Keeper.BlockValidatorUpdates({{0x2e7d688, 0xc00150a560}, {0x2ea19e8, 0xc000fd4490}, {0x2e98020, 0xc0000f8120}, {0x7f1f2b147690, 0xc000f44dc0}, {0x2ea18a0, 0xc000f62390}, ...}, ...)
github.com/cosmos/cosmos-sdk@v0.45.9/x/staking/keeper/val_state_change.go:27 +0x98
github.com/cosmos/cosmos-sdk/x/staking.EndBlocker({{0x2e947d8, 0xc000052050}, {0x2ea3480, 0xc018c5dc40}, {{0xb, 0x0}, {0xc018445860, 0xf}, 0xb3d1a, {0x25e5234b, ...}, ...}, ...}, ...)
github.com/cosmos/cosmos-sdk@v0.45.9/x/staking/abci.go:26 +0x118
github.com/cosmos/cosmos-sdk/x/staking.AppModule.EndBlock(...)
github.com/cosmos/cosmos-sdk@v0.45.9/x/staking/module.go:174
github.com/cosmos/cosmos-sdk/types/module.(*Manager).EndBlock(_, {{0x2e947d8, 0xc000052050}, {0x2ea3480, 0xc018c5dc40}, {{0xb, 0x0}, {0xc018445860, 0xf}, 0xb3d1a, ...}, ...}, ...)
github.com/cosmos/cosmos-sdk@v0.45.9/types/module/module.go:512 +0x4c2
github.com/quasarlabs/quasarnode/app.(*App).EndBlocker(_, {{0x2e947d8, 0xc000052050}, {0x2ea3480, 0xc018c5dc40}, {{0xb, 0x0}, {0xc018445860, 0xf}, 0xb3d1a, ...}, ...}, ...)
github.com/quasarlabs/quasarnode/app/app.go:967 +0x98
github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).EndBlock(0xc000cc9500, {0x20?})
github.com/cosmos/cosmos-sdk@v0.45.9/baseapp/abci.go:202 +0x1fe
github.com/tendermint/tendermint/abci/client.(*localClient).EndBlockSync(0xc000f608a0, {0xc000f608a0?})
github.com/tendermint/tendermint@v0.34.21/abci/client/local_client.go:288 +0xdf
github.com/tendermint/tendermint/proxy.(*appConnConsensus).EndBlockSync(0xc018c42240?, {0x20?})
github.com/tendermint/tendermint@v0.34.21/proxy/app_conn.go:89 +0x24
github.com/tendermint/tendermint/state.execBlockOnProxyApp({0x2e958b0?, 0xc0019201e0}, {0x2e9b940, 0xc000fbd150}, 0xc01853c780, {0x2ea2c80, 0xc00daa82e8}, 0xb3d19?)
github.com/tendermint/tendermint@v0.34.21/state/execution.go:327 +0x54e
github.com/tendermint/tendermint/state.(*BlockExecutor).ApplyBlock(_, {{{0xb, 0x0}, {0xc003d2f610, 0x7}}, {0xc003d2f620, 0xf}, 0x1, 0xb3d19, {{0xc013fd3660, ...}, ...}, ...}, ...)
github.com/tendermint/tendermint@v0.34.21/state/execution.go:140 +0x171
github.com/tendermint/tendermint/consensus.(*State).finalizeCommit(0xc000173c00, 0xb3d1a)
github.com/tendermint/tendermint@v0.34.21/consensus/state.go:1659 +0xafd
github.com/tendermint/tendermint/consensus.(*State).tryFinalizeCommit(0xc000173c00, 0xb3d1a)
github.com/tendermint/tendermint@v0.34.21/consensus/state.go:1568 +0x2ff
github.com/tendermint/tendermint/consensus.(*State).enterCommit.func1()
github.com/tendermint/tendermint@v0.34.21/consensus/state.go:1503 +0x87
github.com/tendermint/tendermint/consensus.(*State).enterCommit(0xc000173c00, 0xb3d1a, 0x0)
github.com/tendermint/tendermint@v0.34.21/consensus/state.go:1541 +0xcb7
github.com/tendermint/tendermint/consensus.(*State).addVote(0xc000173c00, 0xc018ccc140, {0xc0105325a0, 0x28})
github.com/tendermint/tendermint@v0.34.21/consensus/state.go:2155 +0xb7c
github.com/tendermint/tendermint/consensus.(*State).tryAddVote(0xc000173c00, 0xc018ccc140, {0xc0105325a0?, 0xc0106a3500?})
github.com/tendermint/tendermint@v0.34.21/consensus/state.go:1953 +0x2c
github.com/tendermint/tendermint/consensus.(*State).handleMsg(0xc000173c00, {{0x2e78660?, 0xc001b1af98?}, {0xc0105325a0?, 0x0?}})
github.com/tendermint/tendermint@v0.34.21/consensus/state.go:856 +0x44b
github.com/tendermint/tendermint/consensus.(*State).receiveRoutine(0xc000173c00, 0x0)
github.com/tendermint/tendermint@v0.34.21/consensus/state.go:763 +0x419
created by github.com/tendermint/tendermint/consensus.(*State).OnStart
github.com/tendermint/tendermint@v0.34.21/consensus/state.go:379 +0x12d
"