Add system tests for chain upgrade (#1643)

* Start chain upgrade tests

* Fix stakeunstake test

* Make test pass

* Better stop chain

* Test chain upgrade

* Set upgrade handler order

* Fix app for chain upgrade

* Minor cleanup

* Check contract state

* Updates

* Gov constitution migration will be handled by the sdk

* Deactivate upgrade test

* Helper

* Better upgrade structure an minor updates

(cherry picked from commit 32a01da)

* Updates

* Gci formatting

* Updates

* Testnet commit timeout

* Update

* Store artifacts on system test failure

* Better circleci setup

* Artifact path

* x

* Fix upgrade

* Generic upgrade handler

* Fix imports

* Update tests/system/cli.go

Co-authored-by: pinosu <[email protected]>

---------

Co-authored-by: Pino' Surace <[email protected]>
Co-authored-by: pinosu <[email protected]>

Author: alpe

.circleci/config.yml

@@ -112,6 +112,13 @@ jobs:
       - run:
           name: Build and run system tests
           command: make test-system
+      - run:
+          command: |
+            mkdir -p /tmp/system-test-workspace
+            mv  /home/circleci/project/tests/system/testnet /tmp/system-test-workspace
+          when: on_fail
+      - store_artifacts:
+          path: /tmp/system-test-workspace
 
   benchmark:
     executor: golang

Makefile

@@ -161,7 +161,7 @@ test-system: install
 format-tools:
 	go install mvdan.cc/[email protected]
 	go install github.com/client9/misspell/cmd/[email protected]
-	go install golang.org/x/tools/cmd/goimports@latest
+	go install github.com/daixiang0/[email protected]
 
 lint: format-tools
 	golangci-lint run --tests=false
@@ -170,7 +170,7 @@ lint: format-tools
 format: format-tools
 	find . -name '*.go' -type f -not -path "./vendor*" -not -path "./tests/system/vendor*" -not -path "*.git*" -not -path "./client/lcd/statik/statik.go" | xargs gofumpt -w
 	find . -name '*.go' -type f -not -path "./vendor*" -not -path "./tests/system/vendor*" -not -path "*.git*" -not -path "./client/lcd/statik/statik.go" | xargs misspell -w
-	find . -name '*.go' -type f -not -path "./vendor*" -not -path "./tests/system/vendor*" -not -path "*.git*" -not -path "./client/lcd/statik/statik.go" | xargs goimports -w -local github.com/CosmWasm/wasmd
+	find . -name '*.go' -type f -not -path "./vendor*" -not -path "./tests/system/vendor*" -not -path "*.git*" -not -path "./client/lcd/statik/statik.go" | xargs gci write --skip-generated -s standard -s default -s "prefix(cosmossdk.io)" -s "prefix(github.com/cosmos/cosmos-sdk)" -s "prefix(github.com/CosmWasm/wasmd)" --custom-order
 
 
 ###############################################################################

app/app.go

@@ -842,6 +842,7 @@ func NewWasmApp(
 
 	// initialize BaseApp
 	app.SetInitChainer(app.InitChainer)
+	app.SetPreBlocker(app.PreBlocker)
 	app.SetBeginBlocker(app.BeginBlocker)
 	app.SetEndBlocker(app.EndBlocker)
 	app.setAnteHandler(txConfig, wasmConfig, keys[wasmtypes.StoreKey])
@@ -949,6 +950,11 @@ func (app *WasmApp) setPostHandler() {
 // Name returns the name of the App
 func (app *WasmApp) Name() string { return app.BaseApp.Name() }
 
+// PreBlocker application updates every pre block
+func (app *WasmApp) PreBlocker(ctx sdk.Context, _ *abci.RequestFinalizeBlock) (*sdk.ResponsePreBlock, error) {
+	return app.ModuleManager.PreBlock(ctx)
+}
+
 // BeginBlocker application updates every begin block
 func (app *WasmApp) BeginBlocker(ctx sdk.Context) (sdk.BeginBlock, error) {
 	return app.ModuleManager.BeginBlock(ctx)
@@ -1164,6 +1170,5 @@ func initParamsKeeper(appCodec codec.BinaryCodec, legacyAmino *codec.LegacyAmino
 	paramsKeeper.Subspace(icahosttypes.SubModuleName)
 	paramsKeeper.Subspace(icacontrollertypes.SubModuleName)
 	paramsKeeper.Subspace(wasmtypes.ModuleName)
-
 	return paramsKeeper
 }

app/upgrades.go

@@ -1,39 +1,90 @@
 package app
 
 import (
-	"context"
+	"fmt"
+
+	icacontrollertypes "github.com/cosmos/ibc-go/v8/modules/apps/27-interchain-accounts/controller/types"
+	icahosttypes "github.com/cosmos/ibc-go/v8/modules/apps/27-interchain-accounts/host/types"
+	ibctransfertypes "github.com/cosmos/ibc-go/v8/modules/apps/transfer/types"
+	ibcclienttypes "github.com/cosmos/ibc-go/v8/modules/core/02-client/types" //nolint:staticcheck
+	ibcconnectiontypes "github.com/cosmos/ibc-go/v8/modules/core/03-connection/types"
+	ibcexported "github.com/cosmos/ibc-go/v8/modules/core/exported"
 
-	storetypes "cosmossdk.io/store/types"
 	upgradetypes "cosmossdk.io/x/upgrade/types"
 
-	"github.com/cosmos/cosmos-sdk/types/module"
+	paramskeeper "github.com/cosmos/cosmos-sdk/x/params/keeper"
+	paramstypes "github.com/cosmos/cosmos-sdk/x/params/types"
+
+	"github.com/CosmWasm/wasmd/app/upgrades"
+	"github.com/CosmWasm/wasmd/app/upgrades/noop"
+	v050 "github.com/CosmWasm/wasmd/app/upgrades/v050"
 )
 
-// UpgradeName defines the on-chain upgrade name for the sample SimApp upgrade
-// from v047 to v050.
-//
-// NOTE: This upgrade defines a reference implementation of what an upgrade
-// could look like when an application is migrating from Cosmos SDK version
-// v0.47.x to v0.50.x.
-const UpgradeName = "v047-to-v050"
+// Upgrades list of chain upgrades
+var Upgrades = []upgrades.Upgrade{v050.Upgrade}
 
+// RegisterUpgradeHandlers registers the chain upgrade handlers
 func (app WasmApp) RegisterUpgradeHandlers() {
-	app.UpgradeKeeper.SetUpgradeHandler(
-		UpgradeName,
-		func(ctx context.Context, _ upgradetypes.Plan, fromVM module.VersionMap) (module.VersionMap, error) {
-			return app.ModuleManager.RunMigrations(ctx, app.Configurator(), fromVM)
-		},
-	)
+	setupLegacyKeyTables(app.ParamsKeeper)
+	if len(Upgrades) == 0 {
+		// always have a unique upgrade registered for the current version to test in system tests
+		Upgrades = append(Upgrades, noop.NewUpgrade(app.Version()))
+	}
+
+	keepers := upgrades.AppKeepers{AccountKeeper: app.AccountKeeper}
+	// register all upgrade handlers
+	for _, upgrade := range Upgrades {
+		app.UpgradeKeeper.SetUpgradeHandler(
+			upgrade.UpgradeName,
+			upgrade.CreateUpgradeHandler(
+				app.ModuleManager,
+				app.configurator,
+				&keepers,
+			),
+		)
+	}
 
 	upgradeInfo, err := app.UpgradeKeeper.ReadUpgradeInfoFromDisk()
 	if err != nil {
-		panic(err)
+		panic(fmt.Sprintf("failed to read upgrade info from disk %s", err))
 	}
 
-	if upgradeInfo.Name == UpgradeName && !app.UpgradeKeeper.IsSkipHeight(upgradeInfo.Height) {
-		storeUpgrades := storetypes.StoreUpgrades{}
+	if app.UpgradeKeeper.IsSkipHeight(upgradeInfo.Height) {
+		return
+	}
+
+	// register store loader for current upgrade
+	for _, upgrade := range Upgrades {
+		if upgradeInfo.Name == upgrade.UpgradeName {
+			app.SetStoreLoader(upgradetypes.UpgradeStoreLoader(upgradeInfo.Height, &upgrade.StoreUpgrades)) // nolint:gosec
+			break
+		}
+	}
+}
+
+func setupLegacyKeyTables(k paramskeeper.Keeper) {
+	// Set param key table for params module migration
+	for _, subspace := range k.GetSubspaces() {
+		subspace := subspace
+
+		var keyTable paramstypes.KeyTable
+		switch subspace.Name() {
+		// ibc types
+		case ibcexported.ModuleName:
+			keyTable = ibcclienttypes.ParamKeyTable()
+			keyTable.RegisterParamSet(&ibcconnectiontypes.Params{})
+		case ibctransfertypes.ModuleName:
+			keyTable = ibctransfertypes.ParamKeyTable()
+		case icahosttypes.SubModuleName:
+			keyTable = icahosttypes.ParamKeyTable()
+		case icacontrollertypes.SubModuleName:
+			keyTable = icacontrollertypes.ParamKeyTable()
+		default:
+			continue
+		}
 
-		// configure store loader that checks if version == upgradeHeight and applies store upgrades
-		app.SetStoreLoader(upgradetypes.UpgradeStoreLoader(upgradeInfo.Height, &storeUpgrades))
+		if !subspace.HasKeyTable() {
+			subspace.WithKeyTable(keyTable)
+		}
 	}
 }

app/upgrades/noop/upgrades.go

@@ -0,0 +1,34 @@
+package noop
+
+import (
+	"context"
+
+	storetypes "cosmossdk.io/store/types"
+	upgradetypes "cosmossdk.io/x/upgrade/types"
+
+	"github.com/cosmos/cosmos-sdk/types/module"
+
+	"github.com/CosmWasm/wasmd/app/upgrades"
+)
+
+// NewUpgrade constructor
+func NewUpgrade(semver string) upgrades.Upgrade {
+	return upgrades.Upgrade{
+		UpgradeName:          semver,
+		CreateUpgradeHandler: CreateUpgradeHandler,
+		StoreUpgrades: storetypes.StoreUpgrades{
+			Added:   []string{},
+			Deleted: []string{},
+		},
+	}
+}
+
+func CreateUpgradeHandler(
+	mm *module.Manager,
+	configurator module.Configurator,
+	ak *upgrades.AppKeepers,
+) upgradetypes.UpgradeHandler {
+	return func(ctx context.Context, plan upgradetypes.Plan, fromVM module.VersionMap) (module.VersionMap, error) {
+		return mm.RunMigrations(ctx, configurator, fromVM)
+	}
+}

app/upgrades/types.go

@@ -0,0 +1,26 @@
+package upgrades
+
+import (
+	storetypes "cosmossdk.io/store/types"
+	upgradetypes "cosmossdk.io/x/upgrade/types"
+
+	"github.com/cosmos/cosmos-sdk/types/module"
+	authkeeper "github.com/cosmos/cosmos-sdk/x/auth/keeper"
+)
+
+type AppKeepers struct {
+	authkeeper.AccountKeeper
+}
+
+// Upgrade defines a struct containing necessary fields that a SoftwareUpgradeProposal
+// must have written, in order for the state migration to go smoothly.
+// An upgrade must implement this struct, and then set it in the app.go.
+// The app.go will then define the handler.
+type Upgrade struct {
+	// Upgrade version name, for the upgrade handler, e.g. `v7`
+	UpgradeName string
+
+	// CreateUpgradeHandler defines the function that creates an upgrade handler
+	CreateUpgradeHandler func(*module.Manager, module.Configurator, *AppKeepers) upgradetypes.UpgradeHandler
+	StoreUpgrades        storetypes.StoreUpgrades
+}

app/upgrades/v050/upgrades.go

@@ -0,0 +1,37 @@
+package v050
+
+import (
+	"context"
+
+	storetypes "cosmossdk.io/store/types"
+	circuittypes "cosmossdk.io/x/circuit/types"
+	upgradetypes "cosmossdk.io/x/upgrade/types"
+
+	"github.com/cosmos/cosmos-sdk/types/module"
+
+	"github.com/CosmWasm/wasmd/app/upgrades"
+)
+
+// UpgradeName defines the on-chain upgrade name
+const UpgradeName = "v0.50"
+
+var Upgrade = upgrades.Upgrade{
+	UpgradeName:          UpgradeName,
+	CreateUpgradeHandler: CreateUpgradeHandler,
+	StoreUpgrades: storetypes.StoreUpgrades{
+		Added: []string{
+			circuittypes.ModuleName,
+		},
+		Deleted: []string{},
+	},
+}
+
+func CreateUpgradeHandler(
+	mm *module.Manager,
+	configurator module.Configurator,
+	ak *upgrades.AppKeepers,
+) upgradetypes.UpgradeHandler {
+	return func(ctx context.Context, plan upgradetypes.Plan, fromVM module.VersionMap) (module.VersionMap, error) {
+		return mm.RunMigrations(ctx, configurator, fromVM)
+	}
+}

cmd/wasmd/testnet.go

@@ -82,6 +82,7 @@ type startArgs struct {
 	outputDir     string
 	printMnemonic bool
 	rpcAddress    string
+	timeoutCommit time.Duration
 }
 
 func addTestnetFlagsToCmd(cmd *cobra.Command) {
@@ -159,14 +160,13 @@ Example:
 			if err != nil {
 				return err
 			}
-
 			return initTestnetFiles(clientCtx, cmd, config, mbm, genBalIterator, clientCtx.TxConfig.SigningContext().ValidatorAddressCodec(), args)
 		},
 	}
 
 	addTestnetFlagsToCmd(cmd)
 	cmd.Flags().String(flagNodeDirPrefix, "node", "Prefix the directory name for each node with (node results in node0, node1, ...)")
-	cmd.Flags().String(flagNodeDaemonHome, "wasmd", "Home directory of the node's daemon configuration")
+	cmd.Flags().String(flagNodeDaemonHome, version.AppName, "Home directory of the node's daemon configuration")
 	cmd.Flags().String(flagStartingIPAddress, "192.168.0.1", "Starting IP address (192.168.0.1 results in persistent peers list [email protected]:46656, [email protected]:46656, ...)")
 	cmd.Flags().String(flags.FlagKeyringBackend, flags.DefaultKeyringBackend, "Select keyring's backend (os|file|test)")
 	cmd.Flags().Duration(flagCommitTimeout, 5*time.Second, "Time to wait after a block commit before starting on the new height")
@@ -553,6 +553,7 @@ func startTestnet(cmd *cobra.Command, args startArgs) error {
 	networkConfig.APIAddress = args.apiAddress
 	networkConfig.GRPCAddress = args.grpcAddress
 	networkConfig.PrintMnemonic = args.printMnemonic
+	networkConfig.TimeoutCommit = args.timeoutCommit
 	networkLogger := network.NewCLILogger(cmd)
 
 	baseDir := fmt.Sprintf("%s/%s", args.outputDir, networkConfig.ChainID)

tests/system/.gitignore

@@ -1 +1,2 @@
 /testnet
+/binaries

tests/system/Makefile

@@ -10,7 +10,6 @@ test:
 format:
 	find . -name '*.go' -type f -not -path "./vendor*" -not -path "*.git*" -not -path "./client/lcd/statik/statik.go" | xargs gofumpt -w
 	find . -name '*.go' -type f -not -path "./vendor*" -not -path "*.git*" -not -path "./client/lcd/statik/statik.go" | xargs misspell -w
-	find . -name '*.go' -type f -not -path "./vendor*" -not -path "*.git*" -not -path "./client/lcd/statik/statik.go" | xargs goimports -w -local github.com/CosmWasm/wasmd
-
+	find . -name '*.go' -type f -not -path "./vendor*" -not -path "./tests/system/vendor*" -not -path "*.git*" -not -path "./client/lcd/statik/statik.go" | xargs gci write --skip-generated -s standard -s default -s "prefix(cosmossdk.io)" -s "prefix(github.com/cosmos/cosmos-sdk)" -s "prefix(github.com/CosmWasm/wasmd)" --custom-order
 
 .PHONY: all test format