Archive node migration v5.7.5 hotfix

app/app.go

@@ -391,6 +391,7 @@ type App struct {
 
 	genesisImportConfig genesistypes.GenesisImportConfig
 
+	stateStore   seidb.StateStore
 	receiptStore seidb.StateStore
 }
 
@@ -419,7 +420,7 @@ func New(
 	cdc := encodingConfig.Amino
 	interfaceRegistry := encodingConfig.InterfaceRegistry
 
-	bAppOptions := SetupSeiDB(logger, homePath, appOpts, baseAppOptions)
+	bAppOptions, stateStore := SetupSeiDB(logger, homePath, appOpts, baseAppOptions)
 
 	bApp := baseapp.NewBaseApp(AppName, logger, db, encodingConfig.TxConfig.TxDecoder(), tmConfig, appOpts, bAppOptions...)
 	bApp.SetCommitMultiStoreTracer(traceStore)
@@ -453,6 +454,7 @@ func New(
 		versionInfo:       version.NewInfo(),
 		metricCounter:     &map[string]float32{},
 		encodingConfig:    encodingConfig,
+		stateStore:        stateStore,
 	}
 
 	for _, option := range appOptions {
@@ -1095,6 +1097,9 @@ func (app *App) Name() string { return app.BaseApp.Name() }
 // GetBaseApp returns the base app of the application
 func (app App) GetBaseApp() *baseapp.BaseApp { return app.BaseApp }
 
+// GetStateStore returns the state store of the application
+func (app App) GetStateStore() seidb.StateStore { return app.stateStore }
+
 // BeginBlocker application updates every begin block
 func (app *App) BeginBlocker(ctx sdk.Context, req abci.RequestBeginBlock) abci.ResponseBeginBlock {
 	metrics.GaugeSeidVersionAndCommit(app.versionInfo.Version, app.versionInfo.GitCommit)

app/seidb.go

@@ -7,6 +7,7 @@ import (
 	servertypes "github.com/cosmos/cosmos-sdk/server/types"
 	"github.com/cosmos/cosmos-sdk/storev2/rootmulti"
 	"github.com/sei-protocol/sei-db/config"
+	seidb "github.com/sei-protocol/sei-db/ss/types"
 	"github.com/spf13/cast"
 	"github.com/tendermint/tendermint/libs/log"
 )
@@ -30,6 +31,8 @@ const (
 	FlagSSKeepRecent        = "state-store.ss-keep-recent"
 	FlagSSPruneInterval     = "state-store.ss-prune-interval"
 	FlagSSImportNumWorkers  = "state-store.ss-import-num-workers"
+	FlagMigrateIAVL         = "migrate-iavl"
+	FlagMigrateHeight       = "migrate-height"
 
 	// Other configs
 	FlagSnapshotInterval = "state-sync.snapshot-interval"
@@ -40,11 +43,11 @@ func SetupSeiDB(
 	homePath string,
 	appOpts servertypes.AppOptions,
 	baseAppOptions []func(*baseapp.BaseApp),
-) []func(*baseapp.BaseApp) {
+) ([]func(*baseapp.BaseApp), seidb.StateStore) {
 	scEnabled := cast.ToBool(appOpts.Get(FlagSCEnable))
 	if !scEnabled {
 		logger.Info("SeiDB is disabled, falling back to IAVL")
-		return baseAppOptions
+		return baseAppOptions, nil
 	}
 	logger.Info("SeiDB SC is enabled, running node with StoreV2 commit store")
 	scConfig := parseSCConfigs(appOpts)
@@ -56,14 +59,21 @@ func SetupSeiDB(
 
 	// cms must be overridden before the other options, because they may use the cms,
 	// make sure the cms aren't be overridden by the other options later on.
-	cms := rootmulti.NewStore(homePath, logger, scConfig, ssConfig)
+	cms := rootmulti.NewStore(homePath, logger, scConfig, ssConfig, cast.ToBool(appOpts.Get("migrate-iavl")))
+	migrationEnabled := cast.ToBool(appOpts.Get(FlagMigrateIAVL))
+	migrationHeight := cast.ToInt64(appOpts.Get(FlagMigrateHeight))
 	baseAppOptions = append([]func(*baseapp.BaseApp){
 		func(baseApp *baseapp.BaseApp) {
+			if migrationEnabled {
+				originalCMS := baseApp.CommitMultiStore()
+				baseApp.SetQueryMultiStore(originalCMS)
+				baseApp.SetMigrationHeight(migrationHeight)
+			}
 			baseApp.SetCMS(cms)
 		},
 	}, baseAppOptions...)
 
-	return baseAppOptions
+	return baseAppOptions, cms.GetStateStore()
 }
 
 func parseSCConfigs(appOpts servertypes.AppOptions) config.StateCommitConfig {

app/test_state_store.go

@@ -106,6 +106,31 @@ func (s *InMemoryStateStore) ReverseIterator(storeKey string, version int64, sta
 	return iter, nil
 }
 
+func (s *InMemoryStateStore) RawImport(ch <-chan types.RawSnapshotNode) error {
+	// TODO: Add set call here
+	return nil
+}
+
+func (s *InMemoryStateStore) SetLatestMigratedModule(module string) error {
+	// TODO: Add set call here
+	return nil
+}
+
+func (s *InMemoryStateStore) GetLatestMigratedModule() (string, error) {
+	// TODO: Add get call here
+	return "", nil
+}
+
+func (s *InMemoryStateStore) SetLatestMigratedKey(key []byte) error {
+	// TODO: Add set call here
+	return nil
+}
+
+func (s *InMemoryStateStore) GetLatestMigratedKey() ([]byte, error) {
+	// TODO: Add get call here
+	return nil, nil
+}
+
 func (s *InMemoryStateStore) RawIterate(storeKey string, fn func([]byte, []byte, int64) bool) (bool, error) {
 	s.mu.RLock()
 	defer s.mu.RUnlock()

app/upgrades.go

@@ -108,6 +108,7 @@ var upgradesList = []string{
 	"v5.7.1",
 	"v5.7.2",
 	"v5.7.4",
+	"v5.7.5",
 }
 
 // if there is an override list, use that instead, for integration tests

cmd/seid/cmd/root.go

@@ -38,6 +38,7 @@ import (
 	"github.com/sei-protocol/sei-chain/app/params"
 	"github.com/sei-protocol/sei-chain/evmrpc"
 	"github.com/sei-protocol/sei-chain/tools"
+	"github.com/sei-protocol/sei-chain/tools/migration/ss"
 	"github.com/sei-protocol/sei-chain/x/evm/blocktest"
 	"github.com/sei-protocol/sei-chain/x/evm/querier"
 	"github.com/sei-protocol/sei-chain/x/evm/replay"
@@ -220,6 +221,8 @@ func txCommand() *cobra.Command {
 
 func addModuleInitFlags(startCmd *cobra.Command) {
 	crisis.AddModuleInitFlags(startCmd)
+	startCmd.Flags().Bool("migrate-iavl", false, "Run migration of IAVL data store to SeiDB State Store")
+	startCmd.Flags().Int64("migrate-height", 0, "Height at which to start the migration")
 }
 
 // newApp creates a new Cosmos SDK app
@@ -266,7 +269,7 @@ func newApp(
 	// This makes it such that the wasm VM gas converts to sdk gas at a 6.66x rate vs that of the previous multiplier
 	wasmGasRegisterConfig.GasMultiplier = 21_000_000
 
-	return app.New(
+	app := app.New(
 		logger,
 		db,
 		traceStore,
@@ -302,6 +305,20 @@ func newApp(
 		baseapp.SetSnapshotDirectory(cast.ToString(appOpts.Get(server.FlagStateSyncSnapshotDir))),
 		baseapp.SetOccEnabled(cast.ToBool(appOpts.Get(baseapp.FlagOccEnabled))),
 	)
+
+	// Start migration if --migrate flag is set
+	if cast.ToBool(appOpts.Get("migrate-iavl")) {
+		go func() {
+			homeDir := cast.ToString(appOpts.Get(flags.FlagHome))
+			stateStore := app.GetStateStore()
+			migrationHeight := cast.ToInt64(appOpts.Get("migrate-height"))
+			migrator := ss.NewMigrator(homeDir, db, stateStore)
+			if err := migrator.Migrate(migrationHeight, homeDir); err != nil {
+				panic(err)
+			}
+		}()
+	}
+	return app
 }
 
 // appExport creates a new simapp (optionally at a given height)

go.mod

@@ -346,12 +346,12 @@ require (
 replace (
 	github.com/CosmWasm/wasmd => github.com/sei-protocol/sei-wasmd v0.2.3
 	github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8.0
-	github.com/cosmos/cosmos-sdk => github.com/sei-protocol/sei-cosmos v0.3.31
-	github.com/cosmos/iavl => github.com/sei-protocol/sei-iavl v0.1.9
+	github.com/cosmos/cosmos-sdk => github.com/sei-protocol/sei-cosmos v0.3.32-0.20241010125343-6e3ae69e9c64
+	github.com/cosmos/iavl => github.com/sei-protocol/sei-iavl v0.2.0
 	github.com/cosmos/ibc-go/v3 => github.com/sei-protocol/sei-ibc-go/v3 v3.3.2
 	github.com/ethereum/go-ethereum => github.com/sei-protocol/go-ethereum v1.13.5-sei-22
 	github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
-	github.com/sei-protocol/sei-db => github.com/sei-protocol/sei-db v0.0.40
+	github.com/sei-protocol/sei-db => github.com/sei-protocol/sei-db v0.0.45-0.20241009171123-c9b6c253819b
 	// Latest goleveldb is broken, we have to stick to this version
 	github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
 	github.com/tendermint/tendermint => github.com/sei-protocol/sei-tendermint v0.3.6

go.sum

@@ -1347,12 +1347,12 @@ github.com/sei-protocol/go-ethereum v1.13.5-sei-22 h1:t/m1qXER+DEMrcpqgoYmUxifkA
 github.com/sei-protocol/go-ethereum v1.13.5-sei-22/go.mod h1:kcRZmuzRn1lVejiFNTz4l4W7imnpq1bDAnuKS/RyhbQ=
 github.com/sei-protocol/goutils v0.0.2 h1:Bfa7Sv+4CVLNM20QcpvGb81B8C5HkQC/kW1CQpIbXDA=
 github.com/sei-protocol/goutils v0.0.2/go.mod h1:iYE2DuJfEnM+APPehr2gOUXfuLuPsVxorcDO+Tzq9q8=
-github.com/sei-protocol/sei-cosmos v0.3.31 h1:8qnubgQd83F2/Pxx5DmNH2vnl1XtTOAFbW44cEHbeCo=
-github.com/sei-protocol/sei-cosmos v0.3.31/go.mod h1:og/KbejR/zSQ8otapODEDU9zYNhFSUDbq9+tgeYePyU=
-github.com/sei-protocol/sei-db v0.0.40 h1:s6B3u9u0r2Ypd67P8Lrz2IR/QU/FXwtS2X/fnYEix2g=
-github.com/sei-protocol/sei-db v0.0.40/go.mod h1:F/ZKZA8HJPcUzSZPA8yt6pfwlGriJ4RDR4eHKSGLStI=
-github.com/sei-protocol/sei-iavl v0.1.9 h1:y4mVYftxLNRs6533zl7N0/Ch+CzRQc04JDfHolIxgBE=
-github.com/sei-protocol/sei-iavl v0.1.9/go.mod h1:7PfkEVT5dcoQE+s/9KWdoXJ8VVVP1QpYYPLdxlkSXFk=
+github.com/sei-protocol/sei-cosmos v0.3.32-0.20241010125343-6e3ae69e9c64 h1:bid2b4ZzISw9qALPyulJ3iBxr/+1AaUBpBYpvYQCVRc=
+github.com/sei-protocol/sei-cosmos v0.3.32-0.20241010125343-6e3ae69e9c64/go.mod h1:og/KbejR/zSQ8otapODEDU9zYNhFSUDbq9+tgeYePyU=
+github.com/sei-protocol/sei-db v0.0.45-0.20241009171123-c9b6c253819b h1:q25KpfxNwTrS8UZgkjdTvQkNI1Wo84KkavDpPA3aHwI=
+github.com/sei-protocol/sei-db v0.0.45-0.20241009171123-c9b6c253819b/go.mod h1:F/ZKZA8HJPcUzSZPA8yt6pfwlGriJ4RDR4eHKSGLStI=
+github.com/sei-protocol/sei-iavl v0.2.0 h1:OisPjXiDT+oe+aeckzDEFgkZCYuUjHgs/PP8DPicN+I=
+github.com/sei-protocol/sei-iavl v0.2.0/go.mod h1:qRf8QYUPfrAO7K6VDB2B2l/N7K5L76OorioGBcJBIbw=
 github.com/sei-protocol/sei-ibc-go/v3 v3.3.2 h1:BaMZ6gjwqe3R/5dLmcJ1TkSZ3omcWy2TjaAZAeOJH44=
 github.com/sei-protocol/sei-ibc-go/v3 v3.3.2/go.mod h1:VwB/vWu4ysT5DN2aF78d17LYmx3omSAdq6gpKvM7XRA=
 github.com/sei-protocol/sei-tendermint v0.3.6 h1:bvvyWHyJcdCo6+b5Da8p9STyC+7tSNKAI9XPQsYgBuU=

tools/cmd.go

@@ -1,7 +1,8 @@
 package tools
 
 import (
-	"github.com/sei-protocol/sei-chain/tools/tx-scanner/cmd"
+	migration "github.com/sei-protocol/sei-chain/tools/migration/cmd"
+	scanner "github.com/sei-protocol/sei-chain/tools/tx-scanner/cmd"
 	"github.com/spf13/cobra"
 )
 
@@ -10,6 +11,8 @@ func ToolCmd() *cobra.Command {
 		Use:   "tools",
 		Short: "A set of useful tools for sei chain",
 	}
-	toolsCmd.AddCommand(cmd.ScanCmd())
+	toolsCmd.AddCommand(scanner.ScanCmd())
+	toolsCmd.AddCommand(migration.MigrateCmd())
+	toolsCmd.AddCommand(migration.VerifyMigrationCmd())
 	return toolsCmd
 }

tools/migration/cmd/cmd.go

@@ -0,0 +1,118 @@
+package cmd
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/cosmos/cosmos-sdk/store/rootmulti"
+	"github.com/sei-protocol/sei-chain/tools/migration/sc"
+	"github.com/sei-protocol/sei-chain/tools/migration/ss"
+	"github.com/sei-protocol/sei-db/config"
+	sstypes "github.com/sei-protocol/sei-db/ss"
+	"github.com/spf13/cobra"
+	"github.com/tendermint/tendermint/libs/log"
+	dbm "github.com/tendermint/tm-db"
+)
+
+func MigrateCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "migrate-iavl",
+		Short: "A tool to migrate full IAVL data store to SeiDB. Use this tool to migrate IAVL to SeiDB SC and SS database.",
+		Run:   execute,
+	}
+	cmd.PersistentFlags().String("home-dir", "/root/.sei", "Sei home directory")
+	cmd.PersistentFlags().String("target-db", "", "Available options: [SS, SC]")
+	return cmd
+}
+
+func execute(cmd *cobra.Command, _ []string) {
+	homeDir, _ := cmd.Flags().GetString("home-dir")
+	target, _ := cmd.Flags().GetString("target-db")
+	dataDir := filepath.Join(homeDir, "data")
+	db, err := dbm.NewGoLevelDB("application", dataDir)
+	if err != nil {
+		panic(err)
+	}
+	latestVersion := rootmulti.GetLatestVersion(db)
+	fmt.Printf("latest version: %d\n", latestVersion)
+	if target == "SS" {
+		if err = migrateSS(latestVersion, homeDir, db); err != nil {
+			panic(err)
+		}
+	} else if target == "SC" {
+		if err = migrateSC(latestVersion, homeDir, db); err != nil {
+			panic(err)
+		}
+	} else {
+		panic("Invalid target-db, either SS or SC should be provided")
+	}
+}
+
+func migrateSC(version int64, homeDir string, db dbm.DB) error {
+	migrator := sc.NewMigrator(homeDir, db)
+	return migrator.Migrate(version)
+}
+
+func migrateSS(version int64, homeDir string, db dbm.DB) error {
+	ssConfig := config.DefaultStateStoreConfig()
+	ssConfig.Enable = true
+
+	stateStore, err := sstypes.NewStateStore(log.NewNopLogger(), homeDir, ssConfig)
+	if err != nil {
+		return err
+	}
+
+	migrator := ss.NewMigrator(homeDir, db, stateStore)
+	return migrator.Migrate(version, homeDir)
+}
+
+func VerifyMigrationCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "verify-migration",
+		Short: "A tool to verify migration of a IAVL data store to SeiDB at a particular height.",
+		Run:   verify,
+	}
+
+	cmd.PersistentFlags().Int64("version", -1, "Version to run migration verification on")
+	cmd.PersistentFlags().String("home-dir", "/root/.sei", "Sei home directory")
+
+	return cmd
+}
+
+func verify(cmd *cobra.Command, _ []string) {
+	homeDir, _ := cmd.Flags().GetString("home-dir")
+	version, _ := cmd.Flags().GetInt64("version")
+
+	fmt.Printf("version %d\n", version)
+
+	if version <= 0 {
+		panic("Must specify version for verification")
+	}
+
+	dataDir := filepath.Join(homeDir, "data")
+	db, err := dbm.NewGoLevelDB("application", dataDir)
+	if err != nil {
+		panic(err)
+	}
+
+	err = verifySS(version, homeDir, db)
+	if err != nil {
+		fmt.Printf("Verification Failed with err: %s\n", err.Error())
+		return
+	}
+
+	fmt.Println("Verification Succeeded")
+}
+
+func verifySS(version int64, homeDir string, db dbm.DB) error {
+	ssConfig := config.DefaultStateStoreConfig()
+	ssConfig.Enable = true
+
+	stateStore, err := sstypes.NewStateStore(log.NewNopLogger(), homeDir, ssConfig)
+	if err != nil {
+		return err
+	}
+
+	migrator := ss.NewMigrator(homeDir, db, stateStore)
+	return migrator.Verify(version)
+}