From af5d2fd216be15d4f28baf1363a3c3fe67a4705e Mon Sep 17 00:00:00 2001 From: Facundo Date: Fri, 29 Nov 2024 11:48:39 +0100 Subject: [PATCH 01/12] feat(indexer/postgres)!: add basic support for header, txs and events --- baseapp/abci.go | 4 +++ baseapp/streaming.go | 25 +++++++++----- go.mod | 1 + go.sum | 2 -- indexer/postgres/base_sql.go | 15 +++++---- indexer/postgres/listener.go | 65 ++++++++++++++++++++++++++++++++++++ schema/appdata/data.go | 6 ++++ simapp/go.mod | 2 ++ 8 files changed, 104 insertions(+), 16 deletions(-) diff --git a/baseapp/abci.go b/baseapp/abci.go index d9921a8e13d5..cd6aaf18e169 100644 --- a/baseapp/abci.go +++ b/baseapp/abci.go @@ -997,6 +997,10 @@ func (app *BaseApp) Commit() (*abci.CommitResponse, error) { app.logger.Error("Commit listening hook failed", "height", blockHeight, "err", err) if app.streamingManager.StopNodeOnErr { err = fmt.Errorf("Commit listening hook failed: %w", err) + if blockHeight == 1 { + // can't rollback to height 0, so just return the error + return nil, fmt.Errorf("failed to commit block 1, can't automatically rollback: %w", err) + } rollbackErr := app.cms.RollbackToVersion(blockHeight - 1) if rollbackErr != nil { return nil, errors.Join(err, rollbackErr) diff --git a/baseapp/streaming.go b/baseapp/streaming.go index 3e30f8888d58..ab9d6efac911 100644 --- a/baseapp/streaming.go +++ b/baseapp/streaming.go @@ -2,6 +2,7 @@ package baseapp import ( "context" + "encoding/json" "fmt" "sort" "strconv" @@ -144,9 +145,10 @@ func exposeStoreKeysSorted(keysStr []string, keys map[string]*storetypes.KVStore return exposeStoreKeys } -func eventToAppDataEvent(event abci.Event) (appdata.Event, error) { +func eventToAppDataEvent(event abci.Event, height int64) (appdata.Event, error) { appdataEvent := appdata.Event{ - Type: event.Type, + BlockNumber: uint64(height), + Type: event.Type, Attributes: func() ([]appdata.EventAttribute, error) { attrs := make([]appdata.EventAttribute, len(event.Attributes)) for j, attr := range event.Attributes { @@ -208,10 +210,16 @@ func NewListenerWrapper(listener appdata.Listener) listenerWrapper { func (p listenerWrapper) ListenFinalizeBlock(_ context.Context, req abci.FinalizeBlockRequest, res abci.FinalizeBlockResponse) error { if p.listener.StartBlock != nil { + // this is to avoid putting all txs along with the block data + reqWithoutTxs := req + reqWithoutTxs.Txs = nil + if err := p.listener.StartBlock(appdata.StartBlockData{ Height: uint64(req.Height), HeaderBytes: nil, // TODO: https://github.com/cosmos/cosmos-sdk/issues/22009 - HeaderJSON: nil, // TODO: https://github.com/cosmos/cosmos-sdk/issues/22009 + HeaderJSON: func() (json.RawMessage, error) { + return json.Marshal(reqWithoutTxs) + }, }); err != nil { return err } @@ -219,9 +227,10 @@ func (p listenerWrapper) ListenFinalizeBlock(_ context.Context, req abci.Finaliz if p.listener.OnTx != nil { for i, tx := range req.Txs { if err := p.listener.OnTx(appdata.TxData{ - TxIndex: int32(i), - Bytes: func() ([]byte, error) { return tx, nil }, - JSON: nil, // TODO: https://github.com/cosmos/cosmos-sdk/issues/22009 + BlockNumber: uint64(req.Height), + TxIndex: int32(i), + Bytes: func() ([]byte, error) { return tx, nil }, + JSON: nil, // TODO: https://github.com/cosmos/cosmos-sdk/issues/22009 }); err != nil { return err } @@ -231,14 +240,14 @@ func (p listenerWrapper) ListenFinalizeBlock(_ context.Context, req abci.Finaliz events := make([]appdata.Event, len(res.Events)) var err error for i, event := range res.Events { - events[i], err = eventToAppDataEvent(event) + events[i], err = eventToAppDataEvent(event, req.Height) if err != nil { return err } } for _, txResult := range res.TxResults { for _, event := range txResult.Events { - appdataEvent, err := eventToAppDataEvent(event) + appdataEvent, err := eventToAppDataEvent(event, req.Height) if err != nil { return err } diff --git a/go.mod b/go.mod index 173cda441476..13286bf26204 100644 --- a/go.mod +++ b/go.mod @@ -186,6 +186,7 @@ require ( // TODO remove after all modules have their own go.mods replace ( cosmossdk.io/api => ./api + cosmossdk.io/schema => ./schema cosmossdk.io/store => ./store cosmossdk.io/x/bank => ./x/bank cosmossdk.io/x/staking => ./x/staking diff --git a/go.sum b/go.sum index 1a89809fe182..48e3dbff3df8 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,6 @@ cosmossdk.io/log v1.5.0 h1:dVdzPJW9kMrnAYyMf1duqacoidB9uZIl+7c6z0mnq0g= cosmossdk.io/log v1.5.0/go.mod h1:Tr46PUJjiUthlwQ+hxYtUtPn4D/oCZXAkYevBeh5+FI= cosmossdk.io/math v1.4.0 h1:XbgExXFnXmF/CccPPEto40gOO7FpWu9yWNAZPN3nkNQ= cosmossdk.io/math v1.4.0/go.mod h1:O5PkD4apz2jZs4zqFdTr16e1dcaQCc5z6lkEnrrppuk= -cosmossdk.io/schema v0.3.1-0.20241128094659-bd76b47e1d8b h1:svpFdulZRrYz+RTHu2u9CeKkMKrIHx5354vjiHerovo= -cosmossdk.io/schema v0.3.1-0.20241128094659-bd76b47e1d8b/go.mod h1:RDAhxIeNB4bYqAlF4NBJwRrgtnciMcyyg0DOKnhNZQQ= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= diff --git a/indexer/postgres/base_sql.go b/indexer/postgres/base_sql.go index ac79b0929a49..076e245dd821 100644 --- a/indexer/postgres/base_sql.go +++ b/indexer/postgres/base_sql.go @@ -17,17 +17,20 @@ CREATE TABLE IF NOT EXISTS tx id BIGSERIAL PRIMARY KEY, block_number BIGINT NOT NULL REFERENCES block (number), index_in_block BIGINT NOT NULL, - data JSONB NOT NULL + data JSONB NULL, + bytes BYTEA NULL ); CREATE TABLE IF NOT EXISTS event ( id BIGSERIAL PRIMARY KEY, block_number BIGINT NOT NULL REFERENCES block (number), - tx_id BIGINT NULL REFERENCES tx (id), - msg_index BIGINT NULL, - event_index BIGINT NULL, - type TEXT NOT NULL, - data JSONB NOT NULL + block_stage INTEGER NOT NULL, + tx_index BIGINT NOT NULL, + msg_index BIGINT NOT NULL, + event_index BIGINT NOT NULL, + type TEXT NULL, + data JSONB NULL, + attributes JSONB NULL ); ` diff --git a/indexer/postgres/listener.go b/indexer/postgres/listener.go index c2e982e2b02f..b377e459fd4a 100644 --- a/indexer/postgres/listener.go +++ b/indexer/postgres/listener.go @@ -1,6 +1,7 @@ package postgres import ( + "encoding/json" "fmt" "cosmossdk.io/schema/appdata" @@ -81,5 +82,69 @@ func (i *indexerImpl) listener() appdata.Listener { i.tx, err = i.db.BeginTx(i.ctx, nil) return nil, err }, + OnTx: func(td appdata.TxData) error { + var bz []byte + if td.Bytes != nil { + var err error + bz, err = td.Bytes() + if err != nil { + return err + } + } + + var jsonData json.RawMessage + if td.JSON != nil { + var err error + jsonData, err = td.JSON() + if err != nil { + return err + } + } + + _, err := i.tx.Exec("INSERT INTO tx (block_number, index_in_block, data, bytes) VALUES ($1, $2, $3, $4)", + td.BlockNumber, td.TxIndex, jsonData, bz) + + return err + }, + OnEvent: func(data appdata.EventData) error { + for _, e := range data.Events { + var ( + jsonData json.RawMessage + jsonAttributes json.RawMessage + ) + + if e.Data != nil { + var err error + jsonData, err = e.Data() + if err != nil { + return fmt.Errorf("failed to get event data: %w", err) + } + } + + if e.Attributes != nil { + attrs, err := e.Attributes() + if err != nil { + return fmt.Errorf("failed to get event attributes: %w", err) + } + + attrsMap := map[string]interface{}{} + for _, attr := range attrs { + attrsMap[attr.Key] = attr.Value + } + + jsonAttributes, err = json.Marshal(attrsMap) + if err != nil { + return fmt.Errorf("failed to marshal event attributes: %w", err) + } + } + + _, err := i.tx.Exec("INSERT INTO event (block_number, block_stage, tx_index, msg_index, event_index, type, data, attributes) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", + e.BlockNumber, e.BlockStage, e.TxIndex, e.MsgIndex, e.EventIndex, e.Type, jsonData, jsonAttributes) + if err != nil { + return fmt.Errorf("failed to index event: %w", err) + } + } + return nil + }, } } diff --git a/schema/appdata/data.go b/schema/appdata/data.go index 9b8ecc058565..1fdb15864b54 100644 --- a/schema/appdata/data.go +++ b/schema/appdata/data.go @@ -31,6 +31,9 @@ type StartBlockData struct { // TxData represents the raw transaction data that is passed to a listener. type TxData struct { + // BlockNumber is the block number to which this event is associated. + BlockNumber uint64 + // TxIndex is the index of the transaction in the block. TxIndex int32 @@ -53,6 +56,9 @@ type Event struct { // If the block stage is unknown, it should be set to UnknownBlockStage. BlockStage BlockStage + // BlockNumber is the block number to which this event is associated. + BlockNumber uint64 + // TxIndex is the 1-based index of the transaction in the block to which this event is associated. // If TxIndex is zero, it means that we do not know the transaction index. // Otherwise, the index should start with 1. diff --git a/simapp/go.mod b/simapp/go.mod index 782e6f79d0c0..d0644f966d7e 100644 --- a/simapp/go.mod +++ b/simapp/go.mod @@ -252,6 +252,8 @@ replace ( cosmossdk.io/api => ../api cosmossdk.io/client/v2 => ../client/v2 cosmossdk.io/collections => ../collections + cosmossdk.io/schema => ../schema + cosmossdk.io/indexer/postgres => ../indexer/postgres cosmossdk.io/store => ../store cosmossdk.io/tools/confix => ../tools/confix cosmossdk.io/x/accounts => ../x/accounts From 6bf2f3ebba79dbb508016a6f12f51f09781835a6 Mon Sep 17 00:00:00 2001 From: Facundo Date: Tue, 3 Dec 2024 16:46:18 +0100 Subject: [PATCH 02/12] add the basic decoded tx --- baseapp/streaming.go | 18 ++++-- indexer/postgres/listener.go | 108 +++++++++++++++++++---------------- 2 files changed, 72 insertions(+), 54 deletions(-) diff --git a/baseapp/streaming.go b/baseapp/streaming.go index ab9d6efac911..acb7f87557ad 100644 --- a/baseapp/streaming.go +++ b/baseapp/streaming.go @@ -21,6 +21,7 @@ import ( "github.com/cosmos/cosmos-sdk/client/flags" servertypes "github.com/cosmos/cosmos-sdk/server/types" + sdk "github.com/cosmos/cosmos-sdk/types" ) const ( @@ -49,7 +50,7 @@ func (app *BaseApp) EnableIndexer(indexerOpts interface{}, keys map[string]*stor app.cms.AddListeners(exposedKeys) app.streamingManager = storetypes.StreamingManager{ - ABCIListeners: []storetypes.ABCIListener{listenerWrapper{listener.Listener}}, + ABCIListeners: []storetypes.ABCIListener{listenerWrapper{listener.Listener, app.txDecoder}}, StopNodeOnErr: true, } @@ -199,7 +200,8 @@ func eventToAppDataEvent(event abci.Event, height int64) (appdata.Event, error) } type listenerWrapper struct { - listener appdata.Listener + listener appdata.Listener + txDecoder sdk.TxDecoder } // NewListenerWrapper creates a new listenerWrapper. @@ -210,7 +212,7 @@ func NewListenerWrapper(listener appdata.Listener) listenerWrapper { func (p listenerWrapper) ListenFinalizeBlock(_ context.Context, req abci.FinalizeBlockRequest, res abci.FinalizeBlockResponse) error { if p.listener.StartBlock != nil { - // this is to avoid putting all txs along with the block data + // clean up redundant data reqWithoutTxs := req reqWithoutTxs.Txs = nil @@ -230,7 +232,15 @@ func (p listenerWrapper) ListenFinalizeBlock(_ context.Context, req abci.Finaliz BlockNumber: uint64(req.Height), TxIndex: int32(i), Bytes: func() ([]byte, error) { return tx, nil }, - JSON: nil, // TODO: https://github.com/cosmos/cosmos-sdk/issues/22009 + JSON: func() (json.RawMessage, error) { + sdkTx, err := p.txDecoder(tx) + if err != nil { + // if the transaction cannot be decoded, return the error as JSON + // as there are some txs that might not be decodeable by the txDecoder + return json.Marshal(err) + } + return json.Marshal(sdkTx) + }, // TODO: https://github.com/cosmos/cosmos-sdk/issues/22009 }); err != nil { return err } diff --git a/indexer/postgres/listener.go b/indexer/postgres/listener.go index b377e459fd4a..6cc03c64727c 100644 --- a/indexer/postgres/listener.go +++ b/indexer/postgres/listener.go @@ -82,69 +82,77 @@ func (i *indexerImpl) listener() appdata.Listener { i.tx, err = i.db.BeginTx(i.ctx, nil) return nil, err }, - OnTx: func(td appdata.TxData) error { - var bz []byte - if td.Bytes != nil { - var err error - bz, err = td.Bytes() - if err != nil { - return err - } + OnTx: txListener(i), + OnEvent: eventListener(i), + } +} + +func txListener(i *indexerImpl) func(data appdata.TxData) error { + return func(td appdata.TxData) error { + var bz []byte + if td.Bytes != nil { + var err error + bz, err = td.Bytes() + if err != nil { + return err + } + } + + var jsonData json.RawMessage + if td.JSON != nil { + var err error + jsonData, err = td.JSON() + if err != nil { + return err } + } + + _, err := i.tx.Exec("INSERT INTO tx (block_number, index_in_block, data, bytes) VALUES ($1, $2, $3, $4)", + td.BlockNumber, td.TxIndex, jsonData, bz) + + return err + } +} + +func eventListener(i *indexerImpl) func(data appdata.EventData) error { + return func(data appdata.EventData) error { + for _, e := range data.Events { + var ( + jsonData json.RawMessage + jsonAttributes json.RawMessage + ) - var jsonData json.RawMessage - if td.JSON != nil { + if e.Data != nil { var err error - jsonData, err = td.JSON() + jsonData, err = e.Data() if err != nil { - return err + return fmt.Errorf("failed to get event data: %w", err) } } - _, err := i.tx.Exec("INSERT INTO tx (block_number, index_in_block, data, bytes) VALUES ($1, $2, $3, $4)", - td.BlockNumber, td.TxIndex, jsonData, bz) - - return err - }, - OnEvent: func(data appdata.EventData) error { - for _, e := range data.Events { - var ( - jsonData json.RawMessage - jsonAttributes json.RawMessage - ) - - if e.Data != nil { - var err error - jsonData, err = e.Data() - if err != nil { - return fmt.Errorf("failed to get event data: %w", err) - } + if e.Attributes != nil { + attrs, err := e.Attributes() + if err != nil { + return fmt.Errorf("failed to get event attributes: %w", err) } - if e.Attributes != nil { - attrs, err := e.Attributes() - if err != nil { - return fmt.Errorf("failed to get event attributes: %w", err) - } - - attrsMap := map[string]interface{}{} - for _, attr := range attrs { - attrsMap[attr.Key] = attr.Value - } - - jsonAttributes, err = json.Marshal(attrsMap) - if err != nil { - return fmt.Errorf("failed to marshal event attributes: %w", err) - } + attrsMap := map[string]interface{}{} + for _, attr := range attrs { + attrsMap[attr.Key] = attr.Value } - _, err := i.tx.Exec("INSERT INTO event (block_number, block_stage, tx_index, msg_index, event_index, type, data, attributes) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", - e.BlockNumber, e.BlockStage, e.TxIndex, e.MsgIndex, e.EventIndex, e.Type, jsonData, jsonAttributes) + jsonAttributes, err = json.Marshal(attrsMap) if err != nil { - return fmt.Errorf("failed to index event: %w", err) + return fmt.Errorf("failed to marshal event attributes: %w", err) } } - return nil - }, + + _, err := i.tx.Exec("INSERT INTO event (block_number, block_stage, tx_index, msg_index, event_index, type, data, attributes) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", + e.BlockNumber, e.BlockStage, e.TxIndex, e.MsgIndex, e.EventIndex, e.Type, jsonData, jsonAttributes) + if err != nil { + return fmt.Errorf("failed to index event: %w", err) + } + } + return nil } } From 4ac5dfe3f4d51b7613b428bcb81d69e1fa7ab54d Mon Sep 17 00:00:00 2001 From: Facundo Date: Tue, 3 Dec 2024 16:47:57 +0100 Subject: [PATCH 03/12] remove todo --- baseapp/streaming.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/baseapp/streaming.go b/baseapp/streaming.go index acb7f87557ad..7d88c2d196f7 100644 --- a/baseapp/streaming.go +++ b/baseapp/streaming.go @@ -240,7 +240,7 @@ func (p listenerWrapper) ListenFinalizeBlock(_ context.Context, req abci.Finaliz return json.Marshal(err) } return json.Marshal(sdkTx) - }, // TODO: https://github.com/cosmos/cosmos-sdk/issues/22009 + }, }); err != nil { return err } From 8dec19f0a94a919fed53df11ac1a1f863b8d70c6 Mon Sep 17 00:00:00 2001 From: Facundo Date: Wed, 4 Dec 2024 12:40:32 +0100 Subject: [PATCH 04/12] added missing things to v2 --- server/v2/cometbft/abci.go | 9 +++++---- server/v2/cometbft/streaming.go | 11 ++++++++--- server/v2/go.mod | 1 + server/v2/stf/stf.go | 5 +++++ 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/server/v2/cometbft/abci.go b/server/v2/cometbft/abci.go index e6a07fd86f07..27247b5e3b6b 100644 --- a/server/v2/cometbft/abci.go +++ b/server/v2/cometbft/abci.go @@ -538,7 +538,7 @@ func (c *consensus[T]) FinalizeBlock( events = append(events, resp.EndBlockEvents...) // listen to state streaming changes in accordance with the block - err = c.streamDeliverBlockChanges(ctx, req.Height, req.Txs, resp.TxResults, events, stateChanges) + err = c.streamDeliverBlockChanges(ctx, req.Height, req.Txs, decodedTxs, resp.TxResults, events, stateChanges) if err != nil { return nil, err } @@ -575,7 +575,7 @@ func (c *consensus[T]) internalFinalizeBlock( // TODO(tip): can we expect some txs to not decode? if so, what we do in this case? this does not seem to be the case, // considering that prepare and process always decode txs, assuming they're the ones providing txs we should never // have a tx that fails decoding. - decodedTxs, err := decodeTxs(req.Txs, c.txCodec) + decodedTxs, err := decodeTxs(c.logger, req.Txs, c.txCodec) if err != nil { return nil, nil, nil, err } @@ -708,12 +708,13 @@ func (c *consensus[T]) ExtendVote(ctx context.Context, req *abciproto.ExtendVote return resp, err } -func decodeTxs[T transaction.Tx](rawTxs [][]byte, codec transaction.Codec[T]) ([]T, error) { +func decodeTxs[T transaction.Tx](logger log.Logger, rawTxs [][]byte, codec transaction.Codec[T]) ([]T, error) { txs := make([]T, len(rawTxs)) for i, rawTx := range rawTxs { tx, err := codec.Decode(rawTx) if err != nil { - return nil, fmt.Errorf("unable to decode tx: %d: %w", i, err) + // do not return an error here, as we want to deliver the block even if some txs are invalid + logger.Debug("failed to decode tx", "err", err) } txs[i] = tx } diff --git a/server/v2/cometbft/streaming.go b/server/v2/cometbft/streaming.go index c9ca7fddcc7e..8ce5f0a0f7ad 100644 --- a/server/v2/cometbft/streaming.go +++ b/server/v2/cometbft/streaming.go @@ -2,6 +2,7 @@ package cometbft import ( "context" + "encoding/json" "cosmossdk.io/core/event" "cosmossdk.io/core/server" @@ -16,6 +17,7 @@ func (c *consensus[T]) streamDeliverBlockChanges( ctx context.Context, height int64, txs [][]byte, + decodedTxs []T, txResults []server.TxResult, events []event.Event, stateChanges []store.StateChanges, @@ -76,9 +78,12 @@ func (c *consensus[T]) streamDeliverBlockChanges( if c.listener.OnTx != nil { for i, tx := range txs { if err := c.listener.OnTx(appdata.TxData{ - TxIndex: int32(i), - Bytes: func() ([]byte, error) { return tx, nil }, - JSON: nil, // TODO: https://github.com/cosmos/cosmos-sdk/issues/22009 + BlockNumber: uint64(height), + TxIndex: int32(i), + Bytes: func() ([]byte, error) { return tx, nil }, + JSON: func() (json.RawMessage, error) { + return json.Marshal(decodedTxs[i]) + }, }); err != nil { return err } diff --git a/server/v2/go.mod b/server/v2/go.mod index 2905d7e23c1c..c7da5d7b3e41 100644 --- a/server/v2/go.mod +++ b/server/v2/go.mod @@ -4,6 +4,7 @@ go 1.23 replace ( cosmossdk.io/api => ../../api + cosmossdk.io/schema => ../../schema cosmossdk.io/server/v2/appmanager => ./appmanager cosmossdk.io/server/v2/stf => ./stf cosmossdk.io/store/v2 => ../../store/v2 diff --git a/server/v2/stf/stf.go b/server/v2/stf/stf.go index 6b6f6b2c53da..ca078782b0a9 100644 --- a/server/v2/stf/stf.go +++ b/server/v2/stf/stf.go @@ -204,6 +204,7 @@ func (s STF[T]) deliverTx( events := make([]event.Event, 0) // set the event indexes, set MsgIndex to 0 in validation events for i, e := range validationEvents { + e.BlockNumber = uint64(hi.Height) e.BlockStage = appdata.TxProcessingStage e.TxIndex = txIndex e.MsgIndex = 0 @@ -214,6 +215,7 @@ func (s STF[T]) deliverTx( execResp, execGas, execEvents, err := s.execTx(ctx, state, gasLimit-validateGas, tx, execMode, hi) // set the TxIndex in the exec events for _, e := range execEvents { + e.BlockNumber = uint64(hi.Height) e.BlockStage = appdata.TxProcessingStage e.TxIndex = txIndex events = append(events, e) @@ -373,6 +375,7 @@ func (s STF[T]) preBlock( } for i := range ctx.events { + ctx.events[i].BlockNumber = uint64(ctx.headerInfo.Height) ctx.events[i].BlockStage = appdata.PreBlockStage ctx.events[i].EventIndex = int32(i + 1) } @@ -390,6 +393,7 @@ func (s STF[T]) beginBlock( } for i := range ctx.events { + ctx.events[i].BlockNumber = uint64(ctx.headerInfo.Height) ctx.events[i].BlockStage = appdata.BeginBlockStage ctx.events[i].EventIndex = int32(i + 1) } @@ -413,6 +417,7 @@ func (s STF[T]) endBlock( } events = append(events, ctx.events...) for i := range events { + ctx.events[i].BlockNumber = uint64(ctx.headerInfo.Height) events[i].BlockStage = appdata.EndBlockStage events[i].EventIndex = int32(i + 1) } From f7985a43c3a6ab445ee2a7efe18ccf60af505caf Mon Sep 17 00:00:00 2001 From: Facundo Date: Thu, 5 Dec 2024 14:32:45 +0100 Subject: [PATCH 05/12] fix app v2 --- codec/collections.go | 1 + runtime/v2/app.go | 5 +++++ schema/decoding/middleware.go | 3 ++- server/v2/stf/go.mod | 2 ++ server/v2/stf/stf.go | 2 +- simapp/v2/app_di.go | 2 ++ simapp/v2/go.mod | 7 +++++++ simapp/v2/go.sum | 10 ++++++++-- x/accounts/keeper.go | 11 ++++++++--- x/accounts/module.go | 8 ++++++++ 10 files changed, 44 insertions(+), 7 deletions(-) diff --git a/codec/collections.go b/codec/collections.go index 3694558ad5c3..d21a9a92484d 100644 --- a/codec/collections.go +++ b/codec/collections.go @@ -398,6 +398,7 @@ func protoCol(f protoreflect.FieldDescriptor) schema.Field { col.Kind = schema.StringKind case protoreflect.BytesKind: col.Kind = schema.BytesKind + col.Nullable = true case protoreflect.EnumKind: // TODO: support enums col.Kind = schema.EnumKind diff --git a/runtime/v2/app.go b/runtime/v2/app.go index 0c017fdcbcd9..f2128b219def 100644 --- a/runtime/v2/app.go +++ b/runtime/v2/app.go @@ -94,6 +94,11 @@ func (a *App[T]) SchemaDecoderResolver() decoding.DecoderResolver { for moduleName, module := range a.moduleManager.Modules() { moduleSet[moduleName] = module } + + for _, overrideKey := range a.config.OverrideStoreKeys { + moduleSet[overrideKey.KvStoreKey] = moduleSet[overrideKey.ModuleName] + } + return decoding.ModuleSetDecoderResolver(moduleSet) } diff --git a/schema/decoding/middleware.go b/schema/decoding/middleware.go index 2c269dcab417..667f893e73d7 100644 --- a/schema/decoding/middleware.go +++ b/schema/decoding/middleware.go @@ -40,7 +40,8 @@ func Middleware(target appdata.Listener, resolver DecoderResolver, opts Middlewa var err error moduleName, err = resolver.DecodeModuleName(kvUpdate.Actor) if err != nil { - return err + // we don't have a codec for this module so continue + continue } moduleNames[string(kvUpdate.Actor)] = moduleName diff --git a/server/v2/stf/go.mod b/server/v2/stf/go.mod index 0352b590ccb4..ab2998b4a6c7 100644 --- a/server/v2/stf/go.mod +++ b/server/v2/stf/go.mod @@ -10,6 +10,8 @@ require ( github.com/tidwall/btree v1.7.0 ) +replace cosmossdk.io/schema => ../../schema + require ( github.com/google/go-cmp v0.6.0 // indirect google.golang.org/protobuf v1.35.2 // indirect diff --git a/server/v2/stf/stf.go b/server/v2/stf/stf.go index ca078782b0a9..4411a0f5fe40 100644 --- a/server/v2/stf/stf.go +++ b/server/v2/stf/stf.go @@ -417,7 +417,7 @@ func (s STF[T]) endBlock( } events = append(events, ctx.events...) for i := range events { - ctx.events[i].BlockNumber = uint64(ctx.headerInfo.Height) + events[i].BlockNumber = uint64(ctx.headerInfo.Height) events[i].BlockStage = appdata.EndBlockStage events[i].EventIndex = int32(i + 1) } diff --git a/simapp/v2/app_di.go b/simapp/v2/app_di.go index 1d028c1025e4..3a8423e05aec 100644 --- a/simapp/v2/app_di.go +++ b/simapp/v2/app_di.go @@ -19,11 +19,13 @@ import ( stakingkeeper "cosmossdk.io/x/staking/keeper" upgradekeeper "cosmossdk.io/x/upgrade/keeper" + _ "cosmossdk.io/indexer/postgres" // register the postgres indexer "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/codec" codectypes "github.com/cosmos/cosmos-sdk/codec/types" "github.com/cosmos/cosmos-sdk/std" _ "github.com/cosmos/cosmos-sdk/x/genutil" + _ "github.com/jackc/pgx/v5/stdlib" // Import and register pgx driver ) // SimApp extends an ABCI application, but with most of its parameters exported. diff --git a/simapp/v2/go.mod b/simapp/v2/go.mod index 0304f938cfd2..d63b9f53e30a 100644 --- a/simapp/v2/go.mod +++ b/simapp/v2/go.mod @@ -43,9 +43,11 @@ require ( ) require ( + cosmossdk.io/indexer/postgres v0.0.0-20241204160609-556102cfa046 cosmossdk.io/x/accounts/defaults/base v0.0.0-00010101000000-000000000000 cosmossdk.io/x/accounts/defaults/lockup v0.0.0-00010101000000-000000000000 cosmossdk.io/x/accounts/defaults/multisig v0.0.0-00010101000000-000000000000 + github.com/jackc/pgx/v5 v5.7.1 ) require ( @@ -156,6 +158,9 @@ require ( github.com/huandu/skiplist v1.2.1 // indirect github.com/iancoleman/strcase v0.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jmhodges/levigo v1.0.0 // indirect github.com/klauspost/compress v1.17.11 // indirect @@ -295,7 +300,9 @@ replace ( replace ( cosmossdk.io/api => ../../api cosmossdk.io/core/testing => ../../core/testing + cosmossdk.io/indexer/postgres => ../../indexer/postgres cosmossdk.io/runtime/v2 => ../../runtime/v2 + cosmossdk.io/schema => ../../schema cosmossdk.io/server/v2 => ../../server/v2 cosmossdk.io/server/v2/appmanager => ../../server/v2/appmanager cosmossdk.io/server/v2/cometbft => ../../server/v2/cometbft diff --git a/simapp/v2/go.sum b/simapp/v2/go.sum index 68882ebba027..f327ebe827a4 100644 --- a/simapp/v2/go.sum +++ b/simapp/v2/go.sum @@ -206,8 +206,6 @@ cosmossdk.io/log v1.5.0 h1:dVdzPJW9kMrnAYyMf1duqacoidB9uZIl+7c6z0mnq0g= cosmossdk.io/log v1.5.0/go.mod h1:Tr46PUJjiUthlwQ+hxYtUtPn4D/oCZXAkYevBeh5+FI= cosmossdk.io/math v1.4.0 h1:XbgExXFnXmF/CccPPEto40gOO7FpWu9yWNAZPN3nkNQ= cosmossdk.io/math v1.4.0/go.mod h1:O5PkD4apz2jZs4zqFdTr16e1dcaQCc5z6lkEnrrppuk= -cosmossdk.io/schema v0.3.1-0.20241128094659-bd76b47e1d8b h1:svpFdulZRrYz+RTHu2u9CeKkMKrIHx5354vjiHerovo= -cosmossdk.io/schema v0.3.1-0.20241128094659-bd76b47e1d8b/go.mod h1:RDAhxIeNB4bYqAlF4NBJwRrgtnciMcyyg0DOKnhNZQQ= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= @@ -603,6 +601,14 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs= +github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jhump/protoreflect v1.17.0 h1:qOEr613fac2lOuTgWN4tPAtLL7fUSbuJL5X5XumQh94= github.com/jhump/protoreflect v1.17.0/go.mod h1:h9+vUUL38jiBzck8ck+6G/aeMX8Z4QUY/NiJPwPNi+8= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= diff --git a/x/accounts/keeper.go b/x/accounts/keeper.go index cf306dab0e01..748bd84be54b 100644 --- a/x/accounts/keeper.go +++ b/x/accounts/keeper.go @@ -62,9 +62,14 @@ func NewKeeper( accounts: nil, Schema: collections.Schema{}, AccountNumber: collections.NewSequence(sb, AccountNumberKey, "account_number"), - AccountsByType: collections.NewMap(sb, AccountTypeKeyPrefix, "accounts_by_type", collections.BytesKey, collections.StringValue), - AccountByNumber: collections.NewMap(sb, AccountByNumber, "account_by_number", collections.BytesKey, collections.Uint64Value), - AccountsState: collections.NewMap(sb, implementation.AccountStatePrefix, "accounts_state", collections.PairKeyCodec(collections.Uint64Key, collections.BytesKey), collections.BytesValue), + AccountsByType: collections.NewMap(sb, AccountTypeKeyPrefix, "accounts_by_type", collections.BytesKey.WithName("address"), collections.StringValue.WithName("type")), + AccountByNumber: collections.NewMap(sb, AccountByNumber, "account_by_number", collections.BytesKey.WithName("address"), collections.Uint64Value.WithName("number")), + AccountsState: collections.NewMap(sb, implementation.AccountStatePrefix, "accounts_state", collections.NamedPairKeyCodec( + "number", + collections.Uint64Key, + "key", + collections.BytesKey, + ), collections.BytesValue), } schema, err := sb.Build() diff --git a/x/accounts/module.go b/x/accounts/module.go index daadebfe91ce..abc0f252a830 100644 --- a/x/accounts/module.go +++ b/x/accounts/module.go @@ -7,9 +7,11 @@ import ( "github.com/spf13/cobra" "google.golang.org/grpc" + "cosmossdk.io/collections" "cosmossdk.io/core/appmodule" appmodulev2 "cosmossdk.io/core/appmodule/v2" "cosmossdk.io/core/registry" + "cosmossdk.io/schema" "cosmossdk.io/x/accounts/cli" v1 "cosmossdk.io/x/accounts/v1" @@ -125,3 +127,9 @@ func (AppModule) GetQueryCmd() *cobra.Command { } func (AppModule) ConsensusVersion() uint64 { return ConsensusVersion } + +// ModuleCodec implements `schema.HasModuleCodec` interface. +// It allows the indexer to decode the module's KVPairUpdate. +func (am AppModule) ModuleCodec() (schema.ModuleCodec, error) { + return am.k.Schema.ModuleCodec(collections.IndexingOptions{}) +} From e29f0ff1473a04851e4b9cfc843b9f35d20c6777 Mon Sep 17 00:00:00 2001 From: Facundo Date: Thu, 5 Dec 2024 14:44:50 +0100 Subject: [PATCH 06/12] should be fixed --- tests/go.mod | 1 + x/accounts/go.mod | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/go.mod b/tests/go.mod index d95f20fb91ac..1536f005fa5f 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -287,6 +287,7 @@ replace ( replace ( // We always want to test against the latest version of the simapp. cosmossdk.io/simapp => ../simapp + cosmossdk.io/schema => ../schema github.com/99designs/keyring => github.com/cosmos/keyring v1.2.0 // We always want to test against the latest version of the SDK. github.com/cosmos/cosmos-sdk => ../. diff --git a/x/accounts/go.mod b/x/accounts/go.mod index 3434ece8e881..892216b659d2 100644 --- a/x/accounts/go.mod +++ b/x/accounts/go.mod @@ -178,6 +178,7 @@ replace github.com/cosmos/cosmos-sdk => ../../. // TODO remove post spinning out all modules replace ( cosmossdk.io/api => ../../api + cosmossdk.io/schema => ../../schema cosmossdk.io/store => ../../store cosmossdk.io/x/bank => ../bank cosmossdk.io/x/staking => ../staking From 7d0333ebaedb22ab67333b65491ec337c2b1703a Mon Sep 17 00:00:00 2001 From: Facundo Date: Thu, 5 Dec 2024 15:27:40 +0100 Subject: [PATCH 07/12] fixes --- server/v2/cometbft/abci_test.go | 22 +++++++++---------- .../cometbft/oe/optimistic_execution_test.go | 5 +++-- server/v2/cometbft/server.go | 2 +- simapp/v2/app_di.go | 5 +++-- x/distribution/go.mod | 1 + 5 files changed, 19 insertions(+), 16 deletions(-) diff --git a/server/v2/cometbft/abci_test.go b/server/v2/cometbft/abci_test.go index 7b20c354d786..ab1fdc722879 100644 --- a/server/v2/cometbft/abci_test.go +++ b/server/v2/cometbft/abci_test.go @@ -2,18 +2,16 @@ package cometbft import ( "context" - "cosmossdk.io/core/server" "crypto/sha256" "encoding/json" "errors" - abci "github.com/cometbft/cometbft/abci/types" "io" "strings" "sync" "testing" "time" - "cosmossdk.io/server/v2/cometbft/oe" + abci "github.com/cometbft/cometbft/abci/types" abciproto "github.com/cometbft/cometbft/api/cometbft/abci/v1" v1 "github.com/cometbft/cometbft/api/cometbft/types/v1" "github.com/cosmos/gogoproto/proto" @@ -21,6 +19,7 @@ import ( "github.com/stretchr/testify/require" appmodulev2 "cosmossdk.io/core/appmodule/v2" + "cosmossdk.io/core/server" "cosmossdk.io/core/store" "cosmossdk.io/core/transaction" "cosmossdk.io/log" @@ -28,6 +27,7 @@ import ( "cosmossdk.io/server/v2/cometbft/handlers" cometmock "cosmossdk.io/server/v2/cometbft/internal/mock" "cosmossdk.io/server/v2/cometbft/mempool" + "cosmossdk.io/server/v2/cometbft/oe" "cosmossdk.io/server/v2/cometbft/types" "cosmossdk.io/server/v2/stf" "cosmossdk.io/server/v2/stf/branch" @@ -60,10 +60,10 @@ func getQueryRouterBuilder[T any, PT interface { *T proto.Message }, -U any, UT interface { - *U - proto.Message -}]( + U any, UT interface { + *U + proto.Message + }]( t *testing.T, handler func(ctx context.Context, msg PT) (UT, error), ) *stf.MsgRouterBuilder { @@ -90,10 +90,10 @@ func getMsgRouterBuilder[T any, PT interface { *T transaction.Msg }, -U any, UT interface { - *U - transaction.Msg -}]( + U any, UT interface { + *U + transaction.Msg + }]( t *testing.T, handler func(ctx context.Context, msg PT) (UT, error), ) *stf.MsgRouterBuilder { diff --git a/server/v2/cometbft/oe/optimistic_execution_test.go b/server/v2/cometbft/oe/optimistic_execution_test.go index c0eb28c2a5e9..cd3c9263c784 100644 --- a/server/v2/cometbft/oe/optimistic_execution_test.go +++ b/server/v2/cometbft/oe/optimistic_execution_test.go @@ -5,12 +5,13 @@ import ( "errors" "testing" + abci "github.com/cometbft/cometbft/api/cometbft/abci/v1" + "github.com/stretchr/testify/assert" + "cosmossdk.io/core/server" "cosmossdk.io/core/store" "cosmossdk.io/core/transaction" "cosmossdk.io/log" - abci "github.com/cometbft/cometbft/api/cometbft/abci/v1" - "github.com/stretchr/testify/assert" ) func testFinalizeBlock[T transaction.Tx](context.Context, *abci.FinalizeBlockRequest) (*server.BlockResponse, store.WriterMap, []T, error) { diff --git a/server/v2/cometbft/server.go b/server/v2/cometbft/server.go index 6567e01caef0..55a38b5e9646 100644 --- a/server/v2/cometbft/server.go +++ b/server/v2/cometbft/server.go @@ -2,7 +2,6 @@ package cometbft import ( "context" - "cosmossdk.io/server/v2/cometbft/oe" "crypto/sha256" "encoding/json" "fmt" @@ -35,6 +34,7 @@ import ( "cosmossdk.io/server/v2/appmanager" cometlog "cosmossdk.io/server/v2/cometbft/log" "cosmossdk.io/server/v2/cometbft/mempool" + "cosmossdk.io/server/v2/cometbft/oe" "cosmossdk.io/server/v2/cometbft/types" "cosmossdk.io/store/v2/snapshots" diff --git a/simapp/v2/app_di.go b/simapp/v2/app_di.go index 3a8423e05aec..bd0a39472a1c 100644 --- a/simapp/v2/app_di.go +++ b/simapp/v2/app_di.go @@ -4,10 +4,13 @@ import ( _ "embed" "fmt" + _ "github.com/jackc/pgx/v5/stdlib" // Import and register pgx driver + "cosmossdk.io/core/registry" "cosmossdk.io/core/server" "cosmossdk.io/core/transaction" "cosmossdk.io/depinject" + _ "cosmossdk.io/indexer/postgres" // register the postgres indexer "cosmossdk.io/log" "cosmossdk.io/runtime/v2" serverstore "cosmossdk.io/server/v2/store" @@ -19,13 +22,11 @@ import ( stakingkeeper "cosmossdk.io/x/staking/keeper" upgradekeeper "cosmossdk.io/x/upgrade/keeper" - _ "cosmossdk.io/indexer/postgres" // register the postgres indexer "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/codec" codectypes "github.com/cosmos/cosmos-sdk/codec/types" "github.com/cosmos/cosmos-sdk/std" _ "github.com/cosmos/cosmos-sdk/x/genutil" - _ "github.com/jackc/pgx/v5/stdlib" // Import and register pgx driver ) // SimApp extends an ABCI application, but with most of its parameters exported. diff --git a/x/distribution/go.mod b/x/distribution/go.mod index 1f87fd3779c2..836049f61b24 100644 --- a/x/distribution/go.mod +++ b/x/distribution/go.mod @@ -175,6 +175,7 @@ replace github.com/cosmos/cosmos-sdk => ../../. // TODO remove post spinning out all modules replace ( cosmossdk.io/api => ../../api + cosmossdk.io/schema => ../../schema cosmossdk.io/store => ../../store cosmossdk.io/x/bank => ../bank cosmossdk.io/x/protocolpool => ../protocolpool From a0469e8af127468ee2710fefef907de9e953b673 Mon Sep 17 00:00:00 2001 From: Facundo Date: Thu, 5 Dec 2024 15:36:49 +0100 Subject: [PATCH 08/12] fixxxxxx --- server/v2/stf/go.mod | 2 +- server/v2/stf/go.sum | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/server/v2/stf/go.mod b/server/v2/stf/go.mod index ab2998b4a6c7..a6824044348b 100644 --- a/server/v2/stf/go.mod +++ b/server/v2/stf/go.mod @@ -10,7 +10,7 @@ require ( github.com/tidwall/btree v1.7.0 ) -replace cosmossdk.io/schema => ../../schema +replace cosmossdk.io/schema => ../../../schema require ( github.com/google/go-cmp v0.6.0 // indirect diff --git a/server/v2/stf/go.sum b/server/v2/stf/go.sum index 33a84a2d1cfe..c1679ad748ba 100644 --- a/server/v2/stf/go.sum +++ b/server/v2/stf/go.sum @@ -2,8 +2,6 @@ cosmossdk.io/core v1.0.0-alpha.6 h1:5ukC4JcQKmemLQXcAgu/QoOvJI50hpBkIIg4ZT2EN8E= cosmossdk.io/core v1.0.0-alpha.6/go.mod h1:3u9cWq1FAVtiiCrDPpo4LhR+9V6k/ycSG4/Y/tREWCY= cosmossdk.io/core/testing v0.0.0-20241108153815-606544c7be7e h1:F+ScucYxwrrDJU8guJXQXpGhdpziYSbxW6HMP2wCNxs= cosmossdk.io/core/testing v0.0.0-20241108153815-606544c7be7e/go.mod h1:3YvVv9aJayjPhdX0DY1IMrGse4sR63hNBWx2VtDWjGQ= -cosmossdk.io/schema v0.3.0 h1:01lcaM4trhzZ1HQTfTV8z6Ma1GziOZ/YmdzBN3F720c= -cosmossdk.io/schema v0.3.0/go.mod h1:RDAhxIeNB4bYqAlF4NBJwRrgtnciMcyyg0DOKnhNZQQ= github.com/cosmos/gogoproto v1.7.0 h1:79USr0oyXAbxg3rspGh/m4SWNyoz/GLaAh0QlCe2fro= github.com/cosmos/gogoproto v1.7.0/go.mod h1:yWChEv5IUEYURQasfyBW5ffkMHR/90hiHgbNgrtp4j0= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= From bccad4fb325e57361d356e9966c0b136a9738b32 Mon Sep 17 00:00:00 2001 From: Facundo Date: Thu, 5 Dec 2024 15:50:25 +0100 Subject: [PATCH 09/12] please work --- server/v2/cometbft/go.mod | 1 + 1 file changed, 1 insertion(+) diff --git a/server/v2/cometbft/go.mod b/server/v2/cometbft/go.mod index aa227ff3ffc6..03a129cade6b 100644 --- a/server/v2/cometbft/go.mod +++ b/server/v2/cometbft/go.mod @@ -3,6 +3,7 @@ module cosmossdk.io/server/v2/cometbft go 1.23.2 replace ( + cosmossdk.io/schema => ../../../schema cosmossdk.io/api => ../../../api cosmossdk.io/core/testing => ../../../core/testing cosmossdk.io/server/v2 => ../ From 689c530ac572c96a2d2872bbdc07fc4e7ca29f74 Mon Sep 17 00:00:00 2001 From: Facundo Date: Thu, 5 Dec 2024 15:55:09 +0100 Subject: [PATCH 10/12] no comment --- runtime/v2/go.mod | 1 + 1 file changed, 1 insertion(+) diff --git a/runtime/v2/go.mod b/runtime/v2/go.mod index e9653bd73ece..66f5dae191ba 100644 --- a/runtime/v2/go.mod +++ b/runtime/v2/go.mod @@ -5,6 +5,7 @@ go 1.23 // server v2 integration replace ( cosmossdk.io/api => ../../api + cosmossdk.io/schema => ../../schema cosmossdk.io/core/testing => ../../core/testing cosmossdk.io/server/v2/appmanager => ../../server/v2/appmanager cosmossdk.io/server/v2/stf => ../../server/v2/stf From 9a575f09f0240d34e4defc290d749c286a70ca8d Mon Sep 17 00:00:00 2001 From: Facundo Date: Mon, 9 Dec 2024 12:27:25 +0100 Subject: [PATCH 11/12] fix --- indexer/postgres/base_sql.go | 3 +-- indexer/postgres/listener.go | 15 +++++---------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/indexer/postgres/base_sql.go b/indexer/postgres/base_sql.go index 076e245dd821..7721d96bf400 100644 --- a/indexer/postgres/base_sql.go +++ b/indexer/postgres/base_sql.go @@ -30,7 +30,6 @@ CREATE TABLE IF NOT EXISTS event msg_index BIGINT NOT NULL, event_index BIGINT NOT NULL, type TEXT NULL, - data JSONB NULL, - attributes JSONB NULL + data JSONB NULL ); ` diff --git a/indexer/postgres/listener.go b/indexer/postgres/listener.go index 6cc03c64727c..b7199ea5f0b0 100644 --- a/indexer/postgres/listener.go +++ b/indexer/postgres/listener.go @@ -117,10 +117,7 @@ func txListener(i *indexerImpl) func(data appdata.TxData) error { func eventListener(i *indexerImpl) func(data appdata.EventData) error { return func(data appdata.EventData) error { for _, e := range data.Events { - var ( - jsonData json.RawMessage - jsonAttributes json.RawMessage - ) + var jsonData json.RawMessage if e.Data != nil { var err error @@ -128,9 +125,7 @@ func eventListener(i *indexerImpl) func(data appdata.EventData) error { if err != nil { return fmt.Errorf("failed to get event data: %w", err) } - } - - if e.Attributes != nil { + } else if e.Attributes != nil { attrs, err := e.Attributes() if err != nil { return fmt.Errorf("failed to get event attributes: %w", err) @@ -141,14 +136,14 @@ func eventListener(i *indexerImpl) func(data appdata.EventData) error { attrsMap[attr.Key] = attr.Value } - jsonAttributes, err = json.Marshal(attrsMap) + jsonData, err = json.Marshal(attrsMap) if err != nil { return fmt.Errorf("failed to marshal event attributes: %w", err) } } - _, err := i.tx.Exec("INSERT INTO event (block_number, block_stage, tx_index, msg_index, event_index, type, data, attributes) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", - e.BlockNumber, e.BlockStage, e.TxIndex, e.MsgIndex, e.EventIndex, e.Type, jsonData, jsonAttributes) + _, err := i.tx.Exec("INSERT INTO event (block_number, block_stage, tx_index, msg_index, event_index, type, data) VALUES ($1, $2, $3, $4, $5, $6, $7)", + e.BlockNumber, e.BlockStage, e.TxIndex, e.MsgIndex, e.EventIndex, e.Type, jsonData) if err != nil { return fmt.Errorf("failed to index event: %w", err) } From 85b6ebeeeef00365cc071af6b0614eeb50a62e92 Mon Sep 17 00:00:00 2001 From: Facundo Date: Mon, 9 Dec 2024 18:42:09 +0100 Subject: [PATCH 12/12] go mod tidy all --- runtime/v2/go.mod | 2 +- runtime/v2/go.sum | 2 -- server/v2/cometbft/go.mod | 2 +- server/v2/cometbft/go.sum | 2 -- server/v2/go.sum | 2 -- simapp/go.mod | 2 +- simapp/go.sum | 5 ----- tests/go.mod | 2 +- tests/go.sum | 3 --- x/accounts/go.mod | 2 +- x/accounts/go.sum | 2 -- x/distribution/go.sum | 2 -- 12 files changed, 5 insertions(+), 23 deletions(-) diff --git a/runtime/v2/go.mod b/runtime/v2/go.mod index 66f5dae191ba..9f21144d9085 100644 --- a/runtime/v2/go.mod +++ b/runtime/v2/go.mod @@ -5,8 +5,8 @@ go 1.23 // server v2 integration replace ( cosmossdk.io/api => ../../api - cosmossdk.io/schema => ../../schema cosmossdk.io/core/testing => ../../core/testing + cosmossdk.io/schema => ../../schema cosmossdk.io/server/v2/appmanager => ../../server/v2/appmanager cosmossdk.io/server/v2/stf => ../../server/v2/stf cosmossdk.io/store/v2 => ../../store/v2 diff --git a/runtime/v2/go.sum b/runtime/v2/go.sum index ffa2e8b402ff..47a8ece5dc11 100644 --- a/runtime/v2/go.sum +++ b/runtime/v2/go.sum @@ -10,8 +10,6 @@ cosmossdk.io/errors/v2 v2.0.0-20240731132947-df72853b3ca5 h1:IQNdY2kB+k+1OM2DvqF cosmossdk.io/errors/v2 v2.0.0-20240731132947-df72853b3ca5/go.mod h1:0CuYKkFHxc1vw2JC+t21THBCALJVROrWVR/3PQ1urpc= cosmossdk.io/log v1.5.0 h1:dVdzPJW9kMrnAYyMf1duqacoidB9uZIl+7c6z0mnq0g= cosmossdk.io/log v1.5.0/go.mod h1:Tr46PUJjiUthlwQ+hxYtUtPn4D/oCZXAkYevBeh5+FI= -cosmossdk.io/schema v0.3.0 h1:01lcaM4trhzZ1HQTfTV8z6Ma1GziOZ/YmdzBN3F720c= -cosmossdk.io/schema v0.3.0/go.mod h1:RDAhxIeNB4bYqAlF4NBJwRrgtnciMcyyg0DOKnhNZQQ= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= diff --git a/server/v2/cometbft/go.mod b/server/v2/cometbft/go.mod index 03a129cade6b..a555dd2e0477 100644 --- a/server/v2/cometbft/go.mod +++ b/server/v2/cometbft/go.mod @@ -3,9 +3,9 @@ module cosmossdk.io/server/v2/cometbft go 1.23.2 replace ( - cosmossdk.io/schema => ../../../schema cosmossdk.io/api => ../../../api cosmossdk.io/core/testing => ../../../core/testing + cosmossdk.io/schema => ../../../schema cosmossdk.io/server/v2 => ../ cosmossdk.io/server/v2/appmanager => ../appmanager cosmossdk.io/server/v2/stf => ../stf diff --git a/server/v2/cometbft/go.sum b/server/v2/cometbft/go.sum index de4a9135bc90..636d06f2f3ee 100644 --- a/server/v2/cometbft/go.sum +++ b/server/v2/cometbft/go.sum @@ -18,8 +18,6 @@ cosmossdk.io/log v1.5.0 h1:dVdzPJW9kMrnAYyMf1duqacoidB9uZIl+7c6z0mnq0g= cosmossdk.io/log v1.5.0/go.mod h1:Tr46PUJjiUthlwQ+hxYtUtPn4D/oCZXAkYevBeh5+FI= cosmossdk.io/math v1.4.0 h1:XbgExXFnXmF/CccPPEto40gOO7FpWu9yWNAZPN3nkNQ= cosmossdk.io/math v1.4.0/go.mod h1:O5PkD4apz2jZs4zqFdTr16e1dcaQCc5z6lkEnrrppuk= -cosmossdk.io/schema v0.3.1-0.20241128094659-bd76b47e1d8b h1:svpFdulZRrYz+RTHu2u9CeKkMKrIHx5354vjiHerovo= -cosmossdk.io/schema v0.3.1-0.20241128094659-bd76b47e1d8b/go.mod h1:RDAhxIeNB4bYqAlF4NBJwRrgtnciMcyyg0DOKnhNZQQ= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= diff --git a/server/v2/go.sum b/server/v2/go.sum index e494b8b56be5..adc8f292ddeb 100644 --- a/server/v2/go.sum +++ b/server/v2/go.sum @@ -8,8 +8,6 @@ cosmossdk.io/errors/v2 v2.0.0-20240731132947-df72853b3ca5 h1:IQNdY2kB+k+1OM2DvqF cosmossdk.io/errors/v2 v2.0.0-20240731132947-df72853b3ca5/go.mod h1:0CuYKkFHxc1vw2JC+t21THBCALJVROrWVR/3PQ1urpc= cosmossdk.io/log v1.5.0 h1:dVdzPJW9kMrnAYyMf1duqacoidB9uZIl+7c6z0mnq0g= cosmossdk.io/log v1.5.0/go.mod h1:Tr46PUJjiUthlwQ+hxYtUtPn4D/oCZXAkYevBeh5+FI= -cosmossdk.io/schema v0.3.1-0.20241010135032-192601639cac h1:3joNZZWZ3k7fMsrBDL1ktuQ2xQwYLZOaDhkruadDFmc= -cosmossdk.io/schema v0.3.1-0.20241010135032-192601639cac/go.mod h1:RDAhxIeNB4bYqAlF4NBJwRrgtnciMcyyg0DOKnhNZQQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/datadog-go v4.8.3+incompatible h1:fNGaYSuObuQb5nzeTQqowRAd9bpDIRRV4/gUtIBjh8Q= diff --git a/simapp/go.mod b/simapp/go.mod index d0644f966d7e..d545a6687609 100644 --- a/simapp/go.mod +++ b/simapp/go.mod @@ -252,8 +252,8 @@ replace ( cosmossdk.io/api => ../api cosmossdk.io/client/v2 => ../client/v2 cosmossdk.io/collections => ../collections - cosmossdk.io/schema => ../schema cosmossdk.io/indexer/postgres => ../indexer/postgres + cosmossdk.io/schema => ../schema cosmossdk.io/store => ../store cosmossdk.io/tools/confix => ../tools/confix cosmossdk.io/x/accounts => ../x/accounts diff --git a/simapp/go.sum b/simapp/go.sum index fbfdbb723105..ca33041412fd 100644 --- a/simapp/go.sum +++ b/simapp/go.sum @@ -200,15 +200,10 @@ cosmossdk.io/depinject v1.1.0 h1:wLan7LG35VM7Yo6ov0jId3RHWCGRhe8E8bsuARorl5E= cosmossdk.io/depinject v1.1.0/go.mod h1:kkI5H9jCGHeKeYWXTqYdruogYrEeWvBQCw1Pj4/eCFI= cosmossdk.io/errors v1.0.1 h1:bzu+Kcr0kS/1DuPBtUFdWjzLqyUuCiyHjyJB6srBV/0= cosmossdk.io/errors v1.0.1/go.mod h1:MeelVSZThMi4bEakzhhhE/CKqVv3nOJDA25bIqRDu/U= -cosmossdk.io/indexer/postgres v0.0.0-20241128094659-bd76b47e1d8b h1:/5zsEUbJNY6guf7a8y0k6Fk7bUlpCgaZuy8bZNMCXTs= -cosmossdk.io/indexer/postgres v0.0.0-20241128094659-bd76b47e1d8b/go.mod h1:vb5uiIC3rDjz+V7UaSLNA3g5TpbRygWi652qtMugWHA= cosmossdk.io/log v1.5.0 h1:dVdzPJW9kMrnAYyMf1duqacoidB9uZIl+7c6z0mnq0g= cosmossdk.io/log v1.5.0/go.mod h1:Tr46PUJjiUthlwQ+hxYtUtPn4D/oCZXAkYevBeh5+FI= cosmossdk.io/math v1.4.0 h1:XbgExXFnXmF/CccPPEto40gOO7FpWu9yWNAZPN3nkNQ= cosmossdk.io/math v1.4.0/go.mod h1:O5PkD4apz2jZs4zqFdTr16e1dcaQCc5z6lkEnrrppuk= -cosmossdk.io/schema v0.3.0/go.mod h1:RDAhxIeNB4bYqAlF4NBJwRrgtnciMcyyg0DOKnhNZQQ= -cosmossdk.io/schema v0.3.1-0.20241128094659-bd76b47e1d8b h1:svpFdulZRrYz+RTHu2u9CeKkMKrIHx5354vjiHerovo= -cosmossdk.io/schema v0.3.1-0.20241128094659-bd76b47e1d8b/go.mod h1:RDAhxIeNB4bYqAlF4NBJwRrgtnciMcyyg0DOKnhNZQQ= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= diff --git a/tests/go.mod b/tests/go.mod index 1536f005fa5f..7f7ac4bcee59 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -285,9 +285,9 @@ replace ( // Below are the long-lived replace for tests. replace ( + cosmossdk.io/schema => ../schema // We always want to test against the latest version of the simapp. cosmossdk.io/simapp => ../simapp - cosmossdk.io/schema => ../schema github.com/99designs/keyring => github.com/cosmos/keyring v1.2.0 // We always want to test against the latest version of the SDK. github.com/cosmos/cosmos-sdk => ../. diff --git a/tests/go.sum b/tests/go.sum index 18f3a2b9b2de..9a1d6ebad18a 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -204,9 +204,6 @@ cosmossdk.io/log v1.5.0 h1:dVdzPJW9kMrnAYyMf1duqacoidB9uZIl+7c6z0mnq0g= cosmossdk.io/log v1.5.0/go.mod h1:Tr46PUJjiUthlwQ+hxYtUtPn4D/oCZXAkYevBeh5+FI= cosmossdk.io/math v1.4.0 h1:XbgExXFnXmF/CccPPEto40gOO7FpWu9yWNAZPN3nkNQ= cosmossdk.io/math v1.4.0/go.mod h1:O5PkD4apz2jZs4zqFdTr16e1dcaQCc5z6lkEnrrppuk= -cosmossdk.io/schema v0.3.0/go.mod h1:RDAhxIeNB4bYqAlF4NBJwRrgtnciMcyyg0DOKnhNZQQ= -cosmossdk.io/schema v0.3.1-0.20241128094659-bd76b47e1d8b h1:svpFdulZRrYz+RTHu2u9CeKkMKrIHx5354vjiHerovo= -cosmossdk.io/schema v0.3.1-0.20241128094659-bd76b47e1d8b/go.mod h1:RDAhxIeNB4bYqAlF4NBJwRrgtnciMcyyg0DOKnhNZQQ= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= diff --git a/x/accounts/go.mod b/x/accounts/go.mod index 892216b659d2..37c012bccf65 100644 --- a/x/accounts/go.mod +++ b/x/accounts/go.mod @@ -39,7 +39,7 @@ require ( cosmossdk.io/errors v1.0.1 cosmossdk.io/log v1.5.0 // indirect cosmossdk.io/math v1.4.0 - cosmossdk.io/schema v0.3.1-0.20241128094659-bd76b47e1d8b // indirect + cosmossdk.io/schema v0.3.1-0.20241128094659-bd76b47e1d8b cosmossdk.io/store v1.1.1-0.20240418092142-896cdf1971bc // indirect cosmossdk.io/x/staking v0.0.0-00010101000000-000000000000 // indirect filippo.io/edwards25519 v1.1.0 // indirect diff --git a/x/accounts/go.sum b/x/accounts/go.sum index 52cb63118a04..2175eeefca9b 100644 --- a/x/accounts/go.sum +++ b/x/accounts/go.sum @@ -18,8 +18,6 @@ cosmossdk.io/log v1.5.0 h1:dVdzPJW9kMrnAYyMf1duqacoidB9uZIl+7c6z0mnq0g= cosmossdk.io/log v1.5.0/go.mod h1:Tr46PUJjiUthlwQ+hxYtUtPn4D/oCZXAkYevBeh5+FI= cosmossdk.io/math v1.4.0 h1:XbgExXFnXmF/CccPPEto40gOO7FpWu9yWNAZPN3nkNQ= cosmossdk.io/math v1.4.0/go.mod h1:O5PkD4apz2jZs4zqFdTr16e1dcaQCc5z6lkEnrrppuk= -cosmossdk.io/schema v0.3.1-0.20241128094659-bd76b47e1d8b h1:svpFdulZRrYz+RTHu2u9CeKkMKrIHx5354vjiHerovo= -cosmossdk.io/schema v0.3.1-0.20241128094659-bd76b47e1d8b/go.mod h1:RDAhxIeNB4bYqAlF4NBJwRrgtnciMcyyg0DOKnhNZQQ= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= diff --git a/x/distribution/go.sum b/x/distribution/go.sum index 52cb63118a04..2175eeefca9b 100644 --- a/x/distribution/go.sum +++ b/x/distribution/go.sum @@ -18,8 +18,6 @@ cosmossdk.io/log v1.5.0 h1:dVdzPJW9kMrnAYyMf1duqacoidB9uZIl+7c6z0mnq0g= cosmossdk.io/log v1.5.0/go.mod h1:Tr46PUJjiUthlwQ+hxYtUtPn4D/oCZXAkYevBeh5+FI= cosmossdk.io/math v1.4.0 h1:XbgExXFnXmF/CccPPEto40gOO7FpWu9yWNAZPN3nkNQ= cosmossdk.io/math v1.4.0/go.mod h1:O5PkD4apz2jZs4zqFdTr16e1dcaQCc5z6lkEnrrppuk= -cosmossdk.io/schema v0.3.1-0.20241128094659-bd76b47e1d8b h1:svpFdulZRrYz+RTHu2u9CeKkMKrIHx5354vjiHerovo= -cosmossdk.io/schema v0.3.1-0.20241128094659-bd76b47e1d8b/go.mod h1:RDAhxIeNB4bYqAlF4NBJwRrgtnciMcyyg0DOKnhNZQQ= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs=