From e37bcdf59f2f4d3c3e9a6b2899eff646247c316e Mon Sep 17 00:00:00 2001 From: Tilen Marc Date: Thu, 22 Aug 2024 11:55:11 +0000 Subject: [PATCH] Support for Ethereum and Avalanche based chains --- .gitlab-ci.yml | 2 +- README.md | 1 + benchmarks/songbird_test.go | 5 +- chain/chain.go | 333 +++++++++++++++++++++++++++++++ config/config.go | 8 +- database/history_drop.go | 11 +- go.mod | 13 +- go.sum | 35 ++-- indexer/blocks.go | 16 +- indexer/indexer.go | 11 +- indexer/indexer_test.go | 6 +- indexer/transactions.go | 27 +-- main.go | 15 +- main_test.go | 19 +- testing/chain_copy/chain_copy.go | 2 +- 15 files changed, 416 insertions(+), 88 deletions(-) create mode 100644 chain/chain.go diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7dfefcb..0ff4cd5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ variables: GOPATH: /go MYSQL_ROOT_PASSWORD: root MYSQL_DATABASE: flare_ftso_indexer_test - GOLANG_VERSION: "1.21.11" + GOLANG_VERSION: "1.21.12" GOLINT_VERSION: "v1.59.1" .gocache: diff --git a/README.md b/README.md index aa6a81e..ac09a39 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,7 @@ console = true [chain] node_url = "http://127.0.0.1:8545/" # or NODE_URL environment variable # api_key = ... or NODE_API_KEY environment variable +# chain_type = 1 # default Avalanche based chain=1, Ethereum based chain=2 [timeout] backoff_max_elapsed_time_seconds = 300 # optional, defaults to 300s = 5 minutes. Affects how long the indexer will keep retrying in case of a complete outage of the node provider. Set to 0 to retry indefinitely. diff --git a/benchmarks/songbird_test.go b/benchmarks/songbird_test.go index f088b7f..d43f894 100644 --- a/benchmarks/songbird_test.go +++ b/benchmarks/songbird_test.go @@ -2,13 +2,12 @@ package benchmarks import ( "context" + "flare-ftso-indexer/chain" "flare-ftso-indexer/config" "flare-ftso-indexer/database" "flare-ftso-indexer/indexer" "flare-ftso-indexer/logger" "testing" - - "github.com/ava-labs/coreth/ethclient" ) func BenchmarkBlockRequests(b *testing.B) { @@ -31,7 +30,7 @@ func BenchmarkBlockRequests(b *testing.B) { logger.Fatal("Database connect and initialize error: %s", err) } - ethClient, err := ethclient.Dial(cfg.Chain.NodeURL) + ethClient, err := chain.DialRPCNode(cfg) if err != nil { logger.Fatal("Eth client error: %s", err) } diff --git a/chain/chain.go b/chain/chain.go new file mode 100644 index 0000000..dedcc67 --- /dev/null +++ b/chain/chain.go @@ -0,0 +1,333 @@ +package chain + +import ( + "context" + "errors" + "flare-ftso-indexer/config" + "fmt" + "math/big" + + avxClient "github.com/ava-labs/coreth/ethclient" + "github.com/ava-labs/coreth/interfaces" + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + ethClient "github.com/ethereum/go-ethereum/ethclient" + + avxTypes "github.com/ava-labs/coreth/core/types" + ethTypes "github.com/ethereum/go-ethereum/core/types" +) + +type ChainType int + +const ( + ChainTypeAvax ChainType = iota + 1 // Add 1 to skip 0 - avoids the zero value defaulting to Avax + ChainTypeEth +) + +type Client struct { + chain ChainType + eth *ethClient.Client + avx avxClient.Client +} + +type Block struct { + chain ChainType + eth *ethTypes.Block + avx *avxTypes.Block +} + +type Header struct { + chain ChainType + eth *ethTypes.Header + avx *avxTypes.Header +} + +type Receipt struct { + chain ChainType + eth *ethTypes.Receipt + avx *avxTypes.Receipt +} + +type Transaction struct { + chain ChainType + eth *ethTypes.Transaction + avx *avxTypes.Transaction +} + +func DialRPCNode(cfg *config.Config) (*Client, error) { + nodeURL, err := cfg.Chain.FullNodeURL() + if err != nil { + return nil, err + } + c := &Client{chain: ChainType(cfg.Chain.ChainType)} + switch c.chain { + case ChainTypeAvax: + c.avx, err = avxClient.Dial(nodeURL.String()) + case ChainTypeEth: + c.eth, err = ethClient.Dial(nodeURL.String()) + default: + return nil, errors.New("invalid chain") + } + + return c, err +} + +func (c *Client) BlockByNumber(ctx context.Context, number *big.Int) (*Block, error) { + block := &Block{chain: c.chain} + var err error + switch c.chain { + case ChainTypeAvax: + block.avx, err = c.avx.BlockByNumber(ctx, number) + case ChainTypeEth: + block.eth, err = c.eth.BlockByNumber(ctx, number) + default: + return nil, errors.New("invalid chain") + } + + return block, err +} + +func (c *Client) HeaderByNumber(ctx context.Context, number *big.Int) (*Header, error) { + block := &Header{chain: c.chain} + var err error + switch c.chain { + case ChainTypeAvax: + block.avx, err = c.avx.HeaderByNumber(ctx, number) + case ChainTypeEth: + block.eth, err = c.eth.HeaderByNumber(ctx, number) + default: + return nil, errors.New("invalid chain") + } + + return block, err +} + +func (c *Client) TransactionReceipt(ctx context.Context, txHash common.Hash) (*Receipt, error) { + receipt := &Receipt{chain: c.chain} + var err error + switch c.chain { + case ChainTypeAvax: + receipt.avx, err = c.avx.TransactionReceipt(ctx, txHash) + case ChainTypeEth: + receipt.eth, err = c.eth.TransactionReceipt(ctx, txHash) + default: + return nil, errors.New("invalid chain") + } + + return receipt, err +} + +func (c *Client) FilterLogs(ctx context.Context, q interfaces.FilterQuery) ([]avxTypes.Log, error) { + switch c.chain { + case ChainTypeAvax: + return c.avx.FilterLogs(ctx, q) + case ChainTypeEth: + ethLogs, err := c.eth.FilterLogs(ctx, ethereum.FilterQuery(q)) + if err != nil { + return nil, err + } + logs := make([]avxTypes.Log, len(ethLogs)) + for i, e := range ethLogs { + logs[i] = avxTypes.Log(e) + } + return logs, nil + + default: + return nil, errors.New("invalid chain") + } +} + +func (b *Header) Number() *big.Int { + switch b.chain { + case ChainTypeAvax: + return b.avx.Number + case ChainTypeEth: + return b.eth.Number + default: + return nil + } +} + +func (b *Header) Time() uint64 { + switch b.chain { + case ChainTypeAvax: + return b.avx.Time + case ChainTypeEth: + return b.eth.Time + default: + return 0 + } +} + +func (b *Block) Number() *big.Int { + switch b.chain { + case ChainTypeAvax: + return b.avx.Number() + case ChainTypeEth: + return b.eth.Number() + default: + return nil + } +} + +func (b *Block) Time() uint64 { + switch b.chain { + case ChainTypeAvax: + return b.avx.Time() + case ChainTypeEth: + return b.eth.Time() + default: + return 0 + } +} + +func (b *Block) Hash() common.Hash { + switch b.chain { + case ChainTypeAvax: + return b.avx.Hash() + case ChainTypeEth: + return b.eth.Hash() + default: + return common.Hash{} + } +} + +func (r *Receipt) Status() uint64 { + switch r.chain { + case ChainTypeAvax: + return r.avx.Status + case ChainTypeEth: + return r.eth.Status + default: + return 0 + } +} + +func (r *Receipt) Logs() []*avxTypes.Log { + switch r.chain { + case ChainTypeAvax: + return r.avx.Logs + case ChainTypeEth: + logs := make([]*avxTypes.Log, len(r.eth.Logs)) + for i, e := range r.eth.Logs { + log := avxTypes.Log(*e) + logs[i] = &log + } + return logs + default: + return nil + } +} + +func (b *Block) Transactions() []*Transaction { + switch b.chain { + case ChainTypeAvax: + txsAvx := b.avx.Transactions() + txs := make([]*Transaction, len(txsAvx)) + for i, e := range txsAvx { + txs[i] = &Transaction{} + txs[i].chain = b.chain + txs[i].avx = e + } + return txs + case ChainTypeEth: + txsEth := b.eth.Transactions() + txs := make([]*Transaction, len(txsEth)) + for i, e := range txsEth { + txs[i] = &Transaction{} + txs[i].chain = b.chain + txs[i].eth = e + } + return txs + default: + return nil + } +} + +func (t *Transaction) Hash() common.Hash { + switch t.chain { + case ChainTypeAvax: + return t.avx.Hash() + case ChainTypeEth: + return t.eth.Hash() + default: + return common.Hash{} + } +} + +func (t *Transaction) To() *common.Address { + switch t.chain { + case ChainTypeAvax: + return t.avx.To() + case ChainTypeEth: + return t.eth.To() + default: + return nil + } +} + +func (t *Transaction) Data() []byte { + switch t.chain { + case ChainTypeAvax: + return t.avx.Data() + case ChainTypeEth: + return t.eth.Data() + default: + return nil + } +} + +func (t *Transaction) ChainId() *big.Int { + switch t.chain { + case ChainTypeAvax: + return t.avx.ChainId() + case ChainTypeEth: + return t.eth.ChainId() + default: + return nil + } +} + +func (t *Transaction) Value() *big.Int { + switch t.chain { + case ChainTypeAvax: + return t.avx.Value() + case ChainTypeEth: + return t.eth.Value() + default: + return nil + } +} + +func (t *Transaction) GasPrice() *big.Int { + switch t.chain { + case ChainTypeAvax: + return t.avx.GasPrice() + case ChainTypeEth: + return t.eth.GasPrice() + default: + return nil + } +} + +func (t *Transaction) Gas() uint64 { + switch t.chain { + case ChainTypeAvax: + return t.avx.Gas() + case ChainTypeEth: + return t.eth.Gas() + default: + return 0 + } +} + +func (t *Transaction) FromAddress() (common.Address, error) { + switch t.chain { + case ChainTypeAvax: + return avxTypes.Sender(avxTypes.LatestSignerForChainID(t.avx.ChainId()), t.avx) + case ChainTypeEth: + return ethTypes.Sender(ethTypes.LatestSignerForChainID(t.eth.ChainId()), t.eth) + default: + return common.Address{}, fmt.Errorf("wrong chain") + } +} diff --git a/config/config.go b/config/config.go index 3759cf4..2b5e220 100644 --- a/config/config.go +++ b/config/config.go @@ -19,6 +19,7 @@ var ( ) const defaultConfirmations = 1 +const defaultChainType = 1 func init() { GlobalConfigCallback.AddCallback(func(config GlobalConfig) { @@ -66,8 +67,9 @@ type DBConfig struct { } type ChainConfig struct { - NodeURL string `toml:"node_url"` - APIKey string `toml:"api_key"` + NodeURL string `toml:"node_url"` + APIKey string `toml:"api_key"` + ChainType int `toml:"chain_type"` } type IndexerConfig struct { @@ -102,7 +104,7 @@ type LogInfo struct { func BuildConfig() (*Config, error) { cfgFileName := *CfgFlag - cfg := &Config{Indexer: IndexerConfig{Confirmations: defaultConfirmations}} + cfg := &Config{Indexer: IndexerConfig{Confirmations: defaultConfirmations}, Chain: ChainConfig{ChainType: defaultChainType}} err := parseConfigFile(cfg, cfgFileName) if err != nil { return nil, err diff --git a/database/history_drop.go b/database/history_drop.go index 0cd482a..13594b6 100644 --- a/database/history_drop.go +++ b/database/history_drop.go @@ -2,20 +2,19 @@ package database import ( "context" + "flare-ftso-indexer/chain" "flare-ftso-indexer/config" "flare-ftso-indexer/logger" "math/big" "time" - "github.com/ava-labs/coreth/core/types" - "github.com/ava-labs/coreth/ethclient" "github.com/cenkalti/backoff/v4" "github.com/pkg/errors" "gorm.io/gorm" ) func DropHistory( - ctx context.Context, db *gorm.DB, intervalSeconds, checkInterval uint64, client ethclient.Client, + ctx context.Context, db *gorm.DB, intervalSeconds, checkInterval uint64, client *chain.Client, ) { for { logger.Info("starting DropHistory iteration") @@ -44,7 +43,7 @@ var deleteOrder []interface{} = []interface{}{ const deleteBatchSize = 1000 func DropHistoryIteration( - ctx context.Context, db *gorm.DB, intervalSeconds uint64, client ethclient.Client, + ctx context.Context, db *gorm.DB, intervalSeconds uint64, client *chain.Client, ) (uint64, error) { lastBlockTime, _, err := getBlockTimestamp(ctx, nil, client) if err != nil { @@ -99,11 +98,11 @@ func deleteInBatches(db *gorm.DB, deleteStart uint64, entity interface{}) error } } -func getBlockTimestamp(ctx context.Context, index *big.Int, client ethclient.Client) (uint64, uint64, error) { +func getBlockTimestamp(ctx context.Context, index *big.Int, client *chain.Client) (uint64, uint64, error) { bOff := backoff.NewExponentialBackOff() bOff.MaxElapsedTime = config.BackoffMaxElapsedTime - var block *types.Block + var block *chain.Block err := backoff.RetryNotify( func() (err error) { ctx, cancelFunc := context.WithTimeout(ctx, config.Timeout) diff --git a/go.mod b/go.mod index 5b3262a..2b66020 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,12 @@ module flare-ftso-indexer -go 1.21.11 +go 1.21.12 toolchain go1.22.4 require ( github.com/BurntSushi/toml v1.2.1 - github.com/ava-labs/coreth v0.13.5 + github.com/ava-labs/coreth v0.13.7 github.com/bradleyjkemp/cupaloy/v2 v2.8.0 github.com/caarlos0/env/v10 v10.0.0 github.com/cenkalti/backoff/v4 v4.2.1 @@ -24,8 +24,9 @@ require ( require ( github.com/DataDog/zstd v1.5.2 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect github.com/VictoriaMetrics/fastcache v1.12.1 // indirect - github.com/ava-labs/avalanchego v1.11.7 // indirect + github.com/ava-labs/avalanchego v1.11.10-prerelease // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.10.0 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect @@ -41,7 +42,7 @@ require ( github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect github.com/google/renameio/v2 v2.0.0 // indirect github.com/google/uuid v1.6.0 // indirect @@ -79,16 +80,18 @@ require ( go.uber.org/multierr v1.10.0 // indirect golang.org/x/crypto v0.21.0 // indirect golang.org/x/exp v0.0.0-20231127185646-65229373498e // indirect + golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.17.0 // indirect gonum.org/v1/gonum v0.11.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect google.golang.org/grpc v1.62.0 // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) diff --git a/go.sum b/go.sum index 633d5fa..d22be91 100644 --- a/go.sum +++ b/go.sum @@ -2,14 +2,16 @@ github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bwt3uRKnkZU40= github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= -github.com/ava-labs/avalanchego v1.11.7 h1:BemCJEa6+f79fzRLdR0+iJAA3v9shOyhVniAKuibFyk= -github.com/ava-labs/avalanchego v1.11.7/go.mod h1:aPYTETkM0KjtC7vFwPO6S8z2L0QTKaXjVUi98pTdNO4= -github.com/ava-labs/coreth v0.13.5 h1:b2LepWFniva3XMJonxptU+k2GUHEgHe/qK95oFEZw/0= -github.com/ava-labs/coreth v0.13.5/go.mod h1:vm9T8qzP7RLo/jR2MKkliPfaiGgWeEpu/PG6fvvPmog= +github.com/ava-labs/avalanchego v1.11.10-prerelease h1:QUhVqvxrwDmsTvXXnKoePe5WU3Eh8GXB/QE2R2xHa0c= +github.com/ava-labs/avalanchego v1.11.10-prerelease/go.mod h1:ryRFbHr7sKmez4792NxzJS7AGiE+vd0Tez+qs2kmezE= +github.com/ava-labs/coreth v0.13.7 h1:k8T9u/ROifl8f7oXjHRc1KvSISRl9txvy7gGVmHEz6g= +github.com/ava-labs/coreth v0.13.7/go.mod h1:tXDujonxXFOF6oK5HS2EmgtSXJK3Gy6RpZxb5WzR9rM= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.10.0 h1:ePXTeiPEazB5+opbv5fr8umg2R/1NlzgDsyepwsSr88= @@ -84,17 +86,17 @@ github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg= @@ -120,6 +122,10 @@ github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZ github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.2.4 h1:jUc4Nk8fm9jZabQuqr2JzednajVmBpC+oiTiXZJEApU= github.com/holiman/uint256 v1.2.4/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= +github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= +github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= +github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= +github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= @@ -172,6 +178,8 @@ github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= +github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sanity-io/litter v1.5.1 h1:dwnrSypP6q56o3lFxTU+t2fwQ9A+U5qrXVO4Qg9KwVU= @@ -232,6 +240,8 @@ golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20231127185646-65229373498e h1:Gvh4YaCaXNs6dKTlfgismwWZKyjVZXwOPfIyUaqU3No= golang.org/x/exp v0.0.0-20231127185646-65229373498e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -251,7 +261,8 @@ golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= +golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E= gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= @@ -262,10 +273,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= google.golang.org/grpc v1.62.0 h1:HQKZ/fa1bXkX1oFOvSjmZEUL8wLSaZTjCcLAlmZRtdk= google.golang.org/grpc v1.62.0/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/indexer/blocks.go b/indexer/blocks.go index b6a2af4..3e0e1b9 100644 --- a/indexer/blocks.go +++ b/indexer/blocks.go @@ -2,6 +2,7 @@ package indexer import ( "context" + "flare-ftso-indexer/chain" "flare-ftso-indexer/config" "flare-ftso-indexer/database" "flare-ftso-indexer/logger" @@ -9,22 +10,21 @@ import ( "sync" "time" - "github.com/ava-labs/coreth/core/types" "github.com/cenkalti/backoff/v4" "github.com/ethereum/go-ethereum/common" "github.com/pkg/errors" ) type blockBatch struct { - blocks []*types.Block + blocks []*chain.Block mu sync.RWMutex } func newBlockBatch(batchSize uint64) *blockBatch { - return &blockBatch{blocks: make([]*types.Block, batchSize)} + return &blockBatch{blocks: make([]*chain.Block, batchSize)} } -func (ci *BlockIndexer) fetchBlock(ctx context.Context, index *uint64) (block *types.Block, err error) { +func (ci *BlockIndexer) fetchBlock(ctx context.Context, index *uint64) (block *chain.Block, err error) { indexBigInt := indexToBigInt(index) bOff := backoff.NewExponentialBackOff() @@ -51,7 +51,7 @@ func (ci *BlockIndexer) fetchBlock(ctx context.Context, index *uint64) (block *t return block, nil } -func (ci *BlockIndexer) fetchBlockHeader(ctx context.Context, index *uint64) (header *types.Header, err error) { +func (ci *BlockIndexer) fetchBlockHeader(ctx context.Context, index *uint64) (header *chain.Header, err error) { indexBigInt := indexToBigInt(index) bOff := backoff.NewExponentialBackOff() @@ -92,13 +92,13 @@ func (ci *BlockIndexer) fetchLastBlockIndex(ctx context.Context) (uint64, uint64 return 0, 0, errors.Wrap(err, "fetchBlockHeader last") } - latestConfirmedNumber := lastBlock.Number.Uint64() - ci.params.Confirmations + latestConfirmedNumber := lastBlock.Number().Uint64() - ci.params.Confirmations latestConfirmedHeader, err := ci.fetchBlockHeader(ctx, &latestConfirmedNumber) if err != nil { return 0, 0, errors.Wrap(err, "fetchBlockHeader latestConfirmed") } - return latestConfirmedNumber, latestConfirmedHeader.Time, nil + return latestConfirmedNumber, latestConfirmedHeader.Time(), nil } func (ci *BlockIndexer) fetchBlockTimestamp(ctx context.Context, index uint64) (uint64, error) { @@ -107,7 +107,7 @@ func (ci *BlockIndexer) fetchBlockTimestamp(ctx context.Context, index uint64) ( return 0, errors.Wrap(err, "fetchBlockHeader") } - return lastBlock.Time, nil + return lastBlock.Time(), nil } func (ci *BlockIndexer) processBlocks( diff --git a/indexer/indexer.go b/indexer/indexer.go index dcd1db7..1ece1c8 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -3,6 +3,7 @@ package indexer import ( "context" "encoding/hex" + "flare-ftso-indexer/chain" "flare-ftso-indexer/config" "flare-ftso-indexer/database" "flare-ftso-indexer/logger" @@ -10,8 +11,6 @@ import ( "strings" "time" - "github.com/ava-labs/coreth/core/types" - "github.com/ava-labs/coreth/ethclient" "github.com/ethereum/go-ethereum/common" "github.com/pkg/errors" "golang.org/x/sync/errgroup" @@ -33,7 +32,7 @@ type BlockIndexer struct { db *gorm.DB params config.IndexerConfig transactions map[common.Address]map[functionSignature]transactionsPolicy - client ethclient.Client + client *chain.Client } type transactionsPolicy struct { @@ -43,7 +42,7 @@ type transactionsPolicy struct { type functionSignature [4]byte -func CreateBlockIndexer(cfg *config.Config, db *gorm.DB, ethClient ethclient.Client) (*BlockIndexer, error) { +func CreateBlockIndexer(cfg *config.Config, db *gorm.DB, client *chain.Client) (*BlockIndexer, error) { txs, err := makeTransactions(cfg.Indexer.CollectTransactions) if err != nil { return nil, err @@ -53,7 +52,7 @@ func CreateBlockIndexer(cfg *config.Config, db *gorm.DB, ethClient ethclient.Cli db: db, params: updateParams(cfg.Indexer), transactions: txs, - client: ethClient, + client: client, }, nil } @@ -495,7 +494,7 @@ func (ci *BlockIndexer) indexContinuousIteration( return errors.Wrap(err, "ci.fetchBlock") } - bBatch := &blockBatch{blocks: []*types.Block{block}} + bBatch := &blockBatch{blocks: []*chain.Block{block}} txBatch := new(transactionsBatch) ci.processBlocks(bBatch, txBatch) diff --git a/indexer/indexer_test.go b/indexer/indexer_test.go index 8a57eb5..9adc475 100644 --- a/indexer/indexer_test.go +++ b/indexer/indexer_test.go @@ -2,6 +2,7 @@ package indexer import ( "context" + "flare-ftso-indexer/chain" "flare-ftso-indexer/config" "flare-ftso-indexer/database" "flare-ftso-indexer/logger" @@ -10,7 +11,6 @@ import ( "testing" "time" - "github.com/ava-labs/coreth/ethclient" "github.com/bradleyjkemp/cupaloy/v2" "github.com/caarlos0/env/v10" "github.com/stretchr/testify/require" @@ -39,7 +39,7 @@ func TestIndexer(t *testing.T) { // set configuration parameters mockChainAddress := fmt.Sprintf("http://localhost:%d", tCfg.MockChainPort) - cfgChain := config.ChainConfig{NodeURL: mockChainAddress} + cfgChain := config.ChainConfig{NodeURL: mockChainAddress, ChainType: int(chain.ChainTypeAvax)} // for the test we do not use finalizations collectTransactions := []config.TransactionInfo{ @@ -135,7 +135,7 @@ func runIndexer(ctx context.Context, mockChain *indexer_testing.MockChain, db *g // set a new starting index based on the history drop interval historyDropIntervalSeconds := uint64(10000) - ethClient, err := ethclient.Dial(cfg.Chain.NodeURL) + ethClient, err := chain.DialRPCNode(cfg) if err != nil { logger.Fatal("Could not connect to the Ethereum node: %s", err) } diff --git a/indexer/transactions.go b/indexer/transactions.go index e9434c7..64bd4d9 100644 --- a/indexer/transactions.go +++ b/indexer/transactions.go @@ -3,6 +3,7 @@ package indexer import ( "context" "encoding/hex" + "flare-ftso-indexer/chain" "flare-ftso-indexer/config" "flare-ftso-indexer/database" "flare-ftso-indexer/logger" @@ -18,19 +19,19 @@ import ( ) type transactionsBatch struct { - transactions []*types.Transaction - toBlock []*types.Block + transactions []*chain.Transaction + toBlock []*chain.Block toIndex []uint64 - toReceipt []*types.Receipt + toReceipt []*chain.Receipt toPolicy []transactionsPolicy mu sync.RWMutex } func (tb *transactionsBatch) Add( - tx *types.Transaction, - block *types.Block, + tx *chain.Transaction, + block *chain.Block, index uint64, - receipt *types.Receipt, + receipt *chain.Receipt, policy transactionsPolicy, ) { tb.mu.Lock() @@ -66,7 +67,7 @@ func (ci *BlockIndexer) getTransactionsReceipt( policy := txBatch.toPolicy[i] txBatch.mu.RUnlock() - var receipt *types.Receipt + var receipt *chain.Receipt if policy.status || policy.collectEvents { err := backoff.RetryNotify( @@ -117,7 +118,7 @@ func (ci *BlockIndexer) processTransactions(txBatch *transactionsBatch, data *da // if it was chosen to get the logs of the transaction we process it if receipt != nil && policy.collectEvents { - for _, log := range receipt.Logs { + for _, log := range receipt.Logs() { dbLog, err := buildDBLog(dbTx, log, block) if err != nil { return err @@ -135,19 +136,19 @@ func (ci *BlockIndexer) processTransactions(txBatch *transactionsBatch, data *da } func buildDBTx( - tx *types.Transaction, receipt *types.Receipt, block *types.Block, txIndex uint64, + tx *chain.Transaction, receipt *chain.Receipt, block *chain.Block, txIndex uint64, ) (*database.Transaction, error) { txData := hex.EncodeToString(tx.Data()) funcSig := txData[:8] - fromAddress, err := types.Sender(types.LatestSignerForChainID(tx.ChainId()), tx) // todo: this is a bit slow + fromAddress, err := tx.FromAddress() // todo: this is a bit slow if err != nil { return nil, errors.Wrap(err, "types.Sender") } status := uint64(2) if receipt != nil { - status = receipt.Status + status = receipt.Status() } base := database.BaseEntity{ID: database.TransactionId.Load()} @@ -156,7 +157,7 @@ func buildDBTx( Hash: tx.Hash().Hex()[2:], FunctionSig: funcSig, Input: txData, - BlockNumber: block.NumberU64(), + BlockNumber: block.Number().Uint64(), BlockHash: block.Hash().Hex()[2:], TransactionIndex: txIndex, FromAddress: strings.ToLower(fromAddress.Hex()[2:]), @@ -169,7 +170,7 @@ func buildDBTx( }, nil } -func buildDBLog(dbTx *database.Transaction, log *types.Log, block *types.Block) (*database.Log, error) { +func buildDBLog(dbTx *database.Transaction, log *types.Log, block *chain.Block) (*database.Log, error) { if blockNum := block.Number(); blockNum.Cmp(new(big.Int).SetUint64(log.BlockNumber)) != 0 { return nil, errors.Errorf("block number mismatch %s != %d", blockNum, log.BlockNumber) } diff --git a/main.go b/main.go index 3fa0f41..4a3bc22 100644 --- a/main.go +++ b/main.go @@ -3,13 +3,13 @@ package main import ( "context" "flag" + "flare-ftso-indexer/chain" "flare-ftso-indexer/config" "flare-ftso-indexer/database" "flare-ftso-indexer/indexer" "flare-ftso-indexer/logger" "time" - "github.com/ava-labs/coreth/ethclient" "github.com/cenkalti/backoff/v4" "github.com/pkg/errors" "gorm.io/gorm" @@ -30,7 +30,7 @@ func run(ctx context.Context) error { config.GlobalConfigCallback.Call(cfg) - ethClient, err := dialRPCNode(cfg) + ethClient, err := chain.DialRPCNode(cfg) if err != nil { return errors.Wrap(err, "Could not connect to the RPC nodes") } @@ -78,16 +78,7 @@ func run(ctx context.Context) error { return runIndexer(ctx, cfg, db, ethClient) } -func dialRPCNode(cfg *config.Config) (ethclient.Client, error) { - nodeURL, err := cfg.Chain.FullNodeURL() - if err != nil { - return nil, err - } - - return ethclient.Dial(nodeURL.String()) -} - -func runIndexer(ctx context.Context, cfg *config.Config, db *gorm.DB, ethClient ethclient.Client) error { +func runIndexer(ctx context.Context, cfg *config.Config, db *gorm.DB, ethClient *chain.Client) error { cIndexer, err := indexer.CreateBlockIndexer(cfg, db, ethClient) if err != nil { return err diff --git a/main_test.go b/main_test.go index 1f9c407..64de306 100644 --- a/main_test.go +++ b/main_test.go @@ -2,6 +2,7 @@ package main_test import ( "context" + "flare-ftso-indexer/chain" "flare-ftso-indexer/config" "flare-ftso-indexer/database" "flare-ftso-indexer/indexer" @@ -9,7 +10,6 @@ import ( "strings" "testing" - "github.com/ava-labs/coreth/ethclient" "github.com/bradleyjkemp/cupaloy/v2" "github.com/caarlos0/env/v10" "github.com/ethereum/go-ethereum/common" @@ -39,7 +39,6 @@ type testConfig struct { func TestIntegration(t *testing.T) { ctx := context.Background() - var tCfg testConfig err := env.Parse(&tCfg) require.NoError(t, err, "Could not parse test config") @@ -111,8 +110,9 @@ func initConfig(tCfg testConfig, history bool) config.Config { CollectLogs: []config.LogInfo{logInfo}, }, Chain: config.ChainConfig{ - NodeURL: tCfg.NodeURL, - APIKey: tCfg.NodeAPIKey, + NodeURL: tCfg.NodeURL, + APIKey: tCfg.NodeAPIKey, + ChainType: int(chain.ChainTypeAvax), }, Logger: config.LoggerConfig{ Level: "DEBUG", @@ -138,7 +138,7 @@ func initConfig(tCfg testConfig, history bool) config.Config { } func createIndexer(cfg *config.Config, db *gorm.DB) (*indexer.BlockIndexer, error) { - ethClient, err := dialRPCNode(cfg) + ethClient, err := chain.DialRPCNode(cfg) if err != nil { return nil, errors.Wrap(err, "Could not connect to the RPC nodes") } @@ -146,15 +146,6 @@ func createIndexer(cfg *config.Config, db *gorm.DB) (*indexer.BlockIndexer, erro return indexer.CreateBlockIndexer(cfg, db, ethClient) } -func dialRPCNode(cfg *config.Config) (ethclient.Client, error) { - nodeURL, err := cfg.Chain.FullNodeURL() - if err != nil { - return nil, err - } - - return ethclient.Dial(nodeURL.String()) -} - func checkDB(ctx context.Context, t *testing.T, db *gorm.DB, cfg *config.Config) { t.Run("check blocks", func(t *testing.T) { var blocks []database.Block diff --git a/testing/chain_copy/chain_copy.go b/testing/chain_copy/chain_copy.go index e599ec7..5709889 100644 --- a/testing/chain_copy/chain_copy.go +++ b/testing/chain_copy/chain_copy.go @@ -10,7 +10,7 @@ import ( "net/http" "os" - "github.com/ava-labs/coreth/ethclient" + "github.com/ethereum/go-ethereum/ethclient" ) type PostToChain struct {