develop -> master (#119)

* fix: yileds calculations because of wrong getting from db

* removes sender from response

* removes author and pbftHash from response

* WIP: migration for dags

* WIP: add ForEachFromKey to storage

* WIP: run migration manager

* finalizes migration implementation

* fixed typo

* simplifies code

* fix: dag migration and related foreach db method

* adds migration to remove pbfthash

* fix: wrong prefix

---------

Co-authored-by: VargaElod23 <[email protected]>
Co-authored-by: Előd Varga <[email protected]>

.gitignore

@@ -18,4 +18,5 @@ taraxa-indexer
 
 .vscode
 
-data
+data
+backup*

api/openapi.yaml

@@ -532,14 +532,11 @@ components:
     Dag:
       type: object
       required:
-        - sender
         - hash
         - level
         - transactionCount
         - timestamp
       properties:
-        sender:
-          $ref: "#/components/schemas/Address"
         hash:
           $ref: "#/components/schemas/Hash"
         level:
@@ -553,7 +550,6 @@ components:
       required:
         - author
         - hash
-        - pbftHash
         - number
         - transactionCount
         - timestamp
@@ -562,8 +558,6 @@ components:
           $ref: "#/components/schemas/Address"
         hash:
           $ref: "#/components/schemas/Hash"
-        pbftHash:
-          $ref: "#/components/schemas/Hash"
         number:
           $ref: "#/components/schemas/Counter"
         transactionCount:

internal/chain/config.go

@@ -0,0 +1,42 @@
+package chain
+
+import (
+	"github.com/Taraxa-project/taraxa-indexer/internal/common"
+)
+
+type initialValidator struct {
+	Address     string            `json:"address"`
+	Delegations map[string]string `json:"delegations"`
+}
+
+type DposConfig struct {
+	BlocksPerYear               string             `json:"blocks_per_year"`
+	DagProposersReward          string             `json:"dag_proposers_reward"`
+	MaxBlockAuthorReward        string             `json:"max_block_author_reward"`
+	EligibilityBalanceThreshold string             `json:"eligibility_balance_threshold"`
+	YieldPercentage             string             `json:"yield_percentage"`
+	InitialValidators           []initialValidator `json:"initial_validators"`
+}
+
+type PbftConfig struct {
+	CommitteeSize string `json:"committee_size"`
+	LambdaMs      string `json:"lambda_ms"`
+}
+
+type GenesisObject struct {
+	DagGenesisBlock DagBlock          `json:"dag_genesis_block"`
+	InitialBalances map[string]string `json:"initial_balances"`
+	Pbft            PbftConfig        `json:"pbft"`
+	Dpos            DposConfig        `json:"dpos"`
+}
+
+func (g *GenesisObject) ToChainConfig() (c *common.ChainConfig) {
+	c = new(common.ChainConfig)
+	c.CommitteeSize = common.ParseStringToBigInt(g.Pbft.CommitteeSize)
+	c.BlocksPerYear = common.ParseStringToBigInt(g.Dpos.BlocksPerYear)
+	c.YieldPercentage = common.ParseStringToBigInt(g.Dpos.YieldPercentage)
+	c.DagProposersReward = common.ParseStringToBigInt(g.Dpos.DagProposersReward)
+	c.MaxBlockAuthorReward = common.ParseStringToBigInt(g.Dpos.MaxBlockAuthorReward)
+	c.EligibilityBalanceThreshold = common.ParseStringToBigInt(g.Dpos.EligibilityBalanceThreshold)
+	return
+}

internal/chain/types.go

@@ -1,49 +1,12 @@
 package chain
 
 import (
-	"log"
 	"math/big"
-	"runtime/debug"
-	"strconv"
 
+	"github.com/Taraxa-project/taraxa-indexer/internal/common"
 	"github.com/Taraxa-project/taraxa-indexer/models"
 )
 
-func ParseUInt(s string) (v uint64) {
-	if len(s) == 0 {
-		return
-	}
-	v, err := strconv.ParseUint(s, 0, 64)
-	if err != nil {
-		debug.PrintStack()
-		log.Fatal(s, "ParseUInt ", err)
-	}
-	return v
-}
-
-func ParseInt(s string) (v int64) {
-	if len(s) == 0 {
-		return
-	}
-	v, err := strconv.ParseInt(s, 0, 64)
-	if err != nil {
-		debug.PrintStack()
-		log.Fatal(s, "ParseUInt ", err)
-	}
-	return v
-}
-
-func parseBool(s string) (v bool) {
-	if len(s) == 0 {
-		return
-	}
-	i, err := strconv.ParseUint(s, 0, 64)
-	if err != nil {
-		log.Fatal("parseBool ", v)
-	}
-	return i > 0
-}
-
 type Block struct {
 	models.Pbft
 	Number       string   `json:"number"`
@@ -54,24 +17,25 @@ type Block struct {
 
 func (b *Block) ToModel() (pbft *models.Pbft) {
 	pbft = &b.Pbft
-	pbft.Timestamp = ParseUInt(b.Timestamp)
-	pbft.Number = ParseUInt(b.Number)
+	pbft.Timestamp = common.ParseUInt(b.Timestamp)
+	pbft.Number = common.ParseUInt(b.Number)
 	pbft.TransactionCount = uint64(len(b.Transactions))
 
 	return
 }
 
 type DagBlock struct {
 	models.Dag
+	Sender       string   `json:"sender"`
 	Level        string   `json:"level"`
 	Timestamp    string   `json:"timestamp"`
 	Transactions []string `json:"transactions"`
 }
 
 func (b *DagBlock) ToModel() (dag *models.Dag) {
 	dag = &b.Dag
-	dag.Timestamp = ParseUInt(b.Timestamp)
-	dag.Level = ParseUInt(b.Level)
+	dag.Timestamp = common.ParseUInt(b.Timestamp)
+	dag.Level = common.ParseUInt(b.Level)
 	dag.TransactionCount = uint64(len(b.Transactions))
 
 	return
@@ -119,12 +83,12 @@ func GetTransactionType(to, input string, internal bool) models.TransactionType
 
 func (t *Transaction) ToModelWithTimestamp(timestamp uint64) (trx models.Transaction) {
 	trx = t.Transaction
-	trx.BlockNumber = ParseUInt(t.BlockNumber)
-	trx.Nonce = ParseUInt(t.Nonce)
-	trx.GasPrice = ParseUInt(t.GasPrice)
-	trx.GasUsed = ParseUInt(t.GasUsed)
-	trx.TransactionIndex = ParseUInt(t.TransactionIndex)
-	trx.Status = parseBool(t.Status)
+	trx.BlockNumber = common.ParseUInt(t.BlockNumber)
+	trx.Nonce = common.ParseUInt(t.Nonce)
+	trx.GasPrice = common.ParseUInt(t.GasPrice)
+	trx.GasUsed = common.ParseUInt(t.GasUsed)
+	trx.TransactionIndex = common.ParseUInt(t.TransactionIndex)
+	trx.Status = common.ParseBool(t.Status)
 	trx.Type = GetTransactionType(trx.To, t.Input, false)
 	if trx.Type == models.ContractCreation {
 		trx.To = t.ContractAddress
@@ -145,13 +109,13 @@ func (t *Transaction) ExtractLogs() (logs []models.EventLog) {
 		eLog := models.EventLog{
 			Address:          log.Address,
 			Data:             log.Data,
-			LogIndex:         ParseUInt(log.LogIndex),
+			LogIndex:         common.ParseUInt(log.LogIndex),
 			Name:             "",
 			Params:           []string{},
 			Removed:          log.Removed,
 			Topics:           log.Topics,
 			TransactionHash:  log.TransactionHash,
-			TransactionIndex: ParseUInt(log.TransactionIndex),
+			TransactionIndex: common.ParseUInt(log.TransactionIndex),
 		}
 		logs = append(logs, eLog)
 	}

internal/chain/ws_client.go

@@ -80,7 +80,7 @@ func (client *WsClient) GetLatestPeriod() (uint64, error) {
 	if err != nil {
 		return 0, err
 	}
-	return ParseUInt(blk.Number), err
+	return common.ParseUInt(blk.Number), err
 }
 
 func (client *WsClient) TraceBlockTransactions(number uint64) (traces []TransactionTrace, err error) {

internal/common/config.go

@@ -5,20 +5,22 @@ import (
 )
 
 type ChainConfig struct {
-	CommitteeSize        *big.Int
-	BlocksPerYear        *big.Int
-	YieldPercentage      *big.Int
-	DagProposersReward   *big.Int
-	MaxBlockAuthorReward *big.Int
+	CommitteeSize               *big.Int
+	BlocksPerYear               *big.Int
+	YieldPercentage             *big.Int
+	DagProposersReward          *big.Int
+	MaxBlockAuthorReward        *big.Int
+	EligibilityBalanceThreshold *big.Int
 }
 
 func DefaultChainConfig() *ChainConfig {
 	return &ChainConfig{
-		CommitteeSize:        big.NewInt(1000),
-		BlocksPerYear:        big.NewInt(365 * 24 * 60 * 15),
-		YieldPercentage:      big.NewInt(20),
-		DagProposersReward:   big.NewInt(50),
-		MaxBlockAuthorReward: big.NewInt(10),
+		CommitteeSize:               big.NewInt(1000),
+		BlocksPerYear:               big.NewInt(365 * 24 * 60 * 15),
+		YieldPercentage:             big.NewInt(20),
+		DagProposersReward:          big.NewInt(50),
+		MaxBlockAuthorReward:        big.NewInt(10),
+		EligibilityBalanceThreshold: ParseStringToBigInt("0x69E10DE76676D0800000"),
 	}
 }
 
@@ -28,6 +30,13 @@ type Config struct {
 	ValidatorsYieldSavingInterval uint64
 }
 
+func (c *Config) IsEligible(stake *big.Int) bool {
+	if c.Chain != nil && c.Chain.EligibilityBalanceThreshold != nil && stake.Cmp(c.Chain.EligibilityBalanceThreshold) >= 0 {
+		return true
+	}
+	return false
+}
+
 func DefaultConfig() *Config {
 	return &Config{
 		Chain:                         DefaultChainConfig(),