diff --git a/api/openapi.yaml b/api/openapi.yaml index ff64201..7aa5472 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -550,7 +550,6 @@ components: required: - author - hash - - pbftHash - number - transactionCount - timestamp @@ -559,8 +558,6 @@ components: $ref: "#/components/schemas/Address" hash: $ref: "#/components/schemas/Hash" - pbftHash: - $ref: "#/components/schemas/Hash" number: $ref: "#/components/schemas/Counter" transactionCount: diff --git a/api/server.gen.go b/api/server.gen.go index 885f3c6..1d07aa6 100644 --- a/api/server.gen.go +++ b/api/server.gen.go @@ -395,49 +395,49 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+xbbXPbNhL+KxjcfWhmaEu286pP58Ztk5s09STqdTI5TwciVxRqEGABULEmo/9+AxAk", - "wTeJlOWcr3P5JJPAYrHPsy9YMF9xKJJUcOBa4dlXnBJJEtAg7V8kiiQodW0emr8jUKGkqaaC4xm+zN8i", - "LdCSMg0SLTb/5jjA1LxNiV7hAHOSAJ4VknCAJfyZUQkRnmmZQYBVuIKEGOl/l7DEM/y3SaXSJH+rJm6t", - "H+06eLsN8IKJ8PZ9lvQo9715jd5nyQJkpdSfGchNpVUhYwESD9Xktch4ocOKqFXP+m+IWiGxRHoFiGpI", - "vtOScEVC8/qJMVkMGkVEE7QUss9qRv7BJjMaWC1TElNOzMI9ul6XA3otVck4WJ9qFQ/FLwC3PVr9BnDb", - "w62GckbIYPyMWLw1a7snZsJlGBpYrQdIkYLUFHwPGEhPbHhJGOEhmBlwR5KUGQ2nOMB6k5qfSkvKY7v5", - "yo6fPQcpBNyUU8TiDwi1EX5ZqeMJv5sO/NfWohTpMDmi4NeEsSuiSduoOWz+ShEwiImG75wZnnQJtLHJ", - "CjAOZX+0xrgHREpiCHJ3EouT4hnftKxuNSkltyxuBAiS0pNQRBADP4E7LcmJJrFdXbIUzzCnzMq1geED", - "qFRwBe1Na6EJGxNcfD3zyV2MKGb45jw7vwjwUsiEaDzDGeX6+dPKoJRriM0aAb4icVtRG3QGxZYAM1jD", - "8E0FWNMElCZJum/OvBxoZlWh83XhpodY0YXTXOkOsb5+Xba+IrFyUQwiH2nC2C9LPPs8KAB6U7c3QcP4", - "kXOYkuHDRBsgtzdN/tso98MauH4n4mPEtkK7ltsxEb/lEdyNoEIRBB7EzQ3uiVgb2MvZCyEYEG6ni5SG", - "Y5aok/DNCA/xpo2zUIO7pYUDL1VYPIJGFCu3VxmhrX2HYl2ML3Z6SE4YkyveCBaBfKTOVRQHPQ721uDF", - "CZtXBlX9aeBAHTzhnXrUyWIXMaPeEaXnfsytcsTzF8+fvzx/MX3WlSp4xhhZmHF5hddIHaPyYieodaMA", - "j0ZEjhVR7+FO5xXjkmRMN9T0PF1pIvWYBHX/LF0sGthtVep2+VerJm6ZhtGE1rd6Me1CLCF3NMkSPDub", - "TgOcUO7+6kr7pVFKmdMDSNCKUFbTzl0ull2ldaZXQo7IPmMKE54f6oYDny6WelRc/69WMs52QVHSlNqX", - "Oz+gvDEwPdIQbBnUE3+vQVIRdQaVK6JhFD6jQ8vIFRowViKCUt3dAeOjJnpndonVOGIFmBGlr0g8H0ro", - "ek5xAgxA95PgZbjDBRk/GLt/z0/UvZyyWjzwgOhaoMtmHUDsMEwXNw5J9O3k4Fcas69Y2HaMyYmfcUgY", - "y0uLplv7jbThli/l7ZtSdBK2AV5KkYzIGTFR15KGMEKrmKhfFYwsSAbnDcrTTHceOLjgo/RUmuis3grq", - "C1UHJisxwtAHn3MKfU3ANhXL52lwFpwHF8HT4NlNg7IvcWcpaiaerIk05yBlCzGjzNK+DQXXkoT6d0O2", - "2t8Sin4mdRX879688llTQMeLQpLJT2vCstHNP5fF6+3olkUd+S0wxUIVXT2uF1zygS/54nQpmNgZSKqV", - "34n4+IeZsiUx4iTjn64eZ42y74i2DfC/CKMR0UIeoxlzSK6ThN/Ws8OApDDotHeS79DHzi7mtysqjbtY", - "VxrnkeJbgdeBbn6TMeTuoVLM2Os3N6uuYSGrrEF9mCjXF+e4fsbbd1QL8AaIrIk8n9a6xC2p59Pz80Fn", - "wBaQtV0ONm9+LxPsgTev9bdd/PlEge3oMZjYaS8FR3UDxs7YGCUa4T84e/Xq1d4ckM8MPD2r9W8sVyhf", - "2oxsEg8JrdtDQijDs+LRPzSR5I6cUlFdjs3tIzQHYlJHJs3wldapmk0m1fBt0Lh9m68Auak2+4BEiqxB", - "IcIYuv7+xzmyqqkAXV3+5H4jwiPkB2okuL0AdYLCFaHcDoK7VCgjjKPL67d2mEjz61LirkXtr5BwtACU", - "KYgasn64S5mQNjoxGoKD3O3557fz1l4Tqk/cyFMh40leF2nm2cht1CRXkCq3w9np9HRqxooUOEkpnuEL", - "+yiwd7WWWhMX4iZf3Y/tJHLBMQbdvtn8ADqTPDelsd4it54CrtFiY3epgEGoIUJOor37NHS2lcbbCM/w", - "T6BdNrgyiwW1u/seZ6uGTGp3+31+541v3iIbH5TO2+xWz6fTgp2QJyWSpoyGds7kD5WfJ6pL2sE3G6or", - "qFuPaHyPgP758Zf3KI8IyLoE5ZTHiCBGlTb0MhZ3l/JNwxu/6zO9XcrFzSaYv3K4S/MJIKW9y7eXy1mS", - "ELnpRdt4t+XI5/JS98bM6+CSSZvDyGT90m0qlSLKQogOYpRtBf1VKdXT5zoCp0ba/2i08tYdwStzINjP", - "K9vURnlX0ezTWyvwnagV+0sDLIUcz0Db6LonA+9Lp10kqjfixjDHmGKHUQfZ7SjMsfiXq1Rl+lD++H2t", - "QeHJn5DHW0MaLcazwyfaXzVM7T7xHiFa1eF4YLI18R/Bs7Km3kkwO6rJZkQ0UimEdElNJDYOtpNXn1wR", - "/rCEqn+6+I3oVD8eddLndSalKYXy6JQblJg4IS2HFLI9JxSKjEWmLE8laL1BCxofiyWdIPZSZZV/NbA/", - "ha2g5P/88sMl0uIWOHLT8yC0AiqR+/SuJ/a4jxRG0+Po8aPxvVZlhZEdjd6vLjrbV718GWVaQyPiKHVS", - "UCslVJb8Ug9JsNFU8MhX4J+TzzrJxyxN2WZgDaXs4G5uzT1pR2VHmGNUKVo1Js7OL54+e/7i5auuBvXe", - "8JDv5hvHB39pDxnfeh46nwYlDqWFhKgW9Nw+mFCgzH6bOcRVbjugPCyR/D8x7Aa+6JEVuHsoO9irCmPy", - "dUXUajsoOUQQishwoJqOvlC98nA3wnrw9vr+YwGv/gPBty8qh5SQ+btFXkLusQ6iPGRZZMcyVtoU1kaB", - "7+iyePLkFBW2fzp9iuhyr+AVUYgL2xY8Zh7oQN2P9kStelk1qV8ZjjkEFTPr5bdYHka+ro8PHz8Ld34y", - "eZ+TzT2MexRada4/mFRMDGwam4Edp7iDYtY7Eav/qbhVu5IeQBZbwPZTBtagOWKxGGHRo3DFBsYKyn1B", - "aF1ejw6iSDW8fbT/AnBblLo9TSi9gk3ZiermUHVfO5o+1f/6erztl13X0btYl9NNLD0EjtlDWftWL2ji", - "QdEky6T8ovnQLm/VkKS8zqF9vJi776EPJsdDNnDr/13riA3c45a8nUsMhL5soe2Fn/KlQGQhMm13V8ro", - "jh17cH/w/tk3Yoj3wcc+dtg04RmxNODxCotugHqZYKSAXBcI1Jf9mVDOQSMO+ouQt627cppfhJ8m+bhT", - "/zuBpqw5KD1Els7H7ZR1BeshoiI7zJd0s/1PAAAA//8W3UppyT4AAA==", + "H4sIAAAAAAAC/+xb3XPbNhL/VzC4e2hmaEu286mnc+O2yU2aehL1OpmcpwORKwo1CLAAqFiT0f9+AxAk", + "wS+JlOWM27k8ySSwu9j97Qd2ma84FEkqOHCt8OwrTokkCWiQ9i8SRRKUujYPzd8RqFDSVFPB8Qxf5m+R", + "FmhJmQaJFpv/chxgat6mRK9wgDlJAM8KSjjAEv7MqIQIz7TMIMAqXEFCDPV/SljiGf7HpBJpkr9VE8fr", + "R8sHb7cBXjAR3r7Pkh7hvjev0fssWYCshPozA7mppCpoLEDioZK8FhkvZFgRterh/4aoFRJLpFeAqIbk", + "Oy0JVyQ0r58YlcWgUUQ0QUsh+7Rm6B+sMiOBlTIlMeXEMO6R9bpc0KupisbB8lRcPCt+Abjtkeo3gNse", + "bDWEM0QG28+QxVvD2z0xGy7D0JjVeoAUKUhNwfeAgfDEBpeEER6C2QF3JEmZkXCKA6w3qfmptKQ8toev", + "9PjZc5CCwE25RSz+gFAb4peVOB7xu+nAf20pSpLOJkck/JowdkU0aSs1N5vPKQIGMdHwnVPDky6CNjZZ", + "Asah7I/WGveASEkMQO5OYnFSPOObltatJCXllsYNAUFSehKKCGLgJ3CnJTnRJLbcJUvxDHPKLF0bGD6A", + "SgVX0D60FpqwMcHFlzPf3IWIYoevzrPziwAvhUyIxjOcUa6fP60USrmG2PAI8BWJ24LaoDMotgSYwRqG", + "HyrAmiagNEnSfXvm5UKzqwqdrws3PUSLLpzmQneQ9eXr0vUViZWLYhD5liaM/bLEs8+DAqC3dXsTNJQf", + "OYcpET6MtDHk9qaJfxvlflgD1+9EfIzYVkjXcjsm4rc8grsRUCiCwIO4ubF7ItbG7OXuhRAMCLfbRUrD", + "MSzqIHwzwkO8beM01MBuqeHASxXWHkEjipXHq5TQlr5DsC7EFyc9JCeMyRVvBItAPlLnKoqDHgd7a+zF", + "CZtXClX9aeBAGTzinXLUwWKZmFXviNJzP+ZWOeL5i+fPX56/mD7rShU8Y4wszLq8wmukjlF5sdOodaUA", + "j0ZEjhVR7+FO5xXjkmRMN8T0PF1pIvWYBHX/LF0wDeyxKnG7/KtVE7dUw2hC60e9mHZZLCF3NMkSPDub", + "TgOcUO7+6kr7pVJKmtMDQNCKUFbSzlMull2ldaZXQo7IPmMKE55f6v4qlYnTRVCUKLy4k46uUYyuH2kc", + "tTDoCaLXIKmIOiPDFdEwyiij48NIDg3bVSSCUtzdXv9RE70zRcRqHJoCzIjSVySeD0VxPTE4AsZA96Pg", + "panDCaWLpR57fs9P1L08sWIeeIboYtClsw5D7FBMFzYOydbtCO+XC7OvWNieiklsn3FIGMvrg6Zb+92w", + "4Zov6e3bUrQDtgFeSpGMCPwxUdeShjBCqpioXxWMrCoGZxfK00x33hq44KPkVJrorN7P6QtVB2YoMULR", + "B19WCnlNwDZlx+dpcBacBxfB0+DZTQOyL3FnPWk2nqyJNJcZZaspI8zSvg0F15KE+ncDttrfEoqmJHVl", + "+O/evvJZk0DHi4KSyU9rwrLRHTyXuus95ZZGHfitYQpGFVw9rBdY8g1f4sXJUiCxM5BUnN+J+Pg3krKv", + "MOI64l+RHmeNsu+etQ3wfwijEdFCHqOjckiuk4Tf1rPDgKQw6Mp2kp/Qt51l5vccKom7UFcq55HatzJe", + "h3XzccSQAUIlmNHXb25XXcKCVlmD+maiXF+c4/pFbd99K8AbILJG8nxaa/W2qJ5Pz88HXeRahqydcrB6", + "8+FKsMe8ea2/7cLPJwpsR6PAxE472Rt1pR+7Y2OEaIT/4OzVq1d7c0C+M/DkrPjfWKxQvrQZ2SQeElq3", + "h4RQhmfFo39pIskdOaWimnDN7SM0B2JSRybN8pXWqZpNJtXybdAYoc1XgNxWm31AIkXWoBBhDF1//+Mc", + "WdFUgK4uf3K/EeER8gM1EtxOMR2hcEUot4vgLhXKEOPo8vqtXSbSfOZJ3GzT/goJRwtAmYKoQeuHu5QJ", + "aaMToyE4k7sz//x23jprQvWJW3kqZDzJ6yLNPB25g5rkClLlejg7nZ5OzVqRAicpxTN8YR8FduBqoTVx", + "IW7y1f3YTiIXHGPQ7fHkB9CZ5LkqjfYWufYUcI0WG3tKBQxCDRFyFO0A08DZVhpvIzzDP4F22eDKMAtq", + "A/geZ6uWTGoD+j6/89Y3R8HGB6XzNnvU8+m0QCfkSYmkKaOh3TP5Q+X3iWrSOng8obqCuvWIxkcF6N8f", + "f3mP8oiArEtQTnmMCGJUaQMvo3E3WW8q3vhdn+otKxc3m8b8lcNdmm8AKe1A3k6IsyQhctNrbePdFiOf", + "y8nsjdnXgSWTNoeByfqlO1QqRZSFEB2EKNsK+rtCqqfPdQRMjdT/0WDl8R2BK3Mh2I8r25lGeVfRnNPj", + "FfhO1Ir9pQKWQo5HoG103ROB94XTLhDVG3FjkGNUsUOpg/R2FORY+5dcqjJ9KH78vtag8ORvyOOtAY0W", + "49HhA+3vGqZ233iPEK3q5nhgsDXtPwJnZU29E2B2VRPNiGikUgjpkppIbBxsJ64+uSL8YQFV//7wG8Gp", + "fj3qhM/rTEpTCuXRKVcoMXFCWgwpZHtOKBQZi0xZnkrQeoMWND4WSjqN2AuVVT7635/CVlDif3754RJp", + "cQscue15EFoBlch9P9cTe9yXBqPhcfT40fjoqtLCyI5G76cTne2rXryMUq2BEXGQOimglRIqS3yphwTY", + "aCh44Cvsn4PPOsnHLE3ZZmANpezibmzNPWpHRUeY26gStGpMnJ1fPH32/MXLV10N6r3hIT/NN44PPmvP", + "Mr72POt8GpQ4lBYSolrQc+dgQoEy523mEFe57TDlYYnk/4lht+GLHllhd8/KzuxVhTH5uiJqtR2UHCII", + "RWQwUG1HX6heeXY3xHrs7fX9xxq8+l8A376oHFJC5u8WeQm5RzuI8pBlkV3LWKlTWBsBvqPL4smTU1To", + "/un0KaLLvYRXRCEubFvwmHmgw+p+tCdq1YuqSX1kOOYSVOysl99ieRj4ur4gfPwo3Pnd431uNvdQ7lFg", + "1cl/MKiYGNg0Ngs7bnEHxax3IlZ/qbhVG0kPAIstYPshA2vQHLFYjNDoUbBiA2Nlyn1BaF2ORwdBpFre", + "vtp/AbgtSt2eJpRewabsRHVjqJrXjoZP9V+3Hm/7Zdc4ehfqcriJpWeBY/ZQ1r7WC5h4pmiCZVJ+lnxo", + "l7dqSFJex9A+XMzdR80Hg+MhG7j1/3N1xAbucUveThYDTV+20Paan/KlQGQhMm1PV9Lojh177P7g/bNv", + "hBDvg4996LBpwlNiqcDjFRbdBupFgqECcl1YoM72Z0I5B4046C9C3rZm5TQfhJ8m+bpT/zuBJq05KD2E", + "ls7X7aR1BeshpCK7zKd0s/1fAAAA//9oaSfXjj4AAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/internal/indexer/block_context.go b/internal/indexer/block_context.go index e725cf4..47bd111 100644 --- a/internal/indexer/block_context.go +++ b/internal/indexer/block_context.go @@ -160,8 +160,6 @@ func (bc *blockContext) processDagsOld() (err error) { if err != nil { return } - bc.block.PbftHash = block_with_dags.BlockHash - tp := common.MakeThreadPool() for i, dag_hash := range block_with_dags.Schedule.DagBlocksOrder { tp.Go(common.MakeTaskWithResult(bc.processDag, dag_hash, &bc.dags[i], &err).Run) diff --git a/internal/storage/pebble/migrations/0_dag_removeSender.go b/internal/storage/pebble/migrations/0_dag_removeSender.go index 0b3ba0d..a7529f3 100644 --- a/internal/storage/pebble/migrations/0_dag_removeSender.go +++ b/internal/storage/pebble/migrations/0_dag_removeSender.go @@ -33,7 +33,7 @@ func (m *RemoveSenderMigration) Apply(s *pebble.Storage) error { for { var o OldDag count := 0 - s.ForEachFromKey([]byte("d"), last_key, func(key, res []byte) (stop bool) { + s.ForEachFromKey([]byte(pebble.DagsPrefix), last_key, func(key, res []byte) (stop bool) { err := rlp.DecodeBytes(res, &o) if err != nil { if err.Error() == "rlp: too few elements for migration.OldDag" { diff --git a/internal/storage/pebble/migrations/2_pbft_remove_pbfthash.go b/internal/storage/pebble/migrations/2_pbft_remove_pbfthash.go new file mode 100644 index 0000000..9dd4fdc --- /dev/null +++ b/internal/storage/pebble/migrations/2_pbft_remove_pbfthash.go @@ -0,0 +1,70 @@ +package migration + +import ( + "github.com/Taraxa-project/taraxa-indexer/internal/storage/pebble" + "github.com/Taraxa-project/taraxa-indexer/models" + "github.com/ethereum/go-ethereum/rlp" + log "github.com/sirupsen/logrus" +) + +type OldPbft struct { + Author models.Address `json:"author"` + Hash models.Hash `json:"hash"` + Number models.Counter `json:"number"` + PbftHash models.Hash `json:"pbftHash"` + Timestamp models.Timestamp `json:"timestamp"` + TransactionCount models.Counter `json:"transactionCount"` +} + +// RemoveSenderMigration is a migration that removes the Sender attribute from the Dag struct. +type RemovePbftHashMigration struct { + id string +} + +func (m *RemovePbftHashMigration) GetId() string { + return m.id +} + +// Apply is the implementation of the Migration interface for the RemoveSenderMigration. +func (m *RemovePbftHashMigration) Apply(s *pebble.Storage) error { + const PBFT_BATCH_THRESHOLD = 1000 + batch := s.NewBatch() + var last_key []byte + + for { + var o OldPbft + count := 0 + s.ForEachFromKey([]byte(pebble.PbftPrefix), last_key, func(key, res []byte) (stop bool) { + err := rlp.DecodeBytes(res, &o) + if err != nil { + if err.Error() == "rlp: too few elements for migration.OldPbft" { + return false + } + log.WithFields(log.Fields{"migration": m.id, "error": err}).Fatal("Error decoding Pbft") + } + pbft := models.Pbft{ + Author: o.Author, + Hash: o.Hash, + Number: o.Number, + Timestamp: o.Timestamp, + TransactionCount: o.TransactionCount, + } + err = batch.AddToBatchFullKey(&pbft, key) + + if err != nil { + log.WithFields(log.Fields{"migration": m.id, "error": err}).Fatal("Error adding Pbft to batch") + } + + last_key = key + count++ + return count == PBFT_BATCH_THRESHOLD + }) + batch.CommitBatch() + batch = s.NewBatch() + if count < PBFT_BATCH_THRESHOLD { + break + } + } + + return nil +} diff --git a/internal/storage/pebble/migrations/manager.go b/internal/storage/pebble/migrations/manager.go index 79c949b..0848b7f 100644 --- a/internal/storage/pebble/migrations/manager.go +++ b/internal/storage/pebble/migrations/manager.go @@ -22,6 +22,7 @@ func NewManager(s *pebble.Storage) *Manager { storage: s, } m.RegisterMigration(&RemoveSenderMigration{id: "0_dag_removeSender"}) + m.RegisterMigration(&RemovePbftHashMigration{id: "2_pbft_removePbftHash"}) return &m } diff --git a/internal/storage/pebble/storage.go b/internal/storage/pebble/storage.go index 670de13..8df63a1 100644 --- a/internal/storage/pebble/storage.go +++ b/internal/storage/pebble/storage.go @@ -17,6 +17,21 @@ import ( log "github.com/sirupsen/logrus" ) +const AccountPrefix = "b" +const LogsPrefix = "e" +const TransactionPrefix = "t" +const PbftPrefix = "p" +const DagsPrefix = "d" +const StatsPrefix = "s" +const FinalizationDataPrefix = "f" +const GenesisHashPrefix = "g" +const WeekStatsPrefix = "w" +const TotalSupplyPrefix = "ts" +const InternalTransactionsPrefix = "i" +const YieldPrefix = "y" +const ValidatorsYieldPrefix = "vy" +const MultipliedYieldPrefix = "my" + type Storage struct { db *pebble.DB path string @@ -81,33 +96,33 @@ func (s *Storage) get(key []byte) ([]byte, io.Closer, error) { func getPrefix(o interface{}) (ret string) { switch tt := o.(type) { case *[]storage.Account, []storage.Account: - ret = "b" + ret = AccountPrefix case *models.TransactionLogsResponse, models.TransactionLogsResponse: - ret = "e" + ret = LogsPrefix case *models.Transaction, models.Transaction: - ret = "t" + ret = TransactionPrefix case *models.Pbft, models.Pbft: - ret = "p" + ret = PbftPrefix case *models.Dag, models.Dag: - ret = "d" + ret = DagsPrefix case *storage.AddressStats, storage.AddressStats: - ret = "s" + ret = StatsPrefix case *storage.FinalizationData, storage.FinalizationData: - ret = "f" + ret = FinalizationDataPrefix case *storage.GenesisHash, storage.GenesisHash: - ret = "g" + ret = GenesisHashPrefix case *storage.WeekStats, storage.WeekStats: - ret = "w" + ret = WeekStatsPrefix case *storage.TotalSupply, storage.TotalSupply: - ret = "ts" + ret = TotalSupplyPrefix case *models.InternalTransactionsResponse, models.InternalTransactionsResponse: - ret = "i" + ret = InternalTransactionsPrefix case *storage.Yield, storage.Yield: - ret = "y" + ret = YieldPrefix case *storage.ValidatorsYield, storage.ValidatorsYield: - ret = "vy" + ret = ValidatorsYieldPrefix case *storage.MultipliedYield, storage.MultipliedYield: - ret = "my" + ret = MultipliedYieldPrefix // hack if we aren't passing original type directly to this function, but passing interface{} from other function case *interface{}: ret = getPrefix(*o.(*interface{})) diff --git a/models/models.gen.go b/models/models.gen.go index c830153..986b4e4 100644 --- a/models/models.gen.go +++ b/models/models.gen.go @@ -96,7 +96,6 @@ type Pbft struct { Author Address `json:"author"` Hash Hash `json:"hash"` Number Counter `json:"number"` - PbftHash Hash `json:"pbftHash"` Timestamp Timestamp `json:"timestamp"` TransactionCount Counter `json:"transactionCount"` }