From 12343bea0bca0efac5fd42c37fb89a71f087d407 Mon Sep 17 00:00:00 2001 From: pk910 Date: Fri, 26 Apr 2024 06:41:59 +0200 Subject: [PATCH] fixes related to new database schema, fix for too lazy finalized checkpoint refresh (#47) --- dbtypes/other.go | 4 +- handlers/epoch.go | 101 ++++++++++++++++++++++++------------- handlers/slot.go | 3 +- indexer/cache_logic.go | 2 +- indexer/client.go | 21 ++++++-- services/beaconservice.go | 14 +++-- templates/epoch/epoch.html | 2 +- 7 files changed, 101 insertions(+), 46 deletions(-) diff --git a/dbtypes/other.go b/dbtypes/other.go index 7ba11e54..9bd0804e 100644 --- a/dbtypes/other.go +++ b/dbtypes/other.go @@ -7,8 +7,8 @@ type AssignedSlot struct { } type BlockStatus struct { - Root []byte `db:"root"` - Status bool `db:"status"` + Root []byte `db:"root"` + Status SlotStatus `db:"status"` } type AssignedBlob struct { diff --git a/handlers/epoch.go b/handlers/epoch.go index 115fb7b0..5ddfd2b3 100644 --- a/handlers/epoch.go +++ b/handlers/epoch.go @@ -10,6 +10,7 @@ import ( "github.com/gorilla/mux" "github.com/sirupsen/logrus" + "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/templates" "github.com/ethpandaops/dora/types/models" @@ -126,6 +127,8 @@ func buildEpochPageData(epoch uint64) (*models.EpochPageData, time.Duration) { pageData.ValidatorCount = dbEpoch.ValidatorCount if dbEpoch.ValidatorCount > 0 { pageData.AverageValidatorBalance = dbEpoch.ValidatorBalance / dbEpoch.ValidatorCount + } else { + pageData.Synchronized = false } if dbEpoch.Eligible > 0 { pageData.TargetVoteParticipation = float64(dbEpoch.VotedTarget) * 100.0 / float64(dbEpoch.Eligible) @@ -137,44 +140,74 @@ func buildEpochPageData(epoch uint64) (*models.EpochPageData, time.Duration) { // load slots pageData.Slots = make([]*models.EpochPageDataSlot, 0) dbSlots := services.GlobalBeaconService.GetDbBlocksForSlots(uint64(lastSlot), uint32(utils.Config.Chain.Config.SlotsPerEpoch), true, true) + dbIdx := 0 + dbCnt := len(dbSlots) blockCount := uint64(0) - for _, dbSlot := range dbSlots { - slot := uint64(dbSlot.Slot) - - if dbSlot.Status == 2 { - pageData.OrphanedCount++ - } else { - pageData.CanonicalCount++ + for slotIdx := int64(lastSlot); slotIdx >= int64(firstSlot); slotIdx-- { + slot := uint64(slotIdx) + haveBlock := false + for dbIdx < dbCnt && dbSlots[dbIdx] != nil && dbSlots[dbIdx].Slot == slot { + dbSlot := dbSlots[dbIdx] + dbIdx++ + + switch dbSlot.Status { + case dbtypes.Orphaned: + pageData.OrphanedCount++ + case dbtypes.Canonical: + pageData.CanonicalCount++ + case dbtypes.Missing: + pageData.MissedCount++ + } + + proposer := dbSlot.Proposer + if proposer == math.MaxInt64 { + proposer = slotAssignments[slot] + } + + slotData := &models.EpochPageDataSlot{ + Slot: slot, + Epoch: utils.EpochOfSlot(slot), + Ts: utils.SlotToTime(slot), + Status: uint8(dbSlot.Status), + Proposer: proposer, + ProposerName: services.GlobalBeaconService.GetValidatorName(proposer), + AttestationCount: dbSlot.AttestationCount, + DepositCount: dbSlot.DepositCount, + ExitCount: dbSlot.ExitCount, + ProposerSlashingCount: dbSlot.ProposerSlashingCount, + AttesterSlashingCount: dbSlot.AttesterSlashingCount, + SyncParticipation: float64(dbSlot.SyncParticipation) * 100, + EthTransactionCount: dbSlot.EthTransactionCount, + Graffiti: dbSlot.Graffiti, + BlockRoot: dbSlot.Root, + } + if dbSlot.EthBlockNumber != nil { + slotData.WithEthBlock = true + slotData.EthBlockNumber = *dbSlot.EthBlockNumber + } + pageData.Slots = append(pageData.Slots, slotData) + blockCount++ + haveBlock = true } - proposer := dbSlot.Proposer - if proposer == math.MaxInt64 { - proposer = slotAssignments[slot] - } - - slotData := &models.EpochPageDataSlot{ - Slot: slot, - Epoch: utils.EpochOfSlot(slot), - Ts: utils.SlotToTime(slot), - Status: uint8(dbSlot.Status), - Proposer: proposer, - ProposerName: services.GlobalBeaconService.GetValidatorName(proposer), - AttestationCount: dbSlot.AttestationCount, - DepositCount: dbSlot.DepositCount, - ExitCount: dbSlot.ExitCount, - ProposerSlashingCount: dbSlot.ProposerSlashingCount, - AttesterSlashingCount: dbSlot.AttesterSlashingCount, - SyncParticipation: float64(dbSlot.SyncParticipation) * 100, - EthTransactionCount: dbSlot.EthTransactionCount, - Graffiti: dbSlot.Graffiti, - BlockRoot: dbSlot.Root, - } - if dbSlot.EthBlockNumber != nil { - slotData.WithEthBlock = true - slotData.EthBlockNumber = *dbSlot.EthBlockNumber + if !haveBlock { + slotData := &models.EpochPageDataSlot{ + Slot: slot, + Epoch: epoch, + Ts: utils.SlotToTime(slot), + Scheduled: slot >= currentSlot, + Status: 0, + Proposer: slotAssignments[slot], + ProposerName: services.GlobalBeaconService.GetValidatorName(slotAssignments[slot]), + } + if slotData.Scheduled { + pageData.ScheduledCount++ + } else { + pageData.MissedCount++ + } + pageData.Slots = append(pageData.Slots, slotData) + blockCount++ } - pageData.Slots = append(pageData.Slots, slotData) - blockCount++ } pageData.BlockCount = uint64(blockCount) diff --git a/handlers/slot.go b/handlers/slot.go index ec33adf8..4242e47c 100644 --- a/handlers/slot.go +++ b/handlers/slot.go @@ -209,7 +209,8 @@ func buildSlotPageData(blockSlot int64, blockRoot []byte, loadDuties bool) (*mod } } else { // check orphaned status - blockData.Orphaned = services.GlobalBeaconService.CheckBlockOrphanedStatus(blockData.Root) + blockStatus := services.GlobalBeaconService.CheckBlockOrphanedStatus(blockData.Root) + blockData.Orphaned = blockStatus == dbtypes.Orphaned } } diff --git a/indexer/cache_logic.go b/indexer/cache_logic.go index 5437bd0e..a1f1a182 100644 --- a/indexer/cache_logic.go +++ b/indexer/cache_logic.go @@ -291,7 +291,7 @@ func (cache *indexerCache) processOrphanedBlocks(processedEpoch int64) error { // check if blocks are already in db for _, blockRef := range db.GetBlockStatus(blockRoots) { - if blockRef.Status { + if blockRef.Status == dbtypes.Orphaned { logger.Debugf("processed duplicate orphaned block: 0x%x", blockRef.Root) } else { logger.Debugf("processed duplicate canonical block in orphaned handler: 0x%x", blockRef.Root) diff --git a/indexer/client.go b/indexer/client.go index 9f1875dd..e4ab0b47 100644 --- a/indexer/client.go +++ b/indexer/client.go @@ -559,9 +559,22 @@ func (client *IndexerClient) processBlockEvent(evt *v1.BlockEvent) error { } func (client *IndexerClient) processFinalizedEvent(evt *v1.FinalizedCheckpointEvent) error { - time.Sleep(100 * time.Millisecond) - client.refreshFinalityCheckpoints() - logger.WithField("client", client.clientName).Debugf("received finalization_checkpoint event: finalized %v [0x%x], justified %v [0x%x]", client.lastFinalizedEpoch, client.lastFinalizedRoot, client.lastJustifiedEpoch, client.lastJustifiedRoot) - client.indexerCache.setFinalizedHead(client.lastFinalizedEpoch, client.lastFinalizedRoot, client.lastJustifiedEpoch, client.lastJustifiedRoot) + go func() { + retry := 0 + for ; retry < 20; retry++ { + client.refreshFinalityCheckpoints() + + if bytes.Equal(client.lastFinalizedRoot, evt.Block[:]) { + break + } + + time.Sleep(3 * time.Second) + } + + logger.WithField("client", client.clientName).Infof("received finalization_checkpoint event: finalized %v [0x%x], retries: %v", evt.Epoch, evt.Block, retry) + logger.WithField("client", client.clientName).Infof("received finalization_checkpoint event: finalized %v [0x%x], justified %v [0x%x]", client.lastFinalizedEpoch, client.lastFinalizedRoot, client.lastJustifiedEpoch, client.lastJustifiedRoot) + client.indexerCache.setFinalizedHead(client.lastFinalizedEpoch, client.lastFinalizedRoot, client.lastJustifiedEpoch, client.lastJustifiedRoot) + }() + return nil } diff --git a/services/beaconservice.go b/services/beaconservice.go index b4e0e283..b40abb1e 100644 --- a/services/beaconservice.go +++ b/services/beaconservice.go @@ -879,13 +879,21 @@ func (bs *BeaconService) GetDbBlocksByParentRoot(parentRoot []byte) []*dbtypes.S return resBlocks } -func (bs *BeaconService) CheckBlockOrphanedStatus(blockRoot []byte) bool { +func (bs *BeaconService) CheckBlockOrphanedStatus(blockRoot []byte) dbtypes.SlotStatus { cachedBlock := bs.indexer.GetCachedBlock(blockRoot) if cachedBlock != nil { - return !cachedBlock.IsCanonical(bs.indexer, nil) + if cachedBlock.IsCanonical(bs.indexer, nil) { + return dbtypes.Canonical + } else { + return dbtypes.Orphaned + } } dbRefs := db.GetBlockStatus([][]byte{blockRoot}) - return len(dbRefs) > 0 && dbRefs[0].Status + if len(dbRefs) > 0 { + return dbRefs[0].Status + } + + return dbtypes.Missing } func (bs *BeaconService) GetValidatorActivity() (map[uint64]uint8, uint64) { diff --git a/templates/epoch/epoch.html b/templates/epoch/epoch.html index f62d3027..18e02f8b 100644 --- a/templates/epoch/epoch.html +++ b/templates/epoch/epoch.html @@ -189,7 +189,7 @@

{{ formatRecentTimeShort $slot.Ts }} {{ if gt $slot.Slot 0 }}{{ formatValidator $slot.Proposer $slot.ProposerName }}{{ end }} - {{ if $epoch.Synchronized }} + {{ if or $epoch.Synchronized (not (eq $slot.Status 0)) }} {{ if not (eq $slot.Status 0) }}{{ $slot.AttestationCount }}{{ end }} {{ if not (eq $slot.Status 0) }}{{ $slot.DepositCount }} / {{ $slot.ExitCount }}{{ end }} {{ if not (eq $slot.Status 0) }}{{ $slot.ProposerSlashingCount }} / {{ $slot.AttesterSlashingCount }}{{ end }}