fixes related to new database schema, fix for too lazy finalized chec…

…kpoint refresh (#47)
ethpandaops · Apr 26, 2024 · 12343be · 12343be
1 parent 4473f41
commit 12343be
Show file tree

Hide file tree

Showing 7 changed files with 101 additions and 46 deletions.
diff --git a/dbtypes/other.go b/dbtypes/other.go
@@ -7,8 +7,8 @@ type AssignedSlot struct {
 }
 
 type BlockStatus struct {
-	Root   []byte `db:"root"`
-	Status bool   `db:"status"`
+	Root   []byte     `db:"root"`
+	Status SlotStatus `db:"status"`
 }
 
 type AssignedBlob struct {

diff --git a/handlers/epoch.go b/handlers/epoch.go
@@ -10,6 +10,7 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/sirupsen/logrus"
 
+	"github.com/ethpandaops/dora/dbtypes"
 	"github.com/ethpandaops/dora/services"
 	"github.com/ethpandaops/dora/templates"
 	"github.com/ethpandaops/dora/types/models"
@@ -126,6 +127,8 @@ func buildEpochPageData(epoch uint64) (*models.EpochPageData, time.Duration) {
 		pageData.ValidatorCount = dbEpoch.ValidatorCount
 		if dbEpoch.ValidatorCount > 0 {
 			pageData.AverageValidatorBalance = dbEpoch.ValidatorBalance / dbEpoch.ValidatorCount
+		} else {
+			pageData.Synchronized = false
 		}
 		if dbEpoch.Eligible > 0 {
 			pageData.TargetVoteParticipation = float64(dbEpoch.VotedTarget) * 100.0 / float64(dbEpoch.Eligible)
@@ -137,44 +140,74 @@ func buildEpochPageData(epoch uint64) (*models.EpochPageData, time.Duration) {
 	// load slots
 	pageData.Slots = make([]*models.EpochPageDataSlot, 0)
 	dbSlots := services.GlobalBeaconService.GetDbBlocksForSlots(uint64(lastSlot), uint32(utils.Config.Chain.Config.SlotsPerEpoch), true, true)
+	dbIdx := 0
+	dbCnt := len(dbSlots)
 	blockCount := uint64(0)
-	for _, dbSlot := range dbSlots {
-		slot := uint64(dbSlot.Slot)
-
-		if dbSlot.Status == 2 {
-			pageData.OrphanedCount++
-		} else {
-			pageData.CanonicalCount++
+	for slotIdx := int64(lastSlot); slotIdx >= int64(firstSlot); slotIdx-- {
+		slot := uint64(slotIdx)
+		haveBlock := false
+		for dbIdx < dbCnt && dbSlots[dbIdx] != nil && dbSlots[dbIdx].Slot == slot {
+			dbSlot := dbSlots[dbIdx]
+			dbIdx++
+
+			switch dbSlot.Status {
+			case dbtypes.Orphaned:
+				pageData.OrphanedCount++
+			case dbtypes.Canonical:
+				pageData.CanonicalCount++
+			case dbtypes.Missing:
+				pageData.MissedCount++
+			}
+
+			proposer := dbSlot.Proposer
+			if proposer == math.MaxInt64 {
+				proposer = slotAssignments[slot]
+			}
+
+			slotData := &models.EpochPageDataSlot{
+				Slot:                  slot,
+				Epoch:                 utils.EpochOfSlot(slot),
+				Ts:                    utils.SlotToTime(slot),
+				Status:                uint8(dbSlot.Status),
+				Proposer:              proposer,
+				ProposerName:          services.GlobalBeaconService.GetValidatorName(proposer),
+				AttestationCount:      dbSlot.AttestationCount,
+				DepositCount:          dbSlot.DepositCount,
+				ExitCount:             dbSlot.ExitCount,
+				ProposerSlashingCount: dbSlot.ProposerSlashingCount,
+				AttesterSlashingCount: dbSlot.AttesterSlashingCount,
+				SyncParticipation:     float64(dbSlot.SyncParticipation) * 100,
+				EthTransactionCount:   dbSlot.EthTransactionCount,
+				Graffiti:              dbSlot.Graffiti,
+				BlockRoot:             dbSlot.Root,
+			}
+			if dbSlot.EthBlockNumber != nil {
+				slotData.WithEthBlock = true
+				slotData.EthBlockNumber = *dbSlot.EthBlockNumber
+			}
+			pageData.Slots = append(pageData.Slots, slotData)
+			blockCount++
+			haveBlock = true
 		}
 
-		proposer := dbSlot.Proposer
-		if proposer == math.MaxInt64 {
-			proposer = slotAssignments[slot]
-		}
-
-		slotData := &models.EpochPageDataSlot{
-			Slot:                  slot,
-			Epoch:                 utils.EpochOfSlot(slot),
-			Ts:                    utils.SlotToTime(slot),
-			Status:                uint8(dbSlot.Status),
-			Proposer:              proposer,
-			ProposerName:          services.GlobalBeaconService.GetValidatorName(proposer),
-			AttestationCount:      dbSlot.AttestationCount,
-			DepositCount:          dbSlot.DepositCount,
-			ExitCount:             dbSlot.ExitCount,
-			ProposerSlashingCount: dbSlot.ProposerSlashingCount,
-			AttesterSlashingCount: dbSlot.AttesterSlashingCount,
-			SyncParticipation:     float64(dbSlot.SyncParticipation) * 100,
-			EthTransactionCount:   dbSlot.EthTransactionCount,
-			Graffiti:              dbSlot.Graffiti,
-			BlockRoot:             dbSlot.Root,
-		}
-		if dbSlot.EthBlockNumber != nil {
-			slotData.WithEthBlock = true
-			slotData.EthBlockNumber = *dbSlot.EthBlockNumber
+		if !haveBlock {
+			slotData := &models.EpochPageDataSlot{
+				Slot:         slot,
+				Epoch:        epoch,
+				Ts:           utils.SlotToTime(slot),
+				Scheduled:    slot >= currentSlot,
+				Status:       0,
+				Proposer:     slotAssignments[slot],
+				ProposerName: services.GlobalBeaconService.GetValidatorName(slotAssignments[slot]),
+			}
+			if slotData.Scheduled {
+				pageData.ScheduledCount++
+			} else {
+				pageData.MissedCount++
+			}
+			pageData.Slots = append(pageData.Slots, slotData)
+			blockCount++
 		}
-		pageData.Slots = append(pageData.Slots, slotData)
-		blockCount++
 	}
 	pageData.BlockCount = uint64(blockCount)
 

diff --git a/handlers/slot.go b/handlers/slot.go
@@ -209,7 +209,8 @@ func buildSlotPageData(blockSlot int64, blockRoot []byte, loadDuties bool) (*mod
 			}
 		} else {
 			// check orphaned status
-			blockData.Orphaned = services.GlobalBeaconService.CheckBlockOrphanedStatus(blockData.Root)
+			blockStatus := services.GlobalBeaconService.CheckBlockOrphanedStatus(blockData.Root)
+			blockData.Orphaned = blockStatus == dbtypes.Orphaned
 		}
 	}
 

diff --git a/indexer/cache_logic.go b/indexer/cache_logic.go
@@ -291,7 +291,7 @@ func (cache *indexerCache) processOrphanedBlocks(processedEpoch int64) error {
 
 	// check if blocks are already in db
 	for _, blockRef := range db.GetBlockStatus(blockRoots) {
-		if blockRef.Status {
+		if blockRef.Status == dbtypes.Orphaned {
 			logger.Debugf("processed duplicate orphaned block: 0x%x", blockRef.Root)
 		} else {
 			logger.Debugf("processed duplicate canonical block in orphaned handler: 0x%x", blockRef.Root)

diff --git a/indexer/client.go b/indexer/client.go
@@ -559,9 +559,22 @@ func (client *IndexerClient) processBlockEvent(evt *v1.BlockEvent) error {
 }
 
 func (client *IndexerClient) processFinalizedEvent(evt *v1.FinalizedCheckpointEvent) error {
-	time.Sleep(100 * time.Millisecond)
-	client.refreshFinalityCheckpoints()
-	logger.WithField("client", client.clientName).Debugf("received finalization_checkpoint event: finalized %v [0x%x], justified %v [0x%x]", client.lastFinalizedEpoch, client.lastFinalizedRoot, client.lastJustifiedEpoch, client.lastJustifiedRoot)
-	client.indexerCache.setFinalizedHead(client.lastFinalizedEpoch, client.lastFinalizedRoot, client.lastJustifiedEpoch, client.lastJustifiedRoot)
+	go func() {
+		retry := 0
+		for ; retry < 20; retry++ {
+			client.refreshFinalityCheckpoints()
+
+			if bytes.Equal(client.lastFinalizedRoot, evt.Block[:]) {
+				break
+			}
+
+			time.Sleep(3 * time.Second)
+		}
+
+		logger.WithField("client", client.clientName).Infof("received finalization_checkpoint event: finalized %v [0x%x], retries: %v", evt.Epoch, evt.Block, retry)
+		logger.WithField("client", client.clientName).Infof("received finalization_checkpoint event: finalized %v [0x%x], justified %v [0x%x]", client.lastFinalizedEpoch, client.lastFinalizedRoot, client.lastJustifiedEpoch, client.lastJustifiedRoot)
+		client.indexerCache.setFinalizedHead(client.lastFinalizedEpoch, client.lastFinalizedRoot, client.lastJustifiedEpoch, client.lastJustifiedRoot)
+	}()
+
 	return nil
 }
diff --git a/services/beaconservice.go b/services/beaconservice.go
@@ -879,13 +879,21 @@ func (bs *BeaconService) GetDbBlocksByParentRoot(parentRoot []byte) []*dbtypes.S
 	return resBlocks
 }
 
-func (bs *BeaconService) CheckBlockOrphanedStatus(blockRoot []byte) bool {
+func (bs *BeaconService) CheckBlockOrphanedStatus(blockRoot []byte) dbtypes.SlotStatus {
 	cachedBlock := bs.indexer.GetCachedBlock(blockRoot)
 	if cachedBlock != nil {
-		return !cachedBlock.IsCanonical(bs.indexer, nil)
+		if cachedBlock.IsCanonical(bs.indexer, nil) {
+			return dbtypes.Canonical
+		} else {
+			return dbtypes.Orphaned
+		}
 	}
 	dbRefs := db.GetBlockStatus([][]byte{blockRoot})
-	return len(dbRefs) > 0 && dbRefs[0].Status
+	if len(dbRefs) > 0 {
+		return dbRefs[0].Status
+	}
+
+	return dbtypes.Missing
 }
 
 func (bs *BeaconService) GetValidatorActivity() (map[uint64]uint8, uint64) {

diff --git a/templates/epoch/epoch.html b/templates/epoch/epoch.html
@@ -189,7 +189,7 @@ <h1 class="h4 my-3 mb-md-0 h1-pager">
                   </td>
                   <td data-timer="{{ $slot.Ts.Unix }}"><span data-bs-toggle="tooltip" data-bs-placement="top" data-bs-title="{{ $slot.Ts }}">{{ formatRecentTimeShort $slot.Ts }}</span></td>
                   <td>{{ if gt $slot.Slot 0 }}{{ formatValidator $slot.Proposer $slot.ProposerName }}{{ end }}</td>
-                  {{ if $epoch.Synchronized }}
+                  {{ if or $epoch.Synchronized (not (eq $slot.Status 0)) }}
                     <td class="d-none d-md-table-cell">{{ if not (eq $slot.Status 0) }}{{ $slot.AttestationCount }}{{ end }}</td>
                     <td>{{ if not (eq $slot.Status 0) }}{{ $slot.DepositCount }} / {{ $slot.ExitCount }}{{ end }}</td>
                     <td>{{ if not (eq $slot.Status 0) }}{{ $slot.ProposerSlashingCount }} / {{ $slot.AttesterSlashingCount }}{{ end }}</td>