From db66236cf7612c87975df88c808aa6b3b4a7d1c6 Mon Sep 17 00:00:00 2001 From: pk910 Date: Sat, 30 Nov 2024 16:42:31 +0100 Subject: [PATCH] add separate config setting for validator activity history length --- .hack/devnet/run.sh | 1 + config/default.config.yml | 3 +++ handlers/validator.go | 8 +++++--- indexer/beacon/epochvotes.go | 10 +++++++--- indexer/beacon/indexer.go | 10 ++++++++-- indexer/beacon/validatorcache.go | 6 +++--- test-config.yaml | 3 +++ types/config.go | 1 + 8 files changed, 31 insertions(+), 11 deletions(-) diff --git a/.hack/devnet/run.sh b/.hack/devnet/run.sh index 477d107f..2d82367a 100755 --- a/.hack/devnet/run.sh +++ b/.hack/devnet/run.sh @@ -94,6 +94,7 @@ $(for node in $EXECUTION_NODES; do done) indexer: inMemoryEpochs: 8 + activityHistoryLength: 6 cachePersistenceDelay: 8 disableIndexWriter: false syncEpochCooldown: 1 diff --git a/config/default.config.yml b/config/default.config.yml index 038705bd..0e625a66 100644 --- a/config/default.config.yml +++ b/config/default.config.yml @@ -65,6 +65,9 @@ indexer: # max number of epochs to keep in memory inMemoryEpochs: 3 + # number of epochs to keep validator activity history for (high memory usage for large validator sets) + activityHistoryLength: 6 + # disable synchronizing historic data disableSynchronizer: false diff --git a/handlers/validator.go b/handlers/validator.go index 8b4756d1..a6024436 100644 --- a/handlers/validator.go +++ b/handlers/validator.go @@ -214,8 +214,8 @@ func buildValidatorPageData(validatorIndex uint64, tabView string) (*models.Vali if pageData.TabView == "attestations" { currentEpoch := uint64(chainState.CurrentEpoch()) cutOffEpoch := uint64(0) - if currentEpoch > uint64(services.GlobalBeaconService.GetBeaconIndexer().GetInMemoryEpochs()) { - cutOffEpoch = currentEpoch - uint64(services.GlobalBeaconService.GetBeaconIndexer().GetInMemoryEpochs()) + if currentEpoch > uint64(services.GlobalBeaconService.GetBeaconIndexer().GetActivityHistoryLength()) { + cutOffEpoch = currentEpoch - uint64(services.GlobalBeaconService.GetBeaconIndexer().GetActivityHistoryLength()) } else { cutOffEpoch = 0 } @@ -248,7 +248,9 @@ func buildValidatorPageData(validatorIndex uint64, tabView string) (*models.Vali pageData.RecentAttestations = append(pageData.RecentAttestations, attestation) validatorActivityIdx++ } - if !found { + + validatorStatus := v1.ValidatorToState(validator.Validator, &validator.Balance, epoch, beacon.FarFutureEpoch) + if !found && strings.HasPrefix(validatorStatus.String(), "active_") { attestation := &models.ValidatorPageDataAttestation{ Epoch: uint64(epoch), Status: 0, diff --git a/indexer/beacon/epochvotes.go b/indexer/beacon/epochvotes.go index 5679c07e..c61aea07 100644 --- a/indexer/beacon/epochvotes.go +++ b/indexer/beacon/epochvotes.go @@ -97,8 +97,12 @@ func (indexer *Indexer) aggregateEpochVotesAndActivity(epoch phase0.Epoch, chain } var epochStatsValues *EpochStatsValues + votesWithPrecalc := false + votesWithValues := false if epochStats != nil { epochStatsValues = epochStats.GetOrLoadValues(indexer, true, false) + votesWithPrecalc = epochStats.precalcValues != nil + votesWithValues = epochStats.ready } specs := chainState.GetSpecs() @@ -150,7 +154,9 @@ func (indexer *Indexer) aggregateEpochVotesAndActivity(epoch phase0.Epoch, chain voteAmount := phase0.Gwei(0) slotIndex := chainState.SlotToSlotIndex(attData.Slot) updateActivity := func(validatorIndex phase0.ValidatorIndex) { - indexer.validatorCache.updateValidatorActivity(validatorIndex, epoch, attData.Slot, block) + if votesWithValues { + indexer.validatorCache.updateValidatorActivity(validatorIndex, epoch, attData.Slot, block) + } } if attVersioned.Version >= spec.DataVersionElectra { @@ -223,8 +229,6 @@ func (indexer *Indexer) aggregateEpochVotesAndActivity(epoch phase0.Epoch, chain votes.TotalVotePercent = float64(votes.CurrentEpoch.TotalVoteAmount+votes.NextEpoch.TotalVoteAmount) * 100 / float64(epochStatsValues.EffectiveBalance) } - votesWithValues := epochStats != nil && epochStats.ready - votesWithPrecalc := epochStats != nil && epochStats.precalcValues != nil votesKey := getEpochVotesKey(epoch, targetRoot, blocks[len(blocks)-1].Root, uint8(len(blocks)), votesWithValues, votesWithPrecalc) indexer.logger.Debugf("aggregated epoch %v votes in %v (blocks: %v) [0x%x]", epoch, time.Since(t1), len(blocks), votesKey[:]) diff --git a/indexer/beacon/indexer.go b/indexer/beacon/indexer.go index 73c6e86c..b1277689 100644 --- a/indexer/beacon/indexer.go +++ b/indexer/beacon/indexer.go @@ -34,6 +34,7 @@ type Indexer struct { disableSync bool blockCompression bool inMemoryEpochs uint16 + activityHistoryLength uint16 maxParallelStateCalls uint16 // caches @@ -71,6 +72,10 @@ func NewIndexer(logger logrus.FieldLogger, consensusPool *consensus.Pool) *Index if inMemoryEpochs < 2 { inMemoryEpochs = 2 } + activityHistoryLength := utils.Config.Indexer.ActivityHistoryLength + if activityHistoryLength == 0 { + activityHistoryLength = 6 + } maxParallelStateCalls := uint16(utils.Config.Indexer.MaxParallelValidatorSetRequests) if maxParallelStateCalls < 2 { maxParallelStateCalls = 2 @@ -88,6 +93,7 @@ func NewIndexer(logger logrus.FieldLogger, consensusPool *consensus.Pool) *Index disableSync: utils.Config.Indexer.DisableSynchronizer, blockCompression: blockCompression, inMemoryEpochs: inMemoryEpochs, + activityHistoryLength: activityHistoryLength, maxParallelStateCalls: maxParallelStateCalls, clients: make([]*Client, 0), @@ -103,8 +109,8 @@ func NewIndexer(logger logrus.FieldLogger, consensusPool *consensus.Pool) *Index return indexer } -func (indexer *Indexer) GetInMemoryEpochs() uint16 { - return indexer.inMemoryEpochs +func (indexer *Indexer) GetActivityHistoryLength() uint16 { + return indexer.activityHistoryLength } func (indexer *Indexer) getMinInMemoryEpoch() phase0.Epoch { diff --git a/indexer/beacon/validatorcache.go b/indexer/beacon/validatorcache.go index 6bb15158..ba554859 100644 --- a/indexer/beacon/validatorcache.go +++ b/indexer/beacon/validatorcache.go @@ -196,8 +196,8 @@ func (cache *validatorCache) updateValidatorActivity(validatorIndex phase0.Valid chainState := cache.indexer.consensusPool.GetChainState() currentEpoch := chainState.CurrentEpoch() cutOffEpoch := phase0.Epoch(0) - if currentEpoch > phase0.Epoch(cache.indexer.inMemoryEpochs) { - cutOffEpoch = currentEpoch - phase0.Epoch(cache.indexer.inMemoryEpochs) + if currentEpoch > phase0.Epoch(cache.indexer.activityHistoryLength) { + cutOffEpoch = currentEpoch - phase0.Epoch(cache.indexer.activityHistoryLength) } if epoch < cutOffEpoch { @@ -227,7 +227,7 @@ func (cache *validatorCache) updateValidatorActivity(validatorIndex phase0.Valid defer cache.activityMutex.Unlock() if cachedValidator.recentActivity == nil { - cachedValidator.recentActivity = make([]ValidatorActivity, 0, cache.indexer.inMemoryEpochs) + cachedValidator.recentActivity = make([]ValidatorActivity, 0, cache.indexer.activityHistoryLength) } replaceIndex := -1 diff --git a/test-config.yaml b/test-config.yaml index 117cabff..d00f2989 100644 --- a/test-config.yaml +++ b/test-config.yaml @@ -69,6 +69,9 @@ indexer: # max number of epochs to keep in memory inMemoryEpochs: 3 + # number of epochs to keep validator activity history for (high memory usage for large validator sets) + activityHistoryLength: 6 + # disable synchronizing historic data disableSynchronizer: false diff --git a/types/config.go b/types/config.go index c9557033..ec9d86ab 100644 --- a/types/config.go +++ b/types/config.go @@ -89,6 +89,7 @@ type Config struct { ResyncForceUpdate bool `yaml:"resyncForceUpdate" envconfig:"INDEXER_RESYNC_FORCE_UPDATE"` InMemoryEpochs uint16 `yaml:"inMemoryEpochs" envconfig:"INDEXER_IN_MEMORY_EPOCHS"` + ActivityHistoryLength uint16 `yaml:"activityHistoryLength" envconfig:"INDEXER_ACTIVITY_HISTORY_LENGTH"` DisableSynchronizer bool `yaml:"disableSynchronizer" envconfig:"INDEXER_DISABLE_SYNCHRONIZER"` SyncEpochCooldown uint `yaml:"syncEpochCooldown" envconfig:"INDEXER_SYNC_EPOCH_COOLDOWN"` MaxParallelValidatorSetRequests uint `yaml:"maxParallelValidatorSetRequests" envconfig:"INDEXER_MAX_PARALLEL_VALIDATOR_SET_REQUESTS"`