From 3cd48aa5414facff8ecb21348ad8ec765c5a1525 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Mon, 28 Oct 2024 15:20:58 +0700 Subject: [PATCH] prune: time-limit `PruneTable` func (#12511) --- core/rawdb/accessors_chain.go | 19 ++++++++++++------- erigon-lib/state/domain_shared.go | 8 ++++---- eth/stagedsync/stage_execute.go | 7 +++++-- turbo/jsonrpc/eth_call_test.go | 4 ++-- 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 1b7b2909ec4..95778085373 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -1204,9 +1204,9 @@ func IsPosBlock(db kv.Getter, blockHash common.Hash) (trans bool, err error) { } // PruneTable has `limit` parameter to avoid too large data deletes per one sync cycle - better delete by small portions to reduce db.FreeList size -func PruneTable(tx kv.RwTx, table string, pruneTo uint64, ctx context.Context, limit int) error { +func PruneTable(tx kv.RwTx, table string, pruneTo uint64, ctx context.Context, limit int, timeout time.Duration) error { + t := time.Now() c, err := tx.RwCursor(table) - if err != nil { return fmt.Errorf("failed to create cursor for pruning %w", err) } @@ -1226,14 +1226,19 @@ func PruneTable(tx kv.RwTx, table string, pruneTo uint64, ctx context.Context, l if blockNum >= pruneTo { break } - select { - case <-ctx.Done(): - return common.ErrStopped - default: - } if err = c.DeleteCurrent(); err != nil { return fmt.Errorf("failed to remove for block %d: %w", blockNum, err) } + if i%100 == 0 { + select { + case <-ctx.Done(): + return common.ErrStopped + default: + } + if time.Since(t) > timeout { + break + } + } } return nil } diff --git a/erigon-lib/state/domain_shared.go b/erigon-lib/state/domain_shared.go index a32da3a0af5..077e2383818 100644 --- a/erigon-lib/state/domain_shared.go +++ b/erigon-lib/state/domain_shared.go @@ -483,19 +483,19 @@ func (sd *SharedDomains) replaceShortenedKeysInBranch(prefix []byte, branch comm acc := sd.aggTx.d[kv.AccountsDomain] storageItem := sto.lookupVisibleFileByItsRange(fStartTxNum, fEndTxNum) if storageItem == nil { - sd.logger.Warn(fmt.Sprintf("visible storage file of steps %d-%d not found\n", fStartTxNum/sd.aggTx.a.aggregationStep, fEndTxNum/sd.aggTx.a.aggregationStep)) + sd.logger.Warn(fmt.Sprintf("visible storage file of steps %d-%d not found", fStartTxNum/sd.aggTx.a.aggregationStep, fEndTxNum/sd.aggTx.a.aggregationStep)) storageItem = sto.lookupDirtyFileByItsRange(fStartTxNum, fEndTxNum) if storageItem == nil { - sd.logger.Crit(fmt.Sprintf("dirty storage file of steps %d-%d not found\n", fStartTxNum/sd.aggTx.a.aggregationStep, fEndTxNum/sd.aggTx.a.aggregationStep)) + sd.logger.Crit(fmt.Sprintf("dirty storage file of steps %d-%d not found", fStartTxNum/sd.aggTx.a.aggregationStep, fEndTxNum/sd.aggTx.a.aggregationStep)) return nil, errors.New("storage file not found") } } accountItem := acc.lookupVisibleFileByItsRange(fStartTxNum, fEndTxNum) if accountItem == nil { - sd.logger.Warn(fmt.Sprintf("visible account file of steps %d-%d not found\n", fStartTxNum/sd.aggTx.a.aggregationStep, fEndTxNum/sd.aggTx.a.aggregationStep)) + sd.logger.Warn(fmt.Sprintf("visible account file of steps %d-%d not found", fStartTxNum/sd.aggTx.a.aggregationStep, fEndTxNum/sd.aggTx.a.aggregationStep)) accountItem = acc.lookupDirtyFileByItsRange(fStartTxNum, fEndTxNum) if accountItem == nil { - sd.logger.Crit(fmt.Sprintf("dirty account file of steps %d-%d not found\n", fStartTxNum/sd.aggTx.a.aggregationStep, fEndTxNum/sd.aggTx.a.aggregationStep)) + sd.logger.Crit(fmt.Sprintf("dirty account file of steps %d-%d not found", fStartTxNum/sd.aggTx.a.aggregationStep, fEndTxNum/sd.aggTx.a.aggregationStep)) return nil, errors.New("account file not found") } } diff --git a/eth/stagedsync/stage_execute.go b/eth/stagedsync/stage_execute.go index 8004a9dc247..290d400ba2c 100644 --- a/eth/stagedsync/stage_execute.go +++ b/eth/stagedsync/stage_execute.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "math" "time" "github.com/c2h5oh/datasize" @@ -393,10 +394,12 @@ func PruneExecutionStage(s *PruneState, tx kv.RwTx, cfg ExecuteBlockCfg, ctx con // (chunkLen is 8Kb) * (1_000 chunks) = 8mb // Some blocks on bor-mainnet have 400 chunks of diff = 3mb var pruneDiffsLimitOnChainTip = 1_000 + pruneTimeout := 250 * time.Millisecond if s.CurrentSyncCycle.IsInitialCycle { - pruneDiffsLimitOnChainTip *= 10 + pruneDiffsLimitOnChainTip = math.MaxInt + pruneTimeout = time.Hour } - if err := rawdb.PruneTable(tx, kv.ChangeSets3, s.ForwardProgress-config3.MaxReorgDepthV3, ctx, pruneDiffsLimitOnChainTip); err != nil { + if err := rawdb.PruneTable(tx, kv.ChangeSets3, s.ForwardProgress-config3.MaxReorgDepthV3, ctx, pruneDiffsLimitOnChainTip, pruneTimeout); err != nil { return err } } diff --git a/turbo/jsonrpc/eth_call_test.go b/turbo/jsonrpc/eth_call_test.go index 7e4f364daf6..5363f4496b4 100644 --- a/turbo/jsonrpc/eth_call_test.go +++ b/turbo/jsonrpc/eth_call_test.go @@ -578,10 +578,10 @@ func doPrune(t *testing.T, db kv.RwDB, pruneTo uint64) { err = rawdb.PruneTableDupSort(tx, kv.StorageChangeSet, "", pruneTo, logEvery, ctx) assert.NoError(t, err) - err = rawdb.PruneTable(tx, kv.Receipts, pruneTo, ctx, math.MaxInt32) + err = rawdb.PruneTable(tx, kv.Receipts, pruneTo, ctx, math.MaxInt32, time.Hour) assert.NoError(t, err) - err = rawdb.PruneTable(tx, kv.Log, pruneTo, ctx, math.MaxInt32) + err = rawdb.PruneTable(tx, kv.Log, pruneTo, ctx, math.MaxInt32, time.Hour) assert.NoError(t, err) err = rawdb.PruneTableDupSort(tx, kv.CallTraceSet, "", pruneTo, logEvery, ctx)