Upstream v2.58.2

.github/workflows/ci.yml

@@ -39,7 +39,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with:
-          go-version: '1.20'
+          go-version: '1.21'
       - name: Install dependencies on Linux
         if: runner.os == 'Linux'
         run: sudo apt update && sudo apt install build-essential
@@ -91,7 +91,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with:
-          go-version: '1.20'
+          go-version: '1.21'
 
       - uses: actions/cache@v3
         with:

.github/workflows/coverage.yml

@@ -12,7 +12,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with:
-          go-version: '1.20'
+          go-version: '1.21'
 
       - name: install dependencies on Linux
         if: runner.os == 'Linux'

.github/workflows/qa-clean-exit-block-downloading.yml

@@ -40,7 +40,8 @@ jobs:
 
     - name: Restore Erigon Testbed Data Directory
       run: |
-        rsync -av --delete $ERIGON_REFERENCE_DATA_DIR/ $ERIGON_TESTBED_DATA_DIR/
+        rm -rf $ERIGON_TESTBED_DATA_DIR/chaindata
+        rsync -a --delete $ERIGON_REFERENCE_DATA_DIR/ $ERIGON_TESTBED_DATA_DIR/
 
     - name: Clean Erigon Build Directory
       run: |
@@ -51,10 +52,6 @@ jobs:
         make erigon
       working-directory: ${{ github.workspace }}
 
-    #- name: Download Python Script for Logs Checking
-    #  run: |
-    #    curl -o check_erigon_exit.py 'https://gist.githubusercontent.com/mriccobene/8db4030a745de34d527f136f2caa104f/raw/3c1a860cb87d61075e78ce399e17f0ab157cacc6/check_erigon_exit.py'
-
     - name: Run Erigon, send ctrl-c and check for clean exiting
       run: |
         # Run Erigon, send ctrl-c and check logs

.github/workflows/qa-clean-exit-snapshot-downloading.yml

@@ -52,10 +52,6 @@ jobs:
         make erigon
       working-directory: ${{ github.workspace }}
 
-    #- name: Download Python Script for Logs Checking
-    #  run: |
-    #    curl -o check_erigon_exit.py 'https://gist.githubusercontent.com/mriccobene/8db4030a745de34d527f136f2caa104f/raw/3c1a860cb87d61075e78ce399e17f0ab157cacc6/check_erigon_exit.py'
-
     - name: Run Erigon, send ctrl-c and check for clean exiting
       run: |
         # Run Erigon, send ctrl-c and check logs

.github/workflows/test-integration-caplin.yml

@@ -28,7 +28,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with:
-          go-version: '1.20'
+          go-version: '1.21'
       - name: Install dependencies on Linux
         if: runner.os == 'Linux'
         run: sudo apt update && sudo apt install build-essential
@@ -46,7 +46,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with:
-          go-version: '1.20'
+          go-version: '1.21'
 
       - uses: actions/cache@v3
         with:

.github/workflows/test-integration.yml

@@ -25,7 +25,7 @@ jobs:
       - run: git submodule update --init --recursive --force
       - uses: actions/setup-go@v4
         with:
-          go-version: '1.20'
+          go-version: '1.21'
       - name: Install dependencies on Linux
         if: runner.os == 'Linux'
         run: sudo apt update && sudo apt install build-essential
@@ -52,7 +52,7 @@ jobs:
       - run: git submodule update --init --recursive --force
       - uses: actions/setup-go@v4
         with:
-          go-version: '1.20'
+          go-version: '1.21'
 
       - uses: actions/cache@v3
         with:

Dockerfile

@@ -1,5 +1,5 @@
 # syntax = docker/dockerfile:1.2
-FROM docker.io/library/golang:1.20-alpine3.17 AS builder
+FROM docker.io/library/golang:1.21-alpine3.17 AS builder
 
 RUN apk --no-cache add build-base linux-headers git bash ca-certificates libstdc++
 
@@ -18,7 +18,7 @@ RUN --mount=type=cache,target=/root/.cache \
     make BUILD_TAGS=nosqlite,noboltdb,nosilkworm all
 
 
-FROM docker.io/library/golang:1.20-alpine3.17 AS tools-builder
+FROM docker.io/library/golang:1.21-alpine3.17 AS tools-builder
 RUN apk --no-cache add build-base linux-headers git bash ca-certificates libstdc++
 WORKDIR /app
 

Dockerfile.debian

@@ -1,5 +1,5 @@
 # syntax = docker/dockerfile:1.2
-FROM docker.io/library/golang:1.20-bullseye AS builder
+FROM docker.io/library/golang:1.21-bullseye AS builder
 
 RUN apt update
 RUN apt install -y build-essential git bash ca-certificates libstdc++6
@@ -17,7 +17,7 @@ RUN --mount=type=cache,target=/root/.cache \
     make all
 
 
-FROM docker.io/library/golang:1.20-alpine3.17 AS tools-builder
+FROM docker.io/library/golang:1.21-alpine3.17 AS tools-builder
 
 RUN apk --no-cache add build-base linux-headers git bash ca-certificates libstdc++
 WORKDIR /app

Makefile

@@ -251,7 +251,7 @@ install:
 	@ls -al "$(DIST)"
 
 PACKAGE_NAME          := github.com/testinprod-io/op-erigon
-GOLANG_CROSS_VERSION  ?= v1.20.7
+GOLANG_CROSS_VERSION  ?= v1.21.6
 
 .PHONY: release-dry-run
 release-dry-run: git-submodules

README.md

@@ -278,7 +278,7 @@ Use `--datadir` to choose where to store data.
 Use `--chain=gnosis` for [Gnosis Chain](https://www.gnosis.io/), `--chain=bor-mainnet` for Polygon Mainnet, 
 `--chain=mumbai` for Polygon Mumbai and `--chain=amoy` for Polygon Amoy.
 For Gnosis Chain you need a [Consensus Layer](#beacon-chain-consensus-layer) client alongside
-Erigon (https://docs.gnosischain.com/node/guide/beacon).
+Erigon (https://docs.gnosischain.com/node/manual/beacon).
 
 Running `make help` will list and describe the convenience commands available in the [Makefile](./Makefile).
 

cl/antiquary/antiquary.go

@@ -209,7 +209,7 @@ func (a *Antiquary) Loop() error {
 			if to-from < snaptype.Erigon2MergeLimit {
 				continue
 			}
-			if err := a.antiquate(a.sn.Version(), from, to); err != nil {
+			if err := a.antiquate(from, to); err != nil {
 				return err
 			}
 		case <-a.ctx.Done():
@@ -218,12 +218,12 @@ func (a *Antiquary) Loop() error {
 }
 
 // Antiquate will antiquate a specific block range (aka. retire snapshots), this should be ran in the background.
-func (a *Antiquary) antiquate(version uint8, from, to uint64) error {
+func (a *Antiquary) antiquate(from, to uint64) error {
 	if a.downloader == nil {
 		return nil // Just skip if we don't have a downloader
 	}
 	log.Info("[Antiquary]: Antiquating", "from", from, "to", to)
-	if err := freezeblocks.DumpBeaconBlocks(a.ctx, a.mainDB, a.beaconDB, version, from, to, snaptype.Erigon2MergeLimit, a.dirs.Tmp, a.dirs.Snap, 1, log.LvlDebug, a.logger); err != nil {
+	if err := freezeblocks.DumpBeaconBlocks(a.ctx, a.mainDB, a.beaconDB, from, to, a.dirs.Tmp, a.dirs.Snap, 1, log.LvlDebug, a.logger); err != nil {
 		return err
 	}
 

cl/antiquary/state_antiquary.go

@@ -70,6 +70,12 @@ func (s *Antiquary) loopStates(ctx context.Context) {
 		return
 	}
 
+	_, beforeFinalized, err := s.readHistoricalProcessingProgress(ctx)
+	if err != nil {
+		s.logger.Error("Failed to read historical processing progress", "err", err)
+		return
+	}
+
 	for {
 		select {
 		// Check if we are behind finalized
@@ -78,15 +84,16 @@ func (s *Antiquary) loopStates(ctx context.Context) {
 				continue
 			}
 			// Check if we are behind finalized
-			progress, finalized, err := s.readHistoricalProcessingProgress(ctx)
+			_, finalized, err := s.readHistoricalProcessingProgress(ctx)
 			if err != nil {
 				s.logger.Error("Failed to read historical processing progress", "err", err)
 				continue
 			}
-			// Stay behind a little bit we rely on forkchoice up until (finalized - 2*slotsPerEpoch)
-			if progress+s.cfg.SlotsPerEpoch/2 >= finalized {
+			// We wait for updated finality.
+			if finalized == beforeFinalized {
 				continue
 			}
+			beforeFinalized = finalized
 			if err := s.IncrementBeaconState(ctx, finalized); err != nil {
 				slot := uint64(0)
 				if s.currentState != nil {
@@ -427,6 +434,9 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error {
 		if err := transition.TransitionState(s.currentState, block, blockRewardsCollector, fullValidation); err != nil {
 			return err
 		}
+		// if s.currentState.Slot() == 4293952 {
+		// 	s.dumpFullBeaconState()
+		// }
 		blocksProcessed++
 
 		first = false

cl/beacon/beaconevents/emitter.go

@@ -0,0 +1,71 @@
+package beaconevents
+
+import (
+	"sync"
+
+	"github.com/google/uuid"
+	"golang.org/x/sync/errgroup"
+)
+
+type Subscription struct {
+	id     string
+	topics map[string]struct{}
+	cb     func(topic string, item any)
+}
+
+type EventName string
+
+// Emitters creates pub/sub connection
+type Emitters struct {
+	cbs map[string]*Subscription
+	mu  sync.RWMutex
+}
+
+func NewEmitters() *Emitters {
+	return &Emitters{
+		cbs: map[string]*Subscription{},
+	}
+}
+
+// publish to all subscribers. each callback is run in a separate goroutine
+func (e *Emitters) Publish(s string, a any) {
+	// forward gossip object
+	e.mu.Lock()
+	values := make([]*Subscription, 0, len(e.cbs))
+	for _, v := range e.cbs {
+		values = append(values, v)
+	}
+	e.mu.Unlock()
+
+	egg := errgroup.Group{}
+	for idx := range values {
+		v := values[idx]
+		exec := func() error { v.cb(s, a); return nil }
+		if _, ok := v.topics["*"]; ok {
+			egg.Go(exec)
+		} else if _, ok := v.topics[s]; ok {
+			egg.Go(exec)
+		}
+	}
+	egg.Wait()
+}
+
+// subscribe with callback. call the returned cancelfunc to unregister the callback
+// publish will block until all callbacks for the message are resolved
+func (e *Emitters) Subscribe(topics []string, cb func(topic string, item any)) (func(), error) {
+	subid := uuid.New().String()
+	sub := &Subscription{
+		id:     subid,
+		topics: map[string]struct{}{},
+		cb:     cb,
+	}
+	for _, v := range topics {
+		sub.topics[v] = struct{}{}
+	}
+	e.cbs[subid] = sub
+	return func() {
+		e.mu.Lock()
+		defer e.mu.Unlock()
+		delete(e.cbs, subid)
+	}, nil
+}

cl/beacon/beaconevents/emitter_test.go

@@ -0,0 +1,54 @@
+package beaconevents_test
+
+import (
+	"sync/atomic"
+	"testing"
+
+	"github.com/ledgerwatch/erigon/cl/beacon/beaconevents"
+	"github.com/stretchr/testify/require"
+)
+
+func TestEmitterSet(t *testing.T) {
+	e := beaconevents.NewEmitters()
+	var called int
+	e.Subscribe([]string{"set"}, func(topic string, item any) {
+		require.EqualValues(t, "set", topic)
+		require.EqualValues(t, "hello", item.(string))
+		called = called + 1
+	})
+	e.Publish("set", "hello")
+	require.EqualValues(t, 1, called)
+}
+func TestEmitterFilters(t *testing.T) {
+	e := beaconevents.NewEmitters()
+	var a atomic.Int64
+	var b atomic.Int64
+	var ab atomic.Int64
+	var wild atomic.Int64
+	e.Subscribe([]string{"a"}, func(topic string, item any) {
+		require.EqualValues(t, topic, item.(string))
+		a.Add(1)
+	})
+	e.Subscribe([]string{"b"}, func(topic string, item any) {
+		require.EqualValues(t, topic, item.(string))
+		b.Add(1)
+	})
+	e.Subscribe([]string{"a", "b"}, func(topic string, item any) {
+		require.EqualValues(t, topic, item.(string))
+		ab.Add(1)
+	})
+	e.Subscribe([]string{"*"}, func(topic string, item any) {
+		require.EqualValues(t, topic, item.(string))
+		wild.Add(1)
+	})
+
+	e.Publish("a", "a")
+	e.Publish("b", "b")
+	e.Publish("b", "b")
+	e.Publish("c", "c")
+
+	require.EqualValues(t, 1, a.Load())
+	require.EqualValues(t, 2, b.Load())
+	require.EqualValues(t, 3, ab.Load())
+	require.EqualValues(t, 4, wild.Load())
+}