refactor: make leaves retriever stateless

api/errors/errors.go

@@ -28,8 +28,8 @@ var ErrGetValueForKey = errors.New("get value for key error")
 // ErrGetKeyValuePairs signals an error in getting the key-value pairs of a key for an account
 var ErrGetKeyValuePairs = errors.New("get key-value pairs error")
 
-// ErrGetKeyValuePairsWithCheckpoint signals an error in getting the key-value pairs of a key for an account with a checkpoint
-var ErrGetKeyValuePairsWithCheckpoint = errors.New("get key-value pairs with checkpoint error")
+// ErrIterateKeys signals an error in iterating over the keys of an account
+var ErrIterateKeys = errors.New("iterate keys error")
 
 // ErrGetESDTBalance signals an error in getting esdt balance for given address
 var ErrGetESDTBalance = errors.New("get esdt balance for account error")

api/groups/addressGroup.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"math/big"
 	"net/http"
-	"strconv"
 	"sync"
 
 	"github.com/gin-gonic/gin"
@@ -24,7 +23,6 @@ const (
 	getUsernamePath                = "/:address/username"
 	getCodeHashPath                = "/:address/code-hash"
 	getKeysPath                    = "/:address/keys"
-	getKeysWithCheckpointPath      = "/:address/num-keys/:numKeys/checkpoint-id/:checkpointId"
 	getKeyPath                     = "/:address/key/:key"
 	getDataTrieMigrationStatusPath = "/:address/is-data-trie-migrated"
 	getESDTTokensPath              = "/:address/esdt"
@@ -34,6 +32,7 @@ const (
 	getRegisteredNFTsPath          = "/:address/registered-nfts"
 	getESDTNFTDataPath             = "/:address/nft/:tokenIdentifier/nonce/:nonce"
 	getGuardianData                = "/:address/guardian-data"
+	iterateKeysPath                = "/iterate-keys"
 	urlParamOnFinalBlock           = "onFinalBlock"
 	urlParamOnStartOfEpoch         = "onStartOfEpoch"
 	urlParamBlockNonce             = "blockNonce"
@@ -57,7 +56,7 @@ type addressFacadeHandler interface {
 	GetESDTsWithRole(address string, role string, options api.AccountQueryOptions) ([]string, api.BlockInfo, error)
 	GetAllESDTTokens(address string, options api.AccountQueryOptions) (map[string]*esdt.ESDigitalToken, api.BlockInfo, error)
 	GetKeyValuePairs(address string, options api.AccountQueryOptions) (map[string]string, api.BlockInfo, error)
-	GetKeyValuePairsWithCheckpoint(address string, checkpointId string, numLeaves int, options api.AccountQueryOptions) (map[string]string, api.BlockInfo, string, error)
+	IterateKeys(address string, numKeys uint, iteratorState [][]byte, options api.AccountQueryOptions) (map[string]string, [][]byte, api.BlockInfo, error)
 	GetGuardianData(address string, options api.AccountQueryOptions) (api.GuardianData, api.BlockInfo, error)
 	IsDataTrieMigrated(address string, options api.AccountQueryOptions) (bool, error)
 	IsInterfaceNil() bool
@@ -138,9 +137,9 @@ func NewAddressGroup(facade addressFacadeHandler) (*addressGroup, error) {
 			Handler: ag.getKeyValuePairs,
 		},
 		{
-			Path:    getKeysWithCheckpointPath,
-			Method:  http.MethodGet,
-			Handler: ag.getKeyValuePairsWithCheckpoint,
+			Path:    iterateKeysPath,
+			Method:  http.MethodPost,
+			Handler: ag.iterateKeys,
 		},
 		{
 			Path:    getESDTBalancePath,
@@ -352,40 +351,45 @@ func (ag *addressGroup) getKeyValuePairs(c *gin.Context) {
 	shared.RespondWithSuccess(c, gin.H{"pairs": value, "blockInfo": blockInfo})
 }
 
-// getKeysWithCheckpoint returns all the key-value pairs for the given address
-func (ag *addressGroup) getKeyValuePairsWithCheckpoint(c *gin.Context) {
-	addr := c.Param("address")
-	if addr == "" {
-		shared.RespondWithValidationError(c, errors.ErrGetKeyValuePairsWithCheckpoint, errors.ErrEmptyAddress)
-		return
-	}
+// IterateKeysRequest defines the request structure for iterating keys
+type IterateKeysRequest struct {
+	Address       string   `json:"address"`
+	NumKeys       uint     `json:"numKeys"`
+	IteratorState [][]byte `json:"iteratorState"`
+}
 
-	options, err := extractAccountQueryOptions(c)
+// iterateKeys iterates keys for the given address
+func (ag *addressGroup) iterateKeys(c *gin.Context) {
+	var iterateKeysRequest = &IterateKeysRequest{}
+	err := c.ShouldBindJSON(&iterateKeysRequest)
 	if err != nil {
-		shared.RespondWithValidationError(c, errors.ErrGetKeyValuePairsWithCheckpoint, err)
+		shared.RespondWithValidationError(c, errors.ErrValidation, err)
 		return
 	}
 
-	numLeavesAsString := c.Param("numKeys")
-	if numLeavesAsString == "" {
-		shared.RespondWithValidationError(c, errors.ErrGetKeyValuePairsWithCheckpoint, errors.ErrEmptyNumKeys)
+	if len(iterateKeysRequest.Address) == 0 {
+		shared.RespondWithValidationError(c, errors.ErrValidation, errors.ErrEmptyAddress)
 		return
 	}
 
-	numLeaves, err := strconv.Atoi(numLeavesAsString)
+	options, err := extractAccountQueryOptions(c)
 	if err != nil {
-		shared.RespondWithValidationError(c, errors.ErrGetKeyValuePairsWithCheckpoint, err)
+		shared.RespondWithValidationError(c, errors.ErrIterateKeys, err)
 		return
 	}
 
-	checkpointId := c.Param("checkpointId")
-	value, blockInfo, newCheckpointId, err := ag.getFacade().GetKeyValuePairsWithCheckpoint(addr, checkpointId, numLeaves, options)
+	value, newIteratorState, blockInfo, err := ag.getFacade().IterateKeys(
+		iterateKeysRequest.Address,
+		iterateKeysRequest.NumKeys,
+		iterateKeysRequest.IteratorState,
+		options,
+	)
 	if err != nil {
-		shared.RespondWithInternalError(c, errors.ErrGetKeyValuePairsWithCheckpoint, err)
+		shared.RespondWithInternalError(c, errors.ErrIterateKeys, err)
 		return
 	}
 
-	shared.RespondWithSuccess(c, gin.H{"pairs": value, "newCheckpointId": newCheckpointId, "blockInfo": blockInfo})
+	shared.RespondWithSuccess(c, gin.H{"pairs": value, "newIteratorState": newIteratorState, "blockInfo": blockInfo})
 }
 
 // getESDTBalance returns the balance for the given address and esdt token

api/groups/addressGroup_test.go

@@ -125,13 +125,13 @@ type keyValuePairsResponse struct {
 	Code  string
 }
 
-type keyValuePairsWithCheckpointResponseData struct {
-	Pairs           map[string]string `json:"pairs"`
-	NewCheckpointId string            `json:"newCheckpointId"`
+type iterateKeysResponseData struct {
+	Pairs            map[string]string `json:"pairs"`
+	NewIteratorState [][]byte          `json:"newIteratorState"`
 }
-type keyValuePairsWithCheckpointResponse struct {
-	Data  keyValuePairsWithCheckpointResponseData `json:"data"`
-	Error string                                  `json:"error"`
+type iterateKeysResponse struct {
+	Data  iterateKeysResponseData `json:"data"`
+	Error string                  `json:"error"`
 	Code  string
 }
 
@@ -672,37 +672,66 @@ func TestAddressGroup_getKeyValuePairs(t *testing.T) {
 	})
 }
 
-func TestAddressGroup_getKeyValuePairsWithCheckpoint(t *testing.T) {
+func TestAddressGroup_iterateKeys(t *testing.T) {
 	t.Parallel()
 
-	t.Run("empty address should error",
-		testErrorScenario("/address//num-keys/10/checkpoint-id/abc", "GET", nil,
-			formatExpectedErr(apiErrors.ErrGetKeyValuePairsWithCheckpoint, apiErrors.ErrEmptyAddress)))
-	t.Run("invalid query options should error",
-		testErrorScenario("/address/erd1alice/num-keys/10/checkpoint-id/abc?blockNonce=not-uint64", "GET", nil,
-			formatExpectedErr(apiErrors.ErrGetKeyValuePairsWithCheckpoint, apiErrors.ErrBadUrlParams)))
-	t.Run("empty num-keys should error",
-		testErrorScenario("/address/erd1alice/num-keys//checkpoint-id/abc", "GET", nil,
-			formatExpectedErr(apiErrors.ErrGetKeyValuePairsWithCheckpoint, apiErrors.ErrEmptyNumKeys)))
-	t.Run("invalid num-keys should error",
-		testErrorScenario("/address/erd1alice/num-keys/not-uint64/checkpoint-id/abc", "GET", nil,
-			formatExpectedErr(apiErrors.ErrGetKeyValuePairsWithCheckpoint, errors.New("strconv.Atoi: parsing \"not-uint64\": invalid syntax"))))
+	t.Run("invalid body should error",
+		testErrorScenario("/address/iterate-keys", "POST", bytes.NewBuffer([]byte("invalid body")),
+			formatExpectedErr(apiErrors.ErrValidation, errors.New("invalid character 'i' looking for beginning of value"))))
+	t.Run("empty address should error", func(t *testing.T) {
+		t.Parallel()
+
+		body := &groups.IterateKeysRequest{
+			Address: "",
+		}
+		bodyBytes, _ := json.Marshal(body)
+		testAddressGroup(
+			t,
+			&mock.FacadeStub{},
+			"/address/iterate-keys",
+			"POST",
+			bytes.NewBuffer(bodyBytes),
+			http.StatusBadRequest,
+			formatExpectedErr(apiErrors.ErrValidation, apiErrors.ErrEmptyAddress),
+		)
+	})
+	t.Run("invalid query options should error", func(t *testing.T) {
+		t.Parallel()
+
+		body := &groups.IterateKeysRequest{
+			Address: "erd1",
+		}
+		bodyBytes, _ := json.Marshal(body)
+		testAddressGroup(
+			t,
+			&mock.FacadeStub{},
+			"/address/iterate-keys?blockNonce=not-uint64",
+			"POST",
+			bytes.NewBuffer(bodyBytes),
+			http.StatusBadRequest,
+			formatExpectedErr(apiErrors.ErrIterateKeys, apiErrors.ErrBadUrlParams),
+		)
+	})
 	t.Run("with node fail should err", func(t *testing.T) {
 		t.Parallel()
 
+		body := &groups.IterateKeysRequest{
+			Address: "erd1",
+		}
+		bodyBytes, _ := json.Marshal(body)
 		facade := &mock.FacadeStub{
-			GetKeyValuePairsWithCheckpointCalled: func(address string, checkpointId string, numLeaves int, options api.AccountQueryOptions) (map[string]string, api.BlockInfo, string, error) {
-				return nil, api.BlockInfo{}, "", expectedErr
+			IterateKeysCalled: func(address string, numKeys uint, iteratorState [][]byte, options api.AccountQueryOptions) (map[string]string, [][]byte, api.BlockInfo, error) {
+				return nil, nil, api.BlockInfo{}, expectedErr
 			},
 		}
 		testAddressGroup(
 			t,
 			facade,
-			"/address/erd1alice/num-keys/10/checkpoint-id/abc",
-			"GET",
-			nil,
+			"/address/iterate-keys",
+			"POST",
+			bytes.NewBuffer(bodyBytes),
 			http.StatusInternalServerError,
-			formatExpectedErr(apiErrors.ErrGetKeyValuePairsWithCheckpoint, expectedErr),
+			formatExpectedErr(apiErrors.ErrIterateKeys, expectedErr),
 		)
 	})
 	t.Run("should work", func(t *testing.T) {
@@ -712,30 +741,34 @@ func TestAddressGroup_getKeyValuePairsWithCheckpoint(t *testing.T) {
 			"k1": "v1",
 			"k2": "v2",
 		}
-		originalCheckpointId := "abc"
-		newCheckpointId := "def"
-		numKeys := "10"
-		addr := "erd1alice"
+
+		body := &groups.IterateKeysRequest{
+			Address:       "erd1",
+			NumKeys:       10,
+			IteratorState: [][]byte{[]byte("starting"), []byte("state")},
+		}
+		newIteratorState := [][]byte{[]byte("new"), []byte("state")}
+		bodyBytes, _ := json.Marshal(body)
 		facade := &mock.FacadeStub{
-			GetKeyValuePairsWithCheckpointCalled: func(address string, checkpointId string, numLeaves int, options api.AccountQueryOptions) (map[string]string, api.BlockInfo, string, error) {
-				assert.Equal(t, addr, address)
-				assert.Equal(t, 10, numLeaves)
-				assert.Equal(t, originalCheckpointId, checkpointId)
-				return pairs, api.BlockInfo{}, newCheckpointId, nil
+			IterateKeysCalled: func(address string, numKeys uint, iteratorState [][]byte, options api.AccountQueryOptions) (map[string]string, [][]byte, api.BlockInfo, error) {
+				assert.Equal(t, body.Address, address)
+				assert.Equal(t, body.NumKeys, numKeys)
+				assert.Equal(t, body.IteratorState, iteratorState)
+				return pairs, newIteratorState, api.BlockInfo{}, nil
 			},
 		}
 
-		response := &keyValuePairsWithCheckpointResponse{}
+		response := &iterateKeysResponse{}
 		loadAddressGroupResponse(
 			t,
 			facade,
-			"/address/"+addr+"/num-keys/"+numKeys+"/checkpoint-id/"+originalCheckpointId,
-			"GET",
-			nil,
+			"/address/iterate-keys",
+			"POST",
+			bytes.NewBuffer(bodyBytes),
 			response,
 		)
 		assert.Equal(t, pairs, response.Data.Pairs)
-		assert.Equal(t, newCheckpointId, response.Data.NewCheckpointId)
+		assert.Equal(t, newIteratorState, response.Data.NewIteratorState)
 	})
 }
 
@@ -1220,7 +1253,7 @@ func getAddressRoutesConfig() config.ApiRoutesConfig {
 					{Name: "/:address/username", Open: true},
 					{Name: "/:address/code-hash", Open: true},
 					{Name: "/:address/keys", Open: true},
-					{Name: "/:address/num-keys/:numKeys/checkpoint-id/:checkpointId", Open: true},
+					{Name: "/iterate-keys", Open: true},
 					{Name: "/:address/key/:key", Open: true},
 					{Name: "/:address/esdt", Open: true},
 					{Name: "/:address/esdts/roles", Open: true},

api/mock/facadeStub.go

@@ -49,7 +49,7 @@ type FacadeStub struct {
 	GetUsernameCalled                           func(address string, options api.AccountQueryOptions) (string, api.BlockInfo, error)
 	GetCodeHashCalled                           func(address string, options api.AccountQueryOptions) ([]byte, api.BlockInfo, error)
 	GetKeyValuePairsCalled                      func(address string, options api.AccountQueryOptions) (map[string]string, api.BlockInfo, error)
-	GetKeyValuePairsWithCheckpointCalled        func(address string, checkpointId string, numLeaves int, options api.AccountQueryOptions) (map[string]string, api.BlockInfo, string, error)
+	IterateKeysCalled                           func(address string, numKeys uint, iteratorState [][]byte, options api.AccountQueryOptions) (map[string]string, [][]byte, api.BlockInfo, error)
 	SimulateTransactionExecutionHandler         func(tx *transaction.Transaction) (*txSimData.SimulationResultsWithVMOutput, error)
 	GetESDTDataCalled                           func(address string, key string, nonce uint64, options api.AccountQueryOptions) (*esdt.ESDigitalToken, api.BlockInfo, error)
 	GetAllESDTTokensCalled                      func(address string, options api.AccountQueryOptions) (map[string]*esdt.ESDigitalToken, api.BlockInfo, error)
@@ -242,13 +242,13 @@ func (f *FacadeStub) GetKeyValuePairs(address string, options api.AccountQueryOp
 	return nil, api.BlockInfo{}, nil
 }
 
-// GetKeyValuePairsWithCheckpoint -
-func (f *FacadeStub) GetKeyValuePairsWithCheckpoint(address string, checkpointId string, numLeaves int, options api.AccountQueryOptions) (map[string]string, api.BlockInfo, string, error) {
-	if f.GetKeyValuePairsWithCheckpointCalled != nil {
-		return f.GetKeyValuePairsWithCheckpointCalled(address, checkpointId, numLeaves, options)
+// IterateKeys -
+func (f *FacadeStub) IterateKeys(address string, numKeys uint, iteratorState [][]byte, options api.AccountQueryOptions) (map[string]string, [][]byte, api.BlockInfo, error) {
+	if f.IterateKeysCalled != nil {
+		return f.IterateKeysCalled(address, numKeys, iteratorState, options)
 	}
 
-	return nil, api.BlockInfo{}, "", nil
+	return nil, nil, api.BlockInfo{}, nil
 }
 
 // GetGuardianData -

api/shared/interface.go

@@ -74,7 +74,7 @@ type FacadeHandler interface {
 	GetESDTsWithRole(address string, role string, options api.AccountQueryOptions) ([]string, api.BlockInfo, error)
 	GetAllESDTTokens(address string, options api.AccountQueryOptions) (map[string]*esdt.ESDigitalToken, api.BlockInfo, error)
 	GetKeyValuePairs(address string, options api.AccountQueryOptions) (map[string]string, api.BlockInfo, error)
-	GetKeyValuePairsWithCheckpoint(address string, checkpointId string, numLeaves int, options api.AccountQueryOptions) (map[string]string, api.BlockInfo, string, error)
+	IterateKeys(address string, numKeys uint, iteratorState [][]byte, options api.AccountQueryOptions) (map[string]string, [][]byte, api.BlockInfo, error)
 	GetGuardianData(address string, options api.AccountQueryOptions) (api.GuardianData, api.BlockInfo, error)
 	GetBlockByHash(hash string, options api.BlockQueryOptions) (*api.Block, error)
 	GetBlockByNonce(nonce uint64, options api.BlockQueryOptions) (*api.Block, error)

cmd/node/config/api.toml

@@ -79,8 +79,8 @@
         # /address/:address/keys will return all the key-value pairs of a given account
         { Name = "/:address/keys", Open = true },
 
-        # address/:address/num-keys/:numKeys/checkpoint-id/:checkpointId will return the given num of key-value pairs for the given account
-        { Name = "/:address/num-keys/:numKeys/checkpoint-id/:checkpointId", Open = true },
+        # address//iterate-keys will return the given num of key-value pairs for the given account. The iteration will start from the given starting state
+        { Name = "/iterate-keys", Open = true },
 
         # /address/:address/key/:key will return the value of a key for a given account
         { Name = "/:address/key/:key", Open = true },

cmd/node/config/config.toml

@@ -665,7 +665,7 @@
 
 [TrieLeavesRetrieverConfig]
     Enabled = false
-    MaxSizeInBytes = 104857600 #100MB
+    MaxSizeInBytes = 10485760 #10MB
 
 [BlockSizeThrottleConfig]
     MinSizeInBytes = 104857 # 104857 is 10% from 1MB

common/interface.go

@@ -80,6 +80,7 @@ type StorageMarker interface {
 type KeyBuilder interface {
 	BuildKey(keyPart []byte)
 	GetKey() ([]byte, error)
+	GetRawKey() []byte
 	DeepClone() KeyBuilder
 	ShallowClone() KeyBuilder
 	Size() uint
@@ -386,16 +387,13 @@ type TrieNodeData interface {
 // DfsIterator is used to iterate the trie nodes in a depth-first search manner
 type DfsIterator interface {
 	GetLeaves(numLeaves int, maxSize uint64, ctx context.Context) (map[string]string, error)
-	GetIteratorId() []byte
-	Clone() DfsIterator
-	FinishedIteration() bool
-	Size() uint64
+	GetIteratorState() [][]byte
 	IsInterfaceNil() bool
 }
 
 // TrieLeavesRetriever is used to retrieve the leaves from the trie. If there is a saved checkpoint for the iterator id,
 // it will continue to iterate from the checkpoint.
 type TrieLeavesRetriever interface {
-	GetLeaves(numLeaves int, rootHash []byte, iteratorID []byte, ctx context.Context) (map[string]string, []byte, error)
+	GetLeaves(numLeaves int, iteratorState [][]byte, ctx context.Context) (map[string]string, [][]byte, error)
 	IsInterfaceNil() bool
 }

facade/initial/initialNodeFacade.go

@@ -346,9 +346,9 @@ func (inf *initialNodeFacade) GetKeyValuePairs(_ string, _ api.AccountQueryOptio
 	return nil, api.BlockInfo{}, errNodeStarting
 }
 
-// GetKeyValuePairsWithCheckpoint returns error
-func (inf *initialNodeFacade) GetKeyValuePairsWithCheckpoint(_ string, _ string, _ int, _ api.AccountQueryOptions) (map[string]string, api.BlockInfo, string, error) {
-	return nil, api.BlockInfo{}, "", errNodeStarting
+// IterateKeys returns error
+func (inf *initialNodeFacade) IterateKeys(_ string, _ uint, _ [][]byte, _ api.AccountQueryOptions) (map[string]string, [][]byte, api.BlockInfo, error) {
+	return nil, nil, api.BlockInfo{}, errNodeStarting
 }
 
 // GetGuardianData returns error

facade/interface.go

@@ -41,8 +41,8 @@ type NodeHandler interface {
 	// GetKeyValuePairs returns the key-value pairs under a given address
 	GetKeyValuePairs(address string, options api.AccountQueryOptions, ctx context.Context) (map[string]string, api.BlockInfo, error)
 
-	// GetKeyValuePairsWithCheckpoint returns the key-value pairs under a given address with a checkpoint
-	GetKeyValuePairsWithCheckpoint(address string, checkpointId string, numLeaves int, options api.AccountQueryOptions, ctx context.Context) (map[string]string, api.BlockInfo, string, error)
+	// IterateKeys returns the key-value pairs under a given address starting from a given state
+	IterateKeys(address string, numKeys uint, iteratorState [][]byte, options api.AccountQueryOptions, ctx context.Context) (map[string]string, [][]byte, api.BlockInfo, error)
 
 	// GetAllIssuedESDTs returns all the issued esdt tokens from esdt system smart contract
 	GetAllIssuedESDTs(tokenType string, ctx context.Context) ([]string, error)