integrate new account storage iterator

api/errors/errors.go

@@ -28,6 +28,9 @@ var ErrGetValueForKey = errors.New("get value for key error")
 // ErrGetKeyValuePairs signals an error in getting the key-value pairs of a key for an account
 var ErrGetKeyValuePairs = errors.New("get key-value pairs error")
 
+// ErrIterateKeys signals an error in iterating over the keys of an account
+var ErrIterateKeys = errors.New("iterate keys error")
+
 // ErrGetESDTBalance signals an error in getting esdt balance for given address
 var ErrGetESDTBalance = errors.New("get esdt balance for account error")
 
@@ -43,6 +46,12 @@ var ErrGetESDTNFTData = errors.New("get esdt nft data for account error")
 // ErrEmptyAddress signals that an empty address was provided
 var ErrEmptyAddress = errors.New("address is empty")
 
+// ErrEmptyNumKeys signals that an empty numKeys was provided
+var ErrEmptyNumKeys = errors.New("numKeys is empty")
+
+// ErrEmptyCheckpointId signals that an empty checkpointId was provided
+var ErrEmptyCheckpointId = errors.New("checkpointId is empty")
+
 // ErrEmptyKey signals that an empty key was provided
 var ErrEmptyKey = errors.New("key is empty")
 

api/groups/addressGroup.go

@@ -32,6 +32,7 @@ const (
 	getRegisteredNFTsPath          = "/:address/registered-nfts"
 	getESDTNFTDataPath             = "/:address/nft/:tokenIdentifier/nonce/:nonce"
 	getGuardianData                = "/:address/guardian-data"
+	iterateKeysPath                = "/iterate-keys"
 	urlParamOnFinalBlock           = "onFinalBlock"
 	urlParamOnStartOfEpoch         = "onStartOfEpoch"
 	urlParamBlockNonce             = "blockNonce"
@@ -55,6 +56,7 @@ type addressFacadeHandler interface {
 	GetESDTsWithRole(address string, role string, options api.AccountQueryOptions) ([]string, api.BlockInfo, error)
 	GetAllESDTTokens(address string, options api.AccountQueryOptions) (map[string]*esdt.ESDigitalToken, api.BlockInfo, error)
 	GetKeyValuePairs(address string, options api.AccountQueryOptions) (map[string]string, api.BlockInfo, error)
+	IterateKeys(address string, numKeys uint, iteratorState [][]byte, options api.AccountQueryOptions) (map[string]string, [][]byte, api.BlockInfo, error)
 	GetGuardianData(address string, options api.AccountQueryOptions) (api.GuardianData, api.BlockInfo, error)
 	IsDataTrieMigrated(address string, options api.AccountQueryOptions) (bool, error)
 	IsInterfaceNil() bool
@@ -134,6 +136,11 @@ func NewAddressGroup(facade addressFacadeHandler) (*addressGroup, error) {
 			Method:  http.MethodGet,
 			Handler: ag.getKeyValuePairs,
 		},
+		{
+			Path:    iterateKeysPath,
+			Method:  http.MethodPost,
+			Handler: ag.iterateKeys,
+		},
 		{
 			Path:    getESDTBalancePath,
 			Method:  http.MethodGet,
@@ -327,7 +334,7 @@ func (ag *addressGroup) getGuardianData(c *gin.Context) {
 	shared.RespondWithSuccess(c, gin.H{"guardianData": guardianData, "blockInfo": blockInfo})
 }
 
-// addressGroup returns all the key-value pairs for the given address
+// getKeyValuePairs returns all the key-value pairs for the given address
 func (ag *addressGroup) getKeyValuePairs(c *gin.Context) {
 	addr, options, err := extractBaseParams(c)
 	if err != nil {
@@ -344,6 +351,47 @@ func (ag *addressGroup) getKeyValuePairs(c *gin.Context) {
 	shared.RespondWithSuccess(c, gin.H{"pairs": value, "blockInfo": blockInfo})
 }
 
+// IterateKeysRequest defines the request structure for iterating keys
+type IterateKeysRequest struct {
+	Address       string   `json:"address"`
+	NumKeys       uint     `json:"numKeys"`
+	IteratorState [][]byte `json:"iteratorState"`
+}
+
+// iterateKeys iterates keys for the given address
+func (ag *addressGroup) iterateKeys(c *gin.Context) {
+	var iterateKeysRequest = &IterateKeysRequest{}
+	err := c.ShouldBindJSON(&iterateKeysRequest)
+	if err != nil {
+		shared.RespondWithValidationError(c, errors.ErrValidation, err)
+		return
+	}
+
+	if len(iterateKeysRequest.Address) == 0 {
+		shared.RespondWithValidationError(c, errors.ErrValidation, errors.ErrEmptyAddress)
+		return
+	}
+
+	options, err := extractAccountQueryOptions(c)
+	if err != nil {
+		shared.RespondWithValidationError(c, errors.ErrIterateKeys, err)
+		return
+	}
+
+	value, newIteratorState, blockInfo, err := ag.getFacade().IterateKeys(
+		iterateKeysRequest.Address,
+		iterateKeysRequest.NumKeys,
+		iterateKeysRequest.IteratorState,
+		options,
+	)
+	if err != nil {
+		shared.RespondWithInternalError(c, errors.ErrIterateKeys, err)
+		return
+	}
+
+	shared.RespondWithSuccess(c, gin.H{"pairs": value, "newIteratorState": newIteratorState, "blockInfo": blockInfo})
+}
+
 // getESDTBalance returns the balance for the given address and esdt token
 func (ag *addressGroup) getESDTBalance(c *gin.Context) {
 	addr, tokenIdentifier, options, err := extractGetESDTBalanceParams(c)

api/groups/addressGroup_test.go

@@ -125,6 +125,16 @@ type keyValuePairsResponse struct {
 	Code  string
 }
 
+type iterateKeysResponseData struct {
+	Pairs            map[string]string `json:"pairs"`
+	NewIteratorState [][]byte          `json:"newIteratorState"`
+}
+type iterateKeysResponse struct {
+	Data  iterateKeysResponseData `json:"data"`
+	Error string                  `json:"error"`
+	Code  string
+}
+
 type esdtRolesResponseData struct {
 	Roles map[string][]string `json:"roles"`
 }
@@ -662,6 +672,106 @@ func TestAddressGroup_getKeyValuePairs(t *testing.T) {
 	})
 }
 
+func TestAddressGroup_iterateKeys(t *testing.T) {
+	t.Parallel()
+
+	t.Run("invalid body should error",
+		testErrorScenario("/address/iterate-keys", "POST", bytes.NewBuffer([]byte("invalid body")),
+			formatExpectedErr(apiErrors.ErrValidation, errors.New("invalid character 'i' looking for beginning of value"))))
+	t.Run("empty address should error", func(t *testing.T) {
+		t.Parallel()
+
+		body := &groups.IterateKeysRequest{
+			Address: "",
+		}
+		bodyBytes, _ := json.Marshal(body)
+		testAddressGroup(
+			t,
+			&mock.FacadeStub{},
+			"/address/iterate-keys",
+			"POST",
+			bytes.NewBuffer(bodyBytes),
+			http.StatusBadRequest,
+			formatExpectedErr(apiErrors.ErrValidation, apiErrors.ErrEmptyAddress),
+		)
+	})
+	t.Run("invalid query options should error", func(t *testing.T) {
+		t.Parallel()
+
+		body := &groups.IterateKeysRequest{
+			Address: "erd1",
+		}
+		bodyBytes, _ := json.Marshal(body)
+		testAddressGroup(
+			t,
+			&mock.FacadeStub{},
+			"/address/iterate-keys?blockNonce=not-uint64",
+			"POST",
+			bytes.NewBuffer(bodyBytes),
+			http.StatusBadRequest,
+			formatExpectedErr(apiErrors.ErrIterateKeys, apiErrors.ErrBadUrlParams),
+		)
+	})
+	t.Run("with node fail should err", func(t *testing.T) {
+		t.Parallel()
+
+		body := &groups.IterateKeysRequest{
+			Address: "erd1",
+		}
+		bodyBytes, _ := json.Marshal(body)
+		facade := &mock.FacadeStub{
+			IterateKeysCalled: func(address string, numKeys uint, iteratorState [][]byte, options api.AccountQueryOptions) (map[string]string, [][]byte, api.BlockInfo, error) {
+				return nil, nil, api.BlockInfo{}, expectedErr
+			},
+		}
+		testAddressGroup(
+			t,
+			facade,
+			"/address/iterate-keys",
+			"POST",
+			bytes.NewBuffer(bodyBytes),
+			http.StatusInternalServerError,
+			formatExpectedErr(apiErrors.ErrIterateKeys, expectedErr),
+		)
+	})
+	t.Run("should work", func(t *testing.T) {
+		t.Parallel()
+
+		pairs := map[string]string{
+			"k1": "v1",
+			"k2": "v2",
+		}
+
+		body := &groups.IterateKeysRequest{
+			Address:       "erd1",
+			NumKeys:       10,
+			IteratorState: [][]byte{[]byte("starting"), []byte("state")},
+		}
+		newIteratorState := [][]byte{[]byte("new"), []byte("state")}
+		bodyBytes, _ := json.Marshal(body)
+		facade := &mock.FacadeStub{
+			IterateKeysCalled: func(address string, numKeys uint, iteratorState [][]byte, options api.AccountQueryOptions) (map[string]string, [][]byte, api.BlockInfo, error) {
+				assert.Equal(t, body.Address, address)
+				assert.Equal(t, body.NumKeys, numKeys)
+				assert.Equal(t, body.IteratorState, iteratorState)
+				return pairs, newIteratorState, api.BlockInfo{}, nil
+			},
+		}
+
+		response := &iterateKeysResponse{}
+		loadAddressGroupResponse(
+			t,
+			facade,
+			"/address/iterate-keys",
+			"POST",
+			bytes.NewBuffer(bodyBytes),
+			response,
+		)
+		assert.Equal(t, pairs, response.Data.Pairs)
+		assert.Equal(t, newIteratorState, response.Data.NewIteratorState)
+	})
+}
+
 func TestAddressGroup_getESDTBalance(t *testing.T) {
 	t.Parallel()
 
@@ -1143,6 +1253,7 @@ func getAddressRoutesConfig() config.ApiRoutesConfig {
 					{Name: "/:address/username", Open: true},
 					{Name: "/:address/code-hash", Open: true},
 					{Name: "/:address/keys", Open: true},
+					{Name: "/iterate-keys", Open: true},
 					{Name: "/:address/key/:key", Open: true},
 					{Name: "/:address/esdt", Open: true},
 					{Name: "/:address/esdts/roles", Open: true},

api/mock/facadeStub.go

@@ -49,6 +49,7 @@ type FacadeStub struct {
 	GetUsernameCalled                           func(address string, options api.AccountQueryOptions) (string, api.BlockInfo, error)
 	GetCodeHashCalled                           func(address string, options api.AccountQueryOptions) ([]byte, api.BlockInfo, error)
 	GetKeyValuePairsCalled                      func(address string, options api.AccountQueryOptions) (map[string]string, api.BlockInfo, error)
+	IterateKeysCalled                           func(address string, numKeys uint, iteratorState [][]byte, options api.AccountQueryOptions) (map[string]string, [][]byte, api.BlockInfo, error)
 	SimulateTransactionExecutionHandler         func(tx *transaction.Transaction) (*txSimData.SimulationResultsWithVMOutput, error)
 	GetESDTDataCalled                           func(address string, key string, nonce uint64, options api.AccountQueryOptions) (*esdt.ESDigitalToken, api.BlockInfo, error)
 	GetAllESDTTokensCalled                      func(address string, options api.AccountQueryOptions) (map[string]*esdt.ESDigitalToken, api.BlockInfo, error)
@@ -241,6 +242,15 @@ func (f *FacadeStub) GetKeyValuePairs(address string, options api.AccountQueryOp
 	return nil, api.BlockInfo{}, nil
 }
 
+// IterateKeys -
+func (f *FacadeStub) IterateKeys(address string, numKeys uint, iteratorState [][]byte, options api.AccountQueryOptions) (map[string]string, [][]byte, api.BlockInfo, error) {
+	if f.IterateKeysCalled != nil {
+		return f.IterateKeysCalled(address, numKeys, iteratorState, options)
+	}
+
+	return nil, nil, api.BlockInfo{}, nil
+}
+
 // GetGuardianData -
 func (f *FacadeStub) GetGuardianData(address string, options api.AccountQueryOptions) (api.GuardianData, api.BlockInfo, error) {
 	if f.GetGuardianDataCalled != nil {

api/shared/interface.go

@@ -74,6 +74,7 @@ type FacadeHandler interface {
 	GetESDTsWithRole(address string, role string, options api.AccountQueryOptions) ([]string, api.BlockInfo, error)
 	GetAllESDTTokens(address string, options api.AccountQueryOptions) (map[string]*esdt.ESDigitalToken, api.BlockInfo, error)
 	GetKeyValuePairs(address string, options api.AccountQueryOptions) (map[string]string, api.BlockInfo, error)
+	IterateKeys(address string, numKeys uint, iteratorState [][]byte, options api.AccountQueryOptions) (map[string]string, [][]byte, api.BlockInfo, error)
 	GetGuardianData(address string, options api.AccountQueryOptions) (api.GuardianData, api.BlockInfo, error)
 	GetBlockByHash(hash string, options api.BlockQueryOptions) (*api.Block, error)
 	GetBlockByNonce(nonce uint64, options api.BlockQueryOptions) (*api.Block, error)

cmd/node/config/api.toml

@@ -79,6 +79,9 @@
         # /address/:address/keys will return all the key-value pairs of a given account
         { Name = "/:address/keys", Open = true },
 
+        # address//iterate-keys will return the given num of key-value pairs for the given account. The iteration will start from the given starting state
+        { Name = "/iterate-keys", Open = true },
+
         # /address/:address/key/:key will return the value of a key for a given account
         { Name = "/:address/key/:key", Open = true },
 

cmd/node/config/config.toml

@@ -663,6 +663,10 @@
     MaxPeerTrieLevelInMemory = 5
     StateStatisticsEnabled = false
 
+[TrieLeavesRetrieverConfig]
+    Enabled = false
+    MaxSizeInBytes = 10485760 #10MB
+
 [BlockSizeThrottleConfig]
     MinSizeInBytes = 104857 # 104857 is 10% from 1MB
     MaxSizeInBytes = 943718 # 943718 is 90% from 1MB

common/interface.go

@@ -80,6 +80,7 @@ type StorageMarker interface {
 type KeyBuilder interface {
 	BuildKey(keyPart []byte)
 	GetKey() ([]byte, error)
+	GetRawKey() []byte
 	DeepClone() KeyBuilder
 	ShallowClone() KeyBuilder
 	Size() uint
@@ -381,21 +382,19 @@ type TrieNodeData interface {
 	GetData() []byte
 	Size() uint64
 	IsLeaf() bool
+	GetVersion() core.TrieNodeVersion
 }
 
 // DfsIterator is used to iterate the trie nodes in a depth-first search manner
 type DfsIterator interface {
-	GetLeaves(numLeaves int, ctx context.Context) (map[string]string, error)
-	GetIteratorId() []byte
-	Clone() DfsIterator
-	FinishedIteration() bool
-	Size() uint64
+	GetLeaves(numLeaves int, maxSize uint64, leavesParser TrieLeafParser, ctx context.Context) (map[string]string, error)
+	GetIteratorState() [][]byte
 	IsInterfaceNil() bool
 }
 
 // TrieLeavesRetriever is used to retrieve the leaves from the trie. If there is a saved checkpoint for the iterator id,
 // it will continue to iterate from the checkpoint.
 type TrieLeavesRetriever interface {
-	GetLeaves(numLeaves int, rootHash []byte, iteratorID []byte, ctx context.Context) (map[string]string, []byte, error)
+	GetLeaves(numLeaves int, iteratorState [][]byte, leavesParser TrieLeafParser, ctx context.Context) (map[string]string, [][]byte, error)
 	IsInterfaceNil() bool
 }

config/config.go

@@ -161,12 +161,13 @@ type Config struct {
 	BootstrapStorage StorageConfig
 	MetaBlockStorage StorageConfig
 
-	AccountsTrieStorage      StorageConfig
-	PeerAccountsTrieStorage  StorageConfig
-	EvictionWaitingList      EvictionWaitingListConfig
-	StateTriesConfig         StateTriesConfig
-	TrieStorageManagerConfig TrieStorageManagerConfig
-	BadBlocksCache           CacheConfig
+	AccountsTrieStorage       StorageConfig
+	PeerAccountsTrieStorage   StorageConfig
+	EvictionWaitingList       EvictionWaitingListConfig
+	StateTriesConfig          StateTriesConfig
+	TrieStorageManagerConfig  TrieStorageManagerConfig
+	TrieLeavesRetrieverConfig TrieLeavesRetrieverConfig
+	BadBlocksCache            CacheConfig
 
 	TxBlockBodyDataPool         CacheConfig
 	PeerBlockBodyDataPool       CacheConfig
@@ -640,3 +641,9 @@ type PoolsCleanersConfig struct {
 type RedundancyConfig struct {
 	MaxRoundsOfInactivityAccepted int
 }
+
+// TrieLeavesRetrieverConfig represents the config options to be used when setting up the trie leaves retriever
+type TrieLeavesRetrieverConfig struct {
+	Enabled        bool
+	MaxSizeInBytes uint64
+}

errors/errors.go

@@ -598,3 +598,6 @@ var ErrNilSentSignatureTracker = errors.New("nil sent signature tracker")
 
 // ErrNilEpochSystemSCProcessor defines the error for setting a nil EpochSystemSCProcessor
 var ErrNilEpochSystemSCProcessor = errors.New("nil epoch system SC processor")
+
+// ErrNilTrieLeavesRetriever defines the error for setting a nil TrieLeavesRetriever
+var ErrNilTrieLeavesRetriever = errors.New("nil trie leaves retriever")

facade/initial/initialNodeFacade.go

@@ -346,6 +346,11 @@ func (inf *initialNodeFacade) GetKeyValuePairs(_ string, _ api.AccountQueryOptio
 	return nil, api.BlockInfo{}, errNodeStarting
 }
 
+// IterateKeys returns error
+func (inf *initialNodeFacade) IterateKeys(_ string, _ uint, _ [][]byte, _ api.AccountQueryOptions) (map[string]string, [][]byte, api.BlockInfo, error) {
+	return nil, nil, api.BlockInfo{}, errNodeStarting
+}
+
 // GetGuardianData returns error
 func (inf *initialNodeFacade) GetGuardianData(_ string, _ api.AccountQueryOptions) (api.GuardianData, api.BlockInfo, error) {
 	return api.GuardianData{}, api.BlockInfo{}, errNodeStarting

facade/interface.go

@@ -41,6 +41,9 @@ type NodeHandler interface {
 	// GetKeyValuePairs returns the key-value pairs under a given address
 	GetKeyValuePairs(address string, options api.AccountQueryOptions, ctx context.Context) (map[string]string, api.BlockInfo, error)
 
+	// IterateKeys returns the key-value pairs under a given address starting from a given state
+	IterateKeys(address string, numKeys uint, iteratorState [][]byte, options api.AccountQueryOptions, ctx context.Context) (map[string]string, [][]byte, api.BlockInfo, error)
+
 	// GetAllIssuedESDTs returns all the issued esdt tokens from esdt system smart contract
 	GetAllIssuedESDTs(tokenType string, ctx context.Context) ([]string, error)