diff --git a/common/reflectcommon/structFieldsUpdate.go b/common/reflectcommon/structFieldsUpdate.go index be8671eff4f..7f8940b0400 100644 --- a/common/reflectcommon/structFieldsUpdate.go +++ b/common/reflectcommon/structFieldsUpdate.go @@ -126,7 +126,7 @@ func trySetTheNewValue(value *reflect.Value, newValue interface{}) error { case reflect.Map: mapValue := reflect.ValueOf(newValue) - return tryUpdateMapValue(value, mapValue) + return trySetMapValue(value, mapValue) default: return fmt.Errorf("unsupported type <%s> when trying to set the value '%v' of type <%s>", valueKind, newValue, reflect.TypeOf(newValue)) } @@ -141,7 +141,7 @@ func trySetSliceValue(value *reflect.Value, newValue interface{}) error { item := sliceVal.Index(i) newItem := reflect.New(value.Type().Elem()).Elem() - err := trySetStructValue(&newItem, item) + err := trySetTheNewValue(&newItem, item.Interface()) if err != nil { return err } @@ -167,7 +167,7 @@ func trySetStructValue(value *reflect.Value, newValue reflect.Value) error { } } -func tryUpdateMapValue(value *reflect.Value, newValue reflect.Value) error { +func trySetMapValue(value *reflect.Value, newValue reflect.Value) error { if value.IsNil() { value.Set(reflect.MakeMap(value.Type())) } diff --git a/common/reflectcommon/structFieldsUpdate_test.go b/common/reflectcommon/structFieldsUpdate_test.go index 44d3ae7d694..27a30ea9d00 100644 --- a/common/reflectcommon/structFieldsUpdate_test.go +++ b/common/reflectcommon/structFieldsUpdate_test.go @@ -1204,6 +1204,79 @@ func TestAdaptStructureValueBasedOnPath(t *testing.T) { require.Equal(t, "unsupported type when trying to add value in type ", err.Error()) }) + t.Run("should work and override string array", func(t *testing.T) { + t.Parallel() + + testConfig, err := loadTestConfig("../../testscommon/toml/config.toml") + require.NoError(t, err) + + expectedArray := []string{"x", "y", "z"} + + err = AdaptStructureValueBasedOnPath(testConfig, "TestArray.Strings", expectedArray) + require.NoError(t, err) + require.Equal(t, expectedArray, testConfig.TestArray.Strings) + }) + + t.Run("should work and override int array", func(t *testing.T) { + t.Parallel() + + testConfig, err := loadTestConfig("../../testscommon/toml/config.toml") + require.NoError(t, err) + + expectedArray := []int{10, 20, 30} + + err = AdaptStructureValueBasedOnPath(testConfig, "TestArray.Ints", expectedArray) + require.NoError(t, err) + require.Equal(t, expectedArray, testConfig.TestArray.Ints) + }) + + t.Run("should work and override string array from toml", func(t *testing.T) { + t.Parallel() + + testConfig, err := loadTestConfig("../../testscommon/toml/config.toml") + require.NoError(t, err) + + overrideConfig, err := loadOverrideConfig("../../testscommon/toml/overwrite.toml") + require.NoError(t, err) + + err = AdaptStructureValueBasedOnPath(testConfig, overrideConfig.OverridableConfigTomlValues[38].Path, overrideConfig.OverridableConfigTomlValues[38].Value) + require.NoError(t, err) + expectedArray := []string{"x", "y", "z"} + require.Equal(t, expectedArray, testConfig.TestArray.Strings) + }) + + t.Run("should work and override int array from toml", func(t *testing.T) { + t.Parallel() + + testConfig, err := loadTestConfig("../../testscommon/toml/config.toml") + require.NoError(t, err) + + overrideConfig, err := loadOverrideConfig("../../testscommon/toml/overwrite.toml") + require.NoError(t, err) + + err = AdaptStructureValueBasedOnPath(testConfig, overrideConfig.OverridableConfigTomlValues[39].Path, overrideConfig.OverridableConfigTomlValues[39].Value) + require.NoError(t, err) + expectedArray := []int{10, 20, 30} + require.Equal(t, expectedArray, testConfig.TestArray.Ints) + }) + + t.Run("should work and override struct of arrays from toml", func(t *testing.T) { + t.Parallel() + + testConfig, err := loadTestConfig("../../testscommon/toml/config.toml") + require.NoError(t, err) + + overrideConfig, err := loadOverrideConfig("../../testscommon/toml/overwrite.toml") + require.NoError(t, err) + expectedStringsArray := []string{"x", "y", "z"} + expectedIntsArray := []int{10, 20, 30} + + err = AdaptStructureValueBasedOnPath(testConfig, overrideConfig.OverridableConfigTomlValues[40].Path, overrideConfig.OverridableConfigTomlValues[40].Value) + require.NoError(t, err) + require.Equal(t, expectedStringsArray, testConfig.TestArray.Strings) + require.Equal(t, expectedIntsArray, testConfig.TestArray.Ints) + }) + } func loadTestConfig(filepath string) (*toml.Config, error) { diff --git a/node/external/transactionAPI/apiTransactionProcessor.go b/node/external/transactionAPI/apiTransactionProcessor.go index a12d5efb974..45666c18b3c 100644 --- a/node/external/transactionAPI/apiTransactionProcessor.go +++ b/node/external/transactionAPI/apiTransactionProcessor.go @@ -394,8 +394,8 @@ func (atp *apiTransactionProcessor) getFieldGettersForTx(wrappedTx *txcache.Wrap rcvUsernameField: wrappedTx.Tx.GetRcvUserName(), dataField: wrappedTx.Tx.GetData(), valueField: getTxValue(wrappedTx), - senderShardID: wrappedTx.SenderShardID, - receiverShardID: wrappedTx.ReceiverShardID, + senderShardID: atp.shardCoordinator.ComputeId(wrappedTx.Tx.GetSndAddr()), + receiverShardID: atp.shardCoordinator.ComputeId(wrappedTx.Tx.GetRcvAddr()), } guardedTx, isGuardedTx := wrappedTx.Tx.(data.GuardedTransactionHandler) diff --git a/node/external/transactionAPI/apiTransactionProcessor_test.go b/node/external/transactionAPI/apiTransactionProcessor_test.go index 40357484880..3d3a894e737 100644 --- a/node/external/transactionAPI/apiTransactionProcessor_test.go +++ b/node/external/transactionAPI/apiTransactionProcessor_test.go @@ -936,6 +936,9 @@ func TestApiTransactionProcessor_GetTransactionsPoolForSender(t *testing.T) { NumberOfShardsCalled: func() uint32 { return 1 }, + ComputeIdCalled: func(address []byte) uint32 { + return 1 // force to return different from 0 + }, } atp, err := NewAPITransactionProcessor(args) require.NoError(t, err) @@ -948,7 +951,17 @@ func TestApiTransactionProcessor_GetTransactionsPoolForSender(t *testing.T) { for i, tx := range res.Transactions { require.Equal(t, expectedHashes[i], tx.TxFields[hashField]) require.Equal(t, expectedValues[i], tx.TxFields[valueField]) - require.Equal(t, sender, tx.TxFields["sender"]) + require.Equal(t, sender, tx.TxFields[senderField]) + require.Equal(t, uint32(1), tx.TxFields[senderShardID]) + require.Equal(t, uint32(1), tx.TxFields[senderShardID]) + } + + res, err = atp.GetTransactionsPoolForSender(sender, "sender,value") // no hash, should be by default + require.NoError(t, err) + for i, tx := range res.Transactions { + require.Equal(t, expectedHashes[i], tx.TxFields[hashField]) + require.Equal(t, expectedValues[i], tx.TxFields[valueField]) + require.Equal(t, sender, tx.TxFields[senderField]) } // if no tx is found in pool for a sender, it isn't an error, but return empty slice diff --git a/node/external/transactionAPI/fieldsHandler.go b/node/external/transactionAPI/fieldsHandler.go index 4f837968cb7..4102d9ffc61 100644 --- a/node/external/transactionAPI/fieldsHandler.go +++ b/node/external/transactionAPI/fieldsHandler.go @@ -38,8 +38,11 @@ func newFieldsHandler(parameters string) fieldsHandler { } parameters = strings.ToLower(parameters) + fieldsMap := sliceToMap(strings.Split(parameters, separator)) + fieldsMap[hashField] = struct{}{} // hashField should always be returned + return fieldsHandler{ - fieldsMap: sliceToMap(strings.Split(parameters, separator)), + fieldsMap: fieldsMap, } } diff --git a/node/external/transactionAPI/fieldsHandler_test.go b/node/external/transactionAPI/fieldsHandler_test.go index fab3b3a41d9..75b3ae6f81a 100644 --- a/node/external/transactionAPI/fieldsHandler_test.go +++ b/node/external/transactionAPI/fieldsHandler_test.go @@ -20,9 +20,11 @@ func Test_newFieldsHandler(t *testing.T) { for _, field := range splitFields { require.True(t, fh.IsFieldSet(field), fmt.Sprintf("field %s is not set", field)) } + require.True(t, fh.IsFieldSet(hashField), "hashField should have been returned by default") fh = newFieldsHandler("*") for _, field := range splitFields { require.True(t, fh.IsFieldSet(field)) } + require.True(t, fh.IsFieldSet(hashField), "hashField should have been returned by default") } diff --git a/storage/pruning/fullHistoryPruningStorer.go b/storage/pruning/fullHistoryPruningStorer.go index 71213b1dcdd..97852aa3bcd 100644 --- a/storage/pruning/fullHistoryPruningStorer.go +++ b/storage/pruning/fullHistoryPruningStorer.go @@ -184,6 +184,7 @@ func (fhps *FullHistoryPruningStorer) getOrOpenPersister(epoch uint32) (storage. } fhps.oldEpochsActivePersistersCache.Put([]byte(epochString), newPdata, 0) + log.Trace("full history pruning storer - init new storer", "epoch", epoch) fhps.persistersMapByEpoch[epoch] = newPdata return newPdata.getPersister(), nil diff --git a/storage/pruning/fullHistoryPruningStorer_test.go b/storage/pruning/fullHistoryPruningStorer_test.go index 0e0d43877e8..d1274499bb9 100644 --- a/storage/pruning/fullHistoryPruningStorer_test.go +++ b/storage/pruning/fullHistoryPruningStorer_test.go @@ -18,6 +18,7 @@ import ( "github.com/multiversx/mx-chain-go/storage/factory" "github.com/multiversx/mx-chain-go/storage/pathmanager" "github.com/multiversx/mx-chain-go/storage/pruning" + "github.com/multiversx/mx-chain-go/testscommon" logger "github.com/multiversx/mx-chain-logger-go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -399,3 +400,33 @@ func TestFullHistoryPruningStorer_IsInterfaceNil(t *testing.T) { fhps, _ = pruning.NewFullHistoryPruningStorer(fhArgs) require.False(t, fhps.IsInterfaceNil()) } + +func TestFullHistoryPruningStorer_changeEpochClosesOldDbs(t *testing.T) { + t.Parallel() + + shouldCleanCalled := false + args := getDefaultArgs() + fhArgs := pruning.FullHistoryStorerArgs{ + StorerArgs: args, + NumOfOldActivePersisters: 2, + } + fhArgs.OldDataCleanerProvider = &testscommon.OldDataCleanerProviderStub{ + ShouldCleanCalled: func() bool { + shouldCleanCalled = true + return true + }, + } + fhps, err := pruning.NewFullHistoryPruningStorer(fhArgs) + require.Nil(t, err) + + numEpochsChanged := 10 + startEpoch := uint32(0) + for i := 0; i < numEpochsChanged; i++ { + startEpoch++ + key := []byte(fmt.Sprintf("key-%d", i)) + _, _ = fhps.GetFromEpoch(key, startEpoch) + err = fhps.ChangeEpochSimple(startEpoch) + require.Nil(t, err) + } + require.True(t, shouldCleanCalled) +} diff --git a/storage/pruning/pruningStorer.go b/storage/pruning/pruningStorer.go index 2007454a7c8..d40680e5c87 100644 --- a/storage/pruning/pruningStorer.go +++ b/storage/pruning/pruningStorer.go @@ -779,7 +779,7 @@ func (ps *PruningStorer) changeEpoch(header data.HeaderHandler) error { } log.Debug("change epoch pruning storer success", "persister", ps.identifier, "epoch", epoch) - return nil + return ps.removeOldPersistersIfNeeded(header) } shardID := core.GetShardIDString(ps.shardCoordinator.SelfId()) @@ -802,6 +802,11 @@ func (ps *PruningStorer) changeEpoch(header data.HeaderHandler) error { ps.activePersisters = append(singleItemPersisters, ps.activePersisters...) ps.persistersMapByEpoch[epoch] = newPersister + return ps.removeOldPersistersIfNeeded(header) +} + +func (ps *PruningStorer) removeOldPersistersIfNeeded(header data.HeaderHandler) error { + epoch := header.GetEpoch() wasExtended := ps.extendSavedEpochsIfNeeded(header) if wasExtended { if len(ps.activePersisters) > int(ps.numOfActivePersisters) { @@ -814,11 +819,12 @@ func (ps *PruningStorer) changeEpoch(header data.HeaderHandler) error { return nil } - err = ps.closeAndDestroyPersisters(epoch) + err := ps.closeAndDestroyPersisters(epoch) if err != nil { log.Warn("closing persisters", "error", err.Error()) return err } + return nil } diff --git a/testscommon/toml/config.go b/testscommon/toml/config.go index 56cfeb1f0ad..7f64ee446d4 100644 --- a/testscommon/toml/config.go +++ b/testscommon/toml/config.go @@ -16,6 +16,7 @@ type Config struct { TestConfigNestedStruct TestMap TestInterface + TestArray } // TestConfigI8 will hold an int8 value for testing @@ -180,3 +181,9 @@ type MapValues struct { type TestInterface struct { Value interface{} } + +// TestArray will hold an array of strings and integers +type TestArray struct { + Strings []string + Ints []int +} diff --git a/testscommon/toml/config.toml b/testscommon/toml/config.toml index 91512d5e664..890e3922789 100644 --- a/testscommon/toml/config.toml +++ b/testscommon/toml/config.toml @@ -51,3 +51,7 @@ [Map] [Map.Key1] Number = 999 + +[TestArray] + Strings = ["a", "b", "c"] + Ints = [0, 1, 2] diff --git a/testscommon/toml/overwrite.toml b/testscommon/toml/overwrite.toml index 63f74b7828c..5e495e5c08b 100644 --- a/testscommon/toml/overwrite.toml +++ b/testscommon/toml/overwrite.toml @@ -1,40 +1,43 @@ OverridableConfigTomlValues = [ - { File = "config.toml", Path = "TestConfigI8.Int8.Number", Value = 127 }, - { File = "config.toml", Path = "TestConfigI8.Int8.Number", Value = 128 }, - { File = "config.toml", Path = "TestConfigI8.Int8.Number", Value = -128 }, - { File = "config.toml", Path = "TestConfigI8.Int8.Number", Value = -129 }, - { File = "config.toml", Path = "TestConfigI16.Int16.Number", Value = 32767 }, - { File = "config.toml", Path = "TestConfigI16.Int16.Number", Value = 32768 }, - { File = "config.toml", Path = "TestConfigI16.Int16.Number", Value = -32768 }, - { File = "config.toml", Path = "TestConfigI16.Int16.Number", Value = -32769 }, - { File = "config.toml", Path = "TestConfigI32.Int32.Number", Value = 2147483647 }, - { File = "config.toml", Path = "TestConfigI32.Int32.Number", Value = 2147483648 }, - { File = "config.toml", Path = "TestConfigI32.Int32.Number", Value = -2147483648 }, - { File = "config.toml", Path = "TestConfigI32.Int32.Number", Value = -2147483649 }, - { File = "config.toml", Path = "TestConfigI64.Int64.Number", Value = 9223372036854775807 }, - { File = "config.toml", Path = "TestConfigI64.Int64.Number", Value = -9223372036854775808 }, - { File = "config.toml", Path = "TestConfigU8.Uint8.Number", Value = 255 }, - { File = "config.toml", Path = "TestConfigU8.Uint8.Number", Value = 256 }, - { File = "config.toml", Path = "TestConfigU8.Uint8.Number", Value = -256 }, - { File = "config.toml", Path = "TestConfigU16.Uint16.Number", Value = 65535 }, - { File = "config.toml", Path = "TestConfigU16.Uint16.Number", Value = 65536 }, - { File = "config.toml", Path = "TestConfigU16.Uint16.Number", Value = -65536 }, - { File = "config.toml", Path = "TestConfigU32.Uint32.Number", Value = 4294967295 }, - { File = "config.toml", Path = "TestConfigU32.Uint32.Number", Value = 4294967296 }, - { File = "config.toml", Path = "TestConfigU32.Uint32.Number", Value = -4294967296 }, - { File = "config.toml", Path = "TestConfigU64.Uint64.Number", Value = 9223372036854775807 }, - { File = "config.toml", Path = "TestConfigU64.Uint64.Number", Value = -9223372036854775808 }, - { File = "config.toml", Path = "TestConfigF32.Float32.Number", Value = 3.4 }, - { File = "config.toml", Path = "TestConfigF32.Float32.Number", Value = 3.4e+39 }, - { File = "config.toml", Path = "TestConfigF32.Float32.Number", Value = -3.4 }, - { File = "config.toml", Path = "TestConfigF32.Float32.Number", Value = -3.4e+40 }, - { File = "config.toml", Path = "TestConfigF64.Float64.Number", Value = 1.7e+308 }, - { File = "config.toml", Path = "TestConfigF64.Float64.Number", Value = -1.7e+308 }, - { File = "config.toml", Path = "TestConfigStruct.ConfigStruct.Description", Value = { Number = 11 } }, - { File = "config.toml", Path = "TestConfigStruct.ConfigStruct.Description", Value = { Nr = 222 } }, - { File = "config.toml", Path = "TestConfigStruct.ConfigStruct.Description", Value = { Number = "11" } }, - { File = "config.toml", Path = "TestConfigNestedStruct.ConfigNestedStruct", Value = { Text = "Overwritten text", Message = { Public = false, MessageDescription = [{ Text = "Overwritten Text1" }] } } }, - { File = "config.toml", Path = "TestConfigNestedStruct.ConfigNestedStruct.Message.MessageDescription", Value = [{ Text = "Overwritten Text1" }, { Text = "Overwritten Text2" }] }, - { File = "config.toml", Path = "TestMap.Map", Value = { "Key1" = { Number = 10 }, "Key2" = { Number = 11 } } }, - { File = "config.toml", Path = "TestMap.Map", Value = { "Key2" = { Number = 2 }, "Key3" = { Number = 3 } } }, + { File = "config.toml", Path = "TestConfigI8.Int8.Number", Value = 127 }, + { File = "config.toml", Path = "TestConfigI8.Int8.Number", Value = 128 }, + { File = "config.toml", Path = "TestConfigI8.Int8.Number", Value = -128 }, + { File = "config.toml", Path = "TestConfigI8.Int8.Number", Value = -129 }, + { File = "config.toml", Path = "TestConfigI16.Int16.Number", Value = 32767 }, + { File = "config.toml", Path = "TestConfigI16.Int16.Number", Value = 32768 }, + { File = "config.toml", Path = "TestConfigI16.Int16.Number", Value = -32768 }, + { File = "config.toml", Path = "TestConfigI16.Int16.Number", Value = -32769 }, + { File = "config.toml", Path = "TestConfigI32.Int32.Number", Value = 2147483647 }, + { File = "config.toml", Path = "TestConfigI32.Int32.Number", Value = 2147483648 }, + { File = "config.toml", Path = "TestConfigI32.Int32.Number", Value = -2147483648 }, + { File = "config.toml", Path = "TestConfigI32.Int32.Number", Value = -2147483649 }, + { File = "config.toml", Path = "TestConfigI64.Int64.Number", Value = 9223372036854775807 }, + { File = "config.toml", Path = "TestConfigI64.Int64.Number", Value = -9223372036854775808 }, + { File = "config.toml", Path = "TestConfigU8.Uint8.Number", Value = 255 }, + { File = "config.toml", Path = "TestConfigU8.Uint8.Number", Value = 256 }, + { File = "config.toml", Path = "TestConfigU8.Uint8.Number", Value = -256 }, + { File = "config.toml", Path = "TestConfigU16.Uint16.Number", Value = 65535 }, + { File = "config.toml", Path = "TestConfigU16.Uint16.Number", Value = 65536 }, + { File = "config.toml", Path = "TestConfigU16.Uint16.Number", Value = -65536 }, + { File = "config.toml", Path = "TestConfigU32.Uint32.Number", Value = 4294967295 }, + { File = "config.toml", Path = "TestConfigU32.Uint32.Number", Value = 4294967296 }, + { File = "config.toml", Path = "TestConfigU32.Uint32.Number", Value = -4294967296 }, + { File = "config.toml", Path = "TestConfigU64.Uint64.Number", Value = 9223372036854775807 }, + { File = "config.toml", Path = "TestConfigU64.Uint64.Number", Value = -9223372036854775808 }, + { File = "config.toml", Path = "TestConfigF32.Float32.Number", Value = 3.4 }, + { File = "config.toml", Path = "TestConfigF32.Float32.Number", Value = 3.4e+39 }, + { File = "config.toml", Path = "TestConfigF32.Float32.Number", Value = -3.4 }, + { File = "config.toml", Path = "TestConfigF32.Float32.Number", Value = -3.4e+40 }, + { File = "config.toml", Path = "TestConfigF64.Float64.Number", Value = 1.7e+308 }, + { File = "config.toml", Path = "TestConfigF64.Float64.Number", Value = -1.7e+308 }, + { File = "config.toml", Path = "TestConfigStruct.ConfigStruct.Description", Value = { Number = 11 } }, + { File = "config.toml", Path = "TestConfigStruct.ConfigStruct.Description", Value = { Nr = 222 } }, + { File = "config.toml", Path = "TestConfigStruct.ConfigStruct.Description", Value = { Number = "11" } }, + { File = "config.toml", Path = "TestConfigNestedStruct.ConfigNestedStruct", Value = { Text = "Overwritten text", Message = { Public = false, MessageDescription = [{ Text = "Overwritten Text1" }] } } }, + { File = "config.toml", Path = "TestConfigNestedStruct.ConfigNestedStruct.Message.MessageDescription", Value = [{ Text = "Overwritten Text1" }, { Text = "Overwritten Text2" }] }, + { File = "config.toml", Path = "TestMap.Map", Value = { "Key1" = { Number = 10 }, "Key2" = { Number = 11 } } }, + { File = "config.toml", Path = "TestMap.Map", Value = { "Key2" = { Number = 2 }, "Key3" = { Number = 3 } } }, + { File = "config.toml", Path = "TestArray.Strings", Value = ["x", "y", "z"] }, + { File = "config.toml", Path = "TestArray.Ints", Value = [10, 20, 30] }, + { File = "config.toml", Path = "TestArray", Value = { Strings = ["x", "y", "z"], Ints = [10, 20, 30] } }, ]