diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java index e08fb29f673..75307d95202 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java @@ -17,8 +17,8 @@ import static com.google.common.base.Preconditions.checkState; import static java.nio.charset.StandardCharsets.UTF_8; -import org.hyperledger.besu.cli.options.DataStorageOptions; import org.hyperledger.besu.cli.options.TransactionPoolOptions; +import org.hyperledger.besu.cli.options.storage.DataStorageOptions; import org.hyperledger.besu.cli.options.unstable.NetworkingOptions; import org.hyperledger.besu.ethereum.api.jsonrpc.ipc.JsonRpcIpcConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration; diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 08f36d7fed5..886e3d68383 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -41,7 +41,6 @@ import org.hyperledger.besu.cli.custom.JsonRPCAllowlistHostsProperty; import org.hyperledger.besu.cli.error.BesuExecutionExceptionHandler; import org.hyperledger.besu.cli.error.BesuParameterExceptionHandler; -import org.hyperledger.besu.cli.options.DataStorageOptions; import org.hyperledger.besu.cli.options.MiningOptions; import org.hyperledger.besu.cli.options.TransactionPoolOptions; import org.hyperledger.besu.cli.options.stable.ApiConfigurationOptions; @@ -57,6 +56,8 @@ import org.hyperledger.besu.cli.options.stable.PermissionsOptions; import org.hyperledger.besu.cli.options.stable.PluginsConfigurationOptions; import org.hyperledger.besu.cli.options.stable.RpcWebsocketOptions; +import org.hyperledger.besu.cli.options.storage.DataStorageOptions; +import org.hyperledger.besu.cli.options.storage.DiffBasedSubStorageOptions; import org.hyperledger.besu.cli.options.unstable.ChainPruningOptions; import org.hyperledger.besu.cli.options.unstable.DnsOptions; import org.hyperledger.besu.cli.options.unstable.EthProtocolOptions; @@ -139,7 +140,9 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder; import org.hyperledger.besu.ethereum.transaction.TransactionSimulator; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; +import org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; +import org.hyperledger.besu.ethereum.worldstate.ImmutableDiffBasedSubStorageConfiguration; import org.hyperledger.besu.evm.precompile.AbstractAltBnPrecompiledContract; import org.hyperledger.besu.evm.precompile.BigIntegerModularExponentiationPrecompiledContract; import org.hyperledger.besu.evm.precompile.KZGPointEvalPrecompiledContract; @@ -1643,7 +1646,11 @@ && isOptionSet(commandLine, "--sync-min-peers")) { CommandLineUtils.failIfOptionDoesntMeetRequirement( commandLine, "--Xsnapsync-synchronizer-flat option can only be used when --Xbonsai-full-flat-db-enabled is true", - dataStorageOptions.toDomainObject().getUnstable().getBonsaiFullFlatDbEnabled(), + dataStorageOptions + .toDomainObject() + .getDiffBasedSubStorageConfiguration() + .getUnstable() + .getFullFlatDbEnabled(), asList( "--Xsnapsync-synchronizer-flat-account-healed-count-per-request", "--Xsnapsync-synchronizer-flat-slot-healed-count-per-request")); @@ -1775,38 +1782,46 @@ public BesuControllerBuilder setupControllerBuilder() { .withMiningParameters(miningParametersSupplier.get()) .withJsonRpcHttpOptions(jsonRpcHttpOptions); final KeyValueStorageProvider storageProvider = keyValueStorageProvider(keyValueStorageName); - return controllerBuilder - .fromEthNetworkConfig(updateNetworkConfig(network), getDefaultSyncModeIfNotSet()) - .synchronizerConfiguration(buildSyncConfig()) - .ethProtocolConfiguration(unstableEthProtocolOptions.toDomainObject()) - .networkConfiguration(unstableNetworkingOptions.toDomainObject()) - .dataDirectory(dataDir()) - .dataStorageConfiguration(getDataStorageConfiguration()) - .miningParameters(miningParametersSupplier.get()) - .transactionPoolConfiguration(buildTransactionPoolConfiguration()) - .nodeKey(new NodeKey(securityModule())) - .metricsSystem((ObservableMetricsSystem) besuComponent.getMetricsSystem()) - .messagePermissioningProviders(permissioningService.getMessagePermissioningProviders()) - .privacyParameters(privacyParameters()) - .clock(Clock.systemUTC()) - .isRevertReasonEnabled(isRevertReasonEnabled) - .isParallelTxProcessingEnabled( - dataStorageConfiguration.getUnstable().isParallelTxProcessingEnabled()) - .storageProvider(storageProvider) - .gasLimitCalculator( - miningParametersSupplier.get().getTargetGasLimit().isPresent() - ? new FrontierTargetingGasLimitCalculator() - : GasLimitCalculator.constant()) - .requiredBlocks(requiredBlocks) - .reorgLoggingThreshold(reorgLoggingThreshold) - .evmConfiguration(unstableEvmOptions.toDomainObject()) - .maxPeers(p2PDiscoveryOptions.maxPeers) - .maxRemotelyInitiatedPeers(maxRemoteInitiatedPeers) - .randomPeerPriority(p2PDiscoveryOptions.randomPeerPriority) - .chainPruningConfiguration(unstableChainPruningOptions.toDomainObject()) - .cacheLastBlocks(numberOfblocksToCache) - .genesisStateHashCacheEnabled(genesisStateHashCacheEnabled) - .besuComponent(besuComponent); + BesuControllerBuilder besuControllerBuilder = + controllerBuilder + .fromEthNetworkConfig(updateNetworkConfig(network), getDefaultSyncModeIfNotSet()) + .synchronizerConfiguration(buildSyncConfig()) + .ethProtocolConfiguration(unstableEthProtocolOptions.toDomainObject()) + .networkConfiguration(unstableNetworkingOptions.toDomainObject()) + .dataDirectory(dataDir()) + .dataStorageConfiguration(getDataStorageConfiguration()) + .miningParameters(miningParametersSupplier.get()) + .transactionPoolConfiguration(buildTransactionPoolConfiguration()) + .nodeKey(new NodeKey(securityModule())) + .metricsSystem((ObservableMetricsSystem) besuComponent.getMetricsSystem()) + .messagePermissioningProviders(permissioningService.getMessagePermissioningProviders()) + .privacyParameters(privacyParameters()) + .clock(Clock.systemUTC()) + .isRevertReasonEnabled(isRevertReasonEnabled) + .storageProvider(storageProvider) + .gasLimitCalculator( + miningParametersSupplier.get().getTargetGasLimit().isPresent() + ? new FrontierTargetingGasLimitCalculator() + : GasLimitCalculator.constant()) + .requiredBlocks(requiredBlocks) + .reorgLoggingThreshold(reorgLoggingThreshold) + .evmConfiguration(unstableEvmOptions.toDomainObject()) + .maxPeers(p2PDiscoveryOptions.maxPeers) + .maxRemotelyInitiatedPeers(maxRemoteInitiatedPeers) + .randomPeerPriority(p2PDiscoveryOptions.randomPeerPriority) + .chainPruningConfiguration(unstableChainPruningOptions.toDomainObject()) + .cacheLastBlocks(numberOfblocksToCache) + .genesisStateHashCacheEnabled(genesisStateHashCacheEnabled) + .besuComponent(besuComponent); + if (DataStorageFormat.BONSAI.equals(getDataStorageConfiguration().getDataStorageFormat())) { + final DiffBasedSubStorageConfiguration subStorageConfiguration = + getDataStorageConfiguration().getDiffBasedSubStorageConfiguration(); + if (subStorageConfiguration.getLimitTrieLogsEnabled()) { + besuControllerBuilder.isParallelTxProcessingEnabled( + subStorageConfiguration.getUnstable().isParallelTxProcessingEnabled()); + } + } + return besuControllerBuilder; } private JsonRpcConfiguration createEngineJsonRpcConfiguration() { @@ -2126,29 +2141,34 @@ public DataStorageConfiguration getDataStorageConfiguration() { } if (SyncMode.FULL.equals(getDefaultSyncModeIfNotSet()) - && DataStorageFormat.BONSAI.equals(dataStorageConfiguration.getDataStorageFormat()) - && dataStorageConfiguration.getBonsaiLimitTrieLogsEnabled()) { + && DataStorageFormat.BONSAI.equals(dataStorageConfiguration.getDataStorageFormat())) { + final DiffBasedSubStorageConfiguration diffBasedSubStorageConfiguration = + dataStorageConfiguration.getDiffBasedSubStorageConfiguration(); + if (diffBasedSubStorageConfiguration.getLimitTrieLogsEnabled()) { + if (CommandLineUtils.isOptionSet( + commandLine, DiffBasedSubStorageOptions.LIMIT_TRIE_LOGS_ENABLED)) { + throw new ParameterException( + commandLine, + String.format( + "Cannot enable %s with --sync-mode=%s and --data-storage-format=%s. You must set %s or use a different sync-mode", + DiffBasedSubStorageOptions.LIMIT_TRIE_LOGS_ENABLED, + SyncMode.FULL, + DataStorageFormat.BONSAI, + DiffBasedSubStorageOptions.LIMIT_TRIE_LOGS_ENABLED + "=false")); + } - if (CommandLineUtils.isOptionSet( - commandLine, DataStorageOptions.BONSAI_LIMIT_TRIE_LOGS_ENABLED)) { - throw new ParameterException( - commandLine, - String.format( - "Cannot enable %s with --sync-mode=%s and --data-storage-format=%s. You must set %s or use a different sync-mode", - DataStorageOptions.BONSAI_LIMIT_TRIE_LOGS_ENABLED, - SyncMode.FULL, - DataStorageFormat.BONSAI, - DataStorageOptions.BONSAI_LIMIT_TRIE_LOGS_ENABLED + "=false")); + dataStorageConfiguration = + ImmutableDataStorageConfiguration.copyOf(dataStorageConfiguration) + .withDiffBasedSubStorageConfiguration( + ImmutableDiffBasedSubStorageConfiguration.copyOf( + dataStorageConfiguration.getDiffBasedSubStorageConfiguration()) + .withLimitTrieLogsEnabled(false)); + logger.warn( + "Forcing {}, since it cannot be enabled with --sync-mode={} and --data-storage-format={}.", + DiffBasedSubStorageOptions.LIMIT_TRIE_LOGS_ENABLED + "=false", + SyncMode.FULL, + DataStorageFormat.BONSAI); } - - dataStorageConfiguration = - ImmutableDataStorageConfiguration.copyOf(dataStorageConfiguration) - .withBonsaiLimitTrieLogsEnabled(false); - logger.warn( - "Forcing {}, since it cannot be enabled with --sync-mode={} and --data-storage-format={}.", - DataStorageOptions.BONSAI_LIMIT_TRIE_LOGS_ENABLED + "=false", - SyncMode.FULL, - DataStorageFormat.BONSAI); } return dataStorageConfiguration; } @@ -2715,12 +2735,14 @@ private String generateConfigurationOverview() { builder.setHighSpecEnabled(); } - if (DataStorageFormat.BONSAI.equals(getDataStorageConfiguration().getDataStorageFormat()) - && getDataStorageConfiguration().getBonsaiLimitTrieLogsEnabled()) { - builder.setLimitTrieLogsEnabled(); - builder.setTrieLogRetentionLimit(getDataStorageConfiguration().getBonsaiMaxLayersToLoad()); - builder.setTrieLogsPruningWindowSize( - getDataStorageConfiguration().getBonsaiTrieLogPruningWindowSize()); + if (DataStorageFormat.BONSAI.equals(getDataStorageConfiguration().getDataStorageFormat())) { + final DiffBasedSubStorageConfiguration subStorageConfiguration = + getDataStorageConfiguration().getDiffBasedSubStorageConfiguration(); + if (subStorageConfiguration.getLimitTrieLogsEnabled()) { + builder.setLimitTrieLogsEnabled(); + builder.setTrieLogRetentionLimit(subStorageConfiguration.getMaxLayersToLoad()); + builder.setTrieLogsPruningWindowSize(subStorageConfiguration.getTrieLogPruningWindowSize()); + } } builder.setSnapServerEnabled(this.unstableSynchronizerOptions.isSnapsyncServerEnabled()); diff --git a/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java b/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java index abaa1272706..b3dfa2a054c 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java @@ -53,7 +53,7 @@ public class ConfigurationOverviewBuilder { private Collection engineApis; private String engineJwtFilePath; private boolean isHighSpec = false; - private boolean isBonsaiLimitTrieLogsEnabled = false; + private boolean isLimitTrieLogsEnabled = false; private long trieLogRetentionLimit = 0; private Integer trieLogsPruningWindowSize = null; private boolean isSnapServerEnabled = false; @@ -220,7 +220,7 @@ public ConfigurationOverviewBuilder setHighSpecEnabled() { * @return the builder */ public ConfigurationOverviewBuilder setLimitTrieLogsEnabled() { - isBonsaiLimitTrieLogsEnabled = true; + isLimitTrieLogsEnabled = true; return this; } @@ -389,7 +389,7 @@ public String build() { lines.add("Experimental Snap Sync for BFT enabled"); } - if (isBonsaiLimitTrieLogsEnabled) { + if (isLimitTrieLogsEnabled) { final StringBuilder trieLogPruningString = new StringBuilder(); trieLogPruningString .append("Limit trie logs enabled: retention: ") diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/DataStorageOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/DataStorageOptions.java deleted file mode 100644 index b442f6ee034..00000000000 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/DataStorageOptions.java +++ /dev/null @@ -1,248 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.cli.options; - -import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED; -import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; -import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE; -import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_RECEIPT_COMPACTION_ENABLED; -import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT; -import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED; -import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_FULL_FLAT_DB_ENABLED; - -import org.hyperledger.besu.cli.util.CommandLineUtils; -import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; -import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; -import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; - -import java.util.List; -import java.util.Locale; - -import org.apache.commons.lang3.StringUtils; -import picocli.CommandLine; -import picocli.CommandLine.Option; - -/** The Data storage CLI options. */ -public class DataStorageOptions implements CLIOptions { - - private static final String DATA_STORAGE_FORMAT = "--data-storage-format"; - - /** The maximum number of historical layers to load. */ - public static final String BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD = - "--bonsai-historical-block-limit"; - - // Use Bonsai DB - @Option( - names = {DATA_STORAGE_FORMAT}, - description = - "Format to store trie data in. Either FOREST or BONSAI (default: ${DEFAULT-VALUE}).", - arity = "1") - private DataStorageFormat dataStorageFormat = DataStorageFormat.BONSAI; - - @Option( - names = {BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD, "--bonsai-maximum-back-layers-to-load"}, - paramLabel = "", - description = - "Limit of historical layers that can be loaded with BONSAI (default: ${DEFAULT-VALUE}). When using " - + BONSAI_LIMIT_TRIE_LOGS_ENABLED - + " it will also be used as the number of layers of trie logs to retain.", - arity = "1") - private Long bonsaiMaxLayersToLoad = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; - - /** The bonsai limit trie logs enabled option name */ - public static final String BONSAI_LIMIT_TRIE_LOGS_ENABLED = "--bonsai-limit-trie-logs-enabled"; - - /** The bonsai trie logs pruning window size. */ - public static final String BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE = - "--bonsai-trie-logs-pruning-window-size"; - - // TODO --Xbonsai-limit-trie-logs-enabled and --Xbonsai-trie-log-pruning-enabled are deprecated, - // remove in a future release - @SuppressWarnings("ExperimentalCliOptionMustBeCorrectlyDisplayed") - @CommandLine.Option( - names = { - BONSAI_LIMIT_TRIE_LOGS_ENABLED, - "--Xbonsai-limit-trie-logs-enabled", // deprecated - "--Xbonsai-trie-log-pruning-enabled" // deprecated - }, - fallbackValue = "true", - description = "Limit the number of trie logs that are retained. (default: ${DEFAULT-VALUE})") - private Boolean bonsaiLimitTrieLogsEnabled = DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED; - - // TODO --Xbonsai-trie-logs-pruning-window-size is deprecated, remove in a future release - @SuppressWarnings("ExperimentalCliOptionMustBeCorrectlyDisplayed") - @CommandLine.Option( - names = { - BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE, - "--Xbonsai-trie-logs-pruning-window-size" // deprecated - }, - description = - "The max number of blocks to load and prune trie logs for at startup. (default: ${DEFAULT-VALUE})") - private Integer bonsaiTrieLogPruningWindowSize = DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE; - - @Option( - names = "--receipt-compaction-enabled", - description = "Enables compact storing of receipts (default: ${DEFAULT-VALUE})", - fallbackValue = "true") - private Boolean receiptCompactionEnabled = DEFAULT_RECEIPT_COMPACTION_ENABLED; - - @CommandLine.ArgGroup(validate = false) - private final DataStorageOptions.Unstable unstableOptions = new Unstable(); - - /** Default Constructor. */ - DataStorageOptions() {} - - /** The unstable options for data storage. */ - public static class Unstable { - - // TODO: --Xsnapsync-synchronizer-flat-db-healing-enabled is deprecated, remove it in a future - // release - @CommandLine.Option( - hidden = true, - names = { - "--Xbonsai-full-flat-db-enabled", - "--Xsnapsync-synchronizer-flat-db-healing-enabled" - }, - arity = "1", - description = "Enables bonsai full flat database strategy. (default: ${DEFAULT-VALUE})") - private Boolean bonsaiFullFlatDbEnabled = DEFAULT_BONSAI_FULL_FLAT_DB_ENABLED; - - @CommandLine.Option( - hidden = true, - names = {"--Xbonsai-code-using-code-hash-enabled"}, - arity = "1", - description = - "Enables code storage using code hash instead of by account hash. (default: ${DEFAULT-VALUE})") - private boolean bonsaiCodeUsingCodeHashEnabled = DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED; - - @CommandLine.Option( - hidden = true, - names = {"--Xbonsai-parallel-tx-processing-enabled"}, - arity = "1", - description = - "Enables parallelization of transactions to optimize processing speed by concurrently loading and executing necessary data in advance. (default: ${DEFAULT-VALUE})") - private Boolean isParallelTxProcessingEnabled = false; - - /** Default Constructor. */ - Unstable() {} - } - - /** - * Create data storage options. - * - * @return the data storage options - */ - public static DataStorageOptions create() { - return new DataStorageOptions(); - } - - /** - * Validates the data storage options - * - * @param commandLine the full commandLine to check all the options specified by the user - */ - public void validate(final CommandLine commandLine) { - if (DataStorageFormat.BONSAI == dataStorageFormat) { - if (bonsaiLimitTrieLogsEnabled) { - if (bonsaiMaxLayersToLoad < MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT) { - throw new CommandLine.ParameterException( - commandLine, - String.format( - BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD + " minimum value is %d", - MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT)); - } - if (bonsaiTrieLogPruningWindowSize <= 0) { - throw new CommandLine.ParameterException( - commandLine, - String.format( - BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE + "=%d must be greater than 0", - bonsaiTrieLogPruningWindowSize)); - } - if (bonsaiTrieLogPruningWindowSize <= bonsaiMaxLayersToLoad) { - throw new CommandLine.ParameterException( - commandLine, - String.format( - BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE - + "=%d must be greater than " - + BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD - + "=%d", - bonsaiTrieLogPruningWindowSize, - bonsaiMaxLayersToLoad)); - } - } - } else { - if (unstableOptions.isParallelTxProcessingEnabled) { - throw new CommandLine.ParameterException( - commandLine, - "Transaction parallelization is not supported unless operating in a 'diffbased' mode, such as Bonsai."); - } - } - } - - /** - * Converts to options from the configuration - * - * @param domainObject to be reversed - * @return the options that correspond to the configuration - */ - public static DataStorageOptions fromConfig(final DataStorageConfiguration domainObject) { - final DataStorageOptions dataStorageOptions = DataStorageOptions.create(); - dataStorageOptions.dataStorageFormat = domainObject.getDataStorageFormat(); - dataStorageOptions.bonsaiMaxLayersToLoad = domainObject.getBonsaiMaxLayersToLoad(); - dataStorageOptions.receiptCompactionEnabled = domainObject.getReceiptCompactionEnabled(); - dataStorageOptions.bonsaiLimitTrieLogsEnabled = domainObject.getBonsaiLimitTrieLogsEnabled(); - dataStorageOptions.bonsaiTrieLogPruningWindowSize = - domainObject.getBonsaiTrieLogPruningWindowSize(); - dataStorageOptions.unstableOptions.bonsaiFullFlatDbEnabled = - domainObject.getUnstable().getBonsaiFullFlatDbEnabled(); - dataStorageOptions.unstableOptions.bonsaiCodeUsingCodeHashEnabled = - domainObject.getUnstable().getBonsaiCodeStoredByCodeHashEnabled(); - dataStorageOptions.unstableOptions.isParallelTxProcessingEnabled = - domainObject.getUnstable().isParallelTxProcessingEnabled(); - - return dataStorageOptions; - } - - @Override - public DataStorageConfiguration toDomainObject() { - return ImmutableDataStorageConfiguration.builder() - .dataStorageFormat(dataStorageFormat) - .bonsaiMaxLayersToLoad(bonsaiMaxLayersToLoad) - .receiptCompactionEnabled(receiptCompactionEnabled) - .bonsaiLimitTrieLogsEnabled(bonsaiLimitTrieLogsEnabled) - .bonsaiTrieLogPruningWindowSize(bonsaiTrieLogPruningWindowSize) - .unstable( - ImmutableDataStorageConfiguration.Unstable.builder() - .bonsaiFullFlatDbEnabled(unstableOptions.bonsaiFullFlatDbEnabled) - .bonsaiCodeStoredByCodeHashEnabled(unstableOptions.bonsaiCodeUsingCodeHashEnabled) - .isParallelTxProcessingEnabled(unstableOptions.isParallelTxProcessingEnabled) - .build()) - .build(); - } - - @Override - public List getCLIOptions() { - return CommandLineUtils.getCLIOptions(this, new DataStorageOptions()); - } - - /** - * Normalize data storage format string. - * - * @return the normalized string - */ - public String normalizeDataStorageFormat() { - return StringUtils.capitalize(dataStorageFormat.toString().toLowerCase(Locale.ROOT)); - } -} diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/storage/DataStorageOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/storage/DataStorageOptions.java new file mode 100644 index 00000000000..fbc57f50def --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/storage/DataStorageOptions.java @@ -0,0 +1,121 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.options.storage; + +import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_RECEIPT_COMPACTION_ENABLED; + +import org.hyperledger.besu.cli.options.CLIOptions; +import org.hyperledger.besu.cli.util.CommandLineUtils; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; +import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; + +import java.util.List; +import java.util.Locale; + +import org.apache.commons.lang3.StringUtils; +import picocli.CommandLine; +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Option; + +/** The Data storage CLI options. */ +public class DataStorageOptions implements CLIOptions { + + private static final String DATA_STORAGE_FORMAT = "--data-storage-format"; + + // Use Bonsai DB + @Option( + names = {DATA_STORAGE_FORMAT}, + description = + "Format to store trie data in. Either FOREST or BONSAI (default: ${DEFAULT-VALUE}).", + arity = "1") + private DataStorageFormat dataStorageFormat = DataStorageFormat.BONSAI; + + @Option( + names = "--receipt-compaction-enabled", + description = "Enables compact storing of receipts (default: ${DEFAULT-VALUE})", + fallbackValue = "true") + private Boolean receiptCompactionEnabled = DEFAULT_RECEIPT_COMPACTION_ENABLED; + + /** + * Options specific to diff-based storage modes. Holds the necessary parameters to configure + * diff-based storage, such as the Bonsai mode or Verkle in the future. + */ + @Mixin + private DiffBasedSubStorageOptions diffBasedSubStorageOptions = + DiffBasedSubStorageOptions.create(); + + /** Default Constructor. */ + DataStorageOptions() {} + + /** + * Create data storage options. + * + * @return the data storage options + */ + public static DataStorageOptions create() { + return new DataStorageOptions(); + } + + /** + * Validates the data storage options + * + * @param commandLine the full commandLine to check all the options specified by the user + */ + public void validate(final CommandLine commandLine) { + diffBasedSubStorageOptions.validate(commandLine, dataStorageFormat); + } + + /** + * Converts to options from the configuration + * + * @param domainObject to be reversed + * @return the options that correspond to the configuration + */ + public static DataStorageOptions fromConfig(final DataStorageConfiguration domainObject) { + final DataStorageOptions dataStorageOptions = DataStorageOptions.create(); + dataStorageOptions.dataStorageFormat = domainObject.getDataStorageFormat(); + dataStorageOptions.receiptCompactionEnabled = domainObject.getReceiptCompactionEnabled(); + dataStorageOptions.diffBasedSubStorageOptions = + DiffBasedSubStorageOptions.fromConfig(domainObject.getDiffBasedSubStorageConfiguration()); + return dataStorageOptions; + } + + @Override + public DataStorageConfiguration toDomainObject() { + final ImmutableDataStorageConfiguration.Builder builder = + ImmutableDataStorageConfiguration.builder() + .dataStorageFormat(dataStorageFormat) + .receiptCompactionEnabled(receiptCompactionEnabled) + .diffBasedSubStorageConfiguration(diffBasedSubStorageOptions.toDomainObject()); + return builder.build(); + } + + @Override + public List getCLIOptions() { + final List cliOptions = CommandLineUtils.getCLIOptions(this, new DataStorageOptions()); + cliOptions.addAll(diffBasedSubStorageOptions.getCLIOptions()); + return cliOptions; + } + + /** + * Normalize data storage format string. + * + * @return the normalized string + */ + public String normalizeDataStorageFormat() { + return StringUtils.capitalize(dataStorageFormat.toString().toLowerCase(Locale.ROOT)); + } +} diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/storage/DiffBasedSubStorageOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/storage/DiffBasedSubStorageOptions.java new file mode 100644 index 00000000000..c95a7668272 --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/storage/DiffBasedSubStorageOptions.java @@ -0,0 +1,217 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.options.storage; + +import static org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration.DEFAULT_LIMIT_TRIE_LOGS_ENABLED; +import static org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration.DEFAULT_MAX_LAYERS_TO_LOAD; +import static org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration.DEFAULT_TRIE_LOG_PRUNING_WINDOW_SIZE; +import static org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration.DiffBasedUnstable.DEFAULT_CODE_USING_CODE_HASH_ENABLED; +import static org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration.DiffBasedUnstable.DEFAULT_FULL_FLAT_DB_ENABLED; +import static org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration.MINIMUM_TRIE_LOG_RETENTION_LIMIT; + +import org.hyperledger.besu.cli.options.CLIOptions; +import org.hyperledger.besu.cli.util.CommandLineUtils; +import org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration; +import org.hyperledger.besu.ethereum.worldstate.ImmutableDiffBasedSubStorageConfiguration; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; + +import java.util.List; + +import picocli.CommandLine; +import picocli.CommandLine.Option; + +/** The Data storage CLI options. */ +public class DiffBasedSubStorageOptions implements CLIOptions { + + /** The maximum number of historical layers to load. */ + public static final String MAX_LAYERS_TO_LOAD = "--bonsai-historical-block-limit"; + + @Option( + names = {MAX_LAYERS_TO_LOAD, "--bonsai-maximum-back-layers-to-load"}, + paramLabel = "", + description = + "Limit of historical layers that can be loaded with BONSAI (default: ${DEFAULT-VALUE}). When using " + + LIMIT_TRIE_LOGS_ENABLED + + " it will also be used as the number of layers of trie logs to retain.", + arity = "1") + private Long maxLayersToLoad = DEFAULT_MAX_LAYERS_TO_LOAD; + + /** The bonsai limit trie logs enabled option name */ + public static final String LIMIT_TRIE_LOGS_ENABLED = "--bonsai-limit-trie-logs-enabled"; + + /** The bonsai trie logs pruning window size. */ + public static final String TRIE_LOG_PRUNING_WINDOW_SIZE = + "--bonsai-trie-logs-pruning-window-size"; + + // TODO --Xbonsai-limit-trie-logs-enabled and --Xbonsai-trie-log-pruning-enabled are deprecated, + // remove in a future release + @SuppressWarnings("ExperimentalCliOptionMustBeCorrectlyDisplayed") + @Option( + names = { + LIMIT_TRIE_LOGS_ENABLED, + "--Xbonsai-limit-trie-logs-enabled", // deprecated + "--Xbonsai-trie-log-pruning-enabled" // deprecated + }, + fallbackValue = "true", + description = "Limit the number of trie logs that are retained. (default: ${DEFAULT-VALUE})") + private Boolean limitTrieLogsEnabled = DEFAULT_LIMIT_TRIE_LOGS_ENABLED; + + // TODO --Xbonsai-trie-logs-pruning-window-size is deprecated, remove in a future release + @SuppressWarnings("ExperimentalCliOptionMustBeCorrectlyDisplayed") + @Option( + names = { + TRIE_LOG_PRUNING_WINDOW_SIZE, + "--Xbonsai-trie-logs-pruning-window-size" // deprecated + }, + description = + "The max number of blocks to load and prune trie logs for at startup. (default: ${DEFAULT-VALUE})") + private Integer trieLogPruningWindowSize = DEFAULT_TRIE_LOG_PRUNING_WINDOW_SIZE; + + @CommandLine.ArgGroup(validate = false) + private final DiffBasedSubStorageOptions.Unstable unstableOptions = new Unstable(); + + /** Default Constructor. */ + DiffBasedSubStorageOptions() {} + + /** The unstable options for data storage. */ + public static class Unstable { + + // TODO: --Xsnapsync-synchronizer-flat-db-healing-enabled is deprecated, remove it in a future + // release + @Option( + hidden = true, + names = { + "--Xbonsai-full-flat-db-enabled", + "--Xsnapsync-synchronizer-flat-db-healing-enabled" + }, + arity = "1", + description = "Enables bonsai full flat database strategy. (default: ${DEFAULT-VALUE})") + private Boolean fullFlatDbEnabled = DEFAULT_FULL_FLAT_DB_ENABLED; + + @Option( + hidden = true, + names = {"--Xbonsai-code-using-code-hash-enabled"}, + arity = "1", + description = + "Enables code storage using code hash instead of by account hash. (default: ${DEFAULT-VALUE})") + private boolean codeUsingCodeHashEnabled = DEFAULT_CODE_USING_CODE_HASH_ENABLED; + + @Option( + hidden = true, + names = {"--Xbonsai-parallel-tx-processing-enabled"}, + arity = "1", + description = + "Enables parallelization of transactions to optimize processing speed by concurrently loading and executing necessary data in advance. (default: ${DEFAULT-VALUE})") + private Boolean isParallelTxProcessingEnabled = false; + + /** Default Constructor. */ + Unstable() {} + } + + /** + * Create data storage options. + * + * @return the data storage options + */ + public static DiffBasedSubStorageOptions create() { + return new DiffBasedSubStorageOptions(); + } + + /** + * Validates the data storage options + * + * @param commandLine the full commandLine to check all the options specified by the user + * @param dataStorageFormat the selected data storage format which determines the validation rules + * to apply. + */ + public void validate(final CommandLine commandLine, final DataStorageFormat dataStorageFormat) { + if (DataStorageFormat.BONSAI == dataStorageFormat) { + if (limitTrieLogsEnabled) { + if (maxLayersToLoad < MINIMUM_TRIE_LOG_RETENTION_LIMIT) { + throw new CommandLine.ParameterException( + commandLine, + String.format( + MAX_LAYERS_TO_LOAD + " minimum value is %d", MINIMUM_TRIE_LOG_RETENTION_LIMIT)); + } + if (trieLogPruningWindowSize <= 0) { + throw new CommandLine.ParameterException( + commandLine, + String.format( + TRIE_LOG_PRUNING_WINDOW_SIZE + "=%d must be greater than 0", + trieLogPruningWindowSize)); + } + if (trieLogPruningWindowSize <= maxLayersToLoad) { + throw new CommandLine.ParameterException( + commandLine, + String.format( + TRIE_LOG_PRUNING_WINDOW_SIZE + + "=%d must be greater than " + + MAX_LAYERS_TO_LOAD + + "=%d", + trieLogPruningWindowSize, + maxLayersToLoad)); + } + } + } else { + if (unstableOptions.isParallelTxProcessingEnabled) { + throw new CommandLine.ParameterException( + commandLine, + "Transaction parallelization is not supported unless operating in a 'diffbased' mode, such as Bonsai."); + } + } + } + + /** + * Converts to options from the configuration + * + * @param domainObject to be reversed + * @return the options that correspond to the configuration + */ + public static DiffBasedSubStorageOptions fromConfig( + final DiffBasedSubStorageConfiguration domainObject) { + final DiffBasedSubStorageOptions dataStorageOptions = DiffBasedSubStorageOptions.create(); + dataStorageOptions.maxLayersToLoad = domainObject.getMaxLayersToLoad(); + dataStorageOptions.limitTrieLogsEnabled = domainObject.getLimitTrieLogsEnabled(); + dataStorageOptions.trieLogPruningWindowSize = domainObject.getTrieLogPruningWindowSize(); + dataStorageOptions.unstableOptions.fullFlatDbEnabled = + domainObject.getUnstable().getFullFlatDbEnabled(); + dataStorageOptions.unstableOptions.codeUsingCodeHashEnabled = + domainObject.getUnstable().getCodeStoredByCodeHashEnabled(); + dataStorageOptions.unstableOptions.isParallelTxProcessingEnabled = + domainObject.getUnstable().isParallelTxProcessingEnabled(); + + return dataStorageOptions; + } + + @Override + public final DiffBasedSubStorageConfiguration toDomainObject() { + return ImmutableDiffBasedSubStorageConfiguration.builder() + .maxLayersToLoad(maxLayersToLoad) + .limitTrieLogsEnabled(limitTrieLogsEnabled) + .trieLogPruningWindowSize(trieLogPruningWindowSize) + .unstable( + ImmutableDiffBasedSubStorageConfiguration.DiffBasedUnstable.builder() + .fullFlatDbEnabled(unstableOptions.fullFlatDbEnabled) + .codeStoredByCodeHashEnabled(unstableOptions.codeUsingCodeHashEnabled) + .isParallelTxProcessingEnabled(unstableOptions.isParallelTxProcessingEnabled) + .build()) + .build(); + } + + @Override + public List getCLIOptions() { + return CommandLineUtils.getCLIOptions(this, new DiffBasedSubStorageOptions()); + } +} diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java index 1ac309c430f..b96fdaca102 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -15,21 +15,22 @@ package org.hyperledger.besu.cli.subcommands.storage; import static com.google.common.base.Preconditions.checkArgument; -import static org.hyperledger.besu.cli.options.DataStorageOptions.BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD; +import static org.hyperledger.besu.cli.options.storage.DiffBasedSubStorageOptions.MAX_LAYERS_TO_LOAD; +import static org.hyperledger.besu.cli.options.storage.DiffBasedSubStorageOptions.TRIE_LOG_PRUNING_WINDOW_SIZE; import static org.hyperledger.besu.controller.BesuController.DATABASE_PATH; -import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE; +import static org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration.DEFAULT_TRIE_LOG_PRUNING_WINDOW_SIZE; -import org.hyperledger.besu.cli.options.DataStorageOptions; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.rlp.RLP; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; +import org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration; import java.io.File; import java.io.FileInputStream; @@ -64,7 +65,7 @@ public TrieLogHelper() {} boolean prune( final DataStorageConfiguration config, - final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage, final MutableBlockchain blockchain, final Path dataDirectoryPath) { @@ -73,7 +74,7 @@ boolean prune( validatePruneConfiguration(config); - final long layersToRetain = config.getBonsaiMaxLayersToLoad(); + final long layersToRetain = config.getDiffBasedSubStorageConfiguration().getMaxLayersToLoad(); final long chainHeight = blockchain.getChainHeadBlockNumber(); @@ -102,7 +103,7 @@ boolean prune( // Should only be layersToRetain left but loading extra just in case of an unforeseen bug final long countAfterPrune = rootWorldStateStorage - .streamTrieLogKeys(layersToRetain + DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE) + .streamTrieLogKeys(layersToRetain + DEFAULT_TRIE_LOG_PRUNING_WINDOW_SIZE) .count(); if (countAfterPrune == layersToRetain) { if (deleteFiles(batchFileNameBase, numberOfBatches)) { @@ -115,15 +116,12 @@ boolean prune( throw new IllegalStateException( String.format( "Remaining trie logs (%d) did not match %s (%d). Trie logs backup files (in %s) have not been deleted, it is safe to rerun the subcommand.", - countAfterPrune, - BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD, - layersToRetain, - batchFileNameBase)); + countAfterPrune, MAX_LAYERS_TO_LOAD, layersToRetain, batchFileNameBase)); } } private void processTrieLogBatches( - final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage, final MutableBlockchain blockchain, final long chainHeight, final long lastBlockNumberToRetainTrieLogsFor, @@ -152,7 +150,7 @@ private void processTrieLogBatches( private void saveTrieLogBatches( final String batchFileName, - final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage, final List trieLogKeys) { try { @@ -164,7 +162,7 @@ private void saveTrieLogBatches( } private void restoreTrieLogBatches( - final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage, final long batchNumber, final String batchFileNameBase) { @@ -217,7 +215,7 @@ private boolean validatePruneRequirements( final MutableBlockchain blockchain, final long chainHeight, final long lastBlockNumberToRetainTrieLogsFor, - final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage, final long layersToRetain) { if (lastBlockNumberToRetainTrieLogsFor < 0) { @@ -231,7 +229,7 @@ private boolean validatePruneRequirements( // plus extra threshold to account forks and orphans final long clampedCountBeforePruning = rootWorldStateStorage - .streamTrieLogKeys(layersToRetain + DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE) + .streamTrieLogKeys(layersToRetain + DEFAULT_TRIE_LOG_PRUNING_WINDOW_SIZE) .count(); if (clampedCountBeforePruning < layersToRetain) { throw new IllegalArgumentException( @@ -257,7 +255,7 @@ private boolean validatePruneRequirements( } private void recreateTrieLogs( - final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage, final long batchNumber, final String batchFileNameBase) throws IOException { @@ -277,7 +275,7 @@ private void processTransactionChunk( final int chunkSize, final List keys, final IdentityHashMap trieLogsToRetain, - final BonsaiWorldStateKeyValueStorage rootWorldStateStorage) { + final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage) { var updater = rootWorldStateStorage.updater(); int endIndex = Math.min(startIndex + chunkSize, keys.size()); @@ -294,31 +292,31 @@ private void processTransactionChunk( @VisibleForTesting void validatePruneConfiguration(final DataStorageConfiguration config) { + final DiffBasedSubStorageConfiguration subStorageConfiguration = + config.getDiffBasedSubStorageConfiguration(); checkArgument( - config.getBonsaiMaxLayersToLoad() - >= DataStorageConfiguration.MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT, + subStorageConfiguration.getMaxLayersToLoad() + >= DiffBasedSubStorageConfiguration.MINIMUM_TRIE_LOG_RETENTION_LIMIT, String.format( - BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD + " minimum value is %d", - DataStorageConfiguration.MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT)); + MAX_LAYERS_TO_LOAD + " minimum value is %d", + DiffBasedSubStorageConfiguration.MINIMUM_TRIE_LOG_RETENTION_LIMIT)); checkArgument( - config.getBonsaiTrieLogPruningWindowSize() > 0, + subStorageConfiguration.getTrieLogPruningWindowSize() > 0, String.format( - DataStorageOptions.BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE + "=%d must be greater than 0", - config.getBonsaiTrieLogPruningWindowSize())); + TRIE_LOG_PRUNING_WINDOW_SIZE + "=%d must be greater than 0", + subStorageConfiguration.getTrieLogPruningWindowSize())); checkArgument( - config.getBonsaiTrieLogPruningWindowSize() > config.getBonsaiMaxLayersToLoad(), + subStorageConfiguration.getTrieLogPruningWindowSize() + > subStorageConfiguration.getMaxLayersToLoad(), String.format( - DataStorageOptions.BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE - + "=%d must be greater than " - + BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD - + "=%d", - config.getBonsaiTrieLogPruningWindowSize(), - config.getBonsaiMaxLayersToLoad())); + TRIE_LOG_PRUNING_WINDOW_SIZE + "=%d must be greater than " + MAX_LAYERS_TO_LOAD + "=%d", + subStorageConfiguration.getTrieLogPruningWindowSize(), + subStorageConfiguration.getMaxLayersToLoad())); } private void saveTrieLogsInFile( final List trieLogsKeys, - final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage, final String batchFileName) throws IOException { @@ -355,7 +353,7 @@ IdentityHashMap readTrieLogsFromFile(final String batchFileName) private void saveTrieLogsAsRlpInFile( final List trieLogsKeys, - final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage, final String batchFileName) { File file = new File(batchFileName); if (file.exists()) { @@ -400,7 +398,8 @@ IdentityHashMap readTrieLogsAsRlpFromFile(final String batchFile } private IdentityHashMap getTrieLogs( - final List trieLogKeys, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage) { + final List trieLogKeys, + final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage) { IdentityHashMap trieLogsToRetain = new IdentityHashMap<>(); LOG.info("Obtaining trielogs from db, this may take a few minutes..."); @@ -413,7 +412,7 @@ private IdentityHashMap getTrieLogs( } TrieLogCount getCount( - final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage, final int limit, final Blockchain blockchain) { final AtomicInteger total = new AtomicInteger(); @@ -454,7 +453,7 @@ void printCount(final PrintWriter out, final TrieLogCount count) { } void importTrieLog( - final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final Path trieLogFilePath) { + final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage, final Path trieLogFilePath) { var trieLog = readTrieLogsAsRlpFromFile(trieLogFilePath.toString()); @@ -464,7 +463,7 @@ void importTrieLog( } void exportTrieLog( - final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage, final List trieLogHash, final Path directoryPath) throws IOException { diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java index 6fe6d058b3f..24636a01dee 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java @@ -29,6 +29,7 @@ import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; +import org.hyperledger.besu.ethereum.worldstate.ImmutableDiffBasedSubStorageConfiguration; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.io.IOException; @@ -89,7 +90,11 @@ private static BesuController createBesuController() { .besuCommand .setupControllerBuilder() .dataStorageConfiguration( - ImmutableDataStorageConfiguration.copyOf(config).withBonsaiLimitTrieLogsEnabled(false)) + ImmutableDataStorageConfiguration.copyOf(config) + .withDiffBasedSubStorageConfiguration( + ImmutableDiffBasedSubStorageConfiguration.copyOf( + config.getDiffBasedSubStorageConfiguration()) + .withLimitTrieLogsEnabled(false))) .build(); } diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index 91ef6f7cfa5..a26bcb9811a 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -89,6 +89,7 @@ import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.trie.forest.ForestWorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; +import org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; @@ -741,15 +742,18 @@ public BesuController build() { final JsonRpcMethods additionalJsonRpcMethodFactory = createAdditionalJsonRpcMethodFactory(protocolContext, protocolSchedule, miningParameters); - if (dataStorageConfiguration.getBonsaiLimitTrieLogsEnabled() - && DataStorageFormat.BONSAI.equals(dataStorageConfiguration.getDataStorageFormat())) { - final TrieLogManager trieLogManager = - ((BonsaiWorldStateProvider) worldStateArchive).getTrieLogManager(); - final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage = - worldStateStorageCoordinator.getStrategy(BonsaiWorldStateKeyValueStorage.class); - final TrieLogPruner trieLogPruner = - createTrieLogPruner(worldStateKeyValueStorage, blockchain, scheduler); - trieLogManager.subscribe(trieLogPruner); + if (DataStorageFormat.BONSAI.equals(dataStorageConfiguration.getDataStorageFormat())) { + final DiffBasedSubStorageConfiguration subStorageConfiguration = + dataStorageConfiguration.getDiffBasedSubStorageConfiguration(); + if (subStorageConfiguration.getLimitTrieLogsEnabled()) { + final TrieLogManager trieLogManager = + ((BonsaiWorldStateProvider) worldStateArchive).getTrieLogManager(); + final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage = + worldStateStorageCoordinator.getStrategy(BonsaiWorldStateKeyValueStorage.class); + final TrieLogPruner trieLogPruner = + createTrieLogPruner(worldStateKeyValueStorage, blockchain, scheduler); + trieLogManager.subscribe(trieLogPruner); + } } final List closeables = new ArrayList<>(); @@ -803,14 +807,15 @@ private TrieLogPruner createTrieLogPruner( final Blockchain blockchain, final EthScheduler scheduler) { final boolean isProofOfStake = genesisConfigOptions.getTerminalTotalDifficulty().isPresent(); - + final DiffBasedSubStorageConfiguration subStorageConfiguration = + dataStorageConfiguration.getDiffBasedSubStorageConfiguration(); final TrieLogPruner trieLogPruner = new TrieLogPruner( (BonsaiWorldStateKeyValueStorage) worldStateStorage, blockchain, scheduler::executeServiceTask, - dataStorageConfiguration.getBonsaiMaxLayersToLoad(), - dataStorageConfiguration.getBonsaiTrieLogPruningWindowSize(), + subStorageConfiguration.getMaxLayersToLoad(), + subStorageConfiguration.getTrieLogPruningWindowSize(), isProofOfStake, metricsSystem); trieLogPruner.initialize(); @@ -1092,10 +1097,14 @@ WorldStateArchive createWorldStateArchive( case BONSAI -> { final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage = worldStateStorageCoordinator.getStrategy(BonsaiWorldStateKeyValueStorage.class); + yield new BonsaiWorldStateProvider( worldStateKeyValueStorage, blockchain, - Optional.of(dataStorageConfiguration.getBonsaiMaxLayersToLoad()), + Optional.of( + dataStorageConfiguration + .getDiffBasedSubStorageConfiguration() + .getMaxLayersToLoad()), bonsaiCachedMerkleTrieLoader, besuComponent.map(BesuComponent::getBesuPluginContext).orElse(null), evmConfiguration); diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index 399b82be124..019e8d83cd3 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -1390,7 +1390,7 @@ public void ethStatsContactOptionCannotBeUsedWithoutEthStatsServerProvided() { } @Test - public void bonsaiLimitTrieLogsEnabledByDefault() { + public void diffbasedLimitTrieLogsEnabledByDefault() { parseCommand(); verify(mockControllerBuilder) .dataStorageConfiguration(dataStorageConfigurationArgumentCaptor.capture()); @@ -1398,7 +1398,11 @@ public void bonsaiLimitTrieLogsEnabledByDefault() { final DataStorageConfiguration dataStorageConfiguration = dataStorageConfigurationArgumentCaptor.getValue(); assertThat(dataStorageConfiguration.getDataStorageFormat()).isEqualTo(BONSAI); - assertThat(dataStorageConfiguration.getBonsaiLimitTrieLogsEnabled()).isTrue(); + assertThat( + dataStorageConfiguration + .getDiffBasedSubStorageConfiguration() + .getLimitTrieLogsEnabled()) + .isTrue(); assertThat(commandOutput.toString(UTF_8)).isEmpty(); assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); } @@ -1413,7 +1417,11 @@ public void bonsaiLimitTrieLogsDisabledWhenFullSyncEnabled() { final DataStorageConfiguration dataStorageConfiguration = dataStorageConfigurationArgumentCaptor.getValue(); assertThat(dataStorageConfiguration.getDataStorageFormat()).isEqualTo(BONSAI); - assertThat(dataStorageConfiguration.getBonsaiLimitTrieLogsEnabled()).isFalse(); + assertThat( + dataStorageConfiguration + .getDiffBasedSubStorageConfiguration() + .getLimitTrieLogsEnabled()) + .isFalse(); verify(mockLogger) .warn( "Forcing {}, since it cannot be enabled with --sync-mode={} and --data-storage-format={}.", @@ -1448,7 +1456,8 @@ public void parsesValidBonsaiHistoricalBlockLimitOption() { final DataStorageConfiguration dataStorageConfiguration = dataStorageConfigurationArgumentCaptor.getValue(); assertThat(dataStorageConfiguration.getDataStorageFormat()).isEqualTo(BONSAI); - assertThat(dataStorageConfiguration.getBonsaiMaxLayersToLoad()).isEqualTo(11); + assertThat(dataStorageConfiguration.getDiffBasedSubStorageConfiguration().getMaxLayersToLoad()) + .isEqualTo(11); assertThat(commandOutput.toString(UTF_8)).isEmpty(); assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); } @@ -2551,8 +2560,9 @@ public void bonsaiFlatDbShouldBeEnabledByDefault() { besuCommand .getDataStorageOptions() .toDomainObject() + .getDiffBasedSubStorageConfiguration() .getUnstable() - .getBonsaiFullFlatDbEnabled()) + .getFullFlatDbEnabled()) .isTrue(); } @@ -2563,8 +2573,9 @@ public void snapsyncHealingOptionShouldWork() { besuCommand .dataStorageOptions .toDomainObject() + .getDiffBasedSubStorageConfiguration() .getUnstable() - .getBonsaiFullFlatDbEnabled()) + .getFullFlatDbEnabled()) .isFalse(); } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java index 8508edc54e6..5c7bf3ce88a 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java @@ -34,10 +34,10 @@ import org.hyperledger.besu.chainimport.JsonBlockImporter; import org.hyperledger.besu.chainimport.RlpBlockImporter; import org.hyperledger.besu.cli.config.EthNetworkConfig; -import org.hyperledger.besu.cli.options.DataStorageOptions; import org.hyperledger.besu.cli.options.MiningOptions; import org.hyperledger.besu.cli.options.TransactionPoolOptions; import org.hyperledger.besu.cli.options.stable.EthstatsOptions; +import org.hyperledger.besu.cli.options.storage.DataStorageOptions; import org.hyperledger.besu.cli.options.unstable.EthProtocolOptions; import org.hyperledger.besu.cli.options.unstable.NetworkingOptions; import org.hyperledger.besu.cli.options.unstable.SynchronizerOptions; diff --git a/besu/src/test/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilderTest.java index be16c4387c6..c2717afc0a3 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilderTest.java @@ -162,7 +162,7 @@ void setHighSpecEnabled() { } @Test - void setBonsaiLimitTrieLogsEnabled() { + void setDiffbasedLimitTrieLogsEnabled() { final String noTrieLogRetentionLimitSet = builder.build(); assertThat(noTrieLogRetentionLimitSet).doesNotContain("Limit trie logs enabled"); diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/AbstractCLIOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/AbstractCLIOptionsTest.java index 1760b957ec6..b88491451ff 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/AbstractCLIOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/AbstractCLIOptionsTest.java @@ -64,7 +64,6 @@ public void getCLIOptions_custom() { private void getCLIOptions(final D domainObject) { T options = optionsFromDomainObject(domainObject); final String[] cliOptions = options.getCLIOptions().toArray(new String[0]); - final TestBesuCommand cmd = parseCommand(cliOptions); final T optionsFromCommand = getOptionsFromBesuCommand(cmd); diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java index 6be6429adb9..b9dc4fa9270 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java @@ -15,12 +15,13 @@ package org.hyperledger.besu.cli.options.stable; import static org.assertj.core.api.Assertions.assertThat; -import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT; +import static org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration.MINIMUM_TRIE_LOG_RETENTION_LIMIT; import org.hyperledger.besu.cli.options.AbstractCLIOptionsTest; -import org.hyperledger.besu.cli.options.DataStorageOptions; +import org.hyperledger.besu.cli.options.storage.DataStorageOptions; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; +import org.hyperledger.besu.ethereum.worldstate.ImmutableDiffBasedSubStorageConfiguration; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.junit.jupiter.api.Test; @@ -32,7 +33,11 @@ public class DataStorageOptionsTest public void bonsaiTrieLogPruningLimitOption() { internalTestSuccess( dataStorageConfiguration -> - assertThat(dataStorageConfiguration.getBonsaiTrieLogPruningWindowSize()).isEqualTo(600), + assertThat( + dataStorageConfiguration + .getDiffBasedSubStorageConfiguration() + .getTrieLogPruningWindowSize()) + .isEqualTo(600), "--bonsai-limit-trie-logs-enabled", "--bonsai-trie-logs-pruning-window-size", "600"); @@ -42,7 +47,11 @@ public void bonsaiTrieLogPruningLimitOption() { public void bonsaiTrieLogPruningLimitLegacyOption() { internalTestSuccess( dataStorageConfiguration -> - assertThat(dataStorageConfiguration.getBonsaiTrieLogPruningWindowSize()).isEqualTo(600), + assertThat( + dataStorageConfiguration + .getDiffBasedSubStorageConfiguration() + .getTrieLogPruningWindowSize()) + .isEqualTo(600), "--Xbonsai-limit-trie-logs-enabled", "--Xbonsai-trie-logs-pruning-window-size", "600"); @@ -52,12 +61,16 @@ public void bonsaiTrieLogPruningLimitLegacyOption() { public void bonsaiTrieLogsEnabled_explicitlySetToFalse() { internalTestSuccess( dataStorageConfiguration -> - assertThat(dataStorageConfiguration.getBonsaiLimitTrieLogsEnabled()).isEqualTo(false), + assertThat( + dataStorageConfiguration + .getDiffBasedSubStorageConfiguration() + .getLimitTrieLogsEnabled()) + .isEqualTo(false), "--bonsai-limit-trie-logs-enabled=false"); } @Test - public void bonsaiTrieLogPruningWindowSizeShouldBePositive() { + public void diffbasedTrieLogPruningWindowSizeShouldBePositive() { internalTestFailure( "--bonsai-trie-logs-pruning-window-size=0 must be greater than 0", "--bonsai-limit-trie-logs-enabled", @@ -66,7 +79,7 @@ public void bonsaiTrieLogPruningWindowSizeShouldBePositive() { } @Test - public void bonsaiTrieLogPruningWindowSizeShouldBeAboveRetentionLimit() { + public void diffbasedTrieLogPruningWindowSizeShouldBeAboveRetentionLimit() { internalTestFailure( "--bonsai-trie-logs-pruning-window-size=512 must be greater than --bonsai-historical-block-limit=512", "--bonsai-limit-trie-logs-enabled", @@ -78,8 +91,11 @@ public void bonsaiTrieLogPruningWindowSizeShouldBeAboveRetentionLimit() { public void bonsaiTrieLogRetentionLimitOption() { internalTestSuccess( dataStorageConfiguration -> - assertThat(dataStorageConfiguration.getBonsaiMaxLayersToLoad()) - .isEqualTo(MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT + 1), + assertThat( + dataStorageConfiguration + .getDiffBasedSubStorageConfiguration() + .getMaxLayersToLoad()) + .isEqualTo(MINIMUM_TRIE_LOG_RETENTION_LIMIT + 1), "--bonsai-limit-trie-logs-enabled", "--bonsai-historical-block-limit", "513"); @@ -89,8 +105,11 @@ public void bonsaiTrieLogRetentionLimitOption() { public void bonsaiTrieLogRetentionLimitOption_boundaryTest() { internalTestSuccess( dataStorageConfiguration -> - assertThat(dataStorageConfiguration.getBonsaiMaxLayersToLoad()) - .isEqualTo(MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT), + assertThat( + dataStorageConfiguration + .getDiffBasedSubStorageConfiguration() + .getMaxLayersToLoad()) + .isEqualTo(MINIMUM_TRIE_LOG_RETENTION_LIMIT), "--bonsai-limit-trie-logs-enabled", "--bonsai-historical-block-limit", "512"); @@ -106,22 +125,28 @@ public void bonsaiTrieLogRetentionLimitShouldBeAboveMinimum() { } @Test - public void bonsaiCodeUsingCodeHashEnabledCanBeEnabled() { + public void diffbasedCodeUsingCodeHashEnabledCanBeEnabled() { internalTestSuccess( dataStorageConfiguration -> assertThat( - dataStorageConfiguration.getUnstable().getBonsaiCodeStoredByCodeHashEnabled()) + dataStorageConfiguration + .getDiffBasedSubStorageConfiguration() + .getUnstable() + .getCodeStoredByCodeHashEnabled()) .isEqualTo(true), "--Xbonsai-code-using-code-hash-enabled", "true"); } @Test - public void bonsaiCodeUsingCodeHashEnabledCanBeDisabled() { + public void diffbasedCodeUsingCodeHashEnabledCanBeDisabled() { internalTestSuccess( dataStorageConfiguration -> assertThat( - dataStorageConfiguration.getUnstable().getBonsaiCodeStoredByCodeHashEnabled()) + dataStorageConfiguration + .getDiffBasedSubStorageConfiguration() + .getUnstable() + .getCodeStoredByCodeHashEnabled()) .isEqualTo(false), "--Xbonsai-code-using-code-hash-enabled", "false"); @@ -160,9 +185,12 @@ protected DataStorageConfiguration createDefaultDomainObject() { protected DataStorageConfiguration createCustomizedDomainObject() { return ImmutableDataStorageConfiguration.builder() .dataStorageFormat(DataStorageFormat.BONSAI) - .bonsaiMaxLayersToLoad(513L) - .bonsaiLimitTrieLogsEnabled(true) - .bonsaiTrieLogPruningWindowSize(514) + .diffBasedSubStorageConfiguration( + ImmutableDiffBasedSubStorageConfiguration.builder() + .maxLayersToLoad(513L) + .limitTrieLogsEnabled(true) + .trieLogPruningWindowSize(514) + .build()) .build(); } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java index 2f3a693f3de..fef10457ca6 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java @@ -17,7 +17,7 @@ import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE; +import static org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration.DEFAULT_TRIE_LOG_PRUNING_WINDOW_SIZE; import static org.hyperledger.besu.plugin.services.storage.DataStorageFormat.BONSAI; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.spy; @@ -35,6 +35,7 @@ import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; +import org.hyperledger.besu.ethereum.worldstate.ImmutableDiffBasedSubStorageConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import java.io.FileNotFoundException; @@ -134,8 +135,11 @@ public void prune(final @TempDir Path dataDir) throws IOException { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) - .bonsaiMaxLayersToLoad(3L) - .bonsaiLimitTrieLogsEnabled(true) + .diffBasedSubStorageConfiguration( + ImmutableDiffBasedSubStorageConfiguration.builder() + .maxLayersToLoad(3L) + .limitTrieLogsEnabled(true) + .build()) .build(); mockBlockchainBase(); @@ -172,8 +176,11 @@ public void cannotPruneIfNoFinalizedIsFound() { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) - .bonsaiMaxLayersToLoad(2L) - .bonsaiLimitTrieLogsEnabled(true) + .diffBasedSubStorageConfiguration( + ImmutableDiffBasedSubStorageConfiguration.builder() + .maxLayersToLoad(2L) + .limitTrieLogsEnabled(true) + .build()) .build(); when(blockchain.getChainHeadBlockNumber()).thenReturn(5L); @@ -192,8 +199,11 @@ public void cannotPruneIfUserRetainsMoreLayersThanExistingChainLength() { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) - .bonsaiMaxLayersToLoad(10L) - .bonsaiLimitTrieLogsEnabled(true) + .diffBasedSubStorageConfiguration( + ImmutableDiffBasedSubStorageConfiguration.builder() + .maxLayersToLoad(10L) + .limitTrieLogsEnabled(true) + .build()) .build(); when(blockchain.getChainHeadBlockNumber()).thenReturn(5L); @@ -212,8 +222,11 @@ public void cannotPruneIfUserRequiredFurtherThanFinalized(final @TempDir Path da DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) - .bonsaiMaxLayersToLoad(2L) - .bonsaiLimitTrieLogsEnabled(true) + .diffBasedSubStorageConfiguration( + ImmutableDiffBasedSubStorageConfiguration.builder() + .maxLayersToLoad(2L) + .limitTrieLogsEnabled(true) + .build()) .build(); mockBlockchainBase(); @@ -233,8 +246,11 @@ public void skipPruningIfTrieLogCountIsLessThanMaxLayersToLoad() { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) - .bonsaiMaxLayersToLoad(6L) - .bonsaiLimitTrieLogsEnabled(true) + .diffBasedSubStorageConfiguration( + ImmutableDiffBasedSubStorageConfiguration.builder() + .maxLayersToLoad(6L) + .limitTrieLogsEnabled(true) + .build()) .build(); when(blockchain.getChainHeadBlockNumber()).thenReturn(5L); @@ -255,8 +271,11 @@ public void mismatchInPrunedTrieLogCountShouldNotDeleteFiles(final @TempDir Path DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) - .bonsaiMaxLayersToLoad(3L) - .bonsaiLimitTrieLogsEnabled(true) + .diffBasedSubStorageConfiguration( + ImmutableDiffBasedSubStorageConfiguration.builder() + .maxLayersToLoad(3L) + .limitTrieLogsEnabled(true) + .build()) .build(); mockBlockchainBase(); @@ -266,7 +285,7 @@ public void mismatchInPrunedTrieLogCountShouldNotDeleteFiles(final @TempDir Path final BonsaiWorldStateKeyValueStorage inMemoryWorldStateSpy = spy(inMemoryWorldState); // force a different value the second time the trie log count is called - when(inMemoryWorldStateSpy.streamTrieLogKeys(3L + DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE)) + when(inMemoryWorldStateSpy.streamTrieLogKeys(3L + DEFAULT_TRIE_LOG_PRUNING_WINDOW_SIZE)) .thenCallRealMethod() .thenReturn(Stream.empty()); assertThatThrownBy( @@ -284,8 +303,11 @@ public void trieLogRetentionLimitShouldBeAboveMinimum() { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) - .bonsaiMaxLayersToLoad(511L) - .bonsaiLimitTrieLogsEnabled(true) + .diffBasedSubStorageConfiguration( + ImmutableDiffBasedSubStorageConfiguration.builder() + .maxLayersToLoad(511L) + .limitTrieLogsEnabled(true) + .build()) .build(); TrieLogHelper helper = new TrieLogHelper(); @@ -302,9 +324,12 @@ public void trieLogPruningWindowSizeShouldBePositive() { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) - .bonsaiMaxLayersToLoad(512L) - .bonsaiLimitTrieLogsEnabled(true) - .bonsaiTrieLogPruningWindowSize(0) + .diffBasedSubStorageConfiguration( + ImmutableDiffBasedSubStorageConfiguration.builder() + .maxLayersToLoad(512L) + .limitTrieLogsEnabled(true) + .trieLogPruningWindowSize(0) + .build()) .build(); TrieLogHelper helper = new TrieLogHelper(); @@ -320,9 +345,12 @@ public void trieLogPruningWindowSizeShouldBeAboveRetentionLimit() { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) - .bonsaiMaxLayersToLoad(512L) - .bonsaiLimitTrieLogsEnabled(true) - .bonsaiTrieLogPruningWindowSize(512) + .diffBasedSubStorageConfiguration( + ImmutableDiffBasedSubStorageConfiguration.builder() + .maxLayersToLoad(512L) + .limitTrieLogsEnabled(true) + .trieLogPruningWindowSize(512) + .build()) .build(); TrieLogHelper helper = new TrieLogHelper(); @@ -340,8 +368,11 @@ public void exceptionWhileSavingFileStopsPruneProcess(final @TempDir Path dataDi DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) - .bonsaiMaxLayersToLoad(3L) - .bonsaiLimitTrieLogsEnabled(true) + .diffBasedSubStorageConfiguration( + ImmutableDiffBasedSubStorageConfiguration.builder() + .maxLayersToLoad(3L) + .limitTrieLogsEnabled(true) + .build()) .build(); mockBlockchainBase(); diff --git a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommandTest.java index 66536070a7b..8e701ec67f0 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommandTest.java @@ -58,8 +58,10 @@ private void assertConfigurationIsDisabledBySubcommand() { .dataStorageConfiguration(dataStorageConfigurationArgumentCaptor.capture()); final List configs = dataStorageConfigurationArgumentCaptor.getAllValues(); - assertThat(configs.get(0).getBonsaiLimitTrieLogsEnabled()).isTrue(); - assertThat(configs.get(1).getBonsaiLimitTrieLogsEnabled()).isFalse(); + assertThat(configs.get(0).getDiffBasedSubStorageConfiguration().getLimitTrieLogsEnabled()) + .isTrue(); + assertThat(configs.get(1).getDiffBasedSubStorageConfiguration().getLimitTrieLogsEnabled()) + .isFalse(); } @Test @@ -69,6 +71,11 @@ void limitTrieLogsDefaultEnabledForBesuMainCommand() { .dataStorageConfiguration(dataStorageConfigurationArgumentCaptor.capture()); final List configs = dataStorageConfigurationArgumentCaptor.getAllValues(); - assertThat(configs).allMatch(DataStorageConfiguration::getBonsaiLimitTrieLogsEnabled); + assertThat(configs) + .allMatch( + dataStorageConfiguration -> + dataStorageConfiguration + .getDiffBasedSubStorageConfiguration() + .getLimitTrieLogsEnabled()); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorage.java index 9f40d2bd350..07ded9ddf61 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorage.java @@ -24,9 +24,10 @@ import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.trie.MerkleTrie; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.BonsaiFlatDbStrategy; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.BonsaiFlatDbStrategyProvider; import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategy; -import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategyProvider; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; @@ -49,7 +50,7 @@ public class BonsaiWorldStateKeyValueStorage extends DiffBasedWorldStateKeyValueStorage implements WorldStateKeyValueStorage { - protected final FlatDbStrategyProvider flatDbStrategyProvider; + protected final BonsaiFlatDbStrategyProvider flatDbStrategyProvider; public BonsaiWorldStateKeyValueStorage( final StorageProvider provider, @@ -61,12 +62,12 @@ public BonsaiWorldStateKeyValueStorage( ACCOUNT_INFO_STATE, CODE_STORAGE, ACCOUNT_STORAGE_STORAGE, TRIE_BRANCH_STORAGE)), provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE)); this.flatDbStrategyProvider = - new FlatDbStrategyProvider(metricsSystem, dataStorageConfiguration); + new BonsaiFlatDbStrategyProvider(metricsSystem, dataStorageConfiguration); flatDbStrategyProvider.loadFlatDbStrategy(composedWorldStateStorage); } public BonsaiWorldStateKeyValueStorage( - final FlatDbStrategyProvider flatDbStrategyProvider, + final BonsaiFlatDbStrategyProvider flatDbStrategyProvider, final SegmentedKeyValueStorage composedWorldStateStorage, final KeyValueStorage trieLogStorage) { super(composedWorldStateStorage, trieLogStorage); @@ -87,15 +88,12 @@ public Optional getCode(final Hash codeHash, final Hash accountHash) { if (codeHash.equals(Hash.EMPTY)) { return Optional.of(Bytes.EMPTY); } else { - return flatDbStrategyProvider - .getFlatDbStrategy(composedWorldStateStorage) - .getFlatCode(codeHash, accountHash, composedWorldStateStorage); + return getFlatDbStrategy().getFlatCode(codeHash, accountHash, composedWorldStateStorage); } } public Optional getAccount(final Hash accountHash) { - return flatDbStrategyProvider - .getFlatDbStrategy(composedWorldStateStorage) + return getFlatDbStrategy() .getFlatAccount( this::getWorldStateRootHash, this::getAccountStateTrieNode, @@ -148,8 +146,7 @@ public Optional getStorageValueByStorageSlotKey( final Supplier> storageRootSupplier, final Hash accountHash, final StorageSlotKey storageSlotKey) { - return flatDbStrategyProvider - .getFlatDbStrategy(composedWorldStateStorage) + return getFlatDbStrategy() .getFlatStorageValueByStorageSlotKey( this::getWorldStateRootHash, storageRootSupplier, @@ -180,8 +177,9 @@ public void clear() { } @Override - public FlatDbStrategy getFlatDbStrategy() { - return flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage); + public BonsaiFlatDbStrategy getFlatDbStrategy() { + return (BonsaiFlatDbStrategy) + flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage); } @Override @@ -189,7 +187,7 @@ public Updater updater() { return new Updater( composedWorldStateStorage.startTransaction(), trieLogStorage.startTransaction(), - flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage)); + getFlatDbStrategy()); } public static class Updater implements DiffBasedWorldStateKeyValueStorage.Updater { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/BonsaiFlatDbStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/BonsaiFlatDbStrategy.java new file mode 100644 index 00000000000..d0330ed2442 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/BonsaiFlatDbStrategy.java @@ -0,0 +1,173 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat; + +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.ethereum.trie.NodeLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.CodeStorageStrategy; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategy; +import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; + +import java.util.Optional; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Stream; + +import kotlin.Pair; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; + +/** + * This class represents a FlatDbReaderStrategy, which is responsible for reading and writing data + * from flat databases. It implements various methods for storing and retrieving account data, code + * data, and storage data from the corresponding KeyValueStorage. + */ +public abstract class BonsaiFlatDbStrategy extends FlatDbStrategy { + + public BonsaiFlatDbStrategy( + final MetricsSystem metricsSystem, final CodeStorageStrategy codeStorageStrategy) { + super(metricsSystem, codeStorageStrategy); + } + + /* + * Retrieves the account data for the given account hash, using the world state root hash supplier and node loader. + */ + public abstract Optional getFlatAccount( + Supplier> worldStateRootHashSupplier, + NodeLoader nodeLoader, + Hash accountHash, + SegmentedKeyValueStorage storage); + + /* + * Retrieves the storage value for the given account hash and storage slot key, using the world state root hash supplier, storage root supplier, and node loader. + */ + + public abstract Optional getFlatStorageValueByStorageSlotKey( + Supplier> worldStateRootHashSupplier, + Supplier> storageRootSupplier, + NodeLoader nodeLoader, + Hash accountHash, + StorageSlotKey storageSlotKey, + SegmentedKeyValueStorage storageStorage); + + @Override + public void putFlatAccount( + final SegmentedKeyValueStorageTransaction transaction, + final Hash accountHash, + final Bytes accountValue) { + transaction.put(ACCOUNT_INFO_STATE, accountHash.toArrayUnsafe(), accountValue.toArrayUnsafe()); + } + + @Override + public void removeFlatAccount( + final SegmentedKeyValueStorageTransaction transaction, final Hash accountHash) { + transaction.remove(ACCOUNT_INFO_STATE, accountHash.toArrayUnsafe()); + } + + @Override + public void putFlatAccountStorageValueByStorageSlotHash( + final SegmentedKeyValueStorageTransaction transaction, + final Hash accountHash, + final Hash slotHash, + final Bytes storage) { + transaction.put( + ACCOUNT_STORAGE_STORAGE, + Bytes.concatenate(accountHash, slotHash).toArrayUnsafe(), + storage.toArrayUnsafe()); + } + + @Override + public void removeFlatAccountStorageValueByStorageSlotHash( + final SegmentedKeyValueStorageTransaction transaction, + final Hash accountHash, + final Hash slotHash) { + transaction.remove( + ACCOUNT_STORAGE_STORAGE, Bytes.concatenate(accountHash, slotHash).toArrayUnsafe()); + } + + @Override + public void clearAll(final SegmentedKeyValueStorage storage) { + storage.clear(ACCOUNT_INFO_STATE); + storage.clear(ACCOUNT_STORAGE_STORAGE); + storage.clear(CODE_STORAGE); + } + + @Override + public void resetOnResync(final SegmentedKeyValueStorage storage) { + storage.clear(ACCOUNT_INFO_STATE); + storage.clear(ACCOUNT_STORAGE_STORAGE); + } + + @Override + protected Stream> storageToPairStream( + final SegmentedKeyValueStorage storage, + final Hash accountHash, + final Bytes startKeyHash, + final Function valueMapper) { + + return storage + .streamFromKey( + ACCOUNT_STORAGE_STORAGE, Bytes.concatenate(accountHash, startKeyHash).toArrayUnsafe()) + .takeWhile(pair -> Bytes.wrap(pair.getKey()).slice(0, Hash.SIZE).equals(accountHash)) + .map( + pair -> + new Pair<>( + Bytes32.wrap(Bytes.wrap(pair.getKey()).slice(Hash.SIZE)), + valueMapper.apply(Bytes.wrap(pair.getValue()).trimLeadingZeros()))); + } + + @Override + protected Stream> storageToPairStream( + final SegmentedKeyValueStorage storage, + final Hash accountHash, + final Bytes startKeyHash, + final Bytes32 endKeyHash, + final Function valueMapper) { + + return storage + .streamFromKey( + ACCOUNT_STORAGE_STORAGE, + Bytes.concatenate(accountHash, startKeyHash).toArrayUnsafe(), + Bytes.concatenate(accountHash, endKeyHash).toArrayUnsafe()) + .map( + pair -> + new Pair<>( + Bytes32.wrap(Bytes.wrap(pair.getKey()).slice(Hash.SIZE)), + valueMapper.apply(Bytes.wrap(pair.getValue()).trimLeadingZeros()))); + } + + @Override + protected Stream> accountsToPairStream( + final SegmentedKeyValueStorage storage, final Bytes startKeyHash, final Bytes32 endKeyHash) { + return storage + .streamFromKey(ACCOUNT_INFO_STATE, startKeyHash.toArrayUnsafe(), endKeyHash.toArrayUnsafe()) + .map(pair -> new Pair<>(Bytes32.wrap(pair.getKey()), Bytes.wrap(pair.getValue()))); + } + + @Override + protected Stream> accountsToPairStream( + final SegmentedKeyValueStorage storage, final Bytes startKeyHash) { + return storage + .streamFromKey(ACCOUNT_INFO_STATE, startKeyHash.toArrayUnsafe()) + .map(pair -> new Pair<>(Bytes32.wrap(pair.getKey()), Bytes.wrap(pair.getValue()))); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/BonsaiFlatDbStrategyProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/BonsaiFlatDbStrategyProvider.java new file mode 100644 index 00000000000..f6157ca2fee --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/BonsaiFlatDbStrategyProvider.java @@ -0,0 +1,88 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat; + +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; + +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.CodeStorageStrategy; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategy; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategyProvider; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; +import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; +import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class BonsaiFlatDbStrategyProvider extends FlatDbStrategyProvider { + + private static final Logger LOG = LoggerFactory.getLogger(BonsaiFlatDbStrategyProvider.class); + + public BonsaiFlatDbStrategyProvider( + final MetricsSystem metricsSystem, final DataStorageConfiguration dataStorageConfiguration) { + super(metricsSystem, dataStorageConfiguration); + } + + @Override + protected FlatDbMode getRequestedFlatDbMode( + final DataStorageConfiguration dataStorageConfiguration) { + return dataStorageConfiguration + .getDiffBasedSubStorageConfiguration() + .getUnstable() + .getFullFlatDbEnabled() + ? FlatDbMode.FULL + : FlatDbMode.PARTIAL; + } + + @Override + protected FlatDbMode alternativeFlatDbModeForExistingDatabase() { + return FlatDbMode.PARTIAL; + } + + public void upgradeToFullFlatDbMode(final SegmentedKeyValueStorage composedWorldStateStorage) { + final SegmentedKeyValueStorageTransaction transaction = + composedWorldStateStorage.startTransaction(); + LOG.info("setting FlatDbStrategy to FULL"); + transaction.put( + TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.FULL.getVersion().toArrayUnsafe()); + transaction.commit(); + loadFlatDbStrategy(composedWorldStateStorage); // force reload of flat db reader strategy + } + + public void downgradeToPartialFlatDbMode( + final SegmentedKeyValueStorage composedWorldStateStorage) { + final SegmentedKeyValueStorageTransaction transaction = + composedWorldStateStorage.startTransaction(); + LOG.info("setting FlatDbStrategy to PARTIAL"); + transaction.put( + TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.PARTIAL.getVersion().toArrayUnsafe()); + transaction.commit(); + loadFlatDbStrategy(composedWorldStateStorage); // force reload of flat db reader strategy + } + + @Override + protected FlatDbStrategy createFlatDbStrategy( + final FlatDbMode flatDbMode, + final MetricsSystem metricsSystem, + final CodeStorageStrategy codeStorageStrategy) { + if (flatDbMode == FlatDbMode.FULL) { + return new BonsaiFullFlatDbStrategy(metricsSystem, codeStorageStrategy); + } else { + return new BonsaiPartialFlatDbStrategy(metricsSystem, codeStorageStrategy); + } + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/FullFlatDbStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/BonsaiFullFlatDbStrategy.java similarity index 95% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/FullFlatDbStrategy.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/BonsaiFullFlatDbStrategy.java index dd4dac5563b..7a251a03aef 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/FullFlatDbStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/BonsaiFullFlatDbStrategy.java @@ -21,7 +21,6 @@ import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.ethereum.trie.NodeLoader; import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.CodeStorageStrategy; -import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategy; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; @@ -32,13 +31,13 @@ import org.apache.tuweni.bytes.Bytes; -public class FullFlatDbStrategy extends FlatDbStrategy { +public class BonsaiFullFlatDbStrategy extends BonsaiFlatDbStrategy { protected final Counter getAccountNotFoundInFlatDatabaseCounter; protected final Counter getStorageValueNotFoundInFlatDatabaseCounter; - public FullFlatDbStrategy( + public BonsaiFullFlatDbStrategy( final MetricsSystem metricsSystem, final CodeStorageStrategy codeStorageStrategy) { super(metricsSystem, codeStorageStrategy); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/PartialFlatDbStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/BonsaiPartialFlatDbStrategy.java similarity index 97% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/PartialFlatDbStrategy.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/BonsaiPartialFlatDbStrategy.java index 250dc7505cb..301536bcc56 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/PartialFlatDbStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/BonsaiPartialFlatDbStrategy.java @@ -21,7 +21,6 @@ import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.ethereum.trie.NodeLoader; import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.CodeStorageStrategy; -import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategy; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.trie.patricia.StoredNodeFactory; import org.hyperledger.besu.metrics.BesuMetricCategory; @@ -45,7 +44,7 @@ * methods, which checks if the data is present in the flat database, and if not, queries the merkle * trie */ -public class PartialFlatDbStrategy extends FlatDbStrategy { +public class BonsaiPartialFlatDbStrategy extends BonsaiFlatDbStrategy { protected final Counter getAccountMerkleTrieCounter; protected final Counter getAccountMissingMerkleTrieCounter; @@ -53,7 +52,7 @@ public class PartialFlatDbStrategy extends FlatDbStrategy { protected final Counter getStorageValueMerkleTrieCounter; protected final Counter getStorageValueMissingMerkleTrieCounter; - public PartialFlatDbStrategy( + public BonsaiPartialFlatDbStrategy( final MetricsSystem metricsSystem, final CodeStorageStrategy codeStorageStrategy) { super(metricsSystem, codeStorageStrategy); getAccountMerkleTrieCounter = diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategy.java index ad631b8da91..046aaa5cc6a 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategy.java @@ -14,13 +14,7 @@ */ package org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat; -import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; -import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; -import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; - import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.datatypes.StorageSlotKey; -import org.hyperledger.besu.ethereum.trie.NodeLoader; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; @@ -33,7 +27,6 @@ import java.util.TreeMap; import java.util.function.Function; import java.util.function.Predicate; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -86,27 +79,6 @@ public FlatDbStrategy( "Number of storage slots found in the flat database"); } - /* - * Retrieves the account data for the given account hash, using the world state root hash supplier and node loader. - */ - public abstract Optional getFlatAccount( - Supplier> worldStateRootHashSupplier, - NodeLoader nodeLoader, - Hash accountHash, - SegmentedKeyValueStorage storage); - - /* - * Retrieves the storage value for the given account hash and storage slot key, using the world state root hash supplier, storage root supplier, and node loader. - */ - - public abstract Optional getFlatStorageValueByStorageSlotKey( - Supplier> worldStateRootHashSupplier, - Supplier> storageRootSupplier, - NodeLoader nodeLoader, - Hash accountHash, - StorageSlotKey storageSlotKey, - SegmentedKeyValueStorage storageStorage); - public boolean isCodeByCodeHash() { return codeStorageStrategy instanceof CodeHashCodeStorageStrategy; } @@ -124,76 +96,57 @@ public Optional getFlatCode( } /* - * Puts the account data for the given account hash, using the world state root hash supplier and node loader. + * Removes code for the given account hash. */ - public void putFlatAccount( + public void removeFlatCode( final SegmentedKeyValueStorageTransaction transaction, final Hash accountHash, - final Bytes accountValue) { - transaction.put(ACCOUNT_INFO_STATE, accountHash.toArrayUnsafe(), accountValue.toArrayUnsafe()); - } - - public void removeFlatAccount( - final SegmentedKeyValueStorageTransaction transaction, final Hash accountHash) { - transaction.remove(ACCOUNT_INFO_STATE, accountHash.toArrayUnsafe()); + final Hash codeHash) { + codeStorageStrategy.removeFlatCode(transaction, accountHash, codeHash); } /* - * Puts the storage value for the given account hash and storage slot key, using the world state root hash supplier, storage root supplier, and node loader. + * Puts the code data for the given code hash and account hash. */ - public void putFlatAccountStorageValueByStorageSlotHash( + public void putFlatCode( final SegmentedKeyValueStorageTransaction transaction, final Hash accountHash, - final Hash slotHash, - final Bytes storage) { - transaction.put( - ACCOUNT_STORAGE_STORAGE, - Bytes.concatenate(accountHash, slotHash).toArrayUnsafe(), - storage.toArrayUnsafe()); + final Hash codeHash, + final Bytes code) { + codeStorageStrategy.putFlatCode(transaction, accountHash, codeHash, code); } /* - * Removes the storage value for the given account hash and storage slot key, using the world state root hash supplier, storage root supplier, and node loader. + * Puts the account data for the given account hash, using the world state root hash supplier and node loader. */ - public void removeFlatAccountStorageValueByStorageSlotHash( + public abstract void putFlatAccount( final SegmentedKeyValueStorageTransaction transaction, final Hash accountHash, - final Hash slotHash) { - transaction.remove( - ACCOUNT_STORAGE_STORAGE, Bytes.concatenate(accountHash, slotHash).toArrayUnsafe()); - } + final Bytes accountValue); + + public abstract void removeFlatAccount( + final SegmentedKeyValueStorageTransaction transaction, final Hash accountHash); /* - * Removes code for the given account hash. + * Puts the storage value for the given account hash and storage slot key, using the world state root hash supplier, storage root supplier, and node loader. */ - public void removeFlatCode( + public abstract void putFlatAccountStorageValueByStorageSlotHash( final SegmentedKeyValueStorageTransaction transaction, final Hash accountHash, - final Hash codeHash) { - codeStorageStrategy.removeFlatCode(transaction, accountHash, codeHash); - } + final Hash slotHash, + final Bytes storage); /* - * Puts the code data for the given code hash and account hash. + * Removes the storage value for the given account hash and storage slot key, using the world state root hash supplier, storage root supplier, and node loader. */ - public void putFlatCode( + public abstract void removeFlatAccountStorageValueByStorageSlotHash( final SegmentedKeyValueStorageTransaction transaction, final Hash accountHash, - final Hash codeHash, - final Bytes code) { - codeStorageStrategy.putFlatCode(transaction, accountHash, codeHash, code); - } + final Hash slotHash); - public void clearAll(final SegmentedKeyValueStorage storage) { - storage.clear(ACCOUNT_INFO_STATE); - storage.clear(ACCOUNT_STORAGE_STORAGE); - storage.clear(CODE_STORAGE); - } + public abstract void clearAll(final SegmentedKeyValueStorage storage); - public void resetOnResync(final SegmentedKeyValueStorage storage) { - storage.clear(ACCOUNT_INFO_STATE); - storage.clear(ACCOUNT_STORAGE_STORAGE); - } + public abstract void resetOnResync(final SegmentedKeyValueStorage storage); public NavigableMap streamAccountFlatDatabase( final SegmentedKeyValueStorage storage, @@ -249,57 +202,26 @@ public NavigableMap streamStorageFlatDatabase( .takeWhile(takeWhile)); } - private static Stream> storageToPairStream( + protected abstract Stream> storageToPairStream( final SegmentedKeyValueStorage storage, final Hash accountHash, final Bytes startKeyHash, - final Function valueMapper) { - - return storage - .streamFromKey( - ACCOUNT_STORAGE_STORAGE, Bytes.concatenate(accountHash, startKeyHash).toArrayUnsafe()) - .takeWhile(pair -> Bytes.wrap(pair.getKey()).slice(0, Hash.SIZE).equals(accountHash)) - .map( - pair -> - new Pair<>( - Bytes32.wrap(Bytes.wrap(pair.getKey()).slice(Hash.SIZE)), - valueMapper.apply(Bytes.wrap(pair.getValue()).trimLeadingZeros()))); - } + final Function valueMapper); - private static Stream> storageToPairStream( + protected abstract Stream> storageToPairStream( final SegmentedKeyValueStorage storage, final Hash accountHash, final Bytes startKeyHash, final Bytes32 endKeyHash, - final Function valueMapper) { - - return storage - .streamFromKey( - ACCOUNT_STORAGE_STORAGE, - Bytes.concatenate(accountHash, startKeyHash).toArrayUnsafe(), - Bytes.concatenate(accountHash, endKeyHash).toArrayUnsafe()) - .map( - pair -> - new Pair<>( - Bytes32.wrap(Bytes.wrap(pair.getKey()).slice(Hash.SIZE)), - valueMapper.apply(Bytes.wrap(pair.getValue()).trimLeadingZeros()))); - } + final Function valueMapper); - private static Stream> accountsToPairStream( - final SegmentedKeyValueStorage storage, final Bytes startKeyHash, final Bytes32 endKeyHash) { - return storage - .streamFromKey(ACCOUNT_INFO_STATE, startKeyHash.toArrayUnsafe(), endKeyHash.toArrayUnsafe()) - .map(pair -> new Pair<>(Bytes32.wrap(pair.getKey()), Bytes.wrap(pair.getValue()))); - } + protected abstract Stream> accountsToPairStream( + final SegmentedKeyValueStorage storage, final Bytes startKeyHash, final Bytes32 endKeyHash); - private static Stream> accountsToPairStream( - final SegmentedKeyValueStorage storage, final Bytes startKeyHash) { - return storage - .streamFromKey(ACCOUNT_INFO_STATE, startKeyHash.toArrayUnsafe()) - .map(pair -> new Pair<>(Bytes32.wrap(pair.getKey()), Bytes.wrap(pair.getValue()))); - } + protected abstract Stream> accountsToPairStream( + final SegmentedKeyValueStorage storage, final Bytes startKeyHash); - private static NavigableMap toNavigableMap( + private NavigableMap toNavigableMap( final Stream> pairStream) { final TreeMap collected = pairStream.collect( diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProvider.java index 3ce51e793a4..31e8598b28d 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProvider.java @@ -18,13 +18,10 @@ import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; import static org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.FullFlatDbStrategy; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.PartialFlatDbStrategy; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; -import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; import java.nio.charset.StandardCharsets; import java.util.Optional; @@ -34,7 +31,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class FlatDbStrategyProvider { +public abstract class FlatDbStrategyProvider { private static final Logger LOG = LoggerFactory.getLogger(FlatDbStrategyProvider.class); // 0x666C61744462537461747573 @@ -61,20 +58,48 @@ public void loadFlatDbStrategy(final SegmentedKeyValueStorage composedWorldState deriveUseCodeStorageByHash(composedWorldStateStorage) ? new CodeHashCodeStorageStrategy() : new AccountHashCodeStorageStrategy(); - if (flatDbMode == FlatDbMode.FULL) { - this.flatDbStrategy = new FullFlatDbStrategy(metricsSystem, codeStorageStrategy); - } else { - this.flatDbStrategy = new PartialFlatDbStrategy(metricsSystem, codeStorageStrategy); - } + this.flatDbStrategy = createFlatDbStrategy(flatDbMode, metricsSystem, codeStorageStrategy); } } + protected boolean deriveUseCodeStorageByHash( + final SegmentedKeyValueStorage composedWorldStateStorage) { + final boolean configCodeUsingHash = + dataStorageConfiguration + .getDiffBasedSubStorageConfiguration() + .getUnstable() + .getCodeStoredByCodeHashEnabled(); + boolean codeUsingCodeByHash = + detectCodeStorageByHash(composedWorldStateStorage) + .map( + dbCodeUsingHash -> { + if (dbCodeUsingHash != configCodeUsingHash) { + LOG.warn( + "Bonsai db is using code storage mode {} but config specifies mode {}. Using mode from database", + dbCodeUsingHash, + configCodeUsingHash); + } + return dbCodeUsingHash; + }) + .orElse(configCodeUsingHash); + LOG.info("DB mode with code stored using code hash enabled = {}", codeUsingCodeByHash); + return codeUsingCodeByHash; + } + + private Optional detectCodeStorageByHash( + final SegmentedKeyValueStorage composedWorldStateStorage) { + return composedWorldStateStorage.stream(CODE_STORAGE) + .limit(1) + .findFirst() + .map( + keypair -> + CodeHashCodeStorageStrategy.isCodeHashValue(keypair.getKey(), keypair.getValue())); + } + @VisibleForTesting - FlatDbMode deriveFlatDbStrategy(final SegmentedKeyValueStorage composedWorldStateStorage) { - final FlatDbMode requestedFlatDbMode = - dataStorageConfiguration.getUnstable().getBonsaiFullFlatDbEnabled() - ? FlatDbMode.FULL - : FlatDbMode.PARTIAL; + private FlatDbMode deriveFlatDbStrategy( + final SegmentedKeyValueStorage composedWorldStateStorage) { + final FlatDbMode requestedFlatDbMode = getRequestedFlatDbMode(dataStorageConfiguration); final var existingTrieData = composedWorldStateStorage.get(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY).isPresent(); @@ -91,7 +116,7 @@ FlatDbMode deriveFlatDbStrategy(final SegmentedKeyValueStorage composedWorldStat // and default to the storage config otherwise var flatDbModeVal = existingTrieData - ? FlatDbMode.PARTIAL.getVersion() + ? alternativeFlatDbModeForExistingDatabase().getVersion() : requestedFlatDbMode.getVersion(); // persist this config in the db var setDbModeTx = composedWorldStateStorage.startTransaction(); @@ -101,42 +126,11 @@ FlatDbMode deriveFlatDbStrategy(final SegmentedKeyValueStorage composedWorldStat return flatDbModeVal; })); - LOG.info("Bonsai flat db mode found {}", flatDbMode); + LOG.info("Flat db mode found {}", flatDbMode); return flatDbMode; } - protected boolean deriveUseCodeStorageByHash( - final SegmentedKeyValueStorage composedWorldStateStorage) { - final boolean configCodeUsingHash = - dataStorageConfiguration.getUnstable().getBonsaiCodeStoredByCodeHashEnabled(); - boolean codeUsingCodeByHash = - detectCodeStorageByHash(composedWorldStateStorage) - .map( - dbCodeUsingHash -> { - if (dbCodeUsingHash != configCodeUsingHash) { - LOG.warn( - "Bonsai db is using code storage mode {} but config specifies mode {}. Using mode from database", - dbCodeUsingHash, - configCodeUsingHash); - } - return dbCodeUsingHash; - }) - .orElse(configCodeUsingHash); - LOG.info("Bonsai db mode with code stored using code hash enabled = {}", codeUsingCodeByHash); - return codeUsingCodeByHash; - } - - private Optional detectCodeStorageByHash( - final SegmentedKeyValueStorage composedWorldStateStorage) { - return composedWorldStateStorage.stream(CODE_STORAGE) - .limit(1) - .findFirst() - .map( - keypair -> - CodeHashCodeStorageStrategy.isCodeHashValue(keypair.getKey(), keypair.getValue())); - } - public FlatDbStrategy getFlatDbStrategy( final SegmentedKeyValueStorage composedWorldStateStorage) { if (flatDbStrategy == null) { @@ -145,28 +139,17 @@ public FlatDbStrategy getFlatDbStrategy( return flatDbStrategy; } - public void upgradeToFullFlatDbMode(final SegmentedKeyValueStorage composedWorldStateStorage) { - final SegmentedKeyValueStorageTransaction transaction = - composedWorldStateStorage.startTransaction(); - LOG.info("setting FlatDbStrategy to FULL"); - transaction.put( - TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.FULL.getVersion().toArrayUnsafe()); - transaction.commit(); - loadFlatDbStrategy(composedWorldStateStorage); // force reload of flat db reader strategy - } - - public void downgradeToPartialFlatDbMode( - final SegmentedKeyValueStorage composedWorldStateStorage) { - final SegmentedKeyValueStorageTransaction transaction = - composedWorldStateStorage.startTransaction(); - LOG.info("setting FlatDbStrategy to PARTIAL"); - transaction.put( - TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.PARTIAL.getVersion().toArrayUnsafe()); - transaction.commit(); - loadFlatDbStrategy(composedWorldStateStorage); // force reload of flat db reader strategy - } - public FlatDbMode getFlatDbMode() { return flatDbMode; } + + protected abstract FlatDbMode getRequestedFlatDbMode( + final DataStorageConfiguration dataStorageConfiguration); + + protected abstract FlatDbMode alternativeFlatDbModeForExistingDatabase(); + + protected abstract FlatDbStrategy createFlatDbStrategy( + final FlatDbMode flatDbMode, + final MetricsSystem metricsSystem, + final CodeStorageStrategy codeStorageStrategy); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java index 320e38733d4..9ce059e94c2 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java @@ -14,6 +14,7 @@ */ package org.hyperledger.besu.ethereum.worldstate; +import org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration.DiffBasedUnstable; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.immutables.value.Value; @@ -22,90 +23,40 @@ @Value.Enclosing public interface DataStorageConfiguration { - long DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD = 512; - boolean DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED = true; - long MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; - int DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE = 5_000; boolean DEFAULT_RECEIPT_COMPACTION_ENABLED = true; DataStorageConfiguration DEFAULT_CONFIG = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(DataStorageFormat.BONSAI) - .bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD) - .unstable(Unstable.DEFAULT) + .diffBasedSubStorageConfiguration(DiffBasedSubStorageConfiguration.DEFAULT) .build(); - DataStorageConfiguration DEFAULT_BONSAI_CONFIG = - ImmutableDataStorageConfiguration.builder() - .dataStorageFormat(DataStorageFormat.BONSAI) - .bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD) - .build(); + DataStorageConfiguration DEFAULT_BONSAI_CONFIG = DEFAULT_CONFIG; DataStorageConfiguration DEFAULT_BONSAI_PARTIAL_DB_CONFIG = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(DataStorageFormat.BONSAI) - .bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD) - .unstable(Unstable.DEFAULT_PARTIAL) + .diffBasedSubStorageConfiguration( + ImmutableDiffBasedSubStorageConfiguration.builder() + .unstable(DiffBasedUnstable.PARTIAL_MODE) + .build()) .build(); DataStorageConfiguration DEFAULT_FOREST_CONFIG = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(DataStorageFormat.FOREST) - .bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD) - .unstable(Unstable.DEFAULT) + .diffBasedSubStorageConfiguration(DiffBasedSubStorageConfiguration.DISABLED) .build(); DataStorageFormat getDataStorageFormat(); - Long getBonsaiMaxLayersToLoad(); - - @Value.Default - default boolean getBonsaiLimitTrieLogsEnabled() { - return DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED; - } - @Value.Default - default int getBonsaiTrieLogPruningWindowSize() { - return DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE; + default DiffBasedSubStorageConfiguration getDiffBasedSubStorageConfiguration() { + return DiffBasedSubStorageConfiguration.DEFAULT; } @Value.Default default boolean getReceiptCompactionEnabled() { return DEFAULT_RECEIPT_COMPACTION_ENABLED; } - - @Value.Default - default Unstable getUnstable() { - return Unstable.DEFAULT; - } - - @Value.Immutable - interface Unstable { - - boolean DEFAULT_BONSAI_FULL_FLAT_DB_ENABLED = true; - boolean DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED = true; - - boolean DEFAULT_PARALLEL_TRX_ENABLED = false; - - DataStorageConfiguration.Unstable DEFAULT = - ImmutableDataStorageConfiguration.Unstable.builder().build(); - - DataStorageConfiguration.Unstable DEFAULT_PARTIAL = - ImmutableDataStorageConfiguration.Unstable.builder().bonsaiFullFlatDbEnabled(false).build(); - - @Value.Default - default boolean getBonsaiFullFlatDbEnabled() { - return DEFAULT_BONSAI_FULL_FLAT_DB_ENABLED; - } - - @Value.Default - default boolean getBonsaiCodeStoredByCodeHashEnabled() { - return DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED; - } - - @Value.Default - default boolean isParallelTxProcessingEnabled() { - return DEFAULT_PARALLEL_TRX_ENABLED; - } - } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DiffBasedSubStorageConfiguration.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DiffBasedSubStorageConfiguration.java new file mode 100644 index 00000000000..fc0f98e21d1 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DiffBasedSubStorageConfiguration.java @@ -0,0 +1,95 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.worldstate; + +import org.immutables.value.Value; + +@Value.Immutable +@Value.Enclosing +public interface DiffBasedSubStorageConfiguration { + + DiffBasedSubStorageConfiguration DEFAULT = + ImmutableDiffBasedSubStorageConfiguration.builder().build(); + + DiffBasedSubStorageConfiguration DISABLED = + ImmutableDiffBasedSubStorageConfiguration.builder() + .limitTrieLogsEnabled(false) + .unstable(DiffBasedUnstable.DISABLED) + .build(); + + long DEFAULT_MAX_LAYERS_TO_LOAD = 512; + boolean DEFAULT_LIMIT_TRIE_LOGS_ENABLED = true; + long MINIMUM_TRIE_LOG_RETENTION_LIMIT = DEFAULT_MAX_LAYERS_TO_LOAD; + int DEFAULT_TRIE_LOG_PRUNING_WINDOW_SIZE = 5_000; + + @Value.Default + default Long getMaxLayersToLoad() { + return DEFAULT_MAX_LAYERS_TO_LOAD; + } + + @Value.Default + default boolean getLimitTrieLogsEnabled() { + return DEFAULT_LIMIT_TRIE_LOGS_ENABLED; + } + + @Value.Default + default int getTrieLogPruningWindowSize() { + return DEFAULT_TRIE_LOG_PRUNING_WINDOW_SIZE; + } + + @Value.Default + default DiffBasedUnstable getUnstable() { + return DiffBasedUnstable.DEFAULT; + } + + @Value.Immutable + interface DiffBasedUnstable { + + DiffBasedSubStorageConfiguration.DiffBasedUnstable DEFAULT = + ImmutableDiffBasedSubStorageConfiguration.DiffBasedUnstable.builder().build(); + + DiffBasedSubStorageConfiguration.DiffBasedUnstable PARTIAL_MODE = + ImmutableDiffBasedSubStorageConfiguration.DiffBasedUnstable.builder() + .fullFlatDbEnabled(false) + .build(); + + DiffBasedSubStorageConfiguration.DiffBasedUnstable DISABLED = + ImmutableDiffBasedSubStorageConfiguration.DiffBasedUnstable.builder() + .fullFlatDbEnabled(false) + .codeStoredByCodeHashEnabled(false) + .isParallelTxProcessingEnabled(false) + .build(); + + boolean DEFAULT_FULL_FLAT_DB_ENABLED = true; + boolean DEFAULT_CODE_USING_CODE_HASH_ENABLED = true; + + boolean DEFAULT_PARALLEL_TRX_ENABLED = false; + + @Value.Default + default boolean getFullFlatDbEnabled() { + return DEFAULT_FULL_FLAT_DB_ENABLED; + } + + @Value.Default + default boolean getCodeStoredByCodeHashEnabled() { + return DEFAULT_CODE_USING_CODE_HASH_ENABLED; + } + + @Value.Default + default boolean isParallelTxProcessingEnabled() { + return DEFAULT_PARALLEL_TRX_ENABLED; + } + } +} diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java index 02818407b46..464f35bae71 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java @@ -17,7 +17,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; import static org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; -import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; +import static org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration.DEFAULT_MAX_LAYERS_TO_LOAD; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; @@ -40,6 +40,7 @@ import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; +import org.hyperledger.besu.ethereum.worldstate.ImmutableDiffBasedSubStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -472,10 +473,13 @@ private BonsaiWorldStateKeyValueStorage emptyStorage(final boolean useCodeHashSt new NoOpMetricsSystem(), ImmutableDataStorageConfiguration.builder() .dataStorageFormat(DataStorageFormat.BONSAI) - .bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD) - .unstable( - ImmutableDataStorageConfiguration.Unstable.builder() - .bonsaiCodeStoredByCodeHashEnabled(useCodeHashStorage) + .diffBasedSubStorageConfiguration( + ImmutableDiffBasedSubStorageConfiguration.builder() + .maxLayersToLoad(DEFAULT_MAX_LAYERS_TO_LOAD) + .unstable( + ImmutableDiffBasedSubStorageConfiguration.DiffBasedUnstable.builder() + .codeStoredByCodeHashEnabled(useCodeHashStorage) + .build()) .build()) .build()); } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProviderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/BonsaiFlatDbStrategyProviderTest.java similarity index 74% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProviderTest.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/BonsaiFlatDbStrategyProviderTest.java index b3c709b89f2..08b821bd67c 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProviderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/BonsaiFlatDbStrategyProviderTest.java @@ -15,15 +15,17 @@ package org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat; import static org.assertj.core.api.Assertions.assertThat; -import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.FullFlatDbStrategy; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.PartialFlatDbStrategy; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.BonsaiFlatDbStrategyProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.BonsaiFullFlatDbStrategy; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.BonsaiPartialFlatDbStrategy; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; +import org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; +import org.hyperledger.besu.ethereum.worldstate.ImmutableDiffBasedSubStorageConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; @@ -42,9 +44,10 @@ import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) -class FlatDbStrategyProviderTest { - private final FlatDbStrategyProvider flatDbStrategyProvider = - new FlatDbStrategyProvider(new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); +class BonsaiFlatDbStrategyProviderTest { + private final BonsaiFlatDbStrategyProvider flatDbStrategyProvider = + new BonsaiFlatDbStrategyProvider( + new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); private final SegmentedKeyValueStorage composedWorldStateStorage = new SegmentedInMemoryKeyValueStorage( List.of( @@ -74,7 +77,7 @@ void upgradesFlatDbStrategyToFullFlatDbMode() { assertThat(flatDbStrategyProvider.flatDbMode).isEqualTo(FlatDbMode.FULL); assertThat(flatDbStrategyProvider.flatDbStrategy).isNotNull(); assertThat(flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage)) - .isInstanceOf(FullFlatDbStrategy.class); + .isInstanceOf(BonsaiFullFlatDbStrategy.class); assertThat(flatDbStrategyProvider.flatDbStrategy.codeStorageStrategy) .isInstanceOf(CodeHashCodeStorageStrategy.class); } @@ -85,14 +88,17 @@ void emptyDbCreatesFlatDbStrategyUsingCodeByHashConfig(final boolean codeByHashE final DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(DataStorageFormat.BONSAI) - .bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD) - .unstable( - ImmutableDataStorageConfiguration.Unstable.builder() - .bonsaiCodeStoredByCodeHashEnabled(codeByHashEnabled) + .diffBasedSubStorageConfiguration( + ImmutableDiffBasedSubStorageConfiguration.builder() + .maxLayersToLoad(DiffBasedSubStorageConfiguration.DEFAULT_MAX_LAYERS_TO_LOAD) + .unstable( + ImmutableDiffBasedSubStorageConfiguration.DiffBasedUnstable.builder() + .codeStoredByCodeHashEnabled(codeByHashEnabled) + .build()) .build()) .build(); - final FlatDbStrategyProvider flatDbStrategyProvider = - new FlatDbStrategyProvider(new NoOpMetricsSystem(), dataStorageConfiguration); + final BonsaiFlatDbStrategyProvider flatDbStrategyProvider = + new BonsaiFlatDbStrategyProvider(new NoOpMetricsSystem(), dataStorageConfiguration); flatDbStrategyProvider.loadFlatDbStrategy(composedWorldStateStorage); final Class expectedCodeStorageClass = @@ -110,14 +116,17 @@ void existingAccountHashDbUsesAccountHash(final boolean codeByHashEnabled) { final DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(DataStorageFormat.BONSAI) - .bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD) - .unstable( - ImmutableDataStorageConfiguration.Unstable.builder() - .bonsaiCodeStoredByCodeHashEnabled(codeByHashEnabled) + .diffBasedSubStorageConfiguration( + ImmutableDiffBasedSubStorageConfiguration.builder() + .maxLayersToLoad(DiffBasedSubStorageConfiguration.DEFAULT_MAX_LAYERS_TO_LOAD) + .unstable( + ImmutableDiffBasedSubStorageConfiguration.DiffBasedUnstable.builder() + .codeStoredByCodeHashEnabled(codeByHashEnabled) + .build()) .build()) .build(); - final FlatDbStrategyProvider flatDbStrategyProvider = - new FlatDbStrategyProvider(new NoOpMetricsSystem(), dataStorageConfiguration); + final BonsaiFlatDbStrategyProvider flatDbStrategyProvider = + new BonsaiFlatDbStrategyProvider(new NoOpMetricsSystem(), dataStorageConfiguration); final SegmentedKeyValueStorageTransaction transaction = composedWorldStateStorage.startTransaction(); @@ -140,14 +149,17 @@ void existingCodeHashDbUsesCodeHash(final boolean codeByHashEnabled) { final DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(DataStorageFormat.BONSAI) - .bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD) - .unstable( - ImmutableDataStorageConfiguration.Unstable.builder() - .bonsaiCodeStoredByCodeHashEnabled(codeByHashEnabled) + .diffBasedSubStorageConfiguration( + ImmutableDiffBasedSubStorageConfiguration.builder() + .maxLayersToLoad(DiffBasedSubStorageConfiguration.DEFAULT_MAX_LAYERS_TO_LOAD) + .unstable( + ImmutableDiffBasedSubStorageConfiguration.DiffBasedUnstable.builder() + .codeStoredByCodeHashEnabled(codeByHashEnabled) + .build()) .build()) .build(); - final FlatDbStrategyProvider flatDbStrategyProvider = - new FlatDbStrategyProvider(new NoOpMetricsSystem(), dataStorageConfiguration); + final BonsaiFlatDbStrategyProvider flatDbStrategyProvider = + new BonsaiFlatDbStrategyProvider(new NoOpMetricsSystem(), dataStorageConfiguration); final SegmentedKeyValueStorageTransaction transaction = composedWorldStateStorage.startTransaction(); @@ -171,7 +183,7 @@ void downgradesFlatDbStrategyToPartiallyFlatDbMode() { assertThat(flatDbStrategyProvider.flatDbMode).isEqualTo(FlatDbMode.PARTIAL); assertThat(flatDbStrategyProvider.flatDbStrategy).isNotNull(); assertThat(flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage)) - .isInstanceOf(PartialFlatDbStrategy.class); + .isInstanceOf(BonsaiPartialFlatDbStrategy.class); } private void updateFlatDbMode(final FlatDbMode flatDbMode) { diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapServerTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapServerTest.java index e168b7e2fe3..28446034296 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapServerTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapServerTest.java @@ -38,7 +38,7 @@ import org.hyperledger.besu.ethereum.trie.CompactEncoding; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategyProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.BonsaiFlatDbStrategyProvider; import org.hyperledger.besu.ethereum.trie.patricia.SimpleMerklePatriciaTrie; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; @@ -85,7 +85,8 @@ Bytes accountRLP() { // force a full flat db with code stored by code hash: final BonsaiWorldStateKeyValueStorage inMemoryStorage = new BonsaiWorldStateKeyValueStorage( - new FlatDbStrategyProvider(noopMetrics, DataStorageConfiguration.DEFAULT_BONSAI_CONFIG) { + new BonsaiFlatDbStrategyProvider( + noopMetrics, DataStorageConfiguration.DEFAULT_BONSAI_CONFIG) { @Override public FlatDbMode getFlatDbMode() { return FlatDbMode.FULL;