From 4b1494b6aad6a1f9a38fc5b478474cd7e1877372 Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Sat, 7 Dec 2024 17:09:21 +0100 Subject: [PATCH 01/33] Provisional execution code --- silkworm/capi/silkworm.cpp | 86 ++++++++++++++++++++++++++++++++++++++ silkworm/capi/silkworm.h | 2 + 2 files changed, 88 insertions(+) diff --git a/silkworm/capi/silkworm.cpp b/silkworm/capi/silkworm.cpp index 20771fbedc..40c7ebf700 100644 --- a/silkworm/capi/silkworm.cpp +++ b/silkworm/capi/silkworm.cpp @@ -49,8 +49,11 @@ #include #include #include +#include #include #include +#include +#include #include "common.hpp" #include "instance.hpp" @@ -729,6 +732,89 @@ int silkworm_execute_blocks_perpetual(SilkwormHandle handle, MDBX_env* mdbx_env, } } +//todo: add available gas, add txn, add block header +SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* txn, uint64_t block_num, uint64_t tx_index, uint64_t* gas_used, uint64_t* blob_gas_used) SILKWORM_NOEXCEPT { + if (!handle) { + return SILKWORM_INVALID_HANDLE; + } + + if (!txn) { + return SILKWORM_INVALID_MDBX_TXN; + } + + if (block_num == 0) { + return SILKWORM_INVALID_BLOCK; + } + + if (tx_index == 0) { + return SILKWORM_INVALID_BLOCK; + } + + if (gas_used) { + *gas_used = 1; + } + + if (blob_gas_used) { + *blob_gas_used = 1; + } + + // const auto chain_info = kKnownChainConfigs.find(chain_id); + const auto chain_info = kKnownChainConfigs.find(1337); + if (!chain_info) { + return SILKWORM_UNKNOWN_CHAIN_ID; + } + + const auto chain_config = *chain_info; + + // + grpc::ChannelArguments channel_args; + // Allow to receive messages up to specified max size + channel_args.SetMaxReceiveMessageSize(64 * 1024 * 1024); + // Allow each client to open its own TCP connection to server (sharing one single connection becomes a bottleneck under high load) + channel_args.SetInt(GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL, 1); + auto grpc_channel = grpc::CreateCustomChannel("localhost:9090", grpc::InsecureChannelCredentials(), channel_args); + + silkworm::rpc::ClientContextPool context_pool{1}; + + auto& context = context_pool.next_context(); + auto& ioc = *context.ioc(); + auto& grpc_context{*context.grpc_context()}; + + auto state_cache{std::make_unique(db::kv::api::CoherentCacheConfig{})}; + + auto backend{std::make_unique(ioc, grpc_channel, grpc_context)}; + auto database = std::make_unique(backend.get(), state_cache.get(), grpc_context, grpc_channel); + + context_pool.start(); + auto _ = gsl::finally([&context_pool] { + context_pool.stop(); + context_pool.join(); + }); + + Block block{}; //todo: get block header + + auto state = concurrency::spawn_future_and_wait(ioc, [&]() -> Task> { + auto kv_transaction = co_await database->begin(); + auto chain_storage = kv_transaction->create_storage(); + auto this_executor = co_await boost::asio::this_coro::executor; + co_return kv_transaction->create_state(this_executor, *chain_storage, block.header.number); + }); + + auto protocol_rule_set_{protocol::rule_set_factory(*chain_config)}; + ExecutionProcessor processor{block, *protocol_rule_set_, *state, *chain_config}; + //add analysis cache, check block exec for more + + silkworm::Transaction transaction{}; //todo: get txn + silkworm::Receipt receipt{}; + const ValidationResult err{protocol::validate_transaction(transaction, processor.intra_block_state(), processor.available_gas())}; + if (err != ValidationResult::kOk) { + return SILKWORM_INVALID_BLOCK; + } + processor.execute_transaction(transaction, receipt); + + return SILKWORM_OK; +} + SILKWORM_EXPORT int silkworm_fini(SilkwormHandle handle) SILKWORM_NOEXCEPT { if (!handle) { return SILKWORM_INVALID_HANDLE; diff --git a/silkworm/capi/silkworm.h b/silkworm/capi/silkworm.h index 679e267bfb..5125cc584f 100644 --- a/silkworm/capi/silkworm.h +++ b/silkworm/capi/silkworm.h @@ -358,6 +358,8 @@ SILKWORM_EXPORT int silkworm_execute_blocks_perpetual(SilkwormHandle handle, MDB bool write_change_sets, bool write_receipts, bool write_call_traces, uint64_t* last_executed_block, int* mdbx_error_code) SILKWORM_NOEXCEPT; +SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* txn, uint64_t block_num, uint64_t tx_index, uint64_t* gas_used, uint64_t* blob_gas_used) SILKWORM_NOEXCEPT; + /** * \brief Finalize the Silkworm C API library. * \param[in] handle A valid Silkworm instance handle got with silkworm_init. From e000cfc41a891c4e6e987bb1cbfd3c1062dc0c61 Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Mon, 9 Dec 2024 12:04:22 +0100 Subject: [PATCH 02/33] Update remote state --- silkworm/capi/silkworm.cpp | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/silkworm/capi/silkworm.cpp b/silkworm/capi/silkworm.cpp index 06f328750d..d1b200b2d9 100644 --- a/silkworm/capi/silkworm.cpp +++ b/silkworm/capi/silkworm.cpp @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -54,6 +55,7 @@ #include #include #include +#include #include "common.hpp" #include "instance.hpp" @@ -732,7 +734,7 @@ int silkworm_execute_blocks_perpetual(SilkwormHandle handle, MDBX_env* mdbx_env, } } -//todo: add available gas, add txn, add block header +// todo: add available gas, add txn, add block header SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* txn, uint64_t block_num, uint64_t tx_index, uint64_t* gas_used, uint64_t* blob_gas_used) SILKWORM_NOEXCEPT { if (!handle) { return SILKWORM_INVALID_HANDLE; @@ -791,20 +793,23 @@ SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* txn, ui context_pool.join(); }); - Block block{}; //todo: get block header + Block block{}; // todo: get block header auto state = concurrency::spawn_future_and_wait(ioc, [&]() -> Task> { auto kv_transaction = co_await database->begin(); - auto chain_storage = kv_transaction->create_storage(); + const auto chain_storage = kv_transaction->create_storage(); auto this_executor = co_await boost::asio::this_coro::executor; - co_return kv_transaction->create_state(this_executor, *chain_storage, block.header.number); + auto state = std::make_unique(this_executor, *kv_transaction, *chain_storage, 1, std::nullopt); + co_return state; + // co_return execution::StateFactory{*kv_transaction}.create_state(this_executor, *chain_storage, block.header.number); + // co_return kv_transaction->create_state(this_executor, *chain_storage, block.header.number); }); auto protocol_rule_set_{protocol::rule_set_factory(*chain_config)}; ExecutionProcessor processor{block, *protocol_rule_set_, *state, *chain_config}; - //add analysis cache, check block exec for more + // add analysis cache, check block exec for more - silkworm::Transaction transaction{}; //todo: get txn + silkworm::Transaction transaction{}; // todo: get txn silkworm::Receipt receipt{}; const ValidationResult err{protocol::validate_transaction(transaction, processor.intra_block_state(), processor.available_gas())}; if (err != ValidationResult::kOk) { From 3ff8c24e83a22b5df1a3ad6b648775134e56c9b0 Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Tue, 10 Dec 2024 12:45:50 +0100 Subject: [PATCH 03/33] Test run --- silkworm/capi/silkworm.cpp | 50 +++++++++++++------ silkworm/db/chain/local_chain_storage.cpp | 1 + silkworm/db/chain/remote_chain_storage.cpp | 4 ++ .../db/kv/grpc/client/remote_transaction.cpp | 1 + silkworm/db/kv/state_reader.cpp | 18 +++++++ silkworm/db/kv/txn_num.cpp | 11 ++++ silkworm/execution/remote_state.cpp | 3 ++ silkworm/rpc/ethbackend/remote_backend.cpp | 2 + silkworm/rpc/ethdb/kv/backend_providers.hpp | 1 + 9 files changed, 76 insertions(+), 15 deletions(-) diff --git a/silkworm/capi/silkworm.cpp b/silkworm/capi/silkworm.cpp index d1b200b2d9..e7a930b44c 100644 --- a/silkworm/capi/silkworm.cpp +++ b/silkworm/capi/silkworm.cpp @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -55,7 +56,6 @@ #include #include #include -#include #include "common.hpp" #include "instance.hpp" @@ -736,6 +736,7 @@ int silkworm_execute_blocks_perpetual(SilkwormHandle handle, MDBX_env* mdbx_env, // todo: add available gas, add txn, add block header SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* txn, uint64_t block_num, uint64_t tx_index, uint64_t* gas_used, uint64_t* blob_gas_used) SILKWORM_NOEXCEPT { + log::Info{"silkworm_execute_tx", {"block_num", std::to_string(block_num), "tx_index", std::to_string(tx_index)}}; if (!handle) { return SILKWORM_INVALID_HANDLE; } @@ -761,13 +762,11 @@ SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* txn, ui } // const auto chain_info = kKnownChainConfigs.find(chain_id); - const auto chain_info = kKnownChainConfigs.find(1337); + const auto chain_info = kKnownChainConfigs.find(1); if (!chain_info) { return SILKWORM_UNKNOWN_CHAIN_ID; } - const auto chain_config = *chain_info; - // grpc::ChannelArguments channel_args; // Allow to receive messages up to specified max size @@ -805,17 +804,38 @@ SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* txn, ui // co_return kv_transaction->create_state(this_executor, *chain_storage, block.header.number); }); - auto protocol_rule_set_{protocol::rule_set_factory(*chain_config)}; - ExecutionProcessor processor{block, *protocol_rule_set_, *state, *chain_config}; - // add analysis cache, check block exec for more - - silkworm::Transaction transaction{}; // todo: get txn - silkworm::Receipt receipt{}; - const ValidationResult err{protocol::validate_transaction(transaction, processor.intra_block_state(), processor.available_gas())}; - if (err != ValidationResult::kOk) { - return SILKWORM_INVALID_BLOCK; - } - processor.execute_transaction(transaction, receipt); + // Hash h = evmc::bytes32{*from_hex("0x0a67937dc901730147bae17de95ae57cf20768933f0ea545e52c78af9a22c1edc")}; + + // auto h = evmc::bytes32{*from_hex("0xa67937dc901730147bae17de95ae57cf20768933f0ea545e52c78af9a22c1edc")} + // auto h2 = to_bytes32(*from_hex("0xa67937dc901730147bae17de95ae57cf20768933f0ea545e52c78af9a22c1edc")); + + // auto header = state->read_header(1, h2); + // if (header) { + // log::Info{"header", {"number", std::to_string(header->number), "gas_used", std::to_string(header->gas_used)}}; + // } + // else { + // return SILKWORM_INVALID_BLOCK; + // } + + auto a = hex_to_address("0x71562b71999873db5b286df957af199ec94617f7"); + auto acc = state->read_account(a); + + if (acc) { + log::Info{"account", {"balance", std::to_string(acc->balance.num_bits)}}; } + + log::Info{"dupa blada, spadam"}; + // const auto chain_config = *chain_info; + // auto protocol_rule_set_{protocol::rule_set_factory(*chain_config)}; + // ExecutionProcessor processor{block, *protocol_rule_set_, *state, *chain_config}; + // // add analysis cache, check block exec for more + + // silkworm::Transaction transaction{}; // todo: get txn + // silkworm::Receipt receipt{}; + // const ValidationResult err{protocol::validate_transaction(transaction, processor.intra_block_state(), processor.available_gas())}; + // if (err != ValidationResult::kOk) { + // return SILKWORM_INVALID_BLOCK; + // } + // processor.execute_transaction(transaction, receipt); return SILKWORM_OK; } diff --git a/silkworm/db/chain/local_chain_storage.cpp b/silkworm/db/chain/local_chain_storage.cpp index b650854d23..35badccc05 100644 --- a/silkworm/db/chain/local_chain_storage.cpp +++ b/silkworm/db/chain/local_chain_storage.cpp @@ -64,6 +64,7 @@ Task> LocalChainStorage::read_header(BlockNum block_n } Task> LocalChainStorage::read_header(BlockNum block_num, const Hash& hash) const { + SILK_DEBUG << "LocalChainStorage::read_header: " << " number: " << block_num; co_return data_model_.read_header(block_num, hash); } diff --git a/silkworm/db/chain/remote_chain_storage.cpp b/silkworm/db/chain/remote_chain_storage.cpp index f89b96cbea..816a6cf429 100644 --- a/silkworm/db/chain/remote_chain_storage.cpp +++ b/silkworm/db/chain/remote_chain_storage.cpp @@ -90,18 +90,22 @@ Task> RemoteChainStorage::read_header(BlockNum block_ const bool success = co_await providers_.block(block_num, hash, /*.read_senders=*/false, block); std::optional header; if (success) { + SILK_DEBUG << "RemoteChainStorage::read_header: success read " << " number: " << block_num; header = std::move(block.header); } co_return header; } Task> RemoteChainStorage::read_header(BlockNum block_num, const Hash& hash) const { + SILK_DEBUG << "RemoteChainStorage::read_header: " << " number: " << block_num; co_return co_await read_header(block_num, hash.bytes); } Task> RemoteChainStorage::read_header(const Hash& hash) const { const auto block_num = co_await providers_.block_num_from_hash(hash.bytes); if (!block_num) { + SILK_DEBUG << "RemoteChainStorage::read_header: " << " failed to get block number: " << *block_num; + co_return std::nullopt; } SILK_DEBUG << "RemoteChainStorage::read_header: " << silkworm::to_hex(hash) << " number: " << *block_num; diff --git a/silkworm/db/kv/grpc/client/remote_transaction.cpp b/silkworm/db/kv/grpc/client/remote_transaction.cpp index e4b5f4da14..ccd0223534 100644 --- a/silkworm/db/kv/grpc/client/remote_transaction.cpp +++ b/silkworm/db/kv/grpc/client/remote_transaction.cpp @@ -113,6 +113,7 @@ std::shared_ptr RemoteTransaction::create_storage() { } Task RemoteTransaction::first_txn_num_in_block(BlockNum block_num) { + SILK_INFO << "RemoteTransaction::first_txn_num_in_block block_num=" << block_num; const auto min_txn_num = co_await txn::min_tx_num(*this, block_num, providers_.canonical_body_for_storage); co_return min_txn_num + /*txn_index*/ 0; } diff --git a/silkworm/db/kv/state_reader.cpp b/silkworm/db/kv/state_reader.cpp index e6831404bf..c9671c30ab 100644 --- a/silkworm/db/kv/state_reader.cpp +++ b/silkworm/db/kv/state_reader.cpp @@ -16,6 +16,8 @@ #include "state_reader.hpp" +#include + #include #include #include @@ -23,6 +25,7 @@ #include #include #include +#include namespace silkworm::db::kv { @@ -32,19 +35,34 @@ StateReader::StateReader(kv::api::Transaction& tx, std::optional block Task> StateReader::read_account(const evmc::address& address) const { if (!txn_number_) { + SILK_DEBUG << "test 1"; + SILK_DEBUG << "StateReader::read_account: " << " first_txn_num_in_block " << *block_num_; + SILK_DEBUG << "test 1"; + if (tx_.is_local()) { + SILK_DEBUG << "StateReader::read_account local: tx_ type: "; + } else { + SILK_DEBUG << "StateReader::read_account: tx_ type: "; + } + txn_number_ = co_await tx_.first_txn_num_in_block(*block_num_); } + SILK_DEBUG << "StateReader::read_account: " << " query"; db::kv::api::GetAsOfQuery query{ .table = table::kAccountDomain, .key = db::account_domain_key(address), .timestamp = static_cast(*txn_number_), }; + SILK_DEBUG << "StateReader::read_account: " << " query2"; const auto result = co_await tx_.get_as_of(std::move(query)); + SILK_DEBUG << "StateReader::read_account: " << " get_as_of"; + if (!result.success) { + SILK_DEBUG << "StateReader::read_account: " << " no success"; co_return std::nullopt; } + SILK_DEBUG << "StateReader::read_account: " << " success" << to_hex(result.value); const auto account{Account::from_encoded_storage_v3(result.value)}; success_or_throw(account); co_return *account; diff --git a/silkworm/db/kv/txn_num.cpp b/silkworm/db/kv/txn_num.cpp index 33a5682658..a4d0c3bf43 100644 --- a/silkworm/db/kv/txn_num.cpp +++ b/silkworm/db/kv/txn_num.cpp @@ -25,6 +25,7 @@ #include #include "../tables.hpp" +#include namespace silkworm::db::txn { @@ -85,16 +86,26 @@ Task min_tx_num(Transaction& tx, BlockNum block_num, chain::CanonicalBody if (block_num == 0) { co_return 0; } + SILK_DEBUG << "min_tx_num 1"; const auto max_tx_num_cursor = co_await tx.cursor(table::kMaxTxNumName); + SILK_DEBUG << "min_tx_num 2"; const std::optional last_tx_num = co_await last_tx_num_for_block(max_tx_num_cursor, (block_num - 1), provider); + SILK_DEBUG << "min_tx_num 3"; if (!last_tx_num) { + SILK_DEBUG << "min_tx_num 4"; const KeyValue key_value = co_await max_tx_num_cursor->last(); + SILK_DEBUG << "min_tx_num 5"; if (key_value.value.empty()) { + SILK_DEBUG << "min_tx_num 6"; co_return 0; } if (key_value.value.size() != sizeof(TxNum)) { + SILK_DEBUG << "min_tx_num 7"; + throw std::length_error("Bad TxNum value size " + std::to_string(key_value.value.size()) + " in db"); } + SILK_DEBUG << "min_tx_num 8"; + co_return endian::load_big_u64(key_value.value.data()); } co_return *last_tx_num + 1; diff --git a/silkworm/execution/remote_state.cpp b/silkworm/execution/remote_state.cpp index bde1a1b97e..56f8b40a2c 100644 --- a/silkworm/execution/remote_state.cpp +++ b/silkworm/execution/remote_state.cpp @@ -54,6 +54,8 @@ Task AsyncRemoteState::previous_incarnation(const evmc::address& /*add } Task> AsyncRemoteState::read_header(BlockNum block_num, const evmc::bytes32& block_hash) const noexcept { + SILK_DEBUG << "AsyncRemoteState::read_header: " << " number: " << block_num; + co_return co_await storage_.read_header(block_num, block_hash); } @@ -83,6 +85,7 @@ std::optional RemoteState::read_account(const evmc::address& address) c SILK_DEBUG << "RemoteState::read_account address=" << address << " start"; try { std::future> result{boost::asio::co_spawn(executor_, async_state_.read_account(address), boost::asio::use_future)}; + SILK_DEBUG << "RemoteState::read_account address=" << address << " spawned"; const auto optional_account{result.get()}; SILK_DEBUG << "RemoteState::read_account account.nonce=" << (optional_account ? optional_account->nonce : 0) << " end"; return optional_account; diff --git a/silkworm/rpc/ethbackend/remote_backend.cpp b/silkworm/rpc/ethbackend/remote_backend.cpp index 28a44b96c2..97de34091d 100644 --- a/silkworm/rpc/ethbackend/remote_backend.cpp +++ b/silkworm/rpc/ethbackend/remote_backend.cpp @@ -151,7 +151,9 @@ Task RemoteBackEnd::get_block(BlockNum block_num, const HashAsSpan& hash, request.set_allocated_block_hash(h256_from_bytes(hash).release()); const auto reply = co_await get_block_rpc.finish_on(executor_, request); ByteView block_rlp{string_view_to_byte_view(reply.block_rlp())}; + SILK_DEBUG << "RemoteBackEnd::get_block block_num=" << block_num << " reply=" << reply.DebugString(); if (const auto decode_result{rlp::decode(block_rlp, block)}; !decode_result) { + SILK_ERROR << "RemoteBackEnd::get_block block_num=" << block_num << " decode error=" << int (decode_result.error()); co_return false; } if (read_senders) { diff --git a/silkworm/rpc/ethdb/kv/backend_providers.hpp b/silkworm/rpc/ethdb/kv/backend_providers.hpp index 778d38b826..9c9ca2e2f7 100644 --- a/silkworm/rpc/ethdb/kv/backend_providers.hpp +++ b/silkworm/rpc/ethdb/kv/backend_providers.hpp @@ -23,6 +23,7 @@ namespace silkworm::rpc::ethdb::kv { inline db::chain::BlockProvider block_provider(ethbackend::BackEnd* backend) { return [backend](auto block_num, HashAsSpan hash, bool read_senders, auto& block) -> Task { + SILK_DEBUG << "db::chain::BlockProvider::block_provider: " << " number: " << block_num; co_return co_await backend->get_block(block_num, hash, read_senders, block); }; } From 3e21459dd5e6a1bffcad7bb5ab9aef950a1a1895 Mon Sep 17 00:00:00 2001 From: Bartosz Zawistowski Date: Tue, 10 Dec 2024 12:58:26 +0100 Subject: [PATCH 04/33] Make code compile --- silkworm/capi/silkworm.cpp | 13 +++++++------ silkworm/capi/silkworm.h | 4 ++-- silkworm/db/chain/local_chain_storage.cpp | 3 ++- silkworm/db/chain/remote_chain_storage.cpp | 9 ++++++--- silkworm/db/kv/state_reader.cpp | 18 ++++++++++++------ silkworm/db/kv/txn_num.cpp | 2 +- silkworm/execution/remote_state.cpp | 3 ++- silkworm/rpc/ethbackend/remote_backend.cpp | 2 +- silkworm/rpc/ethdb/kv/backend_providers.hpp | 3 ++- 9 files changed, 35 insertions(+), 22 deletions(-) diff --git a/silkworm/capi/silkworm.cpp b/silkworm/capi/silkworm.cpp index e7a930b44c..121b060f96 100644 --- a/silkworm/capi/silkworm.cpp +++ b/silkworm/capi/silkworm.cpp @@ -773,7 +773,7 @@ SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* txn, ui channel_args.SetMaxReceiveMessageSize(64 * 1024 * 1024); // Allow each client to open its own TCP connection to server (sharing one single connection becomes a bottleneck under high load) channel_args.SetInt(GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL, 1); - auto grpc_channel = grpc::CreateCustomChannel("localhost:9090", grpc::InsecureChannelCredentials(), channel_args); + auto grpc_erigon_channel = grpc::CreateCustomChannel("localhost:9090", grpc::InsecureChannelCredentials(), channel_args); silkworm::rpc::ClientContextPool context_pool{1}; @@ -783,8 +783,8 @@ SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* txn, ui auto state_cache{std::make_unique(db::kv::api::CoherentCacheConfig{})}; - auto backend{std::make_unique(ioc, grpc_channel, grpc_context)}; - auto database = std::make_unique(backend.get(), state_cache.get(), grpc_context, grpc_channel); + auto backend{std::make_unique(ioc, grpc_erigon_channel, grpc_context)}; + auto database = std::make_unique(backend.get(), state_cache.get(), grpc_context, grpc_erigon_channel); context_pool.start(); auto _ = gsl::finally([&context_pool] { @@ -798,8 +798,8 @@ SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* txn, ui auto kv_transaction = co_await database->begin(); const auto chain_storage = kv_transaction->create_storage(); auto this_executor = co_await boost::asio::this_coro::executor; - auto state = std::make_unique(this_executor, *kv_transaction, *chain_storage, 1, std::nullopt); - co_return state; + auto remote_state = std::make_unique(this_executor, *kv_transaction, *chain_storage, 1, std::nullopt); + co_return remote_state; // co_return execution::StateFactory{*kv_transaction}.create_state(this_executor, *chain_storage, block.header.number); // co_return kv_transaction->create_state(this_executor, *chain_storage, block.header.number); }); @@ -821,7 +821,8 @@ SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* txn, ui auto acc = state->read_account(a); if (acc) { - log::Info{"account", {"balance", std::to_string(acc->balance.num_bits)}}; } + log::Info{"account", {"balance", std::to_string(acc->balance.num_bits)}}; + } log::Info{"dupa blada, spadam"}; // const auto chain_config = *chain_info; diff --git a/silkworm/capi/silkworm.h b/silkworm/capi/silkworm.h index 5125cc584f..e97365e598 100644 --- a/silkworm/capi/silkworm.h +++ b/silkworm/capi/silkworm.h @@ -248,8 +248,8 @@ SILKWORM_EXPORT int silkworm_sentry_stop(SilkwormHandle handle) SILKWORM_NOEXCEP //! Silkworm Fork Validator configuration options struct SilkwormForkValidatorSettings { - size_t batch_size; // Batch size to use in stages - size_t etl_buffer_size; // Buffer size for ETL operations + uint64_t batch_size; // Batch size to use in stages + uint64_t etl_buffer_size; // Buffer size for ETL operations uint32_t sync_loop_throttle_seconds; // Minimum interval amongst sync cycle bool stop_before_senders_stage; // Stop before senders stage }; diff --git a/silkworm/db/chain/local_chain_storage.cpp b/silkworm/db/chain/local_chain_storage.cpp index 35badccc05..bb61c3d8a5 100644 --- a/silkworm/db/chain/local_chain_storage.cpp +++ b/silkworm/db/chain/local_chain_storage.cpp @@ -64,7 +64,8 @@ Task> LocalChainStorage::read_header(BlockNum block_n } Task> LocalChainStorage::read_header(BlockNum block_num, const Hash& hash) const { - SILK_DEBUG << "LocalChainStorage::read_header: " << " number: " << block_num; + SILK_DEBUG << "LocalChainStorage::read_header: " + << " number: " << block_num; co_return data_model_.read_header(block_num, hash); } diff --git a/silkworm/db/chain/remote_chain_storage.cpp b/silkworm/db/chain/remote_chain_storage.cpp index 816a6cf429..7e9eae5349 100644 --- a/silkworm/db/chain/remote_chain_storage.cpp +++ b/silkworm/db/chain/remote_chain_storage.cpp @@ -90,21 +90,24 @@ Task> RemoteChainStorage::read_header(BlockNum block_ const bool success = co_await providers_.block(block_num, hash, /*.read_senders=*/false, block); std::optional header; if (success) { - SILK_DEBUG << "RemoteChainStorage::read_header: success read " << " number: " << block_num; + SILK_DEBUG << "RemoteChainStorage::read_header: success read " + << " number: " << block_num; header = std::move(block.header); } co_return header; } Task> RemoteChainStorage::read_header(BlockNum block_num, const Hash& hash) const { - SILK_DEBUG << "RemoteChainStorage::read_header: " << " number: " << block_num; + SILK_DEBUG << "RemoteChainStorage::read_header: " + << " number: " << block_num; co_return co_await read_header(block_num, hash.bytes); } Task> RemoteChainStorage::read_header(const Hash& hash) const { const auto block_num = co_await providers_.block_num_from_hash(hash.bytes); if (!block_num) { - SILK_DEBUG << "RemoteChainStorage::read_header: " << " failed to get block number: " << *block_num; + SILK_DEBUG << "RemoteChainStorage::read_header: " + << " failed to get block number: " << *block_num; co_return std::nullopt; } diff --git a/silkworm/db/kv/state_reader.cpp b/silkworm/db/kv/state_reader.cpp index c9671c30ab..41199453a1 100644 --- a/silkworm/db/kv/state_reader.cpp +++ b/silkworm/db/kv/state_reader.cpp @@ -36,7 +36,8 @@ StateReader::StateReader(kv::api::Transaction& tx, std::optional block Task> StateReader::read_account(const evmc::address& address) const { if (!txn_number_) { SILK_DEBUG << "test 1"; - SILK_DEBUG << "StateReader::read_account: " << " first_txn_num_in_block " << *block_num_; + SILK_DEBUG << "StateReader::read_account: " + << " first_txn_num_in_block " << *block_num_; SILK_DEBUG << "test 1"; if (tx_.is_local()) { SILK_DEBUG << "StateReader::read_account local: tx_ type: "; @@ -47,22 +48,27 @@ Task> StateReader::read_account(const evmc::address& addr txn_number_ = co_await tx_.first_txn_num_in_block(*block_num_); } - SILK_DEBUG << "StateReader::read_account: " << " query"; + SILK_DEBUG << "StateReader::read_account: " + << " query"; db::kv::api::GetAsOfQuery query{ .table = table::kAccountDomain, .key = db::account_domain_key(address), .timestamp = static_cast(*txn_number_), }; - SILK_DEBUG << "StateReader::read_account: " << " query2"; + SILK_DEBUG << "StateReader::read_account: " + << " query2"; const auto result = co_await tx_.get_as_of(std::move(query)); - SILK_DEBUG << "StateReader::read_account: " << " get_as_of"; + SILK_DEBUG << "StateReader::read_account: " + << " get_as_of"; if (!result.success) { - SILK_DEBUG << "StateReader::read_account: " << " no success"; + SILK_DEBUG << "StateReader::read_account: " + << " no success"; co_return std::nullopt; } - SILK_DEBUG << "StateReader::read_account: " << " success" << to_hex(result.value); + SILK_DEBUG << "StateReader::read_account: " + << " success" << to_hex(result.value); const auto account{Account::from_encoded_storage_v3(result.value)}; success_or_throw(account); co_return *account; diff --git a/silkworm/db/kv/txn_num.cpp b/silkworm/db/kv/txn_num.cpp index a4d0c3bf43..a3e27f4d7b 100644 --- a/silkworm/db/kv/txn_num.cpp +++ b/silkworm/db/kv/txn_num.cpp @@ -23,9 +23,9 @@ #include #include #include +#include #include "../tables.hpp" -#include namespace silkworm::db::txn { diff --git a/silkworm/execution/remote_state.cpp b/silkworm/execution/remote_state.cpp index 56f8b40a2c..2d9ba2ca14 100644 --- a/silkworm/execution/remote_state.cpp +++ b/silkworm/execution/remote_state.cpp @@ -54,7 +54,8 @@ Task AsyncRemoteState::previous_incarnation(const evmc::address& /*add } Task> AsyncRemoteState::read_header(BlockNum block_num, const evmc::bytes32& block_hash) const noexcept { - SILK_DEBUG << "AsyncRemoteState::read_header: " << " number: " << block_num; + SILK_DEBUG << "AsyncRemoteState::read_header: " + << " number: " << block_num; co_return co_await storage_.read_header(block_num, block_hash); } diff --git a/silkworm/rpc/ethbackend/remote_backend.cpp b/silkworm/rpc/ethbackend/remote_backend.cpp index 97de34091d..3c4d25c201 100644 --- a/silkworm/rpc/ethbackend/remote_backend.cpp +++ b/silkworm/rpc/ethbackend/remote_backend.cpp @@ -153,7 +153,7 @@ Task RemoteBackEnd::get_block(BlockNum block_num, const HashAsSpan& hash, ByteView block_rlp{string_view_to_byte_view(reply.block_rlp())}; SILK_DEBUG << "RemoteBackEnd::get_block block_num=" << block_num << " reply=" << reply.DebugString(); if (const auto decode_result{rlp::decode(block_rlp, block)}; !decode_result) { - SILK_ERROR << "RemoteBackEnd::get_block block_num=" << block_num << " decode error=" << int (decode_result.error()); + SILK_ERROR << "RemoteBackEnd::get_block block_num=" << block_num << " decode error=" << int(decode_result.error()); co_return false; } if (read_senders) { diff --git a/silkworm/rpc/ethdb/kv/backend_providers.hpp b/silkworm/rpc/ethdb/kv/backend_providers.hpp index 9c9ca2e2f7..c088ae5207 100644 --- a/silkworm/rpc/ethdb/kv/backend_providers.hpp +++ b/silkworm/rpc/ethdb/kv/backend_providers.hpp @@ -23,7 +23,8 @@ namespace silkworm::rpc::ethdb::kv { inline db::chain::BlockProvider block_provider(ethbackend::BackEnd* backend) { return [backend](auto block_num, HashAsSpan hash, bool read_senders, auto& block) -> Task { - SILK_DEBUG << "db::chain::BlockProvider::block_provider: " << " number: " << block_num; + SILK_DEBUG << "db::chain::BlockProvider::block_provider: " + << " number: " << block_num; co_return co_await backend->get_block(block_num, hash, read_senders, block); }; } From 6102617c26ce4ea52551ef31b158c586e9869762 Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Tue, 10 Dec 2024 16:28:47 +0100 Subject: [PATCH 05/33] Update mdbx --- third_party/erigon-mdbx-go/mdbx-go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/erigon-mdbx-go/mdbx-go b/third_party/erigon-mdbx-go/mdbx-go index 839cb7b1a9..aab789d157 160000 --- a/third_party/erigon-mdbx-go/mdbx-go +++ b/third_party/erigon-mdbx-go/mdbx-go @@ -1 +1 @@ -Subproject commit 839cb7b1a9a5e6f8cff751ec1b25814d24a7ab38 +Subproject commit aab789d15716460c36950ddb3b9f3b7a7c4c8089 From e5d602ca6aa4914db6a531edbc2f781b5d08159e Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Mon, 6 Jan 2025 14:02:58 +0100 Subject: [PATCH 06/33] Update single_tx --- silkworm/capi/silkworm.cpp | 36 ++++++++++++++++++++++----------- silkworm/capi/silkworm_test.cpp | 14 +++++++++++++ 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/silkworm/capi/silkworm.cpp b/silkworm/capi/silkworm.cpp index 121b060f96..bf3be96e74 100644 --- a/silkworm/capi/silkworm.cpp +++ b/silkworm/capi/silkworm.cpp @@ -799,29 +799,41 @@ SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* txn, ui const auto chain_storage = kv_transaction->create_storage(); auto this_executor = co_await boost::asio::this_coro::executor; auto remote_state = std::make_unique(this_executor, *kv_transaction, *chain_storage, 1, std::nullopt); + + // auto a = hex_to_address("0x71562b71999873db5b286df957af199ec94617f7"); + // auto acc = remote_state->read_account(a); + + // if (acc) { + // log::Info{"account", {"balance", std::to_string(acc->balance.num_bits)}}; + // } else { + // log::Info{"account not found"}; + // } + co_return remote_state; // co_return execution::StateFactory{*kv_transaction}.create_state(this_executor, *chain_storage, block.header.number); // co_return kv_transaction->create_state(this_executor, *chain_storage, block.header.number); }); - // Hash h = evmc::bytes32{*from_hex("0x0a67937dc901730147bae17de95ae57cf20768933f0ea545e52c78af9a22c1edc")}; + // auto b_hash = to_bytes32(*from_hex("0xe8b28e4882bcbd6293ef56433c69b34e9e3e5bf512a05ddbed6bb94aa65948f4")); + // auto b_number = BlockNum{2910651}; - // auto h = evmc::bytes32{*from_hex("0xa67937dc901730147bae17de95ae57cf20768933f0ea545e52c78af9a22c1edc")} - // auto h2 = to_bytes32(*from_hex("0xa67937dc901730147bae17de95ae57cf20768933f0ea545e52c78af9a22c1edc")); + auto b_hash = to_bytes32(*from_hex("0x6f81fc8bb897eb2075ffa53a2b28c3216022fd1841d361af7908198f8ff2faa3")); + auto b_number = BlockNum{1}; - // auto header = state->read_header(1, h2); - // if (header) { - // log::Info{"header", {"number", std::to_string(header->number), "gas_used", std::to_string(header->gas_used)}}; - // } - // else { - // return SILKWORM_INVALID_BLOCK; - // } + auto header = state->read_header(b_number, b_hash); + if (header) { + log::Info{"JG header", {"number", std::to_string(header->number), "gas_used", std::to_string(header->gas_used)}}; + } else { + log::Warning{"header not found"}; + return SILKWORM_INVALID_BLOCK; + } auto a = hex_to_address("0x71562b71999873db5b286df957af199ec94617f7"); auto acc = state->read_account(a); - if (acc) { - log::Info{"account", {"balance", std::to_string(acc->balance.num_bits)}}; + log::Info{"account2", {"balance", std::to_string(acc->balance.num_bits)}}; + } else { + log::Info{"account2 not found"}; } log::Info{"dupa blada, spadam"}; diff --git a/silkworm/capi/silkworm_test.cpp b/silkworm/capi/silkworm_test.cpp index 3e2e7744b8..2d16c302e4 100644 --- a/silkworm/capi/silkworm_test.cpp +++ b/silkworm/capi/silkworm_test.cpp @@ -178,6 +178,10 @@ struct SilkwormLibrary { return result; } + int exectute_single_tx(MDBX_txn* txn) const { + return silkworm_execute_tx(handle_, txn, 1, 1, nullptr, nullptr); + } + int add_snapshot(SilkwormChainSnapshot* snapshot) const { return silkworm_add_snapshot(handle_, snapshot); } @@ -987,6 +991,16 @@ static SilkwormRpcSettings make_rpc_settings_for_test(uint16_t api_listening_por return settings; } +TEST_CASE_METHOD(CApiTest, "CAPI silkworm_tx: single", "[silkworm][capi]") { + // Use Silkworm as a library with silkworm_init/silkworm_fini automated by RAII + SilkwormLibrary silkworm_lib{env_path()}; + + RWTxnManaged external_txn{env}; + auto result = silkworm_lib.exectute_single_tx(*external_txn); + CHECK_NOTHROW(external_txn.commit_and_stop()); + CHECK(result == SILKWORM_OK); +} + static const SilkwormRpcSettings kInvalidRpcSettings{make_rpc_settings_for_test(10)}; static const SilkwormRpcSettings kValidRpcSettings{make_rpc_settings_for_test(8545)}; From 5d30a9b8ae935fd3d2802fa8c2bf2580f9c38ef7 Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Wed, 15 Jan 2025 13:21:42 +0100 Subject: [PATCH 07/33] disable domain accessor index --- silkworm/db/datastore/snapshots/domain.hpp | 2 +- silkworm/db/datastore/snapshots/schema.cpp | 12 ++++++++---- silkworm/db/datastore/snapshots/schema.hpp | 2 ++ silkworm/db/datastore/snapshots/snapshot_bundle.cpp | 5 ++++- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/silkworm/db/datastore/snapshots/domain.hpp b/silkworm/db/datastore/snapshots/domain.hpp index 4cd1abb7dc..0ab8b40af8 100644 --- a/silkworm/db/datastore/snapshots/domain.hpp +++ b/silkworm/db/datastore/snapshots/domain.hpp @@ -28,7 +28,7 @@ namespace silkworm::snapshots { struct Domain { const segment::KVSegmentFileReader& kv_segment; - const rec_split::AccessorIndex& accessor_index; + const rec_split::AccessorIndex* accessor_index{nullptr}; const bloom_filter::BloomFilter& existence_index; const btree::BTreeIndex& btree_index; std::optional history; diff --git a/silkworm/db/datastore/snapshots/schema.cpp b/silkworm/db/datastore/snapshots/schema.cpp index 54f36323a3..21fc5a3f6d 100644 --- a/silkworm/db/datastore/snapshots/schema.cpp +++ b/silkworm/db/datastore/snapshots/schema.cpp @@ -83,10 +83,6 @@ Schema::DomainDef Schema::RepositoryDef::make_domain_schema(datastore::EntityNam .sub_dir_name(kDomainKVSegmentSubDirName) .tag(name2tag(name)) .file_ext(kDomainKVSegmentFileExt); - schema.accessor_index(kDomainAccessorIndexName) - .sub_dir_name(kDomainAccessorIndexSubDirName) - .tag(name2tag(name)) - .file_ext(kDomainAccessorIndexFileExt); schema.existence_index(kDomainExistenceIndexName) .sub_dir_name(kDomainExistenceIndexSubDirName) .tag(name2tag(name)) @@ -99,6 +95,14 @@ Schema::DomainDef Schema::RepositoryDef::make_domain_schema(datastore::EntityNam return schema; } +Schema::DomainDef& Schema::DomainDef::with_accessor_index() { + accessor_index(kDomainAccessorIndexName) + .sub_dir_name(kDomainAccessorIndexSubDirName) + .tag(kv_segment(kDomainKVSegmentName).tag()) + .file_ext(kDomainAccessorIndexFileExt); + return *this; +} + Schema::EntityDef Schema::RepositoryDef::make_history_schema(datastore::EntityName name) { Schema::EntityDef schema; define_history_schema(name, schema); diff --git a/silkworm/db/datastore/snapshots/schema.hpp b/silkworm/db/datastore/snapshots/schema.hpp index 09ca15c87f..fb5aaac956 100644 --- a/silkworm/db/datastore/snapshots/schema.hpp +++ b/silkworm/db/datastore/snapshots/schema.hpp @@ -164,6 +164,8 @@ class Schema { RepositoryDef::undefine_history_schema(*this); return *this; } + + DomainDef& with_accessor_index(); }; class RepositoryDef { diff --git a/silkworm/db/datastore/snapshots/snapshot_bundle.cpp b/silkworm/db/datastore/snapshots/snapshot_bundle.cpp index 65d03bf571..5d9291f2c9 100644 --- a/silkworm/db/datastore/snapshots/snapshot_bundle.cpp +++ b/silkworm/db/datastore/snapshots/snapshot_bundle.cpp @@ -151,11 +151,14 @@ Domain SnapshotBundle::domain(datastore::EntityName name) const { auto& data = data_.entities.at(name); Domain domain{ data.kv_segments.at(Schema::kDomainKVSegmentName), - data.accessor_indexes.at(Schema::kDomainAccessorIndexName), + nullptr, data.existence_indexes.at(Schema::kDomainExistenceIndexName), data.btree_indexes.at(Schema::kDomainBTreeIndexName), std::nullopt, }; + if (data.accessor_indexes.contains(Schema::kDomainAccessorIndexName)) { + domain.accessor_index = &data.accessor_indexes.at(Schema::kDomainAccessorIndexName); + } if (data.segments.contains(Schema::kHistorySegmentName)) { domain.history.emplace(History{ data.segments.at(Schema::kHistorySegmentName), From c6f356e4f834d1349f4ca8b5cbe265f2cb22a133 Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Wed, 15 Jan 2025 13:36:48 +0100 Subject: [PATCH 08/33] remove T prefix from non-template types --- .../datastore/inverted_index_range_by_key_query.hpp | 10 +++++----- silkworm/db/datastore/kvdb/domain_delete_query.hpp | 8 ++++---- .../db/datastore/kvdb/domain_get_latest_query.hpp | 8 ++++---- .../db/datastore/kvdb/domain_put_latest_query.hpp | 6 +++--- silkworm/db/datastore/kvdb/domain_put_query.hpp | 10 +++++----- silkworm/db/datastore/kvdb/history_delete_query.hpp | 4 ++-- silkworm/db/datastore/kvdb/history_put_query.hpp | 6 +++--- .../db/datastore/kvdb/inverted_index_put_query.hpp | 6 +++--- .../kvdb/inverted_index_range_by_key_query.hpp | 8 ++++---- .../snapshots/inverted_index_range_by_key_query.hpp | 12 ++++++------ 10 files changed, 39 insertions(+), 39 deletions(-) diff --git a/silkworm/db/datastore/inverted_index_range_by_key_query.hpp b/silkworm/db/datastore/inverted_index_range_by_key_query.hpp index 312bfe172f..9a740bd40d 100644 --- a/silkworm/db/datastore/inverted_index_range_by_key_query.hpp +++ b/silkworm/db/datastore/inverted_index_range_by_key_query.hpp @@ -45,13 +45,13 @@ struct InvertedIndexRangeByKeyQuery { repository, } {} - using TKey1 = decltype(TKeyEncoder1::value); - using TKey2 = decltype(TKeyEncoder2::value); - static_assert(std::same_as); - using TKey = TKey1; + using Key1 = decltype(TKeyEncoder1::value); + using Key2 = decltype(TKeyEncoder2::value); + static_assert(std::same_as); + using Key = Key1; template - auto exec(TKey key, TimestampRange ts_range) { + auto exec(Key key, TimestampRange ts_range) { if constexpr (ascending) { return silkworm::views::concat( query2_.template exec(key, ts_range), diff --git a/silkworm/db/datastore/kvdb/domain_delete_query.hpp b/silkworm/db/datastore/kvdb/domain_delete_query.hpp index 6e296608f9..022c050acd 100644 --- a/silkworm/db/datastore/kvdb/domain_delete_query.hpp +++ b/silkworm/db/datastore/kvdb/domain_delete_query.hpp @@ -25,13 +25,13 @@ struct DomainDeleteQuery { RWTxn& tx; Domain entity; - using TKey = decltype(TKeyEncoder::value); - using TValue = decltype(TValueEncoder::value); + using Key = decltype(TKeyEncoder::value); + using Value = decltype(TValueEncoder::value); void exec( - const TKey& key, + const Key& key, Timestamp timestamp, - const std::optional& prev_value, + const std::optional& prev_value, Step prev_step) { if (prev_value) { DomainPutQuery> query{tx, entity}; diff --git a/silkworm/db/datastore/kvdb/domain_get_latest_query.hpp b/silkworm/db/datastore/kvdb/domain_get_latest_query.hpp index 757636651c..ba887e5c11 100644 --- a/silkworm/db/datastore/kvdb/domain_get_latest_query.hpp +++ b/silkworm/db/datastore/kvdb/domain_get_latest_query.hpp @@ -31,15 +31,15 @@ struct DomainGetLatestQuery { RWTxn& tx; Domain entity; - using TKey = decltype(TKeyEncoder::value); - using TValue = decltype(TValueDecoder::value); + using Key = decltype(TKeyEncoder::value); + using Value = decltype(TValueDecoder::value); struct Result { - TValue value; + Value value; Step step; }; - std::optional exec(const TKey& key) { + std::optional exec(const Key& key) { DomainKeyEncoder key_encoder{/* has_large_values = */ false}; key_encoder.value.key.value = key; key_encoder.value.timestamp.value = Step{std::numeric_limits::max()}; diff --git a/silkworm/db/datastore/kvdb/domain_put_latest_query.hpp b/silkworm/db/datastore/kvdb/domain_put_latest_query.hpp index 8284991235..526484d06b 100644 --- a/silkworm/db/datastore/kvdb/domain_put_latest_query.hpp +++ b/silkworm/db/datastore/kvdb/domain_put_latest_query.hpp @@ -28,10 +28,10 @@ struct DomainPutLatestQuery { RWTxn& tx; Domain entity; - using TKey = decltype(TKeyEncoder::value); - using TValue = decltype(TValueEncoder::value); + using Key = decltype(TKeyEncoder::value); + using Value = decltype(TValueEncoder::value); - void exec(const TKey& key, const TValue& value, Step step) { + void exec(const Key& key, const Value& value, Step step) { DomainKeyEncoder key_encoder{entity.has_large_values}; key_encoder.value.key.value = key; key_encoder.value.timestamp.value = step; diff --git a/silkworm/db/datastore/kvdb/domain_put_query.hpp b/silkworm/db/datastore/kvdb/domain_put_query.hpp index 3d355fba05..b16b0c41a3 100644 --- a/silkworm/db/datastore/kvdb/domain_put_query.hpp +++ b/silkworm/db/datastore/kvdb/domain_put_query.hpp @@ -27,14 +27,14 @@ struct DomainPutQuery { RWTxn& tx; Domain entity; - using TKey = decltype(TKeyEncoder::value); - using TValue = decltype(TValueEncoder::value); + using Key = decltype(TKeyEncoder::value); + using Value = decltype(TValueEncoder::value); void exec( - const TKey& key, - const TValue& value, + const Key& key, + const Value& value, Timestamp timestamp, - const std::optional& prev_value, + const std::optional& prev_value, Step prev_step) { DomainPutLatestQuery value_query{tx, entity}; value_query.exec(key, value, prev_step); diff --git a/silkworm/db/datastore/kvdb/history_delete_query.hpp b/silkworm/db/datastore/kvdb/history_delete_query.hpp index bf8f604902..50560b7166 100644 --- a/silkworm/db/datastore/kvdb/history_delete_query.hpp +++ b/silkworm/db/datastore/kvdb/history_delete_query.hpp @@ -25,9 +25,9 @@ struct HistoryDeleteQuery { RWTxn& tx; History entity; - using TKey = decltype(TKeyEncoder::value); + using Key = decltype(TKeyEncoder::value); - void exec(const TKey& key, Timestamp timestamp) { + void exec(const Key& key, Timestamp timestamp) { HistoryPutQuery> query{tx, entity}; query.exec(key, ByteView{}, timestamp); } diff --git a/silkworm/db/datastore/kvdb/history_put_query.hpp b/silkworm/db/datastore/kvdb/history_put_query.hpp index c727851da3..38e74ca227 100644 --- a/silkworm/db/datastore/kvdb/history_put_query.hpp +++ b/silkworm/db/datastore/kvdb/history_put_query.hpp @@ -36,10 +36,10 @@ struct HistoryPutQuery { RWTxn& tx; History entity; - using TKey = decltype(TKeyEncoder::value); - using TValue = decltype(TValueEncoder::value); + using Key = decltype(TKeyEncoder::value); + using Value = decltype(TValueEncoder::value); - void exec(const TKey& key, const TValue& value, Timestamp timestamp) { + void exec(const Key& key, const Value& value, Timestamp timestamp) { HistoryKeyEncoder key_encoder{entity.has_large_values}; key_encoder.value.key.value = key; key_encoder.value.timestamp.value = timestamp; diff --git a/silkworm/db/datastore/kvdb/inverted_index_put_query.hpp b/silkworm/db/datastore/kvdb/inverted_index_put_query.hpp index 10a26efd37..292a92e0b1 100644 --- a/silkworm/db/datastore/kvdb/inverted_index_put_query.hpp +++ b/silkworm/db/datastore/kvdb/inverted_index_put_query.hpp @@ -28,14 +28,14 @@ struct InvertedIndexPutQuery { RWTxn& tx; InvertedIndex entity; - using TKey = decltype(TKeyEncoder::value); + using Key = decltype(TKeyEncoder::value); - void exec(const TKey& key, const Timestamp timestamp, bool with_index_update) { + void exec(const Key& key, const Timestamp timestamp, bool with_index_update) { return exec(key, timestamp, with_index_update); } template - void exec(const TKey& key, const TTimestamp& timestamp, bool with_index_update) { + void exec(const Key& key, const TTimestamp& timestamp, bool with_index_update) { TKeyEncoder key_encoder; key_encoder.value = key; Slice key_slice = key_encoder.encode(); diff --git a/silkworm/db/datastore/kvdb/inverted_index_range_by_key_query.hpp b/silkworm/db/datastore/kvdb/inverted_index_range_by_key_query.hpp index 922c863017..31a27ff53b 100644 --- a/silkworm/db/datastore/kvdb/inverted_index_range_by_key_query.hpp +++ b/silkworm/db/datastore/kvdb/inverted_index_range_by_key_query.hpp @@ -34,14 +34,14 @@ struct InvertedIndexRangeByKeyQuery { ROTxn& tx; InvertedIndex entity; - using TKey = decltype(TKeyEncoder::value); + using Key = decltype(TKeyEncoder::value); //! A range of timestamps using Timestamps = std::ranges::filter_view< std::ranges::subrange>, decltype(std::declval().contains_predicate())>; - CursorValuesIterator begin(TKey key, TimestampRange ts_range, bool ascending) { + CursorValuesIterator begin(Key key, TimestampRange ts_range, bool ascending) { auto cursor = tx.ro_cursor_dup_sort(entity.index_table); TKeyEncoder key_encoder; @@ -76,13 +76,13 @@ struct InvertedIndexRangeByKeyQuery { return {}; } - Timestamps exec_with_eager_begin(TKey key, TimestampRange ts_range, bool ascending) { + Timestamps exec_with_eager_begin(Key key, TimestampRange ts_range, bool ascending) { auto begin_it = begin(std::move(key), ts_range, ascending); return std::ranges::subrange{std::move(begin_it), CursorValuesIterator{}} | std::views::filter(ts_range.contains_predicate()); } - auto exec(TKey key, TimestampRange ts_range, bool ascending) { + auto exec(Key key, TimestampRange ts_range, bool ascending) { auto exec_func = [query = *this, key = std::move(key), ts_range, ascending](std::monostate) mutable { return query.exec_with_eager_begin(std::move(key), ts_range, ascending); }; diff --git a/silkworm/db/datastore/snapshots/inverted_index_range_by_key_query.hpp b/silkworm/db/datastore/snapshots/inverted_index_range_by_key_query.hpp index 12e8dad0b0..758cdc63ef 100644 --- a/silkworm/db/datastore/snapshots/inverted_index_range_by_key_query.hpp +++ b/silkworm/db/datastore/snapshots/inverted_index_range_by_key_query.hpp @@ -46,9 +46,9 @@ struct InvertedIndexFindByKeySegmentQuery { datastore::EntityName inverted_index_name) : inverted_index_{bundle.inverted_index(inverted_index_name)} {} - using TKey = decltype(TKeyEncoder::value); + using Key = decltype(TKeyEncoder::value); - std::optional exec(TKey key) { + std::optional exec(Key key) { TKeyEncoder key_encoder; key_encoder.value = std::move(key); ByteView key_data = key_encoder.encode_word(); @@ -59,7 +59,7 @@ struct InvertedIndexFindByKeySegmentQuery { } auto reader = inverted_index_.kv_segment_reader>(); - std::optional> result = reader.seek_one(*offset); + std::optional> result = reader.seek_one(*offset); // ensure that the found key matches to avoid lookup_by_key false positives if (result && (result->first == key_data)) { @@ -70,7 +70,7 @@ struct InvertedIndexFindByKeySegmentQuery { } template - auto exec_filter(TKey key, datastore::TimestampRange ts_range) { + auto exec_filter(Key key, datastore::TimestampRange ts_range) { return timestamp_range_filter(exec(std::move(key)).value_or(elias_fano::EliasFanoList32::empty_list()), ts_range); } @@ -86,10 +86,10 @@ struct InvertedIndexRangeByKeyQuery { : repository_{repository}, inverted_index_name_{inverted_index_name} {} - using TKey = decltype(TKeyEncoder::value); + using Key = decltype(TKeyEncoder::value); template - auto exec(TKey key, datastore::TimestampRange ts_range) { + auto exec(Key key, datastore::TimestampRange ts_range) { auto timestamps_in_bundle = [inverted_index_name = inverted_index_name_, key = std::move(key), ts_range](std::shared_ptr bundle) { InvertedIndexFindByKeySegmentQuery query{*bundle, inverted_index_name}; return query.template exec_filter(key, ts_range); From cbe767fb9cc3a7c457368fdf9e4e14940961dc99 Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Wed, 15 Jan 2025 13:31:59 +0100 Subject: [PATCH 09/33] rename InvertedIndexRangeByKeyQuery to entity/entity_name for unification with DomainGetLatestQuery --- .../inverted_index_range_by_key_query.hpp | 14 +++++------ .../inverted_index_range_by_key_query.hpp | 24 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/silkworm/db/datastore/inverted_index_range_by_key_query.hpp b/silkworm/db/datastore/inverted_index_range_by_key_query.hpp index 9a740bd40d..d022ca0afc 100644 --- a/silkworm/db/datastore/inverted_index_range_by_key_query.hpp +++ b/silkworm/db/datastore/inverted_index_range_by_key_query.hpp @@ -26,21 +26,21 @@ namespace silkworm::datastore { template struct InvertedIndexRangeByKeyQuery { InvertedIndexRangeByKeyQuery( - datastore::EntityName inverted_index_name, - kvdb::InvertedIndex kvdb_inverted_index, + datastore::EntityName entity_name, + kvdb::InvertedIndex kvdb_entity, kvdb::ROTxn& tx, const snapshots::SnapshotRepositoryROAccess& repository) - : query1_{tx, kvdb_inverted_index}, - query2_{repository, inverted_index_name} {} + : query1_{tx, kvdb_entity}, + query2_{repository, entity_name} {} InvertedIndexRangeByKeyQuery( - datastore::EntityName inverted_index_name, + datastore::EntityName entity_name, kvdb::DatabaseRef database, kvdb::ROTxn& tx, const snapshots::SnapshotRepositoryROAccess& repository) : InvertedIndexRangeByKeyQuery{ - inverted_index_name, - database.inverted_index(inverted_index_name), + entity_name, + database.inverted_index(entity_name), tx, repository, } {} diff --git a/silkworm/db/datastore/snapshots/inverted_index_range_by_key_query.hpp b/silkworm/db/datastore/snapshots/inverted_index_range_by_key_query.hpp index 758cdc63ef..1f02858ecb 100644 --- a/silkworm/db/datastore/snapshots/inverted_index_range_by_key_query.hpp +++ b/silkworm/db/datastore/snapshots/inverted_index_range_by_key_query.hpp @@ -39,12 +39,12 @@ auto timestamp_range_filter(elias_fano::EliasFanoList32 list, datastore::Timesta template struct InvertedIndexFindByKeySegmentQuery { explicit InvertedIndexFindByKeySegmentQuery( - InvertedIndex inverted_index) - : inverted_index_{inverted_index} {} + InvertedIndex entity) + : entity_{entity} {} explicit InvertedIndexFindByKeySegmentQuery( const SnapshotBundle& bundle, - datastore::EntityName inverted_index_name) - : inverted_index_{bundle.inverted_index(inverted_index_name)} {} + datastore::EntityName entity_name) + : entity_{bundle.inverted_index(entity_name)} {} using Key = decltype(TKeyEncoder::value); @@ -53,12 +53,12 @@ struct InvertedIndexFindByKeySegmentQuery { key_encoder.value = std::move(key); ByteView key_data = key_encoder.encode_word(); - auto offset = inverted_index_.accessor_index.lookup_by_key(key_data); + auto offset = entity_.accessor_index.lookup_by_key(key_data); if (!offset) { return std::nullopt; } - auto reader = inverted_index_.kv_segment_reader>(); + auto reader = entity_.kv_segment_reader>(); std::optional> result = reader.seek_one(*offset); // ensure that the found key matches to avoid lookup_by_key false positives @@ -75,23 +75,23 @@ struct InvertedIndexFindByKeySegmentQuery { } private: - InvertedIndex inverted_index_; + InvertedIndex entity_; }; template struct InvertedIndexRangeByKeyQuery { explicit InvertedIndexRangeByKeyQuery( const SnapshotRepositoryROAccess& repository, - datastore::EntityName inverted_index_name) + datastore::EntityName entity_name) : repository_{repository}, - inverted_index_name_{inverted_index_name} {} + entity_name_{entity_name} {} using Key = decltype(TKeyEncoder::value); template auto exec(Key key, datastore::TimestampRange ts_range) { - auto timestamps_in_bundle = [inverted_index_name = inverted_index_name_, key = std::move(key), ts_range](std::shared_ptr bundle) { - InvertedIndexFindByKeySegmentQuery query{*bundle, inverted_index_name}; + auto timestamps_in_bundle = [entity_name = entity_name_, key = std::move(key), ts_range](std::shared_ptr bundle) { + InvertedIndexFindByKeySegmentQuery query{*bundle, entity_name}; return query.template exec_filter(key, ts_range); }; @@ -102,7 +102,7 @@ struct InvertedIndexRangeByKeyQuery { private: const SnapshotRepositoryROAccess& repository_; - datastore::EntityName inverted_index_name_; + datastore::EntityName entity_name_; }; } // namespace silkworm::snapshots From 5b1ebc69353747990237798447b2c91fc0e7de2d Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Thu, 16 Jan 2025 10:56:40 +0100 Subject: [PATCH 10/33] move murmur_hash3 to snapshots/common --- silkworm/db/cli/snapshots.cpp | 17 +++++++++-------- .../encoding}/murmur_hash3.cpp | 4 ++-- .../encoding}/murmur_hash3.hpp | 4 ++-- .../encoding}/murmur_hash3_test.cpp | 4 ++-- .../datastore/snapshots/rec_split/rec_split.hpp | 3 ++- 5 files changed, 17 insertions(+), 15 deletions(-) rename silkworm/db/datastore/snapshots/{rec_split => common/encoding}/murmur_hash3.cpp (98%) rename silkworm/db/datastore/snapshots/{rec_split => common/encoding}/murmur_hash3.hpp (94%) rename silkworm/db/datastore/snapshots/{rec_split => common/encoding}/murmur_hash3_test.cpp (95%) diff --git a/silkworm/db/cli/snapshots.cpp b/silkworm/db/cli/snapshots.cpp index 42bfc87287..2fbc31eb0c 100644 --- a/silkworm/db/cli/snapshots.cpp +++ b/silkworm/db/cli/snapshots.cpp @@ -44,9 +44,9 @@ #include #include #include +#include #include -#include -#include // TODO(canepat) refactor to extract Hash128 to murmur_hash3.hpp +#include #include #include #include @@ -516,6 +516,7 @@ void open_existence_index(const SnapshotSubcommandSettings& settings) { salt_stream.read(salt_bytes.data(), salt_bytes.size()); const uint32_t salt = endian::load_big_u32(reinterpret_cast(salt_bytes.data())); SILK_INFO << "Snapshot salt " << salt << " from " << salt_path.filename().string(); + encoding::Murmur3 key_hasher{salt}; std::chrono::time_point start{std::chrono::steady_clock::now()}; seg::Decompressor kv_decompressor{settings.input_file_path}; bloom_filter::BloomFilter existence_index{existence_index_file_path}; @@ -545,18 +546,18 @@ void open_existence_index(const SnapshotSubcommandSettings& settings) { SILK_TRACE << "KV: previous_key=" << to_hex(previous_key) << " key=" << to_hex(key) << " nonexistent_key=" << to_hex(nonexistent_key); // Hash the nonexistent key using murmur3 and check its presence in existence filter - rec_split::Hash128 key_hash{}; - snapshots::rec_split::murmur_hash3_x64_128(nonexistent_key.data(), nonexistent_key.size(), salt, &key_hash); - if (const bool key_found = existence_index.contains_hash(key_hash.first); key_found) { + Bytes key_hash(sizeof(uint64_t) * 2, 0); + key_hasher.hash_x64_128(nonexistent_key.data(), nonexistent_key.size(), key_hash.data()); + if (const bool key_found = existence_index.contains_hash(key_hash); key_found) { ++nonexistent_found_count; } } ++key_count; } else { value = *kv_iterator; - rec_split::Hash128 key_hash{}; - snapshots::rec_split::murmur_hash3_x64_128(key.data(), key.size(), salt, &key_hash); - const bool key_found = existence_index.contains_hash(key_hash.first); + Bytes key_hash(sizeof(uint64_t) * 2, 0); + key_hasher.hash_x64_128(key.data(), key.size(), key_hash.data()); + const bool key_found = existence_index.contains_hash(key_hash); SILK_DEBUG << "KV: key=" << to_hex(key) << " value=" << to_hex(value); ensure(key_found, [&]() { return "open_existence_index: unexpected not found key=" + to_hex(key) + diff --git a/silkworm/db/datastore/snapshots/rec_split/murmur_hash3.cpp b/silkworm/db/datastore/snapshots/common/encoding/murmur_hash3.cpp similarity index 98% rename from silkworm/db/datastore/snapshots/rec_split/murmur_hash3.cpp rename to silkworm/db/datastore/snapshots/common/encoding/murmur_hash3.cpp index 3539e0a48d..75cea50c09 100644 --- a/silkworm/db/datastore/snapshots/rec_split/murmur_hash3.cpp +++ b/silkworm/db/datastore/snapshots/common/encoding/murmur_hash3.cpp @@ -21,7 +21,7 @@ #include -namespace silkworm::snapshots::rec_split { +namespace silkworm::snapshots::encoding { // Platform-specific functions and macros @@ -193,4 +193,4 @@ void murmur_hash3_x64_128(const void* key, const uint64_t len, reinterpret_cast(out)[1] = h2; } -} // namespace silkworm::snapshots::rec_split +} // namespace silkworm::snapshots::encoding diff --git a/silkworm/db/datastore/snapshots/rec_split/murmur_hash3.hpp b/silkworm/db/datastore/snapshots/common/encoding/murmur_hash3.hpp similarity index 94% rename from silkworm/db/datastore/snapshots/rec_split/murmur_hash3.hpp rename to silkworm/db/datastore/snapshots/common/encoding/murmur_hash3.hpp index 47787b184f..e667cdcc3e 100644 --- a/silkworm/db/datastore/snapshots/rec_split/murmur_hash3.hpp +++ b/silkworm/db/datastore/snapshots/common/encoding/murmur_hash3.hpp @@ -31,7 +31,7 @@ typedef unsigned __int64 uint64_t; #include #endif // !defined(_MSC_VER) -namespace silkworm::snapshots::rec_split { +namespace silkworm::snapshots::encoding { void murmur_hash3_x64_128(const void* key, uint64_t len, uint32_t seed, void* out); @@ -51,4 +51,4 @@ class Murmur3 { uint32_t seed_; }; -} // namespace silkworm::snapshots::rec_split +} // namespace silkworm::snapshots::encoding diff --git a/silkworm/db/datastore/snapshots/rec_split/murmur_hash3_test.cpp b/silkworm/db/datastore/snapshots/common/encoding/murmur_hash3_test.cpp similarity index 95% rename from silkworm/db/datastore/snapshots/rec_split/murmur_hash3_test.cpp rename to silkworm/db/datastore/snapshots/common/encoding/murmur_hash3_test.cpp index 7da20da5ef..1663af8175 100644 --- a/silkworm/db/datastore/snapshots/rec_split/murmur_hash3_test.cpp +++ b/silkworm/db/datastore/snapshots/common/encoding/murmur_hash3_test.cpp @@ -21,7 +21,7 @@ #include -namespace silkworm::snapshots::rec_split { +namespace silkworm::snapshots::encoding { TEST_CASE("murmur_hash3_x64_128", "[silkworm][recsplit][support]") { constexpr size_t kHashBits{128}; @@ -67,4 +67,4 @@ TEST_CASE("Murmur3", "[silkworm][recsplit][support]") { delete[] hashed; } -} // namespace silkworm::snapshots::rec_split +} // namespace silkworm::snapshots::encoding diff --git a/silkworm/db/datastore/snapshots/rec_split/rec_split.hpp b/silkworm/db/datastore/snapshots/rec_split/rec_split.hpp index 699f019238..4293b8f659 100644 --- a/silkworm/db/datastore/snapshots/rec_split/rec_split.hpp +++ b/silkworm/db/datastore/snapshots/rec_split/rec_split.hpp @@ -76,11 +76,11 @@ #include #include +#include "../common/encoding/murmur_hash3.hpp" #include "../common/util/bitmask_operators.hpp" #include "../elias_fano/double_elias_fano_list.hpp" #include "../elias_fano/elias_fano_list.hpp" #include "golomb_rice.hpp" -#include "murmur_hash3.hpp" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wsign-conversion" @@ -216,6 +216,7 @@ class RecSplit { using GolombRiceBuilder = GolombRiceVector::Builder; using EliasFano = elias_fano::EliasFanoList32; using DoubleEliasFano = elias_fano::DoubleEliasFanoList16; + using Murmur3 = encoding::Murmur3; //! The base class for RecSplit building strategies struct BuildingStrategy { From c368efc9494b5bc5d41ab6204d632b4dd73d2111 Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Thu, 16 Jan 2025 11:40:49 +0100 Subject: [PATCH 11/33] salt file --- silkworm/capi/silkworm.cpp | 10 ++++- silkworm/db/blocks/schema_config.cpp | 7 ++- silkworm/db/blocks/schema_config.hpp | 4 +- silkworm/db/cli/snapshots.cpp | 9 ++-- .../datastore/snapshots/index_salt_file.cpp | 44 +++++++++++++++++++ .../datastore/snapshots/index_salt_file.hpp | 35 +++++++++++++++ silkworm/db/datastore/snapshots/schema.hpp | 7 +++ .../snapshots/snapshot_repository.cpp | 10 +++++ .../snapshots/snapshot_repository.hpp | 5 +++ silkworm/db/state/schema_config.cpp | 7 ++- silkworm/db/state/schema_config.hpp | 3 +- 11 files changed, 131 insertions(+), 10 deletions(-) create mode 100644 silkworm/db/datastore/snapshots/index_salt_file.cpp create mode 100644 silkworm/db/datastore/snapshots/index_salt_file.hpp diff --git a/silkworm/capi/silkworm.cpp b/silkworm/capi/silkworm.cpp index 432cfa8bf5..a99290927c 100644 --- a/silkworm/capi/silkworm.cpp +++ b/silkworm/capi/silkworm.cpp @@ -228,8 +228,14 @@ SILKWORM_EXPORT int silkworm_init(SilkwormHandle* handle, const struct SilkwormS auto data_dir_path = parse_path(settings->data_dir_path); auto snapshots_dir_path = DataDirectory{data_dir_path}.snapshots().path(); - auto blocks_repository = db::blocks::make_blocks_repository(snapshots_dir_path, /* open = */ false); - auto state_repository = db::state::make_state_repository(snapshots_dir_path, /* open = */ false); + auto blocks_repository = db::blocks::make_blocks_repository( + snapshots_dir_path, + /* open = */ false, + /* index_salt = */ 0); // TODO: pass from erigon + auto state_repository = db::state::make_state_repository( + snapshots_dir_path, + /* open = */ false, + /* index_salt = */ 0); // TODO: pass from erigon // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) *handle = new SilkwormInstance{ diff --git a/silkworm/db/blocks/schema_config.cpp b/silkworm/db/blocks/schema_config.cpp index b8b29ceb3d..1624f7820b 100644 --- a/silkworm/db/blocks/schema_config.cpp +++ b/silkworm/db/blocks/schema_config.cpp @@ -22,6 +22,7 @@ namespace silkworm::db::blocks { snapshots::Schema::RepositoryDef make_blocks_repository_schema() { snapshots::Schema::RepositoryDef repository_schema; + repository_schema.index_salt_file_name("salt-blocks.txt"); snapshots::Schema::EntityDef& schema = repository_schema.default_entity(); schema.segment(kHeaderSegmentName) @@ -55,12 +56,16 @@ std::unique_ptr make_blocks_index_builders_fact return std::make_unique(make_blocks_repository_schema()); } -snapshots::SnapshotRepository make_blocks_repository(std::filesystem::path dir_path, bool open) { +snapshots::SnapshotRepository make_blocks_repository( + std::filesystem::path dir_path, + bool open, + std::optional index_salt) { return snapshots::SnapshotRepository{ std::move(dir_path), open, make_blocks_repository_schema(), std::make_unique(), + std::move(index_salt), make_blocks_index_builders_factory(), }; } diff --git a/silkworm/db/blocks/schema_config.hpp b/silkworm/db/blocks/schema_config.hpp index 55bb413482..0dde147124 100644 --- a/silkworm/db/blocks/schema_config.hpp +++ b/silkworm/db/blocks/schema_config.hpp @@ -17,6 +17,7 @@ #pragma once #include +#include #include "../datastore/common/entity_name.hpp" #include "../datastore/snapshots/index_builders_factory.hpp" @@ -36,7 +37,8 @@ std::unique_ptr make_blocks_index_builders_fact snapshots::SnapshotRepository make_blocks_repository( std::filesystem::path dir_path, - bool open = true); + bool open = true, + std::optional index_salt = std::nullopt); inline constexpr datastore::EntityName kHeaderSegmentName{"headers"}; inline constexpr std::string_view kHeaderSegmentTag = kHeaderSegmentName.name; diff --git a/silkworm/db/cli/snapshots.cpp b/silkworm/db/cli/snapshots.cpp index 2fbc31eb0c..dcb51de058 100644 --- a/silkworm/db/cli/snapshots.cpp +++ b/silkworm/db/cli/snapshots.cpp @@ -46,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -510,12 +511,12 @@ void open_existence_index(const SnapshotSubcommandSettings& settings) { std::filesystem::path existence_index_file_path = settings.input_file_path; existence_index_file_path.replace_extension(".kvei"); SILK_INFO << "KV file: " << settings.input_file_path.string() << " KVEI file: " << existence_index_file_path.string(); + const auto salt_path = existence_index_file_path.parent_path().parent_path() / "salt-state.txt"; - std::ifstream salt_stream{salt_path, std::ios::in | std::ios::binary}; - std::array salt_bytes{}; - salt_stream.read(salt_bytes.data(), salt_bytes.size()); - const uint32_t salt = endian::load_big_u32(reinterpret_cast(salt_bytes.data())); + snapshots::IndexSaltFile salt_file{salt_path}; + const uint32_t salt = salt_file.load(); SILK_INFO << "Snapshot salt " << salt << " from " << salt_path.filename().string(); + encoding::Murmur3 key_hasher{salt}; std::chrono::time_point start{std::chrono::steady_clock::now()}; seg::Decompressor kv_decompressor{settings.input_file_path}; diff --git a/silkworm/db/datastore/snapshots/index_salt_file.cpp b/silkworm/db/datastore/snapshots/index_salt_file.cpp new file mode 100644 index 0000000000..6cac0648ee --- /dev/null +++ b/silkworm/db/datastore/snapshots/index_salt_file.cpp @@ -0,0 +1,44 @@ +/* + Copyright 2024 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "index_salt_file.hpp" + +#include + +#include +#include + +namespace silkworm::snapshots { + +using namespace std; + +uint32_t IndexSaltFile::load() const { + Bytes data(sizeof(uint32_t), 0); + ifstream file{path_, std::ios::binary}; + file.exceptions(ios::failbit | ios::badbit); + file.read(byte_ptr_cast(data.data()), static_cast(data.size())); + return endian::load_big_u32(data.data()); +} + +void IndexSaltFile::save(uint32_t value) const { + Bytes data(sizeof(uint32_t), 0); + endian::store_big_u32(data.data(), value); + ofstream file{path_, std::ios::binary | std::ios::trunc}; + file.exceptions(ios::failbit | ios::badbit); + file.write(byte_ptr_cast(data.data()), static_cast(data.size())); +} + +} // namespace silkworm::snapshots diff --git a/silkworm/db/datastore/snapshots/index_salt_file.hpp b/silkworm/db/datastore/snapshots/index_salt_file.hpp new file mode 100644 index 0000000000..2a540c2bde --- /dev/null +++ b/silkworm/db/datastore/snapshots/index_salt_file.hpp @@ -0,0 +1,35 @@ +/* + Copyright 2024 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#include + +namespace silkworm::snapshots { + +class IndexSaltFile { + public: + explicit IndexSaltFile(std::filesystem::path path) : path_{std::move(path)} {} + + uint32_t load() const; + void save(uint32_t value) const; + bool exists() const { return std::filesystem::exists(path_); } + + private: + std::filesystem::path path_; +}; + +} // namespace silkworm::snapshots diff --git a/silkworm/db/datastore/snapshots/schema.hpp b/silkworm/db/datastore/snapshots/schema.hpp index fb5aaac956..ae11039ba3 100644 --- a/silkworm/db/datastore/snapshots/schema.hpp +++ b/silkworm/db/datastore/snapshots/schema.hpp @@ -185,9 +185,15 @@ class Schema { return *entity_defs_.at(name); } + RepositoryDef& index_salt_file_name(std::string_view value) { + index_salt_file_name_ = value; + return *this; + } + const std::map>& entities() const { return entity_defs_; } std::vector file_extensions() const; std::optional> entity_name_by_path(const SnapshotPath& path) const; + const std::string& index_salt_file_name() const { return index_salt_file_name_.value(); } private: friend DomainDef; @@ -200,6 +206,7 @@ class Schema { static void undefine_inverted_index_schema(EntityDef& schema); std::map> entity_defs_; + std::optional index_salt_file_name_; }; RepositoryDef& repository(datastore::EntityName name) { diff --git a/silkworm/db/datastore/snapshots/snapshot_repository.cpp b/silkworm/db/datastore/snapshots/snapshot_repository.cpp index 0bcdcbafaf..7a10a1ddf4 100644 --- a/silkworm/db/datastore/snapshots/snapshot_repository.cpp +++ b/silkworm/db/datastore/snapshots/snapshot_repository.cpp @@ -24,6 +24,7 @@ #include #include "index_builders_factory.hpp" +#include "index_salt_file.hpp" namespace silkworm::snapshots { @@ -35,10 +36,12 @@ SnapshotRepository::SnapshotRepository( bool open, Schema::RepositoryDef schema, std::unique_ptr step_converter, + std::optional index_salt, std::unique_ptr index_builders_factory) : dir_path_(std::move(dir_path)), schema_(std::move(schema)), step_converter_(std::move(step_converter)), + index_salt_(std::move(index_salt)), index_builders_factory_(std::move(index_builders_factory)), bundles_(std::make_shared()), bundles_mutex_(std::make_unique()) { @@ -118,6 +121,8 @@ std::vector> SnapshotRepository::missing_indexes() void SnapshotRepository::reopen_folder() { SILK_INFO << "Reopen snapshot repository folder: " << dir_path_.string(); + index_salt_ = load_index_salt(); + auto file_ranges = list_dir_file_ranges(); if (file_ranges.empty()) return; @@ -285,6 +290,11 @@ void SnapshotRepository::remove_stale_indexes() const { } } +std::optional SnapshotRepository::load_index_salt() const { + IndexSaltFile file{this->dir_path_ / schema_.index_salt_file_name()}; + return file.exists() ? file.load() : std::optional{}; +} + void SnapshotRepository::build_indexes(const SnapshotBundlePaths& bundle) const { std::vector segment_paths; for (auto& entry : bundle.segment_paths()) diff --git a/silkworm/db/datastore/snapshots/snapshot_repository.hpp b/silkworm/db/datastore/snapshots/snapshot_repository.hpp index 46ad9d2e5d..723b15cb50 100644 --- a/silkworm/db/datastore/snapshots/snapshot_repository.hpp +++ b/silkworm/db/datastore/snapshots/snapshot_repository.hpp @@ -54,6 +54,7 @@ class SnapshotRepository : public SnapshotRepositoryROAccess { bool open, Schema::RepositoryDef schema, std::unique_ptr step_converter, + std::optional index_salt, std::unique_ptr index_builders_factory); SnapshotRepository(SnapshotRepository&&) = default; @@ -107,6 +108,7 @@ class SnapshotRepository : public SnapshotRepositoryROAccess { bool is_stale_index_path(const SnapshotPath& index_path) const; SnapshotPathList stale_index_paths() const; + std::optional load_index_salt() const; //! Path to the snapshots directory std::filesystem::path dir_path_; @@ -117,6 +119,9 @@ class SnapshotRepository : public SnapshotRepositoryROAccess { //! Converts timestamp units to steps std::unique_ptr step_converter_; + //! Index salt + std::optional index_salt_; + //! Creates index builders std::unique_ptr index_builders_factory_; diff --git a/silkworm/db/state/schema_config.cpp b/silkworm/db/state/schema_config.cpp index c4e9f22dc0..08459ca835 100644 --- a/silkworm/db/state/schema_config.cpp +++ b/silkworm/db/state/schema_config.cpp @@ -22,6 +22,7 @@ namespace silkworm::db::state { snapshots::Schema::RepositoryDef make_state_repository_schema() { snapshots::Schema::RepositoryDef schema; + schema.index_salt_file_name("salt-state.txt"); schema.domain(kDomainNameAccounts) .tag_override(kDomainAccountsTag); @@ -68,12 +69,16 @@ std::unique_ptr make_state_index_builders_facto return std::make_unique(make_state_repository_schema()); } -snapshots::SnapshotRepository make_state_repository(std::filesystem::path dir_path, bool open) { +snapshots::SnapshotRepository make_state_repository( + std::filesystem::path dir_path, + bool open, + std::optional index_salt) { return snapshots::SnapshotRepository{ std::move(dir_path), open, make_state_repository_schema(), std::make_unique(), + std::move(index_salt), make_state_index_builders_factory(), }; } diff --git a/silkworm/db/state/schema_config.hpp b/silkworm/db/state/schema_config.hpp index 44aceea977..4a3e998656 100644 --- a/silkworm/db/state/schema_config.hpp +++ b/silkworm/db/state/schema_config.hpp @@ -38,7 +38,8 @@ std::unique_ptr make_state_index_builders_facto snapshots::SnapshotRepository make_state_repository( std::filesystem::path dir_path, - bool open = true); + bool open = true, + std::optional index_salt = std::nullopt); inline constexpr datastore::EntityName kDomainNameAccounts{"Account"}; inline constexpr datastore::EntityName kDomainNameStorage{"Storage"}; From 225351d3d712236fad575eb6d8520e5c1c2f1a37 Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Wed, 15 Jan 2025 14:28:10 +0100 Subject: [PATCH 12/33] DomainGetLatestQuery --- silkworm/db/cli/snapshots.cpp | 13 +-- .../db/datastore/domain_get_latest_query.hpp | 75 ++++++++++++++++ .../kvdb/domain_get_latest_query.hpp | 2 +- silkworm/db/datastore/snapshot_merger.cpp | 2 +- .../snapshots/bloom_filter/bloom_filter.cpp | 11 ++- .../snapshots/bloom_filter/bloom_filter.hpp | 13 ++- .../bloom_filter/bloom_filter_key_hasher.cpp | 31 +++++++ .../bloom_filter/bloom_filter_key_hasher.hpp | 32 +++++++ .../bloom_filter_key_hasher_test.cpp | 30 +++++++ .../datastore/snapshots/btree/btree_index.cpp | 4 +- .../datastore/snapshots/btree/btree_index.hpp | 8 +- .../snapshots/domain_get_latest_query.hpp | 90 +++++++++++++++++++ .../datastore/snapshots/snapshot_bundle.cpp | 11 ++- .../datastore/snapshots/snapshot_bundle.hpp | 8 +- .../snapshots/snapshot_repository.cpp | 6 +- .../snapshots/snapshot_repository.hpp | 4 + silkworm/db/freezer.cpp | 2 +- silkworm/db/snapshot_sync_test.cpp | 5 +- silkworm/db/state/account_codecs.hpp | 19 ++-- silkworm/db/state/accounts_domain.hpp | 7 +- silkworm/db/state/address_codecs.hpp | 15 +++- silkworm/db/state/address_codecs_test.cpp | 11 ++- silkworm/db/state/code_domain.hpp | 7 +- silkworm/db/state/commitment_domain.hpp | 5 +- silkworm/db/state/hash_decoder.hpp | 8 +- silkworm/db/state/hash_decoder_test.cpp | 4 +- .../db/state/log_address_inverted_index.hpp | 2 +- .../db/state/log_topics_inverted_index.hpp | 2 +- silkworm/db/state/receipts_domain.hpp | 23 ++--- silkworm/db/state/receipts_domain_test.cpp | 4 +- silkworm/db/state/storage_codecs.cpp | 18 ++++ silkworm/db/state/storage_codecs.hpp | 45 ++++++---- silkworm/db/state/storage_domain.hpp | 7 +- silkworm/db/state/storage_domain_test.cpp | 22 ++++- .../db/state/traces_from_inverted_index.hpp | 2 +- .../db/state/traces_to_inverted_index.hpp | 2 +- silkworm/execution/local_state.cpp | 22 ++++- 37 files changed, 471 insertions(+), 101 deletions(-) create mode 100644 silkworm/db/datastore/domain_get_latest_query.hpp create mode 100644 silkworm/db/datastore/snapshots/bloom_filter/bloom_filter_key_hasher.cpp create mode 100644 silkworm/db/datastore/snapshots/bloom_filter/bloom_filter_key_hasher.hpp create mode 100644 silkworm/db/datastore/snapshots/bloom_filter/bloom_filter_key_hasher_test.cpp create mode 100644 silkworm/db/datastore/snapshots/domain_get_latest_query.hpp diff --git a/silkworm/db/cli/snapshots.cpp b/silkworm/db/cli/snapshots.cpp index dcb51de058..63376ac3ce 100644 --- a/silkworm/db/cli/snapshots.cpp +++ b/silkworm/db/cli/snapshots.cpp @@ -44,7 +44,6 @@ #include #include #include -#include #include #include #include @@ -517,10 +516,9 @@ void open_existence_index(const SnapshotSubcommandSettings& settings) { const uint32_t salt = salt_file.load(); SILK_INFO << "Snapshot salt " << salt << " from " << salt_path.filename().string(); - encoding::Murmur3 key_hasher{salt}; std::chrono::time_point start{std::chrono::steady_clock::now()}; seg::Decompressor kv_decompressor{settings.input_file_path}; - bloom_filter::BloomFilter existence_index{existence_index_file_path}; + bloom_filter::BloomFilter existence_index{existence_index_file_path, bloom_filter::BloomFilterKeyHasher{salt}}; SILK_INFO << "Starting KV scan and existence index check"; size_t key_count{0}, found_count{0}, nonexistent_count{0}, nonexistent_found_count{0}; @@ -546,19 +544,14 @@ void open_existence_index(const SnapshotSubcommandSettings& settings) { ByteView nonexistent_key = {full_be + kSizeDiff, sizeof(intx::uint256) - kSizeDiff}; SILK_TRACE << "KV: previous_key=" << to_hex(previous_key) << " key=" << to_hex(key) << " nonexistent_key=" << to_hex(nonexistent_key); - // Hash the nonexistent key using murmur3 and check its presence in existence filter - Bytes key_hash(sizeof(uint64_t) * 2, 0); - key_hasher.hash_x64_128(nonexistent_key.data(), nonexistent_key.size(), key_hash.data()); - if (const bool key_found = existence_index.contains_hash(key_hash); key_found) { + if (const bool key_found = existence_index.contains(nonexistent_key); key_found) { ++nonexistent_found_count; } } ++key_count; } else { value = *kv_iterator; - Bytes key_hash(sizeof(uint64_t) * 2, 0); - key_hasher.hash_x64_128(key.data(), key.size(), key_hash.data()); - const bool key_found = existence_index.contains_hash(key_hash); + const bool key_found = existence_index.contains(key); SILK_DEBUG << "KV: key=" << to_hex(key) << " value=" << to_hex(value); ensure(key_found, [&]() { return "open_existence_index: unexpected not found key=" + to_hex(key) + diff --git a/silkworm/db/datastore/domain_get_latest_query.hpp b/silkworm/db/datastore/domain_get_latest_query.hpp new file mode 100644 index 0000000000..f87e4824e0 --- /dev/null +++ b/silkworm/db/datastore/domain_get_latest_query.hpp @@ -0,0 +1,75 @@ +/* + Copyright 2024 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#include "kvdb/database.hpp" +#include "kvdb/domain_get_latest_query.hpp" +#include "snapshots/domain_get_latest_query.hpp" + +namespace silkworm::datastore { + +template < + kvdb::EncoderConcept TKeyEncoder1, snapshots::EncoderConcept TKeyEncoder2, + kvdb::DecoderConcept TValueDecoder1, snapshots::DecoderConcept TValueDecoder2> +struct DomainGetLatestQuery { + DomainGetLatestQuery( + datastore::EntityName entity_name, + kvdb::Domain kvdb_entity, + kvdb::ROTxn& tx, + const snapshots::SnapshotRepositoryROAccess& repository) + : query1_{tx, kvdb_entity}, + query2_{repository, entity_name} {} + + DomainGetLatestQuery( + datastore::EntityName entity_name, + kvdb::DatabaseRef database, + kvdb::ROTxn& tx, + const snapshots::SnapshotRepositoryROAccess& repository) + : DomainGetLatestQuery{ + entity_name, + database.domain(entity_name), + tx, + repository, + } {} + + using Key1 = decltype(TKeyEncoder1::value); + using Key2 = decltype(TKeyEncoder2::value); + static_assert(std::same_as); + using Key = Key1; + + using Result1 = typename kvdb::DomainGetLatestQuery::Result; + using Result2 = typename snapshots::DomainGetLatestQuery::Result; + using Result = Result1; + + std::optional exec(const Key& key) { + auto result1 = query1_.exec(key); + if (result1) { + return result1; + } + auto result2 = query2_.exec(key); + if (result2) { + return Result{std::move(result2->value), result2->step}; + } + return std::nullopt; + } + + private: + kvdb::DomainGetLatestQuery query1_; + snapshots::DomainGetLatestQuery query2_; +}; + +} // namespace silkworm::datastore \ No newline at end of file diff --git a/silkworm/db/datastore/kvdb/domain_get_latest_query.hpp b/silkworm/db/datastore/kvdb/domain_get_latest_query.hpp index ba887e5c11..be7259e7cc 100644 --- a/silkworm/db/datastore/kvdb/domain_get_latest_query.hpp +++ b/silkworm/db/datastore/kvdb/domain_get_latest_query.hpp @@ -28,7 +28,7 @@ namespace silkworm::datastore::kvdb { template struct DomainGetLatestQuery { - RWTxn& tx; + ROTxn& tx; Domain entity; using Key = decltype(TKeyEncoder::value); diff --git a/silkworm/db/datastore/snapshot_merger.cpp b/silkworm/db/datastore/snapshot_merger.cpp index c892a8b927..93154f4803 100644 --- a/silkworm/db/datastore/snapshot_merger.cpp +++ b/silkworm/db/datastore/snapshot_merger.cpp @@ -126,7 +126,7 @@ void SnapshotMerger::commit(std::shared_ptr result) { move_files(bundle.files(), snapshots_.path()); - SnapshotBundle final_bundle{snapshots_.schema(), snapshots_.path(), bundle.step_range()}; + SnapshotBundle final_bundle = snapshots_.open_bundle(bundle.step_range()); snapshots_.replace_snapshot_bundles(std::move(final_bundle)); for (auto& merged_bundle : merged_bundles) { diff --git a/silkworm/db/datastore/snapshots/bloom_filter/bloom_filter.cpp b/silkworm/db/datastore/snapshots/bloom_filter/bloom_filter.cpp index 8b990300d6..3499ea841b 100644 --- a/silkworm/db/datastore/snapshots/bloom_filter/bloom_filter.cpp +++ b/silkworm/db/datastore/snapshots/bloom_filter/bloom_filter.cpp @@ -49,7 +49,9 @@ uint64_t BloomFilter::optimal_bits_count(uint64_t max_key_count, double p) { return static_cast(std::ceil(-static_cast(max_key_count) * std::log(p) / (ln2 * ln2))); } -BloomFilter::BloomFilter(std::filesystem::path path) +BloomFilter::BloomFilter( + std::filesystem::path path, + std::optional data_key_hasher) : BloomFilter{kMinimumBitsCount, new_random_keys()} { if (!std::filesystem::exists(path)) { throw std::runtime_error("index file " + path.filename().string() + " doesn't exist"); @@ -62,6 +64,7 @@ BloomFilter::BloomFilter(std::filesystem::path path) file_stream >> *this; path_ = std::move(path); + data_key_hasher_ = std::move(data_key_hasher); } BloomFilter::BloomFilter() @@ -86,7 +89,7 @@ void BloomFilter::add_hash(uint64_t hash) { ++inserted_count_; } -bool BloomFilter::contains_hash(uint64_t hash) { +bool BloomFilter::contains_hash(uint64_t hash) const { uint64_t r{1}; for (size_t n = 0; n < kHardCodedK; ++n) { hash = ((hash << kRotation) | (hash >> kRotationOf64)) ^ keys_[n]; @@ -96,6 +99,10 @@ bool BloomFilter::contains_hash(uint64_t hash) { return r != 0; } +bool BloomFilter::contains(ByteView data_key) const { + return contains_hash(data_key_hasher_->hash(data_key)); +} + void BloomFilter::ensure_min_bits_count(uint64_t bits_count) { if (bits_count < kMinimumBitsCount) { throw std::runtime_error{"number of bits must be >= " + std::to_string(kMinimumBitsCount) + diff --git a/silkworm/db/datastore/snapshots/bloom_filter/bloom_filter.hpp b/silkworm/db/datastore/snapshots/bloom_filter/bloom_filter.hpp index 62e1fbd783..d5ff8ca680 100644 --- a/silkworm/db/datastore/snapshots/bloom_filter/bloom_filter.hpp +++ b/silkworm/db/datastore/snapshots/bloom_filter/bloom_filter.hpp @@ -20,15 +20,20 @@ #include #include #include +#include #include +#include "bloom_filter_key_hasher.hpp" + namespace silkworm::snapshots::bloom_filter { //! Bloom filter implementation (https://en.wikipedia.org/wiki/Bloom_filter) //! \remark Serialized binary format compatible with: https://github.com/holiman/bloomfilter class BloomFilter { public: - explicit BloomFilter(std::filesystem::path path); + explicit BloomFilter( + std::filesystem::path path, + std::optional data_key_hasher = std::nullopt); BloomFilter(); BloomFilter(uint64_t max_key_count, double p); @@ -43,7 +48,8 @@ class BloomFilter { //! Checks if filter contains the give \p hash value //! \param hash the value to check for presence //! \return false means "definitely does not contain value", true means "maybe contains value" - bool contains_hash(uint64_t hash); + bool contains_hash(uint64_t hash) const; + bool contains(ByteView data_key) const; friend std::istream& operator>>(std::istream& is, BloomFilter& filter); @@ -63,6 +69,9 @@ class BloomFilter { //! The index file path std::filesystem::path path_; + //! Data key hasher + std::optional data_key_hasher_; + //! The number of bits that the bitmap should be able to track uint64_t bits_count_; diff --git a/silkworm/db/datastore/snapshots/bloom_filter/bloom_filter_key_hasher.cpp b/silkworm/db/datastore/snapshots/bloom_filter/bloom_filter_key_hasher.cpp new file mode 100644 index 0000000000..108dd23979 --- /dev/null +++ b/silkworm/db/datastore/snapshots/bloom_filter/bloom_filter_key_hasher.cpp @@ -0,0 +1,31 @@ +/* + Copyright 2024 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "bloom_filter_key_hasher.hpp" + +#include + +#include "../common/encoding/murmur_hash3.hpp" + +namespace silkworm::snapshots::bloom_filter { + +uint64_t BloomFilterKeyHasher::hash(ByteView key) const { + std::array hash = {0, 0}; + encoding::Murmur3{salt_}.hash_x64_128(key.data(), key.size(), hash.data()); + return hash[0]; +} + +} // namespace silkworm::snapshots::bloom_filter diff --git a/silkworm/db/datastore/snapshots/bloom_filter/bloom_filter_key_hasher.hpp b/silkworm/db/datastore/snapshots/bloom_filter/bloom_filter_key_hasher.hpp new file mode 100644 index 0000000000..1f7e3af7be --- /dev/null +++ b/silkworm/db/datastore/snapshots/bloom_filter/bloom_filter_key_hasher.hpp @@ -0,0 +1,32 @@ +/* + Copyright 2024 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#include + +namespace silkworm::snapshots::bloom_filter { + +class BloomFilterKeyHasher { + public: + explicit BloomFilterKeyHasher(uint32_t salt) : salt_{salt} {} + uint64_t hash(ByteView key) const; + + private: + uint32_t salt_; +}; + +} // namespace silkworm::snapshots::bloom_filter diff --git a/silkworm/db/datastore/snapshots/bloom_filter/bloom_filter_key_hasher_test.cpp b/silkworm/db/datastore/snapshots/bloom_filter/bloom_filter_key_hasher_test.cpp new file mode 100644 index 0000000000..32b7700cc9 --- /dev/null +++ b/silkworm/db/datastore/snapshots/bloom_filter/bloom_filter_key_hasher_test.cpp @@ -0,0 +1,30 @@ +/* + Copyright 2024 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "bloom_filter_key_hasher.hpp" + +#include + +#include + +namespace silkworm::snapshots::bloom_filter { + +TEST_CASE("BloomFilterKeyHasher") { + CHECK(BloomFilterKeyHasher{0}.hash(*from_hex("CAFEBABE")) == 2809309899937206063u); + CHECK(BloomFilterKeyHasher{12345}.hash(*from_hex("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")) == 17810263873480351644u); +} + +} // namespace silkworm::snapshots::bloom_filter diff --git a/silkworm/db/datastore/snapshots/btree/btree_index.cpp b/silkworm/db/datastore/snapshots/btree/btree_index.cpp index 7258038c32..9232e9d0f8 100644 --- a/silkworm/db/datastore/snapshots/btree/btree_index.cpp +++ b/silkworm/db/datastore/snapshots/btree/btree_index.cpp @@ -70,7 +70,7 @@ void BTreeIndex::warmup_if_empty_or_check(const KVSegmentReader& kv_segment) { } } -std::optional BTreeIndex::seek(ByteView seek_key, const KVSegmentReader& kv_segment) { +std::optional BTreeIndex::seek(ByteView seek_key, const KVSegmentReader& kv_segment) const { KeyValueIndex index{kv_segment, data_offsets_, file_path_}; auto [found, key, value, data_index] = btree_->seek(seek_key, index); if (key.compare(seek_key) >= 0) { @@ -85,7 +85,7 @@ std::optional BTreeIndex::seek(ByteView seek_key, const KVSe return std::nullopt; } -std::optional BTreeIndex::get(ByteView key, const KVSegmentReader& kv_segment) { +std::optional BTreeIndex::get(ByteView key, const KVSegmentReader& kv_segment) const { KeyValueIndex index{kv_segment, data_offsets_, file_path_}; auto result = btree_->get(key, index); if (!result) { diff --git a/silkworm/db/datastore/snapshots/btree/btree_index.hpp b/silkworm/db/datastore/snapshots/btree/btree_index.hpp index d74a972f06..121211c74b 100644 --- a/silkworm/db/datastore/snapshots/btree/btree_index.hpp +++ b/silkworm/db/datastore/snapshots/btree/btree_index.hpp @@ -50,7 +50,7 @@ class BTreeIndex { friend class BTreeIndex; Cursor( - BTreeIndex* index, + const BTreeIndex* index, Bytes key, Bytes value, DataIndex data_index, @@ -61,7 +61,7 @@ class BTreeIndex { data_index_{data_index}, kv_segment_{kv_segment} {} - BTreeIndex* index_; + const BTreeIndex* index_; Bytes key_; Bytes value_; DataIndex data_index_; @@ -89,13 +89,13 @@ class BTreeIndex { //! \return a cursor positioned at key >= \p seek_key or nullptr //! \details if \p seek_key is empty, first key is returned //! \details if \p seek_key is greater than any other key, std::nullopt is returned - std::optional seek(ByteView seek_key, const KVSegmentReader& kv_segment); + std::optional seek(ByteView seek_key, const KVSegmentReader& kv_segment) const; //! Get the value associated to the given key with exact match //! \param key the data key to match exactly //! \param kv_segment reader of the key-value data sequence //! \return the value associated at \p key or std::nullopt if not found - std::optional get(ByteView key, const KVSegmentReader& kv_segment); + std::optional get(ByteView key, const KVSegmentReader& kv_segment) const; private: class KeyValueIndex : public BTree::KeyValueIndex { diff --git a/silkworm/db/datastore/snapshots/domain_get_latest_query.hpp b/silkworm/db/datastore/snapshots/domain_get_latest_query.hpp new file mode 100644 index 0000000000..cbc00b71b6 --- /dev/null +++ b/silkworm/db/datastore/snapshots/domain_get_latest_query.hpp @@ -0,0 +1,90 @@ +/* + Copyright 2024 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#include "../common/step.hpp" +#include "common/codec.hpp" +#include "common/raw_codec.hpp" +#include "domain.hpp" +#include "segment/kv_segment_reader.hpp" +#include "snapshot_bundle.hpp" +#include "snapshot_repository_ro_access.hpp" + +namespace silkworm::snapshots { + +template +struct DomainGetLatestSegmentQuery { + explicit DomainGetLatestSegmentQuery(Domain entity) + : entity_{std::move(entity)} {} + explicit DomainGetLatestSegmentQuery( + const SnapshotBundle& bundle, + datastore::EntityName entity_name) + : entity_{bundle.domain(entity_name)} {} + + using Key = decltype(TKeyEncoder::value); + using Value = decltype(TValueDecoder::value); + + std::optional exec(const Key& key) { + TKeyEncoder key_encoder; + key_encoder.value = key; + ByteView key_data = key_encoder.encode_word(); + + if (!entity_.existence_index.contains(key_data)) { + return std::nullopt; + } + + std::optional value_data = entity_.btree_index.get(key_data, entity_.kv_segment); + if (!value_data) { + return std::nullopt; + } + + TValueDecoder value_decoder; + value_decoder.decode_word(*value_data); + return std::move(value_decoder.value); + } + + private: + Domain entity_; +}; + +template +struct DomainGetLatestQuery { + const SnapshotRepositoryROAccess& repository; + datastore::EntityName entity_name; + + using Key = decltype(TKeyEncoder::value); + using Value = decltype(TValueDecoder::value); + + struct Result { + Value value; + datastore::Step step; + }; + + std::optional exec(const Key& key) { + for (auto& bundle_ptr : repository.view_bundles_reverse()) { + const SnapshotBundle& bundle = *bundle_ptr; + DomainGetLatestSegmentQuery query{bundle, entity_name}; + auto value = query.exec(key); + if (value) { + return Result{std::move(*value), bundle.step_range().end}; + } + } + return std::nullopt; + } +}; + +} // namespace silkworm::snapshots diff --git a/silkworm/db/datastore/snapshots/snapshot_bundle.cpp b/silkworm/db/datastore/snapshots/snapshot_bundle.cpp index 5d9291f2c9..448ba17944 100644 --- a/silkworm/db/datastore/snapshots/snapshot_bundle.cpp +++ b/silkworm/db/datastore/snapshots/snapshot_bundle.cpp @@ -81,11 +81,13 @@ static std::map open_accessor_indexes( static std::map open_existence_indexes( const Schema::EntityDef& entity, const std::filesystem::path& dir_path, - StepRange range) { + StepRange range, + std::optional salt) { std::map results; for (auto& [name, path] : make_snapshot_paths(Schema::SnapshotFileDef::Format::kExistenceIndex, entity, dir_path, range)) { SILK_TRACE << "make_existence_indexes opens " << name.to_string() << " at " << path.filename(); - results.emplace(name, bloom_filter::BloomFilter{path.path()}); + SILKWORM_ASSERT(salt); + results.emplace(name, bloom_filter::BloomFilter{path.path(), bloom_filter::BloomFilterKeyHasher{*salt}}); } return results; } @@ -105,7 +107,8 @@ static std::map open_btree_indexes( SnapshotBundleData open_bundle_data( const Schema::RepositoryDef& schema, const std::filesystem::path& dir_path, - StepRange step_range) { + StepRange step_range, + std::optional index_salt) { SnapshotBundleData data; for (auto& [name, entity_schema_ptr] : schema.entities()) { auto& entity_schema = *entity_schema_ptr; @@ -115,7 +118,7 @@ SnapshotBundleData open_bundle_data( open_segments(entity_schema, dir_path, step_range), open_kv_segments(entity_schema, dir_path, step_range), open_accessor_indexes(entity_schema, dir_path, step_range), - open_existence_indexes(entity_schema, dir_path, step_range), + open_existence_indexes(entity_schema, dir_path, step_range, index_salt), open_btree_indexes(entity_schema, dir_path, step_range), }); } diff --git a/silkworm/db/datastore/snapshots/snapshot_bundle.hpp b/silkworm/db/datastore/snapshots/snapshot_bundle.hpp index ceb2cfa8c9..284c5ee637 100644 --- a/silkworm/db/datastore/snapshots/snapshot_bundle.hpp +++ b/silkworm/db/datastore/snapshots/snapshot_bundle.hpp @@ -49,7 +49,8 @@ struct SnapshotBundleData { SnapshotBundleData open_bundle_data( const Schema::RepositoryDef& schema, const std::filesystem::path& dir_path, - datastore::StepRange step_range); + datastore::StepRange step_range, + std::optional index_salt); struct SnapshotBundlePaths { using StepRange = datastore::StepRange; @@ -84,10 +85,11 @@ struct SnapshotBundle : public SegmentAndAccessorIndexProvider { SnapshotBundle( const Schema::RepositoryDef& schema, const std::filesystem::path& dir_path, - StepRange range) + StepRange range, + std::optional index_salt) : SnapshotBundle{ range, - open_bundle_data(schema, dir_path, range), + open_bundle_data(schema, dir_path, range, std::move(index_salt)), } {} ~SnapshotBundle() override; diff --git a/silkworm/db/datastore/snapshots/snapshot_repository.cpp b/silkworm/db/datastore/snapshots/snapshot_repository.cpp index 7a10a1ddf4..cfd0f03c72 100644 --- a/silkworm/db/datastore/snapshots/snapshot_repository.cpp +++ b/silkworm/db/datastore/snapshots/snapshot_repository.cpp @@ -145,7 +145,7 @@ void SnapshotRepository::reopen_folder() { SnapshotBundlePaths bundle_paths{schema_, dir_path_, range}; // if all bundle paths exist if (std::ranges::all_of(bundle_paths.files(), [](const fs::path& p) { return fs::exists(p); })) { - SnapshotBundle bundle{schema_, dir_path_, range}; + SnapshotBundle bundle = open_bundle(range); bundles->insert_or_assign(num, std::make_shared(std::move(bundle))); } } @@ -161,6 +161,10 @@ void SnapshotRepository::reopen_folder() { << " max block available: " << max_block_available(); } +SnapshotBundle SnapshotRepository::open_bundle(StepRange range) const { + return SnapshotBundle{schema_, dir_path_, range, index_salt_}; +} + std::shared_ptr SnapshotRepository::find_bundle(Timestamp t) const { return find_bundle(step_converter_->step_from_timestamp(t)); } diff --git a/silkworm/db/datastore/snapshots/snapshot_repository.hpp b/silkworm/db/datastore/snapshots/snapshot_repository.hpp index 723b15cb50..6560925b23 100644 --- a/silkworm/db/datastore/snapshots/snapshot_repository.hpp +++ b/silkworm/db/datastore/snapshots/snapshot_repository.hpp @@ -67,6 +67,9 @@ class SnapshotRepository : public SnapshotRepositoryROAccess { void reopen_folder(); + //! Opens a detached bundle of snapshot files. Use add_snapshot_bundle or replace_snapshot_bundles to add it. + SnapshotBundle open_bundle(StepRange range) const; + void add_snapshot_bundle(SnapshotBundle bundle); //! Replace bundles whose ranges are contained within the given bundle @@ -79,6 +82,7 @@ class SnapshotRepository : public SnapshotRepositoryROAccess { std::vector> missing_indexes() const; void remove_stale_indexes() const; + const std::optional& index_salt() const { return index_salt_; } void build_indexes(const SnapshotBundlePaths& bundle) const; BundlesView> view_bundles() const override { diff --git a/silkworm/db/freezer.cpp b/silkworm/db/freezer.cpp index 3949e0bf3a..23c28a67e2 100644 --- a/silkworm/db/freezer.cpp +++ b/silkworm/db/freezer.cpp @@ -134,7 +134,7 @@ void Freezer::commit(std::shared_ptr result) { auto& bundle = freezer_result.bundle_paths; move_files(bundle.files(), snapshots_.path()); - SnapshotBundle final_bundle{snapshots_.schema(), snapshots_.path(), bundle.step_range()}; + SnapshotBundle final_bundle = snapshots_.open_bundle(bundle.step_range()); snapshots_.add_snapshot_bundle(std::move(final_bundle)); } diff --git a/silkworm/db/snapshot_sync_test.cpp b/silkworm/db/snapshot_sync_test.cpp index 008e0ef509..15fb2002f7 100644 --- a/silkworm/db/snapshot_sync_test.cpp +++ b/silkworm/db/snapshot_sync_test.cpp @@ -133,11 +133,8 @@ TEST_CASE("SnapshotSync::update_block_headers", "[db][snapshot][sync]") { // Add a sample Snapshot bundle to the repository auto step_range = datastore::StepRange::from_block_num_range(snapshots::test_util::kSampleSnapshotBlockRange); - SnapshotBundle bundle{ - step_range, - open_bundle_data(blocks::make_blocks_repository_schema(), tmp_dir_path, step_range), - }; auto& repository = snapshot_sync.blocks_repository(); + SnapshotBundle bundle = repository.open_bundle(step_range); repository.add_snapshot_bundle(std::move(bundle)); // Update the block headers in the database according to the repository content diff --git a/silkworm/db/state/account_codecs.hpp b/silkworm/db/state/account_codecs.hpp index e1d03b28f9..ad3be199c0 100644 --- a/silkworm/db/state/account_codecs.hpp +++ b/silkworm/db/state/account_codecs.hpp @@ -45,19 +45,26 @@ struct AccountKVDBCodec : public datastore::kvdb::Codec { static_assert(datastore::kvdb::EncoderConcept); static_assert(datastore::kvdb::DecoderConcept); -struct AccountDecoder : public snapshots::Decoder { +struct AccountSnapshotsCodec : public snapshots::Codec { Account value; + Bytes word; - ~AccountDecoder() override = default; + ~AccountSnapshotsCodec() override = default; - void decode_word(ByteView word) override { - auto account = AccountCodec::from_encoded_storage_v3(word); + ByteView encode_word() override { + word = AccountCodec::encode_for_storage_v3(value); + return word; + } + + void decode_word(ByteView input_word) override { + auto account = AccountCodec::from_encoded_storage_v3(input_word); if (!account) - throw DecodingException{account.error(), "AccountDecoder failed to decode Account"}; + throw DecodingException{account.error(), "AccountSnapshotsCodec failed to decode Account"}; value = std::move(*account); } }; -static_assert(snapshots::DecoderConcept); +static_assert(snapshots::EncoderConcept); +static_assert(snapshots::DecoderConcept); } // namespace silkworm::db::state diff --git a/silkworm/db/state/accounts_domain.hpp b/silkworm/db/state/accounts_domain.hpp index 7204a02fbf..6f5a04f4a6 100644 --- a/silkworm/db/state/accounts_domain.hpp +++ b/silkworm/db/state/accounts_domain.hpp @@ -16,6 +16,7 @@ #pragma once +#include #include #include @@ -24,10 +25,12 @@ namespace silkworm::db::state { -using AccountsDomainGetLatestQuery = datastore::kvdb::DomainGetLatestQuery; +using AccountsDomainGetLatestQuery = datastore::DomainGetLatestQuery< + AddressKVDBEncoder, AddressSnapshotsEncoder, + AccountKVDBCodec, AccountSnapshotsCodec>; using AccountsDomainPutQuery = datastore::kvdb::DomainPutQuery; using AccountsDomainDeleteQuery = datastore::kvdb::DomainDeleteQuery; -using AccountsDomainKVSegmentReader = snapshots::segment::KVSegmentReader; +using AccountsDomainKVSegmentReader = snapshots::segment::KVSegmentReader; } // namespace silkworm::db::state diff --git a/silkworm/db/state/address_codecs.hpp b/silkworm/db/state/address_codecs.hpp index ca9d736faa..ecd7753984 100644 --- a/silkworm/db/state/address_codecs.hpp +++ b/silkworm/db/state/address_codecs.hpp @@ -37,18 +37,25 @@ struct AddressKVDBEncoder : public datastore::kvdb::Encoder { static_assert(datastore::kvdb::EncoderConcept); -struct AddressDecoder : public snapshots::Decoder { +struct AddressSnapshotsCodec : public snapshots::Codec { evmc::address value; + ~AddressSnapshotsCodec() override = default; - ~AddressDecoder() override = default; + ByteView encode_word() override { + return ByteView{reinterpret_cast(&value.bytes), kAddressLength}; + } void decode_word(ByteView word) override { if (word.size() < kAddressLength) - throw std::runtime_error{"AddressDecoder failed to decode"}; + throw std::runtime_error{"AddressSnapshotsDecoder failed to decode"}; std::memcpy(value.bytes, word.data(), kAddressLength); } }; -static_assert(snapshots::DecoderConcept); +static_assert(snapshots::EncoderConcept); +static_assert(snapshots::DecoderConcept); + +using AddressSnapshotsEncoder = AddressSnapshotsCodec; +using AddressSnapshotsDecoder = AddressSnapshotsCodec; } // namespace silkworm::db::state diff --git a/silkworm/db/state/address_codecs_test.cpp b/silkworm/db/state/address_codecs_test.cpp index 1cfa562f08..5a172b3aca 100644 --- a/silkworm/db/state/address_codecs_test.cpp +++ b/silkworm/db/state/address_codecs_test.cpp @@ -22,13 +22,20 @@ namespace silkworm::db::state { -TEST_CASE("AddressDecoder") { +TEST_CASE("AddressSnapshotsDecoder") { using evmc::literals::operator""_address; - AddressDecoder decoder; + AddressSnapshotsDecoder decoder; decoder.decode_word(*from_hex("0x000000000000000000636f6e736f6c652e6c6f67")); CHECK(decoder.value == 0x000000000000000000636f6e736f6c652e6c6f67_address); CHECK_THROWS_AS(decoder.decode_word({}), std::runtime_error); } +TEST_CASE("AddressSnapshotsEncoder") { + using evmc::literals::operator""_address; + AddressSnapshotsEncoder encoder; + encoder.value = 0x000000000000000000636f6e736f6c652e6c6f67_address; + CHECK(encoder.encode_word() == *from_hex("0x000000000000000000636f6e736f6c652e6c6f67")); +} + } // namespace silkworm::db::state diff --git a/silkworm/db/state/code_domain.hpp b/silkworm/db/state/code_domain.hpp index e2bc7804ee..0a390b5c38 100644 --- a/silkworm/db/state/code_domain.hpp +++ b/silkworm/db/state/code_domain.hpp @@ -16,6 +16,7 @@ #pragma once +#include #include #include #include @@ -24,10 +25,12 @@ namespace silkworm::db::state { -using CodeDomainGetLatestQuery = datastore::kvdb::DomainGetLatestQuery>; +using CodeDomainGetLatestQuery = datastore::DomainGetLatestQuery< + AddressKVDBEncoder, AddressSnapshotsEncoder, + datastore::kvdb::RawDecoder, snapshots::RawDecoder>; using CodeDomainPutQuery = datastore::kvdb::DomainPutQuery>; using CodeDomainDeleteQuery = datastore::kvdb::DomainDeleteQuery>; -using CodeDomainKVSegmentReader = snapshots::segment::KVSegmentReader>; +using CodeDomainKVSegmentReader = snapshots::segment::KVSegmentReader>; } // namespace silkworm::db::state diff --git a/silkworm/db/state/commitment_domain.hpp b/silkworm/db/state/commitment_domain.hpp index e5b58cdc8b..41199d6efe 100644 --- a/silkworm/db/state/commitment_domain.hpp +++ b/silkworm/db/state/commitment_domain.hpp @@ -16,13 +16,16 @@ #pragma once +#include #include #include #include namespace silkworm::db::state { -using CommitmentDomainGetLatestQuery = datastore::kvdb::DomainGetLatestQuery, datastore::kvdb::RawDecoder>; +using CommitmentDomainGetLatestQuery = datastore::DomainGetLatestQuery< + datastore::kvdb::RawEncoder, snapshots::RawEncoder, + datastore::kvdb::RawDecoder, snapshots::RawDecoder>; using CommitmentDomainPutQuery = datastore::kvdb::DomainPutQuery, datastore::kvdb::RawEncoder>; using CommitmentDomainDeleteQuery = datastore::kvdb::DomainDeleteQuery, datastore::kvdb::RawEncoder>; diff --git a/silkworm/db/state/hash_decoder.hpp b/silkworm/db/state/hash_decoder.hpp index a35491db32..b641b8cc4e 100644 --- a/silkworm/db/state/hash_decoder.hpp +++ b/silkworm/db/state/hash_decoder.hpp @@ -23,18 +23,18 @@ namespace silkworm::db::state { -struct HashDecoder : public snapshots::Decoder { +struct HashSnapshotsDecoder : public snapshots::Decoder { Hash value; - ~HashDecoder() override = default; + ~HashSnapshotsDecoder() override = default; void decode_word(ByteView word) override { if (word.size() < kHashLength) - throw std::runtime_error{"HashDecoder failed to decode"}; + throw std::runtime_error{"HashSnapshotsDecoder failed to decode"}; value = Hash{word}; } }; -static_assert(snapshots::DecoderConcept); +static_assert(snapshots::DecoderConcept); } // namespace silkworm::db::state diff --git a/silkworm/db/state/hash_decoder_test.cpp b/silkworm/db/state/hash_decoder_test.cpp index ca2b762711..9077c47bd3 100644 --- a/silkworm/db/state/hash_decoder_test.cpp +++ b/silkworm/db/state/hash_decoder_test.cpp @@ -22,9 +22,9 @@ namespace silkworm::db::state { -TEST_CASE("HashDecoder") { +TEST_CASE("HashSnapshotsDecoder") { using evmc::literals::operator""_bytes32; - HashDecoder decoder; + HashSnapshotsDecoder decoder; decoder.decode_word(*from_hex("0xb397a22bb95bf14753ec174f02f99df3f0bdf70d1851cdff813ebf745f5aeb55")); CHECK(decoder.value == 0xb397a22bb95bf14753ec174f02f99df3f0bdf70d1851cdff813ebf745f5aeb55_bytes32); diff --git a/silkworm/db/state/log_address_inverted_index.hpp b/silkworm/db/state/log_address_inverted_index.hpp index f80299d014..30fa0c70f1 100644 --- a/silkworm/db/state/log_address_inverted_index.hpp +++ b/silkworm/db/state/log_address_inverted_index.hpp @@ -23,6 +23,6 @@ namespace silkworm::db::state { -using LogAddressInvertedIndexKVSegmentReader = snapshots::segment::KVSegmentReader>; +using LogAddressInvertedIndexKVSegmentReader = snapshots::segment::KVSegmentReader>; } // namespace silkworm::db::state diff --git a/silkworm/db/state/log_topics_inverted_index.hpp b/silkworm/db/state/log_topics_inverted_index.hpp index 9f0e20a033..05945a9c61 100644 --- a/silkworm/db/state/log_topics_inverted_index.hpp +++ b/silkworm/db/state/log_topics_inverted_index.hpp @@ -23,6 +23,6 @@ namespace silkworm::db::state { -using LogTopicsInvertedIndexKVSegmentReader = snapshots::segment::KVSegmentReader>; +using LogTopicsInvertedIndexKVSegmentReader = snapshots::segment::KVSegmentReader>; } // namespace silkworm::db::state diff --git a/silkworm/db/state/receipts_domain.hpp b/silkworm/db/state/receipts_domain.hpp index 9ee51d5a43..ad4f3db397 100644 --- a/silkworm/db/state/receipts_domain.hpp +++ b/silkworm/db/state/receipts_domain.hpp @@ -18,13 +18,16 @@ #include +#include #include #include #include namespace silkworm::db::state { -using ReceiptsDomainGetLatestQuery = datastore::kvdb::DomainGetLatestQuery, datastore::kvdb::RawDecoder>; +using ReceiptsDomainGetLatestQuery = datastore::DomainGetLatestQuery< + datastore::kvdb::RawEncoder, snapshots::RawEncoder, + datastore::kvdb::RawDecoder, snapshots::RawDecoder>; using ReceiptsDomainPutQuery = datastore::kvdb::DomainPutQuery, datastore::kvdb::RawEncoder>; using ReceiptsDomainDeleteQuery = datastore::kvdb::DomainDeleteQuery, datastore::kvdb::RawEncoder>; @@ -34,31 +37,31 @@ enum class ReceiptsDomainKey : uint8_t { kFirstLogIndexKey = 2, }; -struct ReceiptsDomainKeyDecoder : public snapshots::Decoder { +struct ReceiptsDomainKeySnapshotsDecoder : public snapshots::Decoder { ReceiptsDomainKey value{}; - ~ReceiptsDomainKeyDecoder() override = default; + ~ReceiptsDomainKeySnapshotsDecoder() override = default; void decode_word(ByteView word) override { if (word.empty()) - throw std::runtime_error{"ReceiptsDomainKeyDecoder failed to decode an empty word"}; + throw std::runtime_error{"ReceiptsDomainKeySnapshotsDecoder failed to decode an empty word"}; value = static_cast(word[0]); } }; -static_assert(snapshots::DecoderConcept); +static_assert(snapshots::DecoderConcept); -struct VarintDecoder : public snapshots::Decoder { +struct VarintSnapshotsDecoder : public snapshots::Decoder { uint64_t value{}; - ~VarintDecoder() override = default; + ~VarintSnapshotsDecoder() override = default; void decode_word(ByteView word) override { auto value_opt = snapshots::seg::varint::decode(word); if (!value_opt) - throw std::runtime_error{"VarintDecoder failed to decode"}; + throw std::runtime_error{"VarintSnapshotsDecoder failed to decode"}; value = *value_opt; } }; -static_assert(snapshots::DecoderConcept); +static_assert(snapshots::DecoderConcept); -using ReceiptsDomainKVSegmentReader = snapshots::segment::KVSegmentReader; +using ReceiptsDomainKVSegmentReader = snapshots::segment::KVSegmentReader; } // namespace silkworm::db::state diff --git a/silkworm/db/state/receipts_domain_test.cpp b/silkworm/db/state/receipts_domain_test.cpp index ee74e01953..8c15d976bf 100644 --- a/silkworm/db/state/receipts_domain_test.cpp +++ b/silkworm/db/state/receipts_domain_test.cpp @@ -20,8 +20,8 @@ namespace silkworm::db::state { -TEST_CASE("ReceiptsDomainKeyDecoder") { - ReceiptsDomainKeyDecoder decoder; +TEST_CASE("ReceiptsDomainKeySnapshotsDecoder") { + ReceiptsDomainKeySnapshotsDecoder decoder; decoder.decode_word(Bytes{1}); CHECK(decoder.value == ReceiptsDomainKey::kCumulativeBlobGasUsedInBlockKey); diff --git a/silkworm/db/state/storage_codecs.cpp b/silkworm/db/state/storage_codecs.cpp index 39c4873161..0078bb5eba 100644 --- a/silkworm/db/state/storage_codecs.cpp +++ b/silkworm/db/state/storage_codecs.cpp @@ -30,4 +30,22 @@ datastore::kvdb::Slice StorageAddressAndLocationKVDBEncoder::encode() { return datastore::kvdb::to_slice(data); } +ByteView StorageAddressAndLocationSnapshotsCodec::encode_word() { + // TODO: this extra copy could be avoided if encoders are able to contain a reference + codec.address.value = value.address; + codec.location_hash.value = value.location_hash; + + word.clear(); + word.reserve(kAddressLength + kHashLength); + word.append(codec.address.encode_word()); + word.append(codec.location_hash.encode_word()); + return word; +} + +void StorageAddressAndLocationSnapshotsCodec::decode_word(ByteView input_word) { + codec.address.decode_word(input_word); + codec.location_hash.decode_word(input_word.substr(kAddressLength)); + value = {codec.address.value, codec.location_hash.value}; +} + } // namespace silkworm::db::state diff --git a/silkworm/db/state/storage_codecs.hpp b/silkworm/db/state/storage_codecs.hpp index 86b90e5a08..c41b9dcc71 100644 --- a/silkworm/db/state/storage_codecs.hpp +++ b/silkworm/db/state/storage_codecs.hpp @@ -38,23 +38,29 @@ struct Bytes32KVDBCodec : public datastore::kvdb::Codec { static_assert(datastore::kvdb::EncoderConcept); static_assert(datastore::kvdb::DecoderConcept); -struct Bytes32Decoder : public snapshots::Decoder { +struct Bytes32SnapshotsCodec : public snapshots::Codec { evmc::bytes32 value; - ~Bytes32Decoder() override = default; + ~Bytes32SnapshotsCodec() override = default; + ByteView encode_word() override { + return ByteView{reinterpret_cast(&value.bytes), sizeof(value.bytes)}; + } void decode_word(ByteView word) override { if (word.size() < sizeof(value.bytes)) - throw std::runtime_error{"Bytes32Decoder failed to decode"}; + throw std::runtime_error{"Bytes32SnapshotsCodec failed to decode"}; std::memcpy(value.bytes, word.data(), sizeof(value.bytes)); } }; -static_assert(snapshots::DecoderConcept); +static_assert(snapshots::EncoderConcept); +static_assert(snapshots::DecoderConcept); + +struct StorageAddressAndLocation { + evmc::address address; + evmc::bytes32 location_hash; +}; struct StorageAddressAndLocationKVDBEncoder : public datastore::kvdb::Encoder { - struct { - evmc::address address; - evmc::bytes32 location_hash; - } value; + StorageAddressAndLocation value; struct { AddressKVDBEncoder address; @@ -70,20 +76,23 @@ struct StorageAddressAndLocationKVDBEncoder : public datastore::kvdb::Encoder { static_assert(datastore::kvdb::EncoderConcept); -struct StorageAddressAndLocationDecoder : public snapshots::Decoder { +struct StorageAddressAndLocationSnapshotsCodec : public snapshots::Codec { + StorageAddressAndLocation value; + struct { - AddressDecoder address; - Bytes32Decoder location_hash; - } value; + AddressSnapshotsCodec address; + Bytes32SnapshotsCodec location_hash; + } codec; - ~StorageAddressAndLocationDecoder() override = default; + Bytes word; - void decode_word(ByteView word) override { - value.address.decode_word(word); - value.location_hash.decode_word(word.substr(kAddressLength)); - } + ~StorageAddressAndLocationSnapshotsCodec() override = default; + + ByteView encode_word() override; + void decode_word(ByteView input_word) override; }; -static_assert(snapshots::DecoderConcept); +static_assert(snapshots::EncoderConcept); +static_assert(snapshots::DecoderConcept); } // namespace silkworm::db::state diff --git a/silkworm/db/state/storage_domain.hpp b/silkworm/db/state/storage_domain.hpp index 456c4988e1..4b66cf29e9 100644 --- a/silkworm/db/state/storage_domain.hpp +++ b/silkworm/db/state/storage_domain.hpp @@ -16,6 +16,7 @@ #pragma once +#include #include #include @@ -23,10 +24,12 @@ namespace silkworm::db::state { -using StorageDomainGetLatestQuery = datastore::kvdb::DomainGetLatestQuery; +using StorageDomainGetLatestQuery = datastore::DomainGetLatestQuery< + StorageAddressAndLocationKVDBEncoder, StorageAddressAndLocationSnapshotsCodec, + Bytes32KVDBCodec, Bytes32SnapshotsCodec>; using StorageDomainPutQuery = datastore::kvdb::DomainPutQuery; using StorageDomainDeleteQuery = datastore::kvdb::DomainDeleteQuery; -using StorageDomainKVSegmentReader = snapshots::segment::KVSegmentReader; +using StorageDomainKVSegmentReader = snapshots::segment::KVSegmentReader; } // namespace silkworm::db::state diff --git a/silkworm/db/state/storage_domain_test.cpp b/silkworm/db/state/storage_domain_test.cpp index 9b492fe85f..e40327a031 100644 --- a/silkworm/db/state/storage_domain_test.cpp +++ b/silkworm/db/state/storage_domain_test.cpp @@ -22,19 +22,33 @@ namespace silkworm::db::state { -TEST_CASE("StorageAddressAndLocationDecoder") { +TEST_CASE("StorageAddressAndLocationSnapshotsCodec.decode_word") { using evmc::literals::operator""_address; using evmc::literals::operator""_bytes32; - StorageAddressAndLocationDecoder decoder; + StorageAddressAndLocationSnapshotsCodec decoder; decoder.decode_word( *from_hex( "000000000000000000636f6e736f6c652e6c6f67" "000000000000000000000000000000000000000000005666856076ebaf477f07")); - CHECK(decoder.value.address.value == 0x000000000000000000636f6e736f6c652e6c6f67_address); - CHECK(decoder.value.location_hash.value == 0x000000000000000000000000000000000000000000005666856076ebaf477f07_bytes32); + CHECK(decoder.value.address == 0x000000000000000000636f6e736f6c652e6c6f67_address); + CHECK(decoder.value.location_hash == 0x000000000000000000000000000000000000000000005666856076ebaf477f07_bytes32); CHECK_THROWS_AS(decoder.decode_word({}), std::runtime_error); } +TEST_CASE("StorageAddressAndLocationSnapshotsCodec.encode_word") { + using evmc::literals::operator""_address; + using evmc::literals::operator""_bytes32; + StorageAddressAndLocationSnapshotsCodec encoder; + + encoder.value.address = 0x000000000000000000636f6e736f6c652e6c6f67_address; + encoder.value.location_hash = 0x000000000000000000000000000000000000000000005666856076ebaf477f07_bytes32; + CHECK( + encoder.encode_word() == + *from_hex( + "000000000000000000636f6e736f6c652e6c6f67" + "000000000000000000000000000000000000000000005666856076ebaf477f07")); +} + } // namespace silkworm::db::state diff --git a/silkworm/db/state/traces_from_inverted_index.hpp b/silkworm/db/state/traces_from_inverted_index.hpp index 5c1de4f0b1..1bd9a4ce28 100644 --- a/silkworm/db/state/traces_from_inverted_index.hpp +++ b/silkworm/db/state/traces_from_inverted_index.hpp @@ -23,6 +23,6 @@ namespace silkworm::db::state { -using TracesFromInvertedIndexKVSegmentReader = snapshots::segment::KVSegmentReader>; +using TracesFromInvertedIndexKVSegmentReader = snapshots::segment::KVSegmentReader>; } // namespace silkworm::db::state diff --git a/silkworm/db/state/traces_to_inverted_index.hpp b/silkworm/db/state/traces_to_inverted_index.hpp index 676a415569..d3b0e8d709 100644 --- a/silkworm/db/state/traces_to_inverted_index.hpp +++ b/silkworm/db/state/traces_to_inverted_index.hpp @@ -23,6 +23,6 @@ namespace silkworm::db::state { -using TracesToInvertedIndexKVSegmentReader = snapshots::segment::KVSegmentReader>; +using TracesToInvertedIndexKVSegmentReader = snapshots::segment::KVSegmentReader>; } // namespace silkworm::db::state diff --git a/silkworm/execution/local_state.cpp b/silkworm/execution/local_state.cpp index 72d0b66b7f..713847d643 100644 --- a/silkworm/execution/local_state.cpp +++ b/silkworm/execution/local_state.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include namespace silkworm::execution { @@ -27,7 +28,12 @@ using namespace db::state; using namespace datastore; std::optional LocalState::read_account(const evmc::address& address) const noexcept { - AccountsDomainGetLatestQuery query{tx_, data_store_.state_db().accounts_domain()}; + AccountsDomainGetLatestQuery query{ + db::state::kDomainNameAccounts, + data_store_.chaindata, + tx_, + data_store_.state_repository, + }; auto result = query.exec(address); if (result) { return std::move(result->value); @@ -36,7 +42,12 @@ std::optional LocalState::read_account(const evmc::address& address) co } ByteView LocalState::read_code(const evmc::address& address, const evmc::bytes32& /*code_hash*/) const noexcept { - CodeDomainGetLatestQuery query{tx_, data_store_.state_db().code_domain()}; + CodeDomainGetLatestQuery query{ + db::state::kDomainNameCode, + data_store_.chaindata, + tx_, + data_store_.state_repository, + }; auto result = query.exec(address); if (result) { return std::move(result->value); @@ -48,7 +59,12 @@ evmc::bytes32 LocalState::read_storage( const evmc::address& address, uint64_t /*incarnation*/, const evmc::bytes32& location) const noexcept { - StorageDomainGetLatestQuery query{tx_, data_store_.state_db().storage_domain()}; + StorageDomainGetLatestQuery query{ + db::state::kDomainNameStorage, + data_store_.chaindata, + tx_, + data_store_.state_repository, + }; auto result = query.exec({address, location}); if (result) { return std::move(result->value); From 50c05dac0cf8955c38bb60ae38dcb10455f32d0f Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Fri, 17 Jan 2025 09:42:32 +0100 Subject: [PATCH 13/33] Update to latest code changes --- silkworm/capi/silkworm.cpp | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/silkworm/capi/silkworm.cpp b/silkworm/capi/silkworm.cpp index 576e6f8f22..3c2880fbcc 100644 --- a/silkworm/capi/silkworm.cpp +++ b/silkworm/capi/silkworm.cpp @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -54,16 +55,19 @@ #include #include #include -#include -#include #include #include +#include +#include +#include #include "common.hpp" #include "instance.hpp" using namespace std::chrono_literals; using namespace silkworm; +// using namespace silkworm::db; +// using namespace silkworm::rpc; static MemoryMappedRegion make_region(const SilkwormMemoryMappedFile& mmf) { return {mmf.memory_address, mmf.memory_length}; @@ -780,6 +784,9 @@ SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* txn, ui // Allow each client to open its own TCP connection to server (sharing one single connection becomes a bottleneck under high load) channel_args.SetInt(GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL, 1); auto grpc_erigon_channel = grpc::CreateCustomChannel("localhost:9090", grpc::InsecureChannelCredentials(), channel_args); + silkworm::rpc::ChannelFactory create_channel = [&]() { + return grpc::CreateCustomChannel("localhost:9090", grpc::InsecureChannelCredentials(), channel_args); + }; silkworm::rpc::ClientContextPool context_pool{1}; @@ -787,10 +794,13 @@ SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* txn, ui auto& ioc = *context.ioc(); auto& grpc_context{*context.grpc_context()}; - auto state_cache{std::make_unique(db::kv::api::CoherentCacheConfig{})}; + // auto state_cache{std::make_unique(db::kv::api::CoherentCacheConfig{})}; + silkworm::db::kv::api::CoherentStateCache state_cache; + + auto backend{std::make_unique(grpc_erigon_channel, grpc_context)}; - auto backend{std::make_unique(ioc, grpc_erigon_channel, grpc_context)}; - auto database = std::make_unique(backend.get(), state_cache.get(), grpc_context, grpc_erigon_channel); + auto database = std::make_unique( + create_channel, grpc_context, &state_cache, silkworm::rpc::ethdb::kv::make_backend_providers(backend.get())); context_pool.start(); auto _ = gsl::finally([&context_pool] { @@ -801,10 +811,10 @@ SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* txn, ui Block block{}; // todo: get block header auto state = concurrency::spawn_future_and_wait(ioc, [&]() -> Task> { - auto kv_transaction = co_await database->begin(); + auto kv_transaction = co_await database->service()->begin_transaction(); const auto chain_storage = kv_transaction->create_storage(); auto this_executor = co_await boost::asio::this_coro::executor; - auto remote_state = std::make_unique(this_executor, *kv_transaction, *chain_storage, 1, std::nullopt); + auto remote_state = std::make_unique(this_executor, *kv_transaction, *chain_storage, 1); // auto a = hex_to_address("0x71562b71999873db5b286df957af199ec94617f7"); // auto acc = remote_state->read_account(a); @@ -842,10 +852,9 @@ SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* txn, ui log::Info{"account2 not found"}; } - log::Info{"dupa blada, spadam"}; - // const auto chain_config = *chain_info; - // auto protocol_rule_set_{protocol::rule_set_factory(*chain_config)}; - // ExecutionProcessor processor{block, *protocol_rule_set_, *state, *chain_config}; + const auto chain_config = *chain_info; + auto protocol_rule_set_{protocol::rule_set_factory(*chain_config)}; + ExecutionProcessor processor{block, *protocol_rule_set_, *state, *chain_config, false}; // // add analysis cache, check block exec for more // silkworm::Transaction transaction{}; // todo: get txn From befd4b192e97dbcaa3e1b6564f521cec8fb07c64 Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Sun, 19 Jan 2025 21:12:59 +0100 Subject: [PATCH 14/33] Add DomainState and tests --- silkworm/capi/instance.hpp | 2 + silkworm/capi/silkworm.cpp | 175 +++++++++----------- silkworm/capi/silkworm.h | 18 +- silkworm/db/cli/snapshots.cpp | 2 +- silkworm/db/datastore/kvdb/database.hpp | 34 ++-- silkworm/db/datastore/kvdb/mdbx.hpp | 14 +- silkworm/db/kv/state_reader.cpp | 31 +--- silkworm/db/test_util/mock_txn.hpp | 20 +++ silkworm/execution/CMakeLists.txt | 4 + silkworm/execution/database_tcontext.cpp | 201 +++++++++++++++++++++++ silkworm/execution/database_tcontext.hpp | 96 +++++++++++ silkworm/execution/domain_state.cpp | 190 +++++++++++++++++++++ silkworm/execution/domain_state.hpp | 122 ++++++++++++++ silkworm/execution/domain_state_test.cpp | 189 +++++++++++++++++++++ 14 files changed, 945 insertions(+), 153 deletions(-) create mode 100644 silkworm/execution/database_tcontext.cpp create mode 100644 silkworm/execution/database_tcontext.hpp create mode 100644 silkworm/execution/domain_state.cpp create mode 100644 silkworm/execution/domain_state.hpp create mode 100644 silkworm/execution/domain_state_test.cpp diff --git a/silkworm/capi/instance.hpp b/silkworm/capi/instance.hpp index 59b0c5fb02..11bf0e302a 100644 --- a/silkworm/capi/instance.hpp +++ b/silkworm/capi/instance.hpp @@ -53,6 +53,8 @@ struct SilkwormInstance { std::unique_ptr rpcdaemon; std::unique_ptr execution_engine; + std::optional chain_config; + // sentry std::unique_ptr sentry_thread; boost::asio::cancellation_signal sentry_stop_signal; diff --git a/silkworm/capi/silkworm.cpp b/silkworm/capi/silkworm.cpp index 2f6bc3b683..5908ca1ea0 100644 --- a/silkworm/capi/silkworm.cpp +++ b/silkworm/capi/silkworm.cpp @@ -42,9 +42,10 @@ #include #include #include +#include #include #include -#include +#include #include #include #include @@ -54,12 +55,12 @@ #include #include #include +#include #include #include #include #include #include -#include #include "common.hpp" #include "instance.hpp" @@ -751,125 +752,105 @@ int silkworm_execute_blocks_perpetual(SilkwormHandle handle, MDBX_env* mdbx_env, } // todo: add available gas, add txn, add block header -SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* txn, uint64_t block_num, uint64_t tx_index, uint64_t* gas_used, uint64_t* blob_gas_used) SILKWORM_NOEXCEPT { - log::Info{"silkworm_execute_tx", {"block_num", std::to_string(block_num), "tx_index", std::to_string(tx_index)}}; +SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* mdbx_tx, uint64_t block_num, struct SilkwormBytes32 head_hash_bytes, uint64_t txn_num, uint64_t txn_id, uint64_t* gas_used, uint64_t* blob_gas_used) SILKWORM_NOEXCEPT { + log::Info{"silkworm_execute_tx", {"block_num", std::to_string(block_num), "txn_num", std::to_string(txn_num)}}; if (!handle) { return SILKWORM_INVALID_HANDLE; } - if (!txn) { + if (!mdbx_tx) { return SILKWORM_INVALID_MDBX_TXN; } - if (block_num == 0) { - return SILKWORM_INVALID_BLOCK; - } - - if (tx_index == 0) { - return SILKWORM_INVALID_BLOCK; - } - if (gas_used) { - *gas_used = 1; + *gas_used = 0; } if (blob_gas_used) { - *blob_gas_used = 1; + *blob_gas_used = 0; } - // const auto chain_info = kKnownChainConfigs.find(chain_id); - const auto chain_info = kKnownChainConfigs.find(1); - if (!chain_info) { - return SILKWORM_UNKNOWN_CHAIN_ID; - } - - // - grpc::ChannelArguments channel_args; - // Allow to receive messages up to specified max size - channel_args.SetMaxReceiveMessageSize(64 * 1024 * 1024); - // Allow each client to open its own TCP connection to server (sharing one single connection becomes a bottleneck under high load) - channel_args.SetInt(GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL, 1); - auto grpc_erigon_channel = grpc::CreateCustomChannel("localhost:9090", grpc::InsecureChannelCredentials(), channel_args); - silkworm::rpc::ChannelFactory create_channel = [&]() { - return grpc::CreateCustomChannel("localhost:9090", grpc::InsecureChannelCredentials(), channel_args); - }; - - silkworm::rpc::ClientContextPool context_pool{1}; - - auto& context = context_pool.next_context(); - auto& ioc = *context.ioc(); - auto& grpc_context{*context.grpc_context()}; - - // auto state_cache{std::make_unique(db::kv::api::CoherentCacheConfig{})}; - silkworm::db::kv::api::CoherentStateCache state_cache; - - auto backend{std::make_unique(grpc_erigon_channel, grpc_context)}; - - auto database = std::make_unique( - create_channel, grpc_context, &state_cache, silkworm::rpc::ethdb::kv::make_backend_providers(backend.get())); - - context_pool.start(); - auto _ = gsl::finally([&context_pool] { - context_pool.stop(); - context_pool.join(); - }); - - Block block{}; // todo: get block header - - auto state = concurrency::spawn_future_and_wait(ioc, [&]() -> Task> { - auto kv_transaction = co_await database->service()->begin_transaction(); - const auto chain_storage = kv_transaction->create_storage(); - auto this_executor = co_await boost::asio::this_coro::executor; - auto remote_state = std::make_unique(this_executor, *kv_transaction, *chain_storage, 1); + silkworm::Hash head_hash{}; + memcpy(head_hash.bytes, head_hash_bytes.bytes, sizeof(head_hash.bytes)); + BlockNum block_number{block_num}; + TxnId txn_id_{txn_id}; - // auto a = hex_to_address("0x71562b71999873db5b286df957af199ec94617f7"); - // auto acc = remote_state->read_account(a); - - // if (acc) { - // log::Info{"account", {"balance", std::to_string(acc->balance.num_bits)}}; - // } else { - // log::Info{"account not found"}; - // } - - co_return remote_state; - // co_return execution::StateFactory{*kv_transaction}.create_state(this_executor, *chain_storage, block.header.number); - // co_return kv_transaction->create_state(this_executor, *chain_storage, block.header.number); - }); + auto unmanaged_tx = datastore::kvdb::RWTxnUnmanaged{mdbx_tx}; + auto unmanaged_env = unmanaged_tx.unmanaged_env(); + auto chain_db = db::DataStore::make_chaindata_database(std::move(unmanaged_env)); + auto state = silkworm::execution::DomainState{txn_id_, unmanaged_tx, chain_db, *handle->blocks_repository, *handle->state_repository}; + if (!handle->chain_config) { + handle->chain_config = db::read_chain_config(unmanaged_tx); + } + //! Manual tests: remove when done // auto b_hash = to_bytes32(*from_hex("0xe8b28e4882bcbd6293ef56433c69b34e9e3e5bf512a05ddbed6bb94aa65948f4")); // auto b_number = BlockNum{2910651}; + // auto b_hash = to_bytes32(*from_hex("0x6f81fc8bb897eb2075ffa53a2b28c3216022fd1841d361af7908198f8ff2faa3")); + // auto b_number = BlockNum{1}; + // auto header = state.read_header(b_number, b_hash); + // if (header) { + // log::Info{"JG header", {"number", std::to_string(header->number), "gas_used", std::to_string(header->gas_used)}}; + // } else { + // log::Warning{"header not found"}; + // return SILKWORM_INVALID_BLOCK; + // } + // silkworm::Block block{}; + // auto block_read_ok = state.read_body(b_number, b_hash, block); + // if (block_read_ok) { + // log::Info{"JG body", {"number", std::to_string(block.header.number), "transactions", std::to_string(block.transactions.size())}}; + // } else { + // log::Warning{"body not found"}; + // return SILKWORM_INVALID_BLOCK; + // } + // block.header = header.value(); + // auto a = hex_to_address("0x71562b71999873db5b286df957af199ec94617f7"); + // auto acc = state.read_account(a); + // if (acc) { + // log::Info{"account2", {"balance", std::to_string(acc->balance.num_bits)}}; + // } else { + // log::Info{"account2 not found"}; + // } - auto b_hash = to_bytes32(*from_hex("0x6f81fc8bb897eb2075ffa53a2b28c3216022fd1841d361af7908198f8ff2faa3")); - auto b_number = BlockNum{1}; - - auto header = state->read_header(b_number, b_hash); - if (header) { - log::Info{"JG header", {"number", std::to_string(header->number), "gas_used", std::to_string(header->gas_used)}}; - } else { - log::Warning{"header not found"}; + //TODO: cache block, also consider preloading + silkworm::Block block{}; + auto block_read_ok = state.read_body(block_number, head_hash, block); + if (!block_read_ok) { + SILK_ERROR << "Block not found" << " block_number: " << block_number << " head_hash: " << head_hash; + return SILKWORM_INVALID_BLOCK; + } + auto header = state.read_header(block_number, head_hash); + if (!header) { + SILK_ERROR << "Header not found" << " block_number: " << block_number << " head_hash: " << head_hash; return SILKWORM_INVALID_BLOCK; } + block.header = header.value(); - auto a = hex_to_address("0x71562b71999873db5b286df957af199ec94617f7"); - auto acc = state->read_account(a); - if (acc) { - log::Info{"account2", {"balance", std::to_string(acc->balance.num_bits)}}; - } else { - log::Info{"account2 not found"}; + if (txn_num >= block.transactions.size()) { + SILK_ERROR << "Transaction not found" << " txn_num: " << txn_num; + return SILKWORM_INVALID_BLOCK; } - const auto chain_config = *chain_info; - auto protocol_rule_set_{protocol::rule_set_factory(*chain_config)}; - ExecutionProcessor processor{block, *protocol_rule_set_, *state, *chain_config, false}; - // // add analysis cache, check block exec for more + auto& transaction = block.transactions[txn_num]; - // silkworm::Transaction transaction{}; // todo: get txn - // silkworm::Receipt receipt{}; - // const ValidationResult err{protocol::validate_transaction(transaction, processor.intra_block_state(), processor.available_gas())}; - // if (err != ValidationResult::kOk) { - // return SILKWORM_INVALID_BLOCK; - // } - // processor.execute_transaction(transaction, receipt); + auto protocol_rule_set_{protocol::rule_set_factory(*handle->chain_config)}; + ExecutionProcessor processor{block, *protocol_rule_set_, state, *handle->chain_config, false}; + //TODO: add analysis cache, check block exec for more + + silkworm::Receipt receipt{}; + const ValidationResult err{protocol::validate_transaction(transaction, processor.intra_block_state(), processor.available_gas())}; + if (err != ValidationResult::kOk) { + return SILKWORM_INVALID_BLOCK; + } + processor.execute_transaction(transaction, receipt); + state.insert_receipts(block_number, std::vector{receipt}); + + if (gas_used) { + *gas_used = receipt.cumulative_gas_used; + } + if (blob_gas_used) { + *blob_gas_used = transaction.total_blob_gas(); + } return SILKWORM_OK; } diff --git a/silkworm/capi/silkworm.h b/silkworm/capi/silkworm.h index e97365e598..258a3cef2b 100644 --- a/silkworm/capi/silkworm.h +++ b/silkworm/capi/silkworm.h @@ -248,8 +248,8 @@ SILKWORM_EXPORT int silkworm_sentry_stop(SilkwormHandle handle) SILKWORM_NOEXCEP //! Silkworm Fork Validator configuration options struct SilkwormForkValidatorSettings { - uint64_t batch_size; // Batch size to use in stages - uint64_t etl_buffer_size; // Buffer size for ETL operations + uint64_t batch_size; // Batch size to use in stages + uint64_t etl_buffer_size; // Buffer size for ETL operations uint32_t sync_loop_throttle_seconds; // Minimum interval amongst sync cycle bool stop_before_senders_stage; // Stop before senders stage }; @@ -358,7 +358,19 @@ SILKWORM_EXPORT int silkworm_execute_blocks_perpetual(SilkwormHandle handle, MDB bool write_change_sets, bool write_receipts, bool write_call_traces, uint64_t* last_executed_block, int* mdbx_error_code) SILKWORM_NOEXCEPT; -SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* txn, uint64_t block_num, uint64_t tx_index, uint64_t* gas_used, uint64_t* blob_gas_used) SILKWORM_NOEXCEPT; +/** + * \brief Execute a transaction in a block. + * \param[in] handle A valid Silkworm instance handle, got with silkworm_init. + * \param[in] mdbx_tx A valid external read-write MDBX transaction. + * \param[in] block_num The block number. + * \param[in] head_hash_bytes The hash of the head block. + * \param[in] txn_num The transaction number in the block. + * \param[in] txn_id The transaction ID. + * \param[out] gas_used The gas used by the transaction. + * \param[out] blob_gas_used The blob gas used by the transaction. + * \return SILKWORM_OK (=0) on success, a non-zero error value on failure. + */ +SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* mdbx_tx, uint64_t block_num, struct SilkwormBytes32 head_hash_bytes, uint64_t txn_num, uint64_t txn_id, uint64_t* gas_used, uint64_t* blob_gas_used) SILKWORM_NOEXCEPT; /** * \brief Finalize the Silkworm C API library. diff --git a/silkworm/db/cli/snapshots.cpp b/silkworm/db/cli/snapshots.cpp index 63376ac3ce..61997d516e 100644 --- a/silkworm/db/cli/snapshots.cpp +++ b/silkworm/db/cli/snapshots.cpp @@ -544,7 +544,7 @@ void open_existence_index(const SnapshotSubcommandSettings& settings) { ByteView nonexistent_key = {full_be + kSizeDiff, sizeof(intx::uint256) - kSizeDiff}; SILK_TRACE << "KV: previous_key=" << to_hex(previous_key) << " key=" << to_hex(key) << " nonexistent_key=" << to_hex(nonexistent_key); - if (const bool key_found = existence_index.contains(nonexistent_key); key_found) { + if (existence_index.contains(nonexistent_key)) { ++nonexistent_found_count; } } diff --git a/silkworm/db/datastore/kvdb/database.hpp b/silkworm/db/datastore/kvdb/database.hpp index 36c42af19c..821d1e1ac4 100644 --- a/silkworm/db/datastore/kvdb/database.hpp +++ b/silkworm/db/datastore/kvdb/database.hpp @@ -17,6 +17,7 @@ #pragma once #include +#include // for std::move #include "domain.hpp" #include "inverted_index.hpp" @@ -58,14 +59,9 @@ class DatabaseRef { DatabaseRef::EntitiesMap make_entities(const Schema::DatabaseDef& schema); -class Database { +class DatabaseBase { public: - Database( - mdbx::env_managed env, - Schema::DatabaseDef schema) - : env_{std::move(env)}, - schema_{std::move(schema)}, - entities_{make_entities(schema_)} {} + virtual ~DatabaseBase() = default; ROAccess access_ro() const { return ref().access_ro(); } RWAccess access_rw() const { return ref().access_rw(); } @@ -73,7 +69,19 @@ class Database { Domain domain(datastore::EntityName name) const { return ref().domain(name); } InvertedIndex inverted_index(datastore::EntityName name) { return ref().inverted_index(name); } - DatabaseRef ref() const { return {env_, schema_, entities_}; } // NOLINT(cppcoreguidelines-slicing) + virtual DatabaseRef ref() const = 0; +}; + +class Database : public DatabaseBase { + public: + Database( + mdbx::env_managed env, + Schema::DatabaseDef schema) + : env_{std::move(env)}, + schema_{std::move(schema)}, + entities_{make_entities(schema_)} {} + + DatabaseRef ref() const override { return {env_, schema_, entities_}; } // NOLINT(cppcoreguidelines-slicing) void create_tables(); @@ -83,7 +91,7 @@ class Database { std::map> entities_; }; -class DatabaseUnmanaged { +class DatabaseUnmanaged : public DatabaseBase { public: DatabaseUnmanaged( EnvUnmanaged env, @@ -92,13 +100,7 @@ class DatabaseUnmanaged { schema_{std::move(schema)}, entities_{make_entities(schema_)} {} - ROAccess access_ro() const { return ref().access_ro(); } - RWAccess access_rw() const { return ref().access_rw(); } - - Domain domain(datastore::EntityName name) const { return ref().domain(name); } - InvertedIndex inverted_index(datastore::EntityName name) { return ref().inverted_index(name); } - - DatabaseRef ref() const { return {env_, schema_, entities_}; } // NOLINT(cppcoreguidelines-slicing) + DatabaseRef ref() const override { return {env_, schema_, entities_}; } // NOLINT(cppcoreguidelines-slicing) private: EnvUnmanaged env_; diff --git a/silkworm/db/datastore/kvdb/mdbx.hpp b/silkworm/db/datastore/kvdb/mdbx.hpp index 697feffb50..912b782ff4 100644 --- a/silkworm/db/datastore/kvdb/mdbx.hpp +++ b/silkworm/db/datastore/kvdb/mdbx.hpp @@ -324,6 +324,12 @@ class RWTxnManaged : public RWTxn { mdbx::txn_managed managed_txn_; }; +//! \brief EnvUnmanaged wraps an *unmanaged* MDBX environment, which means the underlying environment +//! lifecycle is not touched by this class. +struct EnvUnmanaged : public ::mdbx::env { + explicit EnvUnmanaged(MDBX_env* ptr) : ::mdbx::env{ptr} {} +}; + //! \brief RWTxnUnmanaged wraps an *unmanaged* read-write transaction, which means the underlying transaction //! lifecycle is not touched by this class: the transaction is neither committed nor aborted. class RWTxnUnmanaged : public RWTxn, protected ::mdbx::txn { @@ -334,6 +340,8 @@ class RWTxnUnmanaged : public RWTxn, protected ::mdbx::txn { void abort() override { throw std::runtime_error{"RWTxnUnmanaged must not be aborted"}; } void commit_and_renew() override { throw std::runtime_error{"RWTxnUnmanaged must not be committed"}; } void commit_and_stop() override { throw std::runtime_error{"RWTxnUnmanaged must not be committed"}; } + + EnvUnmanaged unmanaged_env() const { return EnvUnmanaged{env()}; } }; //! \brief This class create ROTxn(s) on demand, it is used to enforce in some method signatures the type of db access @@ -383,12 +391,6 @@ struct EnvConfig { uint32_t max_readers{100}; // Default max number of readers }; -//! \brief EnvUnmanaged wraps an *unmanaged* MDBX environment, which means the underlying environment -//! lifecycle is not touched by this class. -struct EnvUnmanaged : public ::mdbx::env { - explicit EnvUnmanaged(MDBX_env* ptr) : ::mdbx::env{ptr} {} -}; - //! \brief Opens an mdbx environment using the provided environment config //! \param [in] config : A structure containing essential environment settings //! \return A handler to mdbx::env_managed class diff --git a/silkworm/db/kv/state_reader.cpp b/silkworm/db/kv/state_reader.cpp index c983b665d7..fe9c7b142f 100644 --- a/silkworm/db/kv/state_reader.cpp +++ b/silkworm/db/kv/state_reader.cpp @@ -16,8 +16,6 @@ #include "state_reader.hpp" -#include - #include #include #include @@ -26,7 +24,6 @@ #include #include #include -#include namespace silkworm::db::kv { @@ -34,42 +31,16 @@ StateReader::StateReader(kv::api::Transaction& tx, TxnId txn_id) : tx_(tx), txn_ } Task> StateReader::read_account(const evmc::address& address) const { - if (!txn_number_) { - SILK_DEBUG << "test 1"; - SILK_DEBUG << "StateReader::read_account: " - << " first_txn_num_in_block " << *block_num_; - SILK_DEBUG << "test 1"; - if (tx_.is_local()) { - SILK_DEBUG << "StateReader::read_account local: tx_ type: "; - } else { - SILK_DEBUG << "StateReader::read_account: tx_ type: "; - } - - txn_number_ = co_await tx_.first_txn_num_in_block(*block_num_); - } - - SILK_DEBUG << "StateReader::read_account: " - << " query"; db::kv::api::GetAsOfQuery query{ .table = table::kAccountDomain, .key = db::account_domain_key(address), .timestamp = static_cast(txn_number_), }; - SILK_DEBUG << "StateReader::read_account: " - << " query2"; const auto result = co_await tx_.get_as_of(std::move(query)); - SILK_DEBUG << "StateReader::read_account: " - << " get_as_of"; - if (!result.success) { - SILK_DEBUG << "StateReader::read_account: " - << " no success"; co_return std::nullopt; } - SILK_DEBUG << "StateReader::read_account: " - << " success" << to_hex(result.value); - // const auto account{Account::from_encoded_storage_v3(result.value)}; const auto account = db::state::AccountCodec::from_encoded_storage_v3(result.value); success_or_throw(account); co_return *account; @@ -107,4 +78,4 @@ Task> StateReader::read_code(const evmc::address& address, co_return result.value; } -} // namespace silkworm::db::kv +} // namespace silkworm::db::kv \ No newline at end of file diff --git a/silkworm/db/test_util/mock_txn.hpp b/silkworm/db/test_util/mock_txn.hpp index aa8347a6fb..7fe6fa910f 100644 --- a/silkworm/db/test_util/mock_txn.hpp +++ b/silkworm/db/test_util/mock_txn.hpp @@ -36,4 +36,24 @@ class MockROTxn : public datastore::kvdb::ROTxn { ::mdbx::txn txn_; }; +class MockRwTxn : public datastore::kvdb::RWTxn { + public: + explicit MockRwTxn() : datastore::kvdb::RWTxn(txn_) {} + + MOCK_METHOD((bool), is_open, (), (const, override)); + MOCK_METHOD((mdbx::env), db, (), (const, override)); + MOCK_METHOD((std::unique_ptr), ro_cursor, (const datastore::kvdb::MapConfig&), (override)); + MOCK_METHOD((std::unique_ptr), ro_cursor_dup_sort, (const datastore::kvdb::MapConfig&), (override)); + MOCK_METHOD((std::unique_ptr), rw_cursor, (const datastore::kvdb::MapConfig&), ()); + MOCK_METHOD((std::unique_ptr), rw_cursor_dup_sort, (const datastore::kvdb::MapConfig&), ()); + MOCK_METHOD((void), commit, (), ()); + MOCK_METHOD((void), abort, (), (override)); + MOCK_METHOD((void), commit_and_renew, (), ()); + MOCK_METHOD((void), commit_and_stop, (), ()); + + private: + ::mdbx::txn txn_; + // silkworm::datastore::kvdb::RWCursor aa; +}; + } // namespace silkworm::db::test_util diff --git a/silkworm/execution/CMakeLists.txt b/silkworm/execution/CMakeLists.txt index ff783102dc..aac33d663e 100644 --- a/silkworm/execution/CMakeLists.txt +++ b/silkworm/execution/CMakeLists.txt @@ -19,6 +19,7 @@ include("${SILKWORM_MAIN_DIR}/cmake/common/targets.cmake") find_package(asio-grpc REQUIRED) find_package(gRPC REQUIRED) find_package(GTest REQUIRED) +find_package(nlohmann_json REQUIRED) # cmake-format: off set(LIBS_PRIVATE @@ -30,9 +31,12 @@ set(LIBS_PRIVATE # cmake-format: off set(LIBS_PUBLIC asio-grpc::asio-grpc + nlohmann_json::nlohmann_json silkworm_core silkworm_infra silkworm_db + silkworm_block_execution + silkworm_snapshots ) # cmake-format: on diff --git a/silkworm/execution/database_tcontext.cpp b/silkworm/execution/database_tcontext.cpp new file mode 100644 index 0000000000..9cf5d271d1 --- /dev/null +++ b/silkworm/execution/database_tcontext.cpp @@ -0,0 +1,201 @@ +/* + Copyright 2025 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "database_tcontext.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace silkworm::execution { + +using namespace silkworm::datastore::kvdb; + +std::filesystem::path get_tests_dir() { + auto working_dir = std::filesystem::current_path(); + + while (working_dir != "/") { + if (std::filesystem::exists(working_dir / "third_party" / "execution-apis")) { + return working_dir / "third_party" / "execution-apis" / "tests"; + } + + if (std::filesystem::exists(working_dir / "silkworm" / "third_party" / "execution-apis")) { + return working_dir / "silkworm" / "third_party" / "execution-apis" / "tests"; + } + + if (std::filesystem::exists(working_dir / "project" / "third_party" / "execution-apis")) { + return working_dir / "project" / "third_party" / "execution-apis" / "tests"; + } + + working_dir = working_dir.parent_path(); + } + + throw std::logic_error("Failed to find tests directory"); +} + +InMemoryState populate_genesis(RWTxn& txn, const std::filesystem::path& tests_dir) { + auto genesis_json_path = tests_dir / "genesis.json"; + std::ifstream genesis_json_input_file(genesis_json_path); + nlohmann::json genesis_json; + genesis_json_input_file >> genesis_json; + + InMemoryState state = read_genesis_allocation(genesis_json.at("alloc")); + silkworm::db::write_genesis_allocation_to_db(txn, state); + + BlockHeader header{read_genesis_header(genesis_json, state.state_root_hash())}; + BlockBody block_body{ + .withdrawals = std::vector{0}, + }; + + // FIX 2: set empty receipts root, should be done in the main code, requires https://github.com/erigontech/silkworm/issues/1348 + header.withdrawals_root = kEmptyRoot; + + auto block_hash{header.hash()}; + auto block_hash_key{silkworm::db::block_key(header.number, block_hash.bytes)}; + silkworm::db::write_header(txn, header, /*with_header_numbers=*/true); // Write table::kHeaders and table::kHeaderNumbers + silkworm::db::write_canonical_header_hash(txn, block_hash.bytes, header.number); // Insert header hash as canonical + silkworm::db::write_total_difficulty(txn, block_hash_key, header.difficulty); // Write initial difficulty + + silkworm::db::write_body(txn, block_body, block_hash.bytes, header.number); // Write block body (empty) + silkworm::db::write_head_header_hash(txn, block_hash.bytes); // Update head header in config + + const uint8_t genesis_null_receipts[] = {0xf6}; // <- cbor encoded + open_cursor(txn, silkworm::db::table::kBlockReceipts) + .upsert(datastore::kvdb::to_slice(block_hash_key).safe_middle(0, 8), datastore::kvdb::to_slice(Bytes(genesis_null_receipts, 1))); + + // Write Chain Settings + auto config_data{genesis_json["config"].dump()}; + open_cursor(txn, silkworm::db::table::kConfig) + .upsert(to_slice(ByteView{block_hash.bytes}), mdbx::slice{config_data.data()}); + + return state; +} + +void populate_blocks(RWTxn& txn, const std::filesystem::path& tests_dir, InMemoryState& state_buffer) { + auto rlp_path = tests_dir / "chain.rlp"; + std::ifstream file(rlp_path, std::ios::binary); + if (!file) { + throw std::logic_error("Failed to open the file: " + rlp_path.string()); + } + std::vector rlps; + std::vector line; + + Bytes rlp_buffer(std::istreambuf_iterator(file), {}); + file.close(); + ByteView rlp_view{rlp_buffer}; + + auto chain_config = silkworm::db::read_chain_config(txn); + + if (!chain_config.has_value()) { + throw std::logic_error("Failed to read chain config"); + } + auto rule_set = protocol::rule_set_factory(*chain_config); + + while (!rlp_view.empty()) { + silkworm::Block block; + + if (!silkworm::rlp::decode(rlp_view, block, silkworm::rlp::Leftover::kAllow)) { + throw std::logic_error("Failed to decode RLP file"); + } + + // store original hashes + auto block_hash = block.header.hash(); + auto block_hash_key = silkworm::db::block_key(block.header.number, block_hash.bytes); + + // FIX 3: populate senders table + silkworm::db::write_senders(txn, block_hash, block.header.number, block); + + // FIX 4a: populate tx lookup table and create receipts + silkworm::db::write_tx_lookup(txn, block); + + // FIX 4b: populate receipts and logs table + std::vector receipts; + ExecutionProcessor processor{block, *rule_set, state_buffer, *chain_config, true}; + silkworm::db::Buffer db_buffer{txn, std::make_unique(txn)}; + for (auto& block_txn : block.transactions) { + silkworm::Receipt receipt{}; + processor.execute_transaction(block_txn, receipt); + receipts.emplace_back(receipt); + } + processor.evm().state().write_to_db(block.header.number); + db_buffer.insert_receipts(block.header.number, receipts); + db_buffer.write_history_to_db(); + + // FIX 5: insert system transactions + intx::uint256 max_priority_fee_per_gas = block.transactions.empty() ? block.header.base_fee_per_gas.value_or(0) : block.transactions[0].max_priority_fee_per_gas; + intx::uint256 max_fee_per_gas = block.transactions.empty() ? block.header.base_fee_per_gas.value_or(0) : block.transactions[0].max_fee_per_gas; + silkworm::Transaction system_transaction; + system_transaction.max_priority_fee_per_gas = max_priority_fee_per_gas; + system_transaction.max_fee_per_gas = max_fee_per_gas; + block.transactions.emplace(block.transactions.begin(), system_transaction); + block.transactions.emplace_back(system_transaction); + + silkworm::db::write_header(txn, block.header, /*with_header_numbers=*/true); // Write table::kHeaders and table::kHeaderNumbers + silkworm::db::write_canonical_header_hash(txn, block_hash.bytes, block.header.number); // Insert header hash as canonical + + // TODO: find how to decode total difficulty + // write_total_difficulty(txn, block_hash_key, block.header.difficulty); // Write initial difficulty + silkworm::db::write_total_difficulty(txn, block_hash_key, 1); // Write initial difficulty + + silkworm::db::write_raw_body(txn, block, block_hash, block.header.number); + silkworm::db::write_head_header_hash(txn, block_hash.bytes); // Update head header in config + silkworm::db::write_last_head_block(txn, block_hash); // Update head block in config + silkworm::db::write_last_safe_block(txn, block_hash); // Update last safe block in config + silkworm::db::write_last_finalized_block(txn, block_hash); // Update last finalized block in config + } +} + +namespace { + mdbx::env_managed initialize_test_database(const std::filesystem::path& chaindata_dir_path) { + const auto tests_dir = get_tests_dir(); + + EnvConfig env_config{ + .path = chaindata_dir_path.string(), + .create = true, + .exclusive = true, + .in_memory = true, + .shared = false, + }; + auto env = open_env(env_config); + + RWTxnManaged txn{env}; + silkworm::db::table::check_or_create_chaindata_tables(txn); + auto state_buffer = populate_genesis(txn, tests_dir); + populate_blocks(txn, tests_dir, state_buffer); + txn.commit_and_stop(); + + return env; + } + +} // namespace + +TestDatabaseContext::TestDatabaseContext(const TemporaryDirectory& tmp_dir) + : chaindata_dir_path_{DataDirectory{tmp_dir.path()}.chaindata().path()}, + env_{std::make_unique(initialize_test_database(chaindata_dir_path_))} {} + +silkworm::ChainConfig TestDatabaseContext::get_chain_config() const { + ROTxnManaged txn = chaindata().start_ro_tx(); + auto chain_config = silkworm::db::read_chain_config(txn); + return chain_config ? *chain_config : silkworm::ChainConfig{}; +} + +} // namespace silkworm::execution diff --git a/silkworm/execution/database_tcontext.hpp b/silkworm/execution/database_tcontext.hpp new file mode 100644 index 0000000000..9242b6c75c --- /dev/null +++ b/silkworm/execution/database_tcontext.hpp @@ -0,0 +1,96 @@ +/* + Copyright 2025 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include + +//TODO: this is a duplicate of test_database_context.hpp +//check linking issues and remove this file + +//TODO: populate_genesis and populate_blocks should write to the domain tables + +namespace silkworm::execution { + +std::filesystem::path get_tests_dir(); + +InMemoryState populate_genesis(db::RWTxn& txn, const std::filesystem::path& tests_dir); + +void populate_blocks(db::RWTxn& txn, const std::filesystem::path& tests_dir, InMemoryState& state_buffer); + +class TestDatabaseContext { + public: + explicit TestDatabaseContext(const TemporaryDirectory& tmp_dir); + virtual ~TestDatabaseContext() = default; + + virtual datastore::kvdb::ROAccess chaindata() const { + return datastore::kvdb::ROAccess{*env_}; + } + virtual datastore::kvdb::RWAccess chaindata_rw() const { + return datastore::kvdb::RWAccess{*env_}; + } + + silkworm::ChainConfig get_chain_config() const; + const std::filesystem::path& chaindata_dir_path() const { return chaindata_dir_path_; } + + protected: + mdbx::env_managed move_env() { + mdbx::env_managed env{std::move(*env_)}; + env_.reset(); + return env; + } + + std::filesystem::path chaindata_dir_path_; + std::unique_ptr env_; +}; + +class TestDataStore : public TestDatabaseContext { + public: + explicit TestDataStore(const TemporaryDirectory& tmp_dir) + : TestDatabaseContext{tmp_dir}, + data_store_{ + move_env(), + DataDirectory{tmp_dir.path(), true}.snapshots().path(), + } {} + ~TestDataStore() override = default; + + db::DataStore& operator*() { return data_store_; } + db::DataStore* operator->() { return &data_store_; } + + datastore::kvdb::ROAccess chaindata() const override { + return data_store_.chaindata().access_ro(); + } + datastore::kvdb::RWAccess chaindata_rw() const override { + return data_store_.chaindata().access_rw(); + } + + db::DataModelFactory data_model_factory() { + return db::DataModelFactory{data_store_.ref()}; + } + + private: + db::DataStore data_store_; +}; + +} // namespace silkworm::execution diff --git a/silkworm/execution/domain_state.cpp b/silkworm/execution/domain_state.cpp new file mode 100644 index 0000000000..856f9c6d8c --- /dev/null +++ b/silkworm/execution/domain_state.cpp @@ -0,0 +1,190 @@ +/* + Copyright 2025 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "domain_state.hpp" + +#include +#include +#include +#include +#include + +namespace silkworm::execution { + +using namespace db::state; +using namespace datastore; + +//TODO: +// - implement: state_root_hash, canonize_block, decanonize_block, insert_call_traces +// - add begin_txn method (replacing begin_block?) +// - insert_receipts should write to domain receipts table db::state::kDomainNameReceipts +// - add buffer saving previous steps for values in accounts, code and storage domains - for updates +// - extend transaction to include txn_id, or +// - add base_txn_id to block and get_txn_by_id method + +std::optional DomainState::read_account(const evmc::address& address) const noexcept { + AccountsDomainGetLatestQuery query{ + db::state::kDomainNameAccounts, + database_.domain(db::state::kDomainNameAccounts), + tx_, + state_repository_, + }; + auto result = query.exec(address); + if (result) { + return std::move(result->value); + } + return std::nullopt; +} + +ByteView DomainState::read_code(const evmc::address& address, const evmc::bytes32& /*code_hash*/) const noexcept { + CodeDomainGetLatestQuery query{ + db::state::kDomainNameCode, + database_.domain(db::state::kDomainNameCode), + tx_, + state_repository_, + }; + auto result = query.exec(address); + if (result) { + return std::move(result->value); + } + return ByteView{}; +} + +evmc::bytes32 DomainState::read_storage( + const evmc::address& address, + uint64_t /*incarnation*/, + const evmc::bytes32& location) const noexcept { + StorageDomainGetLatestQuery query{ + db::state::kDomainNameStorage, + database_.domain(db::state::kDomainNameStorage), + tx_, + state_repository_, + }; + auto result = query.exec({address, location}); + if (result) { + return std::move(result->value); + } + return {}; +} + +uint64_t DomainState::previous_incarnation(const evmc::address& address) const noexcept { + auto prev_incarnation = db::read_previous_incarnation(tx_, address); + return prev_incarnation.value_or(0); +} + +std::optional DomainState::read_header(BlockNum block_num, const evmc::bytes32& block_hash) const noexcept { + return data_model_.read_header(block_num, block_hash); +} + +bool DomainState::read_body(BlockNum block_num, const evmc::bytes32& block_hash, BlockBody& out) const noexcept { + return data_model_.read_body(block_hash, block_num, out); +} + +std::optional DomainState::total_difficulty(BlockNum block_num, const evmc::bytes32& block_hash) const noexcept { + return data_model_.read_total_difficulty(block_num, block_hash); +} + +evmc::bytes32 DomainState::state_root_hash() const { + // TODO: implement + return evmc::bytes32{}; +} + +BlockNum DomainState::current_canonical_block() const { + auto head = db::read_canonical_head(tx_); + return std::get<0>(head); +} + +std::optional DomainState::canonical_hash(BlockNum block_num) const { + return data_model_.read_canonical_header_hash(block_num); +} + +void DomainState::insert_receipts(BlockNum block_num, const std::vector& receipts) { + // TODO: write to domain receipts table db::state::kDomainNameReceipts + db::write_receipts(tx_, receipts, block_num); +} + +void DomainState::update_account( + const evmc::address& address, + std::optional initial, + std::optional current) { + AccountsDomainGetLatestQuery query_prev{ + db::state::kDomainNameAccounts, + database_.domain(db::state::kDomainNameAccounts), + tx_, + state_repository_, + }; + auto result_prev = query_prev.exec(address); + + Step prev_step{0}; + if (result_prev) { + prev_step = result_prev->step; + } + + // TODO: handle current == nullopt + AccountsDomainPutQuery query{tx_, database_.domain(db::state::kDomainNameAccounts)}; + query.exec(address, *current, txn_id_, initial, prev_step); +} + +void DomainState::update_account_code( + const evmc::address& address, + uint64_t /*incarnation*/, + const evmc::bytes32& /*code_hash*/, + ByteView code) { + CodeDomainGetLatestQuery query_prev{ + db::state::kDomainNameCode, + database_.domain(db::state::kDomainNameCode), + tx_, + state_repository_, + }; + auto result_prev = query_prev.exec(address); + + Step prev_step{0}; + std::optional initial_code = std::nullopt; + if (result_prev) { + prev_step = result_prev->step; + initial_code = result_prev->value; + } + + CodeDomainPutQuery query{tx_, database_.domain(db::state::kDomainNameCode)}; + query.exec(address, code, txn_id_, initial_code, prev_step); +} + +void DomainState::update_storage( + const evmc::address& address, + uint64_t /*incarnation*/, + const evmc::bytes32& location, + const evmc::bytes32& initial, + const evmc::bytes32& current) { + + + StorageDomainGetLatestQuery query_prev{ + db::state::kDomainNameStorage, + database_.domain(db::state::kDomainNameStorage), + tx_, + state_repository_, + }; + auto result_prev = query_prev.exec({address, location}); + + Step prev_step{0}; + if (result_prev) { + prev_step = result_prev->step; + } + + StorageDomainPutQuery query{tx_, database_.domain(db::state::kDomainNameStorage)}; + query.exec({address, location}, current, txn_id_, initial, prev_step); +} + +} // namespace silkworm::execution diff --git a/silkworm/execution/domain_state.hpp b/silkworm/execution/domain_state.hpp new file mode 100644 index 0000000000..20ee810af4 --- /dev/null +++ b/silkworm/execution/domain_state.hpp @@ -0,0 +1,122 @@ +/* + Copyright 2025 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +namespace silkworm::execution { + +class DomainState : public State { + public: + explicit DomainState( + TxnId txn_id, + datastore::kvdb::RWTxn& tx, + datastore::kvdb::DatabaseBase& database, + snapshots::SnapshotRepository& blocks_repository, + snapshots::SnapshotRepository& state_repository) + : txn_id_{txn_id}, + tx_{tx}, + database_{database}, + state_repository_{state_repository}, + data_model_{db::DataModel{tx_, blocks_repository}} {} + + explicit DomainState( + TxnId txn_id, + datastore::kvdb::RWTxn& tx, + datastore::kvdb::DatabaseBase& database, + snapshots::SnapshotRepository& state_repository, + db::DataModel& data_model) + + : txn_id_{txn_id}, + tx_{tx}, + database_{database}, + state_repository_{state_repository}, + data_model_{data_model} {} + + std::optional read_account(const evmc::address& address) const noexcept override; + + ByteView read_code(const evmc::address& address, const evmc::bytes32& code_hash) const noexcept override; + + evmc::bytes32 read_storage(const evmc::address& address, uint64_t incarnation, const evmc::bytes32& location) const noexcept override; + + uint64_t previous_incarnation(const evmc::address& address) const noexcept override; + + std::optional read_header(BlockNum block_num, const evmc::bytes32& block_hash) const noexcept override; + + bool read_body(BlockNum block_num, const evmc::bytes32& block_hash, BlockBody& out) const noexcept override; + + std::optional total_difficulty(BlockNum block_num, const evmc::bytes32& block_hash) const noexcept override; + + evmc::bytes32 state_root_hash() const override; + + BlockNum current_canonical_block() const override; + + std::optional canonical_hash(BlockNum block_num) const override; + + void insert_block(const Block& /*block*/, const evmc::bytes32& /*hash*/) override {} + + void canonize_block(BlockNum /*block_num*/, const evmc::bytes32& /*block_hash*/) override {} + + void decanonize_block(BlockNum /*block_num*/) override {} + + void insert_receipts(BlockNum block_num, const std::vector& receipts) override; + + void insert_call_traces(BlockNum /*block_num*/, const CallTraces& /*traces*/) override {} + + void begin_block(BlockNum /*block_num*/, size_t /*updated_accounts_count*/) override {} + + void update_account( + const evmc::address& address, + std::optional initial, + std::optional current) override; + + void update_account_code( + const evmc::address& address, + uint64_t incarnation, + const evmc::bytes32& code_hash, + ByteView code) override; + + void update_storage( + const evmc::address& address, + uint64_t incarnation, + const evmc::bytes32& location, + const evmc::bytes32& initial, + const evmc::bytes32& current) override; + + void unwind_state_changes(BlockNum /*block_num*/) override {} + + private: + TxnId txn_id_; + datastore::kvdb::RWTxn& tx_; + datastore::kvdb::DatabaseBase& database_; + snapshots::SnapshotRepository& state_repository_; + db::DataModel data_model_; +}; + +} // namespace silkworm::execution diff --git a/silkworm/execution/domain_state_test.cpp b/silkworm/execution/domain_state_test.cpp new file mode 100644 index 0000000000..2b64df2f80 --- /dev/null +++ b/silkworm/execution/domain_state_test.cpp @@ -0,0 +1,189 @@ +/* + Copyright 2025 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "domain_state.hpp" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace silkworm::execution { + +using testing::_; +using testing::Invoke; +using testing::InvokeWithoutArgs; +using testing::Unused; + +TEST_CASE("DomainState data access", "[execution][domain][state]") { + TemporaryDirectory tmp_dir; + TestDataStore ds_context{tmp_dir}; + + auto rw_tx = ds_context.chaindata_rw().start_rw_tx(); + + auto sut = DomainState{1, rw_tx, ds_context->chaindata(), ds_context->blocks_repository(), ds_context->state_repository()}; + auto header0_hash = sut.canonical_hash(0); + auto header0 = sut.read_header(0, header0_hash.value()); + + const silkworm::Hash header1_hash{0x7cb4dd3daba1f739d0c1ec7d998b4a2f6fd83019116455afa54ca4f49dfa0ad4_bytes32}; + + SECTION("reads existing block") { + auto header1 = sut.read_header(1, header1_hash); + CHECK(header1.has_value()); + CHECK(header1->number == 1); + CHECK(header1->hash() == header1_hash); + CHECK(header1->parent_hash == header0_hash); + } + + SECTION("reads non-existing block") { + auto header2 = sut.read_header(2, header1_hash); + CHECK_FALSE(header2.has_value()); + } + + SECTION("reads existing block body") { + BlockBody body1{}; + auto body_read_ok = sut.read_body(1, header1_hash, body1); + CHECK(body_read_ok); + CHECK(body1.transactions.size() == 1); + } + + SECTION("reads non-existing block body") { + BlockBody body2{}; + auto body_read_ok = sut.read_body(2, header1_hash, body2); + CHECK_FALSE(body_read_ok); + } + + SECTION("reads existing total difficulty") { + auto td1 = sut.total_difficulty(1, header1_hash); + CHECK(td1.has_value()); + CHECK(*td1 == 17'171'480'576); + } + + SECTION("reads head canonical block number") { + auto head_block_num = sut.current_canonical_block(); + CHECK(head_block_num == 1); + } + + //! Current genesis preloading does not store data to domain tables + + // SECTION("read_account preloaded block") { + // auto account_65 = sut.read_account(0x658bdf435d810c91414ec09147daa6db62406379_address); + // CHECK(account_65.has_value()); + // CHECK(account_65->balance == intx::uint256{72, 8834426692912283648}); + // } + + // SECTION("read_code preloaded block") { + // auto code_aa = sut.read_code( + // 0xaa00000000000000000000000000000000000000_address, + // 0x39a32aa611e90196e88985d1de5179d967b0cab8d198f6186d5f4d3f073d4fbe_bytes32); + // CHECK(code_aa.size() > 0); + // CHECK(code_aa == from_hex("0x6042")); + // } + + // SECTION("read_storage preloaded block") { + // auto storage_bb_01 = sut.read_storage( + // 0xbb00000000000000000000000000000000000000_address, + // 0, + // 0x0100000000000000000000000000000000000000000000000000000000000000_bytes32); + // CHECK(storage_bb_01 == 0x0100000000000000000000000000000000000000000000000000000000000000_bytes32); + // } + + SECTION("update_account") { + Account account_66{ + .nonce = 123, + .balance = 456, + .incarnation = kDefaultIncarnation, + }; + + sut.update_account( + 0x668bdf435d810c91414ec09147daa6db62406379_address, + account_66, + {}); + + auto account_66_read = sut.read_account(0x668bdf435d810c91414ec09147daa6db62406379_address); + CHECK(account_66_read.has_value()); + CHECK(account_66_read->incarnation == kDefaultIncarnation); + CHECK(account_66_read->nonce == 123); + CHECK(account_66_read->balance == 456); + } + + SECTION("update_account_code") { + auto code_66 = *from_hex("0x6042"); + auto code_hash_66 = std::bit_cast(keccak256(code_66)); + + sut.update_account_code( + 0x668bdf435d810c91414ec09147daa6db62406379_address, + kDefaultIncarnation, + code_hash_66, + code_66); + + auto code_66_read = sut.read_code(0x668bdf435d810c91414ec09147daa6db62406379_address, code_hash_66); + CHECK(code_66_read.size() > 0); + CHECK(code_66_read == code_66); + } + + SECTION("update_storage") { + sut.update_storage( + 0x668bdf435d810c91414ec09147daa6db62406379_address, + kDefaultIncarnation, + 0x010_bytes32, + evmc::bytes32{}, + 0x0123_bytes32); + + auto storage_66_01 = sut.read_storage( + 0x668bdf435d810c91414ec09147daa6db62406379_address, + kDefaultIncarnation, + 0x010_bytes32); + + CHECK(storage_66_01 == 0x0123_bytes32); + } + +} + +TEST_CASE("DomainState empty overriden methods do nothing", "[execution][domain][state]") { + TemporaryDirectory tmp_dir; + TestDataStore ds_context{tmp_dir}; + + auto rw_tx = ds_context.chaindata_rw().start_rw_tx(); + + auto sut = DomainState{1, rw_tx, ds_context->chaindata(), ds_context->blocks_repository(), ds_context->state_repository()}; + + CHECK_NOTHROW(sut.insert_block(Block{}, evmc::bytes32{})); + CHECK_NOTHROW(sut.canonize_block(0, evmc::bytes32{})); + CHECK_NOTHROW(sut.decanonize_block(0)); + CHECK_NOTHROW(sut.insert_call_traces(0, CallTraces{})); + CHECK_NOTHROW(sut.begin_block(0, 0)); + CHECK_NOTHROW(sut.unwind_state_changes(0)); + + auto state_root_hash = sut.state_root_hash(); + CHECK(state_root_hash == evmc::bytes32{}); +} +} // namespace silkworm::execution From 34a0efed100c249fc47e1a6b5132f383fda4825e Mon Sep 17 00:00:00 2001 From: GitHub Date: Sun, 19 Jan 2025 20:15:02 +0000 Subject: [PATCH 15/33] make fmt --- silkworm/capi/silkworm.cpp | 17 ++++++++++------- silkworm/execution/database_tcontext.hpp | 6 +++--- silkworm/execution/domain_state.cpp | 16 +++++++--------- silkworm/execution/domain_state_test.cpp | 1 - 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/silkworm/capi/silkworm.cpp b/silkworm/capi/silkworm.cpp index 5908ca1ea0..0fe3d6f6c7 100644 --- a/silkworm/capi/silkworm.cpp +++ b/silkworm/capi/silkworm.cpp @@ -781,7 +781,7 @@ SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* mdbx_tx auto state = silkworm::execution::DomainState{txn_id_, unmanaged_tx, chain_db, *handle->blocks_repository, *handle->state_repository}; if (!handle->chain_config) { handle->chain_config = db::read_chain_config(unmanaged_tx); - } + } //! Manual tests: remove when done // auto b_hash = to_bytes32(*from_hex("0xe8b28e4882bcbd6293ef56433c69b34e9e3e5bf512a05ddbed6bb94aa65948f4")); @@ -812,22 +812,25 @@ SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* mdbx_tx // log::Info{"account2 not found"}; // } - //TODO: cache block, also consider preloading + // TODO: cache block, also consider preloading silkworm::Block block{}; auto block_read_ok = state.read_body(block_number, head_hash, block); if (!block_read_ok) { - SILK_ERROR << "Block not found" << " block_number: " << block_number << " head_hash: " << head_hash; + SILK_ERROR << "Block not found" + << " block_number: " << block_number << " head_hash: " << head_hash; return SILKWORM_INVALID_BLOCK; } auto header = state.read_header(block_number, head_hash); if (!header) { - SILK_ERROR << "Header not found" << " block_number: " << block_number << " head_hash: " << head_hash; + SILK_ERROR << "Header not found" + << " block_number: " << block_number << " head_hash: " << head_hash; return SILKWORM_INVALID_BLOCK; } block.header = header.value(); if (txn_num >= block.transactions.size()) { - SILK_ERROR << "Transaction not found" << " txn_num: " << txn_num; + SILK_ERROR << "Transaction not found" + << " txn_num: " << txn_num; return SILKWORM_INVALID_BLOCK; } @@ -835,7 +838,7 @@ SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* mdbx_tx auto protocol_rule_set_{protocol::rule_set_factory(*handle->chain_config)}; ExecutionProcessor processor{block, *protocol_rule_set_, state, *handle->chain_config, false}; - //TODO: add analysis cache, check block exec for more + // TODO: add analysis cache, check block exec for more silkworm::Receipt receipt{}; const ValidationResult err{protocol::validate_transaction(transaction, processor.intra_block_state(), processor.available_gas())}; @@ -850,7 +853,7 @@ SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* mdbx_tx } if (blob_gas_used) { *blob_gas_used = transaction.total_blob_gas(); - } + } return SILKWORM_OK; } diff --git a/silkworm/execution/database_tcontext.hpp b/silkworm/execution/database_tcontext.hpp index 9242b6c75c..427b9da231 100644 --- a/silkworm/execution/database_tcontext.hpp +++ b/silkworm/execution/database_tcontext.hpp @@ -26,10 +26,10 @@ #include #include -//TODO: this is a duplicate of test_database_context.hpp -//check linking issues and remove this file +// TODO: this is a duplicate of test_database_context.hpp +// check linking issues and remove this file -//TODO: populate_genesis and populate_blocks should write to the domain tables +// TODO: populate_genesis and populate_blocks should write to the domain tables namespace silkworm::execution { diff --git a/silkworm/execution/domain_state.cpp b/silkworm/execution/domain_state.cpp index 856f9c6d8c..897191875b 100644 --- a/silkworm/execution/domain_state.cpp +++ b/silkworm/execution/domain_state.cpp @@ -27,13 +27,13 @@ namespace silkworm::execution { using namespace db::state; using namespace datastore; -//TODO: -// - implement: state_root_hash, canonize_block, decanonize_block, insert_call_traces -// - add begin_txn method (replacing begin_block?) -// - insert_receipts should write to domain receipts table db::state::kDomainNameReceipts -// - add buffer saving previous steps for values in accounts, code and storage domains - for updates -// - extend transaction to include txn_id, or -// - add base_txn_id to block and get_txn_by_id method +// TODO: +// - implement: state_root_hash, canonize_block, decanonize_block, insert_call_traces +// - add begin_txn method (replacing begin_block?) +// - insert_receipts should write to domain receipts table db::state::kDomainNameReceipts +// - add buffer saving previous steps for values in accounts, code and storage domains - for updates +// - extend transaction to include txn_id, or +// - add base_txn_id to block and get_txn_by_id method std::optional DomainState::read_account(const evmc::address& address) const noexcept { AccountsDomainGetLatestQuery query{ @@ -168,8 +168,6 @@ void DomainState::update_storage( const evmc::bytes32& location, const evmc::bytes32& initial, const evmc::bytes32& current) { - - StorageDomainGetLatestQuery query_prev{ db::state::kDomainNameStorage, database_.domain(db::state::kDomainNameStorage), diff --git a/silkworm/execution/domain_state_test.cpp b/silkworm/execution/domain_state_test.cpp index 2b64df2f80..0f641ab91e 100644 --- a/silkworm/execution/domain_state_test.cpp +++ b/silkworm/execution/domain_state_test.cpp @@ -165,7 +165,6 @@ TEST_CASE("DomainState data access", "[execution][domain][state]") { CHECK(storage_66_01 == 0x0123_bytes32); } - } TEST_CASE("DomainState empty overriden methods do nothing", "[execution][domain][state]") { From ec6a7e58dfe9cda0339f9d9bead8c73e267b41b1 Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Sun, 19 Jan 2025 21:32:23 +0100 Subject: [PATCH 16/33] Update copyright --- silkworm/execution/database_tcontext.cpp | 2 +- silkworm/execution/database_tcontext.hpp | 2 +- silkworm/execution/domain_state.cpp | 2 +- silkworm/execution/domain_state.hpp | 2 +- silkworm/execution/domain_state_test.cpp | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/silkworm/execution/database_tcontext.cpp b/silkworm/execution/database_tcontext.cpp index 9cf5d271d1..1020d6ad59 100644 --- a/silkworm/execution/database_tcontext.cpp +++ b/silkworm/execution/database_tcontext.cpp @@ -1,5 +1,5 @@ /* - Copyright 2025 The Silkworm Authors + Copyright 2024 The Silkworm Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/silkworm/execution/database_tcontext.hpp b/silkworm/execution/database_tcontext.hpp index 427b9da231..5383692829 100644 --- a/silkworm/execution/database_tcontext.hpp +++ b/silkworm/execution/database_tcontext.hpp @@ -1,5 +1,5 @@ /* - Copyright 2025 The Silkworm Authors + Copyright 2024 The Silkworm Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/silkworm/execution/domain_state.cpp b/silkworm/execution/domain_state.cpp index 897191875b..40be09111f 100644 --- a/silkworm/execution/domain_state.cpp +++ b/silkworm/execution/domain_state.cpp @@ -1,5 +1,5 @@ /* - Copyright 2025 The Silkworm Authors + Copyright 2024 The Silkworm Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/silkworm/execution/domain_state.hpp b/silkworm/execution/domain_state.hpp index 20ee810af4..556ed75a48 100644 --- a/silkworm/execution/domain_state.hpp +++ b/silkworm/execution/domain_state.hpp @@ -1,5 +1,5 @@ /* - Copyright 2025 The Silkworm Authors + Copyright 2024 The Silkworm Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/silkworm/execution/domain_state_test.cpp b/silkworm/execution/domain_state_test.cpp index 0f641ab91e..2c3f2b793e 100644 --- a/silkworm/execution/domain_state_test.cpp +++ b/silkworm/execution/domain_state_test.cpp @@ -1,5 +1,5 @@ /* - Copyright 2025 The Silkworm Authors + Copyright 2024 The Silkworm Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From 4777cb0d79e4c7973e52a2f04a2fc4381795a4a2 Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Sun, 19 Jan 2025 22:19:40 +0100 Subject: [PATCH 17/33] Remove temp log comments --- silkworm/db/chain/local_chain_storage.cpp | 2 -- silkworm/db/chain/remote_chain_storage.cpp | 7 ------- silkworm/db/kv/grpc/client/remote_transaction.cpp | 1 - silkworm/db/kv/state_reader.cpp | 2 +- silkworm/db/kv/txn_num.cpp | 10 ---------- silkworm/execution/remote_state.cpp | 4 ---- silkworm/rpc/ethbackend/remote_backend.cpp | 2 -- silkworm/rpc/ethdb/kv/backend_providers.hpp | 2 -- 8 files changed, 1 insertion(+), 29 deletions(-) diff --git a/silkworm/db/chain/local_chain_storage.cpp b/silkworm/db/chain/local_chain_storage.cpp index 7d010de890..395566ae79 100644 --- a/silkworm/db/chain/local_chain_storage.cpp +++ b/silkworm/db/chain/local_chain_storage.cpp @@ -64,8 +64,6 @@ Task> LocalChainStorage::read_header(BlockNum block_n } Task> LocalChainStorage::read_header(BlockNum block_num, const Hash& hash) const { - SILK_DEBUG << "LocalChainStorage::read_header: " - << " number: " << block_num; co_return data_model_.read_header(block_num, hash); } diff --git a/silkworm/db/chain/remote_chain_storage.cpp b/silkworm/db/chain/remote_chain_storage.cpp index a11e3bc397..65b996a9e4 100644 --- a/silkworm/db/chain/remote_chain_storage.cpp +++ b/silkworm/db/chain/remote_chain_storage.cpp @@ -89,25 +89,18 @@ Task> RemoteChainStorage::read_header(BlockNum block_ const bool success = co_await providers_.block(block_num, hash, /*.read_senders=*/false, block); std::optional header; if (success) { - SILK_DEBUG << "RemoteChainStorage::read_header: success read " - << " number: " << block_num; header = std::move(block.header); } co_return header; } Task> RemoteChainStorage::read_header(BlockNum block_num, const Hash& hash) const { - SILK_DEBUG << "RemoteChainStorage::read_header: " - << " number: " << block_num; co_return co_await read_header(block_num, hash.bytes); } Task> RemoteChainStorage::read_header(const Hash& hash) const { const auto block_num = co_await providers_.block_num_from_hash(hash.bytes); if (!block_num) { - SILK_DEBUG << "RemoteChainStorage::read_header: " - << " failed to get block number: " << *block_num; - co_return std::nullopt; } SILK_DEBUG << "RemoteChainStorage::read_header: " << silkworm::to_hex(hash) << " number: " << *block_num; diff --git a/silkworm/db/kv/grpc/client/remote_transaction.cpp b/silkworm/db/kv/grpc/client/remote_transaction.cpp index ccd0223534..e4b5f4da14 100644 --- a/silkworm/db/kv/grpc/client/remote_transaction.cpp +++ b/silkworm/db/kv/grpc/client/remote_transaction.cpp @@ -113,7 +113,6 @@ std::shared_ptr RemoteTransaction::create_storage() { } Task RemoteTransaction::first_txn_num_in_block(BlockNum block_num) { - SILK_INFO << "RemoteTransaction::first_txn_num_in_block block_num=" << block_num; const auto min_txn_num = co_await txn::min_tx_num(*this, block_num, providers_.canonical_body_for_storage); co_return min_txn_num + /*txn_index*/ 0; } diff --git a/silkworm/db/kv/state_reader.cpp b/silkworm/db/kv/state_reader.cpp index 18de1bd34a..e73fc02b38 100644 --- a/silkworm/db/kv/state_reader.cpp +++ b/silkworm/db/kv/state_reader.cpp @@ -108,4 +108,4 @@ Task> StateReader::read_code(const evmc::address& address, co_return result.value; } -} // namespace silkworm::db::kv \ No newline at end of file +} // namespace silkworm::db::kv diff --git a/silkworm/db/kv/txn_num.cpp b/silkworm/db/kv/txn_num.cpp index 6f9aa5583f..0e1a07fcca 100644 --- a/silkworm/db/kv/txn_num.cpp +++ b/silkworm/db/kv/txn_num.cpp @@ -86,26 +86,16 @@ Task min_tx_num(Transaction& tx, BlockNum block_num, chain::CanonicalBody if (block_num == 0) { co_return 0; } - SILK_DEBUG << "min_tx_num 1"; const auto max_tx_num_cursor = co_await tx.cursor(table::kMaxTxNumName); - SILK_DEBUG << "min_tx_num 2"; const std::optional last_tx_num = co_await last_tx_num_for_block(max_tx_num_cursor, (block_num - 1), provider); - SILK_DEBUG << "min_tx_num 3"; if (!last_tx_num) { - SILK_DEBUG << "min_tx_num 4"; const KeyValue key_value = co_await max_tx_num_cursor->last(); - SILK_DEBUG << "min_tx_num 5"; if (key_value.value.empty()) { - SILK_DEBUG << "min_tx_num 6"; co_return 0; } if (key_value.value.size() != sizeof(TxNum)) { - SILK_DEBUG << "min_tx_num 7"; - throw std::length_error("Bad TxNum value size " + std::to_string(key_value.value.size()) + " in db"); } - SILK_DEBUG << "min_tx_num 8"; - co_return endian::load_big_u64(key_value.value.data()); } co_return *last_tx_num + 1; diff --git a/silkworm/execution/remote_state.cpp b/silkworm/execution/remote_state.cpp index aae1b7733c..0cbdd5965b 100644 --- a/silkworm/execution/remote_state.cpp +++ b/silkworm/execution/remote_state.cpp @@ -55,9 +55,6 @@ Task AsyncRemoteState::previous_incarnation(const evmc::address& /*add } Task> AsyncRemoteState::read_header(BlockNum block_num, const evmc::bytes32& block_hash) const noexcept { - SILK_DEBUG << "AsyncRemoteState::read_header: " - << " number: " << block_num; - co_return co_await storage_.read_header(block_num, block_hash); } @@ -87,7 +84,6 @@ std::optional RemoteState::read_account(const evmc::address& address) c SILK_DEBUG << "RemoteState::read_account address=" << address << " start"; try { std::future> result{boost::asio::co_spawn(executor_, async_state_.read_account(address), boost::asio::use_future)}; - SILK_DEBUG << "RemoteState::read_account address=" << address << " spawned"; const auto optional_account{result.get()}; SILK_DEBUG << "RemoteState::read_account account.nonce=" << (optional_account ? optional_account->nonce : 0) << " end"; return optional_account; diff --git a/silkworm/rpc/ethbackend/remote_backend.cpp b/silkworm/rpc/ethbackend/remote_backend.cpp index 2f4608f331..9287be10ff 100644 --- a/silkworm/rpc/ethbackend/remote_backend.cpp +++ b/silkworm/rpc/ethbackend/remote_backend.cpp @@ -147,9 +147,7 @@ Task RemoteBackEnd::get_block(BlockNum block_num, const HashAsSpan& hash, request.set_allocated_block_hash(h256_from_bytes(hash).release()); const auto reply = co_await rpc::unary_rpc(&Stub::AsyncBlock, *stub_, request, grpc_context_); ByteView block_rlp{string_view_to_byte_view(reply.block_rlp())}; - SILK_DEBUG << "RemoteBackEnd::get_block block_num=" << block_num << " reply=" << reply.DebugString(); if (const auto decode_result{rlp::decode(block_rlp, block)}; !decode_result) { - SILK_ERROR << "RemoteBackEnd::get_block block_num=" << block_num << " decode error=" << int(decode_result.error()); co_return false; } if (read_senders) { diff --git a/silkworm/rpc/ethdb/kv/backend_providers.hpp b/silkworm/rpc/ethdb/kv/backend_providers.hpp index c088ae5207..778d38b826 100644 --- a/silkworm/rpc/ethdb/kv/backend_providers.hpp +++ b/silkworm/rpc/ethdb/kv/backend_providers.hpp @@ -23,8 +23,6 @@ namespace silkworm::rpc::ethdb::kv { inline db::chain::BlockProvider block_provider(ethbackend::BackEnd* backend) { return [backend](auto block_num, HashAsSpan hash, bool read_senders, auto& block) -> Task { - SILK_DEBUG << "db::chain::BlockProvider::block_provider: " - << " number: " << block_num; co_return co_await backend->get_block(block_num, hash, read_senders, block); }; } From 2e159cca1496bfaf8cd9faab0634700f3394f3a9 Mon Sep 17 00:00:00 2001 From: GitHub Date: Sun, 19 Jan 2025 21:20:18 +0000 Subject: [PATCH 18/33] make fmt --- silkworm/db/kv/state_reader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/silkworm/db/kv/state_reader.cpp b/silkworm/db/kv/state_reader.cpp index e73fc02b38..3691d210f9 100644 --- a/silkworm/db/kv/state_reader.cpp +++ b/silkworm/db/kv/state_reader.cpp @@ -108,4 +108,4 @@ Task> StateReader::read_code(const evmc::address& address, co_return result.value; } -} // namespace silkworm::db::kv +} // namespace silkworm::db::kv From 6ef32fd9e2641fbdcaa32ff4058685d01af2b8bc Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Sun, 19 Jan 2025 22:35:58 +0100 Subject: [PATCH 19/33] Fix test execution --- silkworm/capi/silkworm_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/silkworm/capi/silkworm_test.cpp b/silkworm/capi/silkworm_test.cpp index 6ad44c9f33..419598394c 100644 --- a/silkworm/capi/silkworm_test.cpp +++ b/silkworm/capi/silkworm_test.cpp @@ -178,7 +178,7 @@ struct SilkwormLibrary { } int exectute_single_tx(MDBX_txn* txn) const { - return silkworm_execute_tx(handle_, txn, 1, 1, nullptr, nullptr); + return silkworm_execute_tx(handle_, txn, 1, SilkwormBytes32{}, 1, 1, nullptr, nullptr); } int add_snapshot(SilkwormChainSnapshot* snapshot) const { From 222e78d6bd4343c91bbbec5641aaf00fb84af773 Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Thu, 23 Jan 2025 23:18:18 +0100 Subject: [PATCH 20/33] Add unit tests and optimize domain state queries --- .../kvdb/domain_get_latest_query.hpp | 21 ++- .../kvdb/domain_put_latest_query.hpp | 12 +- .../db/datastore/kvdb/domain_put_query.hpp | 6 +- silkworm/execution/domain_state.cpp | 4 +- silkworm/execution/domain_state_test.cpp | 154 ++++++++++++++++-- 5 files changed, 177 insertions(+), 20 deletions(-) diff --git a/silkworm/db/datastore/kvdb/domain_get_latest_query.hpp b/silkworm/db/datastore/kvdb/domain_get_latest_query.hpp index a691c37fe0..d592ae0996 100644 --- a/silkworm/db/datastore/kvdb/domain_get_latest_query.hpp +++ b/silkworm/db/datastore/kvdb/domain_get_latest_query.hpp @@ -44,23 +44,36 @@ struct DomainGetLatestQuery { key_encoder.value.key.value = key; key_encoder.value.timestamp.value = Step{std::numeric_limits::max()}; Slice key_slice = key_encoder.encode(); + SILK_DEBUG << "Querying " << entity.values_table.name << " with key " << to_hex(datastore::kvdb::from_slice(key_slice), true) << " "; - auto result = tx.ro_cursor(entity.values_table)->lower_bound(key_slice, false); - if (!result) return std::nullopt; + auto result = entity.has_large_values ? tx.ro_cursor(entity.values_table)->lower_bound(key_slice, false) : tx.ro_cursor(entity.values_table)->find(key_slice, false); + + if (!result) { + SILK_DEBUG << "key not found" << std::endl; + return std::nullopt; + } DomainKeyDecoder> key_decoder{entity.has_large_values}; key_decoder.decode(result.key); - if (key_decoder.value.key.value != from_slice(key_slice)) return std::nullopt; + if (key_decoder.value.key.value != from_slice(key_slice)) { + SILK_DEBUG << "key not found (decoded to different value)" << std::endl; + return std::nullopt; + } DomainValueDecoder> empty_value_decoder{entity.has_large_values}; empty_value_decoder.decode(result.value); - if (empty_value_decoder.value.value.value.empty()) return std::nullopt; + if (empty_value_decoder.value.value.value.empty()) { + SILK_DEBUG << "empty value" << std::endl; + return std::nullopt; + } DomainValueDecoder value_decoder{entity.has_large_values}; value_decoder.decode(result.value); Step step = Step(key_decoder.value.timestamp.value.value | value_decoder.value.timestamp.value.value); + SILK_DEBUG << "found value " << to_hex(datastore::kvdb::from_slice(result.value), true) << " at step " << step.to_string() << " resulting key " << to_hex(datastore::kvdb::from_slice(result.key), true) << std::endl; + return Result{std::move(value_decoder.value.value.value), step}; } }; diff --git a/silkworm/db/datastore/kvdb/domain_put_latest_query.hpp b/silkworm/db/datastore/kvdb/domain_put_latest_query.hpp index 526484d06b..098c312284 100644 --- a/silkworm/db/datastore/kvdb/domain_put_latest_query.hpp +++ b/silkworm/db/datastore/kvdb/domain_put_latest_query.hpp @@ -42,6 +42,8 @@ struct DomainPutLatestQuery { value_encoder.value.timestamp.value = step; Slice value_data = value_encoder.encode(); + SILK_DEBUG << "Putting " << entity.values_table.name << " with key " << to_hex(datastore::kvdb::from_slice(key_data), true) << " value " << to_hex(datastore::kvdb::from_slice(value_data), true) << std::endl; + if (entity.values_table.value_mode == mdbx::value_mode::multi) { auto cursor = tx.rw_cursor_dup_sort(entity.values_table); @@ -51,8 +53,11 @@ struct DomainPutLatestQuery { same_step_value_encoder.value.timestamp.value = step; Slice same_step_value = same_step_value_encoder.encode(); - CursorResult result = cursor->find_multivalue(key_data, same_step_value, false); + SILK_DEBUG << "DupSort looking for existing value " << to_hex(datastore::kvdb::from_slice(same_step_value), true); + + CursorResult result = cursor->lower_bound_multivalue(key_data, same_step_value, false); if (result) { + SILK_DEBUG << " found and "; // the found value will have the same key, but the step part can be different, // let's decode it ignoring the data part DomainValueDecoder> existing_value_decoder{entity.has_large_values}; @@ -60,7 +65,12 @@ struct DomainPutLatestQuery { Step existing_value_step = existing_value_decoder.value.timestamp.value; if (existing_value_step == step) { cursor->erase(); + SILK_DEBUG << "earasing" << std::endl; + } else { + SILK_DEBUG << "not earasing, existing_value_step: " << existing_value_step.to_string() << std::endl; } + } else { + SILK_DEBUG << " not found" << std::endl; } cursor->upsert(key_data, value_data); diff --git a/silkworm/db/datastore/kvdb/domain_put_query.hpp b/silkworm/db/datastore/kvdb/domain_put_query.hpp index b16b0c41a3..8bd0f07040 100644 --- a/silkworm/db/datastore/kvdb/domain_put_query.hpp +++ b/silkworm/db/datastore/kvdb/domain_put_query.hpp @@ -37,15 +37,15 @@ struct DomainPutQuery { const std::optional& prev_value, Step prev_step) { DomainPutLatestQuery value_query{tx, entity}; - value_query.exec(key, value, prev_step); + value_query.exec(key, value, Step::from_txn_id(timestamp)); if (entity.history) { if (prev_value) { HistoryPutQuery history_query{tx, *entity.history}; - history_query.exec(key, *prev_value, timestamp); + history_query.exec(key, *prev_value, prev_step.value); } else { HistoryDeleteQuery history_query{tx, *entity.history}; - history_query.exec(key, timestamp); + history_query.exec(key, prev_step.value); } } } diff --git a/silkworm/execution/domain_state.cpp b/silkworm/execution/domain_state.cpp index 40be09111f..8d822167b6 100644 --- a/silkworm/execution/domain_state.cpp +++ b/silkworm/execution/domain_state.cpp @@ -28,7 +28,8 @@ using namespace db::state; using namespace datastore; // TODO: -// - implement: state_root_hash, canonize_block, decanonize_block, insert_call_traces +// - implement: state_root_hash, canonize_block, decanonize_block (optional) +// - implement insert_call_traces (mandatory) // - add begin_txn method (replacing begin_block?) // - insert_receipts should write to domain receipts table db::state::kDomainNameReceipts // - add buffer saving previous steps for values in accounts, code and storage domains - for updates @@ -178,6 +179,7 @@ void DomainState::update_storage( Step prev_step{0}; if (result_prev) { + SILK_DEBUG << "Found previous value " << to_hex(result_prev->value, true) << " step " << result_prev->step.to_string() << std::endl; prev_step = result_prev->step; } diff --git a/silkworm/execution/domain_state_test.cpp b/silkworm/execution/domain_state_test.cpp index 2c3f2b793e..1709a28932 100644 --- a/silkworm/execution/domain_state_test.cpp +++ b/silkworm/execution/domain_state_test.cpp @@ -84,12 +84,12 @@ TEST_CASE("DomainState data access", "[execution][domain][state]") { SECTION("reads existing total difficulty") { auto td1 = sut.total_difficulty(1, header1_hash); CHECK(td1.has_value()); - CHECK(*td1 == 17'171'480'576); + CHECK(*td1 == 1); } SECTION("reads head canonical block number") { auto head_block_num = sut.current_canonical_block(); - CHECK(head_block_num == 1); + CHECK(head_block_num == 9); } //! Current genesis preloading does not store data to domain tables @@ -118,27 +118,92 @@ TEST_CASE("DomainState data access", "[execution][domain][state]") { SECTION("update_account") { Account account_66{ - .nonce = 123, - .balance = 456, + .nonce = 8, + .balance = 260, .incarnation = kDefaultIncarnation, }; + sut.update_account( + 0x668bdf435d810c91414ec09147daa6db62406379_address, + {}, + account_66); + + auto account_66_read = sut.read_account(0x668bdf435d810c91414ec09147daa6db62406379_address); + CHECK(account_66_read.has_value()); + CHECK(account_66_read->incarnation == account_66.incarnation); + CHECK(account_66_read->nonce == account_66.nonce); + CHECK(account_66_read->balance == account_66.balance); + } + SECTION("update_account with different steps") { + Account account_66{ + .nonce = 8, + .balance = 260, + .incarnation = kDefaultIncarnation, + }; sut.update_account( 0x668bdf435d810c91414ec09147daa6db62406379_address, - account_66, - {}); + {}, + account_66); auto account_66_read = sut.read_account(0x668bdf435d810c91414ec09147daa6db62406379_address); CHECK(account_66_read.has_value()); - CHECK(account_66_read->incarnation == kDefaultIncarnation); - CHECK(account_66_read->nonce == 123); - CHECK(account_66_read->balance == 456); + CHECK(account_66_read->incarnation == account_66.incarnation); + CHECK(account_66_read->nonce == account_66.nonce); + CHECK(account_66_read->balance == account_66.balance); + + Account account_66_v2{ + .nonce = 9, + .balance = 261552435, + .incarnation = kDefaultIncarnation, + }; + sut.update_account( + 0x668bdf435d810c91414ec09147daa6db62406379_address, + account_66, + account_66_v2); + + account_66_read = sut.read_account(0x668bdf435d810c91414ec09147daa6db62406379_address); + CHECK(account_66_read.has_value()); + CHECK(account_66_read->incarnation == account_66_v2.incarnation); + CHECK(account_66_read->nonce == account_66_v2.nonce); + CHECK(account_66_read->balance == account_66_v2.balance); + + auto next_step_txn_id = silkworm::datastore::kStepSizeForTemporalSnapshots + 1; + auto sut2 = DomainState{next_step_txn_id, rw_tx, ds_context->chaindata(), ds_context->blocks_repository(), ds_context->state_repository()}; + Account account_66_v3{ + .nonce = 10, + .balance = 262, + .incarnation = kDefaultIncarnation, + }; + + sut2.update_account( + 0x668bdf435d810c91414ec09147daa6db62406379_address, + account_66_v2, + account_66_v3); + + account_66_read = sut2.read_account(0x668bdf435d810c91414ec09147daa6db62406379_address); + CHECK(account_66_read.has_value()); + CHECK(account_66_read->incarnation == account_66_v3.incarnation); + CHECK(account_66_read->nonce == account_66_v3.nonce); + CHECK(account_66_read->balance == account_66_v3.balance); } SECTION("update_account_code") { auto code_66 = *from_hex("0x6042"); auto code_hash_66 = std::bit_cast(keccak256(code_66)); + sut.update_account_code( + 0x668bdf435d810c91414ec09147daa6db62406379_address, + kDefaultIncarnation, + code_hash_66, + code_66); + auto code_66_read = sut.read_code(0x668bdf435d810c91414ec09147daa6db62406379_address, code_hash_66); + CHECK(code_66_read.size() > 0); + CHECK(code_66_read == code_66); + } + + SECTION("update_account_code with different steps") { + auto code_66 = *from_hex("0x6042"); + auto code_hash_66 = std::bit_cast(keccak256(code_66)); sut.update_account_code( 0x668bdf435d810c91414ec09147daa6db62406379_address, kDefaultIncarnation, @@ -148,22 +213,89 @@ TEST_CASE("DomainState data access", "[execution][domain][state]") { auto code_66_read = sut.read_code(0x668bdf435d810c91414ec09147daa6db62406379_address, code_hash_66); CHECK(code_66_read.size() > 0); CHECK(code_66_read == code_66); + + code_66 = *from_hex("0x6043"); + code_hash_66 = std::bit_cast(keccak256(code_66)); + sut.update_account_code( + 0x668bdf435d810c91414ec09147daa6db62406379_address, + kDefaultIncarnation, + code_hash_66, + code_66); + + code_66_read = sut.read_code(0x668bdf435d810c91414ec09147daa6db62406379_address, code_hash_66); + CHECK(code_66_read.size() > 0); + CHECK(code_66_read == code_66); + + auto next_step_txn_id = silkworm::datastore::kStepSizeForTemporalSnapshots + 1; + auto sut2 = DomainState{next_step_txn_id, rw_tx, ds_context->chaindata(), ds_context->blocks_repository(), ds_context->state_repository()}; + code_66 = *from_hex("0x6044"); + code_hash_66 = std::bit_cast(keccak256(code_66)); + sut2.update_account_code( + 0x668bdf435d810c91414ec09147daa6db62406379_address, + kDefaultIncarnation, + code_hash_66, + code_66); + + code_66_read = sut2.read_code(0x668bdf435d810c91414ec09147daa6db62406379_address, code_hash_66); + CHECK(code_66_read.size() > 0); + CHECK(code_66_read == code_66); } SECTION("update_storage") { sut.update_storage( 0x668bdf435d810c91414ec09147daa6db62406379_address, kDefaultIncarnation, - 0x010_bytes32, + 0x0100_bytes32, evmc::bytes32{}, 0x0123_bytes32); auto storage_66_01 = sut.read_storage( 0x668bdf435d810c91414ec09147daa6db62406379_address, kDefaultIncarnation, - 0x010_bytes32); + 0x0100_bytes32); + CHECK(storage_66_01 == 0x0123_bytes32); + } + + SECTION("update_storage with different steps") { + sut.update_storage( + 0x668bdf435d810c91414ec09147daa6db62406379_address, + kDefaultIncarnation, + 0x0100_bytes32, + evmc::bytes32{}, + 0x0123_bytes32); + auto storage_66_01 = sut.read_storage( + 0x668bdf435d810c91414ec09147daa6db62406379_address, + kDefaultIncarnation, + 0x0100_bytes32); CHECK(storage_66_01 == 0x0123_bytes32); + + sut.update_storage( + 0x668bdf435d810c91414ec09147daa6db62406379_address, + kDefaultIncarnation, + 0x0100_bytes32, + 0x0123_bytes32, + 0x0124_bytes32); + + storage_66_01 = sut.read_storage( + 0x668bdf435d810c91414ec09147daa6db62406379_address, + kDefaultIncarnation, + 0x0100_bytes32); + CHECK(storage_66_01 == 0x0124_bytes32); + + auto next_step_txn_id = silkworm::datastore::kStepSizeForTemporalSnapshots + 1; + auto sut2 = DomainState{next_step_txn_id, rw_tx, ds_context->chaindata(), ds_context->blocks_repository(), ds_context->state_repository()}; + sut2.update_storage( + 0x668bdf435d810c91414ec09147daa6db62406379_address, + kDefaultIncarnation, + 0x0100_bytes32, + 0x0124_bytes32, + 0x0456_bytes32); + storage_66_01 = sut2.read_storage( + 0x668bdf435d810c91414ec09147daa6db62406379_address, + kDefaultIncarnation, + 0x0100_bytes32); + CHECK(storage_66_01 == 0x0456_bytes32); } } From 17367961f8c71e3b19feb209818a993163a54876 Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Fri, 24 Jan 2025 09:41:08 +0100 Subject: [PATCH 21/33] Fix silkworm capi test --- silkworm/capi/silkworm_test.cpp | 56 +++++++++++++++---- .../kvdb/domain_get_latest_query.hpp | 8 +-- .../kvdb/domain_put_latest_query.hpp | 8 +-- silkworm/execution/domain_state.cpp | 2 +- silkworm/execution/domain_state_test.cpp | 1 + 5 files changed, 54 insertions(+), 21 deletions(-) diff --git a/silkworm/capi/silkworm_test.cpp b/silkworm/capi/silkworm_test.cpp index 419598394c..e649864467 100644 --- a/silkworm/capi/silkworm_test.cpp +++ b/silkworm/capi/silkworm_test.cpp @@ -177,8 +177,11 @@ struct SilkwormLibrary { return result; } - int exectute_single_tx(MDBX_txn* txn) const { - return silkworm_execute_tx(handle_, txn, 1, SilkwormBytes32{}, 1, 1, nullptr, nullptr); + int exectute_single_tx(MDBX_txn* txn, uint64_t block_num, silkworm::Hash head_hash, uint64_t txn_index, uint64_t txn_id) const { + SilkwormBytes32 head_hash_bytes{}; + std::memcpy(head_hash_bytes.bytes, head_hash.bytes, 32); + + return silkworm_execute_tx(handle_, txn, block_num, head_hash_bytes, txn_index, txn_id, nullptr, nullptr); } int add_snapshot(SilkwormChainSnapshot* snapshot) const { @@ -990,16 +993,6 @@ static SilkwormRpcSettings make_rpc_settings_for_test(uint16_t api_listening_por return settings; } -TEST_CASE_METHOD(CApiTest, "CAPI silkworm_tx: single", "[silkworm][capi]") { - // Use Silkworm as a library with silkworm_init/silkworm_fini automated by RAII - SilkwormLibrary silkworm_lib{env_path()}; - - RWTxnManaged external_txn{env}; - auto result = silkworm_lib.exectute_single_tx(*external_txn); - CHECK_NOTHROW(external_txn.commit_and_stop()); - CHECK(result == SILKWORM_OK); -} - static const SilkwormRpcSettings kInvalidRpcSettings{make_rpc_settings_for_test(10)}; static const SilkwormRpcSettings kValidRpcSettings{make_rpc_settings_for_test(8545)}; @@ -1167,4 +1160,43 @@ TEST_CASE_METHOD(CApiTest, "CAPI silkworm_fork_validator", "[silkworm][capi]") { } } +TEST_CASE_METHOD(CApiTest, "CAPI silkworm_tx: single", "[silkworm][capi]") { + // Use Silkworm as a library with silkworm_init/silkworm_fini automated by RAII + SilkwormLibrary silkworm_lib{env_path()}; + silkworm_lib.start_fork_validator(env, &kValidForkValidatorSettings); + + // Prepare and insert block 10 (just 1 tx w/ value transfer) + evmc::address from{0x658bdf435d810c91414ec09147daa6db62406379_address}; // funded in genesis + evmc::address to{0x8b299e2b7d7f43c0ce3068263545309ff4ffb521_address}; // untouched address + intx::uint256 value{1 * kEther}; + + Block block{}; + block.header.number = 10; + block.header.gas_limit = 5'000'000; + block.header.gas_used = 21'000; + + static constexpr auto kEncoder = [](Bytes& dest, const Receipt& r) { rlp::encode(dest, r); }; + std::vector receipts{ + {TransactionType::kLegacy, true, block.header.gas_used, {}, {}}, + }; + block.header.receipts_root = trie::root_hash(receipts, kEncoder); + block.transactions.resize(1); + block.transactions[0].to = to; + block.transactions[0].gas_limit = block.header.gas_limit; + block.transactions[0].type = TransactionType::kLegacy; + block.transactions[0].max_priority_fee_per_gas = 0; + block.transactions[0].max_fee_per_gas = 20 * kGiga; + block.transactions[0].value = value; + block.transactions[0].r = 1; // dummy + block.transactions[0].s = 1; // dummy + block.transactions[0].set_sender(from); + + insert_block(env, block); + + RWTxnManaged external_txn{env}; + auto result = silkworm_lib.exectute_single_tx(*external_txn, 10, block.header.hash(), 0, 9); + CHECK(result == SILKWORM_INVALID_BLOCK); + CHECK_NOTHROW(external_txn.abort()); +} + } // namespace silkworm diff --git a/silkworm/db/datastore/kvdb/domain_get_latest_query.hpp b/silkworm/db/datastore/kvdb/domain_get_latest_query.hpp index d592ae0996..d04fcf119a 100644 --- a/silkworm/db/datastore/kvdb/domain_get_latest_query.hpp +++ b/silkworm/db/datastore/kvdb/domain_get_latest_query.hpp @@ -49,21 +49,21 @@ struct DomainGetLatestQuery { auto result = entity.has_large_values ? tx.ro_cursor(entity.values_table)->lower_bound(key_slice, false) : tx.ro_cursor(entity.values_table)->find(key_slice, false); if (!result) { - SILK_DEBUG << "key not found" << std::endl; + SILK_DEBUG << "key not found"; return std::nullopt; } DomainKeyDecoder> key_decoder{entity.has_large_values}; key_decoder.decode(result.key); if (key_decoder.value.key.value != from_slice(key_slice)) { - SILK_DEBUG << "key not found (decoded to different value)" << std::endl; + SILK_DEBUG << "key not found (decoded to different value)"; return std::nullopt; } DomainValueDecoder> empty_value_decoder{entity.has_large_values}; empty_value_decoder.decode(result.value); if (empty_value_decoder.value.value.value.empty()) { - SILK_DEBUG << "empty value" << std::endl; + SILK_DEBUG << "empty value"; return std::nullopt; } @@ -72,7 +72,7 @@ struct DomainGetLatestQuery { Step step = Step(key_decoder.value.timestamp.value.value | value_decoder.value.timestamp.value.value); - SILK_DEBUG << "found value " << to_hex(datastore::kvdb::from_slice(result.value), true) << " at step " << step.to_string() << " resulting key " << to_hex(datastore::kvdb::from_slice(result.key), true) << std::endl; + SILK_DEBUG << "found value " << to_hex(datastore::kvdb::from_slice(result.value), true) << " at step " << step.to_string() << " resulting key " << to_hex(datastore::kvdb::from_slice(result.key), true); return Result{std::move(value_decoder.value.value.value), step}; } diff --git a/silkworm/db/datastore/kvdb/domain_put_latest_query.hpp b/silkworm/db/datastore/kvdb/domain_put_latest_query.hpp index 098c312284..a5cca5a882 100644 --- a/silkworm/db/datastore/kvdb/domain_put_latest_query.hpp +++ b/silkworm/db/datastore/kvdb/domain_put_latest_query.hpp @@ -42,7 +42,7 @@ struct DomainPutLatestQuery { value_encoder.value.timestamp.value = step; Slice value_data = value_encoder.encode(); - SILK_DEBUG << "Putting " << entity.values_table.name << " with key " << to_hex(datastore::kvdb::from_slice(key_data), true) << " value " << to_hex(datastore::kvdb::from_slice(value_data), true) << std::endl; + SILK_DEBUG << "Putting " << entity.values_table.name << " with key " << to_hex(datastore::kvdb::from_slice(key_data), true) << " value " << to_hex(datastore::kvdb::from_slice(value_data), true); if (entity.values_table.value_mode == mdbx::value_mode::multi) { auto cursor = tx.rw_cursor_dup_sort(entity.values_table); @@ -65,12 +65,12 @@ struct DomainPutLatestQuery { Step existing_value_step = existing_value_decoder.value.timestamp.value; if (existing_value_step == step) { cursor->erase(); - SILK_DEBUG << "earasing" << std::endl; + SILK_DEBUG << "earasing"; } else { - SILK_DEBUG << "not earasing, existing_value_step: " << existing_value_step.to_string() << std::endl; + SILK_DEBUG << "not earasing, existing_value_step: " << existing_value_step.to_string(); } } else { - SILK_DEBUG << " not found" << std::endl; + SILK_DEBUG << " not found"; } cursor->upsert(key_data, value_data); diff --git a/silkworm/execution/domain_state.cpp b/silkworm/execution/domain_state.cpp index 8d822167b6..0c920f313e 100644 --- a/silkworm/execution/domain_state.cpp +++ b/silkworm/execution/domain_state.cpp @@ -179,7 +179,7 @@ void DomainState::update_storage( Step prev_step{0}; if (result_prev) { - SILK_DEBUG << "Found previous value " << to_hex(result_prev->value, true) << " step " << result_prev->step.to_string() << std::endl; + SILK_DEBUG << "Found previous value " << to_hex(result_prev->value, true) << " step " << result_prev->step.to_string(); prev_step = result_prev->step; } diff --git a/silkworm/execution/domain_state_test.cpp b/silkworm/execution/domain_state_test.cpp index 1709a28932..25a6f495ee 100644 --- a/silkworm/execution/domain_state_test.cpp +++ b/silkworm/execution/domain_state_test.cpp @@ -46,6 +46,7 @@ using testing::Unused; TEST_CASE("DomainState data access", "[execution][domain][state]") { TemporaryDirectory tmp_dir; TestDataStore ds_context{tmp_dir}; + log::init(log::Settings{.log_verbosity = log::Level::kDebug}); auto rw_tx = ds_context.chaindata_rw().start_rw_tx(); From 896908346f29a04ceb5828ac0a3493497613505b Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Fri, 24 Jan 2025 09:43:26 +0100 Subject: [PATCH 22/33] Consistent naming for txn index --- silkworm/capi/silkworm.cpp | 10 +++++----- silkworm/capi/silkworm.h | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/silkworm/capi/silkworm.cpp b/silkworm/capi/silkworm.cpp index 0fe3d6f6c7..255dff7f67 100644 --- a/silkworm/capi/silkworm.cpp +++ b/silkworm/capi/silkworm.cpp @@ -752,8 +752,8 @@ int silkworm_execute_blocks_perpetual(SilkwormHandle handle, MDBX_env* mdbx_env, } // todo: add available gas, add txn, add block header -SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* mdbx_tx, uint64_t block_num, struct SilkwormBytes32 head_hash_bytes, uint64_t txn_num, uint64_t txn_id, uint64_t* gas_used, uint64_t* blob_gas_used) SILKWORM_NOEXCEPT { - log::Info{"silkworm_execute_tx", {"block_num", std::to_string(block_num), "txn_num", std::to_string(txn_num)}}; +SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* mdbx_tx, uint64_t block_num, struct SilkwormBytes32 head_hash_bytes, uint64_t txn_index, uint64_t txn_id, uint64_t* gas_used, uint64_t* blob_gas_used) SILKWORM_NOEXCEPT { + log::Info{"silkworm_execute_tx", {"block_num", std::to_string(block_num), "txn_index", std::to_string(txn_index)}}; if (!handle) { return SILKWORM_INVALID_HANDLE; } @@ -828,13 +828,13 @@ SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* mdbx_tx } block.header = header.value(); - if (txn_num >= block.transactions.size()) { + if (txn_index >= block.transactions.size()) { SILK_ERROR << "Transaction not found" - << " txn_num: " << txn_num; + << " txn_index: " << txn_index; return SILKWORM_INVALID_BLOCK; } - auto& transaction = block.transactions[txn_num]; + auto& transaction = block.transactions[txn_index]; auto protocol_rule_set_{protocol::rule_set_factory(*handle->chain_config)}; ExecutionProcessor processor{block, *protocol_rule_set_, state, *handle->chain_config, false}; diff --git a/silkworm/capi/silkworm.h b/silkworm/capi/silkworm.h index 258a3cef2b..c7320cf43c 100644 --- a/silkworm/capi/silkworm.h +++ b/silkworm/capi/silkworm.h @@ -370,7 +370,7 @@ SILKWORM_EXPORT int silkworm_execute_blocks_perpetual(SilkwormHandle handle, MDB * \param[out] blob_gas_used The blob gas used by the transaction. * \return SILKWORM_OK (=0) on success, a non-zero error value on failure. */ -SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* mdbx_tx, uint64_t block_num, struct SilkwormBytes32 head_hash_bytes, uint64_t txn_num, uint64_t txn_id, uint64_t* gas_used, uint64_t* blob_gas_used) SILKWORM_NOEXCEPT; +SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* mdbx_tx, uint64_t block_num, struct SilkwormBytes32 head_hash_bytes, uint64_t txn_index, uint64_t txn_id, uint64_t* gas_used, uint64_t* blob_gas_used) SILKWORM_NOEXCEPT; /** * \brief Finalize the Silkworm C API library. From c7160b8cce72c39c48a7f167427852301f9f19f1 Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Fri, 24 Jan 2025 09:58:52 +0100 Subject: [PATCH 23/33] Remove redundant debug messages --- silkworm/db/datastore/kvdb/domain_get_latest_query.hpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/silkworm/db/datastore/kvdb/domain_get_latest_query.hpp b/silkworm/db/datastore/kvdb/domain_get_latest_query.hpp index d04fcf119a..9012978a2f 100644 --- a/silkworm/db/datastore/kvdb/domain_get_latest_query.hpp +++ b/silkworm/db/datastore/kvdb/domain_get_latest_query.hpp @@ -44,26 +44,22 @@ struct DomainGetLatestQuery { key_encoder.value.key.value = key; key_encoder.value.timestamp.value = Step{std::numeric_limits::max()}; Slice key_slice = key_encoder.encode(); - SILK_DEBUG << "Querying " << entity.values_table.name << " with key " << to_hex(datastore::kvdb::from_slice(key_slice), true) << " "; auto result = entity.has_large_values ? tx.ro_cursor(entity.values_table)->lower_bound(key_slice, false) : tx.ro_cursor(entity.values_table)->find(key_slice, false); if (!result) { - SILK_DEBUG << "key not found"; return std::nullopt; } DomainKeyDecoder> key_decoder{entity.has_large_values}; key_decoder.decode(result.key); if (key_decoder.value.key.value != from_slice(key_slice)) { - SILK_DEBUG << "key not found (decoded to different value)"; return std::nullopt; } DomainValueDecoder> empty_value_decoder{entity.has_large_values}; empty_value_decoder.decode(result.value); if (empty_value_decoder.value.value.value.empty()) { - SILK_DEBUG << "empty value"; return std::nullopt; } @@ -72,8 +68,6 @@ struct DomainGetLatestQuery { Step step = Step(key_decoder.value.timestamp.value.value | value_decoder.value.timestamp.value.value); - SILK_DEBUG << "found value " << to_hex(datastore::kvdb::from_slice(result.value), true) << " at step " << step.to_string() << " resulting key " << to_hex(datastore::kvdb::from_slice(result.key), true); - return Result{std::move(value_decoder.value.value.value), step}; } }; From 4f6c8d263755948936df2f7336a2d4d3a2377a74 Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Wed, 29 Jan 2025 11:12:03 +0100 Subject: [PATCH 24/33] Simplify domain queries --- silkworm/capi/silkworm.cpp | 3 +- silkworm/db/datastore/kvdb/database.hpp | 33 ++++--- .../kvdb/domain_put_latest_query.hpp | 12 +-- silkworm/db/state/accounts_domain.hpp | 24 +++-- silkworm/db/state/code_domain.hpp | 24 +++-- silkworm/db/state/storage_domain.hpp | 27 ++++-- silkworm/execution/domain_state.cpp | 93 ++++++++----------- silkworm/execution/domain_state.hpp | 6 +- silkworm/execution/domain_state_test.cpp | 12 ++- silkworm/execution/local_state.cpp | 6 +- 10 files changed, 123 insertions(+), 117 deletions(-) diff --git a/silkworm/capi/silkworm.cpp b/silkworm/capi/silkworm.cpp index 255dff7f67..9bd42b54c6 100644 --- a/silkworm/capi/silkworm.cpp +++ b/silkworm/capi/silkworm.cpp @@ -778,7 +778,8 @@ SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* mdbx_tx auto unmanaged_tx = datastore::kvdb::RWTxnUnmanaged{mdbx_tx}; auto unmanaged_env = unmanaged_tx.unmanaged_env(); auto chain_db = db::DataStore::make_chaindata_database(std::move(unmanaged_env)); - auto state = silkworm::execution::DomainState{txn_id_, unmanaged_tx, chain_db, *handle->blocks_repository, *handle->state_repository}; + auto db_ref = chain_db.ref(); + auto state = silkworm::execution::DomainState{txn_id_, unmanaged_tx, db_ref, *handle->blocks_repository, *handle->state_repository}; if (!handle->chain_config) { handle->chain_config = db::read_chain_config(unmanaged_tx); } diff --git a/silkworm/db/datastore/kvdb/database.hpp b/silkworm/db/datastore/kvdb/database.hpp index 930bb2a4d7..a56c8dfa28 100644 --- a/silkworm/db/datastore/kvdb/database.hpp +++ b/silkworm/db/datastore/kvdb/database.hpp @@ -59,20 +59,7 @@ class DatabaseRef { DatabaseRef::EntitiesMap make_entities(const Schema::DatabaseDef& schema); -class DatabaseBase { - public: - virtual ~DatabaseBase() = default; - - ROAccess access_ro() const { return ref().access_ro(); } - RWAccess access_rw() const { return ref().access_rw(); } - - Domain domain(datastore::EntityName name) const { return ref().domain(name); } - InvertedIndex inverted_index(datastore::EntityName name) { return ref().inverted_index(name); } - - virtual DatabaseRef ref() const = 0; -}; - -class Database : public DatabaseBase { +class Database { public: Database( mdbx::env_managed env, @@ -81,7 +68,13 @@ class Database : public DatabaseBase { schema_{std::move(schema)}, entities_{make_entities(schema_)} {} - DatabaseRef ref() const override { return {env_, schema_, entities_}; } // NOLINT(cppcoreguidelines-slicing) + ROAccess access_ro() const { return ref().access_ro(); } + RWAccess access_rw() const { return ref().access_rw(); } + + Domain domain(datastore::EntityName name) const { return ref().domain(name); } + InvertedIndex inverted_index(datastore::EntityName name) const { return ref().inverted_index(name); } + + DatabaseRef ref() const { return {env_, schema_, entities_}; } // NOLINT(cppcoreguidelines-slicing) void create_tables(); @@ -91,7 +84,7 @@ class Database : public DatabaseBase { std::map> entities_; }; -class DatabaseUnmanaged : public DatabaseBase { +class DatabaseUnmanaged { public: DatabaseUnmanaged( EnvUnmanaged env, @@ -100,7 +93,13 @@ class DatabaseUnmanaged : public DatabaseBase { schema_{std::move(schema)}, entities_{make_entities(schema_)} {} - DatabaseRef ref() const override { return {env_, schema_, entities_}; } // NOLINT(cppcoreguidelines-slicing) + ROAccess access_ro() const { return ref().access_ro(); } + RWAccess access_rw() const { return ref().access_rw(); } + + Domain domain(datastore::EntityName name) const { return ref().domain(name); } + InvertedIndex inverted_index(datastore::EntityName name) const { return ref().inverted_index(name); } + + DatabaseRef ref() const { return {env_, schema_, entities_}; } // NOLINT(cppcoreguidelines-slicing) private: EnvUnmanaged env_; diff --git a/silkworm/db/datastore/kvdb/domain_put_latest_query.hpp b/silkworm/db/datastore/kvdb/domain_put_latest_query.hpp index a5cca5a882..7c5c1396ae 100644 --- a/silkworm/db/datastore/kvdb/domain_put_latest_query.hpp +++ b/silkworm/db/datastore/kvdb/domain_put_latest_query.hpp @@ -42,7 +42,7 @@ struct DomainPutLatestQuery { value_encoder.value.timestamp.value = step; Slice value_data = value_encoder.encode(); - SILK_DEBUG << "Putting " << entity.values_table.name << " with key " << to_hex(datastore::kvdb::from_slice(key_data), true) << " value " << to_hex(datastore::kvdb::from_slice(value_data), true); + // SILK_DEBUG << "Putting " << entity.values_table.name << " with key " << to_hex(datastore::kvdb::from_slice(key_data), true) << " value " << to_hex(datastore::kvdb::from_slice(value_data), true); if (entity.values_table.value_mode == mdbx::value_mode::multi) { auto cursor = tx.rw_cursor_dup_sort(entity.values_table); @@ -53,11 +53,11 @@ struct DomainPutLatestQuery { same_step_value_encoder.value.timestamp.value = step; Slice same_step_value = same_step_value_encoder.encode(); - SILK_DEBUG << "DupSort looking for existing value " << to_hex(datastore::kvdb::from_slice(same_step_value), true); + // SILK_DEBUG << "DupSort looking for existing value " << to_hex(datastore::kvdb::from_slice(same_step_value), true); CursorResult result = cursor->lower_bound_multivalue(key_data, same_step_value, false); if (result) { - SILK_DEBUG << " found and "; + // SILK_DEBUG << " found and "; // the found value will have the same key, but the step part can be different, // let's decode it ignoring the data part DomainValueDecoder> existing_value_decoder{entity.has_large_values}; @@ -65,12 +65,12 @@ struct DomainPutLatestQuery { Step existing_value_step = existing_value_decoder.value.timestamp.value; if (existing_value_step == step) { cursor->erase(); - SILK_DEBUG << "earasing"; + // SILK_DEBUG << "earasing"; } else { - SILK_DEBUG << "not earasing, existing_value_step: " << existing_value_step.to_string(); + // SILK_DEBUG << "not earasing, existing_value_step: " << existing_value_step.to_string(); } } else { - SILK_DEBUG << " not found"; + // SILK_DEBUG << " not found"; } cursor->upsert(key_data, value_data); diff --git a/silkworm/db/state/accounts_domain.hpp b/silkworm/db/state/accounts_domain.hpp index 7d52f9a37a..510ff91a85 100644 --- a/silkworm/db/state/accounts_domain.hpp +++ b/silkworm/db/state/accounts_domain.hpp @@ -26,24 +26,32 @@ namespace silkworm::db::state { -using AccountsDomainGetLatestQueryBase = datastore::DomainGetLatestQuery< - AddressKVDBEncoder, AddressSnapshotsEncoder, - AccountKVDBCodec, AccountSnapshotsCodec>; - -struct AccountsDomainGetLatestQuery : public AccountsDomainGetLatestQueryBase { +struct AccountsDomainGetLatestQuery : public datastore::DomainGetLatestQuery< + AddressKVDBEncoder, AddressSnapshotsEncoder, + AccountKVDBCodec, AccountSnapshotsCodec> { AccountsDomainGetLatestQuery( const datastore::kvdb::DatabaseRef& database, datastore::kvdb::ROTxn& tx, const snapshots::SnapshotRepositoryROAccess& repository) - : AccountsDomainGetLatestQueryBase{ + : datastore::DomainGetLatestQuery< + AddressKVDBEncoder, AddressSnapshotsEncoder, + AccountKVDBCodec, AccountSnapshotsCodec>{ db::state::kDomainNameAccounts, - database, + database.domain(db::state::kDomainNameAccounts), tx, repository, } {} }; -using AccountsDomainPutQuery = datastore::kvdb::DomainPutQuery; +struct AccountsDomainPutQuery : public datastore::kvdb::DomainPutQuery { + AccountsDomainPutQuery( + const datastore::kvdb::DatabaseRef& database, + datastore::kvdb::RWTxn& tx) + : datastore::kvdb::DomainPutQuery{ + tx, + database.domain(db::state::kDomainNameAccounts)} {} +}; + using AccountsDomainDeleteQuery = datastore::kvdb::DomainDeleteQuery; using AccountsDomainKVSegmentReader = snapshots::segment::KVSegmentReader; diff --git a/silkworm/db/state/code_domain.hpp b/silkworm/db/state/code_domain.hpp index 8a606a3cf8..d864a322a9 100644 --- a/silkworm/db/state/code_domain.hpp +++ b/silkworm/db/state/code_domain.hpp @@ -26,24 +26,32 @@ namespace silkworm::db::state { -using CodeDomainGetLatestQueryBase = datastore::DomainGetLatestQuery< - AddressKVDBEncoder, AddressSnapshotsEncoder, - datastore::kvdb::RawDecoder, snapshots::RawDecoder>; - -struct CodeDomainGetLatestQuery : public CodeDomainGetLatestQueryBase { +struct CodeDomainGetLatestQuery : public datastore::DomainGetLatestQuery< + AddressKVDBEncoder, AddressSnapshotsEncoder, + datastore::kvdb::RawDecoder, snapshots::RawDecoder> { CodeDomainGetLatestQuery( const datastore::kvdb::DatabaseRef& database, datastore::kvdb::ROTxn& tx, const snapshots::SnapshotRepositoryROAccess& repository) - : CodeDomainGetLatestQueryBase{ + : datastore::DomainGetLatestQuery< + AddressKVDBEncoder, AddressSnapshotsEncoder, + datastore::kvdb::RawDecoder, snapshots::RawDecoder>{ db::state::kDomainNameCode, - database, + database.domain(db::state::kDomainNameCode), tx, repository, } {} }; -using CodeDomainPutQuery = datastore::kvdb::DomainPutQuery>; +struct CodeDomainPutQuery : public datastore::kvdb::DomainPutQuery> { + CodeDomainPutQuery( + const datastore::kvdb::DatabaseRef& database, + datastore::kvdb::RWTxn& tx) + : datastore::kvdb::DomainPutQuery>{ + tx, + database.domain(db::state::kDomainNameCode)} {} +}; + using CodeDomainDeleteQuery = datastore::kvdb::DomainDeleteQuery>; using CodeDomainKVSegmentReader = snapshots::segment::KVSegmentReader>; diff --git a/silkworm/db/state/storage_domain.hpp b/silkworm/db/state/storage_domain.hpp index efffa8a9c1..2b924bb2eb 100644 --- a/silkworm/db/state/storage_domain.hpp +++ b/silkworm/db/state/storage_domain.hpp @@ -25,24 +25,31 @@ namespace silkworm::db::state { -using StorageDomainGetLatestQueryBase = datastore::DomainGetLatestQuery< - StorageAddressAndLocationKVDBEncoder, StorageAddressAndLocationSnapshotsCodec, - Bytes32KVDBCodec, Bytes32SnapshotsCodec>; - -struct StorageDomainGetLatestQuery : public StorageDomainGetLatestQueryBase { +struct StorageDomainGetLatestQuery : public datastore::DomainGetLatestQuery< + StorageAddressAndLocationKVDBEncoder, StorageAddressAndLocationSnapshotsCodec, + Bytes32KVDBCodec, Bytes32SnapshotsCodec> { StorageDomainGetLatestQuery( const datastore::kvdb::DatabaseRef& database, datastore::kvdb::ROTxn& tx, const snapshots::SnapshotRepositoryROAccess& repository) - : StorageDomainGetLatestQueryBase{ + : datastore::DomainGetLatestQuery< + StorageAddressAndLocationKVDBEncoder, StorageAddressAndLocationSnapshotsCodec, + Bytes32KVDBCodec, Bytes32SnapshotsCodec>{ db::state::kDomainNameStorage, - database, + database.domain(db::state::kDomainNameStorage), + tx, + repository} {} +}; + +struct StorageDomainPutQuery : public datastore::kvdb::DomainPutQuery { + StorageDomainPutQuery( + const datastore::kvdb::DatabaseRef& database, + datastore::kvdb::RWTxn& tx) + : datastore::kvdb::DomainPutQuery{ tx, - repository, - } {} + database.domain(db::state::kDomainNameStorage)} {} }; -using StorageDomainPutQuery = datastore::kvdb::DomainPutQuery; using StorageDomainDeleteQuery = datastore::kvdb::DomainDeleteQuery; using StorageDomainKVSegmentReader = snapshots::segment::KVSegmentReader; diff --git a/silkworm/execution/domain_state.cpp b/silkworm/execution/domain_state.cpp index 0c920f313e..eb0b7696cc 100644 --- a/silkworm/execution/domain_state.cpp +++ b/silkworm/execution/domain_state.cpp @@ -37,12 +37,7 @@ using namespace datastore; // - add base_txn_id to block and get_txn_by_id method std::optional DomainState::read_account(const evmc::address& address) const noexcept { - AccountsDomainGetLatestQuery query{ - db::state::kDomainNameAccounts, - database_.domain(db::state::kDomainNameAccounts), - tx_, - state_repository_, - }; + AccountsDomainGetLatestQuery query{database_, tx_, state_repository_}; auto result = query.exec(address); if (result) { return std::move(result->value); @@ -51,12 +46,7 @@ std::optional DomainState::read_account(const evmc::address& address) c } ByteView DomainState::read_code(const evmc::address& address, const evmc::bytes32& /*code_hash*/) const noexcept { - CodeDomainGetLatestQuery query{ - db::state::kDomainNameCode, - database_.domain(db::state::kDomainNameCode), - tx_, - state_repository_, - }; + CodeDomainGetLatestQuery query{database_, tx_, state_repository_}; auto result = query.exec(address); if (result) { return std::move(result->value); @@ -68,12 +58,7 @@ evmc::bytes32 DomainState::read_storage( const evmc::address& address, uint64_t /*incarnation*/, const evmc::bytes32& location) const noexcept { - StorageDomainGetLatestQuery query{ - db::state::kDomainNameStorage, - database_.domain(db::state::kDomainNameStorage), - tx_, - state_repository_, - }; + StorageDomainGetLatestQuery query{database_, tx_, state_repository_}; auto result = query.exec({address, location}); if (result) { return std::move(result->value); @@ -119,24 +104,25 @@ void DomainState::insert_receipts(BlockNum block_num, const std::vector void DomainState::update_account( const evmc::address& address, - std::optional initial, + std::optional original, std::optional current) { - AccountsDomainGetLatestQuery query_prev{ - db::state::kDomainNameAccounts, - database_.domain(db::state::kDomainNameAccounts), - tx_, - state_repository_, - }; - auto result_prev = query_prev.exec(address); - - Step prev_step{0}; - if (result_prev) { - prev_step = result_prev->step; + if (!original) { + AccountsDomainGetLatestQuery query_prev{database_, tx_, state_repository_}; + auto result_prev = query_prev.exec(address); + if (result_prev) { + original = result_prev->value; + } } - // TODO: handle current == nullopt - AccountsDomainPutQuery query{tx_, database_.domain(db::state::kDomainNameAccounts)}; - query.exec(address, *current, txn_id_, initial, prev_step); + auto prev_step = Step{0}; // TODO: JG remove + + if (current) { + AccountsDomainPutQuery query{database_, tx_}; + query.exec(address, *current, txn_id_, original, prev_step); + } else { + AccountsDomainDeleteQuery query{tx_, database_.domain(kDomainNameAccounts)}; + query.exec(address, txn_id_, original, prev_step); + } } void DomainState::update_account_code( @@ -144,23 +130,18 @@ void DomainState::update_account_code( uint64_t /*incarnation*/, const evmc::bytes32& /*code_hash*/, ByteView code) { - CodeDomainGetLatestQuery query_prev{ - db::state::kDomainNameCode, - database_.domain(db::state::kDomainNameCode), - tx_, - state_repository_, - }; + CodeDomainGetLatestQuery query_prev{database_, tx_, state_repository_}; auto result_prev = query_prev.exec(address); Step prev_step{0}; - std::optional initial_code = std::nullopt; + std::optional original_code = std::nullopt; if (result_prev) { prev_step = result_prev->step; - initial_code = result_prev->value; + original_code = result_prev->value; } - CodeDomainPutQuery query{tx_, database_.domain(db::state::kDomainNameCode)}; - query.exec(address, code, txn_id_, initial_code, prev_step); + CodeDomainPutQuery query{database_, tx_}; + query.exec(address, code, txn_id_, original_code, prev_step); } void DomainState::update_storage( @@ -169,22 +150,22 @@ void DomainState::update_storage( const evmc::bytes32& location, const evmc::bytes32& initial, const evmc::bytes32& current) { - StorageDomainGetLatestQuery query_prev{ - db::state::kDomainNameStorage, - database_.domain(db::state::kDomainNameStorage), - tx_, - state_repository_, - }; - auto result_prev = query_prev.exec({address, location}); + evmc::bytes32 original_value{}; + + if (initial == evmc::bytes32{}) { + StorageDomainGetLatestQuery query_prev{database_, tx_, state_repository_}; + auto result_prev = query_prev.exec({address, location}); + if (result_prev) { + original_value = result_prev->value; + } + } else { + original_value = initial; + } Step prev_step{0}; - if (result_prev) { - SILK_DEBUG << "Found previous value " << to_hex(result_prev->value, true) << " step " << result_prev->step.to_string(); - prev_step = result_prev->step; - } - StorageDomainPutQuery query{tx_, database_.domain(db::state::kDomainNameStorage)}; - query.exec({address, location}, current, txn_id_, initial, prev_step); + StorageDomainPutQuery query{database_, tx_}; + query.exec({address, location}, current, txn_id_, original_value, prev_step); } } // namespace silkworm::execution diff --git a/silkworm/execution/domain_state.hpp b/silkworm/execution/domain_state.hpp index 556ed75a48..fd0eff98e4 100644 --- a/silkworm/execution/domain_state.hpp +++ b/silkworm/execution/domain_state.hpp @@ -37,7 +37,7 @@ class DomainState : public State { explicit DomainState( TxnId txn_id, datastore::kvdb::RWTxn& tx, - datastore::kvdb::DatabaseBase& database, + datastore::kvdb::DatabaseRef& database, snapshots::SnapshotRepository& blocks_repository, snapshots::SnapshotRepository& state_repository) : txn_id_{txn_id}, @@ -49,7 +49,7 @@ class DomainState : public State { explicit DomainState( TxnId txn_id, datastore::kvdb::RWTxn& tx, - datastore::kvdb::DatabaseBase& database, + datastore::kvdb::DatabaseRef& database, snapshots::SnapshotRepository& state_repository, db::DataModel& data_model) @@ -114,7 +114,7 @@ class DomainState : public State { private: TxnId txn_id_; datastore::kvdb::RWTxn& tx_; - datastore::kvdb::DatabaseBase& database_; + datastore::kvdb::DatabaseRef& database_; snapshots::SnapshotRepository& state_repository_; db::DataModel data_model_; }; diff --git a/silkworm/execution/domain_state_test.cpp b/silkworm/execution/domain_state_test.cpp index 25a6f495ee..18422cf01e 100644 --- a/silkworm/execution/domain_state_test.cpp +++ b/silkworm/execution/domain_state_test.cpp @@ -50,7 +50,8 @@ TEST_CASE("DomainState data access", "[execution][domain][state]") { auto rw_tx = ds_context.chaindata_rw().start_rw_tx(); - auto sut = DomainState{1, rw_tx, ds_context->chaindata(), ds_context->blocks_repository(), ds_context->state_repository()}; + auto db_ref = ds_context->chaindata().ref(); + auto sut = DomainState{1, rw_tx, db_ref, ds_context->blocks_repository(), ds_context->state_repository()}; auto header0_hash = sut.canonical_hash(0); auto header0 = sut.read_header(0, header0_hash.value()); @@ -169,7 +170,7 @@ TEST_CASE("DomainState data access", "[execution][domain][state]") { CHECK(account_66_read->balance == account_66_v2.balance); auto next_step_txn_id = silkworm::datastore::kStepSizeForTemporalSnapshots + 1; - auto sut2 = DomainState{next_step_txn_id, rw_tx, ds_context->chaindata(), ds_context->blocks_repository(), ds_context->state_repository()}; + auto sut2 = DomainState{next_step_txn_id, rw_tx, db_ref, ds_context->blocks_repository(), ds_context->state_repository()}; Account account_66_v3{ .nonce = 10, .balance = 262, @@ -228,7 +229,7 @@ TEST_CASE("DomainState data access", "[execution][domain][state]") { CHECK(code_66_read == code_66); auto next_step_txn_id = silkworm::datastore::kStepSizeForTemporalSnapshots + 1; - auto sut2 = DomainState{next_step_txn_id, rw_tx, ds_context->chaindata(), ds_context->blocks_repository(), ds_context->state_repository()}; + auto sut2 = DomainState{next_step_txn_id, rw_tx, db_ref, ds_context->blocks_repository(), ds_context->state_repository()}; code_66 = *from_hex("0x6044"); code_hash_66 = std::bit_cast(keccak256(code_66)); sut2.update_account_code( @@ -285,7 +286,7 @@ TEST_CASE("DomainState data access", "[execution][domain][state]") { CHECK(storage_66_01 == 0x0124_bytes32); auto next_step_txn_id = silkworm::datastore::kStepSizeForTemporalSnapshots + 1; - auto sut2 = DomainState{next_step_txn_id, rw_tx, ds_context->chaindata(), ds_context->blocks_repository(), ds_context->state_repository()}; + auto sut2 = DomainState{next_step_txn_id, rw_tx, db_ref, ds_context->blocks_repository(), ds_context->state_repository()}; sut2.update_storage( 0x668bdf435d810c91414ec09147daa6db62406379_address, kDefaultIncarnation, @@ -306,7 +307,8 @@ TEST_CASE("DomainState empty overriden methods do nothing", "[execution][domain] auto rw_tx = ds_context.chaindata_rw().start_rw_tx(); - auto sut = DomainState{1, rw_tx, ds_context->chaindata(), ds_context->blocks_repository(), ds_context->state_repository()}; + auto db_ref = ds_context->chaindata().ref(); + auto sut = DomainState{1, rw_tx, db_ref, ds_context->blocks_repository(), ds_context->state_repository()}; CHECK_NOTHROW(sut.insert_block(Block{}, evmc::bytes32{})); CHECK_NOTHROW(sut.canonize_block(0, evmc::bytes32{})); diff --git a/silkworm/execution/local_state.cpp b/silkworm/execution/local_state.cpp index 3fb04dafcc..ec2fcd558b 100644 --- a/silkworm/execution/local_state.cpp +++ b/silkworm/execution/local_state.cpp @@ -120,7 +120,7 @@ void LocalState::update_account( Step current_step = Step::from_txn_id(*txn_id_); if (current) { - AccountsDomainPutQuery query{tx_, data_store_.state_db().accounts_domain()}; + AccountsDomainPutQuery query{data_store_.chaindata, tx_}; query.exec(address, *current, *txn_id_, initial, current_step); } else { AccountsDomainDeleteQuery query{tx_, data_store_.state_db().accounts_domain()}; @@ -136,7 +136,7 @@ void LocalState::update_account_code( /* should be managed request on Latest(txn_id == nullopt) and historical (txn_id != nullopt) */ Step current_step = Step::from_txn_id(*txn_id_); - CodeDomainPutQuery query{tx_, data_store_.state_db().code_domain()}; + CodeDomainPutQuery query{data_store_.chaindata, tx_}; std::optional initial_code = read_code(address, evmc::bytes32{}); if (initial_code && initial_code->empty()) initial_code = std::nullopt; @@ -152,7 +152,7 @@ void LocalState::update_storage( /* should be managed request on Latest(txn_id == nullopt) and historical (txn_id != nullopt) */ Step current_step = Step::from_txn_id(*txn_id_); - StorageDomainPutQuery query{tx_, data_store_.state_db().storage_domain()}; + StorageDomainPutQuery query{data_store_.chaindata, tx_}; query.exec({address, location}, current, *txn_id_, initial, current_step); } From 2ce87c207e461298f32beaa25dca039c1b41274e Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Wed, 29 Jan 2025 12:11:03 +0100 Subject: [PATCH 25/33] Fix CI build errors --- silkworm/db/state/accounts_domain.hpp | 4 ++-- silkworm/db/state/code_domain.hpp | 4 ++-- silkworm/db/state/storage_domain.hpp | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/silkworm/db/state/accounts_domain.hpp b/silkworm/db/state/accounts_domain.hpp index 510ff91a85..f376058814 100644 --- a/silkworm/db/state/accounts_domain.hpp +++ b/silkworm/db/state/accounts_domain.hpp @@ -46,9 +46,9 @@ struct AccountsDomainGetLatestQuery : public datastore::DomainGetLatestQuery< struct AccountsDomainPutQuery : public datastore::kvdb::DomainPutQuery { AccountsDomainPutQuery( const datastore::kvdb::DatabaseRef& database, - datastore::kvdb::RWTxn& tx) + datastore::kvdb::RWTxn& rw_tx) : datastore::kvdb::DomainPutQuery{ - tx, + rw_tx, database.domain(db::state::kDomainNameAccounts)} {} }; diff --git a/silkworm/db/state/code_domain.hpp b/silkworm/db/state/code_domain.hpp index d864a322a9..07f9709b17 100644 --- a/silkworm/db/state/code_domain.hpp +++ b/silkworm/db/state/code_domain.hpp @@ -46,9 +46,9 @@ struct CodeDomainGetLatestQuery : public datastore::DomainGetLatestQuery< struct CodeDomainPutQuery : public datastore::kvdb::DomainPutQuery> { CodeDomainPutQuery( const datastore::kvdb::DatabaseRef& database, - datastore::kvdb::RWTxn& tx) + datastore::kvdb::RWTxn& rw_tx) : datastore::kvdb::DomainPutQuery>{ - tx, + rw_tx, database.domain(db::state::kDomainNameCode)} {} }; diff --git a/silkworm/db/state/storage_domain.hpp b/silkworm/db/state/storage_domain.hpp index 2b924bb2eb..68df26bb89 100644 --- a/silkworm/db/state/storage_domain.hpp +++ b/silkworm/db/state/storage_domain.hpp @@ -44,9 +44,9 @@ struct StorageDomainGetLatestQuery : public datastore::DomainGetLatestQuery< struct StorageDomainPutQuery : public datastore::kvdb::DomainPutQuery { StorageDomainPutQuery( const datastore::kvdb::DatabaseRef& database, - datastore::kvdb::RWTxn& tx) + datastore::kvdb::RWTxn& rw_tx) : datastore::kvdb::DomainPutQuery{ - tx, + rw_tx, database.domain(db::state::kDomainNameStorage)} {} }; From 6f776096c3e6f85c247dc8d493006830e2067626 Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Wed, 29 Jan 2025 14:51:34 +0100 Subject: [PATCH 26/33] Cleanup code --- .../db/datastore/kvdb/domain_delete_query.hpp | 5 ++--- .../kvdb/domain_put_latest_query.hpp | 10 ---------- .../db/datastore/kvdb/domain_put_query.hpp | 9 +++++---- silkworm/db/state/accounts_domain.hpp | 5 ++--- silkworm/db/state/code_domain.hpp | 5 ++--- silkworm/db/state/storage_domain.hpp | 4 ++-- silkworm/execution/CMakeLists.txt | 2 -- silkworm/execution/domain_state.cpp | 20 ++++++++++--------- 8 files changed, 24 insertions(+), 36 deletions(-) diff --git a/silkworm/db/datastore/kvdb/domain_delete_query.hpp b/silkworm/db/datastore/kvdb/domain_delete_query.hpp index 1d9e36b1b3..804e8cee22 100644 --- a/silkworm/db/datastore/kvdb/domain_delete_query.hpp +++ b/silkworm/db/datastore/kvdb/domain_delete_query.hpp @@ -31,8 +31,7 @@ struct DomainDeleteQuery { void exec( const Key& key, Timestamp timestamp, - const std::optional& prev_value, - Step current_step) { + const std::optional& prev_value) { if (prev_value) { TValueEncoder prev_value_encoder; prev_value_encoder.value = std::move(*prev_value); @@ -43,7 +42,7 @@ struct DomainDeleteQuery { ByteView prev_value_data = prev_value_slice_decoder.value; DomainPutQuery> query{tx, entity}; - query.exec(key, ByteView{}, timestamp, prev_value_data, current_step); + query.exec(key, ByteView{}, timestamp, prev_value_data); } } }; diff --git a/silkworm/db/datastore/kvdb/domain_put_latest_query.hpp b/silkworm/db/datastore/kvdb/domain_put_latest_query.hpp index 7c5c1396ae..0d8492b163 100644 --- a/silkworm/db/datastore/kvdb/domain_put_latest_query.hpp +++ b/silkworm/db/datastore/kvdb/domain_put_latest_query.hpp @@ -42,8 +42,6 @@ struct DomainPutLatestQuery { value_encoder.value.timestamp.value = step; Slice value_data = value_encoder.encode(); - // SILK_DEBUG << "Putting " << entity.values_table.name << " with key " << to_hex(datastore::kvdb::from_slice(key_data), true) << " value " << to_hex(datastore::kvdb::from_slice(value_data), true); - if (entity.values_table.value_mode == mdbx::value_mode::multi) { auto cursor = tx.rw_cursor_dup_sort(entity.values_table); @@ -53,11 +51,8 @@ struct DomainPutLatestQuery { same_step_value_encoder.value.timestamp.value = step; Slice same_step_value = same_step_value_encoder.encode(); - // SILK_DEBUG << "DupSort looking for existing value " << to_hex(datastore::kvdb::from_slice(same_step_value), true); - CursorResult result = cursor->lower_bound_multivalue(key_data, same_step_value, false); if (result) { - // SILK_DEBUG << " found and "; // the found value will have the same key, but the step part can be different, // let's decode it ignoring the data part DomainValueDecoder> existing_value_decoder{entity.has_large_values}; @@ -65,12 +60,7 @@ struct DomainPutLatestQuery { Step existing_value_step = existing_value_decoder.value.timestamp.value; if (existing_value_step == step) { cursor->erase(); - // SILK_DEBUG << "earasing"; - } else { - // SILK_DEBUG << "not earasing, existing_value_step: " << existing_value_step.to_string(); } - } else { - // SILK_DEBUG << " not found"; } cursor->upsert(key_data, value_data); diff --git a/silkworm/db/datastore/kvdb/domain_put_query.hpp b/silkworm/db/datastore/kvdb/domain_put_query.hpp index 8bd0f07040..8ea36090e4 100644 --- a/silkworm/db/datastore/kvdb/domain_put_query.hpp +++ b/silkworm/db/datastore/kvdb/domain_put_query.hpp @@ -34,18 +34,19 @@ struct DomainPutQuery { const Key& key, const Value& value, Timestamp timestamp, - const std::optional& prev_value, - Step prev_step) { + const std::optional& prev_value) { + DomainPutLatestQuery value_query{tx, entity}; value_query.exec(key, value, Step::from_txn_id(timestamp)); + if (entity.history) { if (prev_value) { HistoryPutQuery history_query{tx, *entity.history}; - history_query.exec(key, *prev_value, prev_step.value); + history_query.exec(key, *prev_value, timestamp); } else { HistoryDeleteQuery history_query{tx, *entity.history}; - history_query.exec(key, prev_step.value); + history_query.exec(key, timestamp); } } } diff --git a/silkworm/db/state/accounts_domain.hpp b/silkworm/db/state/accounts_domain.hpp index f376058814..df2d362a37 100644 --- a/silkworm/db/state/accounts_domain.hpp +++ b/silkworm/db/state/accounts_domain.hpp @@ -35,12 +35,11 @@ struct AccountsDomainGetLatestQuery : public datastore::DomainGetLatestQuery< const snapshots::SnapshotRepositoryROAccess& repository) : datastore::DomainGetLatestQuery< AddressKVDBEncoder, AddressSnapshotsEncoder, - AccountKVDBCodec, AccountSnapshotsCodec>{ + AccountKVDBCodec, AccountSnapshotsCodec>( db::state::kDomainNameAccounts, database.domain(db::state::kDomainNameAccounts), tx, - repository, - } {} + repository ) {} }; struct AccountsDomainPutQuery : public datastore::kvdb::DomainPutQuery { diff --git a/silkworm/db/state/code_domain.hpp b/silkworm/db/state/code_domain.hpp index 07f9709b17..aae6041c0a 100644 --- a/silkworm/db/state/code_domain.hpp +++ b/silkworm/db/state/code_domain.hpp @@ -35,12 +35,11 @@ struct CodeDomainGetLatestQuery : public datastore::DomainGetLatestQuery< const snapshots::SnapshotRepositoryROAccess& repository) : datastore::DomainGetLatestQuery< AddressKVDBEncoder, AddressSnapshotsEncoder, - datastore::kvdb::RawDecoder, snapshots::RawDecoder>{ + datastore::kvdb::RawDecoder, snapshots::RawDecoder>( db::state::kDomainNameCode, database.domain(db::state::kDomainNameCode), tx, - repository, - } {} + repository) {} }; struct CodeDomainPutQuery : public datastore::kvdb::DomainPutQuery> { diff --git a/silkworm/db/state/storage_domain.hpp b/silkworm/db/state/storage_domain.hpp index 68df26bb89..532ea03a9b 100644 --- a/silkworm/db/state/storage_domain.hpp +++ b/silkworm/db/state/storage_domain.hpp @@ -34,11 +34,11 @@ struct StorageDomainGetLatestQuery : public datastore::DomainGetLatestQuery< const snapshots::SnapshotRepositoryROAccess& repository) : datastore::DomainGetLatestQuery< StorageAddressAndLocationKVDBEncoder, StorageAddressAndLocationSnapshotsCodec, - Bytes32KVDBCodec, Bytes32SnapshotsCodec>{ + Bytes32KVDBCodec, Bytes32SnapshotsCodec>( db::state::kDomainNameStorage, database.domain(db::state::kDomainNameStorage), tx, - repository} {} + repository) {} }; struct StorageDomainPutQuery : public datastore::kvdb::DomainPutQuery { diff --git a/silkworm/execution/CMakeLists.txt b/silkworm/execution/CMakeLists.txt index aac33d663e..7ed4d7faaf 100644 --- a/silkworm/execution/CMakeLists.txt +++ b/silkworm/execution/CMakeLists.txt @@ -35,8 +35,6 @@ set(LIBS_PUBLIC silkworm_core silkworm_infra silkworm_db - silkworm_block_execution - silkworm_snapshots ) # cmake-format: on diff --git a/silkworm/execution/domain_state.cpp b/silkworm/execution/domain_state.cpp index eb0b7696cc..21acc2add6 100644 --- a/silkworm/execution/domain_state.cpp +++ b/silkworm/execution/domain_state.cpp @@ -58,6 +58,7 @@ evmc::bytes32 DomainState::read_storage( const evmc::address& address, uint64_t /*incarnation*/, const evmc::bytes32& location) const noexcept { + StorageDomainGetLatestQuery query{database_, tx_, state_repository_}; auto result = query.exec({address, location}); if (result) { @@ -106,11 +107,12 @@ void DomainState::update_account( const evmc::address& address, std::optional original, std::optional current) { + if (!original) { AccountsDomainGetLatestQuery query_prev{database_, tx_, state_repository_}; auto result_prev = query_prev.exec(address); if (result_prev) { - original = result_prev->value; + original = std::move(result_prev->value); } } @@ -118,10 +120,10 @@ void DomainState::update_account( if (current) { AccountsDomainPutQuery query{database_, tx_}; - query.exec(address, *current, txn_id_, original, prev_step); + query.exec(address, *current, txn_id_, original); } else { AccountsDomainDeleteQuery query{tx_, database_.domain(kDomainNameAccounts)}; - query.exec(address, txn_id_, original, prev_step); + query.exec(address, txn_id_, original); } } @@ -130,18 +132,17 @@ void DomainState::update_account_code( uint64_t /*incarnation*/, const evmc::bytes32& /*code_hash*/, ByteView code) { + CodeDomainGetLatestQuery query_prev{database_, tx_, state_repository_}; auto result_prev = query_prev.exec(address); - Step prev_step{0}; std::optional original_code = std::nullopt; if (result_prev) { - prev_step = result_prev->step; - original_code = result_prev->value; + original_code = std::move(result_prev->value); } CodeDomainPutQuery query{database_, tx_}; - query.exec(address, code, txn_id_, original_code, prev_step); + query.exec(address, code, txn_id_, original_code); } void DomainState::update_storage( @@ -150,13 +151,14 @@ void DomainState::update_storage( const evmc::bytes32& location, const evmc::bytes32& initial, const evmc::bytes32& current) { + evmc::bytes32 original_value{}; if (initial == evmc::bytes32{}) { StorageDomainGetLatestQuery query_prev{database_, tx_, state_repository_}; auto result_prev = query_prev.exec({address, location}); if (result_prev) { - original_value = result_prev->value; + original_value = std::move(result_prev->value); } } else { original_value = initial; @@ -165,7 +167,7 @@ void DomainState::update_storage( Step prev_step{0}; StorageDomainPutQuery query{database_, tx_}; - query.exec({address, location}, current, txn_id_, original_value, prev_step); + query.exec({address, location}, current, txn_id_, original_value); } } // namespace silkworm::execution From ce38609f905811c76987e9877098ef686b6ad0b8 Mon Sep 17 00:00:00 2001 From: GitHub Date: Wed, 29 Jan 2025 13:52:10 +0000 Subject: [PATCH 27/33] make fmt --- silkworm/db/datastore/kvdb/domain_put_query.hpp | 2 -- silkworm/db/state/accounts_domain.hpp | 2 +- silkworm/execution/domain_state.cpp | 4 ---- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/silkworm/db/datastore/kvdb/domain_put_query.hpp b/silkworm/db/datastore/kvdb/domain_put_query.hpp index 8ea36090e4..bd488d8843 100644 --- a/silkworm/db/datastore/kvdb/domain_put_query.hpp +++ b/silkworm/db/datastore/kvdb/domain_put_query.hpp @@ -35,11 +35,9 @@ struct DomainPutQuery { const Value& value, Timestamp timestamp, const std::optional& prev_value) { - DomainPutLatestQuery value_query{tx, entity}; value_query.exec(key, value, Step::from_txn_id(timestamp)); - if (entity.history) { if (prev_value) { HistoryPutQuery history_query{tx, *entity.history}; diff --git a/silkworm/db/state/accounts_domain.hpp b/silkworm/db/state/accounts_domain.hpp index df2d362a37..0771c9b5b3 100644 --- a/silkworm/db/state/accounts_domain.hpp +++ b/silkworm/db/state/accounts_domain.hpp @@ -39,7 +39,7 @@ struct AccountsDomainGetLatestQuery : public datastore::DomainGetLatestQuery< db::state::kDomainNameAccounts, database.domain(db::state::kDomainNameAccounts), tx, - repository ) {} + repository) {} }; struct AccountsDomainPutQuery : public datastore::kvdb::DomainPutQuery { diff --git a/silkworm/execution/domain_state.cpp b/silkworm/execution/domain_state.cpp index 21acc2add6..f1e6ef44be 100644 --- a/silkworm/execution/domain_state.cpp +++ b/silkworm/execution/domain_state.cpp @@ -58,7 +58,6 @@ evmc::bytes32 DomainState::read_storage( const evmc::address& address, uint64_t /*incarnation*/, const evmc::bytes32& location) const noexcept { - StorageDomainGetLatestQuery query{database_, tx_, state_repository_}; auto result = query.exec({address, location}); if (result) { @@ -107,7 +106,6 @@ void DomainState::update_account( const evmc::address& address, std::optional original, std::optional current) { - if (!original) { AccountsDomainGetLatestQuery query_prev{database_, tx_, state_repository_}; auto result_prev = query_prev.exec(address); @@ -132,7 +130,6 @@ void DomainState::update_account_code( uint64_t /*incarnation*/, const evmc::bytes32& /*code_hash*/, ByteView code) { - CodeDomainGetLatestQuery query_prev{database_, tx_, state_repository_}; auto result_prev = query_prev.exec(address); @@ -151,7 +148,6 @@ void DomainState::update_storage( const evmc::bytes32& location, const evmc::bytes32& initial, const evmc::bytes32& current) { - evmc::bytes32 original_value{}; if (initial == evmc::bytes32{}) { From eaadfd4c4895c86c28fabaf4a190c4002e1cd386 Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Wed, 29 Jan 2025 16:06:05 +0100 Subject: [PATCH 28/33] Remove redundant var --- silkworm/execution/domain_state.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/silkworm/execution/domain_state.cpp b/silkworm/execution/domain_state.cpp index 21acc2add6..44512318e7 100644 --- a/silkworm/execution/domain_state.cpp +++ b/silkworm/execution/domain_state.cpp @@ -116,8 +116,6 @@ void DomainState::update_account( } } - auto prev_step = Step{0}; // TODO: JG remove - if (current) { AccountsDomainPutQuery query{database_, tx_}; query.exec(address, *current, txn_id_, original); From 57948595661759b628cd4fb0ab3df3fa056124f4 Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Thu, 30 Jan 2025 13:05:55 +0100 Subject: [PATCH 29/33] Changes from code review --- silkworm/capi/silkworm.cpp | 50 +---- silkworm/capi/silkworm.h | 10 +- silkworm/capi/silkworm_test.cpp | 6 +- .../db/datastore/kvdb/domain_delete_query.hpp | 5 +- .../kvdb/domain_get_latest_query.hpp | 3 +- .../db/datastore/kvdb/domain_put_query.hpp | 5 +- silkworm/db/datastore/kvdb/mdbx.hpp | 14 +- silkworm/db/state/accounts_domain.hpp | 9 +- silkworm/db/state/code_domain.hpp | 9 +- silkworm/db/state/commitment_domain.hpp | 36 +++- silkworm/db/state/receipts_domain.hpp | 34 ++- silkworm/db/state/storage_domain.hpp | 9 +- silkworm/execution/CMakeLists.txt | 2 +- silkworm/execution/database_tcontext.cpp | 201 ------------------ silkworm/execution/database_tcontext.hpp | 96 --------- silkworm/execution/domain_state.cpp | 10 +- silkworm/execution/domain_state_test.cpp | 7 +- 17 files changed, 114 insertions(+), 392 deletions(-) delete mode 100644 silkworm/execution/database_tcontext.cpp delete mode 100644 silkworm/execution/database_tcontext.hpp diff --git a/silkworm/capi/silkworm.cpp b/silkworm/capi/silkworm.cpp index 9bd42b54c6..ab86806aa9 100644 --- a/silkworm/capi/silkworm.cpp +++ b/silkworm/capi/silkworm.cpp @@ -751,9 +751,8 @@ int silkworm_execute_blocks_perpetual(SilkwormHandle handle, MDBX_env* mdbx_env, } } -// todo: add available gas, add txn, add block header -SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* mdbx_tx, uint64_t block_num, struct SilkwormBytes32 head_hash_bytes, uint64_t txn_index, uint64_t txn_id, uint64_t* gas_used, uint64_t* blob_gas_used) SILKWORM_NOEXCEPT { - log::Info{"silkworm_execute_tx", {"block_num", std::to_string(block_num), "txn_index", std::to_string(txn_index)}}; +SILKWORM_EXPORT int silkworm_execute_txn(SilkwormHandle handle, MDBX_txn* mdbx_tx, uint64_t block_num, struct SilkwormBytes32 block_hash, uint64_t txn_index, uint64_t txn_num, uint64_t* gas_used, uint64_t* blob_gas_used) SILKWORM_NOEXCEPT { + log::Info{"silkworm_execute_txn", {"block_num", std::to_string(block_num), "txn_index", std::to_string(txn_index)}}; if (!handle) { return SILKWORM_INVALID_HANDLE; } @@ -770,13 +769,13 @@ SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* mdbx_tx *blob_gas_used = 0; } - silkworm::Hash head_hash{}; - memcpy(head_hash.bytes, head_hash_bytes.bytes, sizeof(head_hash.bytes)); + silkworm::Hash block_header_hash{}; + memcpy(block_header_hash.bytes, block_hash.bytes, sizeof(block_hash.bytes)); BlockNum block_number{block_num}; - TxnId txn_id_{txn_id}; + TxnId txn_id_{txn_num}; auto unmanaged_tx = datastore::kvdb::RWTxnUnmanaged{mdbx_tx}; - auto unmanaged_env = unmanaged_tx.unmanaged_env(); + auto unmanaged_env = silkworm::datastore::kvdb::EnvUnmanaged{::mdbx_txn_env(mdbx_tx)}; auto chain_db = db::DataStore::make_chaindata_database(std::move(unmanaged_env)); auto db_ref = chain_db.ref(); auto state = silkworm::execution::DomainState{txn_id_, unmanaged_tx, db_ref, *handle->blocks_repository, *handle->state_repository}; @@ -784,47 +783,18 @@ SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* mdbx_tx handle->chain_config = db::read_chain_config(unmanaged_tx); } - //! Manual tests: remove when done - // auto b_hash = to_bytes32(*from_hex("0xe8b28e4882bcbd6293ef56433c69b34e9e3e5bf512a05ddbed6bb94aa65948f4")); - // auto b_number = BlockNum{2910651}; - // auto b_hash = to_bytes32(*from_hex("0x6f81fc8bb897eb2075ffa53a2b28c3216022fd1841d361af7908198f8ff2faa3")); - // auto b_number = BlockNum{1}; - // auto header = state.read_header(b_number, b_hash); - // if (header) { - // log::Info{"JG header", {"number", std::to_string(header->number), "gas_used", std::to_string(header->gas_used)}}; - // } else { - // log::Warning{"header not found"}; - // return SILKWORM_INVALID_BLOCK; - // } - // silkworm::Block block{}; - // auto block_read_ok = state.read_body(b_number, b_hash, block); - // if (block_read_ok) { - // log::Info{"JG body", {"number", std::to_string(block.header.number), "transactions", std::to_string(block.transactions.size())}}; - // } else { - // log::Warning{"body not found"}; - // return SILKWORM_INVALID_BLOCK; - // } - // block.header = header.value(); - // auto a = hex_to_address("0x71562b71999873db5b286df957af199ec94617f7"); - // auto acc = state.read_account(a); - // if (acc) { - // log::Info{"account2", {"balance", std::to_string(acc->balance.num_bits)}}; - // } else { - // log::Info{"account2 not found"}; - // } - // TODO: cache block, also consider preloading silkworm::Block block{}; - auto block_read_ok = state.read_body(block_number, head_hash, block); + auto block_read_ok = state.read_body(block_number, block_header_hash, block); if (!block_read_ok) { SILK_ERROR << "Block not found" - << " block_number: " << block_number << " head_hash: " << head_hash; + << " block_number: " << block_number << " block_hash: " << block_header_hash; return SILKWORM_INVALID_BLOCK; } - auto header = state.read_header(block_number, head_hash); + auto header = state.read_header(block_number, block_header_hash); if (!header) { SILK_ERROR << "Header not found" - << " block_number: " << block_number << " head_hash: " << head_hash; + << " block_number: " << block_number << " block_hash: " << block_header_hash; return SILKWORM_INVALID_BLOCK; } block.header = header.value(); diff --git a/silkworm/capi/silkworm.h b/silkworm/capi/silkworm.h index c7320cf43c..47c3d8df0b 100644 --- a/silkworm/capi/silkworm.h +++ b/silkworm/capi/silkworm.h @@ -362,15 +362,15 @@ SILKWORM_EXPORT int silkworm_execute_blocks_perpetual(SilkwormHandle handle, MDB * \brief Execute a transaction in a block. * \param[in] handle A valid Silkworm instance handle, got with silkworm_init. * \param[in] mdbx_tx A valid external read-write MDBX transaction. - * \param[in] block_num The block number. - * \param[in] head_hash_bytes The hash of the head block. - * \param[in] txn_num The transaction number in the block. - * \param[in] txn_id The transaction ID. + * \param[in] block_num The number of the block containing the transaction. + * \param[in] block_hash The hash of the block. + * \param[in] txn_index The transaction number in the block. + * \param[in] txn_num The canonical transaction ID. * \param[out] gas_used The gas used by the transaction. * \param[out] blob_gas_used The blob gas used by the transaction. * \return SILKWORM_OK (=0) on success, a non-zero error value on failure. */ -SILKWORM_EXPORT int silkworm_execute_tx(SilkwormHandle handle, MDBX_txn* mdbx_tx, uint64_t block_num, struct SilkwormBytes32 head_hash_bytes, uint64_t txn_index, uint64_t txn_id, uint64_t* gas_used, uint64_t* blob_gas_used) SILKWORM_NOEXCEPT; +SILKWORM_EXPORT int silkworm_execute_txn(SilkwormHandle handle, MDBX_txn* mdbx_tx, uint64_t block_num, struct SilkwormBytes32 block_hash, uint64_t txn_index, uint64_t txn_num, uint64_t* gas_used, uint64_t* blob_gas_used) SILKWORM_NOEXCEPT; /** * \brief Finalize the Silkworm C API library. diff --git a/silkworm/capi/silkworm_test.cpp b/silkworm/capi/silkworm_test.cpp index e649864467..50e4589ab0 100644 --- a/silkworm/capi/silkworm_test.cpp +++ b/silkworm/capi/silkworm_test.cpp @@ -177,11 +177,11 @@ struct SilkwormLibrary { return result; } - int exectute_single_tx(MDBX_txn* txn, uint64_t block_num, silkworm::Hash head_hash, uint64_t txn_index, uint64_t txn_id) const { + int execute_txn(MDBX_txn* txn, uint64_t block_num, silkworm::Hash head_hash, uint64_t txn_index, uint64_t txn_id) const { SilkwormBytes32 head_hash_bytes{}; std::memcpy(head_hash_bytes.bytes, head_hash.bytes, 32); - return silkworm_execute_tx(handle_, txn, block_num, head_hash_bytes, txn_index, txn_id, nullptr, nullptr); + return silkworm_execute_txn(handle_, txn, block_num, head_hash_bytes, txn_index, txn_id, nullptr, nullptr); } int add_snapshot(SilkwormChainSnapshot* snapshot) const { @@ -1194,7 +1194,7 @@ TEST_CASE_METHOD(CApiTest, "CAPI silkworm_tx: single", "[silkworm][capi]") { insert_block(env, block); RWTxnManaged external_txn{env}; - auto result = silkworm_lib.exectute_single_tx(*external_txn, 10, block.header.hash(), 0, 9); + auto result = silkworm_lib.execute_txn(*external_txn, 10, block.header.hash(), 0, 9); CHECK(result == SILKWORM_INVALID_BLOCK); CHECK_NOTHROW(external_txn.abort()); } diff --git a/silkworm/db/datastore/kvdb/domain_delete_query.hpp b/silkworm/db/datastore/kvdb/domain_delete_query.hpp index 804e8cee22..1d9e36b1b3 100644 --- a/silkworm/db/datastore/kvdb/domain_delete_query.hpp +++ b/silkworm/db/datastore/kvdb/domain_delete_query.hpp @@ -31,7 +31,8 @@ struct DomainDeleteQuery { void exec( const Key& key, Timestamp timestamp, - const std::optional& prev_value) { + const std::optional& prev_value, + Step current_step) { if (prev_value) { TValueEncoder prev_value_encoder; prev_value_encoder.value = std::move(*prev_value); @@ -42,7 +43,7 @@ struct DomainDeleteQuery { ByteView prev_value_data = prev_value_slice_decoder.value; DomainPutQuery> query{tx, entity}; - query.exec(key, ByteView{}, timestamp, prev_value_data); + query.exec(key, ByteView{}, timestamp, prev_value_data, current_step); } } }; diff --git a/silkworm/db/datastore/kvdb/domain_get_latest_query.hpp b/silkworm/db/datastore/kvdb/domain_get_latest_query.hpp index 9012978a2f..275d0d204e 100644 --- a/silkworm/db/datastore/kvdb/domain_get_latest_query.hpp +++ b/silkworm/db/datastore/kvdb/domain_get_latest_query.hpp @@ -45,7 +45,8 @@ struct DomainGetLatestQuery { key_encoder.value.timestamp.value = Step{std::numeric_limits::max()}; Slice key_slice = key_encoder.encode(); - auto result = entity.has_large_values ? tx.ro_cursor(entity.values_table)->lower_bound(key_slice, false) : tx.ro_cursor(entity.values_table)->find(key_slice, false); + auto db_cursor = tx.ro_cursor(entity.values_table); + auto result = entity.has_large_values ? db_cursor->lower_bound(key_slice, false) : db_cursor->find(key_slice, false); if (!result) { return std::nullopt; diff --git a/silkworm/db/datastore/kvdb/domain_put_query.hpp b/silkworm/db/datastore/kvdb/domain_put_query.hpp index bd488d8843..8fe676f934 100644 --- a/silkworm/db/datastore/kvdb/domain_put_query.hpp +++ b/silkworm/db/datastore/kvdb/domain_put_query.hpp @@ -34,9 +34,10 @@ struct DomainPutQuery { const Key& key, const Value& value, Timestamp timestamp, - const std::optional& prev_value) { + const std::optional& prev_value, + Step current_step) { DomainPutLatestQuery value_query{tx, entity}; - value_query.exec(key, value, Step::from_txn_id(timestamp)); + value_query.exec(key, value, current_step); if (entity.history) { if (prev_value) { diff --git a/silkworm/db/datastore/kvdb/mdbx.hpp b/silkworm/db/datastore/kvdb/mdbx.hpp index 912b782ff4..697feffb50 100644 --- a/silkworm/db/datastore/kvdb/mdbx.hpp +++ b/silkworm/db/datastore/kvdb/mdbx.hpp @@ -324,12 +324,6 @@ class RWTxnManaged : public RWTxn { mdbx::txn_managed managed_txn_; }; -//! \brief EnvUnmanaged wraps an *unmanaged* MDBX environment, which means the underlying environment -//! lifecycle is not touched by this class. -struct EnvUnmanaged : public ::mdbx::env { - explicit EnvUnmanaged(MDBX_env* ptr) : ::mdbx::env{ptr} {} -}; - //! \brief RWTxnUnmanaged wraps an *unmanaged* read-write transaction, which means the underlying transaction //! lifecycle is not touched by this class: the transaction is neither committed nor aborted. class RWTxnUnmanaged : public RWTxn, protected ::mdbx::txn { @@ -340,8 +334,6 @@ class RWTxnUnmanaged : public RWTxn, protected ::mdbx::txn { void abort() override { throw std::runtime_error{"RWTxnUnmanaged must not be aborted"}; } void commit_and_renew() override { throw std::runtime_error{"RWTxnUnmanaged must not be committed"}; } void commit_and_stop() override { throw std::runtime_error{"RWTxnUnmanaged must not be committed"}; } - - EnvUnmanaged unmanaged_env() const { return EnvUnmanaged{env()}; } }; //! \brief This class create ROTxn(s) on demand, it is used to enforce in some method signatures the type of db access @@ -391,6 +383,12 @@ struct EnvConfig { uint32_t max_readers{100}; // Default max number of readers }; +//! \brief EnvUnmanaged wraps an *unmanaged* MDBX environment, which means the underlying environment +//! lifecycle is not touched by this class. +struct EnvUnmanaged : public ::mdbx::env { + explicit EnvUnmanaged(MDBX_env* ptr) : ::mdbx::env{ptr} {} +}; + //! \brief Opens an mdbx environment using the provided environment config //! \param [in] config : A structure containing essential environment settings //! \return A handler to mdbx::env_managed class diff --git a/silkworm/db/state/accounts_domain.hpp b/silkworm/db/state/accounts_domain.hpp index 0771c9b5b3..11b0236d78 100644 --- a/silkworm/db/state/accounts_domain.hpp +++ b/silkworm/db/state/accounts_domain.hpp @@ -51,7 +51,14 @@ struct AccountsDomainPutQuery : public datastore::kvdb::DomainPutQuery; +struct AccountsDomainDeleteQuery : datastore::kvdb::DomainDeleteQuery{ + AccountsDomainDeleteQuery( + const datastore::kvdb::DatabaseRef& database, + datastore::kvdb::RWTxn& rw_tx) + : datastore::kvdb::DomainDeleteQuery{ + rw_tx, + database.domain(db::state::kDomainNameAccounts)} {} +}; using AccountsDomainKVSegmentReader = snapshots::segment::KVSegmentReader; diff --git a/silkworm/db/state/code_domain.hpp b/silkworm/db/state/code_domain.hpp index aae6041c0a..4103c03c95 100644 --- a/silkworm/db/state/code_domain.hpp +++ b/silkworm/db/state/code_domain.hpp @@ -51,7 +51,14 @@ struct CodeDomainPutQuery : public datastore::kvdb::DomainPutQuery>; +struct CodeDomainDeleteQuery : datastore::kvdb::DomainDeleteQuery> { + CodeDomainDeleteQuery( + const datastore::kvdb::DatabaseRef& database, + datastore::kvdb::RWTxn& rw_tx) + : datastore::kvdb::DomainDeleteQuery>{ + rw_tx, + database.domain(db::state::kDomainNameCode)} {} +}; using CodeDomainKVSegmentReader = snapshots::segment::KVSegmentReader>; diff --git a/silkworm/db/state/commitment_domain.hpp b/silkworm/db/state/commitment_domain.hpp index d675c794aa..5d777d20e1 100644 --- a/silkworm/db/state/commitment_domain.hpp +++ b/silkworm/db/state/commitment_domain.hpp @@ -25,25 +25,39 @@ namespace silkworm::db::state { -using CommitmentDomainGetLatestQueryBase = datastore::DomainGetLatestQuery< - datastore::kvdb::RawEncoder, snapshots::RawEncoder, - datastore::kvdb::RawDecoder, snapshots::RawDecoder>; - -struct CommitmentDomainGetLatestQuery : public CommitmentDomainGetLatestQueryBase { +struct CommitmentDomainGetLatestQuery : public datastore::DomainGetLatestQuery< + datastore::kvdb::RawEncoder, snapshots::RawEncoder, + datastore::kvdb::RawDecoder, snapshots::RawDecoder> { CommitmentDomainGetLatestQuery( const datastore::kvdb::DatabaseRef& database, datastore::kvdb::ROTxn& tx, const snapshots::SnapshotRepositoryROAccess& repository) - : CommitmentDomainGetLatestQueryBase{ + : datastore::DomainGetLatestQuery< + datastore::kvdb::RawEncoder, snapshots::RawEncoder, + datastore::kvdb::RawDecoder, snapshots::RawDecoder>( db::state::kDomainNameCommitment, - database, + database.domain(db::state::kDomainNameCommitment), tx, - repository, - } {} + repository) {} +}; + +struct CommitmentDomainPutQuery : public datastore::kvdb::DomainPutQuery, datastore::kvdb::RawEncoder> { + CommitmentDomainPutQuery( + const datastore::kvdb::DatabaseRef& database, + datastore::kvdb::RWTxn& rw_tx) + : datastore::kvdb::DomainPutQuery, datastore::kvdb::RawEncoder>{ + rw_tx, + database.domain(db::state::kDomainNameCommitment)} {} }; -using CommitmentDomainPutQuery = datastore::kvdb::DomainPutQuery, datastore::kvdb::RawEncoder>; -using CommitmentDomainDeleteQuery = datastore::kvdb::DomainDeleteQuery, datastore::kvdb::RawEncoder>; +struct CommitmentDomainDeleteQuery : datastore::kvdb::DomainDeleteQuery, datastore::kvdb::RawEncoder> { + CommitmentDomainDeleteQuery( + const datastore::kvdb::DatabaseRef& database, + datastore::kvdb::RWTxn& rw_tx) + : datastore::kvdb::DomainDeleteQuery, datastore::kvdb::RawEncoder>{ + rw_tx, + database.domain(db::state::kDomainNameCommitment)} {} +}; using CommitmentDomainKVSegmentReader = snapshots::segment::KVSegmentReader, snapshots::RawDecoder>; diff --git a/silkworm/db/state/receipts_domain.hpp b/silkworm/db/state/receipts_domain.hpp index 6b1f50e08e..f3b0c586bd 100644 --- a/silkworm/db/state/receipts_domain.hpp +++ b/silkworm/db/state/receipts_domain.hpp @@ -27,25 +27,39 @@ namespace silkworm::db::state { -using ReceiptsDomainGetLatestQueryBase = datastore::DomainGetLatestQuery< +struct ReceiptsDomainGetLatestQuery: public datastore::DomainGetLatestQuery< datastore::kvdb::RawEncoder, snapshots::RawEncoder, - datastore::kvdb::RawDecoder, snapshots::RawDecoder>; - -struct ReceiptsDomainGetLatestQuery : public ReceiptsDomainGetLatestQueryBase { + datastore::kvdb::RawDecoder, snapshots::RawDecoder> { ReceiptsDomainGetLatestQuery( const datastore::kvdb::DatabaseRef& database, datastore::kvdb::ROTxn& tx, const snapshots::SnapshotRepositoryROAccess& repository) - : ReceiptsDomainGetLatestQueryBase{ + : datastore::DomainGetLatestQuery< + datastore::kvdb::RawEncoder, snapshots::RawEncoder, + datastore::kvdb::RawDecoder, snapshots::RawDecoder>( db::state::kDomainNameReceipts, - database, + database.domain(db::state::kDomainNameReceipts), tx, - repository, - } {} + repository) {} +}; + +struct ReceiptsDomainPutQuery: public datastore::kvdb::DomainPutQuery, datastore::kvdb::RawEncoder> { + ReceiptsDomainPutQuery( + const datastore::kvdb::DatabaseRef& database, + datastore::kvdb::RWTxn& rw_tx) + : datastore::kvdb::DomainPutQuery, datastore::kvdb::RawEncoder>{ + rw_tx, + database.domain(db::state::kDomainNameReceipts)} {} }; -using ReceiptsDomainPutQuery = datastore::kvdb::DomainPutQuery, datastore::kvdb::RawEncoder>; -using ReceiptsDomainDeleteQuery = datastore::kvdb::DomainDeleteQuery, datastore::kvdb::RawEncoder>; +struct ReceiptsDomainDeleteQuery: datastore::kvdb::DomainDeleteQuery, datastore::kvdb::RawEncoder> { + ReceiptsDomainDeleteQuery( + const datastore::kvdb::DatabaseRef& database, + datastore::kvdb::RWTxn& rw_tx) + : datastore::kvdb::DomainDeleteQuery, datastore::kvdb::RawEncoder>{ + rw_tx, + database.domain(db::state::kDomainNameReceipts)} {} +}; enum class ReceiptsDomainKey : uint8_t { kCumulativeGasUsedInBlockKey = 0, diff --git a/silkworm/db/state/storage_domain.hpp b/silkworm/db/state/storage_domain.hpp index 532ea03a9b..86c9f2232a 100644 --- a/silkworm/db/state/storage_domain.hpp +++ b/silkworm/db/state/storage_domain.hpp @@ -50,7 +50,14 @@ struct StorageDomainPutQuery : public datastore::kvdb::DomainPutQuery; +struct StorageDomainDeleteQuery : datastore::kvdb::DomainDeleteQuery { + StorageDomainDeleteQuery( + const datastore::kvdb::DatabaseRef& database, + datastore::kvdb::RWTxn& rw_tx) + : datastore::kvdb::DomainDeleteQuery{ + rw_tx, + database.domain(db::state::kDomainNameStorage)} {} +}; using StorageDomainKVSegmentReader = snapshots::segment::KVSegmentReader; diff --git a/silkworm/execution/CMakeLists.txt b/silkworm/execution/CMakeLists.txt index 7ed4d7faaf..b857616a2b 100644 --- a/silkworm/execution/CMakeLists.txt +++ b/silkworm/execution/CMakeLists.txt @@ -44,4 +44,4 @@ silkworm_library( PRIVATE ${LIBS_PRIVATE} ) -target_link_libraries(silkworm_execution_test PRIVATE GTest::gmock silkworm_infra_test_util) +target_link_libraries(silkworm_execution_test PRIVATE GTest::gmock silkworm_infra_test_util silkworm_db_test_util) diff --git a/silkworm/execution/database_tcontext.cpp b/silkworm/execution/database_tcontext.cpp deleted file mode 100644 index 1020d6ad59..0000000000 --- a/silkworm/execution/database_tcontext.cpp +++ /dev/null @@ -1,201 +0,0 @@ -/* - Copyright 2024 The Silkworm Authors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#include "database_tcontext.hpp" - -#include - -#include -#include -#include -#include -#include -#include -#include - -namespace silkworm::execution { - -using namespace silkworm::datastore::kvdb; - -std::filesystem::path get_tests_dir() { - auto working_dir = std::filesystem::current_path(); - - while (working_dir != "/") { - if (std::filesystem::exists(working_dir / "third_party" / "execution-apis")) { - return working_dir / "third_party" / "execution-apis" / "tests"; - } - - if (std::filesystem::exists(working_dir / "silkworm" / "third_party" / "execution-apis")) { - return working_dir / "silkworm" / "third_party" / "execution-apis" / "tests"; - } - - if (std::filesystem::exists(working_dir / "project" / "third_party" / "execution-apis")) { - return working_dir / "project" / "third_party" / "execution-apis" / "tests"; - } - - working_dir = working_dir.parent_path(); - } - - throw std::logic_error("Failed to find tests directory"); -} - -InMemoryState populate_genesis(RWTxn& txn, const std::filesystem::path& tests_dir) { - auto genesis_json_path = tests_dir / "genesis.json"; - std::ifstream genesis_json_input_file(genesis_json_path); - nlohmann::json genesis_json; - genesis_json_input_file >> genesis_json; - - InMemoryState state = read_genesis_allocation(genesis_json.at("alloc")); - silkworm::db::write_genesis_allocation_to_db(txn, state); - - BlockHeader header{read_genesis_header(genesis_json, state.state_root_hash())}; - BlockBody block_body{ - .withdrawals = std::vector{0}, - }; - - // FIX 2: set empty receipts root, should be done in the main code, requires https://github.com/erigontech/silkworm/issues/1348 - header.withdrawals_root = kEmptyRoot; - - auto block_hash{header.hash()}; - auto block_hash_key{silkworm::db::block_key(header.number, block_hash.bytes)}; - silkworm::db::write_header(txn, header, /*with_header_numbers=*/true); // Write table::kHeaders and table::kHeaderNumbers - silkworm::db::write_canonical_header_hash(txn, block_hash.bytes, header.number); // Insert header hash as canonical - silkworm::db::write_total_difficulty(txn, block_hash_key, header.difficulty); // Write initial difficulty - - silkworm::db::write_body(txn, block_body, block_hash.bytes, header.number); // Write block body (empty) - silkworm::db::write_head_header_hash(txn, block_hash.bytes); // Update head header in config - - const uint8_t genesis_null_receipts[] = {0xf6}; // <- cbor encoded - open_cursor(txn, silkworm::db::table::kBlockReceipts) - .upsert(datastore::kvdb::to_slice(block_hash_key).safe_middle(0, 8), datastore::kvdb::to_slice(Bytes(genesis_null_receipts, 1))); - - // Write Chain Settings - auto config_data{genesis_json["config"].dump()}; - open_cursor(txn, silkworm::db::table::kConfig) - .upsert(to_slice(ByteView{block_hash.bytes}), mdbx::slice{config_data.data()}); - - return state; -} - -void populate_blocks(RWTxn& txn, const std::filesystem::path& tests_dir, InMemoryState& state_buffer) { - auto rlp_path = tests_dir / "chain.rlp"; - std::ifstream file(rlp_path, std::ios::binary); - if (!file) { - throw std::logic_error("Failed to open the file: " + rlp_path.string()); - } - std::vector rlps; - std::vector line; - - Bytes rlp_buffer(std::istreambuf_iterator(file), {}); - file.close(); - ByteView rlp_view{rlp_buffer}; - - auto chain_config = silkworm::db::read_chain_config(txn); - - if (!chain_config.has_value()) { - throw std::logic_error("Failed to read chain config"); - } - auto rule_set = protocol::rule_set_factory(*chain_config); - - while (!rlp_view.empty()) { - silkworm::Block block; - - if (!silkworm::rlp::decode(rlp_view, block, silkworm::rlp::Leftover::kAllow)) { - throw std::logic_error("Failed to decode RLP file"); - } - - // store original hashes - auto block_hash = block.header.hash(); - auto block_hash_key = silkworm::db::block_key(block.header.number, block_hash.bytes); - - // FIX 3: populate senders table - silkworm::db::write_senders(txn, block_hash, block.header.number, block); - - // FIX 4a: populate tx lookup table and create receipts - silkworm::db::write_tx_lookup(txn, block); - - // FIX 4b: populate receipts and logs table - std::vector receipts; - ExecutionProcessor processor{block, *rule_set, state_buffer, *chain_config, true}; - silkworm::db::Buffer db_buffer{txn, std::make_unique(txn)}; - for (auto& block_txn : block.transactions) { - silkworm::Receipt receipt{}; - processor.execute_transaction(block_txn, receipt); - receipts.emplace_back(receipt); - } - processor.evm().state().write_to_db(block.header.number); - db_buffer.insert_receipts(block.header.number, receipts); - db_buffer.write_history_to_db(); - - // FIX 5: insert system transactions - intx::uint256 max_priority_fee_per_gas = block.transactions.empty() ? block.header.base_fee_per_gas.value_or(0) : block.transactions[0].max_priority_fee_per_gas; - intx::uint256 max_fee_per_gas = block.transactions.empty() ? block.header.base_fee_per_gas.value_or(0) : block.transactions[0].max_fee_per_gas; - silkworm::Transaction system_transaction; - system_transaction.max_priority_fee_per_gas = max_priority_fee_per_gas; - system_transaction.max_fee_per_gas = max_fee_per_gas; - block.transactions.emplace(block.transactions.begin(), system_transaction); - block.transactions.emplace_back(system_transaction); - - silkworm::db::write_header(txn, block.header, /*with_header_numbers=*/true); // Write table::kHeaders and table::kHeaderNumbers - silkworm::db::write_canonical_header_hash(txn, block_hash.bytes, block.header.number); // Insert header hash as canonical - - // TODO: find how to decode total difficulty - // write_total_difficulty(txn, block_hash_key, block.header.difficulty); // Write initial difficulty - silkworm::db::write_total_difficulty(txn, block_hash_key, 1); // Write initial difficulty - - silkworm::db::write_raw_body(txn, block, block_hash, block.header.number); - silkworm::db::write_head_header_hash(txn, block_hash.bytes); // Update head header in config - silkworm::db::write_last_head_block(txn, block_hash); // Update head block in config - silkworm::db::write_last_safe_block(txn, block_hash); // Update last safe block in config - silkworm::db::write_last_finalized_block(txn, block_hash); // Update last finalized block in config - } -} - -namespace { - mdbx::env_managed initialize_test_database(const std::filesystem::path& chaindata_dir_path) { - const auto tests_dir = get_tests_dir(); - - EnvConfig env_config{ - .path = chaindata_dir_path.string(), - .create = true, - .exclusive = true, - .in_memory = true, - .shared = false, - }; - auto env = open_env(env_config); - - RWTxnManaged txn{env}; - silkworm::db::table::check_or_create_chaindata_tables(txn); - auto state_buffer = populate_genesis(txn, tests_dir); - populate_blocks(txn, tests_dir, state_buffer); - txn.commit_and_stop(); - - return env; - } - -} // namespace - -TestDatabaseContext::TestDatabaseContext(const TemporaryDirectory& tmp_dir) - : chaindata_dir_path_{DataDirectory{tmp_dir.path()}.chaindata().path()}, - env_{std::make_unique(initialize_test_database(chaindata_dir_path_))} {} - -silkworm::ChainConfig TestDatabaseContext::get_chain_config() const { - ROTxnManaged txn = chaindata().start_ro_tx(); - auto chain_config = silkworm::db::read_chain_config(txn); - return chain_config ? *chain_config : silkworm::ChainConfig{}; -} - -} // namespace silkworm::execution diff --git a/silkworm/execution/database_tcontext.hpp b/silkworm/execution/database_tcontext.hpp deleted file mode 100644 index 5383692829..0000000000 --- a/silkworm/execution/database_tcontext.hpp +++ /dev/null @@ -1,96 +0,0 @@ -/* - Copyright 2024 The Silkworm Authors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#pragma once - -#include - -#include -#include -#include -#include -#include -#include -#include - -// TODO: this is a duplicate of test_database_context.hpp -// check linking issues and remove this file - -// TODO: populate_genesis and populate_blocks should write to the domain tables - -namespace silkworm::execution { - -std::filesystem::path get_tests_dir(); - -InMemoryState populate_genesis(db::RWTxn& txn, const std::filesystem::path& tests_dir); - -void populate_blocks(db::RWTxn& txn, const std::filesystem::path& tests_dir, InMemoryState& state_buffer); - -class TestDatabaseContext { - public: - explicit TestDatabaseContext(const TemporaryDirectory& tmp_dir); - virtual ~TestDatabaseContext() = default; - - virtual datastore::kvdb::ROAccess chaindata() const { - return datastore::kvdb::ROAccess{*env_}; - } - virtual datastore::kvdb::RWAccess chaindata_rw() const { - return datastore::kvdb::RWAccess{*env_}; - } - - silkworm::ChainConfig get_chain_config() const; - const std::filesystem::path& chaindata_dir_path() const { return chaindata_dir_path_; } - - protected: - mdbx::env_managed move_env() { - mdbx::env_managed env{std::move(*env_)}; - env_.reset(); - return env; - } - - std::filesystem::path chaindata_dir_path_; - std::unique_ptr env_; -}; - -class TestDataStore : public TestDatabaseContext { - public: - explicit TestDataStore(const TemporaryDirectory& tmp_dir) - : TestDatabaseContext{tmp_dir}, - data_store_{ - move_env(), - DataDirectory{tmp_dir.path(), true}.snapshots().path(), - } {} - ~TestDataStore() override = default; - - db::DataStore& operator*() { return data_store_; } - db::DataStore* operator->() { return &data_store_; } - - datastore::kvdb::ROAccess chaindata() const override { - return data_store_.chaindata().access_ro(); - } - datastore::kvdb::RWAccess chaindata_rw() const override { - return data_store_.chaindata().access_rw(); - } - - db::DataModelFactory data_model_factory() { - return db::DataModelFactory{data_store_.ref()}; - } - - private: - db::DataStore data_store_; -}; - -} // namespace silkworm::execution diff --git a/silkworm/execution/domain_state.cpp b/silkworm/execution/domain_state.cpp index 46f16831ce..0fe76a8226 100644 --- a/silkworm/execution/domain_state.cpp +++ b/silkworm/execution/domain_state.cpp @@ -116,10 +116,10 @@ void DomainState::update_account( if (current) { AccountsDomainPutQuery query{database_, tx_}; - query.exec(address, *current, txn_id_, original); + query.exec(address, *current, txn_id_, original, Step::from_txn_id(txn_id_)); } else { - AccountsDomainDeleteQuery query{tx_, database_.domain(kDomainNameAccounts)}; - query.exec(address, txn_id_, original); + AccountsDomainDeleteQuery query{database_, tx_}; + query.exec(address, txn_id_, original, Step::from_txn_id(txn_id_)); } } @@ -137,7 +137,7 @@ void DomainState::update_account_code( } CodeDomainPutQuery query{database_, tx_}; - query.exec(address, code, txn_id_, original_code); + query.exec(address, code, txn_id_, original_code, Step::from_txn_id(txn_id_)); } void DomainState::update_storage( @@ -161,7 +161,7 @@ void DomainState::update_storage( Step prev_step{0}; StorageDomainPutQuery query{database_, tx_}; - query.exec({address, location}, current, txn_id_, original_value); + query.exec({address, location}, current, txn_id_, original_value, Step::from_txn_id(txn_id_)); } } // namespace silkworm::execution diff --git a/silkworm/execution/domain_state_test.cpp b/silkworm/execution/domain_state_test.cpp index 18422cf01e..bdfcabbb95 100644 --- a/silkworm/execution/domain_state_test.cpp +++ b/silkworm/execution/domain_state_test.cpp @@ -30,10 +30,9 @@ #include #include #include -#include +#include #include #include -#include #include namespace silkworm::execution { @@ -45,7 +44,7 @@ using testing::Unused; TEST_CASE("DomainState data access", "[execution][domain][state]") { TemporaryDirectory tmp_dir; - TestDataStore ds_context{tmp_dir}; + silkworm::db::test_util::TestDataStore ds_context{tmp_dir}; log::init(log::Settings{.log_verbosity = log::Level::kDebug}); auto rw_tx = ds_context.chaindata_rw().start_rw_tx(); @@ -303,7 +302,7 @@ TEST_CASE("DomainState data access", "[execution][domain][state]") { TEST_CASE("DomainState empty overriden methods do nothing", "[execution][domain][state]") { TemporaryDirectory tmp_dir; - TestDataStore ds_context{tmp_dir}; + silkworm::db::test_util::TestDataStore ds_context{tmp_dir}; auto rw_tx = ds_context.chaindata_rw().start_rw_tx(); From 4e79e967fe581398d0aa357ef049a3bdf4afffda Mon Sep 17 00:00:00 2001 From: GitHub Date: Thu, 30 Jan 2025 12:06:39 +0000 Subject: [PATCH 30/33] make fmt --- silkworm/db/state/accounts_domain.hpp | 6 +++--- silkworm/db/state/code_domain.hpp | 4 ++-- silkworm/db/state/receipts_domain.hpp | 14 +++++++------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/silkworm/db/state/accounts_domain.hpp b/silkworm/db/state/accounts_domain.hpp index 11b0236d78..ae06001fa9 100644 --- a/silkworm/db/state/accounts_domain.hpp +++ b/silkworm/db/state/accounts_domain.hpp @@ -51,13 +51,13 @@ struct AccountsDomainPutQuery : public datastore::kvdb::DomainPutQuery{ +struct AccountsDomainDeleteQuery : datastore::kvdb::DomainDeleteQuery { AccountsDomainDeleteQuery( const datastore::kvdb::DatabaseRef& database, datastore::kvdb::RWTxn& rw_tx) : datastore::kvdb::DomainDeleteQuery{ - rw_tx, - database.domain(db::state::kDomainNameAccounts)} {} + rw_tx, + database.domain(db::state::kDomainNameAccounts)} {} }; using AccountsDomainKVSegmentReader = snapshots::segment::KVSegmentReader; diff --git a/silkworm/db/state/code_domain.hpp b/silkworm/db/state/code_domain.hpp index 4103c03c95..2935b7a19f 100644 --- a/silkworm/db/state/code_domain.hpp +++ b/silkworm/db/state/code_domain.hpp @@ -56,8 +56,8 @@ struct CodeDomainDeleteQuery : datastore::kvdb::DomainDeleteQuery>{ - rw_tx, - database.domain(db::state::kDomainNameCode)} {} + rw_tx, + database.domain(db::state::kDomainNameCode)} {} }; using CodeDomainKVSegmentReader = snapshots::segment::KVSegmentReader>; diff --git a/silkworm/db/state/receipts_domain.hpp b/silkworm/db/state/receipts_domain.hpp index f3b0c586bd..ca78b04c37 100644 --- a/silkworm/db/state/receipts_domain.hpp +++ b/silkworm/db/state/receipts_domain.hpp @@ -27,9 +27,9 @@ namespace silkworm::db::state { -struct ReceiptsDomainGetLatestQuery: public datastore::DomainGetLatestQuery< - datastore::kvdb::RawEncoder, snapshots::RawEncoder, - datastore::kvdb::RawDecoder, snapshots::RawDecoder> { +struct ReceiptsDomainGetLatestQuery : public datastore::DomainGetLatestQuery< + datastore::kvdb::RawEncoder, snapshots::RawEncoder, + datastore::kvdb::RawDecoder, snapshots::RawDecoder> { ReceiptsDomainGetLatestQuery( const datastore::kvdb::DatabaseRef& database, datastore::kvdb::ROTxn& tx, @@ -43,7 +43,7 @@ struct ReceiptsDomainGetLatestQuery: public datastore::DomainGetLatestQuery< repository) {} }; -struct ReceiptsDomainPutQuery: public datastore::kvdb::DomainPutQuery, datastore::kvdb::RawEncoder> { +struct ReceiptsDomainPutQuery : public datastore::kvdb::DomainPutQuery, datastore::kvdb::RawEncoder> { ReceiptsDomainPutQuery( const datastore::kvdb::DatabaseRef& database, datastore::kvdb::RWTxn& rw_tx) @@ -52,13 +52,13 @@ struct ReceiptsDomainPutQuery: public datastore::kvdb::DomainPutQuery, datastore::kvdb::RawEncoder> { +struct ReceiptsDomainDeleteQuery : datastore::kvdb::DomainDeleteQuery, datastore::kvdb::RawEncoder> { ReceiptsDomainDeleteQuery( const datastore::kvdb::DatabaseRef& database, datastore::kvdb::RWTxn& rw_tx) : datastore::kvdb::DomainDeleteQuery, datastore::kvdb::RawEncoder>{ - rw_tx, - database.domain(db::state::kDomainNameReceipts)} {} + rw_tx, + database.domain(db::state::kDomainNameReceipts)} {} }; enum class ReceiptsDomainKey : uint8_t { From 40856f68e2986ef10b0b1f6ca7668afa235174d8 Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Thu, 30 Jan 2025 15:09:11 +0100 Subject: [PATCH 31/33] Fix build --- silkworm/db/state/receipts_domain.hpp | 70 ++++++++++----------------- 1 file changed, 25 insertions(+), 45 deletions(-) diff --git a/silkworm/db/state/receipts_domain.hpp b/silkworm/db/state/receipts_domain.hpp index 59824510b9..d508a40228 100644 --- a/silkworm/db/state/receipts_domain.hpp +++ b/silkworm/db/state/receipts_domain.hpp @@ -29,40 +29,6 @@ namespace silkworm::db::state { -struct ReceiptsDomainGetLatestQuery : public datastore::DomainGetLatestQuery< - datastore::kvdb::RawEncoder, snapshots::RawEncoder, - datastore::kvdb::RawDecoder, snapshots::RawDecoder> { - ReceiptsDomainGetLatestQuery( - const datastore::kvdb::DatabaseRef& database, - datastore::kvdb::ROTxn& tx, - const snapshots::SnapshotRepositoryROAccess& repository) - : datastore::DomainGetLatestQuery< - datastore::kvdb::RawEncoder, snapshots::RawEncoder, - datastore::kvdb::RawDecoder, snapshots::RawDecoder>( - db::state::kDomainNameReceipts, - database.domain(db::state::kDomainNameReceipts), - tx, - repository) {} -}; - -struct ReceiptsDomainPutQuery : public datastore::kvdb::DomainPutQuery, datastore::kvdb::RawEncoder> { - ReceiptsDomainPutQuery( - const datastore::kvdb::DatabaseRef& database, - datastore::kvdb::RWTxn& rw_tx) - : datastore::kvdb::DomainPutQuery, datastore::kvdb::RawEncoder>{ - rw_tx, - database.domain(db::state::kDomainNameReceipts)} {} -}; - -struct ReceiptsDomainDeleteQuery : datastore::kvdb::DomainDeleteQuery, datastore::kvdb::RawEncoder> { - ReceiptsDomainDeleteQuery( - const datastore::kvdb::DatabaseRef& database, - datastore::kvdb::RWTxn& rw_tx) - : datastore::kvdb::DomainDeleteQuery, datastore::kvdb::RawEncoder>{ - rw_tx, - database.domain(db::state::kDomainNameReceipts)} {} -}; - enum class ReceiptsDomainKey : uint8_t { kCumulativeGasUsedInBlockKey = 0, kCumulativeBlobGasUsedInBlockKey = 1, @@ -96,25 +62,39 @@ static_assert(snapshots::DecoderConcept); using ReceiptsDomainKVSegmentReader = snapshots::segment::KVSegmentReader; -using ReceiptsDomainGetLatestQueryBase = datastore::DomainGetLatestQuery< - datastore::kvdb::RawEncoder, snapshots::RawEncoder, - datastore::kvdb::RawDecoder, snapshots::RawDecoder>; - -struct ReceiptsDomainGetLatestQuery : public ReceiptsDomainGetLatestQueryBase { +struct ReceiptsDomainGetLatestQuery : public datastore::DomainGetLatestQuery< + datastore::kvdb::RawEncoder, snapshots::RawEncoder, + datastore::kvdb::RawDecoder, snapshots::RawDecoder> { ReceiptsDomainGetLatestQuery( const datastore::kvdb::DatabaseRef& database, datastore::kvdb::ROTxn& tx, const snapshots::SnapshotRepositoryROAccess& repository) - : ReceiptsDomainGetLatestQueryBase{ + : datastore::DomainGetLatestQuery< + datastore::kvdb::RawEncoder, snapshots::RawEncoder, + datastore::kvdb::RawDecoder, snapshots::RawDecoder>( db::state::kDomainNameReceipts, - database, + database.domain(db::state::kDomainNameReceipts), tx, - repository, - } {} + repository) {} +}; + +struct ReceiptsDomainPutQuery : public datastore::kvdb::DomainPutQuery, datastore::kvdb::RawEncoder> { + ReceiptsDomainPutQuery( + const datastore::kvdb::DatabaseRef& database, + datastore::kvdb::RWTxn& rw_tx) + : datastore::kvdb::DomainPutQuery, datastore::kvdb::RawEncoder>{ + rw_tx, + database.domain(db::state::kDomainNameReceipts)} {} }; -using ReceiptsDomainPutQuery = datastore::kvdb::DomainPutQuery, datastore::kvdb::RawEncoder>; -using ReceiptsDomainDeleteQuery = datastore::kvdb::DomainDeleteQuery, datastore::kvdb::RawEncoder>; +struct ReceiptsDomainDeleteQuery : datastore::kvdb::DomainDeleteQuery, datastore::kvdb::RawEncoder> { + ReceiptsDomainDeleteQuery( + const datastore::kvdb::DatabaseRef& database, + datastore::kvdb::RWTxn& rw_tx) + : datastore::kvdb::DomainDeleteQuery, datastore::kvdb::RawEncoder>{ + rw_tx, + database.domain(db::state::kDomainNameReceipts)} {} +}; using ReceiptsHistoryGetQuery = datastore::HistoryGetQuery< datastore::kvdb::RawEncoder, snapshots::RawEncoder, From 6bd9213e44912f4d94f692c7f9d729049e724042 Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Thu, 30 Jan 2025 16:16:28 +0100 Subject: [PATCH 32/33] Changes from code review --- silkworm/capi/silkworm.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/silkworm/capi/silkworm.h b/silkworm/capi/silkworm.h index 47c3d8df0b..953e186124 100644 --- a/silkworm/capi/silkworm.h +++ b/silkworm/capi/silkworm.h @@ -248,8 +248,8 @@ SILKWORM_EXPORT int silkworm_sentry_stop(SilkwormHandle handle) SILKWORM_NOEXCEP //! Silkworm Fork Validator configuration options struct SilkwormForkValidatorSettings { - uint64_t batch_size; // Batch size to use in stages - uint64_t etl_buffer_size; // Buffer size for ETL operations + uint_t batch_size; // Batch size to use in stages + uint_t etl_buffer_size; // Buffer size for ETL operations uint32_t sync_loop_throttle_seconds; // Minimum interval amongst sync cycle bool stop_before_senders_stage; // Stop before senders stage }; From 7314d13d5e3ebfc423a6bfc15ea6ffa15364686e Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Thu, 30 Jan 2025 23:27:49 +0100 Subject: [PATCH 33/33] Changes from code review --- silkworm/capi/silkworm.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/silkworm/capi/silkworm.h b/silkworm/capi/silkworm.h index 953e186124..25b65beb0e 100644 --- a/silkworm/capi/silkworm.h +++ b/silkworm/capi/silkworm.h @@ -248,8 +248,8 @@ SILKWORM_EXPORT int silkworm_sentry_stop(SilkwormHandle handle) SILKWORM_NOEXCEP //! Silkworm Fork Validator configuration options struct SilkwormForkValidatorSettings { - uint_t batch_size; // Batch size to use in stages - uint_t etl_buffer_size; // Buffer size for ETL operations + size_t batch_size; // Batch size to use in stages + size_t etl_buffer_size; // Buffer size for ETL operations uint32_t sync_loop_throttle_seconds; // Minimum interval amongst sync cycle bool stop_before_senders_stage; // Stop before senders stage };