From 877b52b410d5b2979b4f838c71476b8180adc03e Mon Sep 17 00:00:00 2001 From: Aaryamann Challani <43716372+rymnc@users.noreply.github.com> Date: Mon, 9 Sep 2024 18:40:53 +0530 Subject: [PATCH] fix(db_lookup_times): rework core logic of benchmark (#2159) >[!NOTE] > This PR follows up #2142 with comments after merge ## Linked Issues/PRs - #2142 - #2023 ## Description - refactors the test database to use a custom set of columns that will not create any new tables/columns in `fuel-core` - refactors the functions to return `Result`s instead of unwrapping everywhere - automated cleaning up of benchmark databases - `KeyValueMutate` impl of `RocksDb` is now behind the `test-helpers` feature flag - moved the `db_lookup_times_utils` to the `src` directory in the benches crate so we can test our insertion/fetching logic ## Checklist - [x] Breaking changes are clearly marked as such in the PR description and changelog - [x] New behavior is reflected in tests - [x] [The specification](https://github.com/FuelLabs/fuel-specs/) matches the implemented behavior (link update PR if changes are needed) ### Before requesting review - [x] I have reviewed the code myself - [ ] I have created follow-up issues caused by this PR and linked them here ### After merging, notify other teams [Add or remove entries as needed] - [ ] [Rust SDK](https://github.com/FuelLabs/fuels-rs/) - [ ] [Sway compiler](https://github.com/FuelLabs/sway/) - [ ] [Platform documentation](https://github.com/FuelLabs/devrel-requests/issues/new?assignees=&labels=new+request&projects=&template=NEW-REQUEST.yml&title=%5BRequest%5D%3A+) (for out-of-organization contributors, the person merging the PR will do this) - [ ] Someone else? --------- Co-authored-by: Green Baneling --- Cargo.lock | 5 + benches/Cargo.toml | 5 + benches/benches/db_lookup_times.rs | 98 +++++----- .../benches/db_lookup_times_utils/matrix.rs | 10 -- benches/benches/db_lookup_times_utils/mod.rs | 3 - benches/benches/db_lookup_times_utils/seed.rs | 151 ---------------- .../benches/db_lookup_times_utils/utils.rs | 100 ----------- .../db_lookup_times_utils/full_block_table.rs | 60 +++++++ benches/src/db_lookup_times_utils/matrix.rs | 12 ++ benches/src/db_lookup_times_utils/mod.rs | 83 +++++++++ benches/src/db_lookup_times_utils/seed.rs | 168 ++++++++++++++++++ benches/src/db_lookup_times_utils/utils.rs | 127 +++++++++++++ benches/src/lib.rs | 2 + crates/fuel-core/src/state/rocks_db.rs | 61 ++++--- crates/storage/src/column.rs | 3 - .../storage/src/structured_storage/blocks.rs | 16 -- crates/storage/src/tables.rs | 15 +- 17 files changed, 554 insertions(+), 365 deletions(-) delete mode 100644 benches/benches/db_lookup_times_utils/matrix.rs delete mode 100644 benches/benches/db_lookup_times_utils/mod.rs delete mode 100644 benches/benches/db_lookup_times_utils/seed.rs delete mode 100644 benches/benches/db_lookup_times_utils/utils.rs create mode 100644 benches/src/db_lookup_times_utils/full_block_table.rs create mode 100644 benches/src/db_lookup_times_utils/matrix.rs create mode 100644 benches/src/db_lookup_times_utils/mod.rs create mode 100644 benches/src/db_lookup_times_utils/seed.rs create mode 100644 benches/src/db_lookup_times_utils/utils.rs diff --git a/Cargo.lock b/Cargo.lock index 1aa5b927fb4..de233bfee92 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3212,6 +3212,7 @@ dependencies = [ "criterion", "ctrlc", "ed25519-dalek", + "enum-iterator", "ethnum", "fuel-core", "fuel-core-chain-config", @@ -3221,6 +3222,8 @@ dependencies = [ "fuel-core-sync", "fuel-core-types", "futures", + "itertools 0.12.1", + "num_enum", "p256", "postcard", "primitive-types", @@ -3229,6 +3232,8 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "strum 0.25.0", + "strum_macros 0.25.3", "test-helpers", "tikv-jemallocator", "tokio", diff --git a/benches/Cargo.toml b/benches/Cargo.toml index eeb92dbc481..7005debf627 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -16,6 +16,7 @@ criterion = { version = "0.5", features = [ ] } ctrlc = "3.2.3" ed25519-dalek = { version = "2.0", features = ["rand_core"] } +enum-iterator = { workspace = true } ethnum = "1.3" fuel-core = { path = "../crates/fuel-core", default-features = false, features = [ "smt", @@ -30,6 +31,8 @@ fuel-core-sync = { path = "./../crates/services/sync", features = [ ] } fuel-core-types = { path = "./../crates/types", features = ["test-helpers"] } futures = { workspace = true } +itertools = { workspace = true } +num_enum = { workspace = true } p256 = { version = "0.13", default-features = false, features = [ "digest", "ecdsa", @@ -41,6 +44,8 @@ rand = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } serde_yaml = "0.9.13" +strum = { workspace = true } +strum_macros = { workspace = true } test-helpers = { path = "../tests/test-helpers" } tikv-jemallocator = { workspace = true } tokio = { workspace = true, features = ["full"] } diff --git a/benches/benches/db_lookup_times.rs b/benches/benches/db_lookup_times.rs index 4f54aeeaefe..bd07d4607d8 100644 --- a/benches/benches/db_lookup_times.rs +++ b/benches/benches/db_lookup_times.rs @@ -1,96 +1,114 @@ -use crate::db_lookup_times_utils::{ - matrix::matrix, - utils::{ - get_full_block, - get_random_block_height, - multi_get_block, - open_db, - open_raw_rocksdb, - }, -}; use criterion::{ criterion_group, criterion_main, Criterion, }; -use db_lookup_times_utils::seed::{ - seed_compressed_blocks_and_transactions_matrix, - seed_full_block_matrix, +use fuel_core_benches::db_lookup_times_utils::{ + matrix::matrix, + seed::{ + seed_compressed_blocks_and_transactions_matrix, + seed_full_block_matrix, + }, + utils::{ + get_random_block_height, + open_rocks_db, + LookupMethod, + Result as DbLookupBenchResult, + }, }; -use fuel_core_storage::transactional::AtomicView; -use rand::thread_rng; -mod db_lookup_times_utils; +use fuel_core_benches::utils::ShallowTempDir; +use rand::thread_rng; -pub fn header_and_tx_lookup(c: &mut Criterion) { - let method = "header_and_tx"; +pub fn header_and_tx_lookup(c: &mut Criterion) -> DbLookupBenchResult<()> { + let method = LookupMethod::HeaderAndTx; let mut rng = thread_rng(); - seed_compressed_blocks_and_transactions_matrix(method); - let mut group = c.benchmark_group(method); + let mut group = c.benchmark_group(method.as_ref()); for (block_count, tx_count) in matrix() { - let database = open_db(block_count, tx_count, method); - let view = database.latest_view().unwrap(); + let db_path = ShallowTempDir::new(); + let mut database = open_rocks_db(db_path.path())?; + seed_compressed_blocks_and_transactions_matrix( + &mut database, + block_count, + tx_count, + )?; + group.bench_function(format!("{block_count}/{tx_count}"), |b| { b.iter(|| { let height = get_random_block_height(&mut rng, block_count); - let block = view.get_full_block(&height); + let block = method.get_block(&database, height); assert!(block.is_ok()); - assert!(block.unwrap().is_some()); }); }); } group.finish(); + Ok(()) } -pub fn multi_get_lookup(c: &mut Criterion) { - let method = "multi_get"; +pub fn multi_get_lookup(c: &mut Criterion) -> DbLookupBenchResult<()> { + let method = LookupMethod::MultiGet; let mut rng = thread_rng(); - seed_compressed_blocks_and_transactions_matrix(method); - let mut group = c.benchmark_group(method); + let mut group = c.benchmark_group(method.as_ref()); for (block_count, tx_count) in matrix() { - let database = open_raw_rocksdb(block_count, tx_count, method); + let db_path = ShallowTempDir::new(); + let mut database = open_rocks_db(db_path.path())?; + seed_compressed_blocks_and_transactions_matrix( + &mut database, + block_count, + tx_count, + )?; + group.bench_function(format!("{block_count}/{tx_count}"), |b| { b.iter(|| { let height = get_random_block_height(&mut rng, block_count); - assert!(multi_get_block(&database, height).is_ok()); + let block = method.get_block(&database, height); + assert!(block.is_ok()); }); }); } group.finish(); + Ok(()) } -pub fn full_block_lookup(c: &mut Criterion) { - let method = "full_block"; +pub fn full_block_lookup(c: &mut Criterion) -> DbLookupBenchResult<()> { + let method = LookupMethod::FullBlock; let mut rng = thread_rng(); - seed_full_block_matrix(); - let mut group = c.benchmark_group(method); + let mut group = c.benchmark_group(method.as_ref()); for (block_count, tx_count) in matrix() { - let database = open_db(block_count, tx_count, method); - let view = database.latest_view().unwrap(); + let db_path = ShallowTempDir::new(); + let mut database = open_rocks_db(db_path.path())?; + seed_full_block_matrix(&mut database, block_count, tx_count)?; + group.bench_function(format!("{block_count}/{tx_count}"), |b| { b.iter(|| { let height = get_random_block_height(&mut rng, block_count); - let full_block = get_full_block(&view, &height); + let full_block = method.get_block(&database, height); assert!(full_block.is_ok()); - assert!(full_block.unwrap().is_some()); }); }); } group.finish(); + Ok(()) +} + +fn construct_and_run_benchmarks(c: &mut Criterion) { + header_and_tx_lookup(c).unwrap(); + multi_get_lookup(c).unwrap(); + full_block_lookup(c).unwrap(); } criterion_group! { name = benches; - config = Criterion::default().sample_size(100_000).measurement_time(std::time::Duration::from_secs(100)); - targets = header_and_tx_lookup, multi_get_lookup, full_block_lookup + config = Criterion::default().sample_size(10).measurement_time(std::time::Duration::from_secs(10)); + targets = construct_and_run_benchmarks } criterion_main!(benches); diff --git a/benches/benches/db_lookup_times_utils/matrix.rs b/benches/benches/db_lookup_times_utils/matrix.rs deleted file mode 100644 index 4849309381c..00000000000 --- a/benches/benches/db_lookup_times_utils/matrix.rs +++ /dev/null @@ -1,10 +0,0 @@ -pub const BLOCK_COUNT_MATRIX: [u32; 2] = [10, 100]; -pub const TX_COUNT_MATRIX: [u32; 2] = [100, 1000]; - -pub fn matrix() -> impl Iterator { - BLOCK_COUNT_MATRIX.iter().flat_map(|&block_count| { - TX_COUNT_MATRIX - .iter() - .map(move |&tx_count| (block_count, tx_count)) - }) -} diff --git a/benches/benches/db_lookup_times_utils/mod.rs b/benches/benches/db_lookup_times_utils/mod.rs deleted file mode 100644 index 2bb42a84cbd..00000000000 --- a/benches/benches/db_lookup_times_utils/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod matrix; -pub mod seed; -pub mod utils; diff --git a/benches/benches/db_lookup_times_utils/seed.rs b/benches/benches/db_lookup_times_utils/seed.rs deleted file mode 100644 index 3c330b2e221..00000000000 --- a/benches/benches/db_lookup_times_utils/seed.rs +++ /dev/null @@ -1,151 +0,0 @@ -use crate::db_lookup_times_utils::{ - matrix::matrix, - utils::{ - chain_id, - open_raw_rocksdb, - }, -}; -use fuel_core::{ - database::database_description::on_chain::OnChain, - state::rocks_db::RocksDb, -}; -use fuel_core_storage::{ - column::Column, - kv_store::{ - KeyValueMutate, - Value, - }, -}; -use fuel_core_types::{ - blockchain::{ - block::{ - Block, - PartialFuelBlock, - }, - header::{ - ConsensusHeader, - PartialBlockHeader, - }, - primitives::Empty, - }, - fuel_tx::{ - Transaction, - UniqueIdentifier, - }, - fuel_types::BlockHeight, -}; - -pub fn seed_compressed_blocks_and_transactions_matrix(method: &str) { - for (block_count, tx_count) in matrix() { - let mut database = open_raw_rocksdb(block_count, tx_count, method); - let _ = - seed_compressed_blocks_and_transactions(&mut database, block_count, tx_count); - } -} - -pub fn seed_full_block_matrix() { - for (block_count, tx_count) in matrix() { - let mut database = open_raw_rocksdb(block_count, tx_count, "full_block"); - seed_full_blocks(&mut database, block_count, tx_count); - } -} - -fn generate_bench_block(height: u32, tx_count: u32) -> Block { - let header = PartialBlockHeader { - application: Default::default(), - consensus: ConsensusHeader:: { - height: BlockHeight::from(height), - ..Default::default() - }, - }; - let txes = generate_bench_transactions(tx_count); - let block = PartialFuelBlock::new(header, txes); - block.generate(&[], Default::default()).unwrap() -} - -fn generate_bench_transactions(tx_count: u32) -> Vec { - let mut txes = vec![]; - for _ in 0..tx_count { - txes.push(Transaction::default_test_tx()); - } - txes -} - -fn height_key(block_height: u32) -> Vec { - BlockHeight::from(block_height).to_bytes().to_vec() -} - -fn insert_compressed_block( - database: &mut RocksDb, - height: u32, - tx_count: u32, -) -> Block { - let block = generate_bench_block(height, tx_count); - - let compressed_block = block.compress(&chain_id()); - let height_key = height_key(height); - - let raw_compressed_block = postcard::to_allocvec(&compressed_block).unwrap().to_vec(); - let raw_transactions = block - .transactions() - .iter() - .map(|tx| { - ( - tx.id(&chain_id()), - postcard::to_allocvec(tx).unwrap().to_vec(), - ) - }) - .collect::>(); - - // 1. insert into CompressedBlocks table - database - .put( - height_key.as_slice(), - Column::FuelBlocks, - Value::new(raw_compressed_block), - ) - .unwrap(); - // 2. insert into Transactions table - for (tx_id, tx) in raw_transactions { - database - .put(tx_id.as_slice(), Column::Transactions, Value::new(tx)) - .unwrap(); - } - - block -} - -fn insert_full_block(database: &mut RocksDb, height: u32, tx_count: u32) { - // we seed compressed blocks and transactions to not affect individual - // lookup times - let block = insert_compressed_block(database, height, tx_count); - - let height_key = height_key(height); - let raw_full_block = postcard::to_allocvec(&block).unwrap().to_vec(); - - database - .put( - height_key.as_slice(), - Column::FullFuelBlocks, - Value::new(raw_full_block), - ) - .unwrap(); -} - -fn seed_compressed_blocks_and_transactions( - database: &mut RocksDb, - block_count: u32, - tx_count: u32, -) -> Vec { - let mut blocks = vec![]; - for block_number in 0..block_count { - blocks.push(insert_compressed_block(database, block_number, tx_count)); - } - blocks -} - -fn seed_full_blocks(database: &mut RocksDb, block_count: u32, tx_count: u32) { - for block_number in 0..block_count { - insert_full_block(database, block_number, tx_count); - } -} diff --git a/benches/benches/db_lookup_times_utils/utils.rs b/benches/benches/db_lookup_times_utils/utils.rs deleted file mode 100644 index dfbc570906c..00000000000 --- a/benches/benches/db_lookup_times_utils/utils.rs +++ /dev/null @@ -1,100 +0,0 @@ -use anyhow::anyhow; -use fuel_core::{ - database::{ - database_description::on_chain::OnChain, - Database, - }, - state::{ - historical_rocksdb::StateRewindPolicy, - rocks_db::RocksDb, - IterableKeyValueView, - }, -}; -use fuel_core_storage::{ - column::Column, - kv_store::{ - KeyValueInspect, - StorageColumn, - }, - tables::FullFuelBlocks, - StorageAsRef, -}; -use fuel_core_types::{ - blockchain::block::{ - Block, - CompressedBlock, - }, - fuel_tx::Transaction, - fuel_types::{ - BlockHeight, - ChainId, - }, -}; -use rand::{ - rngs::ThreadRng, - Rng, -}; -use std::{ - borrow::Cow, - path::Path, -}; - -pub fn get_random_block_height(rng: &mut ThreadRng, block_count: u32) -> BlockHeight { - BlockHeight::from(rng.gen_range(0..block_count)) -} - -pub fn open_db(block_count: u32, tx_count: u32, method: &str) -> Database { - Database::open_rocksdb( - Path::new(format!("./{block_count}/{method}/{tx_count}").as_str()), - None, // no caching - StateRewindPolicy::NoRewind, - ) - .unwrap() -} - -pub fn open_raw_rocksdb( - block_count: u32, - tx_count: u32, - method: &str, -) -> RocksDb { - RocksDb::default_open( - Path::new(format!("./{block_count}/{method}/{tx_count}").as_str()), - None, - ) - .unwrap() -} - -pub fn chain_id() -> ChainId { - ChainId::default() -} - -pub fn get_full_block( - view: &IterableKeyValueView, - height: &BlockHeight, -) -> anyhow::Result> { - let db_block = view.storage::().get(height)?; - if let Some(block) = db_block { - Ok(Some(Cow::into_owned(block))) - } else { - Ok(None) - } -} - -pub fn multi_get_block( - database: &RocksDb, - height: BlockHeight, -) -> anyhow::Result<()> { - let height_key = height.to_bytes(); - - let raw_block = database - .get(&height_key, Column::FuelBlocks)? - .ok_or(anyhow!("empty raw block"))?; - let block: CompressedBlock = postcard::from_bytes(raw_block.as_slice())?; - let tx_ids = block.transactions().iter(); - let raw_txs = database.multi_get(Column::Transactions.id(), tx_ids)?; - for raw_tx in raw_txs.iter().flatten() { - let _: Transaction = postcard::from_bytes(raw_tx.as_slice())?; - } - - Ok(()) -} diff --git a/benches/src/db_lookup_times_utils/full_block_table.rs b/benches/src/db_lookup_times_utils/full_block_table.rs new file mode 100644 index 00000000000..997f638cf15 --- /dev/null +++ b/benches/src/db_lookup_times_utils/full_block_table.rs @@ -0,0 +1,60 @@ +use fuel_core::database::database_description::DatabaseDescription; +use fuel_core_storage::kv_store::StorageColumn; +use fuel_core_types::fuel_types::BlockHeight; + +#[repr(u32)] +#[derive( + Copy, + Clone, + Debug, + strum_macros::EnumCount, + strum_macros::IntoStaticStr, + PartialEq, + Eq, + enum_iterator::Sequence, + Hash, + num_enum::TryFromPrimitive, +)] +pub enum BenchDbColumn { + /// See [`Transactions`](crate::tables::Transactions) + Transactions = 0, + /// See [`FuelBlocks`](crate::tables::FuelBlocks) + FuelBlocks = 1, + FullFuelBlocks = 2, + Metadata = 3, +} + +impl StorageColumn for BenchDbColumn { + fn name(&self) -> String { + let str: &str = self.into(); + str.to_string() + } + + fn id(&self) -> u32 { + *self as u32 + } +} + +#[derive(Copy, Clone, Debug)] +pub struct BenchDatabase; + +impl DatabaseDescription for BenchDatabase { + type Column = BenchDbColumn; + type Height = BlockHeight; + + fn version() -> u32 { + 0 + } + + fn name() -> String { + "bench_database".to_string() + } + + fn metadata_column() -> Self::Column { + Self::Column::Metadata + } + + fn prefix(_column: &Self::Column) -> Option { + None + } +} diff --git a/benches/src/db_lookup_times_utils/matrix.rs b/benches/src/db_lookup_times_utils/matrix.rs new file mode 100644 index 00000000000..45efc98d142 --- /dev/null +++ b/benches/src/db_lookup_times_utils/matrix.rs @@ -0,0 +1,12 @@ +use fuel_core_types::fuel_types::BlockHeight; + +pub const BLOCK_COUNT_MATRIX: [u32; 5] = [10, 100, 1000, 5000, 10_000]; +pub const TX_COUNT_MATRIX: [u32; 3] = [500, 1000, 2000]; + +pub fn matrix() -> impl Iterator { + BLOCK_COUNT_MATRIX.iter().flat_map(|&block_count| { + TX_COUNT_MATRIX + .iter() + .map(move |&tx_count| (block_count.into(), tx_count)) + }) +} diff --git a/benches/src/db_lookup_times_utils/mod.rs b/benches/src/db_lookup_times_utils/mod.rs new file mode 100644 index 00000000000..9bb12fb7169 --- /dev/null +++ b/benches/src/db_lookup_times_utils/mod.rs @@ -0,0 +1,83 @@ +pub mod full_block_table; +pub mod matrix; +pub mod seed; +pub mod utils; + +#[cfg(test)] +mod tests { + use crate::db_lookup_times_utils::{ + full_block_table::BenchDatabase, + utils::{ + LookupMethod, + Result as DbLookupBenchResult, + }, + }; + use fuel_core::state::rocks_db::RocksDb; + + use crate::{ + db_lookup_times_utils::seed::{ + insert_compressed_block, + insert_full_block, + }, + utils::ShallowTempDir, + }; + + const TEST_HEIGHT: u32 = 1; + const TEST_TX_COUNT: u32 = 10; + + fn setup_test_db() -> RocksDb { + let temp_dir = ShallowTempDir::new(); + RocksDb::default_open(temp_dir.path(), None).unwrap() + } + + #[test] + fn test_insert_and_fetch_compressed_block() -> DbLookupBenchResult<()> { + // given + let mut db = setup_test_db(); + + // when + let inserted_block = + insert_compressed_block(&mut db, TEST_HEIGHT.into(), TEST_TX_COUNT)?; + + // then + let fetched_block = + LookupMethod::get_block(&LookupMethod::HeaderAndTx, &db, TEST_HEIGHT.into())?; + + assert_eq!(inserted_block, fetched_block); + Ok(()) + } + + #[test] + fn test_insert_and_fetch_full_block() -> DbLookupBenchResult<()> { + // given + let mut db = setup_test_db(); + + // when + let inserted_block = + insert_full_block(&mut db, TEST_HEIGHT.into(), TEST_TX_COUNT)?; + + // then + let fetched_block = + LookupMethod::get_block(&LookupMethod::FullBlock, &db, TEST_HEIGHT.into())?; + + assert_eq!(inserted_block, fetched_block); + Ok(()) + } + + #[test] + fn test_insert_and_multi_get_block() -> DbLookupBenchResult<()> { + // given + let mut db = setup_test_db(); + + // when + let inserted_block = + insert_compressed_block(&mut db, TEST_HEIGHT.into(), TEST_TX_COUNT)?; + + // then + let fetched_block = + LookupMethod::get_block(&LookupMethod::MultiGet, &db, TEST_HEIGHT.into())?; + + assert_eq!(inserted_block, fetched_block); + Ok(()) + } +} diff --git a/benches/src/db_lookup_times_utils/seed.rs b/benches/src/db_lookup_times_utils/seed.rs new file mode 100644 index 00000000000..e5d68c4450f --- /dev/null +++ b/benches/src/db_lookup_times_utils/seed.rs @@ -0,0 +1,168 @@ +use crate::db_lookup_times_utils::{ + full_block_table::{ + BenchDatabase, + BenchDbColumn, + }, + utils::Result as DbLookupBenchResult, +}; +use anyhow::anyhow; +use fuel_core::state::rocks_db::RocksDb; +use fuel_core_storage::kv_store::{ + KeyValueMutate, + Value, +}; +use fuel_core_types::{ + blockchain::{ + block::{ + Block, + PartialFuelBlock, + }, + header::{ + ConsensusHeader, + PartialBlockHeader, + }, + primitives::Empty, + }, + fuel_tx::{ + Bytes32, + Transaction, + UniqueIdentifier, + }, + fuel_types::{ + BlockHeight, + ChainId, + }, +}; +use itertools::Itertools; + +pub fn seed_compressed_blocks_and_transactions_matrix( + database: &mut RocksDb, + block_count: BlockHeight, + tx_count: u32, +) -> DbLookupBenchResult<()> { + seed_compressed_blocks_and_transactions(database, block_count, tx_count) +} + +pub fn seed_full_block_matrix( + database: &mut RocksDb, + block_count: BlockHeight, + tx_count: u32, +) -> DbLookupBenchResult<()> { + seed_full_blocks(database, block_count, tx_count) +} + +fn generate_bench_block( + height: BlockHeight, + tx_count: u32, +) -> DbLookupBenchResult { + let header = PartialBlockHeader { + application: Default::default(), + consensus: ConsensusHeader:: { + height, + ..Default::default() + }, + }; + let txs = generate_bench_transactions(tx_count); + let block = PartialFuelBlock::new(header, txs); + block + .generate(&[], Default::default()) + .map_err(|err| anyhow!(err)) +} + +fn generate_bench_transactions(tx_count: u32) -> Vec { + let mut txs = Vec::with_capacity(tx_count as usize); + for _ in 0..tx_count { + txs.push(Transaction::default_test_tx()); + } + txs +} + +fn height_key(block_height: BlockHeight) -> Vec { + block_height.to_bytes().to_vec() +} + +pub fn insert_compressed_block( + database: &mut RocksDb, + height: BlockHeight, + tx_count: u32, +) -> DbLookupBenchResult { + let block = generate_bench_block(height, tx_count)?; + + let compressed_block = block.compress(&ChainId::default()); + let height_key = height_key(height); + + let raw_compressed_block = postcard::to_allocvec(&compressed_block)?.to_vec(); + let raw_transactions: Vec<(Bytes32, Vec)> = block + .transactions() + .iter() + .map(|tx| -> DbLookupBenchResult<(Bytes32, Vec)> { + let tx_id = tx.id(&ChainId::default()); + let raw_tx = postcard::to_allocvec(tx)?.to_vec(); + Ok((tx_id, raw_tx)) + }) + .try_collect()?; + + // 1. insert into CompressedBlocks table + database.put( + height_key.as_slice(), + BenchDbColumn::FuelBlocks, + Value::new(raw_compressed_block), + )?; + // 2. insert into Transactions table + for (tx_id, tx) in raw_transactions { + database.put( + tx_id.as_slice(), + BenchDbColumn::Transactions, + Value::new(tx), + )?; + } + + Ok(block) +} + +pub fn insert_full_block( + database: &mut RocksDb, + height: BlockHeight, + tx_count: u32, +) -> DbLookupBenchResult { + // we seed compressed blocks and transactions to not affect individual + // lookup times + let block = insert_compressed_block(database, height, tx_count)?; + + let height_key = height_key(height); + let raw_full_block = postcard::to_allocvec(&block)?.to_vec(); + + database + .put( + height_key.as_slice(), + BenchDbColumn::FullFuelBlocks, + Value::new(raw_full_block), + ) + .map_err(|err| anyhow!(err))?; + + Ok(block) +} + +fn seed_compressed_blocks_and_transactions( + database: &mut RocksDb, + block_count: BlockHeight, + tx_count: u32, +) -> DbLookupBenchResult<()> { + for block_number in 0..block_count.into() { + insert_compressed_block(database, block_number.into(), tx_count)?; + } + + Ok(()) +} + +fn seed_full_blocks( + full_block_db: &mut RocksDb, + block_count: BlockHeight, + tx_count: u32, +) -> DbLookupBenchResult<()> { + for block_number in 0..block_count.into() { + insert_full_block(full_block_db, block_number.into(), tx_count)?; + } + + Ok(()) +} diff --git a/benches/src/db_lookup_times_utils/utils.rs b/benches/src/db_lookup_times_utils/utils.rs new file mode 100644 index 00000000000..5845079e2b5 --- /dev/null +++ b/benches/src/db_lookup_times_utils/utils.rs @@ -0,0 +1,127 @@ +use crate::db_lookup_times_utils::full_block_table::{ + BenchDatabase, + BenchDbColumn, +}; +use anyhow::anyhow; +use fuel_core::{ + database::database_description::DatabaseDescription, + state::rocks_db::RocksDb, +}; +use fuel_core_storage::kv_store::{ + KeyValueInspect, + StorageColumn, +}; +use fuel_core_types::{ + blockchain::block::{ + Block, + CompressedBlock, + }, + fuel_tx::Transaction, + fuel_types::BlockHeight, +}; +use itertools::Itertools; +use rand::{ + rngs::ThreadRng, + Rng, +}; +use std::path::Path; +use strum_macros::AsRefStr; + +pub use anyhow::Result; + +pub fn get_random_block_height( + rng: &mut ThreadRng, + block_count: BlockHeight, +) -> BlockHeight { + BlockHeight::from(rng.gen_range(0..block_count.into())) +} + +pub fn open_rocks_db( + path: &Path, +) -> Result> { + let db = RocksDb::default_open(path, None)?; + Ok(db) +} + +#[derive(Copy, Clone, AsRefStr)] +pub enum LookupMethod { + FullBlock, + MultiGet, + HeaderAndTx, +} + +impl LookupMethod { + pub fn get_block( + &self, + database: &RocksDb, + height: BlockHeight, + ) -> Result { + match self { + LookupMethod::FullBlock => get_block_full_block_method(database, height), + LookupMethod::MultiGet => get_block_multi_get_method(database, height), + LookupMethod::HeaderAndTx => { + get_block_headers_and_tx_method(database, height) + } + } + } +} + +fn get_block_full_block_method( + database: &RocksDb, + height: BlockHeight, +) -> Result { + let height_key = height.to_bytes(); + let raw_block = database + .get(&height_key, BenchDbColumn::FullFuelBlocks)? + .ok_or(anyhow!("empty raw full block"))?; + + let block: Block = postcard::from_bytes(raw_block.as_slice())?; + Ok(block) +} + +fn get_block_multi_get_method( + database: &RocksDb, + height: BlockHeight, +) -> Result { + let height_key = height.to_bytes(); + + let raw_block = database + .get(&height_key, BenchDbColumn::FuelBlocks)? + .ok_or(anyhow!("empty raw block"))?; + let block: CompressedBlock = postcard::from_bytes(raw_block.as_slice())?; + let tx_ids = block.transactions().iter(); + let raw_txs = database.multi_get(BenchDbColumn::Transactions.id(), tx_ids)?; + let txs: Vec = raw_txs + .iter() + .flatten() + .map(|raw_tx| postcard::from_bytes::(raw_tx.as_slice())) + .try_collect()?; + + Ok(block.uncompress(txs)) +} + +fn get_block_headers_and_tx_method( + database: &RocksDb, + height: BlockHeight, +) -> Result { + let height_key = height.to_bytes(); + + let raw_block = database + .get(&height_key, BenchDbColumn::FuelBlocks)? + .ok_or(anyhow!("empty raw block"))?; + let block: CompressedBlock = postcard::from_bytes(raw_block.as_slice())?; + + let txs: Vec = block + .transactions() + .iter() + .map(|tx_id| { + let raw_tx = database + .get(tx_id.as_slice(), BenchDbColumn::Transactions)? + .ok_or(anyhow!("empty transaction"))?; + postcard::from_bytes::(raw_tx.as_slice()) + .map_err(|err| anyhow!(err)) + }) + .try_collect()?; + + Ok(block.uncompress(txs)) +} diff --git a/benches/src/lib.rs b/benches/src/lib.rs index edf5f7d73dc..a06e26d1eee 100644 --- a/benches/src/lib.rs +++ b/benches/src/lib.rs @@ -40,6 +40,8 @@ pub mod default_gas_costs; pub mod import; pub mod utils; +pub mod db_lookup_times_utils; + pub use fuel_core_storage::vm_storage::VmStorage; pub use rand::Rng; diff --git a/crates/fuel-core/src/state/rocks_db.rs b/crates/fuel-core/src/state/rocks_db.rs index 9edb402d590..c54ee3765c1 100644 --- a/crates/fuel-core/src/state/rocks_db.rs +++ b/crates/fuel-core/src/state/rocks_db.rs @@ -23,15 +23,11 @@ use fuel_core_storage::{ KVItem, KeyItem, KeyValueInspect, - KeyValueMutate, StorageColumn, Value, WriteOperation, }, - transactional::{ - Changes, - ReadTransaction, - }, + transactional::Changes, Result as StorageResult, }; use itertools::Itertools; @@ -830,30 +826,39 @@ fn next_prefix(mut prefix: Vec) -> Option> { None } -impl KeyValueMutate for RocksDb -where - Description: DatabaseDescription, -{ - fn write( - &mut self, - key: &[u8], - column: Self::Column, - buf: &[u8], - ) -> StorageResult { - let mut transaction = self.read_transaction(); - let len = transaction.write(key, column, buf)?; - let changes = transaction.into_changes(); - self.commit_changes(&changes)?; - - Ok(len) - } +#[cfg(feature = "test-helpers")] +pub mod test_helpers { + use super::*; + use fuel_core_storage::{ + kv_store::KeyValueMutate, + transactional::ReadTransaction, + }; - fn delete(&mut self, key: &[u8], column: Self::Column) -> StorageResult<()> { - let mut transaction = self.read_transaction(); - transaction.delete(key, column)?; - let changes = transaction.into_changes(); - self.commit_changes(&changes)?; - Ok(()) + impl KeyValueMutate for RocksDb + where + Description: DatabaseDescription, + { + fn write( + &mut self, + key: &[u8], + column: Self::Column, + buf: &[u8], + ) -> StorageResult { + let mut transaction = self.read_transaction(); + let len = transaction.write(key, column, buf)?; + let changes = transaction.into_changes(); + self.commit_changes(&changes)?; + + Ok(len) + } + + fn delete(&mut self, key: &[u8], column: Self::Column) -> StorageResult<()> { + let mut transaction = self.read_transaction(); + transaction.delete(key, column)?; + let changes = transaction.into_changes(); + self.commit_changes(&changes)?; + Ok(()) + } } } diff --git a/crates/storage/src/column.rs b/crates/storage/src/column.rs index 974dfe56438..6557dc35b4d 100644 --- a/crates/storage/src/column.rs +++ b/crates/storage/src/column.rs @@ -67,9 +67,6 @@ pub enum Column { UploadedBytecodes = 19, /// See [`Blobs`](fuel_vm_private::storage::BlobData) Blobs = 20, - /// See [`FullFuelBlocks`](crate::tables::FullFuelBlocks) - /// we don't use this table at the moment, only used for benchmarks - FullFuelBlocks = 22, // TODO: Remove this column and use `Metadata` column instead. /// Table for genesis state import progress tracking. diff --git a/crates/storage/src/structured_storage/blocks.rs b/crates/storage/src/structured_storage/blocks.rs index 3257b6a1125..ddd5884cf5a 100644 --- a/crates/storage/src/structured_storage/blocks.rs +++ b/crates/storage/src/structured_storage/blocks.rs @@ -15,7 +15,6 @@ use crate::{ FuelBlockMerkleMetadata, }, FuelBlocks, - FullFuelBlocks, }, }; use fuel_core_types::blockchain::block::{ @@ -60,21 +59,6 @@ impl TableWithBlueprint for FuelBlocks { } } -impl TableWithBlueprint for FullFuelBlocks { - type Blueprint = Merklized< - Primitive<4>, - Postcard, - FuelBlockMerkleMetadata, - FuelBlockMerkleData, - BlockEncoder, - >; - type Column = Column; - - fn column() -> Column { - Column::FullFuelBlocks - } -} - #[cfg(test)] mod tests { use crate::{ diff --git a/crates/storage/src/tables.rs b/crates/storage/src/tables.rs index 74e1ac2d6bb..0eda7e5e645 100644 --- a/crates/storage/src/tables.rs +++ b/crates/storage/src/tables.rs @@ -4,10 +4,7 @@ use crate::Mappable; use fuel_core_types::{ blockchain::{ - block::{ - Block, - CompressedBlock, - }, + block::CompressedBlock, consensus::Consensus, header::{ ConsensusParametersVersion, @@ -44,16 +41,6 @@ pub use fuel_vm_private::storage::{ /// Right now, we have only that type of block, but we will support others in the future. pub struct FuelBlocks; -/// Full block table. Contains all the information about the block. -pub struct FullFuelBlocks; - -impl Mappable for FullFuelBlocks { - type Key = Self::OwnedKey; - type OwnedKey = BlockHeight; - type Value = Self::OwnedValue; - type OwnedValue = Block; -} - impl Mappable for FuelBlocks { /// Unique identifier of the fuel block. type Key = Self::OwnedKey;