diff --git a/Cargo.lock b/Cargo.lock index c399a9585..31c150c61 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,6 +28,18 @@ dependencies = [ "cpufeatures", ] +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.2" @@ -37,6 +49,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -290,6 +308,15 @@ dependencies = [ "serde", ] +[[package]] +name = "bitcoin_slices" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8041a1be831c809ada090db2e3bd1469c65b72321bb2f31d7f56261eefc8321" +dependencies = [ + "hashbrown 0.14.5", +] + [[package]] name = "bitcoincore-rpc" version = "0.18.0" @@ -738,6 +765,7 @@ dependencies = [ "bincode", "bitcoin 0.31.2", "bitcoin-test-data", + "bitcoin_slices", "bitcoind", "clap 2.34.0", "criterion", @@ -1022,6 +1050,16 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + [[package]] name = "hermit-abi" version = "0.1.19" @@ -1194,7 +1232,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.3", ] [[package]] @@ -3077,6 +3115,26 @@ dependencies = [ "rustix", ] +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "zip" version = "0.6.6" diff --git a/Cargo.toml b/Cargo.toml index 6bae3b716..8a473968f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ keywords = ["bitcoin", "electrum", "server", "index", "database"] documentation = "https://docs.rs/electrs/" readme = "README.md" edition = "2018" +default-run = "electrs" [features] liquid = ["elements"] @@ -56,6 +57,7 @@ tokio = { version = "1", features = ["sync", "macros"] } # optional dependencies for electrum-discovery electrum-client = { version = "0.8", optional = true } +bitcoin_slices = { version = "0.8.0", features = ["slice_cache"] } [dev-dependencies] diff --git a/src/config.rs b/src/config.rs index 8696ecf8f..60720f384 100644 --- a/src/config.rs +++ b/src/config.rs @@ -41,6 +41,15 @@ pub struct Config { pub electrum_banner: String, pub electrum_rpc_logging: Option, + /// Tx cache size in megabytes + pub tx_cache_size: usize, + + /// Enable compaction during initial sync + /// + /// By default compaction is off until initial sync is finished for performance reasons, + /// however, this requires much more disk space. + pub initial_sync_compaction: bool, + #[cfg(feature = "liquid")] pub parent_network: BNetwork, #[cfg(feature = "liquid")] @@ -191,6 +200,15 @@ impl Config { .long("electrum-rpc-logging") .help(&rpc_logging_help) .takes_value(true), + ).arg( + Arg::with_name("tx_cache_size") + .long("tx-cache-size") + .help("The amount of MB for a in-memory cache for transactions.") + .default_value("1000") + ).arg( + Arg::with_name("initial_sync_compaction") + .long("initial-sync-compaction") + .help("Perform compaction during initial sync (slower but less disk space required)") ); #[cfg(unix)] @@ -403,6 +421,8 @@ impl Config { index_unspendables: m.is_present("index_unspendables"), cors: m.value_of("cors").map(|s| s.to_string()), precache_scripts: m.value_of("precache_scripts").map(|s| s.to_string()), + tx_cache_size: value_t_or_exit!(m, "tx_cache_size", usize), + initial_sync_compaction: m.is_present("initial_sync_compaction"), #[cfg(feature = "liquid")] parent_network, diff --git a/src/daemon.rs b/src/daemon.rs index 457bf4230..133896c66 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -424,7 +424,7 @@ impl Daemon { loop { match self.handle_request_batch(method, params_list) { Err(Error(ErrorKind::Connection(msg), _)) => { - warn!("reconnecting to bitcoind: {}", msg); + warn!("reconnecting to bitcoind: {msg}\nmethod was:{method}\nwith_params:{params_list:?}"); self.signal.wait(Duration::from_secs(3), false)?; let mut conn = self.conn.lock().unwrap(); *conn = conn.reconnect()?; diff --git a/src/new_index/db.rs b/src/new_index/db.rs index f68da233c..b6617d425 100644 --- a/src/new_index/db.rs +++ b/src/new_index/db.rs @@ -90,7 +90,7 @@ impl DB { db_opts.set_compression_type(rocksdb::DBCompressionType::Snappy); db_opts.set_target_file_size_base(1_073_741_824); db_opts.set_write_buffer_size(256 << 20); - db_opts.set_disable_auto_compactions(true); // for initial bulk load + db_opts.set_disable_auto_compactions(!config.initial_sync_compaction); // for initial bulk load // db_opts.set_advise_random_on_open(???); db_opts.set_compaction_readahead_size(1 << 20); @@ -154,7 +154,7 @@ impl DB { } pub fn write(&self, mut rows: Vec, flush: DBFlush) { - debug!( + log::trace!( "writing {} rows to {:?}, flush={:?}", rows.len(), self.db, diff --git a/src/new_index/fetch.rs b/src/new_index/fetch.rs index 11843f5d7..d7637ee5c 100644 --- a/src/new_index/fetch.rs +++ b/src/new_index/fetch.rs @@ -99,6 +99,7 @@ fn bitcoind_fetcher( sender .send(block_entries) .expect("failed to send fetched blocks"); + log::debug!("last fetch {:?}", entries.last()); } }), )) diff --git a/src/new_index/mempool.rs b/src/new_index/mempool.rs index 94ba7a41d..50d55d18f 100644 --- a/src/new_index/mempool.rs +++ b/src/new_index/mempool.rs @@ -309,6 +309,18 @@ impl Mempool { txids.push(txid); self.txstore.insert(txid, tx); } + + // Populate tx cache + let txid_misses = self.chain.txs_cache_miss(&txids); + let mut tx_misses = vec![]; + for txid in txid_misses { + if let Some(tx) = self.txstore.get(&txid) { + let bytes = serialize(tx); + tx_misses.push((txid, bytes)); + } + } + self.chain.add_txs_to_cache(&tx_misses); + // Phase 2: index history and spend edges (can fail if some txos cannot be found) let txos = match self.lookup_txos(self.get_prevouts(&txids)) { Ok(txos) => txos, diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index 1ea2a97ef..4612f64d4 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -2,10 +2,12 @@ use bitcoin::hashes::sha256d::Hash as Sha256dHash; #[cfg(not(feature = "liquid"))] use bitcoin::merkle_tree::MerkleBlock; use bitcoin::VarInt; +use bitcoin_slices::SliceCache; use crypto::digest::Digest; use crypto::sha2::Sha256; use hex::FromHex; use itertools::Itertools; +use prometheus::IntCounter; use rayon::prelude::*; #[cfg(not(feature = "liquid"))] @@ -17,9 +19,12 @@ use elements::{ AssetId, }; -use std::collections::{BTreeSet, HashMap, HashSet}; use std::path::Path; use std::sync::{Arc, RwLock}; +use std::{ + collections::{BTreeSet, HashMap, HashSet}, + sync::Mutex, +}; use crate::chain::{ BlockHash, BlockHeader, Network, OutPoint, Script, Transaction, TxOut, Txid, Value, @@ -198,6 +203,11 @@ pub struct ChainQuery { light_mode: bool, duration: HistogramVec, network: Network, + + /// By default 1GB of cached transactions, `Txid -> Transaction` + txs_cache: Mutex>, + cache_hit: IntCounter, + cache_miss: IntCounter, } // TODO: &[Block] should be an iterator / a queue. @@ -360,6 +370,9 @@ impl ChainQuery { HistogramOpts::new("query_duration", "Index query duration (in seconds)"), &["name"], ), + txs_cache: Mutex::new(SliceCache::new(config.tx_cache_size << 20)), + cache_hit: metrics.counter(MetricOpts::new("tx_cache_hit", "Tx cache Hit")), + cache_miss: metrics.counter(MetricOpts::new("tx_cache_miss", "Tx cache Miss")), } } @@ -838,7 +851,16 @@ impl ChainQuery { pub fn lookup_raw_txn(&self, txid: &Txid, blockhash: Option<&BlockHash>) -> Option { let _timer = self.start_timer("lookup_raw_txn"); - if self.light_mode { + if let Ok(cache) = self.txs_cache.lock() { + if let Some(bytes) = cache.get(txid) { + self.cache_hit.inc(); + return Some(bytes.to_vec()); + } else { + self.cache_miss.inc(); + } + } + + let result = if self.light_mode { let queried_blockhash = blockhash.map_or_else(|| self.tx_confirming_block(txid).map(|b| b.hash), |_| None); let blockhash = blockhash.or_else(|| queried_blockhash.as_ref())?; @@ -848,9 +870,35 @@ impl ChainQuery { .gettransaction_raw(txid, blockhash, false) .ok()?; let txhex = txval.as_str().expect("valid tx from bitcoind"); - Some(Bytes::from_hex(txhex).expect("valid tx from bitcoind")) + let vec = Bytes::from_hex(txhex).expect("valid tx from bitcoind"); + + Some(vec) } else { self.store.txstore_db.get(&TxRow::key(&txid[..])) + }; + if let Some(result) = result.as_ref() { + self.add_txs_to_cache(&[(*txid, result)]); + } + result + } + + pub fn txs_cache_miss(&self, txids: &[Txid]) -> Vec { + let mut result = vec![]; + if let Ok(cache) = self.txs_cache.lock() { + for txid in txids { + if !cache.contains(txid) { + result.push(*txid); + } + } + } + result + } + + pub fn add_txs_to_cache>(&self, txs: &[(Txid, T)]) { + if let Ok(mut cache) = self.txs_cache.lock() { + for (txid, tx) in txs { + let _ = cache.insert(*txid, &tx.as_ref()); + } } } diff --git a/tests/common.rs b/tests/common.rs index 23c7ce1e2..32026da7d 100644 --- a/tests/common.rs +++ b/tests/common.rs @@ -113,6 +113,8 @@ impl TestRunner { asset_db_path: None, // XXX #[cfg(feature = "liquid")] parent_network: bitcoin::Network::Regtest, + tx_cache_size: 100, + initial_sync_compaction: false, //#[cfg(feature = "electrum-discovery")] //electrum_public_hosts: Option, //#[cfg(feature = "electrum-discovery")]