From 085872d5038f9a8b8c70a9a958c55d9eb25d0c67 Mon Sep 17 00:00:00 2001 From: Daniel Savu Date: Thu, 29 Oct 2020 02:45:37 +0200 Subject: [PATCH 01/47] Add index-unspendables CLI flag (#28) --- README.md | 1 + doc/schema.md | 2 +- src/config.rs | 7 +++++++ src/new_index/mempool.rs | 4 +++- src/new_index/schema.rs | 4 +++- 5 files changed, 15 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 71a1dd57e..5cf685fff 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,7 @@ In addition to electrs's original configuration options, a few new options are a - `--lightmode` - enable light mode (see above) - `--cors ` - origins allowed to make cross-site request (optional, defaults to none). - `--address-search` - enables the by-prefix address search index. +- `--index-unspendables` - enables indexing of provably unspendable outputs. - `--utxos-limit ` - maximum number of utxos to return per address. - `--electrum-txs-limit ` - maximum number of txs to return per address in the electrum server (does not apply for the http api). - `--electrum-banner ` - welcome banner text for electrum server. diff --git a/doc/schema.md b/doc/schema.md index 851c67544..4875cb4df 100644 --- a/doc/schema.md +++ b/doc/schema.md @@ -43,7 +43,7 @@ When the indexer is synced up to the tip of the chain, the hash of the tip is sa ### `history` -Each funding output (except for provably unspendable ones) results in the following new rows (`H` is for history, `F` is for funding): +Each funding output (except for provably unspendable ones when `--index-unspendables` is not enabled) results in the following new rows (`H` is for history, `F` is for funding): * `"H{funding-scripthash}{funding-height}F{funding-txid:vout}{value}" → ""` * `"a{funding-address-str}" → ""` (for prefix address search, only saved when `--address-search` is enabled) diff --git a/src/config.rs b/src/config.rs index 48adaf05a..189ac5579 100644 --- a/src/config.rs +++ b/src/config.rs @@ -31,6 +31,7 @@ pub struct Config { pub jsonrpc_import: bool, pub light_mode: bool, pub address_search: bool, + pub index_unspendables: bool, pub cors: Option, pub precache_scripts: Option, pub utxos_limit: usize, @@ -148,6 +149,11 @@ impl Config { .long("address-search") .help("Enable prefix address search") ) + .arg( + Arg::with_name("index_unspendables") + .long("index-unspendables") + .help("Enable indexing of provably unspendable outputs") + ) .arg( Arg::with_name("cors") .long("cors") @@ -362,6 +368,7 @@ impl Config { jsonrpc_import: m.is_present("jsonrpc_import"), light_mode: m.is_present("light_mode"), address_search: m.is_present("address_search"), + index_unspendables: m.is_present("index_unspendables"), cors: m.value_of("cors").map(|s| s.to_string()), precache_scripts: m.value_of("precache_scripts").map(|s| s.to_string()), diff --git a/src/new_index/mempool.rs b/src/new_index/mempool.rs index 16a771ce7..cf67790bf 100644 --- a/src/new_index/mempool.rs +++ b/src/new_index/mempool.rs @@ -378,12 +378,14 @@ impl Mempool { ) }); + let config = &self.config; + // An iterator over (ScriptHash, TxHistoryInfo) let funding = tx .output .iter() .enumerate() - .filter(|(_, txo)| is_spendable(txo)) + .filter(|(_, txo)| is_spendable(txo) || config.index_unspendables) .map(|(index, txo)| { ( compute_script_hash(&txo.script_pubkey), diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index facc9f363..bb85bda00 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -169,6 +169,7 @@ pub struct Indexer { struct IndexerConfig { light_mode: bool, address_search: bool, + index_unspendables: bool, network: Network, #[cfg(feature = "liquid")] parent_network: Network, @@ -179,6 +180,7 @@ impl From<&Config> for IndexerConfig { IndexerConfig { light_mode: config.light_mode, address_search: config.address_search, + index_unspendables: config.index_unspendables, network: config.network_type, #[cfg(feature = "liquid")] parent_network: config.parent_network, @@ -1072,7 +1074,7 @@ fn index_transaction( // S{funding-txid:vout}{spending-txid:vin} → "" let txid = full_hash(&tx.txid()[..]); for (txo_index, txo) in tx.output.iter().enumerate() { - if is_spendable(txo) { + if is_spendable(txo) || iconfig.index_unspendables { let history = TxHistoryRow::new( &txo.script_pubkey, confirmed_height, From 91076cff8bef143922abe455031ff80424de4d9c Mon Sep 17 00:00:00 2001 From: Nadav Ivgi Date: Wed, 4 Nov 2020 02:17:54 +0200 Subject: [PATCH 02/47] http: Implement GET /block/:hash/header --- src/new_index/schema.rs | 5 +++++ src/rest.rs | 12 +++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index bb85bda00..a7a16e818 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -420,6 +420,11 @@ impl ChainQuery { } } + pub fn get_block_header(&self, hash: &BlockHash) -> Option { + let _timer = self.start_timer("get_block_header"); + Some(*self.header_by_hash(hash)?.header()) + } + pub fn get_block_with_meta(&self, hash: &BlockHash) -> Option { let _timer = self.start_timer("get_block_with_meta"); Some(BlockHeaderMeta { diff --git a/src/rest.rs b/src/rest.rs index fee6aba8e..ef653b305 100644 --- a/src/rest.rs +++ b/src/rest.rs @@ -673,6 +673,16 @@ fn handle_request( .ok_or_else(|| HttpError::not_found("Block not found".to_string()))?; json_response(txids, TTL_LONG) } + (&Method::GET, Some(&"block"), Some(hash), Some(&"header"), None, None) => { + let hash = BlockHash::from_hex(hash)?; + let header = query + .chain() + .get_block_header(&hash) + .ok_or_else(|| HttpError::not_found("Block not found".to_string()))?; + + let header_hex = hex::encode(encode::serialize(&header)); + http_message(StatusCode::OK, header_hex, TTL_LONG) + } (&Method::GET, Some(&"block"), Some(hash), Some(&"raw"), None, None) => { let hash = BlockHash::from_hex(hash)?; let raw = query @@ -718,7 +728,7 @@ fn handle_request( ))); } - // header_by_hash() only returns the BlockId for non-orphaned blocks, + // blockid_by_hash() only returns the BlockId for non-orphaned blocks, // or None for orphaned let confirmed_blockid = query.chain().blockid_by_hash(&hash); From 78adcdb3504c61055aae8680220d680279a5ca0e Mon Sep 17 00:00:00 2001 From: Nadav Ivgi Date: Thu, 5 Nov 2020 00:09:26 +0200 Subject: [PATCH 03/47] Add median time past to blocks json --- src/new_index/schema.rs | 9 ++++++++- src/rest.rs | 2 ++ src/util/block.rs | 20 ++++++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index a7a16e818..707f15816 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -425,11 +425,18 @@ impl ChainQuery { Some(*self.header_by_hash(hash)?.header()) } + pub fn get_mtp(&self, height: usize) -> u32 { + let _timer = self.start_timer("get_block_mtp"); + self.store.indexed_headers.read().unwrap().get_mtp(height) + } + pub fn get_block_with_meta(&self, hash: &BlockHash) -> Option { let _timer = self.start_timer("get_block_with_meta"); + let header_entry = self.header_by_hash(hash)?; Some(BlockHeaderMeta { - header_entry: self.header_by_hash(hash)?, meta: self.get_block_meta(hash)?, + mtp: self.get_mtp(header_entry.height()), + header_entry, }) } diff --git a/src/rest.rs b/src/rest.rs index ef653b305..74df2fadd 100644 --- a/src/rest.rs +++ b/src/rest.rs @@ -65,6 +65,7 @@ struct BlockValue { weight: u32, merkle_root: String, previousblockhash: Option, + mediantime: u32, #[cfg(not(feature = "liquid"))] nonce: u32, @@ -96,6 +97,7 @@ impl BlockValue { } else { None }, + mediantime: blockhm.mtp, #[cfg(not(feature = "liquid"))] bits: header.bits, diff --git a/src/util/block.rs b/src/util/block.rs index 112ef4e2b..1bb7cd294 100644 --- a/src/util/block.rs +++ b/src/util/block.rs @@ -10,6 +10,8 @@ use std::iter::FromIterator; use std::slice; use time::OffsetDateTime as DateTime; +const MTP_SPAN: usize = 11; + #[derive(Debug, Serialize, Deserialize, Clone)] pub struct BlockId { pub height: usize, @@ -231,6 +233,23 @@ impl HeaderList { pub fn iter(&self) -> slice::Iter { self.headers.iter() } + + /// Get the Median Time Past + pub fn get_mtp(&self, height: usize) -> u32 { + // Use the timestamp as the mtp of the genesis block. + // Matches bitcoind's behaviour: bitcoin-cli getblock `bitcoin-cli getblockhash 0` | jq '.time == .mediantime' + if height == 0 { + self.headers.get(0).unwrap().header.time + } else if height > self.len() - 1 { + 0 + } else { + let mut timestamps = (height.saturating_sub(MTP_SPAN - 1)..=height) + .map(|p_height| self.headers.get(p_height).unwrap().header.time) + .collect::>(); + timestamps.sort_unstable(); + timestamps[timestamps.len() / 2] + } + } } #[derive(Serialize, Deserialize)] @@ -269,6 +288,7 @@ pub struct BlockMeta { pub struct BlockHeaderMeta { pub header_entry: HeaderEntry, pub meta: BlockMeta, + pub mtp: u32, } impl From<&BlockEntry> for BlockMeta { From 627b12467cec777b7935f3398b87e79ffa678619 Mon Sep 17 00:00:00 2001 From: Nadav Ivgi Date: Sat, 7 Nov 2020 01:30:13 +0200 Subject: [PATCH 04/47] Make asset sorting case insensitive The domain name doesn't need lower-casing because its guaranteed to be in all lowercase. --- src/elements/registry.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/elements/registry.rs b/src/elements/registry.rs index 50507054c..3d1bba43d 100644 --- a/src/elements/registry.rs +++ b/src/elements/registry.rs @@ -141,10 +141,10 @@ impl AssetSorting { AssetSortField::Name => { // Order by name first, use asset id as a tie breaker. the other sorting fields // don't require this because they're guaranteed to be unique. - Box::new(|a, b| a.1.name.cmp(&b.1.name).then_with(|| a.0.cmp(b.0))) + Box::new(|a, b| lc_cmp(&a.1.name, &b.1.name).then_with(|| a.0.cmp(b.0))) } AssetSortField::Domain => Box::new(|a, b| a.1.domain().cmp(&b.1.domain())), - AssetSortField::Ticker => Box::new(|a, b| a.1.ticker.cmp(&b.1.ticker)), + AssetSortField::Ticker => Box::new(|a, b| lc_cmp(&a.1.ticker, &b.1.ticker)), }; match self.1 { @@ -172,3 +172,7 @@ impl AssetSorting { Ok(Self(field, dir)) } } + +fn lc_cmp(a: &str, b: &str) -> cmp::Ordering { + a.to_lowercase().cmp(&b.to_lowercase()) +} From 1e69fa6376037deab19cb6fc6254f280c57bafc7 Mon Sep 17 00:00:00 2001 From: Nadav Ivgi Date: Sat, 7 Nov 2020 06:03:53 +0200 Subject: [PATCH 05/47] Fix get_fee_histogram https://github.com/romanz/electrs/commit/c88a0dc331eb16163276becf98fcc020565d97eb --- src/util/fees.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/util/fees.rs b/src/util/fees.rs index ae9166265..1e28ac784 100644 --- a/src/util/fees.rs +++ b/src/util/fees.rs @@ -43,18 +43,18 @@ pub fn make_fee_histogram(mut entries: Vec<&TxFeeInfo>) -> Vec<(f32, u32)> { let mut histogram = vec![]; let mut bin_size = 0; - let mut last_fee_rate = None; + let mut last_fee_rate = 0.0; for e in entries.iter().rev() { - bin_size += e.vsize; - if bin_size > VSIZE_BIN_WIDTH && last_fee_rate.map_or(true, |last| e.fee_per_vbyte < last) { - // vsize of transactions paying >= e.fee_per_vbyte - histogram.push((e.fee_per_vbyte, bin_size)); + if bin_size > VSIZE_BIN_WIDTH && last_fee_rate != e.fee_per_vbyte { + // vsize of transactions paying >= last_fee_rate + histogram.push((last_fee_rate, bin_size)); bin_size = 0; } - last_fee_rate = Some(e.fee_per_vbyte); + last_fee_rate = e.fee_per_vbyte; + bin_size += e.vsize; } - if let Some(fee_rate) = last_fee_rate { - histogram.push((fee_rate, bin_size)); + if bin_size > 0 { + histogram.push((last_fee_rate, bin_size)); } histogram } From 641a99f1871dd42481c7cca684ffbedfb6bd3873 Mon Sep 17 00:00:00 2001 From: Nadav Ivgi Date: Thu, 12 Nov 2020 14:57:38 +0200 Subject: [PATCH 06/47] Fix ordering by asset ticker --- src/elements/registry.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/elements/registry.rs b/src/elements/registry.rs index 3d1bba43d..8554d9e13 100644 --- a/src/elements/registry.rs +++ b/src/elements/registry.rs @@ -144,7 +144,7 @@ impl AssetSorting { Box::new(|a, b| lc_cmp(&a.1.name, &b.1.name).then_with(|| a.0.cmp(b.0))) } AssetSortField::Domain => Box::new(|a, b| a.1.domain().cmp(&b.1.domain())), - AssetSortField::Ticker => Box::new(|a, b| lc_cmp(&a.1.ticker, &b.1.ticker)), + AssetSortField::Ticker => Box::new(|a, b| lc_cmp_opt(&a.1.ticker, &b.1.ticker)), }; match self.1 { @@ -176,3 +176,8 @@ impl AssetSorting { fn lc_cmp(a: &str, b: &str) -> cmp::Ordering { a.to_lowercase().cmp(&b.to_lowercase()) } +fn lc_cmp_opt(a: &Option, b: &Option) -> cmp::Ordering { + a.as_ref() + .map(|a| a.to_lowercase()) + .cmp(&b.as_ref().map(|b| b.to_lowercase())) +} From 85e0d9f2600675d1210bc954f87d1473e545ad71 Mon Sep 17 00:00:00 2001 From: Nadav Ivgi Date: Thu, 12 Nov 2020 15:02:33 +0200 Subject: [PATCH 07/47] Fix get_block_header() in liquid mode Liquid's BlockHeaders are not Copyable --- src/new_index/schema.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index 707f15816..bee8273ba 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -422,7 +422,7 @@ impl ChainQuery { pub fn get_block_header(&self, hash: &BlockHash) -> Option { let _timer = self.start_timer("get_block_header"); - Some(*self.header_by_hash(hash)?.header()) + Some(self.header_by_hash(hash)?.header().clone()) } pub fn get_mtp(&self, height: usize) -> u32 { From 1cbc86fbfdb5680fea752377bf8f18ed417f0edd Mon Sep 17 00:00:00 2001 From: Nadav Ivgi Date: Fri, 13 Nov 2020 21:15:44 +0200 Subject: [PATCH 08/47] Return the total number of available registered assets As the `X-Total-Results` header, in reply to `GET /assets/registry` --- src/elements/registry.rs | 12 ++++++++++-- src/new_index/query.rs | 20 ++++++++++---------- src/rest.rs | 10 ++++++++-- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/elements/registry.rs b/src/elements/registry.rs index 8554d9e13..8c1fc6969 100644 --- a/src/elements/registry.rs +++ b/src/elements/registry.rs @@ -35,14 +35,22 @@ impl AssetRegistry { .map(|(_, metadata)| metadata) } - pub fn list(&self, start_index: usize, limit: usize, sorting: AssetSorting) -> Vec { + pub fn list( + &self, + start_index: usize, + limit: usize, + sorting: AssetSorting, + ) -> (usize, Vec) { let mut assets: Vec = self .assets_cache .iter() .map(|(asset_id, (_, metadata))| (asset_id, metadata)) .collect(); assets.sort_by(sorting.as_comparator()); - assets.into_iter().skip(start_index).take(limit).collect() + ( + assets.len(), + assets.into_iter().skip(start_index).take(limit).collect(), + ) } pub fn fs_sync(&mut self) -> Result<()> { diff --git a/src/new_index/query.rs b/src/new_index/query.rs index fa1ddf6ef..c7211af43 100644 --- a/src/new_index/query.rs +++ b/src/new_index/query.rs @@ -245,20 +245,20 @@ impl Query { start_index: usize, limit: usize, sorting: AssetSorting, - ) -> Result> { + ) -> Result<(usize, Vec)> { let asset_db = match &self.asset_db { - None => return Ok(vec![]), + None => return Ok((0, vec![])), Some(db) => db.read().unwrap(), }; - Ok(asset_db - .list(start_index, limit, sorting) + let (total_num, results) = asset_db.list(start_index, limit, sorting); + // Attach on-chain information alongside the registry metadata + let results = results .into_iter() - .filter_map(|(asset_id, metadata)| { - // Attach on-chain information alongside the registry metadata - lookup_asset(&self, None, asset_id, Some(metadata)) - .ok() - .flatten() + .map(|(asset_id, metadata)| { + Ok(lookup_asset(&self, None, asset_id, Some(metadata))? + .chain_err(|| "missing registered asset")?) }) - .collect()) + .collect::>>()?; + Ok((total_num, results)) } } diff --git a/src/rest.rs b/src/rest.rs index 74df2fadd..00a971df8 100644 --- a/src/rest.rs +++ b/src/rest.rs @@ -1047,9 +1047,15 @@ fn handle_request( let sorting = AssetSorting::from_query_params(&query_params)?; - let assets = query.list_registry_assets(start_index, limit, sorting)?; + let (total_num, assets) = query.list_registry_assets(start_index, limit, sorting)?; - json_response(assets, TTL_SHORT) + Ok(Response::builder() + // Disable caching because we don't currently support caching with query string params + .header("Cache-Control", "no-store") + .header("Content-Type", "application/json") + .header("X-Total-Results", total_num.to_string()) + .body(Body::from(serde_json::to_string(&assets)?)) + .unwrap()) } #[cfg(feature = "liquid")] From 773291f3684924a9ec1de27364bb520fd39655c2 Mon Sep 17 00:00:00 2001 From: Nadav Ivgi Date: Mon, 30 Nov 2020 07:02:36 +0200 Subject: [PATCH 09/47] Can derive Default --- src/elements/asset.rs | 32 ++------------------------------ 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/src/elements/asset.rs b/src/elements/asset.rs index 188e939f1..a5717fa03 100644 --- a/src/elements/asset.rs +++ b/src/elements/asset.rs @@ -408,7 +408,7 @@ pub fn get_issuance_entropy(txin: &TxIn) -> Result { // Asset stats // -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Default)] pub struct IssuedAssetStats { pub tx_count: usize, pub issuance_count: usize, @@ -419,21 +419,7 @@ pub struct IssuedAssetStats { pub burned_reissuance_tokens: u64, } -impl Default for IssuedAssetStats { - fn default() -> Self { - Self { - tx_count: 0, - issuance_count: 0, - issued_amount: 0, - burned_amount: 0, - has_blinded_issuances: false, - reissuance_tokens: None, - burned_reissuance_tokens: 0, - } - } -} - -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Default)] pub struct NativeAssetStats { pub tx_count: usize, pub peg_in_count: usize, @@ -444,20 +430,6 @@ pub struct NativeAssetStats { pub burned_amount: u64, } -impl Default for NativeAssetStats { - fn default() -> Self { - Self { - tx_count: 0, - peg_in_count: 0, - peg_in_amount: 0, - peg_out_count: 0, - peg_out_amount: 0, - burn_count: 0, - burned_amount: 0, - } - } -} - type AssetStatApplyFn = fn(&TxHistoryInfo, &mut T, &mut HashSet); fn asset_cache_key(asset_id: &AssetId) -> Bytes { From 65e20f9e1196c0039d56307b34ea7bcfbe375b99 Mon Sep 17 00:00:00 2001 From: Nadav Ivgi Date: Sat, 19 Dec 2020 19:31:48 +0200 Subject: [PATCH 10/47] Update crate dependencies tokio and hyper were not upgraded because hyperlocal doesn't support their latest versions. --- Cargo.lock | 298 +++++++++++++++++++++++++++++------------------------ Cargo.toml | 34 +++--- 2 files changed, 183 insertions(+), 149 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6d0aadac5..99031f2db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -87,7 +87,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46254cf2fdcdf1badb5934448c1bcbe046a56537b3987d96c51a7afc5d03f293" dependencies = [ "addr2line", - "cfg-if", + "cfg-if 0.1.10", "libc", "miniz_oxide", "object", @@ -117,9 +117,9 @@ checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" [[package]] name = "base64" -version = "0.12.3" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" [[package]] name = "bech32" @@ -139,17 +139,17 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.53.3" +version = "0.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c72a978d268b1d70b0e963217e60fdabd9523a941457a6c42a7315d15c7e89e5" +checksum = "66c0bb6167449588ff70803f4127f0684f9063097eca5016f37eb52b92c2cf36" dependencies = [ "bitflags", "cexpr", - "cfg-if", + "cfg-if 0.1.10", "clang-sys", "clap", "env_logger", - "lazy_static 1.4.0", + "lazy_static", "lazycell", "log", "peeking_take_while", @@ -244,6 +244,12 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + [[package]] name = "chrono" version = "0.4.13" @@ -274,9 +280,9 @@ dependencies = [ [[package]] name = "clap" -version = "2.33.1" +version = "2.33.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdfa80d47f954d53a35a64987ca1422f495b8d6483c0fe9f7117b36c2a792129" +checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" dependencies = [ "ansi_term", "atty", @@ -287,6 +293,12 @@ dependencies = [ "vec_map", ] +[[package]] +name = "const_fn" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd51eab21ab4fd6a3bf889e2d0958c0a6e3a61ad04260325e919e652a2a62826" + [[package]] name = "constant_time_eq" version = "0.1.5" @@ -295,60 +307,59 @@ checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" [[package]] name = "crossbeam-channel" -version = "0.4.3" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ee0cc8804d5393478d743b035099520087a5186f3b93fa58cec08fa62407b6" +checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775" dependencies = [ - "cfg-if", - "crossbeam-utils", + "cfg-if 1.0.0", + "crossbeam-utils 0.8.1", ] [[package]] name = "crossbeam-deque" -version = "0.7.3" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f02af974daeee82218205558e51ec8768b48cf524bd01d550abe5573a608285" +checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" dependencies = [ + "cfg-if 1.0.0", "crossbeam-epoch", - "crossbeam-utils", - "maybe-uninit", + "crossbeam-utils 0.8.1", ] [[package]] name = "crossbeam-epoch" -version = "0.8.2" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" +checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d" dependencies = [ - "autocfg", - "cfg-if", - "crossbeam-utils", - "lazy_static 1.4.0", - "maybe-uninit", + "cfg-if 1.0.0", + "const_fn", + "crossbeam-utils 0.8.1", + "lazy_static", "memoffset", "scopeguard", ] [[package]] -name = "crossbeam-queue" -version = "0.2.3" +name = "crossbeam-utils" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" dependencies = [ - "cfg-if", - "crossbeam-utils", - "maybe-uninit", + "autocfg", + "cfg-if 0.1.10", + "lazy_static", ] [[package]] name = "crossbeam-utils" -version = "0.7.2" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d" dependencies = [ "autocfg", - "cfg-if", - "lazy_static 1.4.0", + "cfg-if 1.0.0", + "lazy_static", ] [[package]] @@ -389,7 +400,7 @@ version = "0.4.1" dependencies = [ "arraydeque", "arrayref", - "base64 0.12.3", + "base64 0.13.0", "bincode", "bitcoin", "clap", @@ -403,7 +414,7 @@ dependencies = [ "hyper", "hyperlocal", "itertools", - "lazy_static 1.4.0", + "lazy_static", "libc", "log", "num_cpus", @@ -411,7 +422,7 @@ dependencies = [ "prometheus", "rayon", "rocksdb 0.12.4", - "rocksdb 0.14.0", + "rocksdb 0.15.0", "rust-crypto", "serde", "serde_derive", @@ -421,7 +432,7 @@ dependencies = [ "stderrlog", "sysconf", "tempfile", - "time 0.2.16", + "time 0.2.23", "tiny_http", "tokio", "url", @@ -488,9 +499,9 @@ dependencies = [ [[package]] name = "error-chain" -version = "0.12.2" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d371106cc88ffdfb1eabd7111e432da544f16f3e2d7bf1dfe8bf575f1df045cd" +checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc" dependencies = [ "backtrace", "version_check", @@ -502,6 +513,16 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "form_urlencoded" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece68d15c92e84fa4f19d3780f1294e5ca82a78a6d515f1efaabcc144688be00" +dependencies = [ + "matches", + "percent-encoding", +] + [[package]] name = "fuchsia-cprng" version = "0.1.1" @@ -607,7 +628,7 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "libc", "wasi", ] @@ -757,6 +778,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "instant" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "iovec" version = "0.1.4" @@ -800,12 +830,6 @@ dependencies = [ "winapi-build", ] -[[package]] -name = "lazy_static" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" - [[package]] name = "lazy_static" version = "1.4.0" @@ -820,9 +844,9 @@ checksum = "b294d6fa9ee409a054354afc4352b0b9ef7ca222c69b8812cbea9e7d2bf3783f" [[package]] name = "libc" -version = "0.2.71" +version = "0.2.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9457b06509d27052635f90d6466700c65095fdf75409b3fbdd903e988b886f49" +checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" [[package]] name = "libloading" @@ -836,9 +860,9 @@ dependencies = [ [[package]] name = "librocksdb-sys" -version = "6.7.4" +version = "6.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "883213ae3d09bfc3d104aefe94b25ebb183b6f4d3a515b23b14817e1f4854005" +checksum = "eb5b56f651c204634b936be2f92dbb42c36867e00ff7fe2405591f3b9fa66f09" dependencies = [ "bindgen", "cc", @@ -846,13 +870,22 @@ dependencies = [ "libc", ] +[[package]] +name = "lock_api" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" -version = "0.4.8" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" +checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", ] [[package]] @@ -861,7 +894,7 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ecc775857611e1df29abba5c41355cdf540e7e9d4acfdf0f355eefee82330b7" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "generator", "scoped-tls", ] @@ -872,12 +905,6 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" -[[package]] -name = "maybe-uninit" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" - [[package]] name = "memchr" version = "2.3.3" @@ -886,9 +913,9 @@ checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" [[package]] name = "memoffset" -version = "0.5.5" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c198b026e1bbf08a937e94c6c60f9ec4a2267f5b0d2eec9c1b21b061ce2be55f" +checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87" dependencies = [ "autocfg", ] @@ -908,7 +935,7 @@ version = "0.6.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fce347092656428bc8eaf6201042cb551b8d67855af7374542a92a0fbfcac430" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "fuchsia-zircon", "fuchsia-zircon-sys", "iovec", @@ -950,7 +977,7 @@ version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ba7c918ac76704fb42afcbbb43891e72731f3dcca3bef2a19786297baf14af7" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "libc", "winapi 0.3.9", ] @@ -1022,6 +1049,31 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "parking_lot" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c6d9b8427445284a09c55be860a15855ab580a417ccad9da88f5a06787ced0" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi 0.3.9", +] + [[package]] name = "peeking_take_while" version = "0.1.2" @@ -1095,15 +1147,16 @@ dependencies = [ [[package]] name = "prometheus" -version = "0.9.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0ced56dee39a6e960c15c74dc48849d614586db2eaada6497477af7c7811cd" +checksum = "c8425533e7122f0c3cc7a37e6244b16ad3a2cc32ae7ac6276e2a75da0d9c200d" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "fnv", - "lazy_static 1.4.0", + "lazy_static", + "parking_lot", "protobuf", - "spin", + "regex", "thiserror", ] @@ -1209,9 +1262,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.3.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62f02856753d04e03e26929f820d0a0a337ebe71f849801eea335d464b349080" +checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" dependencies = [ "autocfg", "crossbeam-deque", @@ -1221,14 +1274,14 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.7.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e92e15d89083484e11353891f1af602cc661426deb9564c298b270c726973280" +checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" dependencies = [ + "crossbeam-channel", "crossbeam-deque", - "crossbeam-queue", - "crossbeam-utils", - "lazy_static 1.4.0", + "crossbeam-utils 0.8.1", + "lazy_static", "num_cpus", ] @@ -1276,7 +1329,7 @@ dependencies = [ "aho-corasick", "memchr", "regex-syntax", - "thread_local 1.0.1", + "thread_local", ] [[package]] @@ -1301,7 +1354,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ba5a8ec64ee89a76c98c549af81ff14813df09c3e6dc4766c3856da48597a0c" dependencies = [ "cc", - "lazy_static 1.4.0", + "lazy_static", "libc", "spin", "untrusted", @@ -1321,9 +1374,9 @@ dependencies = [ [[package]] name = "rocksdb" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61aa17a99a2413cd71c1106691bf59dad7de0cd5099127f90e9d99c429c40d4a" +checksum = "23d83c02c429044d58474eaf5ae31e062d0de894e21125b47437ec0edc1397e6" dependencies = [ "libc", "librocksdb-sys", @@ -1338,7 +1391,7 @@ dependencies = [ "base64 0.11.0", "blake2b_simd", "constant_time_eq", - "crossbeam-utils", + "crossbeam-utils 0.7.2", ] [[package]] @@ -1458,18 +1511,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.114" +version = "1.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5317f7588f0a5078ee60ef675ef96735a1442132dc645eb1d12c018620ed8cd3" +checksum = "06c64263859d87aa2eb554587e2d23183398d617427327cf2b3d0ed8c69e4800" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.114" +version = "1.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0be94b04690fbaed37cddffc5c134bf537c8e3329d53e982fe04c374978f8e" +checksum = "c84d3526699cd55261af4b941e4e725444df67aa4f9e6a3564f18030d12672df" dependencies = [ "proc-macro2", "quote", @@ -1478,9 +1531,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.56" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3433e879a558dde8b5e8feb2a04899cf34fdde1fafb894687e52105fc1162ac3" +checksum = "1500e84d27fe482ed1dc791a56eddc2f230046a040fa908c08bda1d9fb615779" dependencies = [ "itoa", "ryu", @@ -1501,9 +1554,9 @@ checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" [[package]] name = "signal-hook" -version = "0.1.16" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "604508c1418b99dfe1925ca9224829bb2a8a9a04dda655cc01fcad46f4ab05ed" +checksum = "8133fd06d2c721d4168f9b76a9a7fd3a0bfc96df58cf7316c7fb9f23bd677f4e" dependencies = [ "libc", "signal-hook-registry", @@ -1525,15 +1578,20 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +[[package]] +name = "smallvec" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae524f056d7d770e174287294f562e95044c68e88dec909a00d2094805db9d75" + [[package]] name = "socket2" -version = "0.3.12" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03088793f677dce356f3ccc2edb1b314ad191ab702a5de3faf49304f7e104918" +checksum = "97e0e9fd577458a4f61fb91fcb559ea2afecc54c934119421f9f5d3d5b1a1057" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", - "redox_syscall", "winapi 0.3.9", ] @@ -1566,15 +1624,15 @@ dependencies = [ [[package]] name = "stderrlog" -version = "0.4.3" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32e5ee9b90a5452c570a0b0ac1c99ae9498db7e56e33d74366de7f2a7add7f25" +checksum = "b02f316286ae558d83acc93dd81eaba096e746987a7961d4a9ae026842bae67f" dependencies = [ "atty", "chrono", "log", "termcolor", - "thread_local 0.3.4", + "thread_local", ] [[package]] @@ -1661,7 +1719,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "libc", "rand 0.7.3", "redox_syscall", @@ -1719,23 +1777,13 @@ dependencies = [ "syn", ] -[[package]] -name = "thread_local" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1697c4b57aeeb7a536b647165a2825faddffb1d3bad386d507709bd51a90bb14" -dependencies = [ - "lazy_static 0.2.11", - "unreachable", -] - [[package]] name = "thread_local" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" dependencies = [ - "lazy_static 1.4.0", + "lazy_static", ] [[package]] @@ -1750,11 +1798,11 @@ dependencies = [ [[package]] name = "time" -version = "0.2.16" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a51cadc5b1eec673a685ff7c33192ff7b7603d0b75446fb354939ee615acb15" +checksum = "bcdaeea317915d59b2b4cd3b5efcd156c309108664277793f5351700c02ce98b" dependencies = [ - "cfg-if", + "const_fn", "libc", "standback", "stdweb", @@ -1815,7 +1863,7 @@ dependencies = [ "fnv", "futures-core", "iovec", - "lazy_static 1.4.0", + "lazy_static", "libc", "memchr", "mio", @@ -1892,15 +1940,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" -[[package]] -name = "unreachable" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" -dependencies = [ - "void", -] - [[package]] name = "untrusted" version = "0.7.1" @@ -1909,10 +1948,11 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829d4a8476c35c9bf0bbce5a3b23f4106f79728039b726d292bb93bc106787cb" +checksum = "5909f2b0817350449ed73e8bcd81c8c3c8d9a7a5d8acba4b27db277f1868976e" dependencies = [ + "form_urlencoded", "idna", "matches", "percent-encoding", @@ -1930,12 +1970,6 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" -[[package]] -name = "void" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" - [[package]] name = "want" version = "0.3.0" @@ -1958,7 +1992,7 @@ version = "0.2.64" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a634620115e4a229108b71bde263bb4220c483b3f07f5ba514ee8d15064c4c2" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "wasm-bindgen-macro", ] @@ -1969,7 +2003,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e53963b583d18a5aa3aaae4b4c1cb535218246131ba22a71f05b518098571df" dependencies = [ "bumpalo", - "lazy_static 1.4.0", + "lazy_static", "log", "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index a04d1fb34..8fcfdf2cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,35 +20,35 @@ oldcpu = [ "rocksdb-oldcpu" ] [dependencies] arraydeque = "0.4" arrayref = "0.3.6" -base64 = "0.12.3" +base64 = "0.13.0" bincode = "1.3.1" -clap = "2.31" -crossbeam-channel = "0.4" +clap = "2.33.3" +crossbeam-channel = "0.5.0" dirs = "3.0.1" -error-chain = "0.12.2" +error-chain = "0.12.4" glob = "0.3" hex = "0.4.2" itertools = "0.9.0" lazy_static = "1.3.0" -libc = "0.2.68" -log = "0.4" -socket2 = { version = "0.3.12", features = ["reuseport"] } +libc = "0.2.81" +log = "0.4.11" +socket2 = { version = "0.3.18", features = ["reuseport"] } num_cpus = "1.12.0" page_size = "0.4.2" -prometheus = "0.9" -rayon = "1.3.1" -rocksdb = { version = "0.14.0", optional = true } +prometheus = "0.11.0" +rayon = "1.5.0" +rocksdb = { version = "0.15.0", optional = true } rocksdb-oldcpu = { version = "0.12.4", optional = true, package = "rocksdb" } rust-crypto = "0.2" -serde = "1.0.114" -serde_derive = "1.0.114" -serde_json = "1.0.56" -signal-hook = "0.1" -stderrlog = "0.4.3" +serde = "1.0.118" +serde_derive = "1.0.118" +serde_json = "1.0.60" +signal-hook = "0.2.2" +stderrlog = "0.5.0" sysconf = ">=0.3.4" -time = "0.2.16" +time = "0.2.23" tiny_http = "0.7.0" -url = "2.1.1" +url = "2.2.0" hyper = "0.13.6" hyperlocal = "0.7" # close to same tokio version as dependent by hyper v0.13.6 and hyperlocal 0.7 -- things can go awry if they mismatch From c484efa2ebb2cbcdc15cb2c5b890ede3a0b77096 Mon Sep 17 00:00:00 2001 From: Nadav Ivgi Date: Thu, 31 Dec 2020 20:04:13 +0200 Subject: [PATCH 11/47] Don't add mempool transaction that already exists Doing this doesn't affect the indexes, but may result in multiple duplciated entries showing in the `recent` transactions. --- src/new_index/mempool.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/new_index/mempool.rs b/src/new_index/mempool.rs index cf67790bf..5caf6e8cb 100644 --- a/src/new_index/mempool.rs +++ b/src/new_index/mempool.rs @@ -317,8 +317,10 @@ impl Mempool { } pub fn add_by_txid(&mut self, daemon: &Daemon, txid: &Txid) { - if let Ok(tx) = daemon.getmempooltx(&txid) { - self.add(vec![tx]) + if self.txstore.get(txid).is_none() { + if let Ok(tx) = daemon.getmempooltx(&txid) { + self.add(vec![tx]) + } } } From bae488118de24d571042fb94e18789f0a9d1f69f Mon Sep 17 00:00:00 2001 From: Nadav Ivgi Date: Mon, 15 Feb 2021 22:50:09 +0200 Subject: [PATCH 12/47] Properly handle negative balanaces in the Electrum RPC Refs https://github.com/bitcoindevkit/rust-electrum-client/issues/45 --- src/electrum/server.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/electrum/server.rs b/src/electrum/server.rs index b35ed3f6c..96d4213aa 100644 --- a/src/electrum/server.rs +++ b/src/electrum/server.rs @@ -286,8 +286,8 @@ impl Connection { let (chain_stats, mempool_stats) = self.query.stats(&script_hash[..]); Ok(json!({ - "confirmed": chain_stats.funded_txo_sum - chain_stats.spent_txo_sum, - "unconfirmed": mempool_stats.funded_txo_sum - mempool_stats.spent_txo_sum, + "confirmed": chain_stats.funded_txo_sum as i64 - chain_stats.spent_txo_sum as i64, + "unconfirmed": mempool_stats.funded_txo_sum as i64 - mempool_stats.spent_txo_sum as i64, })) } From d1fd717d5e5d15ae43c938d08d71e0567323d505 Mon Sep 17 00:00:00 2001 From: Nadav Ivgi Date: Mon, 15 Feb 2021 23:51:48 +0200 Subject: [PATCH 13/47] Confirmed balances have to be positive h/t @sgeisler https://github.com/bitcoindevkit/rust-electrum-client/issues/45#issuecomment-779453735 --- src/electrum/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/electrum/server.rs b/src/electrum/server.rs index 96d4213aa..6157fbef4 100644 --- a/src/electrum/server.rs +++ b/src/electrum/server.rs @@ -286,7 +286,7 @@ impl Connection { let (chain_stats, mempool_stats) = self.query.stats(&script_hash[..]); Ok(json!({ - "confirmed": chain_stats.funded_txo_sum as i64 - chain_stats.spent_txo_sum as i64, + "confirmed": chain_stats.funded_txo_sum - chain_stats.spent_txo_sum, "unconfirmed": mempool_stats.funded_txo_sum as i64 - mempool_stats.spent_txo_sum as i64, })) } From a33e97e1a1fc63fa9c20a116bb92579bbf43b254 Mon Sep 17 00:00:00 2001 From: Nadav Ivgi Date: Sat, 20 Feb 2021 16:45:07 +0200 Subject: [PATCH 14/47] Display warning message with just the number of transactions The full list of txids can get pretty huge and is not really useful. --- src/new_index/mempool.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/new_index/mempool.rs b/src/new_index/mempool.rs index 5caf6e8cb..7963c5060 100644 --- a/src/new_index/mempool.rs +++ b/src/new_index/mempool.rs @@ -291,7 +291,7 @@ impl Mempool { let to_add = match daemon.gettransactions(&txids) { Ok(txs) => txs, Err(err) => { - warn!("failed to get transactions {:?}: {}", txids, err); // e.g. new block or RBF + warn!("failed to get {} transactions: {}", txids.len(), err); // e.g. new block or RBF return Ok(()); // keep the mempool until next update() } }; From abfbce73eb0504630a09f1e6599114ef4fa25bda Mon Sep 17 00:00:00 2001 From: Nadav Ivgi Date: Fri, 25 Jun 2021 22:07:08 +0300 Subject: [PATCH 15/47] Report the indexed tip height as a Prometheus metric (#37) --- src/new_index/schema.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index bee8273ba..120eae454 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -24,7 +24,7 @@ use crate::chain::{BlockHeader, Network, OutPoint, Transaction, TxOut, Value}; use crate::config::Config; use crate::daemon::Daemon; use crate::errors::*; -use crate::metrics::{HistogramOpts, HistogramTimer, HistogramVec, Metrics}; +use crate::metrics::{Gauge, HistogramOpts, HistogramTimer, HistogramVec, MetricOpts, Metrics}; use crate::util::{ full_hash, has_prevout, is_spendable, script_to_address, BlockHeaderMeta, BlockId, BlockMeta, BlockStatus, Bytes, HeaderEntry, HeaderList, @@ -164,6 +164,7 @@ pub struct Indexer { from: FetchFrom, iconfig: IndexerConfig, duration: HistogramVec, + tip_metric: Gauge, } struct IndexerConfig { @@ -208,6 +209,7 @@ impl Indexer { HistogramOpts::new("index_duration", "Index update duration (in seconds)"), &["step"], ), + tip_metric: metrics.gauge(MetricOpts::new("tip_height", "Current chain tip height")), } } @@ -296,6 +298,8 @@ impl Indexer { self.from = FetchFrom::Bitcoind; } + self.tip_metric.set(headers.len() as i64 - 1); + Ok(tip) } From 6b7416b4417b85b3a205165772c5183b2de5910c Mon Sep 17 00:00:00 2001 From: Nadav Ivgi Date: Thu, 21 Jan 2021 05:53:51 +0200 Subject: [PATCH 16/47] Upgrade to rust-bitcoin v0.25 and elements v0.15 --- Cargo.lock | 148 +++++++++++++++++++++++++++++++++-------- Cargo.toml | 19 +----- src/chain.rs | 3 +- src/daemon.rs | 5 +- src/new_index/fetch.rs | 4 +- src/rest.rs | 10 +-- src/util/block.rs | 6 +- src/util/script.rs | 2 +- 8 files changed, 136 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 99031f2db..faa5177d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -134,7 +134,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f30d3a39baa26f9651f17b375061f3233dde33424a8b72b0dbe93a68a0bc896d" dependencies = [ "byteorder", - "serde", + "serde 1.0.118", ] [[package]] @@ -164,13 +164,26 @@ dependencies = [ [[package]] name = "bitcoin" version = "0.23.0" -source = "git+https://github.com/blockstream/rust-bitcoin?rev=0.23-electrs#dd131855022c2cdd691a531c1d5c2f5ab707e4d4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a32c9d2fa897cfbb0db45d71e3d2838666194abc4828c0f994e4b5c3bf85ba4" dependencies = [ "bech32", - "bitcoin_hashes", + "bitcoin_hashes 0.7.6", "hex 0.3.2", - "secp256k1", - "serde", + "secp256k1 0.17.2", + "serde 1.0.118", +] + +[[package]] +name = "bitcoin" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aefc9be9f17185f4ebccae6575d342063f775924d57df0000edb1880c0fb7095" +dependencies = [ + "bech32", + "bitcoin_hashes 0.9.4", + "secp256k1 0.19.0", + "serde 1.0.118", ] [[package]] @@ -179,7 +192,16 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b375d62f341cef9cd9e77793ec8f1db3fc9ce2e4d57e982c8fe697a2c16af3b6" dependencies = [ - "serde", + "serde 1.0.118", +] + +[[package]] +name = "bitcoin_hashes" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aaf87b776808e26ae93289bc7d025092b6d909c193f0cdee0b3a86e7bd3c776" +dependencies = [ + "serde 1.0.118", ] [[package]] @@ -257,7 +279,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c74d84029116787153e02106bf53e66828452a4b325cc8652b788b5967c0a0b6" dependencies = [ "num-integer", - "num-traits", + "num-traits 0.2.12", "time 0.1.43", ] @@ -388,6 +410,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" +[[package]] +name = "dtoa" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d7ed2934d741c6b37e33e3832298e8850b53fd2d2bea03873375596c7cea4e" + [[package]] name = "either" version = "1.5.3" @@ -402,7 +430,7 @@ dependencies = [ "arrayref", "base64 0.13.0", "bincode", - "bitcoin", + "bitcoin 0.25.2", "clap", "crossbeam-channel", "dirs", @@ -424,9 +452,9 @@ dependencies = [ "rocksdb 0.12.4", "rocksdb 0.15.0", "rust-crypto", - "serde", + "serde 1.0.118", "serde_derive", - "serde_json", + "serde_json 1.0.60", "signal-hook", "socket2", "stderrlog", @@ -443,11 +471,11 @@ name = "electrum-client" version = "0.1.0-beta.6" source = "git+https://github.com/Blockstream/rust-electrum-client?rev=bd783aa70604fa1a0d8b29a3459326c640001354#bd783aa70604fa1a0d8b29a3459326c640001354" dependencies = [ - "bitcoin", + "bitcoin 0.23.0", "log", "rustls", - "serde", - "serde_json", + "serde 1.0.118", + "serde_json 1.0.60", "socks", "webpki", "webpki-roots", @@ -455,12 +483,15 @@ dependencies = [ [[package]] name = "elements" -version = "0.12.1" -source = "git+https://github.com/elementsproject/rust-elements?rev=98e0c4ba1b55cd876d1d8bcff64bcf01c4c370b6#98e0c4ba1b55cd876d1d8bcff64bcf01c4c370b6" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd3865be0d3cabe1c92022d310b747e08b6308de5d81a70c921c35701133cd8" dependencies = [ - "bitcoin", - "bitcoin_hashes", - "serde", + "bitcoin 0.25.2", + "bitcoin_hashes 0.9.4", + "serde 1.0.118", + "serde_json 0.9.10", + "slip21", ] [[package]] @@ -693,7 +724,7 @@ checksum = "28d569972648b2c512421b5f2a405ad6ac9666547189d0c5477a3f200f3e02f9" dependencies = [ "bytes", "fnv", - "itoa", + "itoa 0.4.6", ] [[package]] @@ -735,7 +766,7 @@ dependencies = [ "http", "http-body", "httparse", - "itoa", + "itoa 0.4.6", "log", "pin-project", "socket2", @@ -805,6 +836,12 @@ dependencies = [ "either", ] +[[package]] +name = "itoa" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8324a32baf01e2ae060e9de58ed0bc2320c9a2833491ee36cd3b4c414de4db8c" + [[package]] name = "itoa" version = "0.4.6" @@ -999,7 +1036,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b" dependencies = [ "autocfg", - "num-traits", + "num-traits 0.2.12", +] + +[[package]] +name = "num-traits" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" +dependencies = [ + "num-traits 0.2.12", ] [[package]] @@ -1481,8 +1527,18 @@ version = "0.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2932dc07acd2066ff2e3921a4419606b220ba6cd03a9935123856cc534877056" dependencies = [ - "secp256k1-sys", - "serde", + "secp256k1-sys 0.1.2", + "serde 1.0.118", +] + +[[package]] +name = "secp256k1" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6179428c22c73ac0fbb7b5579a56353ce78ba29759b3b8575183336ea74cdfb" +dependencies = [ + "secp256k1-sys 0.3.0", + "serde 1.0.118", ] [[package]] @@ -1494,6 +1550,15 @@ dependencies = [ "cc", ] +[[package]] +name = "secp256k1-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11553d210db090930f4432bea123b31f70bbf693ace14504ea2a35e796c28dd2" +dependencies = [ + "cc", +] + [[package]] name = "semver" version = "0.9.0" @@ -1509,6 +1574,12 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +[[package]] +name = "serde" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b623917345a631dc9608d5194cc206b3fe6c3554cd1c75b937e55e285254af" + [[package]] name = "serde" version = "1.0.118" @@ -1529,15 +1600,27 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_json" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8bcf487be7d2e15d3d543f04312de991d631cfe1b43ea0ade69e6a8a5b16a1" +dependencies = [ + "dtoa", + "itoa 0.3.4", + "num-traits 0.1.43", + "serde 0.9.15", +] + [[package]] name = "serde_json" version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1500e84d27fe482ed1dc791a56eddc2f230046a040fa908c08bda1d9fb615779" dependencies = [ - "itoa", + "itoa 0.4.6", "ryu", - "serde", + "serde 1.0.118", ] [[package]] @@ -1578,6 +1661,15 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +[[package]] +name = "slip21" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acff9a1a0f4d902c8184bdde5338861dd95eb1397a12c4df78fb0b6ebeed4cbe" +dependencies = [ + "bitcoin_hashes 0.7.6", +] + [[package]] name = "smallvec" version = "1.5.1" @@ -1657,7 +1749,7 @@ checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" dependencies = [ "proc-macro2", "quote", - "serde", + "serde 1.0.118", "serde_derive", "syn", ] @@ -1671,9 +1763,9 @@ dependencies = [ "base-x", "proc-macro2", "quote", - "serde", + "serde 1.0.118", "serde_derive", - "serde_json", + "serde_json 1.0.60", "sha1", "syn", ] diff --git a/Cargo.toml b/Cargo.toml index 8fcfdf2cf..148a11573 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,9 +22,11 @@ arraydeque = "0.4" arrayref = "0.3.6" base64 = "0.13.0" bincode = "1.3.1" +bitcoin = { version = "0.25.0", features = [ "use-serde" ] } clap = "2.33.3" crossbeam-channel = "0.5.0" dirs = "3.0.1" +elements = { version = "0.15.0", features = [ "serde-feature" ], optional = true } error-chain = "0.12.4" glob = "0.3" hex = "0.4.2" @@ -57,15 +59,6 @@ tokio = { version = "=0.2.6", features = ["sync", "macros"] } # optional dependencies for electrum-discovery electrum-client = { version = "0.1.0-beta.6", optional = true } -[dependencies.bitcoin] -version = "0.23.0" -features = ["use-serde"] - -[dependencies.elements] -optional = true -version = "0.12.1" -features = ["serde-feature"] - [dev-dependencies] tempfile = "3.0" @@ -74,14 +67,6 @@ lto = true panic = 'abort' codegen-units = 1 -[patch.crates-io.bitcoin] -git = "https://github.com/blockstream/rust-bitcoin" -rev = "0.23-electrs" - -[patch.crates-io.elements] -git = "https://github.com/elementsproject/rust-elements" -rev = "98e0c4ba1b55cd876d1d8bcff64bcf01c4c370b6" - [patch.crates-io.electrum-client] git = "https://github.com/Blockstream/rust-electrum-client" rev = "bd783aa70604fa1a0d8b29a3459326c640001354" # add-peer diff --git a/src/chain.rs b/src/chain.rs index 5e5e3fd8a..d8f006240 100644 --- a/src/chain.rs +++ b/src/chain.rs @@ -12,7 +12,6 @@ pub use { use bitcoin::blockdata::constants::genesis_block; use bitcoin::network::constants::Network as BNetwork; -use bitcoin::util::hash::BitcoinHash; use bitcoin::BlockHash; use std::collections::HashMap; @@ -46,7 +45,7 @@ impl Network { return *block_hash; } - let block_hash = genesis_block(BNetwork::from(self)).bitcoin_hash(); + let block_hash = genesis_block(BNetwork::from(self)).block_hash(); CACHED_GENESIS.write().unwrap().insert(self, block_hash); block_hash } diff --git a/src/daemon.rs b/src/daemon.rs index 529369954..afffa2041 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -7,7 +7,6 @@ use std::time::Duration; use base64; use bitcoin::hashes::hex::{FromHex, ToHex}; -use bitcoin::util::hash::BitcoinHash; use bitcoin::{BlockHash, Txid}; use glob; use hex; @@ -461,7 +460,7 @@ impl Daemon { let block = block_from_value( self.request("getblock", json!([blockhash.to_hex(), /*verbose=*/ false]))?, )?; - assert_eq!(block.bitcoin_hash(), *blockhash); + assert_eq!(block.block_hash(), *blockhash); Ok(block) } @@ -588,7 +587,7 @@ impl Daemon { let mut blockhash = BlockHash::default(); for header in &result { assert_eq!(header.prev_blockhash, blockhash); - blockhash = header.bitcoin_hash(); + blockhash = header.block_hash(); } assert_eq!(blockhash, *tip); Ok(result) diff --git a/src/new_index/fetch.rs b/src/new_index/fetch.rs index 6b9651445..0cb6375c8 100644 --- a/src/new_index/fetch.rs +++ b/src/new_index/fetch.rs @@ -1,6 +1,6 @@ use crate::chain::Block; -use bitcoin::{BitcoinHash, BlockHash}; +use bitcoin::BlockHash; use rayon::prelude::*; #[cfg(not(feature = "liquid"))] @@ -124,7 +124,7 @@ fn blkfiles_fetcher( let block_entries: Vec = sizedblocks .into_iter() .filter_map(|(block, size)| { - let blockhash = block.bitcoin_hash(); + let blockhash = block.block_hash(); entry_map .remove(&blockhash) .map(|entry| BlockEntry { block, entry, size }) diff --git a/src/rest.rs b/src/rest.rs index 00a971df8..2b280c735 100644 --- a/src/rest.rs +++ b/src/rest.rs @@ -12,7 +12,7 @@ use crate::util::{ use bitcoin::consensus::encode; use bitcoin::hashes::hex::{FromHex, ToHex}; use bitcoin::hashes::Error as HashError; -use bitcoin::{BitcoinHash, BlockHash, Script, Txid}; +use bitcoin::{BlockHash, Script, Txid}; use hex::{self, FromHexError}; use hyper::service::{make_service_fn, service_fn}; use hyper::{Body, Method, Response, Server, StatusCode}; @@ -84,9 +84,9 @@ impl BlockValue { fn new(blockhm: BlockHeaderMeta, network: Network) -> Self { let header = blockhm.header_entry.header(); BlockValue { - id: header.bitcoin_hash().to_hex(), + id: header.block_hash().to_hex(), height: blockhm.header_entry.height() as u32, - version: header.version, + version: header.version as u32, timestamp: header.time, tx_count: blockhm.meta.tx_count, size: blockhm.meta.size, @@ -152,7 +152,7 @@ impl TransactionValue { TransactionValue { txid: tx.txid(), - version: tx.version, + version: tx.version as u32, locktime: tx.lock_time, vin: vins, vout: vouts, @@ -959,7 +959,7 @@ fn handle_request( let height = query .chain() - .height_by_hash(&merkleblock.header.bitcoin_hash()); + .height_by_hash(&merkleblock.header.block_hash()); http_message( StatusCode::OK, diff --git a/src/util/block.rs b/src/util/block.rs index 1bb7cd294..0e2c73ab6 100644 --- a/src/util/block.rs +++ b/src/util/block.rs @@ -2,7 +2,7 @@ use crate::chain::BlockHeader; use crate::errors::*; use crate::new_index::BlockEntry; -use bitcoin::{BitcoinHash, BlockHash}; +use bitcoin::BlockHash; use std::collections::HashMap; use std::fmt; @@ -97,7 +97,7 @@ impl HeaderList { panic!( "missing expected blockhash in headers map: {:?}, pointed from: {:?}", blockhash, - headers_chain.last().map(|h| h.bitcoin_hash()) + headers_chain.last().map(|h| h.block_hash()) ) }); blockhash = header.prev_blockhash; @@ -124,7 +124,7 @@ impl HeaderList { } let hashed_headers = Vec::::from_iter(new_headers.into_iter().map(|header| HashedHeader { - blockhash: header.bitcoin_hash(), + blockhash: header.block_hash(), header, })); for i in 1..hashed_headers.len() { diff --git a/src/util/script.rs b/src/util/script.rs index 5d1bb7958..9faf4c186 100644 --- a/src/util/script.rs +++ b/src/util/script.rs @@ -31,7 +31,7 @@ pub fn get_script_asm(script: &Script) -> String { pub fn get_innerscripts(txin: &TxIn, prevout: &TxOut) -> InnerScripts { // Wrapped redeemScript for P2SH spends let redeem_script = if prevout.script_pubkey.is_p2sh() { - if let Some(PushBytes(redeemscript)) = txin.script_sig.iter(true).last() { + if let Some(Ok(PushBytes(redeemscript))) = txin.script_sig.instructions().last() { Some(Script::from(redeemscript.to_vec())) } else { None From f905a01102d09c02239c2c5e363406e1dc72bc25 Mon Sep 17 00:00:00 2001 From: Nadav Ivgi Date: Thu, 21 Jan 2021 09:51:51 +0200 Subject: [PATCH 17/47] Adjust for rust-elements's new data types --- src/chain.rs | 43 ++++++++++++++++++++----------------- src/config.rs | 11 ++++++---- src/daemon.rs | 3 +-- src/electrum/client.rs | 3 ++- src/electrum/discovery.rs | 4 +--- src/electrum/mod.rs | 3 +-- src/electrum/server.rs | 2 +- src/elements/asset.rs | 9 ++++---- src/elements/peg.rs | 23 +++++++++++--------- src/elements/registry.rs | 2 +- src/new_index/fetch.rs | 4 +--- src/new_index/mempool.rs | 4 +--- src/new_index/query.rs | 4 +--- src/new_index/schema.rs | 15 +++++++------ src/rest.rs | 18 +++++++--------- src/util/block.rs | 4 +--- src/util/electrum_merkle.rs | 5 ++--- src/util/mod.rs | 2 +- src/util/script.rs | 35 ++++++++++++++++-------------- src/util/transaction.rs | 10 ++++----- 20 files changed, 100 insertions(+), 104 deletions(-) diff --git a/src/chain.rs b/src/chain.rs index d8f006240..d35896130 100644 --- a/src/chain.rs +++ b/src/chain.rs @@ -1,18 +1,20 @@ #[cfg(not(feature = "liquid"))] // use regular Bitcoin data structures -pub use bitcoin::{util::address, Block, BlockHeader, OutPoint, Transaction, TxIn, TxOut}; +pub use bitcoin::{ + blockdata::script, consensus::deserialize, util::address, Block, BlockHash, BlockHeader, + OutPoint, Script, Transaction, TxIn, TxOut, Txid, +}; #[cfg(feature = "liquid")] pub use { crate::elements::asset, elements::{ - address, confidential, Address, AssetId, Block, BlockHeader, OutPoint, Transaction, TxIn, - TxOut, + address, confidential, encode::deserialize, script, Address, AssetId, Block, BlockHash, + BlockHeader, OutPoint, Script, Transaction, TxIn, TxOut, Txid, }, }; use bitcoin::blockdata::constants::genesis_block; -use bitcoin::network::constants::Network as BNetwork; -use bitcoin::BlockHash; +pub use bitcoin::network::constants::Network as BNetwork; use std::collections::HashMap; use std::sync::{Arc, RwLock}; @@ -22,11 +24,6 @@ pub type Value = u64; #[cfg(feature = "liquid")] pub use confidential::Value; -lazy_static! { - static ref CACHED_GENESIS: Arc>> = - Arc::new(RwLock::new(HashMap::new())); -} - #[derive(Debug, Copy, Clone, PartialEq, Hash, Serialize, Ord, PartialOrd, Eq)] pub enum Network { Bitcoin, @@ -40,16 +37,6 @@ pub enum Network { } impl Network { - pub fn genesis_hash(self) -> BlockHash { - if let Some(block_hash) = CACHED_GENESIS.read().unwrap().get(&self) { - return *block_hash; - } - - let block_hash = genesis_block(BNetwork::from(self)).block_hash(); - CACHED_GENESIS.write().unwrap().insert(self, block_hash); - block_hash - } - pub fn magic(self) -> u32 { match self { Network::Bitcoin => 0xD9B4_BEF9, @@ -102,6 +89,22 @@ impl Network { } } +// For bitcoin (non-elements) chains only +pub fn genesis_hash(network: BNetwork) -> bitcoin::BlockHash { + lazy_static! { + static ref CACHED_GENESIS: Arc>> = + Arc::new(RwLock::new(HashMap::new())); + } + + if let Some(block_hash) = CACHED_GENESIS.read().unwrap().get(&network) { + return *block_hash; + } + + let block_hash = genesis_block(network).block_hash(); + CACHED_GENESIS.write().unwrap().insert(network, block_hash); + block_hash +} + impl From<&str> for Network { fn from(network_name: &str) -> Self { match network_name { diff --git a/src/config.rs b/src/config.rs index 189ac5579..9a68e4507 100644 --- a/src/config.rs +++ b/src/config.rs @@ -9,9 +9,11 @@ use stderrlog; use crate::chain::Network; use crate::daemon::CookieGetter; - use crate::errors::*; +#[cfg(feature = "liquid")] +use bitcoin::Network as BNetwork; + const ELECTRS_VERSION: &str = env!("CARGO_PKG_VERSION"); #[derive(Debug, Clone)] @@ -39,7 +41,7 @@ pub struct Config { pub electrum_banner: String, #[cfg(feature = "liquid")] - pub parent_network: Network, + pub parent_network: BNetwork, #[cfg(feature = "liquid")] pub asset_db_path: Option, @@ -235,9 +237,10 @@ impl Config { let parent_network = m .value_of("parent_network") .map(Network::from) + .map(BNetwork::from) .unwrap_or_else(|| match network_type { - Network::Liquid => Network::Bitcoin, - Network::LiquidRegtest => Network::Regtest, + Network::Liquid => BNetwork::Bitcoin, + Network::LiquidRegtest => BNetwork::Regtest, _ => panic!("unknown liquid network, --parent-network is required"), }); diff --git a/src/daemon.rs b/src/daemon.rs index afffa2041..eb05680b3 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -7,7 +7,6 @@ use std::time::Duration; use base64; use bitcoin::hashes::hex::{FromHex, ToHex}; -use bitcoin::{BlockHash, Txid}; use glob; use hex; use serde_json::{from_str, from_value, Value}; @@ -17,7 +16,7 @@ use bitcoin::consensus::encode::{deserialize, serialize}; #[cfg(feature = "liquid")] use elements::encode::{deserialize, serialize}; -use crate::chain::{Block, BlockHeader, Network, Transaction}; +use crate::chain::{Block, BlockHash, BlockHeader, Network, Transaction, Txid}; use crate::metrics::{HistogramOpts, HistogramVec, Metrics}; use crate::signal::Waiter; use crate::util::HeaderList; diff --git a/src/electrum/client.rs b/src/electrum/client.rs index f6326e620..436754116 100644 --- a/src/electrum/client.rs +++ b/src/electrum/client.rs @@ -2,13 +2,14 @@ use std::collections::HashMap; use std::convert::{TryFrom, TryInto}; use std::net::ToSocketAddrs; -use bitcoin::{hashes::Hash, BlockHash}; +use bitcoin::hashes::Hash; use electrum_client::client::{ Client as RClient, ElectrumPlaintextStream, ElectrumProxyStream, ElectrumSslStream, }; pub use electrum_client::types::ServerFeaturesRes; pub use electrum_client::Error as ElectrumError; +use crate::chain::BlockHash; use crate::electrum::ServerFeatures; use crate::errors::{Error, ResultExt}; diff --git a/src/electrum/discovery.rs b/src/electrum/discovery.rs index 3eecc55dc..2cc1c45cb 100644 --- a/src/electrum/discovery.rs +++ b/src/electrum/discovery.rs @@ -7,9 +7,7 @@ use std::sync::{Arc, RwLock}; use std::thread; use std::time::{Duration, Instant}; -use bitcoin::BlockHash; - -use crate::chain::Network; +use crate::chain::{BlockHash, Network}; use crate::electrum::{Client, Hostname, Port, ProtocolVersion, ServerFeatures}; use crate::errors::{Result, ResultExt}; use crate::util::spawn_thread; diff --git a/src/electrum/mod.rs b/src/electrum/mod.rs index b618a2ec6..d28426d7d 100644 --- a/src/electrum/mod.rs +++ b/src/electrum/mod.rs @@ -14,8 +14,7 @@ use std::str::FromStr; use serde::{de, Deserialize, Deserializer, Serialize}; -use bitcoin::BlockHash; - +use crate::chain::BlockHash; use crate::errors::ResultExt; use crate::util::BlockId; diff --git a/src/electrum/server.rs b/src/electrum/server.rs index 6157fbef4..cbcf12bef 100644 --- a/src/electrum/server.rs +++ b/src/electrum/server.rs @@ -6,7 +6,6 @@ use std::sync::{Arc, Mutex}; use std::thread; use bitcoin::hashes::sha256d::Hash as Sha256dHash; -use bitcoin::Txid; use crypto::digest::Digest; use crypto::sha2::Sha256; use error_chain::ChainedError; @@ -18,6 +17,7 @@ use bitcoin::consensus::encode::serialize; #[cfg(feature = "liquid")] use elements::encode::serialize; +use crate::chain::Txid; use crate::config::Config; use crate::electrum::{get_electrum_height, ProtocolVersion}; use crate::errors::*; diff --git a/src/elements/asset.rs b/src/elements/asset.rs index a5717fa03..67578bc33 100644 --- a/src/elements/asset.rs +++ b/src/elements/asset.rs @@ -2,12 +2,11 @@ use std::collections::{HashMap, HashSet}; use std::sync::{Arc, RwLock}; use bitcoin::hashes::{hex::FromHex, sha256, Hash}; -use bitcoin::{BlockHash, Txid}; use elements::confidential::{Asset, Value}; use elements::encode::{deserialize, serialize}; use elements::{issuance::ContractHash, AssetId, AssetIssuance, OutPoint, Transaction, TxIn}; -use crate::chain::Network; +use crate::chain::{BNetwork, BlockHash, Network, Txid}; use crate::elements::peg::{get_pegin_data, get_pegout_data, PeginInfo, PegoutInfo}; use crate::elements::registry::{AssetMeta, AssetRegistry}; use crate::errors::*; @@ -169,7 +168,7 @@ pub fn index_confirmed_tx_assets( tx: &Transaction, confirmed_height: u32, network: Network, - parent_network: Network, + parent_network: BNetwork, rows: &mut Vec, ) { let (history, issuances) = index_tx_assets(tx, network, parent_network); @@ -193,7 +192,7 @@ pub fn index_confirmed_tx_assets( pub fn index_mempool_tx_assets( tx: &Transaction, network: Network, - parent_network: Network, + parent_network: BNetwork, asset_history: &mut HashMap>, asset_issuance: &mut HashMap, ) { @@ -231,7 +230,7 @@ pub fn remove_mempool_tx_assets( fn index_tx_assets( tx: &Transaction, network: Network, - parent_network: Network, + parent_network: BNetwork, ) -> (Vec<(AssetId, TxHistoryInfo)>, Vec<(AssetId, AssetRow)>) { let mut history = vec![]; let mut issuances = vec![]; diff --git a/src/elements/peg.rs b/src/elements/peg.rs index f980ae05a..77c058530 100644 --- a/src/elements/peg.rs +++ b/src/elements/peg.rs @@ -1,8 +1,8 @@ -use bitcoin::{hashes::hex::ToHex, Script}; +use bitcoin::hashes::hex::ToHex; use elements::{confidential::Asset, PeginData, PegoutData, TxIn, TxOut}; -use crate::chain::Network; -use crate::util::{get_script_asm, script_to_address, FullHash}; +use crate::chain::{genesis_hash, BNetwork, Network}; +use crate::util::{FullHash, ScriptExt}; pub fn get_pegin_data(txout: &TxIn, network: Network) -> Option { txout @@ -13,11 +13,11 @@ pub fn get_pegin_data(txout: &TxIn, network: Network) -> Option { pub fn get_pegout_data( txout: &TxOut, network: Network, - parent_network: Network, + parent_network: BNetwork, ) -> Option { txout.pegout_data().filter(|pegout| { pegout.asset == Asset::Explicit(*network.native_asset()) - && pegout.genesis_hash == parent_network.genesis_hash() + && pegout.genesis_hash == genesis_hash(parent_network) }) } @@ -25,21 +25,24 @@ pub fn get_pegout_data( #[derive(Serialize, Deserialize, Clone)] pub struct PegoutValue { pub genesis_hash: String, - pub scriptpubkey: Script, + pub scriptpubkey: bitcoin::Script, pub scriptpubkey_asm: String, #[serde(skip_serializing_if = "Option::is_none")] pub scriptpubkey_address: Option, } impl PegoutValue { - pub fn from_txout(txout: &TxOut, network: Network, parent_network: Network) -> Option { + pub fn from_txout(txout: &TxOut, network: Network, parent_network: BNetwork) -> Option { let pegoutdata = get_pegout_data(txout, network, parent_network)?; + // pending https://github.com/ElementsProject/rust-elements/pull/69 is merged + let scriptpubkey = bitcoin::Script::from(pegoutdata.script_pubkey.into_bytes()); + Some(PegoutValue { genesis_hash: pegoutdata.genesis_hash.to_hex(), - scriptpubkey_asm: get_script_asm(&pegoutdata.script_pubkey), - scriptpubkey_address: script_to_address(&pegoutdata.script_pubkey, parent_network), - scriptpubkey: pegoutdata.script_pubkey, + scriptpubkey_asm: scriptpubkey.to_asm(), + scriptpubkey_address: scriptpubkey.to_address(parent_network.into()), + scriptpubkey, }) } } diff --git a/src/elements/registry.rs b/src/elements/registry.rs index 8c1fc6969..e0728320d 100644 --- a/src/elements/registry.rs +++ b/src/elements/registry.rs @@ -5,7 +5,7 @@ use std::{cmp, fs, path, thread}; use serde_json::Value as JsonValue; -use elements::bitcoin_hashes::hex::FromHex; +use bitcoin::hashes::hex::FromHex; use elements::AssetId; use crate::errors::*; diff --git a/src/new_index/fetch.rs b/src/new_index/fetch.rs index 0cb6375c8..2a3112252 100644 --- a/src/new_index/fetch.rs +++ b/src/new_index/fetch.rs @@ -1,6 +1,3 @@ -use crate::chain::Block; - -use bitcoin::BlockHash; use rayon::prelude::*; #[cfg(not(feature = "liquid"))] @@ -15,6 +12,7 @@ use std::path::PathBuf; use std::sync::mpsc::Receiver; use std::thread; +use crate::chain::{Block, BlockHash}; use crate::daemon::Daemon; use crate::errors::*; use crate::util::{spawn_thread, HeaderEntry, SyncChannel}; diff --git a/src/new_index/mempool.rs b/src/new_index/mempool.rs index 7963c5060..9586c5168 100644 --- a/src/new_index/mempool.rs +++ b/src/new_index/mempool.rs @@ -1,6 +1,4 @@ use arraydeque::{ArrayDeque, Wrapping}; -use bitcoin::consensus::encode::deserialize; -use bitcoin::Txid; use itertools::Itertools; #[cfg(not(feature = "liquid"))] @@ -13,7 +11,7 @@ use std::iter::FromIterator; use std::sync::Arc; use std::time::{Duration, Instant}; -use crate::chain::{Network, OutPoint, Transaction, TxOut}; +use crate::chain::{deserialize, Network, OutPoint, Transaction, TxOut, Txid}; use crate::config::Config; use crate::daemon::Daemon; use crate::errors::*; diff --git a/src/new_index/query.rs b/src/new_index/query.rs index c7211af43..387e8f70f 100644 --- a/src/new_index/query.rs +++ b/src/new_index/query.rs @@ -4,15 +4,13 @@ use std::collections::{BTreeSet, HashMap}; use std::sync::{Arc, RwLock, RwLockReadGuard}; use std::time::{Duration, Instant}; -use crate::chain::{Network, OutPoint, Transaction, TxOut}; +use crate::chain::{Network, OutPoint, Transaction, TxOut, Txid}; use crate::config::Config; use crate::daemon::Daemon; use crate::errors::*; use crate::new_index::{ChainQuery, Mempool, ScriptStats, SpendingInput, Utxo}; use crate::util::{is_spendable, BlockId, Bytes, TransactionStatus}; -use bitcoin::Txid; - #[cfg(feature = "liquid")] use crate::{ chain::AssetId, diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index 120eae454..7b69f5135 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -1,8 +1,7 @@ -use bitcoin::blockdata::script::Script; use bitcoin::hashes::sha256d::Hash as Sha256dHash; #[cfg(not(feature = "liquid"))] use bitcoin::util::merkleblock::MerkleBlock; -use bitcoin::{BlockHash, Txid, VarInt}; +use bitcoin::VarInt; use crypto::digest::Digest; use crypto::sha2::Sha256; use itertools::Itertools; @@ -20,14 +19,16 @@ use std::collections::{BTreeSet, HashMap, HashSet}; use std::path::Path; use std::sync::{Arc, RwLock}; -use crate::chain::{BlockHeader, Network, OutPoint, Transaction, TxOut, Value}; +use crate::chain::{ + BlockHash, BlockHeader, Network, OutPoint, Script, Transaction, TxOut, Txid, Value, +}; use crate::config::Config; use crate::daemon::Daemon; use crate::errors::*; use crate::metrics::{Gauge, HistogramOpts, HistogramTimer, HistogramVec, MetricOpts, Metrics}; use crate::util::{ - full_hash, has_prevout, is_spendable, script_to_address, BlockHeaderMeta, BlockId, BlockMeta, - BlockStatus, Bytes, HeaderEntry, HeaderList, + full_hash, has_prevout, is_spendable, BlockHeaderMeta, BlockId, BlockMeta, BlockStatus, Bytes, + HeaderEntry, HeaderList, ScriptExt, }; use crate::new_index::db::{DBFlush, DBRow, ReverseScanIterator, ScanIterator, DB}; @@ -173,7 +174,7 @@ struct IndexerConfig { index_unspendables: bool, network: Network, #[cfg(feature = "liquid")] - parent_network: Network, + parent_network: crate::chain::BNetwork, } impl From<&Config> for IndexerConfig { @@ -1151,7 +1152,7 @@ fn index_transaction( } fn addr_search_row(spk: &Script, network: Network) -> Option { - script_to_address(spk, network).map(|address| DBRow { + spk.to_address(network).map(|address| DBRow { key: [b"a", address.as_bytes()].concat(), value: vec![], }) diff --git a/src/rest.rs b/src/rest.rs index 2b280c735..b7fd38fe5 100644 --- a/src/rest.rs +++ b/src/rest.rs @@ -1,18 +1,16 @@ -use crate::chain::{address, Network, OutPoint, Transaction, TxIn, TxOut}; +use crate::chain::{address, BlockHash, Network, OutPoint, Script, Transaction, TxIn, TxOut, Txid}; use crate::config::Config; use crate::errors; use crate::new_index::{compute_script_hash, Query, SpendingInput, Utxo}; use crate::util::{ - create_socket, electrum_merkle, extract_tx_prevouts, full_hash, get_innerscripts, - get_script_asm, get_tx_fee, has_prevout, is_coinbase, script_to_address, BlockHeaderMeta, - BlockId, FullHash, TransactionStatus, + create_socket, electrum_merkle, extract_tx_prevouts, full_hash, get_innerscripts, get_tx_fee, + has_prevout, is_coinbase, BlockHeaderMeta, BlockId, FullHash, ScriptExt, TransactionStatus, }; #[cfg(not(feature = "liquid"))] use bitcoin::consensus::encode; use bitcoin::hashes::hex::{FromHex, ToHex}; use bitcoin::hashes::Error as HashError; -use bitcoin::{BlockHash, Script, Txid}; use hex::{self, FromHexError}; use hyper::service::{make_service_fn, service_fn}; use hyper::{Body, Method, Response, Server, StatusCode}; @@ -208,17 +206,17 @@ impl TxInValue { txid: txin.previous_output.txid, vout: txin.previous_output.vout, prevout: prevout.map(|prevout| TxOutValue::new(prevout, config)), - scriptsig_asm: get_script_asm(&txin.script_sig), + scriptsig_asm: txin.script_sig.to_asm(), witness, inner_redeemscript_asm: innerscripts .as_ref() .and_then(|i| i.redeem_script.as_ref()) - .map(get_script_asm), + .map(ScriptExt::to_asm), inner_witnessscript_asm: innerscripts .as_ref() .and_then(|i| i.witness_script.as_ref()) - .map(get_script_asm), + .map(ScriptExt::to_asm), is_coinbase, sequence: txin.sequence, @@ -299,8 +297,8 @@ impl TxOutValue { let is_fee = txout.is_fee(); let script = &txout.script_pubkey; - let script_asm = get_script_asm(&script); - let script_addr = script_to_address(&script, config.network_type); + let script_asm = script.to_asm(); + let script_addr = script.to_address(config.network_type); // TODO should the following something to put inside rust-elements lib? let script_type = if is_fee { diff --git a/src/util/block.rs b/src/util/block.rs index 0e2c73ab6..8dea9c60f 100644 --- a/src/util/block.rs +++ b/src/util/block.rs @@ -1,9 +1,7 @@ -use crate::chain::BlockHeader; +use crate::chain::{BlockHash, BlockHeader}; use crate::errors::*; use crate::new_index::BlockEntry; -use bitcoin::BlockHash; - use std::collections::HashMap; use std::fmt; use std::iter::FromIterator; diff --git a/src/util/electrum_merkle.rs b/src/util/electrum_merkle.rs index a8048b155..954782a7d 100644 --- a/src/util/electrum_merkle.rs +++ b/src/util/electrum_merkle.rs @@ -1,8 +1,7 @@ -use bitcoin::hashes::{sha256d::Hash as Sha256dHash, Hash}; -use bitcoin::{BlockHash, Txid}; - +use crate::chain::{BlockHash, Txid}; use crate::errors::*; use crate::new_index::ChainQuery; +use bitcoin::hashes::{sha256d::Hash as Sha256dHash, Hash}; pub fn get_tx_merkle_proof( chain: &ChainQuery, diff --git a/src/util/mod.rs b/src/util/mod.rs index 6ef04b3f6..c0505d1e2 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -7,7 +7,7 @@ pub mod fees; pub use self::block::{BlockHeaderMeta, BlockId, BlockMeta, BlockStatus, HeaderEntry, HeaderList}; pub use self::fees::get_tx_fee; -pub use self::script::{get_innerscripts, get_script_asm, script_to_address}; +pub use self::script::{get_innerscripts, ScriptExt}; pub use self::transaction::{ extract_tx_prevouts, has_prevout, is_coinbase, is_spendable, TransactionStatus, TxInput, }; diff --git a/src/util/script.rs b/src/util/script.rs index 9faf4c186..09faa84da 100644 --- a/src/util/script.rs +++ b/src/util/script.rs @@ -1,30 +1,33 @@ -use bitcoin::blockdata::script::{Instruction::PushBytes, Script}; - #[cfg(feature = "liquid")] use elements::address as elements_address; -use crate::chain::Network; -use crate::chain::{TxIn, TxOut}; +use crate::chain::{script, Network, Script, TxIn, TxOut}; +use script::Instruction::PushBytes; pub struct InnerScripts { pub redeem_script: Option