From d92bf27bfc3838dc0bafab47f56bf19662e2dda7 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 12 Jun 2024 11:22:14 -0600 Subject: [PATCH 1/4] zcash_client_sqlite: Move transparent-inputs wallet methods into the `wallet::transparent` module. --- zcash_client_sqlite/src/lib.rs | 27 +- zcash_client_sqlite/src/wallet.rs | 721 +---------------- zcash_client_sqlite/src/wallet/transparent.rs | 727 +++++++++++++++++- 3 files changed, 750 insertions(+), 725 deletions(-) diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 092d060d84..b973e7d4da 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -272,7 +272,7 @@ impl, P: consensus::Parameters> InputSource for &self, outpoint: &OutPoint, ) -> Result, Self::Error> { - wallet::get_unspent_transparent_output(self.conn.borrow(), outpoint) + wallet::transparent::get_unspent_transparent_output(self.conn.borrow(), outpoint) } #[cfg(feature = "transparent-inputs")] @@ -282,7 +282,7 @@ impl, P: consensus::Parameters> InputSource for max_height: BlockHeight, exclude: &[OutPoint], ) -> Result, Self::Error> { - wallet::get_unspent_transparent_outputs( + wallet::transparent::get_unspent_transparent_outputs( self.conn.borrow(), &self.params, address, @@ -516,7 +516,7 @@ impl, P: consensus::Parameters> WalletRead for W &self, account: AccountId, ) -> Result>, Self::Error> { - wallet::get_transparent_receivers(self.conn.borrow(), &self.params, account) + wallet::transparent::get_transparent_receivers(self.conn.borrow(), &self.params, account) } #[cfg(feature = "transparent-inputs")] @@ -525,7 +525,12 @@ impl, P: consensus::Parameters> WalletRead for W account: AccountId, max_height: BlockHeight, ) -> Result, Self::Error> { - wallet::get_transparent_balances(self.conn.borrow(), &self.params, account, max_height) + wallet::transparent::get_transparent_address_balances( + self.conn.borrow(), + &self.params, + account, + max_height, + ) } } @@ -1034,7 +1039,11 @@ impl WalletWrite for WalletDb _output: &WalletTransparentOutput, ) -> Result { #[cfg(feature = "transparent-inputs")] - return wallet::put_received_transparent_utxo(&self.conn, &self.params, _output); + return wallet::transparent::put_received_transparent_utxo( + &self.conn, + &self.params, + _output, + ); #[cfg(not(feature = "transparent-inputs"))] panic!( @@ -1228,7 +1237,7 @@ impl WalletWrite for WalletDb .iter() .flat_map(|b| b.vin.iter()) { - wallet::mark_transparent_utxo_spent(wdb.conn.0, tx_ref, &txin.prevout)?; + wallet::transparent::mark_transparent_utxo_spent(wdb.conn.0, tx_ref, &txin.prevout)?; } // If we have some transparent outputs: @@ -1337,7 +1346,11 @@ impl WalletWrite for WalletDb #[cfg(feature = "transparent-inputs")] for utxo_outpoint in sent_tx.utxos_spent() { - wallet::mark_transparent_utxo_spent(wdb.conn.0, tx_ref, utxo_outpoint)?; + wallet::transparent::mark_transparent_utxo_spent( + wdb.conn.0, + tx_ref, + utxo_outpoint, + )?; } for output in sent_tx.outputs() { diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 38a46a364a..257c6e3cfd 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -120,22 +120,6 @@ use self::scanning::{parse_priority_code, priority_code, replace_queue_entries}; #[cfg(feature = "orchard")] use {crate::ORCHARD_TABLES_PREFIX, zcash_client_backend::data_api::ORCHARD_SHARD_HEIGHT}; -#[cfg(feature = "transparent-inputs")] -use { - crate::UtxoId, - rusqlite::Row, - std::collections::BTreeSet, - zcash_address::unified::{Encoding, Ivk, Uivk}, - zcash_client_backend::wallet::{TransparentAddressMetadata, WalletTransparentOutput}, - zcash_primitives::{ - legacy::{ - keys::{IncomingViewingKey, NonHardenedChildIndex}, - Script, TransparentAddress, - }, - transaction::components::{OutPoint, TxOut}, - }, -}; - pub mod commitment_tree; pub(crate) mod common; mod db; @@ -605,116 +589,6 @@ pub(crate) fn insert_address( Ok(()) } -#[cfg(feature = "transparent-inputs")] -pub(crate) fn get_transparent_receivers( - conn: &rusqlite::Connection, - params: &P, - account: AccountId, -) -> Result>, SqliteClientError> { - let mut ret: HashMap> = HashMap::new(); - - // Get all UAs derived - let mut ua_query = conn.prepare( - "SELECT address, diversifier_index_be FROM addresses WHERE account_id = :account", - )?; - let mut rows = ua_query.query(named_params![":account": account.0])?; - - while let Some(row) = rows.next()? { - let ua_str: String = row.get(0)?; - let di_vec: Vec = row.get(1)?; - let mut di: [u8; 11] = di_vec.try_into().map_err(|_| { - SqliteClientError::CorruptedData("Diversifier index is not an 11-byte value".to_owned()) - })?; - di.reverse(); // BE -> LE conversion - - let ua = Address::decode(params, &ua_str) - .ok_or_else(|| { - SqliteClientError::CorruptedData("Not a valid Zcash recipient address".to_owned()) - }) - .and_then(|addr| match addr { - Address::Unified(ua) => Ok(ua), - _ => Err(SqliteClientError::CorruptedData(format!( - "Addresses table contains {} which is not a unified address", - ua_str, - ))), - })?; - - if let Some(taddr) = ua.transparent() { - let index = NonHardenedChildIndex::from_index( - DiversifierIndex::from(di).try_into().map_err(|_| { - SqliteClientError::CorruptedData( - "Unable to get diversifier for transparent address.".to_string(), - ) - })?, - ) - .ok_or_else(|| { - SqliteClientError::CorruptedData( - "Unexpected hardened index for transparent address.".to_string(), - ) - })?; - - ret.insert( - *taddr, - Some(TransparentAddressMetadata::new( - Scope::External.into(), - index, - )), - ); - } - } - - if let Some((taddr, address_index)) = get_legacy_transparent_address(params, conn, account)? { - ret.insert( - taddr, - Some(TransparentAddressMetadata::new( - Scope::External.into(), - address_index, - )), - ); - } - - Ok(ret) -} - -#[cfg(feature = "transparent-inputs")] -pub(crate) fn get_legacy_transparent_address( - params: &P, - conn: &rusqlite::Connection, - account_id: AccountId, -) -> Result, SqliteClientError> { - use zcash_address::unified::Container; - use zcash_primitives::legacy::keys::ExternalIvk; - - // Get the UIVK for the account. - let uivk_str: Option = conn - .query_row( - "SELECT uivk FROM accounts WHERE id = :account", - [account_id.0], - |row| row.get(0), - ) - .optional()?; - - if let Some(uivk_str) = uivk_str { - let (network, uivk) = Uivk::decode(&uivk_str) - .map_err(|e| SqliteClientError::CorruptedData(format!("Unable to parse UIVK: {e}")))?; - if params.network_type() != network { - return Err(SqliteClientError::CorruptedData( - "Network type mismatch".to_owned(), - )); - } - - // Derive the default transparent address (if it wasn't already part of a derived UA). - for item in uivk.items() { - if let Ivk::P2pkh(tivk_bytes) = item { - let tivk = ExternalIvk::deserialize(&tivk_bytes)?; - return Ok(Some(tivk.default_address())); - } - } - } - - Ok(None) -} - /// Returns the [`UnifiedFullViewingKey`]s for the wallet. pub(crate) fn get_unified_full_viewing_keys( conn: &rusqlite::Connection, @@ -1293,45 +1167,12 @@ pub(crate) fn get_wallet_summary( drop(sapling_trace); #[cfg(feature = "transparent-inputs")] - { - let transparent_trace = tracing::info_span!("stmt_transparent_balances").entered(); - let zero_conf_height = (chain_tip_height + 1).saturating_sub(min_confirmations); - let stable_height = chain_tip_height.saturating_sub(PRUNING_DEPTH); - - let mut stmt_transparent_balances = tx.prepare( - "SELECT u.received_by_account_id, SUM(u.value_zat) - FROM utxos u - WHERE u.height <= :max_height - -- and the received txo is unspent - AND u.id NOT IN ( - SELECT transparent_received_output_id - FROM transparent_received_output_spends txo_spends - JOIN transactions tx - ON tx.id_tx = txo_spends.transaction_id - WHERE tx.block IS NOT NULL -- the spending tx is mined - OR tx.expiry_height IS NULL -- the spending tx will not expire - OR tx.expiry_height > :stable_height -- the spending tx is unexpired - ) - GROUP BY u.received_by_account_id", - )?; - let mut rows = stmt_transparent_balances.query(named_params![ - ":max_height": u32::from(zero_conf_height), - ":stable_height": u32::from(stable_height) - ])?; - - while let Some(row) = rows.next()? { - let account = AccountId(row.get(0)?); - let raw_value = row.get(1)?; - let value = NonNegativeAmount::from_nonnegative_i64(raw_value).map_err(|_| { - SqliteClientError::CorruptedData(format!("Negative UTXO value {:?}", raw_value)) - })?; - - if let Some(balances) = account_balances.get_mut(&account) { - balances.add_unshielded_value(value)?; - } - } - drop(transparent_trace); - } + transparent::add_transparent_account_balances( + tx, + chain_tip_height, + min_confirmations, + &mut account_balances, + )?; // The approach used here for Sapling and Orchard subtree indexing was a quick hack // that has not yet been replaced. TODO: Make less hacky. @@ -2076,172 +1917,6 @@ pub(crate) fn truncate_to_height( Ok(()) } -#[cfg(feature = "transparent-inputs")] -fn to_unspent_transparent_output(row: &Row) -> Result { - let txid: Vec = row.get("prevout_txid")?; - let mut txid_bytes = [0u8; 32]; - txid_bytes.copy_from_slice(&txid); - - let index: u32 = row.get("prevout_idx")?; - let script_pubkey = Script(row.get("script")?); - let raw_value: i64 = row.get("value_zat")?; - let value = NonNegativeAmount::from_nonnegative_i64(raw_value).map_err(|_| { - SqliteClientError::CorruptedData(format!("Invalid UTXO value: {}", raw_value)) - })?; - let height: u32 = row.get("height")?; - - let outpoint = OutPoint::new(txid_bytes, index); - WalletTransparentOutput::from_parts( - outpoint, - TxOut { - value, - script_pubkey, - }, - BlockHeight::from(height), - ) - .ok_or_else(|| { - SqliteClientError::CorruptedData( - "Txout script_pubkey value did not correspond to a P2PKH or P2SH address".to_string(), - ) - }) -} - -#[cfg(feature = "transparent-inputs")] -pub(crate) fn get_unspent_transparent_output( - conn: &rusqlite::Connection, - outpoint: &OutPoint, -) -> Result, SqliteClientError> { - let mut stmt_select_utxo = conn.prepare_cached( - "SELECT u.prevout_txid, u.prevout_idx, u.script, u.value_zat, u.height - FROM utxos u - WHERE u.prevout_txid = :txid - AND u.prevout_idx = :output_index - AND u.id NOT IN ( - SELECT txo_spends.transparent_received_output_id - FROM transparent_received_output_spends txo_spends - JOIN transactions tx ON tx.id_tx = txo_spends.transaction_id - WHERE tx.block IS NOT NULL -- the spending tx is mined - OR tx.expiry_height IS NULL -- the spending tx will not expire - )", - )?; - - let result: Result, SqliteClientError> = stmt_select_utxo - .query_and_then( - named_params![ - ":txid": outpoint.hash(), - ":output_index": outpoint.n() - ], - to_unspent_transparent_output, - )? - .next() - .transpose(); - - result -} - -/// Returns unspent transparent outputs that have been received by this wallet at the given -/// transparent address, such that the block that included the transaction was mined at a -/// height less than or equal to the provided `max_height`. -#[cfg(feature = "transparent-inputs")] -pub(crate) fn get_unspent_transparent_outputs( - conn: &rusqlite::Connection, - params: &P, - address: &TransparentAddress, - max_height: BlockHeight, - exclude: &[OutPoint], -) -> Result, SqliteClientError> { - let chain_tip_height = scan_queue_extrema(conn)?.map(|range| *range.end()); - let stable_height = chain_tip_height - .unwrap_or(max_height) - .saturating_sub(PRUNING_DEPTH); - - let mut stmt_utxos = conn.prepare( - "SELECT u.prevout_txid, u.prevout_idx, u.script, - u.value_zat, u.height - FROM utxos u - WHERE u.address = :address - AND u.height <= :max_height - AND u.id NOT IN ( - SELECT txo_spends.transparent_received_output_id - FROM transparent_received_output_spends txo_spends - JOIN transactions tx ON tx.id_tx = txo_spends.transaction_id - WHERE - tx.block IS NOT NULL -- the spending tx is mined - OR tx.expiry_height IS NULL -- the spending tx will not expire - OR tx.expiry_height > :stable_height -- the spending tx is unexpired - )", - )?; - - let addr_str = address.encode(params); - - let mut utxos = Vec::::new(); - let mut rows = stmt_utxos.query(named_params![ - ":address": addr_str, - ":max_height": u32::from(max_height), - ":stable_height": u32::from(stable_height), - ])?; - let excluded: BTreeSet = exclude.iter().cloned().collect(); - while let Some(row) = rows.next()? { - let output = to_unspent_transparent_output(row)?; - if excluded.contains(output.outpoint()) { - continue; - } - - utxos.push(output); - } - - Ok(utxos) -} - -/// Returns the unspent balance for each transparent address associated with the specified account, -/// such that the block that included the transaction was mined at a height less than or equal to -/// the provided `max_height`. -#[cfg(feature = "transparent-inputs")] -pub(crate) fn get_transparent_balances( - conn: &rusqlite::Connection, - params: &P, - account: AccountId, - max_height: BlockHeight, -) -> Result, SqliteClientError> { - let chain_tip_height = scan_queue_extrema(conn)?.map(|range| *range.end()); - let stable_height = chain_tip_height - .unwrap_or(max_height) - .saturating_sub(PRUNING_DEPTH); - - let mut stmt_blocks = conn.prepare( - "SELECT u.address, SUM(u.value_zat) - FROM utxos u - WHERE u.received_by_account_id = :account_id - AND u.height <= :max_height - AND u.id NOT IN ( - SELECT txo_spends.transparent_received_output_id - FROM transparent_received_output_spends txo_spends - JOIN transactions tx ON tx.id_tx = txo_spends.transaction_id - WHERE - tx.block IS NOT NULL -- the spending tx is mined - OR tx.expiry_height IS NULL -- the spending tx will not expire - OR tx.expiry_height > :stable_height -- the spending tx is unexpired - ) - GROUP BY u.address", - )?; - - let mut res = HashMap::new(); - let mut rows = stmt_blocks.query(named_params![ - ":account_id": account.0, - ":max_height": u32::from(max_height), - ":stable_height": u32::from(stable_height), - ])?; - while let Some(row) = rows.next()? { - let taddr_str: String = row.get(0)?; - let taddr = TransparentAddress::decode(params, &taddr_str)?; - let value = NonNegativeAmount::from_nonnegative_i64(row.get(1)?)?; - - res.insert(taddr, value); - } - - Ok(res) -} - /// Returns a vector with the IDs of all accounts known to this wallet. pub(crate) fn get_account_ids( conn: &rusqlite::Connection, @@ -2443,119 +2118,6 @@ pub(crate) fn put_tx_data( .map_err(SqliteClientError::from) } -/// Marks the given UTXO as having been spent. -#[cfg(feature = "transparent-inputs")] -pub(crate) fn mark_transparent_utxo_spent( - conn: &rusqlite::Connection, - tx_ref: i64, - outpoint: &OutPoint, -) -> Result<(), SqliteClientError> { - let mut stmt_mark_transparent_utxo_spent = conn.prepare_cached( - "INSERT INTO transparent_received_output_spends (transparent_received_output_id, transaction_id) - SELECT txo.id, :spent_in_tx - FROM utxos txo - WHERE txo.prevout_txid = :prevout_txid - AND txo.prevout_idx = :prevout_idx - ON CONFLICT (transparent_received_output_id, transaction_id) DO NOTHING", - )?; - - let sql_args = named_params![ - ":spent_in_tx": &tx_ref, - ":prevout_txid": &outpoint.hash().to_vec(), - ":prevout_idx": &outpoint.n(), - ]; - - stmt_mark_transparent_utxo_spent.execute(sql_args)?; - Ok(()) -} - -/// Adds the given received UTXO to the datastore. -#[cfg(feature = "transparent-inputs")] -pub(crate) fn put_received_transparent_utxo( - conn: &rusqlite::Connection, - params: &P, - output: &WalletTransparentOutput, -) -> Result { - let address_str = output.recipient_address().encode(params); - let account_id = conn - .query_row( - "SELECT account_id FROM addresses WHERE cached_transparent_receiver_address = :address", - named_params![":address": &address_str], - |row| Ok(AccountId(row.get(0)?)), - ) - .optional()?; - - if let Some(account) = account_id { - Ok(put_legacy_transparent_utxo(conn, params, output, account)?) - } else { - // If the UTXO is received at the legacy transparent address (at BIP 44 address - // index 0 within its particular account, which we specifically ensure is returned - // from `get_transparent_receivers`), there may be no entry in the addresses table - // that can be used to tie the address to a particular account. In this case, we - // look up the legacy address for each account in the wallet, and check whether it - // matches the address for the received UTXO; if so, insert/update it directly. - get_account_ids(conn)? - .into_iter() - .find_map( - |account| match get_legacy_transparent_address(params, conn, account) { - Ok(Some((legacy_taddr, _))) if &legacy_taddr == output.recipient_address() => { - Some( - put_legacy_transparent_utxo(conn, params, output, account) - .map_err(SqliteClientError::from), - ) - } - Ok(_) => None, - Err(e) => Some(Err(e)), - }, - ) - // The UTXO was not for any of the legacy transparent addresses. - .unwrap_or_else(|| { - Err(SqliteClientError::AddressNotRecognized( - *output.recipient_address(), - )) - }) - } -} - -#[cfg(feature = "transparent-inputs")] -pub(crate) fn put_legacy_transparent_utxo( - conn: &rusqlite::Connection, - params: &P, - output: &WalletTransparentOutput, - received_by_account: AccountId, -) -> Result { - #[cfg(feature = "transparent-inputs")] - let mut stmt_upsert_legacy_transparent_utxo = conn.prepare_cached( - "INSERT INTO utxos ( - prevout_txid, prevout_idx, - received_by_account_id, address, script, - value_zat, height) - VALUES - (:prevout_txid, :prevout_idx, - :received_by_account_id, :address, :script, - :value_zat, :height) - ON CONFLICT (prevout_txid, prevout_idx) DO UPDATE - SET received_by_account_id = :received_by_account_id, - height = :height, - address = :address, - script = :script, - value_zat = :value_zat - RETURNING id", - )?; - - let sql_args = named_params![ - ":prevout_txid": &output.outpoint().hash().to_vec(), - ":prevout_idx": &output.outpoint().n(), - ":received_by_account_id": received_by_account.0, - ":address": &output.recipient_address().encode(params), - ":script": &output.txout().script_pubkey.0, - ":value_zat": &i64::from(Amount::from(output.txout().value)), - ":height": &u32::from(output.height()), - ]; - - stmt_upsert_legacy_transparent_utxo.query_row(sql_args, |row| row.get::<_, i64>(0).map(UtxoId)) -} - // A utility function for creation of parameters for use in `insert_sent_output` // and `put_sent_output` fn recipient_params( @@ -2841,24 +2403,6 @@ mod tests { use super::account_birthday; - #[cfg(feature = "transparent-inputs")] - use { - crate::PRUNING_DEPTH, - zcash_client_backend::{ - data_api::{wallet::input_selection::GreedyInputSelector, InputSource, WalletWrite}, - encoding::AddressCodec, - fees::{fixed, DustOutputPolicy}, - wallet::WalletTransparentOutput, - }, - zcash_primitives::{ - consensus::BlockHeight, - transaction::{ - components::{OutPoint, TxOut}, - fees::fixed::FeeRule as FixedFeeRule, - }, - }, - }; - #[test] fn empty_database_has_no_balance() { let st = TestBuilder::new() @@ -2891,106 +2435,6 @@ mod tests { ); } - #[test] - #[cfg(feature = "transparent-inputs")] - fn put_received_transparent_utxo() { - use crate::testing::TestBuilder; - - let mut st = TestBuilder::new() - .with_account_from_sapling_activation(BlockHash([0; 32])) - .build(); - - let account_id = st.test_account().unwrap().account_id(); - let uaddr = st - .wallet() - .get_current_address(account_id) - .unwrap() - .unwrap(); - let taddr = uaddr.transparent().unwrap(); - - let height_1 = BlockHeight::from_u32(12345); - let bal_absent = st - .wallet() - .get_transparent_balances(account_id, height_1) - .unwrap(); - assert!(bal_absent.is_empty()); - - // Create a fake transparent output. - let value = NonNegativeAmount::const_from_u64(100000); - let outpoint = OutPoint::fake(); - let txout = TxOut { - value, - script_pubkey: taddr.script(), - }; - - // Pretend the output's transaction was mined at `height_1`. - let utxo = - WalletTransparentOutput::from_parts(outpoint.clone(), txout.clone(), height_1).unwrap(); - let res0 = st.wallet_mut().put_received_transparent_utxo(&utxo); - assert_matches!(res0, Ok(_)); - - // Confirm that we see the output unspent as of `height_1`. - assert_matches!( - st.wallet().get_unspent_transparent_outputs( - taddr, - height_1, - &[] - ).as_deref(), - Ok([ret]) if (ret.outpoint(), ret.txout(), ret.height()) == (utxo.outpoint(), utxo.txout(), height_1) - ); - assert_matches!( - st.wallet().get_unspent_transparent_output(utxo.outpoint()), - Ok(Some(ret)) if (ret.outpoint(), ret.txout(), ret.height()) == (utxo.outpoint(), utxo.txout(), height_1) - ); - - // Change the mined height of the UTXO and upsert; we should get back - // the same `UtxoId`. - let height_2 = BlockHeight::from_u32(34567); - let utxo2 = WalletTransparentOutput::from_parts(outpoint, txout, height_2).unwrap(); - let res1 = st.wallet_mut().put_received_transparent_utxo(&utxo2); - assert_matches!(res1, Ok(id) if id == res0.unwrap()); - - // Confirm that we no longer see any unspent outputs as of `height_1`. - assert_matches!( - st.wallet() - .get_unspent_transparent_outputs(taddr, height_1, &[]) - .as_deref(), - Ok(&[]) - ); - - // We can still look up the specific output, and it has the expected height. - assert_matches!( - st.wallet().get_unspent_transparent_output(utxo2.outpoint()), - Ok(Some(ret)) if (ret.outpoint(), ret.txout(), ret.height()) == (utxo2.outpoint(), utxo2.txout(), height_2) - ); - - // If we include `height_2` then the output is returned. - assert_matches!( - st.wallet() - .get_unspent_transparent_outputs(taddr, height_2, &[]) - .as_deref(), - Ok([ret]) if (ret.outpoint(), ret.txout(), ret.height()) == (utxo.outpoint(), utxo.txout(), height_2) - ); - - assert_matches!( - st.wallet().get_transparent_balances(account_id, height_2), - Ok(h) if h.get(taddr) == Some(&value) - ); - - // Artificially delete the address from the addresses table so that - // we can ensure the update fails if the join doesn't work. - st.wallet() - .conn - .execute( - "DELETE FROM addresses WHERE cached_transparent_receiver_address = ?", - [Some(taddr.encode(&st.wallet().params))], - ) - .unwrap(); - - let res2 = st.wallet_mut().put_received_transparent_utxo(&utxo2); - assert_matches!(res2, Err(_)); - } - #[test] fn get_default_account_index() { use crate::testing::TestBuilder; @@ -3027,159 +2471,6 @@ mod tests { } } - #[test] - #[cfg(feature = "transparent-inputs")] - fn transparent_balance_across_shielding() { - use zcash_client_backend::ShieldedProtocol; - - let mut st = TestBuilder::new() - .with_block_cache() - .with_account_from_sapling_activation(BlockHash([0; 32])) - .build(); - - let account = st.test_account().cloned().unwrap(); - let uaddr = st - .wallet() - .get_current_address(account.account_id()) - .unwrap() - .unwrap(); - let taddr = uaddr.transparent().unwrap(); - - // Initialize the wallet with chain data that has no shielded notes for us. - let not_our_key = ExtendedSpendingKey::master(&[]).to_diversifiable_full_viewing_key(); - let not_our_value = NonNegativeAmount::const_from_u64(10000); - let (start_height, _, _) = - st.generate_next_block(¬_our_key, AddressType::DefaultExternal, not_our_value); - for _ in 1..10 { - st.generate_next_block(¬_our_key, AddressType::DefaultExternal, not_our_value); - } - st.scan_cached_blocks(start_height, 10); - - let check_balance = |st: &TestState<_>, min_confirmations: u32, expected| { - // Check the wallet summary returns the expected transparent balance. - let summary = st - .wallet() - .get_wallet_summary(min_confirmations) - .unwrap() - .unwrap(); - let balance = summary - .account_balances() - .get(&account.account_id()) - .unwrap(); - assert_eq!(balance.unshielded(), expected); - - // Check the older APIs for consistency. - let max_height = st.wallet().chain_height().unwrap().unwrap() + 1 - min_confirmations; - assert_eq!( - st.wallet() - .get_transparent_balances(account.account_id(), max_height) - .unwrap() - .get(taddr) - .cloned() - .unwrap_or(NonNegativeAmount::ZERO), - expected, - ); - assert_eq!( - st.wallet() - .get_unspent_transparent_outputs(taddr, max_height, &[]) - .unwrap() - .into_iter() - .map(|utxo| utxo.value()) - .sum::>(), - Some(expected), - ); - }; - - // The wallet starts out with zero balance. - check_balance(&st, 0, NonNegativeAmount::ZERO); - check_balance(&st, 1, NonNegativeAmount::ZERO); - - // Create a fake transparent output. - let value = NonNegativeAmount::from_u64(100000).unwrap(); - let txout = TxOut { - value, - script_pubkey: taddr.script(), - }; - - // Pretend the output was received in the chain tip. - let height = st.wallet().chain_height().unwrap().unwrap(); - let utxo = WalletTransparentOutput::from_parts(OutPoint::fake(), txout, height).unwrap(); - st.wallet_mut() - .put_received_transparent_utxo(&utxo) - .unwrap(); - - // The wallet should detect the balance as having 1 confirmation. - check_balance(&st, 0, value); - check_balance(&st, 1, value); - check_balance(&st, 2, NonNegativeAmount::ZERO); - - // Shield the output. - let input_selector = GreedyInputSelector::new( - fixed::SingleOutputChangeStrategy::new( - FixedFeeRule::non_standard(NonNegativeAmount::ZERO), - None, - ShieldedProtocol::Sapling, - ), - DustOutputPolicy::default(), - ); - let txid = st - .shield_transparent_funds(&input_selector, value, account.usk(), &[*taddr], 1) - .unwrap()[0]; - - // The wallet should have zero transparent balance, because the shielding - // transaction can be mined. - check_balance(&st, 0, NonNegativeAmount::ZERO); - check_balance(&st, 1, NonNegativeAmount::ZERO); - check_balance(&st, 2, NonNegativeAmount::ZERO); - - // Mine the shielding transaction. - let (mined_height, _) = st.generate_next_block_including(txid); - st.scan_cached_blocks(mined_height, 1); - - // The wallet should still have zero transparent balance. - check_balance(&st, 0, NonNegativeAmount::ZERO); - check_balance(&st, 1, NonNegativeAmount::ZERO); - check_balance(&st, 2, NonNegativeAmount::ZERO); - - // Unmine the shielding transaction via a reorg. - st.wallet_mut() - .truncate_to_height(mined_height - 1) - .unwrap(); - assert_eq!(st.wallet().chain_height().unwrap(), Some(mined_height - 1)); - - // The wallet should still have zero transparent balance. - check_balance(&st, 0, NonNegativeAmount::ZERO); - check_balance(&st, 1, NonNegativeAmount::ZERO); - check_balance(&st, 2, NonNegativeAmount::ZERO); - - // Expire the shielding transaction. - let expiry_height = st - .wallet() - .get_transaction(txid) - .unwrap() - .expect("Transaction exists in the wallet.") - .expiry_height(); - st.wallet_mut().update_chain_tip(expiry_height).unwrap(); - - // TODO: Making the transparent output spendable in this situation requires - // changes to the transparent data model, so for now the wallet should still have - // zero transparent balance. https://github.com/zcash/librustzcash/issues/986 - check_balance(&st, 0, NonNegativeAmount::ZERO); - check_balance(&st, 1, NonNegativeAmount::ZERO); - check_balance(&st, 2, NonNegativeAmount::ZERO); - - // Roll forward the chain tip until the transaction's expiry height is in the - // stable block range (so a reorg won't make it spendable again). - st.wallet_mut() - .update_chain_tip(expiry_height + PRUNING_DEPTH) - .unwrap(); - - // The transparent output should be spendable again, with more confirmations. - check_balance(&st, 0, value); - check_balance(&st, 1, value); - check_balance(&st, 2, value); - } - #[test] fn block_fully_scanned() { let mut st = TestBuilder::new() diff --git a/zcash_client_sqlite/src/wallet/transparent.rs b/zcash_client_sqlite/src/wallet/transparent.rs index 63761a7100..d3190fe02c 100644 --- a/zcash_client_sqlite/src/wallet/transparent.rs +++ b/zcash_client_sqlite/src/wallet/transparent.rs @@ -1,10 +1,29 @@ //! Functions for transparent input support in the wallet. +use rusqlite::OptionalExtension; +use rusqlite::{named_params, Connection, Row}; +use std::collections::BTreeSet; +use std::collections::HashMap; use std::collections::HashSet; +use zcash_client_backend::data_api::AccountBalance; +use zcash_keys::address::Address; +use zip32::{DiversifierIndex, Scope}; -use rusqlite::{named_params, Connection}; -use zcash_primitives::transaction::components::OutPoint; +use zcash_address::unified::{Encoding, Ivk, Uivk}; +use zcash_client_backend::wallet::{TransparentAddressMetadata, WalletTransparentOutput}; +use zcash_keys::encoding::AddressCodec; +use zcash_primitives::{ + legacy::{ + keys::{IncomingViewingKey, NonHardenedChildIndex}, + Script, TransparentAddress, + }, + transaction::components::{amount::NonNegativeAmount, Amount, OutPoint, TxOut}, +}; +use zcash_protocol::consensus::{self, BlockHeight}; -use crate::AccountId; +use crate::{error::SqliteClientError, AccountId, UtxoId, PRUNING_DEPTH}; + +use super::get_account_ids; +use super::scan_queue_extrema; pub(crate) fn detect_spending_accounts<'a>( conn: &Connection, @@ -32,3 +51,705 @@ pub(crate) fn detect_spending_accounts<'a>( Ok(acc) } + +pub(crate) fn get_transparent_receivers( + conn: &rusqlite::Connection, + params: &P, + account: AccountId, +) -> Result>, SqliteClientError> { + let mut ret: HashMap> = HashMap::new(); + + // Get all UAs derived + let mut ua_query = conn.prepare( + "SELECT address, diversifier_index_be FROM addresses WHERE account_id = :account", + )?; + let mut rows = ua_query.query(named_params![":account": account.0])?; + + while let Some(row) = rows.next()? { + let ua_str: String = row.get(0)?; + let di_vec: Vec = row.get(1)?; + let mut di: [u8; 11] = di_vec.try_into().map_err(|_| { + SqliteClientError::CorruptedData("Diversifier index is not an 11-byte value".to_owned()) + })?; + di.reverse(); // BE -> LE conversion + + let ua = Address::decode(params, &ua_str) + .ok_or_else(|| { + SqliteClientError::CorruptedData("Not a valid Zcash recipient address".to_owned()) + }) + .and_then(|addr| match addr { + Address::Unified(ua) => Ok(ua), + _ => Err(SqliteClientError::CorruptedData(format!( + "Addresses table contains {} which is not a unified address", + ua_str, + ))), + })?; + + if let Some(taddr) = ua.transparent() { + let index = NonHardenedChildIndex::from_index( + DiversifierIndex::from(di).try_into().map_err(|_| { + SqliteClientError::CorruptedData( + "Unable to get diversifier for transparent address.".to_string(), + ) + })?, + ) + .ok_or_else(|| { + SqliteClientError::CorruptedData( + "Unexpected hardened index for transparent address.".to_string(), + ) + })?; + + ret.insert( + *taddr, + Some(TransparentAddressMetadata::new( + Scope::External.into(), + index, + )), + ); + } + } + + if let Some((taddr, address_index)) = get_legacy_transparent_address(params, conn, account)? { + ret.insert( + taddr, + Some(TransparentAddressMetadata::new( + Scope::External.into(), + address_index, + )), + ); + } + + Ok(ret) +} + +pub(crate) fn get_legacy_transparent_address( + params: &P, + conn: &rusqlite::Connection, + account_id: AccountId, +) -> Result, SqliteClientError> { + use zcash_address::unified::Container; + use zcash_primitives::legacy::keys::ExternalIvk; + + // Get the UIVK for the account. + let uivk_str: Option = conn + .query_row( + "SELECT uivk FROM accounts WHERE id = :account", + [account_id.0], + |row| row.get(0), + ) + .optional()?; + + if let Some(uivk_str) = uivk_str { + let (network, uivk) = Uivk::decode(&uivk_str) + .map_err(|e| SqliteClientError::CorruptedData(format!("Unable to parse UIVK: {e}")))?; + if params.network_type() != network { + return Err(SqliteClientError::CorruptedData( + "Network type mismatch".to_owned(), + )); + } + + // Derive the default transparent address (if it wasn't already part of a derived UA). + for item in uivk.items() { + if let Ivk::P2pkh(tivk_bytes) = item { + let tivk = ExternalIvk::deserialize(&tivk_bytes)?; + return Ok(Some(tivk.default_address())); + } + } + } + + Ok(None) +} + +fn to_unspent_transparent_output(row: &Row) -> Result { + let txid: Vec = row.get("prevout_txid")?; + let mut txid_bytes = [0u8; 32]; + txid_bytes.copy_from_slice(&txid); + + let index: u32 = row.get("prevout_idx")?; + let script_pubkey = Script(row.get("script")?); + let raw_value: i64 = row.get("value_zat")?; + let value = NonNegativeAmount::from_nonnegative_i64(raw_value).map_err(|_| { + SqliteClientError::CorruptedData(format!("Invalid UTXO value: {}", raw_value)) + })?; + let height: u32 = row.get("height")?; + + let outpoint = OutPoint::new(txid_bytes, index); + WalletTransparentOutput::from_parts( + outpoint, + TxOut { + value, + script_pubkey, + }, + BlockHeight::from(height), + ) + .ok_or_else(|| { + SqliteClientError::CorruptedData( + "Txout script_pubkey value did not correspond to a P2PKH or P2SH address".to_string(), + ) + }) +} + +pub(crate) fn get_unspent_transparent_output( + conn: &rusqlite::Connection, + outpoint: &OutPoint, +) -> Result, SqliteClientError> { + let mut stmt_select_utxo = conn.prepare_cached( + "SELECT u.prevout_txid, u.prevout_idx, u.script, u.value_zat, u.height + FROM utxos u + WHERE u.prevout_txid = :txid + AND u.prevout_idx = :output_index + AND u.id NOT IN ( + SELECT txo_spends.transparent_received_output_id + FROM transparent_received_output_spends txo_spends + JOIN transactions tx ON tx.id_tx = txo_spends.transaction_id + WHERE tx.block IS NOT NULL -- the spending tx is mined + OR tx.expiry_height IS NULL -- the spending tx will not expire + )", + )?; + + let result: Result, SqliteClientError> = stmt_select_utxo + .query_and_then( + named_params![ + ":txid": outpoint.hash(), + ":output_index": outpoint.n() + ], + to_unspent_transparent_output, + )? + .next() + .transpose(); + + result +} + +/// Returns unspent transparent outputs that have been received by this wallet at the given +/// transparent address, such that the block that included the transaction was mined at a +/// height less than or equal to the provided `max_height`. +pub(crate) fn get_unspent_transparent_outputs( + conn: &rusqlite::Connection, + params: &P, + address: &TransparentAddress, + max_height: BlockHeight, + exclude: &[OutPoint], +) -> Result, SqliteClientError> { + let chain_tip_height = scan_queue_extrema(conn)?.map(|range| *range.end()); + let stable_height = chain_tip_height + .unwrap_or(max_height) + .saturating_sub(PRUNING_DEPTH); + + let mut stmt_utxos = conn.prepare( + "SELECT u.prevout_txid, u.prevout_idx, u.script, + u.value_zat, u.height + FROM utxos u + WHERE u.address = :address + AND u.height <= :max_height + AND u.id NOT IN ( + SELECT txo_spends.transparent_received_output_id + FROM transparent_received_output_spends txo_spends + JOIN transactions tx ON tx.id_tx = txo_spends.transaction_id + WHERE + tx.block IS NOT NULL -- the spending tx is mined + OR tx.expiry_height IS NULL -- the spending tx will not expire + OR tx.expiry_height > :stable_height -- the spending tx is unexpired + )", + )?; + + let addr_str = address.encode(params); + + let mut utxos = Vec::::new(); + let mut rows = stmt_utxos.query(named_params![ + ":address": addr_str, + ":max_height": u32::from(max_height), + ":stable_height": u32::from(stable_height), + ])?; + let excluded: BTreeSet = exclude.iter().cloned().collect(); + while let Some(row) = rows.next()? { + let output = to_unspent_transparent_output(row)?; + if excluded.contains(output.outpoint()) { + continue; + } + + utxos.push(output); + } + + Ok(utxos) +} + +/// Returns the unspent balance for each transparent address associated with the specified account, +/// such that the block that included the transaction was mined at a height less than or equal to +/// the provided `max_height`. +pub(crate) fn get_transparent_address_balances( + conn: &rusqlite::Connection, + params: &P, + account: AccountId, + max_height: BlockHeight, +) -> Result, SqliteClientError> { + let chain_tip_height = scan_queue_extrema(conn)?.map(|range| *range.end()); + let stable_height = chain_tip_height + .unwrap_or(max_height) + .saturating_sub(PRUNING_DEPTH); + + let mut stmt_address_balances = conn.prepare( + "SELECT u.address, SUM(u.value_zat) + FROM utxos u + WHERE u.received_by_account_id = :account_id + AND u.height <= :max_height + AND u.id NOT IN ( + SELECT txo_spends.transparent_received_output_id + FROM transparent_received_output_spends txo_spends + JOIN transactions tx ON tx.id_tx = txo_spends.transaction_id + WHERE + tx.block IS NOT NULL -- the spending tx is mined + OR tx.expiry_height IS NULL -- the spending tx will not expire + OR tx.expiry_height > :stable_height -- the spending tx is unexpired + ) + GROUP BY u.address", + )?; + + let mut res = HashMap::new(); + let mut rows = stmt_address_balances.query(named_params![ + ":account_id": account.0, + ":max_height": u32::from(max_height), + ":stable_height": u32::from(stable_height), + ])?; + while let Some(row) = rows.next()? { + let taddr_str: String = row.get(0)?; + let taddr = TransparentAddress::decode(params, &taddr_str)?; + let value = NonNegativeAmount::from_nonnegative_i64(row.get(1)?)?; + + res.insert(taddr, value); + } + + Ok(res) +} + +pub(crate) fn add_transparent_account_balances( + conn: &rusqlite::Connection, + chain_tip_height: BlockHeight, + min_confirmations: u32, + account_balances: &mut HashMap, +) -> Result<(), SqliteClientError> { + let transparent_trace = tracing::info_span!("stmt_transparent_balances").entered(); + let zero_conf_height = (chain_tip_height + 1).saturating_sub(min_confirmations); + let stable_height = chain_tip_height.saturating_sub(PRUNING_DEPTH); + + let mut stmt_transparent_balances = conn.prepare( + "SELECT u.received_by_account_id, SUM(u.value_zat) + FROM utxos u + WHERE u.height <= :max_height + -- and the received txo is unspent + AND u.id NOT IN ( + SELECT transparent_received_output_id + FROM transparent_received_output_spends txo_spends + JOIN transactions tx + ON tx.id_tx = txo_spends.transaction_id + WHERE tx.block IS NOT NULL -- the spending tx is mined + OR tx.expiry_height IS NULL -- the spending tx will not expire + OR tx.expiry_height > :stable_height -- the spending tx is unexpired + ) + GROUP BY u.received_by_account_id", + )?; + let mut rows = stmt_transparent_balances.query(named_params![ + ":max_height": u32::from(zero_conf_height), + ":stable_height": u32::from(stable_height) + ])?; + + while let Some(row) = rows.next()? { + let account = AccountId(row.get(0)?); + let raw_value = row.get(1)?; + let value = NonNegativeAmount::from_nonnegative_i64(raw_value).map_err(|_| { + SqliteClientError::CorruptedData(format!("Negative UTXO value {:?}", raw_value)) + })?; + + if let Some(balances) = account_balances.get_mut(&account) { + balances.add_unshielded_value(value)?; + } + } + drop(transparent_trace); + Ok(()) +} + +/// Marks the given UTXO as having been spent. +pub(crate) fn mark_transparent_utxo_spent( + conn: &rusqlite::Connection, + tx_ref: i64, + outpoint: &OutPoint, +) -> Result<(), SqliteClientError> { + let mut stmt_mark_transparent_utxo_spent = conn.prepare_cached( + "INSERT INTO transparent_received_output_spends (transparent_received_output_id, transaction_id) + SELECT txo.id, :spent_in_tx + FROM utxos txo + WHERE txo.prevout_txid = :prevout_txid + AND txo.prevout_idx = :prevout_idx + ON CONFLICT (transparent_received_output_id, transaction_id) DO NOTHING", + )?; + + let sql_args = named_params![ + ":spent_in_tx": &tx_ref, + ":prevout_txid": &outpoint.hash().to_vec(), + ":prevout_idx": &outpoint.n(), + ]; + + stmt_mark_transparent_utxo_spent.execute(sql_args)?; + Ok(()) +} + +/// Adds the given received UTXO to the datastore. +pub(crate) fn put_received_transparent_utxo( + conn: &rusqlite::Connection, + params: &P, + output: &WalletTransparentOutput, +) -> Result { + let address_str = output.recipient_address().encode(params); + let account_id = conn + .query_row( + "SELECT account_id FROM addresses WHERE cached_transparent_receiver_address = :address", + named_params![":address": &address_str], + |row| Ok(AccountId(row.get(0)?)), + ) + .optional()?; + + if let Some(account) = account_id { + Ok(put_legacy_transparent_utxo(conn, params, output, account)?) + } else { + // If the UTXO is received at the legacy transparent address (at BIP 44 address + // index 0 within its particular account, which we specifically ensure is returned + // from `get_transparent_receivers`), there may be no entry in the addresses table + // that can be used to tie the address to a particular account. In this case, we + // look up the legacy address for each account in the wallet, and check whether it + // matches the address for the received UTXO; if so, insert/update it directly. + get_account_ids(conn)? + .into_iter() + .find_map( + |account| match get_legacy_transparent_address(params, conn, account) { + Ok(Some((legacy_taddr, _))) if &legacy_taddr == output.recipient_address() => { + Some( + put_legacy_transparent_utxo(conn, params, output, account) + .map_err(SqliteClientError::from), + ) + } + Ok(_) => None, + Err(e) => Some(Err(e)), + }, + ) + // The UTXO was not for any of the legacy transparent addresses. + .unwrap_or_else(|| { + Err(SqliteClientError::AddressNotRecognized( + *output.recipient_address(), + )) + }) + } +} + +pub(crate) fn put_legacy_transparent_utxo( + conn: &rusqlite::Connection, + params: &P, + output: &WalletTransparentOutput, + received_by_account: AccountId, +) -> Result { + #[cfg(feature = "transparent-inputs")] + let mut stmt_upsert_legacy_transparent_utxo = conn.prepare_cached( + "INSERT INTO utxos ( + prevout_txid, prevout_idx, + received_by_account_id, address, script, + value_zat, height) + VALUES + (:prevout_txid, :prevout_idx, + :received_by_account_id, :address, :script, + :value_zat, :height) + ON CONFLICT (prevout_txid, prevout_idx) DO UPDATE + SET received_by_account_id = :received_by_account_id, + height = :height, + address = :address, + script = :script, + value_zat = :value_zat + RETURNING id", + )?; + + let sql_args = named_params![ + ":prevout_txid": &output.outpoint().hash().to_vec(), + ":prevout_idx": &output.outpoint().n(), + ":received_by_account_id": received_by_account.0, + ":address": &output.recipient_address().encode(params), + ":script": &output.txout().script_pubkey.0, + ":value_zat": &i64::from(Amount::from(output.txout().value)), + ":height": &u32::from(output.height()), + ]; + + stmt_upsert_legacy_transparent_utxo.query_row(sql_args, |row| row.get::<_, i64>(0).map(UtxoId)) +} + +#[cfg(test)] +mod tests { + use crate::{ + testing::{AddressType, TestBuilder, TestState}, + PRUNING_DEPTH, + }; + use sapling::zip32::ExtendedSpendingKey; + use zcash_client_backend::{ + data_api::{ + wallet::input_selection::GreedyInputSelector, InputSource, WalletRead, WalletWrite, + }, + encoding::AddressCodec, + fees::{fixed, DustOutputPolicy}, + wallet::WalletTransparentOutput, + }; + use zcash_primitives::{ + block::BlockHash, + consensus::BlockHeight, + transaction::{ + components::{amount::NonNegativeAmount, OutPoint, TxOut}, + fees::fixed::FeeRule as FixedFeeRule, + }, + }; + + #[test] + fn put_received_transparent_utxo() { + use crate::testing::TestBuilder; + + let mut st = TestBuilder::new() + .with_account_from_sapling_activation(BlockHash([0; 32])) + .build(); + + let account_id = st.test_account().unwrap().account_id(); + let uaddr = st + .wallet() + .get_current_address(account_id) + .unwrap() + .unwrap(); + let taddr = uaddr.transparent().unwrap(); + + let height_1 = BlockHeight::from_u32(12345); + let bal_absent = st + .wallet() + .get_transparent_balances(account_id, height_1) + .unwrap(); + assert!(bal_absent.is_empty()); + + // Create a fake transparent output. + let value = NonNegativeAmount::const_from_u64(100000); + let outpoint = OutPoint::fake(); + let txout = TxOut { + value, + script_pubkey: taddr.script(), + }; + + // Pretend the output's transaction was mined at `height_1`. + let utxo = + WalletTransparentOutput::from_parts(outpoint.clone(), txout.clone(), height_1).unwrap(); + let res0 = st.wallet_mut().put_received_transparent_utxo(&utxo); + assert_matches!(res0, Ok(_)); + + // Confirm that we see the output unspent as of `height_1`. + assert_matches!( + st.wallet().get_unspent_transparent_outputs( + taddr, + height_1, + &[] + ).as_deref(), + Ok([ret]) if (ret.outpoint(), ret.txout(), ret.height()) == (utxo.outpoint(), utxo.txout(), height_1) + ); + assert_matches!( + st.wallet().get_unspent_transparent_output(utxo.outpoint()), + Ok(Some(ret)) if (ret.outpoint(), ret.txout(), ret.height()) == (utxo.outpoint(), utxo.txout(), height_1) + ); + + // Change the mined height of the UTXO and upsert; we should get back + // the same `UtxoId`. + let height_2 = BlockHeight::from_u32(34567); + let utxo2 = WalletTransparentOutput::from_parts(outpoint, txout, height_2).unwrap(); + let res1 = st.wallet_mut().put_received_transparent_utxo(&utxo2); + assert_matches!(res1, Ok(id) if id == res0.unwrap()); + + // Confirm that we no longer see any unspent outputs as of `height_1`. + assert_matches!( + st.wallet() + .get_unspent_transparent_outputs(taddr, height_1, &[]) + .as_deref(), + Ok(&[]) + ); + + // We can still look up the specific output, and it has the expected height. + assert_matches!( + st.wallet().get_unspent_transparent_output(utxo2.outpoint()), + Ok(Some(ret)) if (ret.outpoint(), ret.txout(), ret.height()) == (utxo2.outpoint(), utxo2.txout(), height_2) + ); + + // If we include `height_2` then the output is returned. + assert_matches!( + st.wallet() + .get_unspent_transparent_outputs(taddr, height_2, &[]) + .as_deref(), + Ok([ret]) if (ret.outpoint(), ret.txout(), ret.height()) == (utxo.outpoint(), utxo.txout(), height_2) + ); + + assert_matches!( + st.wallet().get_transparent_balances(account_id, height_2), + Ok(h) if h.get(taddr) == Some(&value) + ); + + // Artificially delete the address from the addresses table so that + // we can ensure the update fails if the join doesn't work. + st.wallet() + .conn + .execute( + "DELETE FROM addresses WHERE cached_transparent_receiver_address = ?", + [Some(taddr.encode(&st.wallet().params))], + ) + .unwrap(); + + let res2 = st.wallet_mut().put_received_transparent_utxo(&utxo2); + assert_matches!(res2, Err(_)); + } + + #[test] + fn transparent_balance_across_shielding() { + use zcash_client_backend::ShieldedProtocol; + + let mut st = TestBuilder::new() + .with_block_cache() + .with_account_from_sapling_activation(BlockHash([0; 32])) + .build(); + + let account = st.test_account().cloned().unwrap(); + let uaddr = st + .wallet() + .get_current_address(account.account_id()) + .unwrap() + .unwrap(); + let taddr = uaddr.transparent().unwrap(); + + // Initialize the wallet with chain data that has no shielded notes for us. + let not_our_key = ExtendedSpendingKey::master(&[]).to_diversifiable_full_viewing_key(); + let not_our_value = NonNegativeAmount::const_from_u64(10000); + let (start_height, _, _) = + st.generate_next_block(¬_our_key, AddressType::DefaultExternal, not_our_value); + for _ in 1..10 { + st.generate_next_block(¬_our_key, AddressType::DefaultExternal, not_our_value); + } + st.scan_cached_blocks(start_height, 10); + + let check_balance = |st: &TestState<_>, min_confirmations: u32, expected| { + // Check the wallet summary returns the expected transparent balance. + let summary = st + .wallet() + .get_wallet_summary(min_confirmations) + .unwrap() + .unwrap(); + let balance = summary + .account_balances() + .get(&account.account_id()) + .unwrap(); + assert_eq!(balance.unshielded(), expected); + + // Check the older APIs for consistency. + let max_height = st.wallet().chain_height().unwrap().unwrap() + 1 - min_confirmations; + assert_eq!( + st.wallet() + .get_transparent_balances(account.account_id(), max_height) + .unwrap() + .get(taddr) + .cloned() + .unwrap_or(NonNegativeAmount::ZERO), + expected, + ); + assert_eq!( + st.wallet() + .get_unspent_transparent_outputs(taddr, max_height, &[]) + .unwrap() + .into_iter() + .map(|utxo| utxo.value()) + .sum::>(), + Some(expected), + ); + }; + + // The wallet starts out with zero balance. + check_balance(&st, 0, NonNegativeAmount::ZERO); + check_balance(&st, 1, NonNegativeAmount::ZERO); + + // Create a fake transparent output. + let value = NonNegativeAmount::from_u64(100000).unwrap(); + let txout = TxOut { + value, + script_pubkey: taddr.script(), + }; + + // Pretend the output was received in the chain tip. + let height = st.wallet().chain_height().unwrap().unwrap(); + let utxo = WalletTransparentOutput::from_parts(OutPoint::fake(), txout, height).unwrap(); + st.wallet_mut() + .put_received_transparent_utxo(&utxo) + .unwrap(); + + // The wallet should detect the balance as having 1 confirmation. + check_balance(&st, 0, value); + check_balance(&st, 1, value); + check_balance(&st, 2, NonNegativeAmount::ZERO); + + // Shield the output. + let input_selector = GreedyInputSelector::new( + fixed::SingleOutputChangeStrategy::new( + FixedFeeRule::non_standard(NonNegativeAmount::ZERO), + None, + ShieldedProtocol::Sapling, + ), + DustOutputPolicy::default(), + ); + let txid = st + .shield_transparent_funds(&input_selector, value, account.usk(), &[*taddr], 1) + .unwrap()[0]; + + // The wallet should have zero transparent balance, because the shielding + // transaction can be mined. + check_balance(&st, 0, NonNegativeAmount::ZERO); + check_balance(&st, 1, NonNegativeAmount::ZERO); + check_balance(&st, 2, NonNegativeAmount::ZERO); + + // Mine the shielding transaction. + let (mined_height, _) = st.generate_next_block_including(txid); + st.scan_cached_blocks(mined_height, 1); + + // The wallet should still have zero transparent balance. + check_balance(&st, 0, NonNegativeAmount::ZERO); + check_balance(&st, 1, NonNegativeAmount::ZERO); + check_balance(&st, 2, NonNegativeAmount::ZERO); + + // Unmine the shielding transaction via a reorg. + st.wallet_mut() + .truncate_to_height(mined_height - 1) + .unwrap(); + assert_eq!(st.wallet().chain_height().unwrap(), Some(mined_height - 1)); + + // The wallet should still have zero transparent balance. + check_balance(&st, 0, NonNegativeAmount::ZERO); + check_balance(&st, 1, NonNegativeAmount::ZERO); + check_balance(&st, 2, NonNegativeAmount::ZERO); + + // Expire the shielding transaction. + let expiry_height = st + .wallet() + .get_transaction(txid) + .unwrap() + .expect("Transaction exists in the wallet.") + .expiry_height(); + st.wallet_mut().update_chain_tip(expiry_height).unwrap(); + + // TODO: Making the transparent output spendable in this situation requires + // changes to the transparent data model, so for now the wallet should still have + // zero transparent balance. https://github.com/zcash/librustzcash/issues/986 + check_balance(&st, 0, NonNegativeAmount::ZERO); + check_balance(&st, 1, NonNegativeAmount::ZERO); + check_balance(&st, 2, NonNegativeAmount::ZERO); + + // Roll forward the chain tip until the transaction's expiry height is in the + // stable block range (so a reorg won't make it spendable again). + st.wallet_mut() + .update_chain_tip(expiry_height + PRUNING_DEPTH) + .unwrap(); + + // The transparent output should be spendable again, with more confirmations. + check_balance(&st, 0, value); + check_balance(&st, 1, value); + check_balance(&st, 2, value); + } +} From 72d8df8e68c2180f06a0efe7a6ebf737b511e44d Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 30 May 2024 18:12:58 -0600 Subject: [PATCH 2/4] zcash_client_sqlite: Align handling of transparent UTXOs with that of shielded notes. Co-authored-by: Daira-Emma Hopwood Co-authored-by: Jack Grigg --- zcash_client_backend/CHANGELOG.md | 9 +- zcash_client_backend/src/data_api.rs | 18 +- .../src/data_api/wallet/input_selection.rs | 6 +- zcash_client_sqlite/src/lib.rs | 16 +- zcash_client_sqlite/src/wallet.rs | 106 +++-- zcash_client_sqlite/src/wallet/db.rs | 422 +++++++++--------- zcash_client_sqlite/src/wallet/init.rs | 8 +- .../src/wallet/init/migrations.rs | 6 +- .../init/migrations/orchard_shardtree.rs | 6 +- .../init/migrations/receiving_key_scopes.rs | 126 +++++- .../wallet/init/migrations/utxos_to_txos.rs | 328 ++++++++++++++ zcash_client_sqlite/src/wallet/transparent.rs | 337 ++++++++------ 12 files changed, 956 insertions(+), 432 deletions(-) create mode 100644 zcash_client_sqlite/src/wallet/init/migrations/utxos_to_txos.rs diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 37d7fb3f1c..b722f206f5 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -10,13 +10,14 @@ and this library adheres to Rust's notion of ### Added - `zcash_client_backend::data_api`: - `chain::BlockCache` trait, behind the `sync` feature flag. + - `WalletRead::get_spendable_transparent_outputs`. - `zcash_client_backend::fees`: - `ChangeValue::{transparent, shielded}` - `sapling::EmptyBundleView` - `orchard::EmptyBundleView` - `zcash_client_backend::scanning`: - `testing` module -- `zcash_client_backend::sync` module, behind the `sync` feature flag. +- `zcash_client_backend::sync` module, behind the `sync` feature flag ### Changed - MSRV is now 1.70.0. @@ -52,6 +53,12 @@ and this library adheres to Rust's notion of `zcash_address::ZcashAddress`. This simplifies the process of tracking the original address to which value was sent. +### Removed +- `zcash_client_backend::data_api`: + - `WalletRead::get_unspent_transparent_outputs` has been removed because its + semantics were unclear and could not be clarified. Use + `WalletRead::get_spendable_transparent_outputs` instead. + ## [0.12.1] - 2024-03-27 ### Fixed diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index a535e50e28..fc89170786 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -665,10 +665,10 @@ pub trait InputSource { exclude: &[Self::NoteRef], ) -> Result, Self::Error>; - /// Fetches a spendable transparent output. + /// Fetches the transparent output corresponding to the provided `outpoint`. /// /// Returns `Ok(None)` if the UTXO is not known to belong to the wallet or is not - /// spendable. + /// spendable as of the chain tip height. #[cfg(feature = "transparent-inputs")] fn get_unspent_transparent_output( &self, @@ -677,14 +677,18 @@ pub trait InputSource { Ok(None) } - /// Returns a list of unspent transparent UTXOs that appear in the chain at heights up to and - /// including `max_height`. + /// Returns the list of transparent outputs received at `address` such that: + /// * The transaction that produced these outputs is mined or mineable as of `max_height`. + /// * Each returned output is unspent as of the current chain tip. + /// + /// The caller should filter these outputs to ensure they respect the desired number of + /// confirmations before attempting to spend them. #[cfg(feature = "transparent-inputs")] - fn get_unspent_transparent_outputs( + fn get_spendable_transparent_outputs( &self, _address: &TransparentAddress, - _max_height: BlockHeight, - _exclude: &[OutPoint], + _target_height: BlockHeight, + _min_confirmations: u32, ) -> Result, Self::Error> { Ok(vec![]) } diff --git a/zcash_client_backend/src/data_api/wallet/input_selection.rs b/zcash_client_backend/src/data_api/wallet/input_selection.rs index 3632fa56cf..e6579aa031 100644 --- a/zcash_client_backend/src/data_api/wallet/input_selection.rs +++ b/zcash_client_backend/src/data_api/wallet/input_selection.rs @@ -580,11 +580,7 @@ where let mut transparent_inputs: Vec = source_addrs .iter() .map(|taddr| { - wallet_db.get_unspent_transparent_outputs( - taddr, - target_height - min_confirmations, - &[], - ) + wallet_db.get_spendable_transparent_outputs(taddr, target_height, min_confirmations) }) .collect::>, _>>() .map_err(InputSelectorError::DataSource)? diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index b973e7d4da..a429eee12b 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -276,18 +276,18 @@ impl, P: consensus::Parameters> InputSource for } #[cfg(feature = "transparent-inputs")] - fn get_unspent_transparent_outputs( + fn get_spendable_transparent_outputs( &self, address: &TransparentAddress, - max_height: BlockHeight, - exclude: &[OutPoint], + target_height: BlockHeight, + min_confirmations: u32, ) -> Result, Self::Error> { - wallet::transparent::get_unspent_transparent_outputs( + wallet::transparent::get_spendable_transparent_outputs( self.conn.borrow(), &self.params, address, - max_height, - exclude, + target_height, + min_confirmations, ) } } @@ -430,9 +430,7 @@ impl, P: consensus::Parameters> WalletRead for W } fn chain_height(&self) -> Result, Self::Error> { - wallet::scan_queue_extrema(self.conn.borrow()) - .map(|h| h.map(|range| *range.end())) - .map_err(SqliteClientError::from) + wallet::chain_tip_height(self.conn.borrow()).map_err(SqliteClientError::from) } fn get_block_hash(&self, block_height: BlockHeight) -> Result, Self::Error> { diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 257c6e3cfd..eb019ea81b 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -490,7 +490,7 @@ pub(crate) fn add_account( // Rewrite the scan ranges from the birthday height up to the chain tip so that we'll ensure we // re-scan to find any notes that might belong to the newly added account. - if let Some(t) = scan_queue_extrema(conn)?.map(|range| *range.end()) { + if let Some(t) = chain_tip_height(conn)? { let rescan_range = birthday.height()..(t + 1); replace_queue_entries::( @@ -952,8 +952,8 @@ pub(crate) fn get_wallet_summary( min_confirmations: u32, progress: &impl ScanProgress, ) -> Result>, SqliteClientError> { - let chain_tip_height = match scan_queue_extrema(tx)? { - Some(range) => *range.end(), + let chain_tip_height = match chain_tip_height(tx)? { + Some(h) => h, None => { return Ok(None); } @@ -1019,7 +1019,7 @@ pub(crate) fn get_wallet_summary( ) -> Result<(), SqliteClientError>, { // If the shard containing the summary height contains any unscanned ranges that start below or - // including that height, none of our balance is currently spendable. + // including that height, none of our shielded balance is currently spendable. #[tracing::instrument(skip_all)] fn is_any_spendable( conn: &rusqlite::Connection, @@ -1167,12 +1167,7 @@ pub(crate) fn get_wallet_summary( drop(sapling_trace); #[cfg(feature = "transparent-inputs")] - transparent::add_transparent_account_balances( - tx, - chain_tip_height, - min_confirmations, - &mut account_balances, - )?; + transparent::add_transparent_account_balances(tx, chain_tip_height + 1, &mut account_balances)?; // The approach used here for Sapling and Orchard subtree indexing was a quick hack // that has not yet been replaced. TODO: Make less hacky. @@ -1503,30 +1498,23 @@ pub(crate) fn get_account( } /// Returns the minimum and maximum heights of blocks in the chain which may be scanned. -pub(crate) fn scan_queue_extrema( +pub(crate) fn chain_tip_height( conn: &rusqlite::Connection, -) -> Result>, rusqlite::Error> { - conn.query_row( - "SELECT MIN(block_range_start), MAX(block_range_end) FROM scan_queue", - [], - |row| { - let min_height: Option = row.get(0)?; - let max_height: Option = row.get(1)?; - - // Scan ranges are end-exclusive, so we subtract 1 from `max_height` to obtain the - // height of the last known chain tip; - Ok(min_height - .zip(max_height.map(|h| h.saturating_sub(1))) - .map(|(min, max)| RangeInclusive::new(min.into(), max.into()))) - }, - ) +) -> Result, rusqlite::Error> { + conn.query_row("SELECT MAX(block_range_end) FROM scan_queue", [], |row| { + let max_height: Option = row.get(0)?; + + // Scan ranges are end-exclusive, so we subtract 1 from `max_height` to obtain the + // height of the last known chain tip; + Ok(max_height.map(|h| BlockHeight::from(h.saturating_sub(1)))) + }) } pub(crate) fn get_target_and_anchor_heights( conn: &rusqlite::Connection, min_confirmations: NonZeroU32, ) -> Result, rusqlite::Error> { - match scan_queue_extrema(conn)?.map(|range| *range.end()) { + match chain_tip_height(conn)? { Some(chain_tip_height) => { let sapling_anchor_height = get_max_checkpointed_height( conn, @@ -1861,9 +1849,29 @@ pub(crate) fn truncate_to_height( named_params![":new_end_height": u32::from(block_height + 1)], )?; - // If we're removing scanned blocks, we need to truncate the note commitment tree, un-mine - // transactions, and remove received transparent outputs and affected block records from the - // database. + // Mark transparent utxos as un-mined. Since the TXO is now not mined, it would ideally be + // considered to have been returned to the mempool; it _might_ be spendable in this state, but + // we must also set its max_observed_unspent_height field to NULL because the transaction may + // be rendered entirely invalid by a reorg that alters anchor(s) used in constructing shielded + // spends in the transaction. + conn.execute( + "UPDATE transparent_received_outputs + SET max_observed_unspent_height = NULL + WHERE max_observed_unspent_height > :height", + named_params![":height": u32::from(block_height)], + )?; + + // Un-mine transactions. This must be done outside of the last_scanned_height check because + // transaction entries may be created as a consequence of receiving transparent TXOs. + conn.execute( + "UPDATE transactions + SET block = NULL, mined_height = NULL, tx_index = NULL + WHERE block > ?", + [u32::from(block_height)], + )?; + + // If we're removing scanned blocks, we need to truncate the note commitment tree and remove + // affected block records from the database. if block_height < last_scanned_height { // Truncate the note commitment trees let mut wdb = WalletDb { @@ -1885,20 +1893,6 @@ pub(crate) fn truncate_to_height( // not recoverable; balance APIs must ensure that un-mined received notes // do not count towards spendability or transaction balalnce. - // Rewind utxos. It is currently necessary to delete these because we do - // not have the full transaction data for the received output. - conn.execute( - "DELETE FROM utxos WHERE height > ?", - [u32::from(block_height)], - )?; - - // Un-mine transactions. - conn.execute( - "UPDATE transactions SET block = NULL, tx_index = NULL - WHERE block IS NOT NULL AND block > ?", - [u32::from(block_height)], - )?; - // Now that they aren't depended on, delete un-mined blocks. conn.execute( "DELETE FROM blocks WHERE height > ?", @@ -2010,6 +2004,25 @@ pub(crate) fn put_block( ":orchard_action_count": orchard_action_count, ])?; + // If we now have a block corresponding to a received transparent output that had not been + // scanned at the time the UTXO was discovered, update the associated transaction record to + // refer to that block. + // + // NOTE: There's a small data corruption hazard here, in that we're relying exclusively upon + // the block height to associate the transaction to the block. This is because CompactBlock + // values only contain CompactTx entries for transactions that contain shielded inputs or + // outputs, and the GetAddressUtxosReply data does not contain the block hash. As such, it's + // necessary to ensure that any chain rollback to below the received height causes that height + // to be set to NULL. + let mut stmt_update_transaction_block_reference = conn.prepare_cached( + "UPDATE transactions + SET block = :height + WHERE mined_height = :height", + )?; + + stmt_update_transaction_block_reference + .execute(named_params![":height": u32::from(block_height),])?; + Ok(()) } @@ -2022,10 +2035,11 @@ pub(crate) fn put_tx_meta( ) -> Result { // It isn't there, so insert our transaction into the database. let mut stmt_upsert_tx_meta = conn.prepare_cached( - "INSERT INTO transactions (txid, block, tx_index) - VALUES (:txid, :block, :tx_index) + "INSERT INTO transactions (txid, block, mined_height, tx_index) + VALUES (:txid, :block, :block, :tx_index) ON CONFLICT (txid) DO UPDATE SET block = :block, + mined_height = :block, tx_index = :tx_index RETURNING id_tx", )?; diff --git a/zcash_client_sqlite/src/wallet/db.rs b/zcash_client_sqlite/src/wallet/db.rs index 17db1d728d..edd4e8b39e 100644 --- a/zcash_client_sqlite/src/wallet/db.rs +++ b/zcash_client_sqlite/src/wallet/db.rs @@ -59,9 +59,9 @@ pub(super) const INDEX_HD_ACCOUNT: &str = /// Stores diversified Unified Addresses that have been generated from accounts in the /// wallet. /// -/// - The `cached_transparent_receiver_address` column contains the transparent receiver -/// component of the UA. It is cached directly in the table to make account lookups for -/// transparent outputs more efficient, enabling joins to [`TABLE_UTXOS`]. +/// - The `cached_transparent_receiver_address` column contains the transparent receiver component +/// of the UA. It is cached directly in the table to make account lookups for transparent outputs +/// more efficient, enabling joins to [`TABLE_TRANSPARENT_RECEIVED_OUTPUTS`]. pub(super) const TABLE_ADDRESSES: &str = r#" CREATE TABLE "addresses" ( account_id INTEGER NOT NULL, @@ -80,7 +80,7 @@ CREATE INDEX "addresses_accounts" ON "addresses" ( /// /// Note that this table does not contain any rows for blocks that the wallet might have /// observed partial information about (for example, a transparent output fetched and -/// stored in [`TABLE_UTXOS`]). This may change in future. +/// stored in [`TABLE_TRANSPARENT_RECEIVED_OUTPUTS`]). This may change in future. pub(super) const TABLE_BLOCKS: &str = " CREATE TABLE blocks ( height INTEGER PRIMARY KEY, @@ -99,22 +99,32 @@ CREATE TABLE blocks ( /// data that is not recoverable from the chain (for example, transactions created by the /// wallet that expired before being mined). /// -/// - The `block` column stores the height (in the wallet's chain view) of the mined block -/// containing the transaction. It is `NULL` for transactions that have not yet been -/// observed in scanned blocks, including transactions in the mempool or that have -/// expired. -pub(super) const TABLE_TRANSACTIONS: &str = " -CREATE TABLE transactions ( +/// ### Columns +/// - `created`: The time at which the transaction was created as a string in the format +/// `yyyy-MM-dd HH:mm:ss.fffffffzzz`. +/// - `block`: stores the height (in the wallet's chain view) of the mined block containing the +/// transaction. It is `NULL` for transactions that have not yet been observed in scanned blocks, +/// including transactions in the mempool or that have expired. +/// - `mined_height`: stores the height (in the wallet's chain view) of the mined block containing +/// the transaction. It is present to allow the block height for a retrieved transaction to be +/// stored without requiring that the entire block containing the transaction be scanned; the +/// foreign key constraint on `block` prevents that column from being populated prior to complete +/// scanning of the block. This is constrained to be equal to the `block` column if `block` is +/// non-null. +pub(super) const TABLE_TRANSACTIONS: &str = r#" +CREATE TABLE "transactions" ( id_tx INTEGER PRIMARY KEY, txid BLOB NOT NULL UNIQUE, created TEXT, block INTEGER, + mined_height INTEGER, tx_index INTEGER, expiry_height INTEGER, raw BLOB, fee INTEGER, - FOREIGN KEY (block) REFERENCES blocks(height) -)"; + FOREIGN KEY (block) REFERENCES blocks(height), + CONSTRAINT height_consistency CHECK (block IS NULL OR mined_height = block) +)"#; /// Stores the Sapling notes received by the wallet. /// @@ -216,54 +226,72 @@ CREATE TABLE orchard_received_note_spends ( UNIQUE (orchard_received_note_id, transaction_id) )"; -/// Stores the current UTXO set for the wallet, as well as any transparent outputs -/// previously observed by the wallet. +/// Stores the transparent outputs received by the wallet. /// /// Originally this table only stored the current UTXO set (as of latest refresh), and the /// table was cleared prior to loading in the latest UTXO set. We now upsert instead of /// insert into the database, meaning that spent outputs are left in the database. This -/// makes it similar to the `*_received_notes` tables in that it can store history, but -/// has several downsides: -/// - The table has incomplete contents for recovered-from-seed wallets. -/// - The table can have inconsistent contents for seeds loaded into multiple wallets +/// makes it similar to the `*_received_notes` tables in that it can store history. +/// Depending upon how transparent TXOs for the wallet are discovered, the following +/// may be true: +/// - The table may have incomplete contents for recovered-from-seed wallets. +/// - The table may have inconsistent contents for seeds loaded into multiple wallets /// simultaneously. -/// - The wallet's transparent balance can be incorrect prior to "transaction enhancement" +/// - The wallet's transparent balance may be incorrect prior to "transaction enhancement" /// (downloading the full transaction containing the transparent output spend). -pub(super) const TABLE_UTXOS: &str = r#" -CREATE TABLE "utxos" ( +/// +/// ### Columns: +/// - `id`: Primary key +/// - `transaction_id`: Reference to the transaction in which this TXO was created +/// - `output_index`: The output index of this TXO in the transaction referred to by `transaction_id` +/// - `account_id`: The account that controls spend authority for this TXO +/// - `address`: The address to which this TXO was sent +/// - `script`: The full txout script +/// - `value_zat`: The value of the TXO in zatoshis +/// - `max_observed_unspent_height`: The maximum block height at which this TXO was either +/// observed to be a member of the UTXO set at the start of the block, or observed +/// to be an output of a transaction mined in the block. This is intended to be used to +/// determine when the TXO is no longer a part of the UTXO set, in the case that the +/// transaction that spends it is not detected by the wallet. +pub(super) const TABLE_TRANSPARENT_RECEIVED_OUTPUTS: &str = r#" +CREATE TABLE transparent_received_outputs ( id INTEGER PRIMARY KEY, - received_by_account_id INTEGER NOT NULL, + transaction_id INTEGER NOT NULL, + output_index INTEGER NOT NULL, + account_id INTEGER NOT NULL, address TEXT NOT NULL, - prevout_txid BLOB NOT NULL, - prevout_idx INTEGER NOT NULL, script BLOB NOT NULL, value_zat INTEGER NOT NULL, - height INTEGER NOT NULL, - FOREIGN KEY (received_by_account_id) REFERENCES accounts(id), - CONSTRAINT tx_outpoint UNIQUE (prevout_txid, prevout_idx) + max_observed_unspent_height INTEGER, + FOREIGN KEY (transaction_id) REFERENCES transactions(id_tx), + FOREIGN KEY (account_id) REFERENCES accounts(id), + CONSTRAINT transparent_output_unique UNIQUE (transaction_id, output_index) )"#; -pub(super) const INDEX_UTXOS_RECEIVED_BY_ACCOUNT: &str = - r#"CREATE INDEX utxos_received_by_account ON "utxos" (received_by_account_id)"#; +pub(super) const INDEX_TRANSPARENT_RECEIVED_OUTPUTS_ACCOUNT_ID: &str = r#" +CREATE INDEX idx_transparent_received_outputs_account_id +ON "transparent_received_outputs" (account_id)"#; -/// A junction table between received transparent outputs and the transactions that spend -/// them. +/// A junction table between received transparent outputs and the transactions that spend them. /// -/// This is identical to [`TABLE_SAPLING_RECEIVED_NOTE_SPENDS`]; see its documentation for -/// details. Note however that [`TABLE_UTXOS`] and [`TABLE_SAPLING_RECEIVED_NOTES`] are -/// not equivalent, and care must be taken when interpreting the result of joining this -/// table to [`TABLE_UTXOS`]. -pub(super) const TABLE_TRANSPARENT_RECEIVED_OUTPUT_SPENDS: &str = " -CREATE TABLE transparent_received_output_spends ( +/// This plays the same role for transparent TXOs as does [`TABLE_SAPLING_RECEIVED_NOTE_SPENDS`] +/// for Sapling notes. However, [`TABLE_TRANSPARENT_RECEIVED_OUTPUTS`] differs from +/// [`TABLE_SAPLING_RECEIVED_NOTES`] and [`TABLE_ORCHARD_RECEIVED_NOTES`] in that an +/// associated `transactions` record may have its `mined_height` set without there existing a +/// corresponding record in the `blocks` table for a block at that height, due to the asymmetries +/// between scanning for shielded notes and retrieving transparent TXOs currently implemented +/// in [`zcash_client_backend`]. +pub(super) const TABLE_TRANSPARENT_RECEIVED_OUTPUT_SPENDS: &str = r#" +CREATE TABLE "transparent_received_output_spends" ( transparent_received_output_id INTEGER NOT NULL, transaction_id INTEGER NOT NULL, FOREIGN KEY (transparent_received_output_id) - REFERENCES utxos(id) + REFERENCES transparent_received_outputs(id) ON DELETE CASCADE, FOREIGN KEY (transaction_id) -- We do not delete transactions, so this does not cascade REFERENCES transactions(id_tx), UNIQUE (transparent_received_output_id, transaction_id) -)"; +)"#; /// Stores the outputs of transactions created by the wallet. /// @@ -497,219 +525,201 @@ pub(super) const TABLE_SQLITE_SEQUENCE: &str = "CREATE TABLE sqlite_sequence(nam // Views // -pub(super) const VIEW_RECEIVED_NOTES: &str = " -CREATE VIEW v_received_notes AS -SELECT - sapling_received_notes.id AS id_within_pool_table, - sapling_received_notes.tx, - 2 AS pool, - sapling_received_notes.output_index AS output_index, - account_id, - sapling_received_notes.value, - is_change, - sapling_received_notes.memo, - sent_notes.id AS sent_note_id -FROM sapling_received_notes -LEFT JOIN sent_notes -ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = - (sapling_received_notes.tx, 2, sapling_received_notes.output_index) +pub(super) const VIEW_RECEIVED_OUTPUTS: &str = " +CREATE VIEW v_received_outputs AS + SELECT + sapling_received_notes.id AS id_within_pool_table, + sapling_received_notes.tx AS transaction_id, + 2 AS pool, + sapling_received_notes.output_index, + account_id, + sapling_received_notes.value, + is_change, + sapling_received_notes.memo, + sent_notes.id AS sent_note_id + FROM sapling_received_notes + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (sapling_received_notes.tx, 2, sapling_received_notes.output_index) UNION -SELECT - orchard_received_notes.id AS id_within_pool_table, - orchard_received_notes.tx, - 3 AS pool, - orchard_received_notes.action_index AS output_index, - account_id, - orchard_received_notes.value, - is_change, - orchard_received_notes.memo, - sent_notes.id AS sent_note_id -FROM orchard_received_notes -LEFT JOIN sent_notes -ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = - (orchard_received_notes.tx, 3, orchard_received_notes.action_index)"; - -pub(super) const VIEW_RECEIVED_NOTE_SPENDS: &str = " -CREATE VIEW v_received_note_spends AS + SELECT + orchard_received_notes.id AS id_within_pool_table, + orchard_received_notes.tx AS transaction_id, + 3 AS pool, + orchard_received_notes.action_index AS output_index, + account_id, + orchard_received_notes.value, + is_change, + orchard_received_notes.memo, + sent_notes.id AS sent_note_id + FROM orchard_received_notes + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (orchard_received_notes.tx, 3, orchard_received_notes.action_index) +UNION + SELECT + u.id AS id_within_pool_table, + u.transaction_id, + 0 AS pool, + u.output_index, + u.account_id, + u.value_zat AS value, + 0 AS is_change, + NULL AS memo, + sent_notes.id AS sent_note_id + FROM transparent_received_outputs u + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (u.transaction_id, 0, u.output_index)"; + +pub(super) const VIEW_RECEIVED_OUTPUT_SPENDS: &str = " +CREATE VIEW v_received_output_spends AS SELECT 2 AS pool, - sapling_received_note_id AS received_note_id, + sapling_received_note_id AS received_output_id, transaction_id FROM sapling_received_note_spends UNION SELECT 3 AS pool, - orchard_received_note_id AS received_note_id, + orchard_received_note_id AS received_output_id, transaction_id -FROM orchard_received_note_spends"; +FROM orchard_received_note_spends +UNION +SELECT + 0 AS pool, + transparent_received_output_id AS received_output_id, + transaction_id +FROM transparent_received_output_spends"; pub(super) const VIEW_TRANSACTIONS: &str = " CREATE VIEW v_transactions AS WITH notes AS ( - -- Shielded notes received in this transaction - SELECT v_received_notes.account_id AS account_id, - transactions.block AS block, - transactions.txid AS txid, - v_received_notes.pool AS pool, - id_within_pool_table, - v_received_notes.value AS value, - CASE - WHEN v_received_notes.is_change THEN 1 + -- Outputs received in this transaction + SELECT ro.account_id AS account_id, + transactions.mined_height AS mined_height, + transactions.txid AS txid, + ro.pool AS pool, + id_within_pool_table, + ro.value AS value, + CASE + WHEN ro.is_change THEN 1 ELSE 0 - END AS is_change, - CASE - WHEN v_received_notes.is_change THEN 0 - ELSE 1 - END AS received_count, - CASE - WHEN (v_received_notes.memo IS NULL OR v_received_notes.memo = X'F6') - THEN 0 + END AS is_change, + CASE + WHEN ro.is_change THEN 0 ELSE 1 - END AS memo_present - FROM v_received_notes + END AS received_count, + CASE + WHEN (ro.memo IS NULL OR ro.memo = X'F6') + THEN 0 + ELSE 1 + END AS memo_present + FROM v_received_outputs ro JOIN transactions - ON transactions.id_tx = v_received_notes.tx + ON transactions.id_tx = ro.transaction_id UNION - -- Transparent TXOs received in this transaction - SELECT utxos.received_by_account_id AS account_id, - utxos.height AS block, - utxos.prevout_txid AS txid, - 0 AS pool, - utxos.id AS id_within_pool_table, - utxos.value_zat AS value, - 0 AS is_change, - 1 AS received_count, - 0 AS memo_present - FROM utxos - UNION - -- Shielded notes spent in this transaction - SELECT v_received_notes.account_id AS account_id, - transactions.block AS block, - transactions.txid AS txid, - v_received_notes.pool AS pool, - id_within_pool_table, - -v_received_notes.value AS value, - 0 AS is_change, - 0 AS received_count, - 0 AS memo_present - FROM v_received_notes - JOIN v_received_note_spends rns - ON rns.pool = v_received_notes.pool - AND rns.received_note_id = v_received_notes.id_within_pool_table - JOIN transactions - ON transactions.id_tx = rns.transaction_id - UNION - -- Transparent TXOs spent in this transaction - SELECT utxos.received_by_account_id AS account_id, - transactions.block AS block, - transactions.txid AS txid, - 0 AS pool, - utxos.id AS id_within_pool_table, - -utxos.value_zat AS value, - 0 AS is_change, - 0 AS received_count, - 0 AS memo_present - FROM utxos - JOIN transparent_received_output_spends tros - ON tros.transparent_received_output_id = utxos.id + -- Outputs spent in this transaction + SELECT ro.account_id AS account_id, + transactions.mined_height AS mined_height, + transactions.txid AS txid, + ro.pool AS pool, + id_within_pool_table, + -ro.value AS value, + 0 AS is_change, + 0 AS received_count, + 0 AS memo_present + FROM v_received_outputs ro + JOIN v_received_output_spends ros + ON ros.pool = ro.pool + AND ros.received_output_id = ro.id_within_pool_table JOIN transactions - ON transactions.id_tx = tros.transaction_id + ON transactions.id_tx = ro.transaction_id ), -- Obtain a count of the notes that the wallet created in each transaction, -- not counting change notes. sent_note_counts AS ( - SELECT sent_notes.from_account_id AS account_id, - transactions.txid AS txid, - COUNT(DISTINCT sent_notes.id) as sent_notes, - SUM( - CASE - WHEN (sent_notes.memo IS NULL OR sent_notes.memo = X'F6' OR v_received_notes.tx IS NOT NULL) - THEN 0 - ELSE 1 - END - ) AS memo_count + SELECT sent_notes.from_account_id AS account_id, + transactions.txid AS txid, + COUNT(DISTINCT sent_notes.id) AS sent_notes, + SUM( + CASE + WHEN (sent_notes.memo IS NULL OR sent_notes.memo = X'F6' OR ro.transaction_id IS NOT NULL) + THEN 0 + ELSE 1 + END + ) AS memo_count FROM sent_notes JOIN transactions - ON transactions.id_tx = sent_notes.tx - LEFT JOIN v_received_notes - ON sent_notes.id = v_received_notes.sent_note_id - WHERE COALESCE(v_received_notes.is_change, 0) = 0 + ON transactions.id_tx = sent_notes.tx + LEFT JOIN v_received_outputs ro + ON sent_notes.id = ro.sent_note_id + WHERE COALESCE(ro.is_change, 0) = 0 GROUP BY account_id, txid ), blocks_max_height AS ( - SELECT MAX(blocks.height) as max_height FROM blocks + SELECT MAX(blocks.height) AS max_height FROM blocks ) -SELECT notes.account_id AS account_id, - notes.block AS mined_height, - notes.txid AS txid, - transactions.tx_index AS tx_index, - transactions.expiry_height AS expiry_height, - transactions.raw AS raw, - SUM(notes.value) AS account_balance_delta, - transactions.fee AS fee_paid, - SUM(notes.is_change) > 0 AS has_change, - MAX(COALESCE(sent_note_counts.sent_notes, 0)) AS sent_note_count, - SUM(notes.received_count) AS received_note_count, - SUM(notes.memo_present) + MAX(COALESCE(sent_note_counts.memo_count, 0)) AS memo_count, - blocks.time AS block_time, - ( +SELECT notes.account_id AS account_id, + notes.mined_height AS mined_height, + notes.txid AS txid, + transactions.tx_index AS tx_index, + transactions.expiry_height AS expiry_height, + transactions.raw AS raw, + SUM(notes.value) AS account_balance_delta, + transactions.fee AS fee_paid, + SUM(notes.is_change) > 0 AS has_change, + MAX(COALESCE(sent_note_counts.sent_notes, 0)) AS sent_note_count, + SUM(notes.received_count) AS received_note_count, + SUM(notes.memo_present) + MAX(COALESCE(sent_note_counts.memo_count, 0)) AS memo_count, + blocks.time AS block_time, + ( blocks.height IS NULL AND transactions.expiry_height BETWEEN 1 AND blocks_max_height.max_height - ) AS expired_unmined + ) AS expired_unmined FROM notes LEFT JOIN transactions - ON notes.txid = transactions.txid + ON notes.txid = transactions.txid JOIN blocks_max_height -LEFT JOIN blocks ON blocks.height = notes.block +LEFT JOIN blocks ON blocks.height = notes.mined_height LEFT JOIN sent_note_counts - ON sent_note_counts.account_id = notes.account_id - AND sent_note_counts.txid = notes.txid + ON sent_note_counts.account_id = notes.account_id + AND sent_note_counts.txid = notes.txid GROUP BY notes.account_id, notes.txid"; pub(super) const VIEW_TX_OUTPUTS: &str = " CREATE VIEW v_tx_outputs AS -SELECT transactions.txid AS txid, - v_received_notes.pool AS output_pool, - v_received_notes.output_index AS output_index, - sent_notes.from_account_id AS from_account_id, - v_received_notes.account_id AS to_account_id, - NULL AS to_address, - v_received_notes.value AS value, - v_received_notes.is_change AS is_change, - v_received_notes.memo AS memo -FROM v_received_notes +SELECT transactions.txid AS txid, + ro.pool AS output_pool, + ro.output_index AS output_index, + sent_notes.from_account_id AS from_account_id, + ro.account_id AS to_account_id, + NULL AS to_address, + ro.value AS value, + ro.is_change AS is_change, + ro.memo AS memo +FROM v_received_outputs ro JOIN transactions - ON transactions.id_tx = v_received_notes.tx + ON transactions.id_tx = ro.transaction_id LEFT JOIN sent_notes - ON sent_notes.id = v_received_notes.sent_note_id -UNION -SELECT utxos.prevout_txid AS txid, - 0 AS output_pool, - utxos.prevout_idx AS output_index, - NULL AS from_account_id, - utxos.received_by_account_id AS to_account_id, - utxos.address AS to_address, - utxos.value_zat AS value, - 0 AS is_change, - NULL AS memo -FROM utxos + ON sent_notes.id = ro.sent_note_id UNION SELECT transactions.txid AS txid, - sent_notes.output_pool AS output_pool, - sent_notes.output_index AS output_index, - sent_notes.from_account_id AS from_account_id, - v_received_notes.account_id AS to_account_id, - sent_notes.to_address AS to_address, - sent_notes.value AS value, - 0 AS is_change, - sent_notes.memo AS memo + sent_notes.output_pool AS output_pool, + sent_notes.output_index AS output_index, + sent_notes.from_account_id AS from_account_id, + ro.account_id AS to_account_id, + sent_notes.to_address AS to_address, + sent_notes.value AS value, + 0 AS is_change, + sent_notes.memo AS memo FROM sent_notes JOIN transactions ON transactions.id_tx = sent_notes.tx -LEFT JOIN v_received_notes - ON sent_notes.id = v_received_notes.sent_note_id -WHERE COALESCE(v_received_notes.is_change, 0) = 0"; +LEFT JOIN v_received_outputs ro + ON sent_notes.id = ro.sent_note_id +WHERE COALESCE(ro.is_change, 0) = 0"; pub(super) fn view_sapling_shard_scan_ranges(params: &P) -> String { format!( diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 0aacc91e53..08838d0486 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -395,8 +395,8 @@ mod tests { db::TABLE_SQLITE_SEQUENCE, db::TABLE_TRANSACTIONS, db::TABLE_TRANSPARENT_RECEIVED_OUTPUT_SPENDS, + db::TABLE_TRANSPARENT_RECEIVED_OUTPUTS, db::TABLE_TX_LOCATOR_MAP, - db::TABLE_UTXOS, ]; let rows = describe_tables(&st.wallet().conn).unwrap(); @@ -421,7 +421,7 @@ mod tests { db::INDEX_SENT_NOTES_FROM_ACCOUNT, db::INDEX_SENT_NOTES_TO_ACCOUNT, db::INDEX_SENT_NOTES_TX, - db::INDEX_UTXOS_RECEIVED_BY_ACCOUNT, + db::INDEX_TRANSPARENT_RECEIVED_OUTPUTS_ACCOUNT_ID, ]; let mut indices_query = st .wallet() @@ -443,8 +443,8 @@ mod tests { db::view_orchard_shard_scan_ranges(&st.network()), db::view_orchard_shard_unscanned_ranges(), db::VIEW_ORCHARD_SHARDS_SCAN_STATE.to_owned(), - db::VIEW_RECEIVED_NOTE_SPENDS.to_owned(), - db::VIEW_RECEIVED_NOTES.to_owned(), + db::VIEW_RECEIVED_OUTPUT_SPENDS.to_owned(), + db::VIEW_RECEIVED_OUTPUTS.to_owned(), db::view_sapling_shard_scan_ranges(&st.network()), db::view_sapling_shard_unscanned_ranges(), db::VIEW_SAPLING_SHARDS_SCAN_STATE.to_owned(), diff --git a/zcash_client_sqlite/src/wallet/init/migrations.rs b/zcash_client_sqlite/src/wallet/init/migrations.rs index 8a09ae2d38..1f4befd996 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations.rs @@ -15,6 +15,7 @@ mod sent_notes_to_internal; mod shardtree_support; mod ufvk_support; mod utxos_table; +mod utxos_to_txos; mod v_sapling_shard_unscanned_ranges; mod v_transactions_net; mod v_transactions_note_uniqueness; @@ -63,8 +64,8 @@ pub(super) fn all_migrations( // -------------------- full_account_ids // | // orchard_received_notes - // | - // ensure_orchard_ua_receiver + // / \ + // ensure_orchard_ua_receiver utxos_to_txos vec![ Box::new(initial_setup::Migration {}), Box::new(utxos_table::Migration {}), @@ -114,6 +115,7 @@ pub(super) fn all_migrations( Box::new(ensure_orchard_ua_receiver::Migration { params: params.clone(), }), + Box::new(utxos_to_txos::Migration), ] } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/orchard_shardtree.rs b/zcash_client_sqlite/src/wallet/init/migrations/orchard_shardtree.rs index 564c345fca..dc44f1e087 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/orchard_shardtree.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/orchard_shardtree.rs @@ -11,7 +11,7 @@ use zcash_client_backend::data_api::scanning::ScanPriority; use zcash_protocol::consensus::{self, BlockHeight, NetworkUpgrade}; use super::shardtree_support; -use crate::wallet::{init::WalletMigrationError, scan_queue_extrema, scanning::priority_code}; +use crate::wallet::{chain_tip_height, init::WalletMigrationError, scanning::priority_code}; pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0x3a6487f7_e068_42bb_9d12_6bb8dbe6da00); @@ -142,10 +142,10 @@ impl RusqliteMigration for Migration

{ // Treat the current best-known chain tip height as the height to use for Orchard // initialization, bounded below by NU5 activation. - if let Some(orchard_init_height) = scan_queue_extrema(transaction)?.and_then(|r| { + if let Some(orchard_init_height) = chain_tip_height(transaction)?.and_then(|h| { self.params .activation_height(NetworkUpgrade::Nu5) - .map(|orchard_activation| std::cmp::max(orchard_activation, *r.end())) + .map(|orchard_activation| std::cmp::max(orchard_activation, h)) }) { // If a scan range exists that contains the Orchard init height, split it in two at the // init height. diff --git a/zcash_client_sqlite/src/wallet/init/migrations/receiving_key_scopes.rs b/zcash_client_sqlite/src/wallet/init/migrations/receiving_key_scopes.rs index 1c1dd6bcc3..cd1f768455 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/receiving_key_scopes.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/receiving_key_scopes.rs @@ -28,9 +28,10 @@ use zcash_primitives::{ use crate::{ wallet::{ + chain_tip_height, commitment_tree::SqliteShardStore, init::{migrations::shardtree_support, WalletMigrationError}, - scan_queue_extrema, scope_code, + scope_code, }, PRUNING_DEPTH, SAPLING_TABLES_PREFIX, }; @@ -154,7 +155,7 @@ impl RusqliteMigration for Migration

{ let zip212_height = tx_height.map_or_else( || { tx_expiry.filter(|h| *h != 0).map_or_else( - || scan_queue_extrema(transaction).map(|extrema| extrema.map(|r| *r.end())), + || chain_tip_height(transaction), |h| Ok(Some(BlockHeight::from(h))), ) }, @@ -286,13 +287,14 @@ mod tests { slice::ParallelSliceMut, }; use rand_core::OsRng; - use rusqlite::{named_params, params, Connection}; + use rusqlite::{named_params, params, Connection, OptionalExtension}; use tempfile::NamedTempFile; use zcash_client_backend::{ data_api::{BlockMetadata, WalletCommitmentTrees, SAPLING_SHARD_HEIGHT}, decrypt_transaction, proto::compact_formats::{CompactBlock, CompactTx}, scanning::{scan_block, Nullifiers, ScanningKeys}, + wallet::WalletTx, TransferType, }; use zcash_keys::keys::{UnifiedFullViewingKey, UnifiedSpendingKey}; @@ -647,7 +649,7 @@ mod tests { } // Insert the block into the database. - crate::wallet::put_block( + put_block( wdb.conn.0, block.height(), block.block_hash(), @@ -661,7 +663,7 @@ mod tests { )?; for tx in block.transactions() { - let tx_row = crate::wallet::put_tx_meta(wdb.conn.0, tx, block.height())?; + let tx_row = put_tx_meta(wdb.conn.0, tx, block.height())?; for output in tx.sapling_outputs() { put_received_note_before_migration(wdb.conn.0, output, tx_row, None)?; @@ -745,4 +747,118 @@ mod tests { } assert_eq!(row_count, 2); } + + /// This is a copy of [`crate::wallet::put_block`] as of the expected database + /// state corresponding to this migration. It is duplicated here as later + /// updates to the database schema require incompatible changes to `put_block`. + #[allow(clippy::too_many_arguments)] + fn put_block( + conn: &rusqlite::Transaction<'_>, + block_height: BlockHeight, + block_hash: BlockHash, + block_time: u32, + sapling_commitment_tree_size: u32, + sapling_output_count: u32, + #[cfg(feature = "orchard")] orchard_commitment_tree_size: u32, + #[cfg(feature = "orchard")] orchard_action_count: u32, + ) -> Result<(), SqliteClientError> { + let block_hash_data = conn + .query_row( + "SELECT hash FROM blocks WHERE height = ?", + [u32::from(block_height)], + |row| row.get::<_, Vec>(0), + ) + .optional()?; + + // Ensure that in the case of an upsert, we don't overwrite block data + // with information for a block with a different hash. + if let Some(bytes) = block_hash_data { + let expected_hash = BlockHash::try_from_slice(&bytes).ok_or_else(|| { + SqliteClientError::CorruptedData(format!( + "Invalid block hash at height {}", + u32::from(block_height) + )) + })?; + if expected_hash != block_hash { + return Err(SqliteClientError::BlockConflict(block_height)); + } + } + + let mut stmt_upsert_block = conn.prepare_cached( + "INSERT INTO blocks ( + height, + hash, + time, + sapling_commitment_tree_size, + sapling_output_count, + sapling_tree, + orchard_commitment_tree_size, + orchard_action_count + ) + VALUES ( + :height, + :hash, + :block_time, + :sapling_commitment_tree_size, + :sapling_output_count, + x'00', + :orchard_commitment_tree_size, + :orchard_action_count + ) + ON CONFLICT (height) DO UPDATE + SET hash = :hash, + time = :block_time, + sapling_commitment_tree_size = :sapling_commitment_tree_size, + sapling_output_count = :sapling_output_count, + orchard_commitment_tree_size = :orchard_commitment_tree_size, + orchard_action_count = :orchard_action_count", + )?; + + #[cfg(not(feature = "orchard"))] + let orchard_commitment_tree_size: Option = None; + #[cfg(not(feature = "orchard"))] + let orchard_action_count: Option = None; + + stmt_upsert_block.execute(named_params![ + ":height": u32::from(block_height), + ":hash": &block_hash.0[..], + ":block_time": block_time, + ":sapling_commitment_tree_size": sapling_commitment_tree_size, + ":sapling_output_count": sapling_output_count, + ":orchard_commitment_tree_size": orchard_commitment_tree_size, + ":orchard_action_count": orchard_action_count, + ])?; + + Ok(()) + } + + /// This is a copy of [`crate::wallet::put_tx_meta`] as of the expected database + /// state corresponding to this migration. It is duplicated here as later + /// updates to the database schema require incompatible changes to `put_tx_meta`. + pub(crate) fn put_tx_meta( + conn: &rusqlite::Connection, + tx: &WalletTx, + height: BlockHeight, + ) -> Result { + // It isn't there, so insert our transaction into the database. + let mut stmt_upsert_tx_meta = conn.prepare_cached( + "INSERT INTO transactions (txid, block, tx_index) + VALUES (:txid, :block, :tx_index) + ON CONFLICT (txid) DO UPDATE + SET block = :block, + tx_index = :tx_index + RETURNING id_tx", + )?; + + let txid_bytes = tx.txid(); + let tx_params = named_params![ + ":txid": &txid_bytes.as_ref()[..], + ":block": u32::from(height), + ":tx_index": i64::try_from(tx.block_index()).expect("transaction indices are representable as i64"), + ]; + + stmt_upsert_tx_meta + .query_row(tx_params, |row| row.get::<_, i64>(0)) + .map_err(SqliteClientError::from) + } } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/utxos_to_txos.rs b/zcash_client_sqlite/src/wallet/init/migrations/utxos_to_txos.rs new file mode 100644 index 0000000000..1d2680b657 --- /dev/null +++ b/zcash_client_sqlite/src/wallet/init/migrations/utxos_to_txos.rs @@ -0,0 +1,328 @@ +//! A migration that brings transparent UTXO handling into line with that for shielded outputs. +use std::collections::HashSet; + +use rusqlite; +use schemer; +use schemer_rusqlite::RusqliteMigration; +use uuid::Uuid; + +use crate::wallet::init::{migrations::orchard_received_notes, WalletMigrationError}; + +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0x3a2562b3_f174_46a1_aa8c_1d122ca2e884); + +pub(super) struct Migration; + +impl schemer::Migration for Migration { + fn id(&self) -> Uuid { + MIGRATION_ID + } + + fn dependencies(&self) -> HashSet { + [orchard_received_notes::MIGRATION_ID].into_iter().collect() + } + + fn description(&self) -> &'static str { + "Updates transparent UTXO handling to be similar to that for shielded notes." + } +} + +impl RusqliteMigration for Migration { + type Error = WalletMigrationError; + + fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { + transaction.execute_batch(r#" + PRAGMA legacy_alter_table = ON; + + CREATE TABLE transactions_new ( + id_tx INTEGER PRIMARY KEY, + txid BLOB NOT NULL UNIQUE, + created TEXT, + block INTEGER, + mined_height INTEGER, + tx_index INTEGER, + expiry_height INTEGER, + raw BLOB, + fee INTEGER, + FOREIGN KEY (block) REFERENCES blocks(height), + CONSTRAINT height_consistency CHECK (block IS NULL OR mined_height = block) + ); + + INSERT INTO transactions_new + SELECT id_tx, txid, created, block, block, tx_index, expiry_height, raw, fee + FROM transactions; + + -- We may initially set the block height to null, which will mean that the + -- transaction may appear to be un-mined until we actually scan the block + -- containing the transaction. + INSERT INTO transactions_new (txid, block, mined_height) + SELECT + utxos.prevout_txid, + blocks.height, + blocks.height + FROM utxos + LEFT OUTER JOIN blocks ON blocks.height = utxos.height + WHERE utxos.prevout_txid NOT IN ( + SELECT txid FROM transactions + ); + + DROP TABLE transactions; + ALTER TABLE transactions_new RENAME TO transactions; + + CREATE TABLE transparent_received_outputs ( + id INTEGER PRIMARY KEY, + transaction_id INTEGER NOT NULL, + output_index INTEGER NOT NULL, + account_id INTEGER NOT NULL, + address TEXT NOT NULL, + script BLOB NOT NULL, + value_zat INTEGER NOT NULL, + max_observed_unspent_height INTEGER, + FOREIGN KEY (transaction_id) REFERENCES transactions(id_tx), + FOREIGN KEY (account_id) REFERENCES accounts(id), + CONSTRAINT transparent_output_unique UNIQUE (transaction_id, output_index) + ); + CREATE INDEX idx_transparent_received_outputs_account_id + ON "transparent_received_outputs" (account_id); + + INSERT INTO transparent_received_outputs SELECT + u.id, + t.id_tx, + prevout_idx, + received_by_account_id, + address, + script, + value_zat, + NULL + FROM utxos u + -- This being a `LEFT OUTER JOIN` provides defense in depth against dropping + -- TXOs that reference missing `transactions` entries (which should never exist + -- given the migrations above). + LEFT OUTER JOIN transactions t ON t.txid = u.prevout_txid; + + CREATE TABLE transparent_received_output_spends_new ( + transparent_received_output_id INTEGER NOT NULL, + transaction_id INTEGER NOT NULL, + FOREIGN KEY (transparent_received_output_id) + REFERENCES transparent_received_outputs(id) + ON DELETE CASCADE, + FOREIGN KEY (transaction_id) + -- We do not delete transactions, so this does not cascade + REFERENCES transactions(id_tx), + UNIQUE (transparent_received_output_id, transaction_id) + ); + + INSERT INTO transparent_received_output_spends_new + SELECT * FROM transparent_received_output_spends; + + DROP VIEW v_tx_outputs; + DROP VIEW v_transactions; + DROP VIEW v_received_notes; + DROP VIEW v_received_note_spends; + DROP TABLE transparent_received_output_spends; + ALTER TABLE transparent_received_output_spends_new + RENAME TO transparent_received_output_spends; + + CREATE VIEW v_received_outputs AS + SELECT + sapling_received_notes.id AS id_within_pool_table, + sapling_received_notes.tx AS transaction_id, + 2 AS pool, + sapling_received_notes.output_index, + account_id, + sapling_received_notes.value, + is_change, + sapling_received_notes.memo, + sent_notes.id AS sent_note_id + FROM sapling_received_notes + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (sapling_received_notes.tx, 2, sapling_received_notes.output_index) + UNION + SELECT + orchard_received_notes.id AS id_within_pool_table, + orchard_received_notes.tx AS transaction_id, + 3 AS pool, + orchard_received_notes.action_index AS output_index, + account_id, + orchard_received_notes.value, + is_change, + orchard_received_notes.memo, + sent_notes.id AS sent_note_id + FROM orchard_received_notes + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (orchard_received_notes.tx, 3, orchard_received_notes.action_index) + UNION + SELECT + u.id AS id_within_pool_table, + u.transaction_id, + 0 AS pool, + u.output_index, + u.account_id, + u.value_zat AS value, + 0 AS is_change, + NULL AS memo, + sent_notes.id AS sent_note_id + FROM transparent_received_outputs u + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (u.transaction_id, 0, u.output_index); + + CREATE VIEW v_received_output_spends AS + SELECT + 2 AS pool, + sapling_received_note_id AS received_output_id, + transaction_id + FROM sapling_received_note_spends + UNION + SELECT + 3 AS pool, + orchard_received_note_id AS received_output_id, + transaction_id + FROM orchard_received_note_spends + UNION + SELECT + 0 AS pool, + transparent_received_output_id AS received_output_id, + transaction_id + FROM transparent_received_output_spends; + + CREATE VIEW v_transactions AS + WITH + notes AS ( + -- Outputs received in this transaction + SELECT ro.account_id AS account_id, + transactions.mined_height AS mined_height, + transactions.txid AS txid, + ro.pool AS pool, + id_within_pool_table, + ro.value AS value, + CASE + WHEN ro.is_change THEN 1 + ELSE 0 + END AS is_change, + CASE + WHEN ro.is_change THEN 0 + ELSE 1 + END AS received_count, + CASE + WHEN (ro.memo IS NULL OR ro.memo = X'F6') + THEN 0 + ELSE 1 + END AS memo_present + FROM v_received_outputs ro + JOIN transactions + ON transactions.id_tx = ro.transaction_id + UNION + -- Outputs spent in this transaction + SELECT ro.account_id AS account_id, + transactions.mined_height AS mined_height, + transactions.txid AS txid, + ro.pool AS pool, + id_within_pool_table, + -ro.value AS value, + 0 AS is_change, + 0 AS received_count, + 0 AS memo_present + FROM v_received_outputs ro + JOIN v_received_output_spends ros + ON ros.pool = ro.pool + AND ros.received_output_id = ro.id_within_pool_table + JOIN transactions + ON transactions.id_tx = ro.transaction_id + ), + -- Obtain a count of the notes that the wallet created in each transaction, + -- not counting change notes. + sent_note_counts AS ( + SELECT sent_notes.from_account_id AS account_id, + transactions.txid AS txid, + COUNT(DISTINCT sent_notes.id) AS sent_notes, + SUM( + CASE + WHEN (sent_notes.memo IS NULL OR sent_notes.memo = X'F6' OR ro.transaction_id IS NOT NULL) + THEN 0 + ELSE 1 + END + ) AS memo_count + FROM sent_notes + JOIN transactions + ON transactions.id_tx = sent_notes.tx + LEFT JOIN v_received_outputs ro + ON sent_notes.id = ro.sent_note_id + WHERE COALESCE(ro.is_change, 0) = 0 + GROUP BY account_id, txid + ), + blocks_max_height AS ( + SELECT MAX(blocks.height) AS max_height FROM blocks + ) + SELECT notes.account_id AS account_id, + notes.mined_height AS mined_height, + notes.txid AS txid, + transactions.tx_index AS tx_index, + transactions.expiry_height AS expiry_height, + transactions.raw AS raw, + SUM(notes.value) AS account_balance_delta, + transactions.fee AS fee_paid, + SUM(notes.is_change) > 0 AS has_change, + MAX(COALESCE(sent_note_counts.sent_notes, 0)) AS sent_note_count, + SUM(notes.received_count) AS received_note_count, + SUM(notes.memo_present) + MAX(COALESCE(sent_note_counts.memo_count, 0)) AS memo_count, + blocks.time AS block_time, + ( + blocks.height IS NULL + AND transactions.expiry_height BETWEEN 1 AND blocks_max_height.max_height + ) AS expired_unmined + FROM notes + LEFT JOIN transactions + ON notes.txid = transactions.txid + JOIN blocks_max_height + LEFT JOIN blocks ON blocks.height = notes.mined_height + LEFT JOIN sent_note_counts + ON sent_note_counts.account_id = notes.account_id + AND sent_note_counts.txid = notes.txid + GROUP BY notes.account_id, notes.txid; + + CREATE VIEW v_tx_outputs AS + SELECT transactions.txid AS txid, + ro.pool AS output_pool, + ro.output_index AS output_index, + sent_notes.from_account_id AS from_account_id, + ro.account_id AS to_account_id, + NULL AS to_address, + ro.value AS value, + ro.is_change AS is_change, + ro.memo AS memo + FROM v_received_outputs ro + JOIN transactions + ON transactions.id_tx = ro.transaction_id + LEFT JOIN sent_notes + ON sent_notes.id = ro.sent_note_id + UNION + SELECT transactions.txid AS txid, + sent_notes.output_pool AS output_pool, + sent_notes.output_index AS output_index, + sent_notes.from_account_id AS from_account_id, + ro.account_id AS to_account_id, + sent_notes.to_address AS to_address, + sent_notes.value AS value, + 0 AS is_change, + sent_notes.memo AS memo + FROM sent_notes + JOIN transactions + ON transactions.id_tx = sent_notes.tx + LEFT JOIN v_received_outputs ro + ON sent_notes.id = ro.sent_note_id + WHERE COALESCE(ro.is_change, 0) = 0; + + DROP TABLE utxos; + + PRAGMA legacy_alter_table = OFF; + "#)?; + + Ok(()) + } + + fn down(&self, _: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { + Err(WalletMigrationError::CannotRevert(MIGRATION_ID)) + } +} diff --git a/zcash_client_sqlite/src/wallet/transparent.rs b/zcash_client_sqlite/src/wallet/transparent.rs index d3190fe02c..c06e3620de 100644 --- a/zcash_client_sqlite/src/wallet/transparent.rs +++ b/zcash_client_sqlite/src/wallet/transparent.rs @@ -1,7 +1,6 @@ //! Functions for transparent input support in the wallet. use rusqlite::OptionalExtension; use rusqlite::{named_params, Connection, Row}; -use std::collections::BTreeSet; use std::collections::HashMap; use std::collections::HashSet; use zcash_client_backend::data_api::AccountBalance; @@ -20,20 +19,20 @@ use zcash_primitives::{ }; use zcash_protocol::consensus::{self, BlockHeight}; -use crate::{error::SqliteClientError, AccountId, UtxoId, PRUNING_DEPTH}; +use crate::{error::SqliteClientError, AccountId, UtxoId}; -use super::get_account_ids; -use super::scan_queue_extrema; +use super::{chain_tip_height, get_account_ids}; pub(crate) fn detect_spending_accounts<'a>( conn: &Connection, spent: impl Iterator, ) -> Result, rusqlite::Error> { let mut account_q = conn.prepare_cached( - "SELECT received_by_account_id - FROM utxos - WHERE prevout_txid = :prevout_txid - AND prevout_idx = :prevout_idx", + "SELECT account_id + FROM transparent_received_outputs o + JOIN transactions t ON t.id_tx = o.transaction_id + WHERE t.txid = :prevout_txid + AND o.output_index = :prevout_idx", )?; let mut acc = HashSet::new(); @@ -161,17 +160,17 @@ pub(crate) fn get_legacy_transparent_address( } fn to_unspent_transparent_output(row: &Row) -> Result { - let txid: Vec = row.get("prevout_txid")?; + let txid: Vec = row.get("txid")?; let mut txid_bytes = [0u8; 32]; txid_bytes.copy_from_slice(&txid); - let index: u32 = row.get("prevout_idx")?; + let index: u32 = row.get("output_index")?; let script_pubkey = Script(row.get("script")?); let raw_value: i64 = row.get("value_zat")?; let value = NonNegativeAmount::from_nonnegative_i64(raw_value).map_err(|_| { SqliteClientError::CorruptedData(format!("Invalid UTXO value: {}", raw_value)) })?; - let height: u32 = row.get("height")?; + let height: u32 = row.get("received_height")?; let outpoint = OutPoint::new(txid_bytes, index); WalletTransparentOutput::from_parts( @@ -189,21 +188,39 @@ fn to_unspent_transparent_output(row: &Row) -> Result Result, SqliteClientError> { + let chain_tip_height = chain_tip_height(conn)?; + + // This could, in very rare circumstances, return as unspent outputs that are actually not + // spendable, if they are the outputs of deshielding transactions where the spend anchors have + // been invalidated by a rewind. There isn't a way to detect this circumstance at present, but + // it should be vanishingly rare as the vast majority of rewinds are of a single block. let mut stmt_select_utxo = conn.prepare_cached( - "SELECT u.prevout_txid, u.prevout_idx, u.script, u.value_zat, u.height - FROM utxos u - WHERE u.prevout_txid = :txid - AND u.prevout_idx = :output_index + "SELECT t.txid, u.output_index, u.script, + u.value_zat, t.mined_height AS received_height + FROM transparent_received_outputs u + JOIN transactions t ON t.id_tx = u.transaction_id + WHERE t.txid = :txid + AND u.output_index = :output_index + -- the transaction that created the output is mined or is definitely unexpired + AND ( + t.mined_height IS NOT NULL -- tx is mined + -- TODO: uncomment the following two lines in order to enable zero-conf spends + -- OR t.expiry_height = 0 -- tx will not expire + -- OR t.expiry_height >= :mempool_height -- tx has not yet expired + ) + -- and the output is unspent AND u.id NOT IN ( SELECT txo_spends.transparent_received_output_id FROM transparent_received_output_spends txo_spends JOIN transactions tx ON tx.id_tx = txo_spends.transaction_id - WHERE tx.block IS NOT NULL -- the spending tx is mined - OR tx.expiry_height IS NULL -- the spending tx will not expire + WHERE tx.mined_height IS NOT NULL -- the spending tx is mined + OR tx.expiry_height = 0 -- the spending tx will not expire + OR tx.expiry_height >= :mempool_height -- the spending tx has not yet expired )", )?; @@ -211,7 +228,8 @@ pub(crate) fn get_unspent_transparent_output( .query_and_then( named_params![ ":txid": outpoint.hash(), - ":output_index": outpoint.n() + ":output_index": outpoint.n(), + ":mempool_height": chain_tip_height.map(|h| u32::from(h) + 1), ], to_unspent_transparent_output, )? @@ -221,53 +239,62 @@ pub(crate) fn get_unspent_transparent_output( result } -/// Returns unspent transparent outputs that have been received by this wallet at the given -/// transparent address, such that the block that included the transaction was mined at a -/// height less than or equal to the provided `max_height`. -pub(crate) fn get_unspent_transparent_outputs( +pub(crate) fn get_spendable_transparent_outputs( conn: &rusqlite::Connection, params: &P, address: &TransparentAddress, - max_height: BlockHeight, - exclude: &[OutPoint], + target_height: BlockHeight, + min_confirmations: u32, ) -> Result, SqliteClientError> { - let chain_tip_height = scan_queue_extrema(conn)?.map(|range| *range.end()); - let stable_height = chain_tip_height - .unwrap_or(max_height) - .saturating_sub(PRUNING_DEPTH); + let confirmed_height = target_height - min_confirmations; + // This could, in very rare circumstances, return as unspent outputs that are actually not + // spendable, if they are the outputs of deshielding transactions where the spend anchors have + // been invalidated by a rewind. There isn't a way to detect this circumstance at present, but + // it should be vanishingly rare as the vast majority of rewinds are of a single block. let mut stmt_utxos = conn.prepare( - "SELECT u.prevout_txid, u.prevout_idx, u.script, - u.value_zat, u.height - FROM utxos u + "SELECT t.txid, u.output_index, u.script, + u.value_zat, t.mined_height AS received_height + FROM transparent_received_outputs u + JOIN transactions t ON t.id_tx = u.transaction_id WHERE u.address = :address - AND u.height <= :max_height + -- the transaction that created the output is mined or unexpired as of `confirmed_height` + AND ( + t.mined_height <= :confirmed_height -- tx is mined + -- TODO: uncomment the following lines in order to enable zero-conf spends + -- OR ( + -- :min_confirmations = 0 + -- AND ( + -- t.expiry_height = 0 -- tx will not expire + -- OR t.expiry_height >= :target_height + -- ) + -- ) + ) + -- and the output is unspent AND u.id NOT IN ( SELECT txo_spends.transparent_received_output_id FROM transparent_received_output_spends txo_spends JOIN transactions tx ON tx.id_tx = txo_spends.transaction_id - WHERE - tx.block IS NOT NULL -- the spending tx is mined - OR tx.expiry_height IS NULL -- the spending tx will not expire - OR tx.expiry_height > :stable_height -- the spending tx is unexpired + WHERE tx.mined_height IS NOT NULL -- the spending transaction is mined + OR tx.expiry_height = 0 -- the spending tx will not expire + OR tx.expiry_height >= :target_height -- the spending tx has not yet expired + -- we are intentionally conservative and exclude outputs that are potentially spent + -- as of the target height, even if they might actually be spendable due to expiry + -- of the spending transaction as of the chain tip )", )?; let addr_str = address.encode(params); - - let mut utxos = Vec::::new(); let mut rows = stmt_utxos.query(named_params![ ":address": addr_str, - ":max_height": u32::from(max_height), - ":stable_height": u32::from(stable_height), + ":confirmed_height": u32::from(confirmed_height), + ":target_height": u32::from(target_height), + //":min_confirmations": min_confirmations ])?; - let excluded: BTreeSet = exclude.iter().cloned().collect(); + + let mut utxos = Vec::::new(); while let Some(row) = rows.next()? { let output = to_unspent_transparent_output(row)?; - if excluded.contains(output.outpoint()) { - continue; - } - utxos.push(output); } @@ -276,31 +303,40 @@ pub(crate) fn get_unspent_transparent_outputs( /// Returns the unspent balance for each transparent address associated with the specified account, /// such that the block that included the transaction was mined at a height less than or equal to -/// the provided `max_height`. +/// the provided `summary_height`. pub(crate) fn get_transparent_address_balances( conn: &rusqlite::Connection, params: &P, account: AccountId, - max_height: BlockHeight, + summary_height: BlockHeight, ) -> Result, SqliteClientError> { - let chain_tip_height = scan_queue_extrema(conn)?.map(|range| *range.end()); - let stable_height = chain_tip_height - .unwrap_or(max_height) - .saturating_sub(PRUNING_DEPTH); + let chain_tip_height = chain_tip_height(conn)?.ok_or(SqliteClientError::ChainHeightUnknown)?; let mut stmt_address_balances = conn.prepare( "SELECT u.address, SUM(u.value_zat) - FROM utxos u - WHERE u.received_by_account_id = :account_id - AND u.height <= :max_height + FROM transparent_received_outputs u + JOIN transactions t + ON t.id_tx = u.transaction_id + WHERE u.account_id = :account_id + -- the transaction that created the output is mined or is definitely unexpired + AND ( + t.mined_height <= :summary_height -- tx is mined + OR ( -- or the caller has requested to include zero-conf funds that are not expired + :summary_height > :chain_tip_height + AND ( + t.expiry_height = 0 -- tx will not expire + OR t.expiry_height >= :summary_height + ) + ) + ) + -- and the output is unspent AND u.id NOT IN ( SELECT txo_spends.transparent_received_output_id FROM transparent_received_output_spends txo_spends JOIN transactions tx ON tx.id_tx = txo_spends.transaction_id - WHERE - tx.block IS NOT NULL -- the spending tx is mined - OR tx.expiry_height IS NULL -- the spending tx will not expire - OR tx.expiry_height > :stable_height -- the spending tx is unexpired + WHERE tx.mined_height IS NOT NULL -- the spending tx is mined + OR tx.expiry_height = 0 -- the spending tx will not expire + OR tx.expiry_height >= :spend_expiry_height -- the spending tx is unexpired ) GROUP BY u.address", )?; @@ -308,8 +344,9 @@ pub(crate) fn get_transparent_address_balances( let mut res = HashMap::new(); let mut rows = stmt_address_balances.query(named_params![ ":account_id": account.0, - ":max_height": u32::from(max_height), - ":stable_height": u32::from(stable_height), + ":summary_height": u32::from(summary_height), + ":chain_tip_height": u32::from(chain_tip_height), + ":spend_expiry_height": u32::from(std::cmp::min(summary_height, chain_tip_height + 1)), ])?; while let Some(row) = rows.next()? { let taddr_str: String = row.get(0)?; @@ -322,36 +359,38 @@ pub(crate) fn get_transparent_address_balances( Ok(res) } +#[tracing::instrument(skip(conn, account_balances))] pub(crate) fn add_transparent_account_balances( conn: &rusqlite::Connection, - chain_tip_height: BlockHeight, - min_confirmations: u32, + mempool_height: BlockHeight, account_balances: &mut HashMap, ) -> Result<(), SqliteClientError> { let transparent_trace = tracing::info_span!("stmt_transparent_balances").entered(); - let zero_conf_height = (chain_tip_height + 1).saturating_sub(min_confirmations); - let stable_height = chain_tip_height.saturating_sub(PRUNING_DEPTH); - - let mut stmt_transparent_balances = conn.prepare( - "SELECT u.received_by_account_id, SUM(u.value_zat) - FROM utxos u - WHERE u.height <= :max_height + let mut stmt_account_balances = conn.prepare( + "SELECT u.account_id, SUM(u.value_zat) + FROM transparent_received_outputs u + JOIN transactions t + ON t.id_tx = u.transaction_id + -- the transaction that created the output is mined or is definitely unexpired + WHERE ( + t.mined_height < :mempool_height -- tx is mined + OR t.expiry_height = 0 -- tx will not expire + OR t.expiry_height >= :mempool_height + ) -- and the received txo is unspent AND u.id NOT IN ( SELECT transparent_received_output_id FROM transparent_received_output_spends txo_spends JOIN transactions tx ON tx.id_tx = txo_spends.transaction_id - WHERE tx.block IS NOT NULL -- the spending tx is mined - OR tx.expiry_height IS NULL -- the spending tx will not expire - OR tx.expiry_height > :stable_height -- the spending tx is unexpired + WHERE tx.mined_height IS NOT NULL -- the spending tx is mined + OR tx.expiry_height = 0 -- the spending tx will not expire + OR tx.expiry_height >= :mempool_height -- the spending tx is unexpired ) - GROUP BY u.received_by_account_id", + GROUP BY u.account_id", )?; - let mut rows = stmt_transparent_balances.query(named_params![ - ":max_height": u32::from(zero_conf_height), - ":stable_height": u32::from(stable_height) - ])?; + let mut rows = stmt_account_balances + .query(named_params![":mempool_height": u32::from(mempool_height),])?; while let Some(row) = rows.next()? { let account = AccountId(row.get(0)?); @@ -360,9 +399,10 @@ pub(crate) fn add_transparent_account_balances( SqliteClientError::CorruptedData(format!("Negative UTXO value {:?}", raw_value)) })?; - if let Some(balances) = account_balances.get_mut(&account) { - balances.add_unshielded_value(value)?; - } + account_balances + .entry(account) + .or_insert(AccountBalance::ZERO) + .add_unshielded_value(value)?; } drop(transparent_trace); Ok(()) @@ -377,9 +417,10 @@ pub(crate) fn mark_transparent_utxo_spent( let mut stmt_mark_transparent_utxo_spent = conn.prepare_cached( "INSERT INTO transparent_received_output_spends (transparent_received_output_id, transaction_id) SELECT txo.id, :spent_in_tx - FROM utxos txo - WHERE txo.prevout_txid = :prevout_txid - AND txo.prevout_idx = :prevout_idx + FROM transparent_received_outputs txo + JOIN transactions t ON t.id_tx = txo.transaction_id + WHERE t.txid = :prevout_txid + AND txo.output_index = :prevout_idx ON CONFLICT (transparent_received_output_id, transaction_id) DO NOTHING", )?; @@ -409,7 +450,7 @@ pub(crate) fn put_received_transparent_utxo( .optional()?; if let Some(account) = account_id { - Ok(put_legacy_transparent_utxo(conn, params, output, account)?) + Ok(put_transparent_output(conn, params, output, account)?) } else { // If the UTXO is received at the legacy transparent address (at BIP 44 address // index 0 within its particular account, which we specifically ensure is returned @@ -423,7 +464,7 @@ pub(crate) fn put_received_transparent_utxo( |account| match get_legacy_transparent_address(params, conn, account) { Ok(Some((legacy_taddr, _))) if &legacy_taddr == output.recipient_address() => { Some( - put_legacy_transparent_utxo(conn, params, output, account) + put_transparent_output(conn, params, output, account) .map_err(SqliteClientError::from), ) } @@ -440,50 +481,74 @@ pub(crate) fn put_received_transparent_utxo( } } -pub(crate) fn put_legacy_transparent_utxo( +pub(crate) fn put_transparent_output( conn: &rusqlite::Connection, params: &P, output: &WalletTransparentOutput, received_by_account: AccountId, ) -> Result { - #[cfg(feature = "transparent-inputs")] - let mut stmt_upsert_legacy_transparent_utxo = conn.prepare_cached( - "INSERT INTO utxos ( - prevout_txid, prevout_idx, - received_by_account_id, address, script, - value_zat, height) - VALUES - (:prevout_txid, :prevout_idx, - :received_by_account_id, :address, :script, - :value_zat, :height) - ON CONFLICT (prevout_txid, prevout_idx) DO UPDATE - SET received_by_account_id = :received_by_account_id, - height = :height, + // Check whether we have an entry in the blocks table for the output height; + // if not, the transaction will be updated with its mined height when the + // associated block is scanned. + let block = conn + .query_row( + "SELECT height FROM blocks WHERE height = :height", + named_params![":height": &u32::from(output.height())], + |row| row.get::<_, u32>(0), + ) + .optional()?; + + let id_tx = conn.query_row( + "INSERT INTO transactions (txid, block, mined_height) + VALUES (:txid, :block, :mined_height) + ON CONFLICT (txid) DO UPDATE + SET block = IFNULL(block, :block), + mined_height = :mined_height + RETURNING id_tx", + named_params![ + ":txid": &output.outpoint().hash().to_vec(), + ":block": block, + ":mined_height": u32::from(output.height()) + ], + |row| row.get::<_, i64>(0), + )?; + + let mut stmt_upsert_transparent_output = conn.prepare_cached( + "INSERT INTO transparent_received_outputs ( + transaction_id, output_index, + account_id, address, script, + value_zat, max_observed_unspent_height + ) + VALUES ( + :transaction_id, :output_index, + :account_id, :address, :script, + :value_zat, :height + ) + ON CONFLICT (transaction_id, output_index) DO UPDATE + SET account_id = :account_id, address = :address, script = :script, - value_zat = :value_zat + value_zat = :value_zat, + max_observed_unspent_height = :height RETURNING id", )?; let sql_args = named_params![ - ":prevout_txid": &output.outpoint().hash().to_vec(), - ":prevout_idx": &output.outpoint().n(), - ":received_by_account_id": received_by_account.0, + ":transaction_id": id_tx, + ":output_index": &output.outpoint().n(), + ":account_id": received_by_account.0, ":address": &output.recipient_address().encode(params), ":script": &output.txout().script_pubkey.0, ":value_zat": &i64::from(Amount::from(output.txout().value)), ":height": &u32::from(output.height()), ]; - stmt_upsert_legacy_transparent_utxo.query_row(sql_args, |row| row.get::<_, i64>(0).map(UtxoId)) + stmt_upsert_transparent_output.query_row(sql_args, |row| row.get::<_, i64>(0).map(UtxoId)) } #[cfg(test)] mod tests { - use crate::{ - testing::{AddressType, TestBuilder, TestState}, - PRUNING_DEPTH, - }; + use crate::testing::{AddressType, TestBuilder, TestState}; use sapling::zip32::ExtendedSpendingKey; use zcash_client_backend::{ data_api::{ @@ -495,7 +560,6 @@ mod tests { }; use zcash_primitives::{ block::BlockHash, - consensus::BlockHeight, transaction::{ components::{amount::NonNegativeAmount, OutPoint, TxOut}, fees::fixed::FeeRule as FixedFeeRule, @@ -510,6 +574,7 @@ mod tests { .with_account_from_sapling_activation(BlockHash([0; 32])) .build(); + let birthday = st.test_account().unwrap().birthday().height(); let account_id = st.test_account().unwrap().account_id(); let uaddr = st .wallet() @@ -518,7 +583,9 @@ mod tests { .unwrap(); let taddr = uaddr.transparent().unwrap(); - let height_1 = BlockHeight::from_u32(12345); + let height_1 = birthday + 12345; + st.wallet_mut().update_chain_tip(height_1).unwrap(); + let bal_absent = st .wallet() .get_transparent_balances(account_id, height_1) @@ -541,10 +608,10 @@ mod tests { // Confirm that we see the output unspent as of `height_1`. assert_matches!( - st.wallet().get_unspent_transparent_outputs( + st.wallet().get_spendable_transparent_outputs( taddr, height_1, - &[] + 0 ).as_deref(), Ok([ret]) if (ret.outpoint(), ret.txout(), ret.height()) == (utxo.outpoint(), utxo.txout(), height_1) ); @@ -555,7 +622,8 @@ mod tests { // Change the mined height of the UTXO and upsert; we should get back // the same `UtxoId`. - let height_2 = BlockHeight::from_u32(34567); + let height_2 = birthday + 34567; + st.wallet_mut().update_chain_tip(height_2).unwrap(); let utxo2 = WalletTransparentOutput::from_parts(outpoint, txout, height_2).unwrap(); let res1 = st.wallet_mut().put_received_transparent_utxo(&utxo2); assert_matches!(res1, Ok(id) if id == res0.unwrap()); @@ -563,7 +631,7 @@ mod tests { // Confirm that we no longer see any unspent outputs as of `height_1`. assert_matches!( st.wallet() - .get_unspent_transparent_outputs(taddr, height_1, &[]) + .get_spendable_transparent_outputs(taddr, height_1, 0) .as_deref(), Ok(&[]) ); @@ -577,7 +645,7 @@ mod tests { // If we include `height_2` then the output is returned. assert_matches!( st.wallet() - .get_unspent_transparent_outputs(taddr, height_2, &[]) + .get_spendable_transparent_outputs(taddr, height_2, 0) .as_deref(), Ok([ret]) if (ret.outpoint(), ret.txout(), ret.height()) == (utxo.outpoint(), utxo.txout(), height_2) ); @@ -639,13 +707,15 @@ mod tests { .account_balances() .get(&account.account_id()) .unwrap(); + // TODO: in the future, we will distinguish between available and total + // balance according to `min_confirmations` assert_eq!(balance.unshielded(), expected); // Check the older APIs for consistency. - let max_height = st.wallet().chain_height().unwrap().unwrap() + 1 - min_confirmations; + let mempool_height = st.wallet().chain_height().unwrap().unwrap() + 1; assert_eq!( st.wallet() - .get_transparent_balances(account.account_id(), max_height) + .get_transparent_balances(account.account_id(), mempool_height) .unwrap() .get(taddr) .cloned() @@ -654,7 +724,7 @@ mod tests { ); assert_eq!( st.wallet() - .get_unspent_transparent_outputs(taddr, max_height, &[]) + .get_spendable_transparent_outputs(taddr, mempool_height, 0) .unwrap() .into_iter() .map(|utxo| utxo.value()) @@ -664,8 +734,11 @@ mod tests { }; // The wallet starts out with zero balance. + // TODO: Once we have refactored `get_wallet_summary` to distinguish between available + // and total balance, we should perform additional checks against available balance; + // we use minconf 0 here because all transparent funds are considered shieldable, + // irrespective of confirmation depth. check_balance(&st, 0, NonNegativeAmount::ZERO); - check_balance(&st, 1, NonNegativeAmount::ZERO); // Create a fake transparent output. let value = NonNegativeAmount::from_u64(100000).unwrap(); @@ -681,10 +754,8 @@ mod tests { .put_received_transparent_utxo(&utxo) .unwrap(); - // The wallet should detect the balance as having 1 confirmation. + // The wallet should detect the balance as available check_balance(&st, 0, value); - check_balance(&st, 1, value); - check_balance(&st, 2, NonNegativeAmount::ZERO); // Shield the output. let input_selector = GreedyInputSelector::new( @@ -702,8 +773,6 @@ mod tests { // The wallet should have zero transparent balance, because the shielding // transaction can be mined. check_balance(&st, 0, NonNegativeAmount::ZERO); - check_balance(&st, 1, NonNegativeAmount::ZERO); - check_balance(&st, 2, NonNegativeAmount::ZERO); // Mine the shielding transaction. let (mined_height, _) = st.generate_next_block_including(txid); @@ -711,8 +780,6 @@ mod tests { // The wallet should still have zero transparent balance. check_balance(&st, 0, NonNegativeAmount::ZERO); - check_balance(&st, 1, NonNegativeAmount::ZERO); - check_balance(&st, 2, NonNegativeAmount::ZERO); // Unmine the shielding transaction via a reorg. st.wallet_mut() @@ -722,8 +789,6 @@ mod tests { // The wallet should still have zero transparent balance. check_balance(&st, 0, NonNegativeAmount::ZERO); - check_balance(&st, 1, NonNegativeAmount::ZERO); - check_balance(&st, 2, NonNegativeAmount::ZERO); // Expire the shielding transaction. let expiry_height = st @@ -734,22 +799,6 @@ mod tests { .expiry_height(); st.wallet_mut().update_chain_tip(expiry_height).unwrap(); - // TODO: Making the transparent output spendable in this situation requires - // changes to the transparent data model, so for now the wallet should still have - // zero transparent balance. https://github.com/zcash/librustzcash/issues/986 - check_balance(&st, 0, NonNegativeAmount::ZERO); - check_balance(&st, 1, NonNegativeAmount::ZERO); - check_balance(&st, 2, NonNegativeAmount::ZERO); - - // Roll forward the chain tip until the transaction's expiry height is in the - // stable block range (so a reorg won't make it spendable again). - st.wallet_mut() - .update_chain_tip(expiry_height + PRUNING_DEPTH) - .unwrap(); - - // The transparent output should be spendable again, with more confirmations. check_balance(&st, 0, value); - check_balance(&st, 1, value); - check_balance(&st, 2, value); } } From 10258b8443f92726bc0ce514e63898d8168e53fc Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 21 Jun 2024 19:04:48 -0600 Subject: [PATCH 3/4] zcash_client_sqlite: Address comments from code review. --- zcash_client_sqlite/src/wallet.rs | 10 ++++++---- zcash_client_sqlite/src/wallet/db.rs | 9 ++++++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index eb019ea81b..d0cf7faf45 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -1856,8 +1856,10 @@ pub(crate) fn truncate_to_height( // spends in the transaction. conn.execute( "UPDATE transparent_received_outputs - SET max_observed_unspent_height = NULL - WHERE max_observed_unspent_height > :height", + SET max_observed_unspent_height = CASE WHEN tx.mined_height <= :height THEN :height ELSE NULL END + FROM transactions tx + WHERE tx.id_tx = transaction_id + AND max_observed_unspent_height > :height", named_params![":height": u32::from(block_height)], )?; @@ -1866,8 +1868,8 @@ pub(crate) fn truncate_to_height( conn.execute( "UPDATE transactions SET block = NULL, mined_height = NULL, tx_index = NULL - WHERE block > ?", - [u32::from(block_height)], + WHERE mined_height > :height", + named_params![":height": u32::from(block_height)], )?; // If we're removing scanned blocks, we need to truncate the note commitment tree and remove diff --git a/zcash_client_sqlite/src/wallet/db.rs b/zcash_client_sqlite/src/wallet/db.rs index edd4e8b39e..27274feba9 100644 --- a/zcash_client_sqlite/src/wallet/db.rs +++ b/zcash_client_sqlite/src/wallet/db.rs @@ -211,8 +211,8 @@ CREATE INDEX orchard_received_notes_tx ON orchard_received_notes ( /// A junction table between received Orchard notes and the transactions that spend them. /// -/// This is identical to [`TABLE_SAPLING_RECEIVED_NOTE_SPENDS`]; see its documentation for -/// details. +/// Thie plays the same role for Orchard notes as does [`TABLE_SAPLING_RECEIVED_NOTE_SPENDS`] for +/// Sapling notes; see its documentation for details. pub(super) const TABLE_ORCHARD_RECEIVED_NOTE_SPENDS: &str = " CREATE TABLE orchard_received_note_spends ( orchard_received_note_id INTEGER NOT NULL, @@ -245,7 +245,10 @@ CREATE TABLE orchard_received_note_spends ( /// - `transaction_id`: Reference to the transaction in which this TXO was created /// - `output_index`: The output index of this TXO in the transaction referred to by `transaction_id` /// - `account_id`: The account that controls spend authority for this TXO -/// - `address`: The address to which this TXO was sent +/// - `address`: The address to which this TXO was sent. We store this address to make querying +/// for UTXOs for a single address easier, because when shielding we always select UTXOs +/// for only a single address at a time to prevent linking addresses in the shielding +/// transaction. /// - `script`: The full txout script /// - `value_zat`: The value of the TXO in zatoshis /// - `max_observed_unspent_height`: The maximum block height at which this TXO was either From 9716617b558bec40f937fbe5c82b84ed3645e229 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 24 Jun 2024 11:40:34 -0600 Subject: [PATCH 4/4] Apply suggestions from code review Co-authored-by: Jack Grigg Co-authored-by: Daira-Emma Hopwood --- zcash_client_sqlite/CHANGELOG.md | 1 + zcash_client_sqlite/src/wallet/db.rs | 35 ++- .../wallet/init/migrations/utxos_to_txos.rs | 240 +++++++++--------- zcash_client_sqlite/src/wallet/transparent.rs | 2 - 4 files changed, 147 insertions(+), 131 deletions(-) diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index 738284fc41..6c3e8df808 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -11,6 +11,7 @@ and this library adheres to Rust's notion of - `zcash_client_sqlite::error::SqliteClientError` has changed variants: - Removed `HdwalletError`. - Added `TransparentDerivation`. +- The `block` column of the `v_transactions` view has been renamed to `mined_height`. ## [0.10.3] - 2024-04-08 diff --git a/zcash_client_sqlite/src/wallet/db.rs b/zcash_client_sqlite/src/wallet/db.rs index 27274feba9..6e54eb3dcf 100644 --- a/zcash_client_sqlite/src/wallet/db.rs +++ b/zcash_client_sqlite/src/wallet/db.rs @@ -609,7 +609,7 @@ notes AS ( CASE WHEN ro.is_change THEN 1 ELSE 0 - END AS is_change, + END AS change_note_count, CASE WHEN ro.is_change THEN 0 ELSE 1 @@ -630,7 +630,7 @@ notes AS ( ro.pool AS pool, id_within_pool_table, -ro.value AS value, - 0 AS is_change, + 0 AS change_note_count, 0 AS received_count, 0 AS memo_present FROM v_received_outputs ro @@ -672,7 +672,7 @@ SELECT notes.account_id AS account_id, transactions.raw AS raw, SUM(notes.value) AS account_balance_delta, transactions.fee AS fee_paid, - SUM(notes.is_change) > 0 AS has_change, + SUM(notes.change_note_count) > 0 AS has_change, MAX(COALESCE(sent_note_counts.sent_notes, 0)) AS sent_note_count, SUM(notes.received_count) AS received_note_count, SUM(notes.memo_present) + MAX(COALESCE(sent_note_counts.memo_count, 0)) AS memo_count, @@ -691,8 +691,22 @@ LEFT JOIN sent_note_counts AND sent_note_counts.txid = notes.txid GROUP BY notes.account_id, notes.txid"; +/// Selects all outputs received by the wallet, plus any outputs sent from the wallet to +/// external recipients. +/// +/// This will contain: +/// * Outputs received from external recipients +/// * Outputs sent to external recipients +/// * Outputs received as part of a wallet-internal operation, including +/// both outputs received as a consequence of wallet-internal transfers +/// and as change. +/// +/// The `to_address` column will only contain an address when the recipient is +/// external. In all other cases, the recipient account id indicates the account +/// that controls the output. pub(super) const VIEW_TX_OUTPUTS: &str = " CREATE VIEW v_tx_outputs AS +-- select all outputs received by the wallet SELECT transactions.txid AS txid, ro.pool AS output_pool, ro.output_index AS output_index, @@ -705,24 +719,25 @@ SELECT transactions.txid AS txid, FROM v_received_outputs ro JOIN transactions ON transactions.id_tx = ro.transaction_id -LEFT JOIN sent_notes - ON sent_notes.id = ro.sent_note_id +-- join to the sent_notes table to obtain `from_account_id` +LEFT JOIN sent_notes ON sent_notes.id = ro.sent_note_id UNION +-- select all outputs sent from the wallet to external recipients SELECT transactions.txid AS txid, sent_notes.output_pool AS output_pool, sent_notes.output_index AS output_index, sent_notes.from_account_id AS from_account_id, - ro.account_id AS to_account_id, + NULL AS to_account_id, sent_notes.to_address AS to_address, sent_notes.value AS value, - 0 AS is_change, + FALSE AS is_change, sent_notes.memo AS memo FROM sent_notes JOIN transactions ON transactions.id_tx = sent_notes.tx -LEFT JOIN v_received_outputs ro - ON sent_notes.id = ro.sent_note_id -WHERE COALESCE(ro.is_change, 0) = 0"; +LEFT JOIN v_received_outputs ro ON ro.sent_note_id = sent_notes.id +-- exclude any sent notes for which a row exists in the v_received_outputs view +WHERE ro.account_id IS NULL"; pub(super) fn view_sapling_shard_scan_ranges(params: &P) -> String { format!( diff --git a/zcash_client_sqlite/src/wallet/init/migrations/utxos_to_txos.rs b/zcash_client_sqlite/src/wallet/init/migrations/utxos_to_txos.rs index 1d2680b657..b1ccbbd7fa 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/utxos_to_txos.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/utxos_to_txos.rs @@ -188,131 +188,133 @@ impl RusqliteMigration for Migration { FROM transparent_received_output_spends; CREATE VIEW v_transactions AS - WITH - notes AS ( - -- Outputs received in this transaction - SELECT ro.account_id AS account_id, - transactions.mined_height AS mined_height, - transactions.txid AS txid, - ro.pool AS pool, - id_within_pool_table, - ro.value AS value, - CASE - WHEN ro.is_change THEN 1 - ELSE 0 - END AS is_change, - CASE - WHEN ro.is_change THEN 0 - ELSE 1 - END AS received_count, - CASE - WHEN (ro.memo IS NULL OR ro.memo = X'F6') - THEN 0 - ELSE 1 - END AS memo_present - FROM v_received_outputs ro - JOIN transactions - ON transactions.id_tx = ro.transaction_id - UNION - -- Outputs spent in this transaction - SELECT ro.account_id AS account_id, - transactions.mined_height AS mined_height, - transactions.txid AS txid, - ro.pool AS pool, - id_within_pool_table, - -ro.value AS value, - 0 AS is_change, - 0 AS received_count, - 0 AS memo_present - FROM v_received_outputs ro - JOIN v_received_output_spends ros - ON ros.pool = ro.pool - AND ros.received_output_id = ro.id_within_pool_table - JOIN transactions - ON transactions.id_tx = ro.transaction_id - ), - -- Obtain a count of the notes that the wallet created in each transaction, - -- not counting change notes. - sent_note_counts AS ( - SELECT sent_notes.from_account_id AS account_id, - transactions.txid AS txid, - COUNT(DISTINCT sent_notes.id) AS sent_notes, - SUM( - CASE - WHEN (sent_notes.memo IS NULL OR sent_notes.memo = X'F6' OR ro.transaction_id IS NOT NULL) - THEN 0 - ELSE 1 - END - ) AS memo_count - FROM sent_notes - JOIN transactions - ON transactions.id_tx = sent_notes.tx - LEFT JOIN v_received_outputs ro - ON sent_notes.id = ro.sent_note_id - WHERE COALESCE(ro.is_change, 0) = 0 - GROUP BY account_id, txid - ), - blocks_max_height AS ( - SELECT MAX(blocks.height) AS max_height FROM blocks - ) - SELECT notes.account_id AS account_id, - notes.mined_height AS mined_height, - notes.txid AS txid, - transactions.tx_index AS tx_index, - transactions.expiry_height AS expiry_height, - transactions.raw AS raw, - SUM(notes.value) AS account_balance_delta, - transactions.fee AS fee_paid, - SUM(notes.is_change) > 0 AS has_change, - MAX(COALESCE(sent_note_counts.sent_notes, 0)) AS sent_note_count, - SUM(notes.received_count) AS received_note_count, - SUM(notes.memo_present) + MAX(COALESCE(sent_note_counts.memo_count, 0)) AS memo_count, - blocks.time AS block_time, - ( - blocks.height IS NULL - AND transactions.expiry_height BETWEEN 1 AND blocks_max_height.max_height - ) AS expired_unmined - FROM notes - LEFT JOIN transactions - ON notes.txid = transactions.txid - JOIN blocks_max_height - LEFT JOIN blocks ON blocks.height = notes.mined_height - LEFT JOIN sent_note_counts - ON sent_note_counts.account_id = notes.account_id - AND sent_note_counts.txid = notes.txid - GROUP BY notes.account_id, notes.txid; - - CREATE VIEW v_tx_outputs AS - SELECT transactions.txid AS txid, - ro.pool AS output_pool, - ro.output_index AS output_index, - sent_notes.from_account_id AS from_account_id, - ro.account_id AS to_account_id, - NULL AS to_address, - ro.value AS value, - ro.is_change AS is_change, - ro.memo AS memo + WITH + notes AS ( + -- Outputs received in this transaction + SELECT ro.account_id AS account_id, + transactions.mined_height AS mined_height, + transactions.txid AS txid, + ro.pool AS pool, + id_within_pool_table, + ro.value AS value, + CASE + WHEN ro.is_change THEN 1 + ELSE 0 + END AS change_note_count, + CASE + WHEN ro.is_change THEN 0 + ELSE 1 + END AS received_count, + CASE + WHEN (ro.memo IS NULL OR ro.memo = X'F6') + THEN 0 + ELSE 1 + END AS memo_present FROM v_received_outputs ro JOIN transactions - ON transactions.id_tx = ro.transaction_id - LEFT JOIN sent_notes - ON sent_notes.id = ro.sent_note_id + ON transactions.id_tx = ro.transaction_id UNION - SELECT transactions.txid AS txid, - sent_notes.output_pool AS output_pool, - sent_notes.output_index AS output_index, - sent_notes.from_account_id AS from_account_id, - ro.account_id AS to_account_id, - sent_notes.to_address AS to_address, - sent_notes.value AS value, - 0 AS is_change, - sent_notes.memo AS memo + -- Outputs spent in this transaction + SELECT ro.account_id AS account_id, + transactions.mined_height AS mined_height, + transactions.txid AS txid, + ro.pool AS pool, + id_within_pool_table, + -ro.value AS value, + 0 AS change_note_count, + 0 AS received_count, + 0 AS memo_present + FROM v_received_outputs ro + JOIN v_received_output_spends ros + ON ros.pool = ro.pool + AND ros.received_output_id = ro.id_within_pool_table + JOIN transactions + ON transactions.id_tx = ro.transaction_id + ), + -- Obtain a count of the notes that the wallet created in each transaction, + -- not counting change notes. + sent_note_counts AS ( + SELECT sent_notes.from_account_id AS account_id, + transactions.txid AS txid, + COUNT(DISTINCT sent_notes.id) AS sent_notes, + SUM( + CASE + WHEN (sent_notes.memo IS NULL OR sent_notes.memo = X'F6' OR ro.transaction_id IS NOT NULL) + THEN 0 + ELSE 1 + END + ) AS memo_count FROM sent_notes JOIN transactions - ON transactions.id_tx = sent_notes.tx + ON transactions.id_tx = sent_notes.tx LEFT JOIN v_received_outputs ro - ON sent_notes.id = ro.sent_note_id - WHERE COALESCE(ro.is_change, 0) = 0; + ON sent_notes.id = ro.sent_note_id + WHERE COALESCE(ro.is_change, 0) = 0 + GROUP BY account_id, txid + ), + blocks_max_height AS ( + SELECT MAX(blocks.height) AS max_height FROM blocks + ) + SELECT notes.account_id AS account_id, + notes.mined_height AS mined_height, + notes.txid AS txid, + transactions.tx_index AS tx_index, + transactions.expiry_height AS expiry_height, + transactions.raw AS raw, + SUM(notes.value) AS account_balance_delta, + transactions.fee AS fee_paid, + SUM(notes.change_note_count) > 0 AS has_change, + MAX(COALESCE(sent_note_counts.sent_notes, 0)) AS sent_note_count, + SUM(notes.received_count) AS received_note_count, + SUM(notes.memo_present) + MAX(COALESCE(sent_note_counts.memo_count, 0)) AS memo_count, + blocks.time AS block_time, + ( + blocks.height IS NULL + AND transactions.expiry_height BETWEEN 1 AND blocks_max_height.max_height + ) AS expired_unmined + FROM notes + LEFT JOIN transactions + ON notes.txid = transactions.txid + JOIN blocks_max_height + LEFT JOIN blocks ON blocks.height = notes.mined_height + LEFT JOIN sent_note_counts + ON sent_note_counts.account_id = notes.account_id + AND sent_note_counts.txid = notes.txid + GROUP BY notes.account_id, notes.txid; + + CREATE VIEW v_tx_outputs AS + -- select all outputs received by the wallet + SELECT transactions.txid AS txid, + ro.pool AS output_pool, + ro.output_index AS output_index, + sent_notes.from_account_id AS from_account_id, + ro.account_id AS to_account_id, + NULL AS to_address, + ro.value AS value, + ro.is_change AS is_change, + ro.memo AS memo + FROM v_received_outputs ro + JOIN transactions + ON transactions.id_tx = ro.transaction_id + -- join to the sent_notes table to obtain `from_account_id` + LEFT JOIN sent_notes ON sent_notes.id = ro.sent_note_id + UNION + -- select all outputs sent from the wallet to external recipients + SELECT transactions.txid AS txid, + sent_notes.output_pool AS output_pool, + sent_notes.output_index AS output_index, + sent_notes.from_account_id AS from_account_id, + NULL AS to_account_id, + sent_notes.to_address AS to_address, + sent_notes.value AS value, + FALSE AS is_change, + sent_notes.memo AS memo + FROM sent_notes + JOIN transactions + ON transactions.id_tx = sent_notes.tx + LEFT JOIN v_received_outputs ro ON ro.sent_note_id = sent_notes.id + -- exclude any sent notes for which a row exists in the v_received_outputs view + WHERE ro.account_id IS NULL; DROP TABLE utxos; diff --git a/zcash_client_sqlite/src/wallet/transparent.rs b/zcash_client_sqlite/src/wallet/transparent.rs index c06e3620de..181bc30c8b 100644 --- a/zcash_client_sqlite/src/wallet/transparent.rs +++ b/zcash_client_sqlite/src/wallet/transparent.rs @@ -365,7 +365,6 @@ pub(crate) fn add_transparent_account_balances( mempool_height: BlockHeight, account_balances: &mut HashMap, ) -> Result<(), SqliteClientError> { - let transparent_trace = tracing::info_span!("stmt_transparent_balances").entered(); let mut stmt_account_balances = conn.prepare( "SELECT u.account_id, SUM(u.value_zat) FROM transparent_received_outputs u @@ -404,7 +403,6 @@ pub(crate) fn add_transparent_account_balances( .or_insert(AccountBalance::ZERO) .add_unshielded_value(value)?; } - drop(transparent_trace); Ok(()) }