diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 37d7fb3f1c..b722f206f5 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -10,13 +10,14 @@ and this library adheres to Rust's notion of ### Added - `zcash_client_backend::data_api`: - `chain::BlockCache` trait, behind the `sync` feature flag. + - `WalletRead::get_spendable_transparent_outputs`. - `zcash_client_backend::fees`: - `ChangeValue::{transparent, shielded}` - `sapling::EmptyBundleView` - `orchard::EmptyBundleView` - `zcash_client_backend::scanning`: - `testing` module -- `zcash_client_backend::sync` module, behind the `sync` feature flag. +- `zcash_client_backend::sync` module, behind the `sync` feature flag ### Changed - MSRV is now 1.70.0. @@ -52,6 +53,12 @@ and this library adheres to Rust's notion of `zcash_address::ZcashAddress`. This simplifies the process of tracking the original address to which value was sent. +### Removed +- `zcash_client_backend::data_api`: + - `WalletRead::get_unspent_transparent_outputs` has been removed because its + semantics were unclear and could not be clarified. Use + `WalletRead::get_spendable_transparent_outputs` instead. + ## [0.12.1] - 2024-03-27 ### Fixed diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index a535e50e28..fc89170786 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -665,10 +665,10 @@ pub trait InputSource { exclude: &[Self::NoteRef], ) -> Result, Self::Error>; - /// Fetches a spendable transparent output. + /// Fetches the transparent output corresponding to the provided `outpoint`. /// /// Returns `Ok(None)` if the UTXO is not known to belong to the wallet or is not - /// spendable. + /// spendable as of the chain tip height. #[cfg(feature = "transparent-inputs")] fn get_unspent_transparent_output( &self, @@ -677,14 +677,18 @@ pub trait InputSource { Ok(None) } - /// Returns a list of unspent transparent UTXOs that appear in the chain at heights up to and - /// including `max_height`. + /// Returns the list of transparent outputs received at `address` such that: + /// * The transaction that produced these outputs is mined or mineable as of `max_height`. + /// * Each returned output is unspent as of the current chain tip. + /// + /// The caller should filter these outputs to ensure they respect the desired number of + /// confirmations before attempting to spend them. #[cfg(feature = "transparent-inputs")] - fn get_unspent_transparent_outputs( + fn get_spendable_transparent_outputs( &self, _address: &TransparentAddress, - _max_height: BlockHeight, - _exclude: &[OutPoint], + _target_height: BlockHeight, + _min_confirmations: u32, ) -> Result, Self::Error> { Ok(vec![]) } diff --git a/zcash_client_backend/src/data_api/wallet/input_selection.rs b/zcash_client_backend/src/data_api/wallet/input_selection.rs index 3632fa56cf..e6579aa031 100644 --- a/zcash_client_backend/src/data_api/wallet/input_selection.rs +++ b/zcash_client_backend/src/data_api/wallet/input_selection.rs @@ -580,11 +580,7 @@ where let mut transparent_inputs: Vec = source_addrs .iter() .map(|taddr| { - wallet_db.get_unspent_transparent_outputs( - taddr, - target_height - min_confirmations, - &[], - ) + wallet_db.get_spendable_transparent_outputs(taddr, target_height, min_confirmations) }) .collect::>, _>>() .map_err(InputSelectorError::DataSource)? diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index b973e7d4da..a429eee12b 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -276,18 +276,18 @@ impl, P: consensus::Parameters> InputSource for } #[cfg(feature = "transparent-inputs")] - fn get_unspent_transparent_outputs( + fn get_spendable_transparent_outputs( &self, address: &TransparentAddress, - max_height: BlockHeight, - exclude: &[OutPoint], + target_height: BlockHeight, + min_confirmations: u32, ) -> Result, Self::Error> { - wallet::transparent::get_unspent_transparent_outputs( + wallet::transparent::get_spendable_transparent_outputs( self.conn.borrow(), &self.params, address, - max_height, - exclude, + target_height, + min_confirmations, ) } } @@ -430,9 +430,7 @@ impl, P: consensus::Parameters> WalletRead for W } fn chain_height(&self) -> Result, Self::Error> { - wallet::scan_queue_extrema(self.conn.borrow()) - .map(|h| h.map(|range| *range.end())) - .map_err(SqliteClientError::from) + wallet::chain_tip_height(self.conn.borrow()).map_err(SqliteClientError::from) } fn get_block_hash(&self, block_height: BlockHeight) -> Result, Self::Error> { diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 257c6e3cfd..eb019ea81b 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -490,7 +490,7 @@ pub(crate) fn add_account( // Rewrite the scan ranges from the birthday height up to the chain tip so that we'll ensure we // re-scan to find any notes that might belong to the newly added account. - if let Some(t) = scan_queue_extrema(conn)?.map(|range| *range.end()) { + if let Some(t) = chain_tip_height(conn)? { let rescan_range = birthday.height()..(t + 1); replace_queue_entries::( @@ -952,8 +952,8 @@ pub(crate) fn get_wallet_summary( min_confirmations: u32, progress: &impl ScanProgress, ) -> Result>, SqliteClientError> { - let chain_tip_height = match scan_queue_extrema(tx)? { - Some(range) => *range.end(), + let chain_tip_height = match chain_tip_height(tx)? { + Some(h) => h, None => { return Ok(None); } @@ -1019,7 +1019,7 @@ pub(crate) fn get_wallet_summary( ) -> Result<(), SqliteClientError>, { // If the shard containing the summary height contains any unscanned ranges that start below or - // including that height, none of our balance is currently spendable. + // including that height, none of our shielded balance is currently spendable. #[tracing::instrument(skip_all)] fn is_any_spendable( conn: &rusqlite::Connection, @@ -1167,12 +1167,7 @@ pub(crate) fn get_wallet_summary( drop(sapling_trace); #[cfg(feature = "transparent-inputs")] - transparent::add_transparent_account_balances( - tx, - chain_tip_height, - min_confirmations, - &mut account_balances, - )?; + transparent::add_transparent_account_balances(tx, chain_tip_height + 1, &mut account_balances)?; // The approach used here for Sapling and Orchard subtree indexing was a quick hack // that has not yet been replaced. TODO: Make less hacky. @@ -1503,30 +1498,23 @@ pub(crate) fn get_account( } /// Returns the minimum and maximum heights of blocks in the chain which may be scanned. -pub(crate) fn scan_queue_extrema( +pub(crate) fn chain_tip_height( conn: &rusqlite::Connection, -) -> Result>, rusqlite::Error> { - conn.query_row( - "SELECT MIN(block_range_start), MAX(block_range_end) FROM scan_queue", - [], - |row| { - let min_height: Option = row.get(0)?; - let max_height: Option = row.get(1)?; - - // Scan ranges are end-exclusive, so we subtract 1 from `max_height` to obtain the - // height of the last known chain tip; - Ok(min_height - .zip(max_height.map(|h| h.saturating_sub(1))) - .map(|(min, max)| RangeInclusive::new(min.into(), max.into()))) - }, - ) +) -> Result, rusqlite::Error> { + conn.query_row("SELECT MAX(block_range_end) FROM scan_queue", [], |row| { + let max_height: Option = row.get(0)?; + + // Scan ranges are end-exclusive, so we subtract 1 from `max_height` to obtain the + // height of the last known chain tip; + Ok(max_height.map(|h| BlockHeight::from(h.saturating_sub(1)))) + }) } pub(crate) fn get_target_and_anchor_heights( conn: &rusqlite::Connection, min_confirmations: NonZeroU32, ) -> Result, rusqlite::Error> { - match scan_queue_extrema(conn)?.map(|range| *range.end()) { + match chain_tip_height(conn)? { Some(chain_tip_height) => { let sapling_anchor_height = get_max_checkpointed_height( conn, @@ -1861,9 +1849,29 @@ pub(crate) fn truncate_to_height( named_params![":new_end_height": u32::from(block_height + 1)], )?; - // If we're removing scanned blocks, we need to truncate the note commitment tree, un-mine - // transactions, and remove received transparent outputs and affected block records from the - // database. + // Mark transparent utxos as un-mined. Since the TXO is now not mined, it would ideally be + // considered to have been returned to the mempool; it _might_ be spendable in this state, but + // we must also set its max_observed_unspent_height field to NULL because the transaction may + // be rendered entirely invalid by a reorg that alters anchor(s) used in constructing shielded + // spends in the transaction. + conn.execute( + "UPDATE transparent_received_outputs + SET max_observed_unspent_height = NULL + WHERE max_observed_unspent_height > :height", + named_params![":height": u32::from(block_height)], + )?; + + // Un-mine transactions. This must be done outside of the last_scanned_height check because + // transaction entries may be created as a consequence of receiving transparent TXOs. + conn.execute( + "UPDATE transactions + SET block = NULL, mined_height = NULL, tx_index = NULL + WHERE block > ?", + [u32::from(block_height)], + )?; + + // If we're removing scanned blocks, we need to truncate the note commitment tree and remove + // affected block records from the database. if block_height < last_scanned_height { // Truncate the note commitment trees let mut wdb = WalletDb { @@ -1885,20 +1893,6 @@ pub(crate) fn truncate_to_height( // not recoverable; balance APIs must ensure that un-mined received notes // do not count towards spendability or transaction balalnce. - // Rewind utxos. It is currently necessary to delete these because we do - // not have the full transaction data for the received output. - conn.execute( - "DELETE FROM utxos WHERE height > ?", - [u32::from(block_height)], - )?; - - // Un-mine transactions. - conn.execute( - "UPDATE transactions SET block = NULL, tx_index = NULL - WHERE block IS NOT NULL AND block > ?", - [u32::from(block_height)], - )?; - // Now that they aren't depended on, delete un-mined blocks. conn.execute( "DELETE FROM blocks WHERE height > ?", @@ -2010,6 +2004,25 @@ pub(crate) fn put_block( ":orchard_action_count": orchard_action_count, ])?; + // If we now have a block corresponding to a received transparent output that had not been + // scanned at the time the UTXO was discovered, update the associated transaction record to + // refer to that block. + // + // NOTE: There's a small data corruption hazard here, in that we're relying exclusively upon + // the block height to associate the transaction to the block. This is because CompactBlock + // values only contain CompactTx entries for transactions that contain shielded inputs or + // outputs, and the GetAddressUtxosReply data does not contain the block hash. As such, it's + // necessary to ensure that any chain rollback to below the received height causes that height + // to be set to NULL. + let mut stmt_update_transaction_block_reference = conn.prepare_cached( + "UPDATE transactions + SET block = :height + WHERE mined_height = :height", + )?; + + stmt_update_transaction_block_reference + .execute(named_params![":height": u32::from(block_height),])?; + Ok(()) } @@ -2022,10 +2035,11 @@ pub(crate) fn put_tx_meta( ) -> Result { // It isn't there, so insert our transaction into the database. let mut stmt_upsert_tx_meta = conn.prepare_cached( - "INSERT INTO transactions (txid, block, tx_index) - VALUES (:txid, :block, :tx_index) + "INSERT INTO transactions (txid, block, mined_height, tx_index) + VALUES (:txid, :block, :block, :tx_index) ON CONFLICT (txid) DO UPDATE SET block = :block, + mined_height = :block, tx_index = :tx_index RETURNING id_tx", )?; diff --git a/zcash_client_sqlite/src/wallet/db.rs b/zcash_client_sqlite/src/wallet/db.rs index 17db1d728d..edd4e8b39e 100644 --- a/zcash_client_sqlite/src/wallet/db.rs +++ b/zcash_client_sqlite/src/wallet/db.rs @@ -59,9 +59,9 @@ pub(super) const INDEX_HD_ACCOUNT: &str = /// Stores diversified Unified Addresses that have been generated from accounts in the /// wallet. /// -/// - The `cached_transparent_receiver_address` column contains the transparent receiver -/// component of the UA. It is cached directly in the table to make account lookups for -/// transparent outputs more efficient, enabling joins to [`TABLE_UTXOS`]. +/// - The `cached_transparent_receiver_address` column contains the transparent receiver component +/// of the UA. It is cached directly in the table to make account lookups for transparent outputs +/// more efficient, enabling joins to [`TABLE_TRANSPARENT_RECEIVED_OUTPUTS`]. pub(super) const TABLE_ADDRESSES: &str = r#" CREATE TABLE "addresses" ( account_id INTEGER NOT NULL, @@ -80,7 +80,7 @@ CREATE INDEX "addresses_accounts" ON "addresses" ( /// /// Note that this table does not contain any rows for blocks that the wallet might have /// observed partial information about (for example, a transparent output fetched and -/// stored in [`TABLE_UTXOS`]). This may change in future. +/// stored in [`TABLE_TRANSPARENT_RECEIVED_OUTPUTS`]). This may change in future. pub(super) const TABLE_BLOCKS: &str = " CREATE TABLE blocks ( height INTEGER PRIMARY KEY, @@ -99,22 +99,32 @@ CREATE TABLE blocks ( /// data that is not recoverable from the chain (for example, transactions created by the /// wallet that expired before being mined). /// -/// - The `block` column stores the height (in the wallet's chain view) of the mined block -/// containing the transaction. It is `NULL` for transactions that have not yet been -/// observed in scanned blocks, including transactions in the mempool or that have -/// expired. -pub(super) const TABLE_TRANSACTIONS: &str = " -CREATE TABLE transactions ( +/// ### Columns +/// - `created`: The time at which the transaction was created as a string in the format +/// `yyyy-MM-dd HH:mm:ss.fffffffzzz`. +/// - `block`: stores the height (in the wallet's chain view) of the mined block containing the +/// transaction. It is `NULL` for transactions that have not yet been observed in scanned blocks, +/// including transactions in the mempool or that have expired. +/// - `mined_height`: stores the height (in the wallet's chain view) of the mined block containing +/// the transaction. It is present to allow the block height for a retrieved transaction to be +/// stored without requiring that the entire block containing the transaction be scanned; the +/// foreign key constraint on `block` prevents that column from being populated prior to complete +/// scanning of the block. This is constrained to be equal to the `block` column if `block` is +/// non-null. +pub(super) const TABLE_TRANSACTIONS: &str = r#" +CREATE TABLE "transactions" ( id_tx INTEGER PRIMARY KEY, txid BLOB NOT NULL UNIQUE, created TEXT, block INTEGER, + mined_height INTEGER, tx_index INTEGER, expiry_height INTEGER, raw BLOB, fee INTEGER, - FOREIGN KEY (block) REFERENCES blocks(height) -)"; + FOREIGN KEY (block) REFERENCES blocks(height), + CONSTRAINT height_consistency CHECK (block IS NULL OR mined_height = block) +)"#; /// Stores the Sapling notes received by the wallet. /// @@ -216,54 +226,72 @@ CREATE TABLE orchard_received_note_spends ( UNIQUE (orchard_received_note_id, transaction_id) )"; -/// Stores the current UTXO set for the wallet, as well as any transparent outputs -/// previously observed by the wallet. +/// Stores the transparent outputs received by the wallet. /// /// Originally this table only stored the current UTXO set (as of latest refresh), and the /// table was cleared prior to loading in the latest UTXO set. We now upsert instead of /// insert into the database, meaning that spent outputs are left in the database. This -/// makes it similar to the `*_received_notes` tables in that it can store history, but -/// has several downsides: -/// - The table has incomplete contents for recovered-from-seed wallets. -/// - The table can have inconsistent contents for seeds loaded into multiple wallets +/// makes it similar to the `*_received_notes` tables in that it can store history. +/// Depending upon how transparent TXOs for the wallet are discovered, the following +/// may be true: +/// - The table may have incomplete contents for recovered-from-seed wallets. +/// - The table may have inconsistent contents for seeds loaded into multiple wallets /// simultaneously. -/// - The wallet's transparent balance can be incorrect prior to "transaction enhancement" +/// - The wallet's transparent balance may be incorrect prior to "transaction enhancement" /// (downloading the full transaction containing the transparent output spend). -pub(super) const TABLE_UTXOS: &str = r#" -CREATE TABLE "utxos" ( +/// +/// ### Columns: +/// - `id`: Primary key +/// - `transaction_id`: Reference to the transaction in which this TXO was created +/// - `output_index`: The output index of this TXO in the transaction referred to by `transaction_id` +/// - `account_id`: The account that controls spend authority for this TXO +/// - `address`: The address to which this TXO was sent +/// - `script`: The full txout script +/// - `value_zat`: The value of the TXO in zatoshis +/// - `max_observed_unspent_height`: The maximum block height at which this TXO was either +/// observed to be a member of the UTXO set at the start of the block, or observed +/// to be an output of a transaction mined in the block. This is intended to be used to +/// determine when the TXO is no longer a part of the UTXO set, in the case that the +/// transaction that spends it is not detected by the wallet. +pub(super) const TABLE_TRANSPARENT_RECEIVED_OUTPUTS: &str = r#" +CREATE TABLE transparent_received_outputs ( id INTEGER PRIMARY KEY, - received_by_account_id INTEGER NOT NULL, + transaction_id INTEGER NOT NULL, + output_index INTEGER NOT NULL, + account_id INTEGER NOT NULL, address TEXT NOT NULL, - prevout_txid BLOB NOT NULL, - prevout_idx INTEGER NOT NULL, script BLOB NOT NULL, value_zat INTEGER NOT NULL, - height INTEGER NOT NULL, - FOREIGN KEY (received_by_account_id) REFERENCES accounts(id), - CONSTRAINT tx_outpoint UNIQUE (prevout_txid, prevout_idx) + max_observed_unspent_height INTEGER, + FOREIGN KEY (transaction_id) REFERENCES transactions(id_tx), + FOREIGN KEY (account_id) REFERENCES accounts(id), + CONSTRAINT transparent_output_unique UNIQUE (transaction_id, output_index) )"#; -pub(super) const INDEX_UTXOS_RECEIVED_BY_ACCOUNT: &str = - r#"CREATE INDEX utxos_received_by_account ON "utxos" (received_by_account_id)"#; +pub(super) const INDEX_TRANSPARENT_RECEIVED_OUTPUTS_ACCOUNT_ID: &str = r#" +CREATE INDEX idx_transparent_received_outputs_account_id +ON "transparent_received_outputs" (account_id)"#; -/// A junction table between received transparent outputs and the transactions that spend -/// them. +/// A junction table between received transparent outputs and the transactions that spend them. /// -/// This is identical to [`TABLE_SAPLING_RECEIVED_NOTE_SPENDS`]; see its documentation for -/// details. Note however that [`TABLE_UTXOS`] and [`TABLE_SAPLING_RECEIVED_NOTES`] are -/// not equivalent, and care must be taken when interpreting the result of joining this -/// table to [`TABLE_UTXOS`]. -pub(super) const TABLE_TRANSPARENT_RECEIVED_OUTPUT_SPENDS: &str = " -CREATE TABLE transparent_received_output_spends ( +/// This plays the same role for transparent TXOs as does [`TABLE_SAPLING_RECEIVED_NOTE_SPENDS`] +/// for Sapling notes. However, [`TABLE_TRANSPARENT_RECEIVED_OUTPUTS`] differs from +/// [`TABLE_SAPLING_RECEIVED_NOTES`] and [`TABLE_ORCHARD_RECEIVED_NOTES`] in that an +/// associated `transactions` record may have its `mined_height` set without there existing a +/// corresponding record in the `blocks` table for a block at that height, due to the asymmetries +/// between scanning for shielded notes and retrieving transparent TXOs currently implemented +/// in [`zcash_client_backend`]. +pub(super) const TABLE_TRANSPARENT_RECEIVED_OUTPUT_SPENDS: &str = r#" +CREATE TABLE "transparent_received_output_spends" ( transparent_received_output_id INTEGER NOT NULL, transaction_id INTEGER NOT NULL, FOREIGN KEY (transparent_received_output_id) - REFERENCES utxos(id) + REFERENCES transparent_received_outputs(id) ON DELETE CASCADE, FOREIGN KEY (transaction_id) -- We do not delete transactions, so this does not cascade REFERENCES transactions(id_tx), UNIQUE (transparent_received_output_id, transaction_id) -)"; +)"#; /// Stores the outputs of transactions created by the wallet. /// @@ -497,219 +525,201 @@ pub(super) const TABLE_SQLITE_SEQUENCE: &str = "CREATE TABLE sqlite_sequence(nam // Views // -pub(super) const VIEW_RECEIVED_NOTES: &str = " -CREATE VIEW v_received_notes AS -SELECT - sapling_received_notes.id AS id_within_pool_table, - sapling_received_notes.tx, - 2 AS pool, - sapling_received_notes.output_index AS output_index, - account_id, - sapling_received_notes.value, - is_change, - sapling_received_notes.memo, - sent_notes.id AS sent_note_id -FROM sapling_received_notes -LEFT JOIN sent_notes -ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = - (sapling_received_notes.tx, 2, sapling_received_notes.output_index) +pub(super) const VIEW_RECEIVED_OUTPUTS: &str = " +CREATE VIEW v_received_outputs AS + SELECT + sapling_received_notes.id AS id_within_pool_table, + sapling_received_notes.tx AS transaction_id, + 2 AS pool, + sapling_received_notes.output_index, + account_id, + sapling_received_notes.value, + is_change, + sapling_received_notes.memo, + sent_notes.id AS sent_note_id + FROM sapling_received_notes + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (sapling_received_notes.tx, 2, sapling_received_notes.output_index) UNION -SELECT - orchard_received_notes.id AS id_within_pool_table, - orchard_received_notes.tx, - 3 AS pool, - orchard_received_notes.action_index AS output_index, - account_id, - orchard_received_notes.value, - is_change, - orchard_received_notes.memo, - sent_notes.id AS sent_note_id -FROM orchard_received_notes -LEFT JOIN sent_notes -ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = - (orchard_received_notes.tx, 3, orchard_received_notes.action_index)"; - -pub(super) const VIEW_RECEIVED_NOTE_SPENDS: &str = " -CREATE VIEW v_received_note_spends AS + SELECT + orchard_received_notes.id AS id_within_pool_table, + orchard_received_notes.tx AS transaction_id, + 3 AS pool, + orchard_received_notes.action_index AS output_index, + account_id, + orchard_received_notes.value, + is_change, + orchard_received_notes.memo, + sent_notes.id AS sent_note_id + FROM orchard_received_notes + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (orchard_received_notes.tx, 3, orchard_received_notes.action_index) +UNION + SELECT + u.id AS id_within_pool_table, + u.transaction_id, + 0 AS pool, + u.output_index, + u.account_id, + u.value_zat AS value, + 0 AS is_change, + NULL AS memo, + sent_notes.id AS sent_note_id + FROM transparent_received_outputs u + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (u.transaction_id, 0, u.output_index)"; + +pub(super) const VIEW_RECEIVED_OUTPUT_SPENDS: &str = " +CREATE VIEW v_received_output_spends AS SELECT 2 AS pool, - sapling_received_note_id AS received_note_id, + sapling_received_note_id AS received_output_id, transaction_id FROM sapling_received_note_spends UNION SELECT 3 AS pool, - orchard_received_note_id AS received_note_id, + orchard_received_note_id AS received_output_id, transaction_id -FROM orchard_received_note_spends"; +FROM orchard_received_note_spends +UNION +SELECT + 0 AS pool, + transparent_received_output_id AS received_output_id, + transaction_id +FROM transparent_received_output_spends"; pub(super) const VIEW_TRANSACTIONS: &str = " CREATE VIEW v_transactions AS WITH notes AS ( - -- Shielded notes received in this transaction - SELECT v_received_notes.account_id AS account_id, - transactions.block AS block, - transactions.txid AS txid, - v_received_notes.pool AS pool, - id_within_pool_table, - v_received_notes.value AS value, - CASE - WHEN v_received_notes.is_change THEN 1 + -- Outputs received in this transaction + SELECT ro.account_id AS account_id, + transactions.mined_height AS mined_height, + transactions.txid AS txid, + ro.pool AS pool, + id_within_pool_table, + ro.value AS value, + CASE + WHEN ro.is_change THEN 1 ELSE 0 - END AS is_change, - CASE - WHEN v_received_notes.is_change THEN 0 - ELSE 1 - END AS received_count, - CASE - WHEN (v_received_notes.memo IS NULL OR v_received_notes.memo = X'F6') - THEN 0 + END AS is_change, + CASE + WHEN ro.is_change THEN 0 ELSE 1 - END AS memo_present - FROM v_received_notes + END AS received_count, + CASE + WHEN (ro.memo IS NULL OR ro.memo = X'F6') + THEN 0 + ELSE 1 + END AS memo_present + FROM v_received_outputs ro JOIN transactions - ON transactions.id_tx = v_received_notes.tx + ON transactions.id_tx = ro.transaction_id UNION - -- Transparent TXOs received in this transaction - SELECT utxos.received_by_account_id AS account_id, - utxos.height AS block, - utxos.prevout_txid AS txid, - 0 AS pool, - utxos.id AS id_within_pool_table, - utxos.value_zat AS value, - 0 AS is_change, - 1 AS received_count, - 0 AS memo_present - FROM utxos - UNION - -- Shielded notes spent in this transaction - SELECT v_received_notes.account_id AS account_id, - transactions.block AS block, - transactions.txid AS txid, - v_received_notes.pool AS pool, - id_within_pool_table, - -v_received_notes.value AS value, - 0 AS is_change, - 0 AS received_count, - 0 AS memo_present - FROM v_received_notes - JOIN v_received_note_spends rns - ON rns.pool = v_received_notes.pool - AND rns.received_note_id = v_received_notes.id_within_pool_table - JOIN transactions - ON transactions.id_tx = rns.transaction_id - UNION - -- Transparent TXOs spent in this transaction - SELECT utxos.received_by_account_id AS account_id, - transactions.block AS block, - transactions.txid AS txid, - 0 AS pool, - utxos.id AS id_within_pool_table, - -utxos.value_zat AS value, - 0 AS is_change, - 0 AS received_count, - 0 AS memo_present - FROM utxos - JOIN transparent_received_output_spends tros - ON tros.transparent_received_output_id = utxos.id + -- Outputs spent in this transaction + SELECT ro.account_id AS account_id, + transactions.mined_height AS mined_height, + transactions.txid AS txid, + ro.pool AS pool, + id_within_pool_table, + -ro.value AS value, + 0 AS is_change, + 0 AS received_count, + 0 AS memo_present + FROM v_received_outputs ro + JOIN v_received_output_spends ros + ON ros.pool = ro.pool + AND ros.received_output_id = ro.id_within_pool_table JOIN transactions - ON transactions.id_tx = tros.transaction_id + ON transactions.id_tx = ro.transaction_id ), -- Obtain a count of the notes that the wallet created in each transaction, -- not counting change notes. sent_note_counts AS ( - SELECT sent_notes.from_account_id AS account_id, - transactions.txid AS txid, - COUNT(DISTINCT sent_notes.id) as sent_notes, - SUM( - CASE - WHEN (sent_notes.memo IS NULL OR sent_notes.memo = X'F6' OR v_received_notes.tx IS NOT NULL) - THEN 0 - ELSE 1 - END - ) AS memo_count + SELECT sent_notes.from_account_id AS account_id, + transactions.txid AS txid, + COUNT(DISTINCT sent_notes.id) AS sent_notes, + SUM( + CASE + WHEN (sent_notes.memo IS NULL OR sent_notes.memo = X'F6' OR ro.transaction_id IS NOT NULL) + THEN 0 + ELSE 1 + END + ) AS memo_count FROM sent_notes JOIN transactions - ON transactions.id_tx = sent_notes.tx - LEFT JOIN v_received_notes - ON sent_notes.id = v_received_notes.sent_note_id - WHERE COALESCE(v_received_notes.is_change, 0) = 0 + ON transactions.id_tx = sent_notes.tx + LEFT JOIN v_received_outputs ro + ON sent_notes.id = ro.sent_note_id + WHERE COALESCE(ro.is_change, 0) = 0 GROUP BY account_id, txid ), blocks_max_height AS ( - SELECT MAX(blocks.height) as max_height FROM blocks + SELECT MAX(blocks.height) AS max_height FROM blocks ) -SELECT notes.account_id AS account_id, - notes.block AS mined_height, - notes.txid AS txid, - transactions.tx_index AS tx_index, - transactions.expiry_height AS expiry_height, - transactions.raw AS raw, - SUM(notes.value) AS account_balance_delta, - transactions.fee AS fee_paid, - SUM(notes.is_change) > 0 AS has_change, - MAX(COALESCE(sent_note_counts.sent_notes, 0)) AS sent_note_count, - SUM(notes.received_count) AS received_note_count, - SUM(notes.memo_present) + MAX(COALESCE(sent_note_counts.memo_count, 0)) AS memo_count, - blocks.time AS block_time, - ( +SELECT notes.account_id AS account_id, + notes.mined_height AS mined_height, + notes.txid AS txid, + transactions.tx_index AS tx_index, + transactions.expiry_height AS expiry_height, + transactions.raw AS raw, + SUM(notes.value) AS account_balance_delta, + transactions.fee AS fee_paid, + SUM(notes.is_change) > 0 AS has_change, + MAX(COALESCE(sent_note_counts.sent_notes, 0)) AS sent_note_count, + SUM(notes.received_count) AS received_note_count, + SUM(notes.memo_present) + MAX(COALESCE(sent_note_counts.memo_count, 0)) AS memo_count, + blocks.time AS block_time, + ( blocks.height IS NULL AND transactions.expiry_height BETWEEN 1 AND blocks_max_height.max_height - ) AS expired_unmined + ) AS expired_unmined FROM notes LEFT JOIN transactions - ON notes.txid = transactions.txid + ON notes.txid = transactions.txid JOIN blocks_max_height -LEFT JOIN blocks ON blocks.height = notes.block +LEFT JOIN blocks ON blocks.height = notes.mined_height LEFT JOIN sent_note_counts - ON sent_note_counts.account_id = notes.account_id - AND sent_note_counts.txid = notes.txid + ON sent_note_counts.account_id = notes.account_id + AND sent_note_counts.txid = notes.txid GROUP BY notes.account_id, notes.txid"; pub(super) const VIEW_TX_OUTPUTS: &str = " CREATE VIEW v_tx_outputs AS -SELECT transactions.txid AS txid, - v_received_notes.pool AS output_pool, - v_received_notes.output_index AS output_index, - sent_notes.from_account_id AS from_account_id, - v_received_notes.account_id AS to_account_id, - NULL AS to_address, - v_received_notes.value AS value, - v_received_notes.is_change AS is_change, - v_received_notes.memo AS memo -FROM v_received_notes +SELECT transactions.txid AS txid, + ro.pool AS output_pool, + ro.output_index AS output_index, + sent_notes.from_account_id AS from_account_id, + ro.account_id AS to_account_id, + NULL AS to_address, + ro.value AS value, + ro.is_change AS is_change, + ro.memo AS memo +FROM v_received_outputs ro JOIN transactions - ON transactions.id_tx = v_received_notes.tx + ON transactions.id_tx = ro.transaction_id LEFT JOIN sent_notes - ON sent_notes.id = v_received_notes.sent_note_id -UNION -SELECT utxos.prevout_txid AS txid, - 0 AS output_pool, - utxos.prevout_idx AS output_index, - NULL AS from_account_id, - utxos.received_by_account_id AS to_account_id, - utxos.address AS to_address, - utxos.value_zat AS value, - 0 AS is_change, - NULL AS memo -FROM utxos + ON sent_notes.id = ro.sent_note_id UNION SELECT transactions.txid AS txid, - sent_notes.output_pool AS output_pool, - sent_notes.output_index AS output_index, - sent_notes.from_account_id AS from_account_id, - v_received_notes.account_id AS to_account_id, - sent_notes.to_address AS to_address, - sent_notes.value AS value, - 0 AS is_change, - sent_notes.memo AS memo + sent_notes.output_pool AS output_pool, + sent_notes.output_index AS output_index, + sent_notes.from_account_id AS from_account_id, + ro.account_id AS to_account_id, + sent_notes.to_address AS to_address, + sent_notes.value AS value, + 0 AS is_change, + sent_notes.memo AS memo FROM sent_notes JOIN transactions ON transactions.id_tx = sent_notes.tx -LEFT JOIN v_received_notes - ON sent_notes.id = v_received_notes.sent_note_id -WHERE COALESCE(v_received_notes.is_change, 0) = 0"; +LEFT JOIN v_received_outputs ro + ON sent_notes.id = ro.sent_note_id +WHERE COALESCE(ro.is_change, 0) = 0"; pub(super) fn view_sapling_shard_scan_ranges(params: &P) -> String { format!( diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 0aacc91e53..08838d0486 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -395,8 +395,8 @@ mod tests { db::TABLE_SQLITE_SEQUENCE, db::TABLE_TRANSACTIONS, db::TABLE_TRANSPARENT_RECEIVED_OUTPUT_SPENDS, + db::TABLE_TRANSPARENT_RECEIVED_OUTPUTS, db::TABLE_TX_LOCATOR_MAP, - db::TABLE_UTXOS, ]; let rows = describe_tables(&st.wallet().conn).unwrap(); @@ -421,7 +421,7 @@ mod tests { db::INDEX_SENT_NOTES_FROM_ACCOUNT, db::INDEX_SENT_NOTES_TO_ACCOUNT, db::INDEX_SENT_NOTES_TX, - db::INDEX_UTXOS_RECEIVED_BY_ACCOUNT, + db::INDEX_TRANSPARENT_RECEIVED_OUTPUTS_ACCOUNT_ID, ]; let mut indices_query = st .wallet() @@ -443,8 +443,8 @@ mod tests { db::view_orchard_shard_scan_ranges(&st.network()), db::view_orchard_shard_unscanned_ranges(), db::VIEW_ORCHARD_SHARDS_SCAN_STATE.to_owned(), - db::VIEW_RECEIVED_NOTE_SPENDS.to_owned(), - db::VIEW_RECEIVED_NOTES.to_owned(), + db::VIEW_RECEIVED_OUTPUT_SPENDS.to_owned(), + db::VIEW_RECEIVED_OUTPUTS.to_owned(), db::view_sapling_shard_scan_ranges(&st.network()), db::view_sapling_shard_unscanned_ranges(), db::VIEW_SAPLING_SHARDS_SCAN_STATE.to_owned(), diff --git a/zcash_client_sqlite/src/wallet/init/migrations.rs b/zcash_client_sqlite/src/wallet/init/migrations.rs index 8a09ae2d38..1f4befd996 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations.rs @@ -15,6 +15,7 @@ mod sent_notes_to_internal; mod shardtree_support; mod ufvk_support; mod utxos_table; +mod utxos_to_txos; mod v_sapling_shard_unscanned_ranges; mod v_transactions_net; mod v_transactions_note_uniqueness; @@ -63,8 +64,8 @@ pub(super) fn all_migrations( // -------------------- full_account_ids // | // orchard_received_notes - // | - // ensure_orchard_ua_receiver + // / \ + // ensure_orchard_ua_receiver utxos_to_txos vec![ Box::new(initial_setup::Migration {}), Box::new(utxos_table::Migration {}), @@ -114,6 +115,7 @@ pub(super) fn all_migrations( Box::new(ensure_orchard_ua_receiver::Migration { params: params.clone(), }), + Box::new(utxos_to_txos::Migration), ] } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/orchard_shardtree.rs b/zcash_client_sqlite/src/wallet/init/migrations/orchard_shardtree.rs index 564c345fca..dc44f1e087 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/orchard_shardtree.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/orchard_shardtree.rs @@ -11,7 +11,7 @@ use zcash_client_backend::data_api::scanning::ScanPriority; use zcash_protocol::consensus::{self, BlockHeight, NetworkUpgrade}; use super::shardtree_support; -use crate::wallet::{init::WalletMigrationError, scan_queue_extrema, scanning::priority_code}; +use crate::wallet::{chain_tip_height, init::WalletMigrationError, scanning::priority_code}; pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0x3a6487f7_e068_42bb_9d12_6bb8dbe6da00); @@ -142,10 +142,10 @@ impl RusqliteMigration for Migration

{ // Treat the current best-known chain tip height as the height to use for Orchard // initialization, bounded below by NU5 activation. - if let Some(orchard_init_height) = scan_queue_extrema(transaction)?.and_then(|r| { + if let Some(orchard_init_height) = chain_tip_height(transaction)?.and_then(|h| { self.params .activation_height(NetworkUpgrade::Nu5) - .map(|orchard_activation| std::cmp::max(orchard_activation, *r.end())) + .map(|orchard_activation| std::cmp::max(orchard_activation, h)) }) { // If a scan range exists that contains the Orchard init height, split it in two at the // init height. diff --git a/zcash_client_sqlite/src/wallet/init/migrations/receiving_key_scopes.rs b/zcash_client_sqlite/src/wallet/init/migrations/receiving_key_scopes.rs index 1c1dd6bcc3..cd1f768455 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/receiving_key_scopes.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/receiving_key_scopes.rs @@ -28,9 +28,10 @@ use zcash_primitives::{ use crate::{ wallet::{ + chain_tip_height, commitment_tree::SqliteShardStore, init::{migrations::shardtree_support, WalletMigrationError}, - scan_queue_extrema, scope_code, + scope_code, }, PRUNING_DEPTH, SAPLING_TABLES_PREFIX, }; @@ -154,7 +155,7 @@ impl RusqliteMigration for Migration

{ let zip212_height = tx_height.map_or_else( || { tx_expiry.filter(|h| *h != 0).map_or_else( - || scan_queue_extrema(transaction).map(|extrema| extrema.map(|r| *r.end())), + || chain_tip_height(transaction), |h| Ok(Some(BlockHeight::from(h))), ) }, @@ -286,13 +287,14 @@ mod tests { slice::ParallelSliceMut, }; use rand_core::OsRng; - use rusqlite::{named_params, params, Connection}; + use rusqlite::{named_params, params, Connection, OptionalExtension}; use tempfile::NamedTempFile; use zcash_client_backend::{ data_api::{BlockMetadata, WalletCommitmentTrees, SAPLING_SHARD_HEIGHT}, decrypt_transaction, proto::compact_formats::{CompactBlock, CompactTx}, scanning::{scan_block, Nullifiers, ScanningKeys}, + wallet::WalletTx, TransferType, }; use zcash_keys::keys::{UnifiedFullViewingKey, UnifiedSpendingKey}; @@ -647,7 +649,7 @@ mod tests { } // Insert the block into the database. - crate::wallet::put_block( + put_block( wdb.conn.0, block.height(), block.block_hash(), @@ -661,7 +663,7 @@ mod tests { )?; for tx in block.transactions() { - let tx_row = crate::wallet::put_tx_meta(wdb.conn.0, tx, block.height())?; + let tx_row = put_tx_meta(wdb.conn.0, tx, block.height())?; for output in tx.sapling_outputs() { put_received_note_before_migration(wdb.conn.0, output, tx_row, None)?; @@ -745,4 +747,118 @@ mod tests { } assert_eq!(row_count, 2); } + + /// This is a copy of [`crate::wallet::put_block`] as of the expected database + /// state corresponding to this migration. It is duplicated here as later + /// updates to the database schema require incompatible changes to `put_block`. + #[allow(clippy::too_many_arguments)] + fn put_block( + conn: &rusqlite::Transaction<'_>, + block_height: BlockHeight, + block_hash: BlockHash, + block_time: u32, + sapling_commitment_tree_size: u32, + sapling_output_count: u32, + #[cfg(feature = "orchard")] orchard_commitment_tree_size: u32, + #[cfg(feature = "orchard")] orchard_action_count: u32, + ) -> Result<(), SqliteClientError> { + let block_hash_data = conn + .query_row( + "SELECT hash FROM blocks WHERE height = ?", + [u32::from(block_height)], + |row| row.get::<_, Vec>(0), + ) + .optional()?; + + // Ensure that in the case of an upsert, we don't overwrite block data + // with information for a block with a different hash. + if let Some(bytes) = block_hash_data { + let expected_hash = BlockHash::try_from_slice(&bytes).ok_or_else(|| { + SqliteClientError::CorruptedData(format!( + "Invalid block hash at height {}", + u32::from(block_height) + )) + })?; + if expected_hash != block_hash { + return Err(SqliteClientError::BlockConflict(block_height)); + } + } + + let mut stmt_upsert_block = conn.prepare_cached( + "INSERT INTO blocks ( + height, + hash, + time, + sapling_commitment_tree_size, + sapling_output_count, + sapling_tree, + orchard_commitment_tree_size, + orchard_action_count + ) + VALUES ( + :height, + :hash, + :block_time, + :sapling_commitment_tree_size, + :sapling_output_count, + x'00', + :orchard_commitment_tree_size, + :orchard_action_count + ) + ON CONFLICT (height) DO UPDATE + SET hash = :hash, + time = :block_time, + sapling_commitment_tree_size = :sapling_commitment_tree_size, + sapling_output_count = :sapling_output_count, + orchard_commitment_tree_size = :orchard_commitment_tree_size, + orchard_action_count = :orchard_action_count", + )?; + + #[cfg(not(feature = "orchard"))] + let orchard_commitment_tree_size: Option = None; + #[cfg(not(feature = "orchard"))] + let orchard_action_count: Option = None; + + stmt_upsert_block.execute(named_params![ + ":height": u32::from(block_height), + ":hash": &block_hash.0[..], + ":block_time": block_time, + ":sapling_commitment_tree_size": sapling_commitment_tree_size, + ":sapling_output_count": sapling_output_count, + ":orchard_commitment_tree_size": orchard_commitment_tree_size, + ":orchard_action_count": orchard_action_count, + ])?; + + Ok(()) + } + + /// This is a copy of [`crate::wallet::put_tx_meta`] as of the expected database + /// state corresponding to this migration. It is duplicated here as later + /// updates to the database schema require incompatible changes to `put_tx_meta`. + pub(crate) fn put_tx_meta( + conn: &rusqlite::Connection, + tx: &WalletTx, + height: BlockHeight, + ) -> Result { + // It isn't there, so insert our transaction into the database. + let mut stmt_upsert_tx_meta = conn.prepare_cached( + "INSERT INTO transactions (txid, block, tx_index) + VALUES (:txid, :block, :tx_index) + ON CONFLICT (txid) DO UPDATE + SET block = :block, + tx_index = :tx_index + RETURNING id_tx", + )?; + + let txid_bytes = tx.txid(); + let tx_params = named_params![ + ":txid": &txid_bytes.as_ref()[..], + ":block": u32::from(height), + ":tx_index": i64::try_from(tx.block_index()).expect("transaction indices are representable as i64"), + ]; + + stmt_upsert_tx_meta + .query_row(tx_params, |row| row.get::<_, i64>(0)) + .map_err(SqliteClientError::from) + } } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/utxos_to_txos.rs b/zcash_client_sqlite/src/wallet/init/migrations/utxos_to_txos.rs new file mode 100644 index 0000000000..1d2680b657 --- /dev/null +++ b/zcash_client_sqlite/src/wallet/init/migrations/utxos_to_txos.rs @@ -0,0 +1,328 @@ +//! A migration that brings transparent UTXO handling into line with that for shielded outputs. +use std::collections::HashSet; + +use rusqlite; +use schemer; +use schemer_rusqlite::RusqliteMigration; +use uuid::Uuid; + +use crate::wallet::init::{migrations::orchard_received_notes, WalletMigrationError}; + +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0x3a2562b3_f174_46a1_aa8c_1d122ca2e884); + +pub(super) struct Migration; + +impl schemer::Migration for Migration { + fn id(&self) -> Uuid { + MIGRATION_ID + } + + fn dependencies(&self) -> HashSet { + [orchard_received_notes::MIGRATION_ID].into_iter().collect() + } + + fn description(&self) -> &'static str { + "Updates transparent UTXO handling to be similar to that for shielded notes." + } +} + +impl RusqliteMigration for Migration { + type Error = WalletMigrationError; + + fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { + transaction.execute_batch(r#" + PRAGMA legacy_alter_table = ON; + + CREATE TABLE transactions_new ( + id_tx INTEGER PRIMARY KEY, + txid BLOB NOT NULL UNIQUE, + created TEXT, + block INTEGER, + mined_height INTEGER, + tx_index INTEGER, + expiry_height INTEGER, + raw BLOB, + fee INTEGER, + FOREIGN KEY (block) REFERENCES blocks(height), + CONSTRAINT height_consistency CHECK (block IS NULL OR mined_height = block) + ); + + INSERT INTO transactions_new + SELECT id_tx, txid, created, block, block, tx_index, expiry_height, raw, fee + FROM transactions; + + -- We may initially set the block height to null, which will mean that the + -- transaction may appear to be un-mined until we actually scan the block + -- containing the transaction. + INSERT INTO transactions_new (txid, block, mined_height) + SELECT + utxos.prevout_txid, + blocks.height, + blocks.height + FROM utxos + LEFT OUTER JOIN blocks ON blocks.height = utxos.height + WHERE utxos.prevout_txid NOT IN ( + SELECT txid FROM transactions + ); + + DROP TABLE transactions; + ALTER TABLE transactions_new RENAME TO transactions; + + CREATE TABLE transparent_received_outputs ( + id INTEGER PRIMARY KEY, + transaction_id INTEGER NOT NULL, + output_index INTEGER NOT NULL, + account_id INTEGER NOT NULL, + address TEXT NOT NULL, + script BLOB NOT NULL, + value_zat INTEGER NOT NULL, + max_observed_unspent_height INTEGER, + FOREIGN KEY (transaction_id) REFERENCES transactions(id_tx), + FOREIGN KEY (account_id) REFERENCES accounts(id), + CONSTRAINT transparent_output_unique UNIQUE (transaction_id, output_index) + ); + CREATE INDEX idx_transparent_received_outputs_account_id + ON "transparent_received_outputs" (account_id); + + INSERT INTO transparent_received_outputs SELECT + u.id, + t.id_tx, + prevout_idx, + received_by_account_id, + address, + script, + value_zat, + NULL + FROM utxos u + -- This being a `LEFT OUTER JOIN` provides defense in depth against dropping + -- TXOs that reference missing `transactions` entries (which should never exist + -- given the migrations above). + LEFT OUTER JOIN transactions t ON t.txid = u.prevout_txid; + + CREATE TABLE transparent_received_output_spends_new ( + transparent_received_output_id INTEGER NOT NULL, + transaction_id INTEGER NOT NULL, + FOREIGN KEY (transparent_received_output_id) + REFERENCES transparent_received_outputs(id) + ON DELETE CASCADE, + FOREIGN KEY (transaction_id) + -- We do not delete transactions, so this does not cascade + REFERENCES transactions(id_tx), + UNIQUE (transparent_received_output_id, transaction_id) + ); + + INSERT INTO transparent_received_output_spends_new + SELECT * FROM transparent_received_output_spends; + + DROP VIEW v_tx_outputs; + DROP VIEW v_transactions; + DROP VIEW v_received_notes; + DROP VIEW v_received_note_spends; + DROP TABLE transparent_received_output_spends; + ALTER TABLE transparent_received_output_spends_new + RENAME TO transparent_received_output_spends; + + CREATE VIEW v_received_outputs AS + SELECT + sapling_received_notes.id AS id_within_pool_table, + sapling_received_notes.tx AS transaction_id, + 2 AS pool, + sapling_received_notes.output_index, + account_id, + sapling_received_notes.value, + is_change, + sapling_received_notes.memo, + sent_notes.id AS sent_note_id + FROM sapling_received_notes + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (sapling_received_notes.tx, 2, sapling_received_notes.output_index) + UNION + SELECT + orchard_received_notes.id AS id_within_pool_table, + orchard_received_notes.tx AS transaction_id, + 3 AS pool, + orchard_received_notes.action_index AS output_index, + account_id, + orchard_received_notes.value, + is_change, + orchard_received_notes.memo, + sent_notes.id AS sent_note_id + FROM orchard_received_notes + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (orchard_received_notes.tx, 3, orchard_received_notes.action_index) + UNION + SELECT + u.id AS id_within_pool_table, + u.transaction_id, + 0 AS pool, + u.output_index, + u.account_id, + u.value_zat AS value, + 0 AS is_change, + NULL AS memo, + sent_notes.id AS sent_note_id + FROM transparent_received_outputs u + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (u.transaction_id, 0, u.output_index); + + CREATE VIEW v_received_output_spends AS + SELECT + 2 AS pool, + sapling_received_note_id AS received_output_id, + transaction_id + FROM sapling_received_note_spends + UNION + SELECT + 3 AS pool, + orchard_received_note_id AS received_output_id, + transaction_id + FROM orchard_received_note_spends + UNION + SELECT + 0 AS pool, + transparent_received_output_id AS received_output_id, + transaction_id + FROM transparent_received_output_spends; + + CREATE VIEW v_transactions AS + WITH + notes AS ( + -- Outputs received in this transaction + SELECT ro.account_id AS account_id, + transactions.mined_height AS mined_height, + transactions.txid AS txid, + ro.pool AS pool, + id_within_pool_table, + ro.value AS value, + CASE + WHEN ro.is_change THEN 1 + ELSE 0 + END AS is_change, + CASE + WHEN ro.is_change THEN 0 + ELSE 1 + END AS received_count, + CASE + WHEN (ro.memo IS NULL OR ro.memo = X'F6') + THEN 0 + ELSE 1 + END AS memo_present + FROM v_received_outputs ro + JOIN transactions + ON transactions.id_tx = ro.transaction_id + UNION + -- Outputs spent in this transaction + SELECT ro.account_id AS account_id, + transactions.mined_height AS mined_height, + transactions.txid AS txid, + ro.pool AS pool, + id_within_pool_table, + -ro.value AS value, + 0 AS is_change, + 0 AS received_count, + 0 AS memo_present + FROM v_received_outputs ro + JOIN v_received_output_spends ros + ON ros.pool = ro.pool + AND ros.received_output_id = ro.id_within_pool_table + JOIN transactions + ON transactions.id_tx = ro.transaction_id + ), + -- Obtain a count of the notes that the wallet created in each transaction, + -- not counting change notes. + sent_note_counts AS ( + SELECT sent_notes.from_account_id AS account_id, + transactions.txid AS txid, + COUNT(DISTINCT sent_notes.id) AS sent_notes, + SUM( + CASE + WHEN (sent_notes.memo IS NULL OR sent_notes.memo = X'F6' OR ro.transaction_id IS NOT NULL) + THEN 0 + ELSE 1 + END + ) AS memo_count + FROM sent_notes + JOIN transactions + ON transactions.id_tx = sent_notes.tx + LEFT JOIN v_received_outputs ro + ON sent_notes.id = ro.sent_note_id + WHERE COALESCE(ro.is_change, 0) = 0 + GROUP BY account_id, txid + ), + blocks_max_height AS ( + SELECT MAX(blocks.height) AS max_height FROM blocks + ) + SELECT notes.account_id AS account_id, + notes.mined_height AS mined_height, + notes.txid AS txid, + transactions.tx_index AS tx_index, + transactions.expiry_height AS expiry_height, + transactions.raw AS raw, + SUM(notes.value) AS account_balance_delta, + transactions.fee AS fee_paid, + SUM(notes.is_change) > 0 AS has_change, + MAX(COALESCE(sent_note_counts.sent_notes, 0)) AS sent_note_count, + SUM(notes.received_count) AS received_note_count, + SUM(notes.memo_present) + MAX(COALESCE(sent_note_counts.memo_count, 0)) AS memo_count, + blocks.time AS block_time, + ( + blocks.height IS NULL + AND transactions.expiry_height BETWEEN 1 AND blocks_max_height.max_height + ) AS expired_unmined + FROM notes + LEFT JOIN transactions + ON notes.txid = transactions.txid + JOIN blocks_max_height + LEFT JOIN blocks ON blocks.height = notes.mined_height + LEFT JOIN sent_note_counts + ON sent_note_counts.account_id = notes.account_id + AND sent_note_counts.txid = notes.txid + GROUP BY notes.account_id, notes.txid; + + CREATE VIEW v_tx_outputs AS + SELECT transactions.txid AS txid, + ro.pool AS output_pool, + ro.output_index AS output_index, + sent_notes.from_account_id AS from_account_id, + ro.account_id AS to_account_id, + NULL AS to_address, + ro.value AS value, + ro.is_change AS is_change, + ro.memo AS memo + FROM v_received_outputs ro + JOIN transactions + ON transactions.id_tx = ro.transaction_id + LEFT JOIN sent_notes + ON sent_notes.id = ro.sent_note_id + UNION + SELECT transactions.txid AS txid, + sent_notes.output_pool AS output_pool, + sent_notes.output_index AS output_index, + sent_notes.from_account_id AS from_account_id, + ro.account_id AS to_account_id, + sent_notes.to_address AS to_address, + sent_notes.value AS value, + 0 AS is_change, + sent_notes.memo AS memo + FROM sent_notes + JOIN transactions + ON transactions.id_tx = sent_notes.tx + LEFT JOIN v_received_outputs ro + ON sent_notes.id = ro.sent_note_id + WHERE COALESCE(ro.is_change, 0) = 0; + + DROP TABLE utxos; + + PRAGMA legacy_alter_table = OFF; + "#)?; + + Ok(()) + } + + fn down(&self, _: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { + Err(WalletMigrationError::CannotRevert(MIGRATION_ID)) + } +} diff --git a/zcash_client_sqlite/src/wallet/transparent.rs b/zcash_client_sqlite/src/wallet/transparent.rs index ee160d565a..78bfd07b12 100644 --- a/zcash_client_sqlite/src/wallet/transparent.rs +++ b/zcash_client_sqlite/src/wallet/transparent.rs @@ -1,7 +1,6 @@ //! Functions for transparent input support in the wallet. use rusqlite::OptionalExtension; use rusqlite::{named_params, Connection, Row}; -use std::collections::BTreeSet; use std::collections::HashMap; use std::collections::HashSet; use zcash_client_backend::data_api::AccountBalance; @@ -20,20 +19,20 @@ use zcash_primitives::{ }; use zcash_protocol::consensus::{self, BlockHeight}; -use crate::{error::SqliteClientError, AccountId, UtxoId, PRUNING_DEPTH}; +use crate::{error::SqliteClientError, AccountId, UtxoId}; -use super::get_account_ids; -use super::scan_queue_extrema; +use super::{chain_tip_height, get_account_ids}; pub(crate) fn detect_spending_accounts<'a>( conn: &Connection, spent: impl Iterator, ) -> Result, rusqlite::Error> { let mut account_q = conn.prepare_cached( - "SELECT received_by_account_id - FROM utxos - WHERE prevout_txid = :prevout_txid - AND prevout_idx = :prevout_idx", + "SELECT account_id + FROM transparent_received_outputs o + JOIN transactions t ON t.id_tx = o.transaction_id + WHERE t.txid = :prevout_txid + AND o.output_index = :prevout_idx", )?; let mut acc = HashSet::new(); @@ -161,17 +160,17 @@ pub(crate) fn get_legacy_transparent_address( } fn to_unspent_transparent_output(row: &Row) -> Result { - let txid: Vec = row.get("prevout_txid")?; + let txid: Vec = row.get("txid")?; let mut txid_bytes = [0u8; 32]; txid_bytes.copy_from_slice(&txid); - let index: u32 = row.get("prevout_idx")?; + let index: u32 = row.get("output_index")?; let script_pubkey = Script(row.get("script")?); let raw_value: i64 = row.get("value_zat")?; let value = NonNegativeAmount::from_nonnegative_i64(raw_value).map_err(|_| { SqliteClientError::CorruptedData(format!("Invalid UTXO value: {}", raw_value)) })?; - let height: u32 = row.get("height")?; + let height: u32 = row.get("received_height")?; let outpoint = OutPoint::new(txid_bytes, index); WalletTransparentOutput::from_parts( @@ -189,21 +188,39 @@ fn to_unspent_transparent_output(row: &Row) -> Result Result, SqliteClientError> { + let chain_tip_height = chain_tip_height(conn)?; + + // This could, in very rare circumstances, return as unspent outputs that are actually not + // spendable, if they are the outputs of deshielding transactions where the spend anchors have + // been invalidated by a rewind. There isn't a way to detect this circumstance at present, but + // it should be vanishingly rare as the vast majority of rewinds are of a single block. let mut stmt_select_utxo = conn.prepare_cached( - "SELECT u.prevout_txid, u.prevout_idx, u.script, u.value_zat, u.height - FROM utxos u - WHERE u.prevout_txid = :txid - AND u.prevout_idx = :output_index + "SELECT t.txid, u.output_index, u.script, + u.value_zat, t.mined_height AS received_height + FROM transparent_received_outputs u + JOIN transactions t ON t.id_tx = u.transaction_id + WHERE t.txid = :txid + AND u.output_index = :output_index + -- the transaction that created the output is mined or is definitely unexpired + AND ( + t.mined_height IS NOT NULL -- tx is mined + -- TODO: uncomment the following two lines in order to enable zero-conf spends + -- OR t.expiry_height = 0 -- tx will not expire + -- OR t.expiry_height >= :mempool_height -- tx has not yet expired + ) + -- and the output is unspent AND u.id NOT IN ( SELECT txo_spends.transparent_received_output_id FROM transparent_received_output_spends txo_spends JOIN transactions tx ON tx.id_tx = txo_spends.transaction_id - WHERE tx.block IS NOT NULL -- the spending tx is mined - OR tx.expiry_height IS NULL -- the spending tx will not expire + WHERE tx.mined_height IS NOT NULL -- the spending tx is mined + OR tx.expiry_height = 0 -- the spending tx will not expire + OR tx.expiry_height >= :mempool_height -- the spending tx has not yet expired )", )?; @@ -211,7 +228,8 @@ pub(crate) fn get_unspent_transparent_output( .query_and_then( named_params![ ":txid": outpoint.hash(), - ":output_index": outpoint.n() + ":output_index": outpoint.n(), + ":mempool_height": chain_tip_height.map(|h| u32::from(h) + 1), ], to_unspent_transparent_output, )? @@ -221,53 +239,62 @@ pub(crate) fn get_unspent_transparent_output( result } -/// Returns unspent transparent outputs that have been received by this wallet at the given -/// transparent address, such that the block that included the transaction was mined at a -/// height less than or equal to the provided `max_height`. -pub(crate) fn get_unspent_transparent_outputs( +pub(crate) fn get_spendable_transparent_outputs( conn: &rusqlite::Connection, params: &P, address: &TransparentAddress, - max_height: BlockHeight, - exclude: &[OutPoint], + target_height: BlockHeight, + min_confirmations: u32, ) -> Result, SqliteClientError> { - let chain_tip_height = scan_queue_extrema(conn)?.map(|range| *range.end()); - let stable_height = chain_tip_height - .unwrap_or(max_height) - .saturating_sub(PRUNING_DEPTH); + let confirmed_height = target_height - min_confirmations; + // This could, in very rare circumstances, return as unspent outputs that are actually not + // spendable, if they are the outputs of deshielding transactions where the spend anchors have + // been invalidated by a rewind. There isn't a way to detect this circumstance at present, but + // it should be vanishingly rare as the vast majority of rewinds are of a single block. let mut stmt_utxos = conn.prepare( - "SELECT u.prevout_txid, u.prevout_idx, u.script, - u.value_zat, u.height - FROM utxos u + "SELECT t.txid, u.output_index, u.script, + u.value_zat, t.mined_height AS received_height + FROM transparent_received_outputs u + JOIN transactions t ON t.id_tx = u.transaction_id WHERE u.address = :address - AND u.height <= :max_height + -- the transaction that created the output is mined or unexpired as of `confirmed_height` + AND ( + t.mined_height <= :confirmed_height -- tx is mined + -- TODO: uncomment the following lines in order to enable zero-conf spends + -- OR ( + -- :min_confirmations = 0 + -- AND ( + -- t.expiry_height = 0 -- tx will not expire + -- OR t.expiry_height >= :target_height + -- ) + -- ) + ) + -- and the output is unspent AND u.id NOT IN ( SELECT txo_spends.transparent_received_output_id FROM transparent_received_output_spends txo_spends JOIN transactions tx ON tx.id_tx = txo_spends.transaction_id - WHERE - tx.block IS NOT NULL -- the spending tx is mined - OR tx.expiry_height IS NULL -- the spending tx will not expire - OR tx.expiry_height > :stable_height -- the spending tx is unexpired + WHERE tx.mined_height IS NOT NULL -- the spending transaction is mined + OR tx.expiry_height = 0 -- the spending tx will not expire + OR tx.expiry_height >= :target_height -- the spending tx has not yet expired + -- we are intentionally conservative and exclude outputs that are potentially spent + -- as of the target height, even if they might actually be spendable due to expiry + -- of the spending transaction as of the chain tip )", )?; let addr_str = address.encode(params); - - let mut utxos = Vec::::new(); let mut rows = stmt_utxos.query(named_params![ ":address": addr_str, - ":max_height": u32::from(max_height), - ":stable_height": u32::from(stable_height), + ":confirmed_height": u32::from(confirmed_height), + ":target_height": u32::from(target_height), + //":min_confirmations": min_confirmations ])?; - let excluded: BTreeSet = exclude.iter().cloned().collect(); + + let mut utxos = Vec::::new(); while let Some(row) = rows.next()? { let output = to_unspent_transparent_output(row)?; - if excluded.contains(output.outpoint()) { - continue; - } - utxos.push(output); } @@ -276,31 +303,40 @@ pub(crate) fn get_unspent_transparent_outputs( /// Returns the unspent balance for each transparent address associated with the specified account, /// such that the block that included the transaction was mined at a height less than or equal to -/// the provided `max_height`. +/// the provided `summary_height`. pub(crate) fn get_transparent_address_balances( conn: &rusqlite::Connection, params: &P, account: AccountId, - max_height: BlockHeight, + summary_height: BlockHeight, ) -> Result, SqliteClientError> { - let chain_tip_height = scan_queue_extrema(conn)?.map(|range| *range.end()); - let stable_height = chain_tip_height - .unwrap_or(max_height) - .saturating_sub(PRUNING_DEPTH); + let chain_tip_height = chain_tip_height(conn)?.ok_or(SqliteClientError::ChainHeightUnknown)?; let mut stmt_address_balances = conn.prepare( "SELECT u.address, SUM(u.value_zat) - FROM utxos u - WHERE u.received_by_account_id = :account_id - AND u.height <= :max_height + FROM transparent_received_outputs u + JOIN transactions t + ON t.id_tx = u.transaction_id + WHERE u.account_id = :account_id + -- the transaction that created the output is mined or is definitely unexpired + AND ( + t.mined_height <= :summary_height -- tx is mined + OR ( -- or the caller has requested to include zero-conf funds that are not expired + :summary_height > :chain_tip_height + AND ( + t.expiry_height = 0 -- tx will not expire + OR t.expiry_height >= :summary_height + ) + ) + ) + -- and the output is unspent AND u.id NOT IN ( SELECT txo_spends.transparent_received_output_id FROM transparent_received_output_spends txo_spends JOIN transactions tx ON tx.id_tx = txo_spends.transaction_id - WHERE - tx.block IS NOT NULL -- the spending tx is mined - OR tx.expiry_height IS NULL -- the spending tx will not expire - OR tx.expiry_height > :stable_height -- the spending tx is unexpired + WHERE tx.mined_height IS NOT NULL -- the spending tx is mined + OR tx.expiry_height = 0 -- the spending tx will not expire + OR tx.expiry_height >= :spend_expiry_height -- the spending tx is unexpired ) GROUP BY u.address", )?; @@ -308,8 +344,9 @@ pub(crate) fn get_transparent_address_balances( let mut res = HashMap::new(); let mut rows = stmt_address_balances.query(named_params![ ":account_id": account.0, - ":max_height": u32::from(max_height), - ":stable_height": u32::from(stable_height), + ":summary_height": u32::from(summary_height), + ":chain_tip_height": u32::from(chain_tip_height), + ":spend_expiry_height": u32::from(std::cmp::min(summary_height, chain_tip_height + 1)), ])?; while let Some(row) = rows.next()? { let taddr_str: String = row.get(0)?; @@ -322,36 +359,38 @@ pub(crate) fn get_transparent_address_balances( Ok(res) } +#[tracing::instrument(skip(conn, account_balances))] pub(crate) fn add_transparent_account_balances( conn: &rusqlite::Connection, - chain_tip_height: BlockHeight, - min_confirmations: u32, + mempool_height: BlockHeight, account_balances: &mut HashMap, ) -> Result<(), SqliteClientError> { let transparent_trace = tracing::info_span!("stmt_transparent_balances").entered(); - let zero_conf_height = (chain_tip_height + 1).saturating_sub(min_confirmations); - let stable_height = chain_tip_height.saturating_sub(PRUNING_DEPTH); - - let mut stmt_transparent_balances = conn.prepare( - "SELECT u.received_by_account_id, SUM(u.value_zat) - FROM utxos u - WHERE u.height <= :max_height + let mut stmt_account_balances = conn.prepare( + "SELECT u.account_id, SUM(u.value_zat) + FROM transparent_received_outputs u + JOIN transactions t + ON t.id_tx = u.transaction_id + -- the transaction that created the output is mined or is definitely unexpired + WHERE ( + t.mined_height < :mempool_height -- tx is mined + OR t.expiry_height = 0 -- tx will not expire + OR t.expiry_height >= :mempool_height + ) -- and the received txo is unspent AND u.id NOT IN ( SELECT transparent_received_output_id FROM transparent_received_output_spends txo_spends JOIN transactions tx ON tx.id_tx = txo_spends.transaction_id - WHERE tx.block IS NOT NULL -- the spending tx is mined - OR tx.expiry_height IS NULL -- the spending tx will not expire - OR tx.expiry_height > :stable_height -- the spending tx is unexpired + WHERE tx.mined_height IS NOT NULL -- the spending tx is mined + OR tx.expiry_height = 0 -- the spending tx will not expire + OR tx.expiry_height >= :mempool_height -- the spending tx is unexpired ) - GROUP BY u.received_by_account_id", + GROUP BY u.account_id", )?; - let mut rows = stmt_transparent_balances.query(named_params![ - ":max_height": u32::from(zero_conf_height), - ":stable_height": u32::from(stable_height) - ])?; + let mut rows = stmt_account_balances + .query(named_params![":mempool_height": u32::from(mempool_height),])?; while let Some(row) = rows.next()? { let account = AccountId(row.get(0)?); @@ -360,9 +399,10 @@ pub(crate) fn add_transparent_account_balances( SqliteClientError::CorruptedData(format!("Negative UTXO value {:?}", raw_value)) })?; - if let Some(balances) = account_balances.get_mut(&account) { - balances.add_unshielded_value(value)?; - } + account_balances + .entry(account) + .or_insert(AccountBalance::ZERO) + .add_unshielded_value(value)?; } drop(transparent_trace); Ok(()) @@ -377,9 +417,10 @@ pub(crate) fn mark_transparent_utxo_spent( let mut stmt_mark_transparent_utxo_spent = conn.prepare_cached( "INSERT INTO transparent_received_output_spends (transparent_received_output_id, transaction_id) SELECT txo.id, :spent_in_tx - FROM utxos txo - WHERE txo.prevout_txid = :prevout_txid - AND txo.prevout_idx = :prevout_idx + FROM transparent_received_outputs txo + JOIN transactions t ON t.id_tx = txo.transaction_id + WHERE t.txid = :prevout_txid + AND txo.output_index = :prevout_idx ON CONFLICT (transparent_received_output_id, transaction_id) DO NOTHING", )?; @@ -409,7 +450,7 @@ pub(crate) fn put_received_transparent_utxo( .optional()?; if let Some(account) = account_id { - Ok(put_legacy_transparent_utxo(conn, params, output, account)?) + Ok(put_transparent_output(conn, params, output, account)?) } else { // If the UTXO is received at the legacy transparent address (at BIP 44 address // index 0 within its particular account, which we specifically ensure is returned @@ -423,7 +464,7 @@ pub(crate) fn put_received_transparent_utxo( |account| match get_legacy_transparent_address(params, conn, account) { Ok(Some((legacy_taddr, _))) if &legacy_taddr == output.recipient_address() => { Some( - put_legacy_transparent_utxo(conn, params, output, account) + put_transparent_output(conn, params, output, account) .map_err(SqliteClientError::from), ) } @@ -440,50 +481,74 @@ pub(crate) fn put_received_transparent_utxo( } } -pub(crate) fn put_legacy_transparent_utxo( +pub(crate) fn put_transparent_output( conn: &rusqlite::Connection, params: &P, output: &WalletTransparentOutput, received_by_account: AccountId, ) -> Result { - #[cfg(feature = "transparent-inputs")] - let mut stmt_upsert_legacy_transparent_utxo = conn.prepare_cached( - "INSERT INTO utxos ( - prevout_txid, prevout_idx, - received_by_account_id, address, script, - value_zat, height) - VALUES - (:prevout_txid, :prevout_idx, - :received_by_account_id, :address, :script, - :value_zat, :height) - ON CONFLICT (prevout_txid, prevout_idx) DO UPDATE - SET received_by_account_id = :received_by_account_id, - height = :height, + // Check whether we have an entry in the blocks table for the output height; + // if not, the transaction will be updated with its mined height when the + // associated block is scanned. + let block = conn + .query_row( + "SELECT height FROM blocks WHERE height = :height", + named_params![":height": &u32::from(output.height())], + |row| row.get::<_, u32>(0), + ) + .optional()?; + + let id_tx = conn.query_row( + "INSERT INTO transactions (txid, block, mined_height) + VALUES (:txid, :block, :mined_height) + ON CONFLICT (txid) DO UPDATE + SET block = IFNULL(block, :block), + mined_height = :mined_height + RETURNING id_tx", + named_params![ + ":txid": &output.outpoint().hash().to_vec(), + ":block": block, + ":mined_height": u32::from(output.height()) + ], + |row| row.get::<_, i64>(0), + )?; + + let mut stmt_upsert_transparent_output = conn.prepare_cached( + "INSERT INTO transparent_received_outputs ( + transaction_id, output_index, + account_id, address, script, + value_zat, max_observed_unspent_height + ) + VALUES ( + :transaction_id, :output_index, + :account_id, :address, :script, + :value_zat, :height + ) + ON CONFLICT (transaction_id, output_index) DO UPDATE + SET account_id = :account_id, address = :address, script = :script, - value_zat = :value_zat + value_zat = :value_zat, + max_observed_unspent_height = :height RETURNING id", )?; let sql_args = named_params![ - ":prevout_txid": &output.outpoint().hash().to_vec(), - ":prevout_idx": &output.outpoint().n(), - ":received_by_account_id": received_by_account.0, + ":transaction_id": id_tx, + ":output_index": &output.outpoint().n(), + ":account_id": received_by_account.0, ":address": &output.recipient_address().encode(params), ":script": &output.txout().script_pubkey.0, ":value_zat": &i64::from(Amount::from(output.txout().value)), ":height": &u32::from(output.height()), ]; - stmt_upsert_legacy_transparent_utxo.query_row(sql_args, |row| row.get::<_, i64>(0).map(UtxoId)) + stmt_upsert_transparent_output.query_row(sql_args, |row| row.get::<_, i64>(0).map(UtxoId)) } #[cfg(test)] mod tests { - use crate::{ - testing::{AddressType, TestBuilder, TestState}, - PRUNING_DEPTH, - }; + use crate::testing::{AddressType, TestBuilder, TestState}; use sapling::zip32::ExtendedSpendingKey; use zcash_client_backend::{ data_api::{ @@ -495,7 +560,6 @@ mod tests { }; use zcash_primitives::{ block::BlockHash, - consensus::BlockHeight, transaction::{ components::{amount::NonNegativeAmount, OutPoint, TxOut}, fees::fixed::FeeRule as FixedFeeRule, @@ -510,6 +574,7 @@ mod tests { .with_account_from_sapling_activation(BlockHash([0; 32])) .build(); + let birthday = st.test_account().unwrap().birthday().height(); let account_id = st.test_account().unwrap().account_id(); let uaddr = st .wallet() @@ -518,7 +583,9 @@ mod tests { .unwrap(); let taddr = uaddr.transparent().unwrap(); - let height_1 = BlockHeight::from_u32(12345); + let height_1 = birthday + 12345; + st.wallet_mut().update_chain_tip(height_1).unwrap(); + let bal_absent = st .wallet() .get_transparent_balances(account_id, height_1) @@ -541,10 +608,10 @@ mod tests { // Confirm that we see the output unspent as of `height_1`. assert_matches!( - st.wallet().get_unspent_transparent_outputs( + st.wallet().get_spendable_transparent_outputs( taddr, height_1, - &[] + 0 ).as_deref(), Ok([ret]) if (ret.outpoint(), ret.txout(), ret.height()) == (utxo.outpoint(), utxo.txout(), height_1) ); @@ -555,7 +622,8 @@ mod tests { // Change the mined height of the UTXO and upsert; we should get back // the same `UtxoId`. - let height_2 = BlockHeight::from_u32(34567); + let height_2 = birthday + 34567; + st.wallet_mut().update_chain_tip(height_2).unwrap(); let utxo2 = WalletTransparentOutput::from_parts(outpoint, txout, height_2).unwrap(); let res1 = st.wallet_mut().put_received_transparent_utxo(&utxo2); assert_matches!(res1, Ok(id) if id == res0.unwrap()); @@ -563,7 +631,7 @@ mod tests { // Confirm that we no longer see any unspent outputs as of `height_1`. assert_matches!( st.wallet() - .get_unspent_transparent_outputs(taddr, height_1, &[]) + .get_spendable_transparent_outputs(taddr, height_1, 0) .as_deref(), Ok(&[]) ); @@ -577,7 +645,7 @@ mod tests { // If we include `height_2` then the output is returned. assert_matches!( st.wallet() - .get_unspent_transparent_outputs(taddr, height_2, &[]) + .get_spendable_transparent_outputs(taddr, height_2, 0) .as_deref(), Ok([ret]) if (ret.outpoint(), ret.txout(), ret.height()) == (utxo.outpoint(), utxo.txout(), height_2) ); @@ -639,13 +707,15 @@ mod tests { .account_balances() .get(&account.account_id()) .unwrap(); + // TODO: in the future, we will distinguish between available and total + // balance according to `min_confirmations` assert_eq!(balance.unshielded(), expected); // Check the older APIs for consistency. - let max_height = st.wallet().chain_height().unwrap().unwrap() + 1 - min_confirmations; + let mempool_height = st.wallet().chain_height().unwrap().unwrap() + 1; assert_eq!( st.wallet() - .get_transparent_balances(account.account_id(), max_height) + .get_transparent_balances(account.account_id(), mempool_height) .unwrap() .get(taddr) .cloned() @@ -654,7 +724,7 @@ mod tests { ); assert_eq!( st.wallet() - .get_unspent_transparent_outputs(taddr, max_height, &[]) + .get_spendable_transparent_outputs(taddr, mempool_height, 0) .unwrap() .into_iter() .map(|utxo| utxo.value()) @@ -664,8 +734,11 @@ mod tests { }; // The wallet starts out with zero balance. + // TODO: Once we have refactored `get_wallet_summary` to distinguish between available + // and total balance, we should perform additional checks against available balance; + // we use minconf 0 here because all transparent funds are considered shieldable, + // irrespective of confirmation depth. check_balance(&st, 0, NonNegativeAmount::ZERO); - check_balance(&st, 1, NonNegativeAmount::ZERO); // Create a fake transparent output. let value = NonNegativeAmount::from_u64(100000).unwrap(); @@ -682,10 +755,8 @@ mod tests { .put_received_transparent_utxo(&utxo) .unwrap(); - // The wallet should detect the balance as having 1 confirmation. + // The wallet should detect the balance as available check_balance(&st, 0, value); - check_balance(&st, 1, value); - check_balance(&st, 2, NonNegativeAmount::ZERO); // Shield the output. let input_selector = GreedyInputSelector::new( @@ -703,8 +774,6 @@ mod tests { // The wallet should have zero transparent balance, because the shielding // transaction can be mined. check_balance(&st, 0, NonNegativeAmount::ZERO); - check_balance(&st, 1, NonNegativeAmount::ZERO); - check_balance(&st, 2, NonNegativeAmount::ZERO); // Mine the shielding transaction. let (mined_height, _) = st.generate_next_block_including(txid); @@ -712,8 +781,6 @@ mod tests { // The wallet should still have zero transparent balance. check_balance(&st, 0, NonNegativeAmount::ZERO); - check_balance(&st, 1, NonNegativeAmount::ZERO); - check_balance(&st, 2, NonNegativeAmount::ZERO); // Unmine the shielding transaction via a reorg. st.wallet_mut() @@ -723,8 +790,6 @@ mod tests { // The wallet should still have zero transparent balance. check_balance(&st, 0, NonNegativeAmount::ZERO); - check_balance(&st, 1, NonNegativeAmount::ZERO); - check_balance(&st, 2, NonNegativeAmount::ZERO); // Expire the shielding transaction. let expiry_height = st @@ -735,22 +800,6 @@ mod tests { .expiry_height(); st.wallet_mut().update_chain_tip(expiry_height).unwrap(); - // TODO: Making the transparent output spendable in this situation requires - // changes to the transparent data model, so for now the wallet should still have - // zero transparent balance. https://github.com/zcash/librustzcash/issues/986 - check_balance(&st, 0, NonNegativeAmount::ZERO); - check_balance(&st, 1, NonNegativeAmount::ZERO); - check_balance(&st, 2, NonNegativeAmount::ZERO); - - // Roll forward the chain tip until the transaction's expiry height is in the - // stable block range (so a reorg won't make it spendable again). - st.wallet_mut() - .update_chain_tip(expiry_height + PRUNING_DEPTH) - .unwrap(); - - // The transparent output should be spendable again, with more confirmations. check_balance(&st, 0, value); - check_balance(&st, 1, value); - check_balance(&st, 2, value); } }