zcash_client_backend: Add transaction data requests.

This adds a mechanism that can be used by the backend to request
additional transaction data or status information from the client,
for the purpose of being to explore transparent transaction history
that is otherwise currently unavailable.

zcash_client_backend/CHANGELOG.md

@@ -28,6 +28,8 @@ funds to those addresses. See [ZIP 320](https://zips.z.cash/zip-0320) for detail
   - `chain::BlockCache` trait, behind the `sync` feature flag.
   - `WalletRead::get_spendable_transparent_outputs`
   - `DecryptedTransaction::mined_height`
+  - `TransactionDataRequest`
+  - `TransactionStatus`
 - `zcash_client_backend::fees`:
   - `EphemeralBalance`
   - `ChangeValue::shielded, is_ephemeral`
@@ -61,12 +63,14 @@ funds to those addresses. See [ZIP 320](https://zips.z.cash/zip-0320) for detail
   have changed as a consequence of this extraction; please see the `zip321`
   CHANGELOG for details.
 - `zcash_client_backend::data_api`:
+  - `WalletRead` has a new `transaction_data_requests` method.
   - `WalletRead` has new `get_known_ephemeral_addresses`,
     `find_account_for_ephemeral_address`, and `get_transparent_address_metadata`
     methods when the "transparent-inputs" feature is enabled.
   - `WalletWrite` has a new `reserve_next_n_ephemeral_addresses` method when
     the "transparent-inputs" feature is enabled.
-  - `WalletWrite` has new methods `import_account_hd` and `import_account_ufvk`.
+  - `WalletWrite` has new methods `import_account_hd`, `import_account_ufvk`,
+    and `set_transaction_status`.
   - `error::Error` has new `Address` and (when the "transparent-inputs" feature
     is enabled) `PaysEphemeralTransparentAddress` variants.
   - The `WalletWrite::store_sent_tx` method has been renamed to

zcash_client_backend/src/data_api.rs

@@ -541,6 +541,51 @@ pub struct SpendableNotes<NoteRef> {
     orchard: Vec<ReceivedNote<NoteRef, orchard::note::Note>>,
 }
 
+/// A request for transaction data enhancement, spentness check, or discovery
+/// of spends from a given transparent address within a specific block range.
+#[derive(Clone, Debug)]
+pub enum TransactionDataRequest {
+    /// Transaction enhancement (download of complete raw transaction data) is requested.
+    ///
+    /// The caller should respond to this request by calling [`wallet::decrypt_and_store_transaction`]
+    /// to provide data for the requested transaction to the wallet backend. If no data is
+    /// available for the specified transaction,
+    Enhancement(TxId),
+    /// Information about the chain's view of a transaction is requested.
+    ///
+    /// The caller should respond to this request by calling [`WalletWrite::set_transaction_status`]
+    /// to provide transaction status information to the wallet backend.
+    GetStatus(TxId),
+    /// Information about transactions that receive or spend funds belonging to the specified
+    /// transparent address is requested.
+    ///
+    /// Fully transparent transactions, and transactions that do not contain either shielded inputs
+    /// or shielded outputs belonging to the wallet, may not be discovered by the process of chain
+    /// scanning; as a consequence, the wallet must actively query to find transactions that spend
+    /// such funds. Ideally we'd be able to query by [`OutPoint`] but this is not currently
+    /// functionality that is supported by the light wallet server.
+    #[cfg(feature = "transparent-inputs")]
+    SpendsFromAddress {
+        address: TransparentAddress,
+        block_range_start: BlockHeight,
+        block_range_end: Option<BlockHeight>,
+    },
+}
+
+/// Metadata about the status of a transaction obtained by inspecting the chain state.
+#[derive(Clone, Copy, Debug)]
+pub enum TransactionStatus {
+    /// The requested transaction ID was not recognized as belonging to a transaction
+    /// either in the main chain or in the mempool.
+    TxidNotRecognized,
+    /// The requested transaction ID corresponds to a transaction in the mempool that has not yet
+    /// been mined.
+    Mempool,
+    /// The requested transaction ID corresponds to a transaction that has been included in the
+    /// block at the provided height.
+    Mined(BlockHeight),
+}
+
 impl<NoteRef> SpendableNotes<NoteRef> {
     /// Construct a new empty [`SpendableNotes`].
     pub fn empty() -> Self {
@@ -1039,6 +1084,10 @@ pub trait WalletRead {
         }
         Ok(None)
     }
+
+    /// Returns a vector of [`TransactionDataRequest`] values that describe information
+    /// needed by the wallet to complete its view of transaction history.
+    fn transaction_data_requests(&self) -> Result<Vec<TransactionDataRequest>, Self::Error>;
 }
 
 /// The relevance of a seed to a given wallet.
@@ -1811,9 +1860,32 @@ pub trait WalletWrite: WalletRead {
     #[cfg(feature = "transparent-inputs")]
     fn reserve_next_n_ephemeral_addresses(
         &mut self,
-        account_id: Self::AccountId,
-        n: usize,
-    ) -> Result<Vec<(TransparentAddress, TransparentAddressMetadata)>, Self::Error>;
+        _account_id: Self::AccountId,
+        _n: usize,
+    ) -> Result<Vec<(TransparentAddress, TransparentAddressMetadata)>, Self::Error> {
+        // Default impl is required for feature-flagged trait methods to prevent
+        // breakage due to inadvertent activation of features by transitive dependencies
+        // of the implementing crate.
+        Ok(vec![])
+    }
+
+    /// Updates the wallet backend with respect to the status of a specific transaction, from the
+    /// perspective of the main chain.
+    ///
+    /// Fully transparent transactions, and transactions that do not contain either shielded inputs
+    /// or shielded outputs belonging to the wallet, may not be discovered by the process of chain
+    /// scanning; as a consequence, the wallet must actively query to determine whether such
+    /// transactions have been mined.
+    fn set_transaction_status(
+        &mut self,
+        _txid: TxId,
+        _status: TransactionStatus,
+    ) -> Result<(), Self::Error> {
+        // Default impl is required for feature-flagged trait methods to prevent
+        // breakage due to inadvertent activation of features by transitive dependencies
+        // of the implementing crate.
+        Ok(())
+    }
 }
 
 /// This trait describes a capability for manipulating wallet note commitment trees.
@@ -1913,8 +1985,8 @@ pub mod testing {
         chain::{ChainState, CommitmentTreeRoot},
         scanning::ScanRange,
         AccountBirthday, BlockMetadata, DecryptedTransaction, InputSource, NullifierQuery,
-        ScannedBlock, SeedRelevance, SentTransaction, SpendableNotes, WalletCommitmentTrees,
-        WalletRead, WalletSummary, WalletWrite, SAPLING_SHARD_HEIGHT,
+        ScannedBlock, SeedRelevance, SentTransaction, SpendableNotes, TransactionDataRequest,
+        WalletCommitmentTrees, WalletRead, WalletSummary, WalletWrite, SAPLING_SHARD_HEIGHT,
     };
 
     #[cfg(feature = "transparent-inputs")]
@@ -2170,6 +2242,10 @@ pub mod testing {
         ) -> Result<Option<Self::AccountId>, Self::Error> {
             Ok(None)
         }
+
+        fn transaction_data_requests(&self) -> Result<Vec<TransactionDataRequest>, Self::Error> {
+            Ok(vec![])
+        }
     }
 
     impl WalletWrite for MockWalletDb {

zcash_client_sqlite/src/lib.rs

@@ -52,7 +52,8 @@ use zcash_client_backend::{
         scanning::{ScanPriority, ScanRange},
         Account, AccountBirthday, AccountSource, BlockMetadata, DecryptedTransaction, InputSource,
         NullifierQuery, ScannedBlock, SeedRelevance, SentTransaction, SpendableNotes,
-        WalletCommitmentTrees, WalletRead, WalletSummary, WalletWrite, SAPLING_SHARD_HEIGHT,
+        TransactionDataRequest, WalletCommitmentTrees, WalletRead, WalletSummary, WalletWrite,
+        SAPLING_SHARD_HEIGHT,
     },
     keys::{
         AddressGenerationError, UnifiedAddressRequest, UnifiedFullViewingKey, UnifiedSpendingKey,
@@ -589,6 +590,16 @@ impl<C: Borrow<rusqlite::Connection>, P: consensus::Parameters> WalletRead for W
             &address.encode(&self.params),
         )
     }
+
+    fn transaction_data_requests(&self) -> Result<Vec<TransactionDataRequest>, Self::Error> {
+        let iter = std::iter::empty();
+
+        #[cfg(feature = "transparent-inputs")]
+        let iter = iter
+            .chain(wallet::transparent::transaction_data_requests(self.conn.borrow())?.into_iter());
+
+        Ok(iter.collect())
+    }
 }
 
 impl<P: consensus::Parameters> WalletWrite for WalletDb<rusqlite::Connection, P> {
@@ -1508,6 +1519,14 @@ impl<P: consensus::Parameters> WalletWrite for WalletDb<rusqlite::Connection, P>
             wallet::transparent::ephemeral::reserve_next_n_ephemeral_addresses(wdb, account_id, n)
         })
     }
+
+    fn set_transaction_status(
+        &mut self,
+        txid: TxId,
+        status: data_api::TransactionStatus,
+    ) -> Result<(), Self::Error> {
+        self.transactionally(|wdb| wallet::set_transaction_status(wdb.conn.0, txid, status))
+    }
 }
 
 impl<P: consensus::Parameters> WalletCommitmentTrees for WalletDb<rusqlite::Connection, P> {

zcash_client_sqlite/src/wallet.rs

@@ -68,6 +68,7 @@ use incrementalmerkletree::{Marking, Retention};
 use rusqlite::{self, named_params, params, OptionalExtension};
 use secrecy::{ExposeSecret, SecretVec};
 use shardtree::{error::ShardTreeError, store::ShardStore, ShardTree};
+use zcash_client_backend::data_api::TransactionStatus;
 use zip32::fingerprint::SeedFingerprint;
 
 use std::collections::{HashMap, HashSet};
@@ -108,13 +109,13 @@ use zcash_primitives::{
 };
 use zip32::{self, DiversifierIndex, Scope};
 
-use crate::TxRef;
 use crate::{
     error::SqliteClientError,
     wallet::commitment_tree::{get_max_checkpointed_height, SqliteShardStore},
     AccountId, SqlTransaction, TransferType, WalletCommitmentTrees, WalletDb, DEFAULT_UA_REQUEST,
     PRUNING_DEPTH, SAPLING_TABLES_PREFIX,
 };
+use crate::{TxRef, VERIFY_LOOKAHEAD};
 
 #[cfg(feature = "transparent-inputs")]
 use zcash_primitives::transaction::components::TxOut;
@@ -1760,7 +1761,7 @@ pub(crate) fn get_tx_height(
 ) -> Result<Option<BlockHeight>, rusqlite::Error> {
     conn.query_row(
         "SELECT block FROM transactions WHERE txid = ?",
-        [txid.as_ref().to_vec()],
+        [txid.as_ref()],
         |row| Ok(row.get::<_, Option<u32>>(0)?.map(BlockHeight::from)),
     )
     .optional()
@@ -1969,6 +1970,89 @@ pub(crate) fn store_transaction_to_be_sent<P: consensus::Parameters>(
     Ok(())
 }
 
+pub(crate) fn set_transaction_status(
+    conn: &rusqlite::Transaction,
+    txid: TxId,
+    status: TransactionStatus,
+) -> Result<(), SqliteClientError> {
+    match status {
+        TransactionStatus::TxidNotRecognized | TransactionStatus::Mempool => {
+            // The transaction is unmined, so it either needs to be resubmitted if it is unexpired
+            // as of the current chain tip, or it needs to be marked as confirmed unmined as of the
+            // chain tip height. If the chain tip height is unknown, we're in a weird ignorable
+            // state; we'll just wait for the transaction status to be queried again later.
+            if let Some(chain_tip) = chain_tip_height(conn)? {
+                let mut stmt = conn.prepare_cached(
+                    "UPDATE transactions
+                     SET confirmed_unmined = :chain_tip
+                     WHERE txid = :txid
+                     RETURNING expiry_height",
+                )?;
+
+                let expiry_height = stmt
+                    .query_row(
+                        named_params![
+                            ":chain_tip": u32::from(chain_tip),
+                            ":txid": txid.as_ref()
+                        ],
+                        |row| {
+                            row.get::<_, Option<u32>>(0)
+                                .map(|opt| opt.map(BlockHeight::from))
+                        },
+                    )
+                    .optional()?
+                    .flatten();
+
+                // If the transaction's expiry height + the confirmation depth is less than the
+                // chain tip height, we know it to be un-mineable and we can remove the reference
+                // to it from the transaction retrieval queue.
+                if expiry_height
+                    .iter()
+                    .any(|expiry| *expiry + VERIFY_LOOKAHEAD < chain_tip)
+                {
+                    conn.execute(
+                        "DELETE FROM tx_retrieval_queue WHERE txid = :txid",
+                        named_params![
+                            ":txid": txid.as_ref()
+                        ],
+                    )?;
+                }
+            }
+        }
+        TransactionStatus::Mined(height) => {
+            // The transaction has been mined, so we can set its mined height, associate it with
+            // the appropriate block, and remove it from the retrieval queue.
+            let sql_args = named_params![
+                ":txid": txid.as_ref(),
+                ":height": u32::from(height)
+            ];
+
+            conn.execute(
+                "UPDATE transactions
+                 SET mined_height = :height
+                 WHERE txid = :txid",
+                sql_args,
+            )?;
+
+            conn.execute(
+                "UPDATE transactions
+                 SET block = blocks.height
+                 FROM blocks
+                 WHERE txid = :txid
+                 AND blocks.height = :height",
+                sql_args,
+            )?;
+
+            conn.execute(
+                "DELETE FROM tx_retrieval_queue WHERE txid = :txid",
+                sql_args,
+            )?;
+        }
+    }
+
+    Ok(())
+}
+
 /// Truncates the database to the given height.
 ///
 /// If the requested height is greater than or equal to the height of the last scanned

zcash_client_sqlite/src/wallet/db.rs

@@ -185,6 +185,7 @@ CREATE TABLE "transactions" (
     expiry_height INTEGER,
     raw BLOB,
     fee INTEGER,
+    confirmed_unmined INTEGER,
     FOREIGN KEY (block) REFERENCES blocks(height),
     CONSTRAINT height_consistency CHECK (block IS NULL OR mined_height = block)
 )"#;

zcash_client_sqlite/src/wallet/init/migrations/tx_retrieval_queue.rs

@@ -35,16 +35,20 @@ impl RusqliteMigration for Migration {
             "CREATE TABLE tx_retrieval_queue (
                 txid BLOB NOT NULL UNIQUE,
                 query_type INTEGER NOT NULL,
-                dependent_transaction_id INTEGER NOT NULL,
+                dependent_transaction_id INTEGER,
                 FOREIGN KEY (dependent_transaction_id) REFERENCES transactions(id_tx)
-            );",
+            );
+            ALTER TABLE transactions ADD COLUMN confirmed_unmined INTEGER;",
         )?;
 
         Ok(())
     }
 
     fn down(&self, transaction: &Transaction) -> Result<(), WalletMigrationError> {
-        transaction.execute_batch("DROP TABLE tx_retrieval_queue;")?;
+        transaction.execute_batch(
+            "ALTER TABLE transactions DROP COLUMN confirmed_unmined;
+             DROP TABLE tx_retrieval_queue;",
+        )?;
 
         Ok(())
     }

zcash_client_sqlite/src/wallet/transparent.rs

@@ -4,7 +4,6 @@ use std::collections::{HashMap, HashSet};
 use rusqlite::OptionalExtension;
 use rusqlite::{named_params, Connection, Row};
 use zcash_client_backend::data_api::TransactionDataRequest;
-use zcash_primitives::transaction::builder::DEFAULT_TX_EXPIRY_DELTA;
 use zcash_primitives::transaction::TxId;
 use zip32::{DiversifierIndex, Scope};
 
@@ -479,6 +478,32 @@ pub(crate) fn put_received_transparent_utxo<P: consensus::Parameters>(
     }
 }
 
+/// Returns the vector of [`TxId`]s for transactions for which spentness state is indeterminate.
+pub(crate) fn transaction_data_requests(
+    conn: &rusqlite::Connection,
+) -> Result<Vec<TransactionDataRequest>, SqliteClientError> {
+    let mut tx_retrieval_stmt =
+        conn.prepare_cached("SELECT txid, query_type FROM tx_retrieval_queue")?;
+
+    let result = tx_retrieval_stmt
+        .query_and_then([], |row| {
+            let txid = row.get(0).map(TxId::from_bytes)?;
+            let query_type = row.get(1).map(TxQueryType::from_code)?.ok_or_else(|| {
+                SqliteClientError::CorruptedData(
+                    "Unrecognized transaction data request type.".to_owned(),
+                )
+            })?;
+
+            Ok::<TransactionDataRequest, SqliteClientError>(match query_type {
+                TxQueryType::Enhancement => TransactionDataRequest::Enhancement(txid),
+                TxQueryType::Status => TransactionDataRequest::GetStatus(txid),
+            })
+        })?
+        .collect::<Result<Vec<_>, _>>()?;
+
+    Ok(result)
+}
+
 pub(crate) fn get_transparent_address_metadata<P: consensus::Parameters>(
     conn: &rusqlite::Connection,
     params: &P,