From 1badfa3a4ef8ca4c311ac37ac1281374fc7f622b Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Fri, 1 Mar 2024 14:38:47 +0100 Subject: [PATCH 01/35] const fn --- sdk/src/wallet/operations/syncing/options.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk/src/wallet/operations/syncing/options.rs b/sdk/src/wallet/operations/syncing/options.rs index f5a1d09816..029557f412 100644 --- a/sdk/src/wallet/operations/syncing/options.rs +++ b/sdk/src/wallet/operations/syncing/options.rs @@ -47,27 +47,27 @@ pub struct SyncOptions { pub sync_implicit_accounts: bool, } -fn default_force_syncing() -> bool { +const fn default_force_syncing() -> bool { DEFAULT_FORCE_SYNCING } -fn default_sync_incoming_transactions() -> bool { +const fn default_sync_incoming_transactions() -> bool { DEFAULT_SYNC_INCOMING_TRANSACTIONS } -fn default_sync_only_most_basic_outputs() -> bool { +const fn default_sync_only_most_basic_outputs() -> bool { DEFAULT_SYNC_ONLY_MOST_BASIC_OUTPUTS } -fn default_sync_pending_transactions() -> bool { +const fn default_sync_pending_transactions() -> bool { DEFAULT_SYNC_PENDING_TRANSACTIONS } -fn default_sync_native_token_foundries() -> bool { +const fn default_sync_native_token_foundries() -> bool { DEFAULT_SYNC_NATIVE_TOKEN_FOUNDRIES } -fn default_sync_implicit_accounts() -> bool { +const fn default_sync_implicit_accounts() -> bool { DEFAULT_SYNC_IMPLICIT_ACCOUNTS } From bb673e2ab1716e2c0dce918d5c2dab2fea73aabf Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Fri, 1 Mar 2024 17:34:04 +0100 Subject: [PATCH 02/35] sync options --- sdk/src/wallet/operations/syncing/options.rs | 106 +++++++++---------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/sdk/src/wallet/operations/syncing/options.rs b/sdk/src/wallet/operations/syncing/options.rs index 029557f412..190c9847a9 100644 --- a/sdk/src/wallet/operations/syncing/options.rs +++ b/sdk/src/wallet/operations/syncing/options.rs @@ -3,13 +3,6 @@ use serde::{Deserialize, Serialize}; -const DEFAULT_FORCE_SYNCING: bool = false; -const DEFAULT_SYNC_INCOMING_TRANSACTIONS: bool = false; -const DEFAULT_SYNC_ONLY_MOST_BASIC_OUTPUTS: bool = false; -const DEFAULT_SYNC_PENDING_TRANSACTIONS: bool = true; -const DEFAULT_SYNC_NATIVE_TOKEN_FOUNDRIES: bool = false; -const DEFAULT_SYNC_IMPLICIT_ACCOUNTS: bool = false; - /// The synchronization options #[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -17,14 +10,14 @@ pub struct SyncOptions { /// Usually syncing is skipped if it's called in between 200ms, because there can only be new changes every /// milestone and calling it twice "at the same time" will not return new data /// When this to true, we will sync anyways, even if it's called 0ms after the las sync finished. - #[serde(default)] + #[serde(default = "no")] pub force_syncing: bool, /// Try to sync transactions from incoming outputs with their inputs. Some data may not be obtained if it has been /// pruned. - #[serde(default = "default_sync_incoming_transactions")] + #[serde(default = "no")] pub sync_incoming_transactions: bool, /// Checks pending transactions and reissues them if necessary. - #[serde(default = "default_sync_pending_transactions")] + #[serde(default = "yes")] pub sync_pending_transactions: bool, /// Specifies what outputs should be synced for the ed25519 address from the wallet. #[serde(default)] @@ -37,52 +30,28 @@ pub struct SyncOptions { pub nft: NftSyncOptions, /// Specifies if only basic outputs with an AddressUnlockCondition alone should be synced, will overwrite /// `wallet`, `account` and `nft` options. - #[serde(default = "default_sync_only_most_basic_outputs")] + #[serde(default = "no")] pub sync_only_most_basic_outputs: bool, /// Sync native token foundries, so their metadata can be returned in the balance. - #[serde(default = "default_sync_native_token_foundries")] + #[serde(default = "no")] pub sync_native_token_foundries: bool, /// Sync implicit accounts. - #[serde(default = "default_sync_implicit_accounts")] + #[serde(default = "no")] pub sync_implicit_accounts: bool, } -const fn default_force_syncing() -> bool { - DEFAULT_FORCE_SYNCING -} - -const fn default_sync_incoming_transactions() -> bool { - DEFAULT_SYNC_INCOMING_TRANSACTIONS -} - -const fn default_sync_only_most_basic_outputs() -> bool { - DEFAULT_SYNC_ONLY_MOST_BASIC_OUTPUTS -} - -const fn default_sync_pending_transactions() -> bool { - DEFAULT_SYNC_PENDING_TRANSACTIONS -} - -const fn default_sync_native_token_foundries() -> bool { - DEFAULT_SYNC_NATIVE_TOKEN_FOUNDRIES -} - -const fn default_sync_implicit_accounts() -> bool { - DEFAULT_SYNC_IMPLICIT_ACCOUNTS -} - impl Default for SyncOptions { fn default() -> Self { Self { - sync_incoming_transactions: default_sync_incoming_transactions(), - sync_pending_transactions: default_sync_pending_transactions(), + force_syncing: no(), + sync_incoming_transactions: no(), + sync_pending_transactions: yes(), wallet: WalletSyncOptions::default(), account: AccountSyncOptions::default(), nft: NftSyncOptions::default(), - sync_only_most_basic_outputs: default_sync_only_most_basic_outputs(), - sync_native_token_foundries: default_sync_native_token_foundries(), - force_syncing: default_force_syncing(), - sync_implicit_accounts: default_sync_implicit_accounts(), + sync_only_most_basic_outputs: no(), + sync_native_token_foundries: no(), + sync_implicit_accounts: no(), } } } @@ -91,19 +60,23 @@ impl Default for SyncOptions { #[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize)] #[serde(default, rename_all = "camelCase")] pub struct WalletSyncOptions { + #[serde(default = "yes")] pub basic_outputs: bool, + #[serde(default = "yes")] pub account_outputs: bool, + #[serde(default = "yes")] pub nft_outputs: bool, + #[serde(default = "yes")] pub delegation_outputs: bool, } impl Default for WalletSyncOptions { fn default() -> Self { Self { - basic_outputs: true, - account_outputs: true, - nft_outputs: true, - delegation_outputs: true, + basic_outputs: yes(), + account_outputs: yes(), + nft_outputs: yes(), + delegation_outputs: yes(), } } } @@ -118,10 +91,15 @@ impl WalletSyncOptions { #[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize)] #[serde(default, rename_all = "camelCase")] pub struct AccountSyncOptions { + #[serde(default = "no")] pub basic_outputs: bool, + #[serde(default = "no")] pub account_outputs: bool, + #[serde(default = "yes")] pub foundry_outputs: bool, + #[serde(default = "no")] pub nft_outputs: bool, + #[serde(default = "no")] pub delegation_outputs: bool, } @@ -129,11 +107,11 @@ impl Default for AccountSyncOptions { // Sync only foundries fn default() -> Self { Self { - basic_outputs: false, - account_outputs: false, - foundry_outputs: true, - nft_outputs: false, - delegation_outputs: false, + basic_outputs: no(), + account_outputs: no(), + foundry_outputs: yes(), + nft_outputs: no(), + delegation_outputs: no(), } } } @@ -149,17 +127,39 @@ impl AccountSyncOptions { } /// Sync options for addresses from NFT outputs -#[derive(Debug, Default, Clone, Eq, PartialEq, Hash, Serialize, Deserialize)] +#[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize)] #[serde(default, rename_all = "camelCase")] pub struct NftSyncOptions { + #[serde(default = "no")] pub basic_outputs: bool, + #[serde(default = "no")] pub account_outputs: bool, + #[serde(default = "no")] pub nft_outputs: bool, + #[serde(default = "no")] pub delegation_outputs: bool, } +impl Default for NftSyncOptions { + fn default() -> Self { + Self { + basic_outputs: no(), + account_outputs: no(), + nft_outputs: no(), + delegation_outputs: no(), + } + } +} impl NftSyncOptions { pub(crate) fn all_outputs(&self) -> bool { self.basic_outputs && self.account_outputs && self.nft_outputs && self.delegation_outputs } } + +const fn yes() -> bool { + true +} + +const fn no() -> bool { + false +} From 8134473cbe981df627b7482ceac70b1f51343bab Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Thu, 14 Mar 2024 09:45:14 +0100 Subject: [PATCH 03/35] move instead of clone --- sdk/src/wallet/operations/syncing/outputs.rs | 40 +++++++++++--------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/sdk/src/wallet/operations/syncing/outputs.rs b/sdk/src/wallet/operations/syncing/outputs.rs index b8e62b6462..88835515f0 100644 --- a/sdk/src/wallet/operations/syncing/outputs.rs +++ b/sdk/src/wallet/operations/syncing/outputs.rs @@ -30,23 +30,29 @@ impl Wallet { Ok(outputs_with_meta .into_iter() - .map(|output_with_meta| { - // check if we know the transaction that created this output and if we created it (if we store incoming - // transactions separated, then this check wouldn't be required) - let remainder = wallet_ledger - .transactions - .get(output_with_meta.metadata().output_id().transaction_id()) - .map_or(false, |tx| !tx.incoming); - - OutputData { - output_id: output_with_meta.metadata().output_id().to_owned(), - metadata: *output_with_meta.metadata(), - output: output_with_meta.output().clone(), - output_id_proof: output_with_meta.output_id_proof().clone(), - network_id, - remainder, - } - }) + .map( + |OutputWithMetadataResponse { + output, + output_id_proof, + metadata, + }| { + // check if we know the transaction that created this output and if we created it (if we store + // incoming transactions separated, then this check wouldn't be required) + let remainder = wallet_ledger + .transactions + .get(metadata.output_id().transaction_id()) + .map_or(false, |tx| !tx.incoming); + + OutputData { + output_id: metadata.output_id().to_owned(), + metadata, + output, + output_id_proof, + network_id, + remainder, + } + }, + ) .collect()) } From 264b2d923ef558aad928a7dbbd9bd2c0058c43a6 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Thu, 14 Mar 2024 11:15:45 +0100 Subject: [PATCH 04/35] combinators --- .../syncing/addresses/output_ids/account_foundry.rs | 8 ++------ .../wallet/operations/syncing/addresses/output_ids/mod.rs | 7 ++----- sdk/src/wallet/operations/syncing/outputs.rs | 1 + sdk/src/wallet/types/mod.rs | 2 ++ 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/sdk/src/wallet/operations/syncing/addresses/output_ids/account_foundry.rs b/sdk/src/wallet/operations/syncing/addresses/output_ids/account_foundry.rs index e67ef0339f..b6d6474b1a 100644 --- a/sdk/src/wallet/operations/syncing/addresses/output_ids/account_foundry.rs +++ b/sdk/src/wallet/operations/syncing/addresses/output_ids/account_foundry.rs @@ -77,13 +77,9 @@ impl Wallet { } } - let mut output_ids = HashSet::new(); let results: Vec> = futures::future::try_join_all(tasks).await?; - - for res in results { - let foundry_output_ids = res?; - output_ids.extend(foundry_output_ids.items); - } + let responses: Vec = results.into_iter().collect::, _>>()?; + let output_ids: HashSet = HashSet::from_iter(responses.into_iter().flat_map(|res| res.items)); Ok(output_ids.into_iter().collect()) } diff --git a/sdk/src/wallet/operations/syncing/addresses/output_ids/mod.rs b/sdk/src/wallet/operations/syncing/addresses/output_ids/mod.rs index 16caf0613c..2f0ae8d355 100644 --- a/sdk/src/wallet/operations/syncing/addresses/output_ids/mod.rs +++ b/sdk/src/wallet/operations/syncing/addresses/output_ids/mod.rs @@ -217,11 +217,8 @@ impl Wallet { let results = futures::future::try_join_all(tasks).await?; // Get all results - let mut output_ids = HashSet::new(); - for res in results { - let found_output_ids = res?; - output_ids.extend(found_output_ids); - } + let output_ids = results.into_iter().collect::, _>>()?; + let output_ids: HashSet = HashSet::from_iter(output_ids.into_iter().flat_map(|v| v.into_iter())); Ok(output_ids.into_iter().collect()) } diff --git a/sdk/src/wallet/operations/syncing/outputs.rs b/sdk/src/wallet/operations/syncing/outputs.rs index 88835515f0..30e57ed3fa 100644 --- a/sdk/src/wallet/operations/syncing/outputs.rs +++ b/sdk/src/wallet/operations/syncing/outputs.rs @@ -64,6 +64,7 @@ impl Wallet { ) -> Result, WalletError> { log::debug!("[SYNC] start get_outputs"); let get_outputs_start_time = Instant::now(); + let mut outputs = Vec::new(); let mut unknown_outputs = Vec::new(); let mut unspent_outputs = Vec::new(); diff --git a/sdk/src/wallet/types/mod.rs b/sdk/src/wallet/types/mod.rs index 543b49055c..029aae3e1b 100644 --- a/sdk/src/wallet/types/mod.rs +++ b/sdk/src/wallet/types/mod.rs @@ -38,6 +38,7 @@ use crate::{ pub struct OutputData { /// The output id pub output_id: OutputId, + /// The output metadata pub metadata: OutputMetadata, /// The actual Output pub output: Output, @@ -45,6 +46,7 @@ pub struct OutputData { pub output_id_proof: OutputIdProof, /// Network ID pub network_id: u64, + /// Whether the output is a remainder pub remainder: bool, } From 010e27616ba7fa4d0908aa692d63daab96c5cb7e Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Thu, 14 Mar 2024 12:06:49 +0100 Subject: [PATCH 05/35] rm unnecessary cloning 1 --- .../syncing/addresses/output_ids/account_foundry.rs | 2 +- sdk/src/wallet/operations/syncing/addresses/outputs.rs | 9 +++------ sdk/src/wallet/operations/syncing/mod.rs | 2 +- sdk/src/wallet/operations/syncing/outputs.rs | 6 +++--- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/sdk/src/wallet/operations/syncing/addresses/output_ids/account_foundry.rs b/sdk/src/wallet/operations/syncing/addresses/output_ids/account_foundry.rs index b6d6474b1a..f73d3585ad 100644 --- a/sdk/src/wallet/operations/syncing/addresses/output_ids/account_foundry.rs +++ b/sdk/src/wallet/operations/syncing/addresses/output_ids/account_foundry.rs @@ -55,7 +55,7 @@ impl Wallet { ) -> Result, WalletError> { log::debug!("[SYNC] get_foundry_output_ids"); // Get account outputs, so we can then get the foundry outputs with the account addresses - let account_outputs_with_meta = self.get_outputs(account_output_ids.to_vec()).await?; + let account_outputs_with_meta = self.get_outputs(account_output_ids).await?; let bech32_hrp = self.client().get_bech32_hrp().await?; diff --git a/sdk/src/wallet/operations/syncing/addresses/outputs.rs b/sdk/src/wallet/operations/syncing/addresses/outputs.rs index 90df1a4d45..6c87037386 100644 --- a/sdk/src/wallet/operations/syncing/addresses/outputs.rs +++ b/sdk/src/wallet/operations/syncing/addresses/outputs.rs @@ -34,9 +34,8 @@ impl Wallet { let wallet = self.clone(); tasks.push(async move { task::spawn(async move { - let unspent_outputs_with_metadata = wallet - .get_outputs(address_with_unspent_outputs.output_ids.clone()) - .await?; + let unspent_outputs_with_metadata = + wallet.get_outputs(&address_with_unspent_outputs.output_ids).await?; let unspent_outputs_data = wallet .output_response_to_output_data(unspent_outputs_with_metadata) .await?; @@ -46,9 +45,7 @@ impl Wallet { }); } let results: Vec> = futures::future::try_join_all(tasks).await?; - for res in results { - addresses_with_outputs.push(res?); - } + let addresses_with_outputs = results.into_iter().collect::, _>>()?; } log::debug!( "[SYNC] finished get_outputs_from_address_output_ids in {:.2?}", diff --git a/sdk/src/wallet/operations/syncing/mod.rs b/sdk/src/wallet/operations/syncing/mod.rs index 405338d7fb..56bdf9e3c1 100644 --- a/sdk/src/wallet/operations/syncing/mod.rs +++ b/sdk/src/wallet/operations/syncing/mod.rs @@ -117,7 +117,7 @@ impl Wallet { .output_ids .extend(account_or_nft_output_ids.clone()); - let account_or_nft_outputs_with_metadata = self.get_outputs(account_or_nft_output_ids).await?; + let account_or_nft_outputs_with_metadata = self.get_outputs(&account_or_nft_output_ids).await?; let account_or_nft_outputs_data = self .output_response_to_output_data(account_or_nft_outputs_with_metadata) .await?; diff --git a/sdk/src/wallet/operations/syncing/outputs.rs b/sdk/src/wallet/operations/syncing/outputs.rs index 30e57ed3fa..def6603e86 100644 --- a/sdk/src/wallet/operations/syncing/outputs.rs +++ b/sdk/src/wallet/operations/syncing/outputs.rs @@ -60,7 +60,7 @@ impl Wallet { /// unspent, because we wouldn't get them from the node if they were spent pub(crate) async fn get_outputs( &self, - output_ids: Vec, + output_ids: &[OutputId], ) -> Result, WalletError> { log::debug!("[SYNC] start get_outputs"); let get_outputs_start_time = Instant::now(); @@ -78,14 +78,14 @@ impl Wallet { log::warn!("Removing spent output metadata for {output_id}, because it's still unspent"); output_data.metadata.spent = None; } - unspent_outputs.push((output_id, output_data.clone())); + unspent_outputs.push((*output_id, output_data.clone())); outputs.push(OutputWithMetadataResponse::new( output_data.output.clone(), output_data.output_id_proof.clone(), output_data.metadata, )); } - None => unknown_outputs.push(output_id), + None => unknown_outputs.push(*output_id), } } // known output is unspent, so insert it to the unspent outputs again, because if it was an From abe5f409e3f25c20421b03fcf4bf8b7b319f697c Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Thu, 14 Mar 2024 12:21:59 +0100 Subject: [PATCH 06/35] rm unnecessary cloning 2 --- .../operations/syncing/addresses/output_ids/mod.rs | 4 ++-- sdk/src/wallet/operations/syncing/mod.rs | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/sdk/src/wallet/operations/syncing/addresses/output_ids/mod.rs b/sdk/src/wallet/operations/syncing/addresses/output_ids/mod.rs index 2f0ae8d355..d7f017fc2e 100644 --- a/sdk/src/wallet/operations/syncing/addresses/output_ids/mod.rs +++ b/sdk/src/wallet/operations/syncing/addresses/output_ids/mod.rs @@ -227,7 +227,7 @@ impl Wallet { /// return spent outputs separated pub(crate) async fn get_output_ids_for_addresses( &self, - addresses_with_unspent_outputs: Vec, + addresses_with_unspent_outputs: &[AddressWithUnspentOutputs], options: &SyncOptions, ) -> Result<(Vec, Vec), WalletError> { log::debug!("[SYNC] start get_output_ids_for_addresses"); @@ -238,7 +238,7 @@ impl Wallet { let mut spent_or_not_anymore_synced_outputs = Vec::new(); // We split the addresses into chunks so we don't get timeouts if we have thousands - for addresses_chunk in &mut addresses_with_unspent_outputs + for addresses_chunk in addresses_with_unspent_outputs .chunks(PARALLEL_REQUESTS_AMOUNT) .map(|x: &[AddressWithUnspentOutputs]| x.to_vec()) { diff --git a/sdk/src/wallet/operations/syncing/mod.rs b/sdk/src/wallet/operations/syncing/mod.rs index 56bdf9e3c1..3d3abdf4f8 100644 --- a/sdk/src/wallet/operations/syncing/mod.rs +++ b/sdk/src/wallet/operations/syncing/mod.rs @@ -46,7 +46,7 @@ impl Wallet { // are found. async fn request_outputs_recursively( &self, - addresses_to_sync: Vec, + addresses_to_sync: &[AddressWithUnspentOutputs], options: &SyncOptions, ) -> Result<(Vec, Vec, Vec), WalletError> { // Cache account and nft addresses with the related Ed25519 address, so we can update the account @@ -58,9 +58,8 @@ impl Wallet { let bech32_hrp = self.client().get_bech32_hrp().await?; // Get the unspent and spent/not-synced output ids per address to sync - let (addresses_to_sync_with_unspent_output_ids, mut spent_or_not_synced_output_ids) = self - .get_output_ids_for_addresses(addresses_to_sync.clone(), options) - .await?; + let (addresses_to_sync_with_unspent_output_ids, mut spent_or_not_synced_output_ids) = + self.get_output_ids_for_addresses(addresses_to_sync, options).await?; // Get the corresponding unspent output data let mut new_unspent_outputs_data = self @@ -224,7 +223,7 @@ where } let (_addresses_with_unspent_outputs, spent_or_not_synced_output_ids, outputs_data) = - self.request_outputs_recursively(addresses_to_sync, options).await?; + self.request_outputs_recursively(&addresses_to_sync, options).await?; // Request possible spent outputs log::debug!("[SYNC] spent_or_not_synced_outputs: {spent_or_not_synced_output_ids:?}"); From c5e38fbcfb2493781d0428f892dc74725fa043f6 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Thu, 14 Mar 2024 17:54:28 +0100 Subject: [PATCH 07/35] slices; impl Into