From e3eb8e39113fd4a37d1f5a0faa9ac117f8d6335e Mon Sep 17 00:00:00 2001 From: danielxiangzl Date: Thu, 12 Dec 2024 18:24:20 -0800 Subject: [PATCH 01/10] test leader equivocation --- consensus/consensus-types/src/block.rs | 10 ++++- consensus/consensus-types/src/block_data.rs | 4 ++ consensus/consensus-types/src/proposal_msg.rs | 9 ++++ consensus/src/round_manager.rs | 43 ++++++++++++++++++- .../src/suites/realistic_environment.rs | 1 + testsuite/forge.py | 2 +- .../consensus/consensus_fault_tolerance.rs | 41 ++++++++++++++++++ testsuite/testcases/src/performance_test.rs | 34 ++++++++++++++- 8 files changed, 139 insertions(+), 5 deletions(-) diff --git a/consensus/consensus-types/src/block.rs b/consensus/consensus-types/src/block.rs index c36946e493e7f..d36868dbcbb33 100644 --- a/consensus/consensus-types/src/block.rs +++ b/consensus/consensus-types/src/block.rs @@ -9,7 +9,7 @@ use crate::{ }; use anyhow::{bail, ensure, format_err}; use aptos_bitvec::BitVec; -use aptos_crypto::{bls12381, hash::CryptoHash, HashValue}; +use aptos_crypto::{bls12381::{self, Signature}, hash::CryptoHash, HashValue}; use aptos_infallible::duration_since_epoch; use aptos_types::{ account_address::AccountAddress, @@ -87,6 +87,14 @@ impl Block { self.is_opt } + pub fn set_timestamp(&mut self, timestamp: u64) { + self.block_data.set_timestamp(timestamp); + } + + pub fn set_signature(&mut self, signature: Signature) { + self.signature = Some(signature); + } + pub fn set_quorum_cert(&mut self, qc: QuorumCert) { self.block_data.set_quorum_cert(qc); } diff --git a/consensus/consensus-types/src/block_data.rs b/consensus/consensus-types/src/block_data.rs index 82b181017f368..dbf2afc4dd0fa 100644 --- a/consensus/consensus-types/src/block_data.rs +++ b/consensus/consensus-types/src/block_data.rs @@ -112,6 +112,10 @@ impl CryptoHash for BlockData { } impl BlockData { + pub fn set_timestamp(&mut self, timestamp: u64) { + self.timestamp_usecs = timestamp; + } + pub fn set_quorum_cert(&mut self, qc: QuorumCert) { self.quorum_cert = qc; } diff --git a/consensus/consensus-types/src/proposal_msg.rs b/consensus/consensus-types/src/proposal_msg.rs index 7a6c089cf27e4..4d52f54aacfbc 100644 --- a/consensus/consensus-types/src/proposal_msg.rs +++ b/consensus/consensus-types/src/proposal_msg.rs @@ -4,6 +4,7 @@ use crate::{block::Block, common::Author, proof_of_store::ProofCache, sync_info::SyncInfo}; use anyhow::{anyhow, ensure, format_err, Context, Ok, Result}; +use aptos_crypto::bls12381::Signature; use aptos_short_hex_str::AsShortHexStr; use aptos_types::validator_verifier::ValidatorVerifier; use serde::{Deserialize, Serialize}; @@ -26,6 +27,14 @@ impl ProposalMsg { } } + pub fn set_timestamp(&mut self, timestamp: u64) { + self.proposal.set_timestamp(timestamp); + } + + pub fn set_signature(&mut self, signature: Signature) { + self.proposal.set_signature(signature); + } + pub fn epoch(&self) -> u64 { self.proposal.epoch() } diff --git a/consensus/src/round_manager.rs b/consensus/src/round_manager.rs index 7a0885fc79eb7..a09f1757fb5e2 100644 --- a/consensus/src/round_manager.rs +++ b/consensus/src/round_manager.rs @@ -459,7 +459,7 @@ impl RoundManager { sync_info, network.clone(), proposal_generator, - safety_rules, + safety_rules.clone(), proposer_election, parent_id, ) @@ -474,9 +474,19 @@ impl RoundManager { { if Self::check_whether_to_inject_reconfiguration_error() { Self::attempt_to_inject_reconfiguration_error( + epoch_state.clone(), + network.clone(), + &proposal_msg, + ) + .await?; + } + + if Self::check_whether_to_equivocate() { + Self::attempt_to_equivocate( epoch_state, network.clone(), &proposal_msg, + safety_rules.clone() ) .await?; } @@ -1922,6 +1932,12 @@ impl RoundManager { false } + #[cfg(feature = "failpoints")] + fn check_whether_to_equivocate() -> bool { + fail_point!("consensus::leader_equivocation", |_| true); + false + } + /// Given R1 <- B2 if R1 has the reconfiguration txn, we inject error on B2 if R1.round + 1 = B2.round /// Direct suffix is checked by parent.has_reconfiguration && !parent.parent.has_reconfiguration /// The error is injected by sending proposals to half of the validators to force a timeout. @@ -1956,4 +1972,29 @@ impl RoundManager { Ok(()) } } + + #[cfg(feature = "failpoints")] + async fn attempt_to_equivocate( + epoch_state: Arc, + network: Arc, + proposal_msg: &ProposalMsg, + safety_rules: Arc>, + ) -> anyhow::Result<()> { + info!("[Test] Leader of epoch {} round {} equivocates", epoch_state.epoch, proposal_msg.proposal().round()); + + let all_peers: Vec<_> = epoch_state + .verifier + .get_ordered_account_addresses_iter() + .collect(); + let mut timestamp = proposal_msg.proposal().block_data().timestamp_usecs(); + for peer in all_peers { + timestamp += 1; + let mut modified_proposal_msg = proposal_msg.clone(); + modified_proposal_msg.set_timestamp(timestamp); + let signature = safety_rules.lock().sign_proposal(modified_proposal_msg.proposal().block_data())?; + modified_proposal_msg.set_signature(signature); + network.send_proposal(modified_proposal_msg.clone(), vec![peer]).await; + } + Err(anyhow::anyhow!("Injected leader equivocation")) + } } diff --git a/testsuite/forge-cli/src/suites/realistic_environment.rs b/testsuite/forge-cli/src/suites/realistic_environment.rs index 2f0475486e9bd..c79914eea39bf 100644 --- a/testsuite/forge-cli/src/suites/realistic_environment.rs +++ b/testsuite/forge-cli/src/suites/realistic_environment.rs @@ -61,6 +61,7 @@ pub(crate) fn realistic_env_sweep_wrap( .with_initial_fullnode_count(num_fullnodes) .with_validator_override_node_config_fn(Arc::new(|config, _| { config.execution.processed_transactions_detailed_counters = true; + config.api.failpoints_enabled = true; })) .add_network_test(test) // Test inherits the main EmitJobRequest, so update here for more precise latency measurements diff --git a/testsuite/forge.py b/testsuite/forge.py index dcf15e3e3233b..e0a1d4f8f88b7 100644 --- a/testsuite/forge.py +++ b/testsuite/forge.py @@ -1560,7 +1560,7 @@ def test( asyncio.run(forge_cluster.write(context.shell)) # These features and profile flags are set as strings - enable_failpoints = forge_enable_failpoints == "true" + enable_failpoints = True enable_performance_profile = forge_enable_performance == "true" # In the below, assume that the image is pushed to all registries diff --git a/testsuite/smoke-test/src/consensus/consensus_fault_tolerance.rs b/testsuite/smoke-test/src/consensus/consensus_fault_tolerance.rs index dc640d98146da..8bbda0172bd26 100644 --- a/testsuite/smoke-test/src/consensus/consensus_fault_tolerance.rs +++ b/testsuite/smoke-test/src/consensus/consensus_fault_tolerance.rs @@ -347,6 +347,47 @@ async fn test_execution_retry() { .unwrap(); } +#[tokio::test] +async fn test_fault_tolerance_of_leader_equivocation() { + let num_validators = 4; + + let swarm = create_swarm(num_validators, 1).await; + let (validator_clients, public_info) = { + ( + swarm.get_validator_clients_with_names(), + swarm.aptos_public_info(), + ) + }; + test_consensus_fault_tolerance( + validator_clients, + public_info, + 3, + 5.0, + 1, + Box::new(FailPointFailureInjection::new(Box::new(move |cycle, _| { + ( + vec![( + cycle % num_validators, + "consensus::leader_equivocation".to_string(), + format!("{}%return", 50), + )], + true, + ) + }))), + Box::new( + move |_, executed_epochs, executed_rounds, executed_transactions, _, _| { + successful_criteria(executed_epochs, executed_rounds, executed_transactions); + Ok(()) + }, + ), + true, + false, + ) + .await + .unwrap(); + panic!("test_fault_tolerance_of_leader_equivocation"); +} + #[tokio::test] async fn test_fault_tolerance_of_network_send() { // Randomly increase network failure rate, until network halts, and check that it comes back afterwards. diff --git a/testsuite/testcases/src/performance_test.rs b/testsuite/testcases/src/performance_test.rs index 63786565d1f98..ebcdead6d6481 100644 --- a/testsuite/testcases/src/performance_test.rs +++ b/testsuite/testcases/src/performance_test.rs @@ -2,8 +2,11 @@ // Parts of the project are originally copyright © Meta Platforms, Inc. // SPDX-License-Identifier: Apache-2.0 +use std::{sync::Arc, time::Duration}; +use anyhow::{anyhow, bail, Context}; + use crate::NetworkLoadTest; -use aptos_forge::{NetworkContextSynchronizer, NetworkTest, Result, Test}; +use aptos_forge::{NetworkContextSynchronizer, NetworkTest, Result, Swarm, SwarmExt, Test, TestReport}; use async_trait::async_trait; pub struct PerformanceBenchmark; @@ -14,7 +17,34 @@ impl Test for PerformanceBenchmark { } } -impl NetworkLoadTest for PerformanceBenchmark {} +#[async_trait] +impl NetworkLoadTest for PerformanceBenchmark { + async fn test( + &self, + swarm: Arc>>, + _report: &mut TestReport, + duration: Duration, + ) -> Result<()> { + let validators = { swarm.read().await.get_validator_clients_with_names() }; + let num_bad_leaders = validators.len() / 5; + for (name, validator) in validators[..num_bad_leaders].iter() { + validator + .set_failpoint( + "consensus::leader_equivocation".to_string(), + "return".to_string(), + ) + .await + .map_err(|e| { + anyhow!( + "set_failpoint to set consensus leader equivocation on {} failed, {:?}", + name, + e + ) + })?; + }; + Ok(()) + } +} #[async_trait] impl NetworkTest for PerformanceBenchmark { From 8c69113e6d842b4211bf5654a26b875fe81aca5c Mon Sep 17 00:00:00 2001 From: danielxiangzl Date: Wed, 18 Dec 2024 11:30:54 -0800 Subject: [PATCH 02/10] test --- testsuite/forge-cli/src/suites/realistic_environment.rs | 4 +--- testsuite/testcases/src/performance_test.rs | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/testsuite/forge-cli/src/suites/realistic_environment.rs b/testsuite/forge-cli/src/suites/realistic_environment.rs index c79914eea39bf..92da302faca8f 100644 --- a/testsuite/forge-cli/src/suites/realistic_environment.rs +++ b/testsuite/forge-cli/src/suites/realistic_environment.rs @@ -83,15 +83,13 @@ pub(crate) fn realistic_env_sweep_wrap( pub(crate) fn realistic_env_load_sweep_test() -> ForgeConfig { realistic_env_sweep_wrap(10, 5, LoadVsPerfBenchmark { test: Box::new(PerformanceBenchmark), - workloads: Workloads::TPS(vec![1000, 5000, 10000, 12000, 13000, 14000, 15000]), + workloads: Workloads::TPS(vec![1000, 2500, 5000, 7500, 10000]), criteria: [ (95, 0.9, 1.1, 1.2, 0), (95, 0.9, 1.1, 1.2, 0), (95, 0.9, 1.1, 1.2, 0), (95, 0.9, 1.1, 1.2, 0), (95, 0.9, 1.1, 1.2, 0), - (95, 0.9, 1.1, 1.2, 0), - (95, 0.9, 1.1, 1.2, 0), ] .into_iter() .map( diff --git a/testsuite/testcases/src/performance_test.rs b/testsuite/testcases/src/performance_test.rs index ebcdead6d6481..151d5e9bdb3fd 100644 --- a/testsuite/testcases/src/performance_test.rs +++ b/testsuite/testcases/src/performance_test.rs @@ -26,7 +26,8 @@ impl NetworkLoadTest for PerformanceBenchmark { duration: Duration, ) -> Result<()> { let validators = { swarm.read().await.get_validator_clients_with_names() }; - let num_bad_leaders = validators.len() / 5; + // 10 vals, test 1,2,3 failures + let num_bad_leaders = 3; for (name, validator) in validators[..num_bad_leaders].iter() { validator .set_failpoint( From 328f7e800601b9b3172a66550b7b609ef6e180db Mon Sep 17 00:00:00 2001 From: danielxiangzl Date: Fri, 13 Dec 2024 15:34:12 -0800 Subject: [PATCH 03/10] use rotating leader --- types/src/on_chain_config/consensus_config.rs | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/types/src/on_chain_config/consensus_config.rs b/types/src/on_chain_config/consensus_config.rs index d46718d6b09b3..da5f95a1c4298 100644 --- a/types/src/on_chain_config/consensus_config.rs +++ b/types/src/on_chain_config/consensus_config.rs @@ -374,22 +374,23 @@ impl Default for ConsensusConfigV1 { back_pressure_limit: 10, exclude_round: 40, max_failed_authors_to_store: 10, - proposer_election_type: ProposerElectionType::LeaderReputation( - LeaderReputationType::ProposerAndVoterV2(ProposerAndVoterConfig { - active_weight: 1000, - inactive_weight: 10, - failed_weight: 1, - failure_threshold_percent: 10, // = 10% - // In each round we get stastics for the single proposer - // and large number of validators. So the window for - // the proposers needs to be significantly larger - // to have enough useful statistics. - proposer_window_num_validators_multiplier: 10, - voter_window_num_validators_multiplier: 1, - weight_by_voting_power: true, - use_history_from_previous_epoch_max_count: 5, - }), - ), + proposer_election_type: ProposerElectionType::RotatingProposer(1), + // ProposerElectionType::LeaderReputation( + // LeaderReputationType::ProposerAndVoterV2(ProposerAndVoterConfig { + // active_weight: 1000, + // inactive_weight: 10, + // failed_weight: 1, + // failure_threshold_percent: 10, // = 10% + // // In each round we get stastics for the single proposer + // // and large number of validators. So the window for + // // the proposers needs to be significantly larger + // // to have enough useful statistics. + // proposer_window_num_validators_multiplier: 10, + // voter_window_num_validators_multiplier: 1, + // weight_by_voting_power: true, + // use_history_from_previous_epoch_max_count: 5, + // }), + // ), } } } From 51d88e925b4bb3969ca289838448be589bc254d4 Mon Sep 17 00:00:00 2001 From: danielxiangzl Date: Wed, 18 Dec 2024 14:35:16 -0800 Subject: [PATCH 04/10] 1 failure --- testsuite/testcases/src/performance_test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testsuite/testcases/src/performance_test.rs b/testsuite/testcases/src/performance_test.rs index 151d5e9bdb3fd..d7472c5b67c02 100644 --- a/testsuite/testcases/src/performance_test.rs +++ b/testsuite/testcases/src/performance_test.rs @@ -27,7 +27,7 @@ impl NetworkLoadTest for PerformanceBenchmark { ) -> Result<()> { let validators = { swarm.read().await.get_validator_clients_with_names() }; // 10 vals, test 1,2,3 failures - let num_bad_leaders = 3; + let num_bad_leaders = 1; for (name, validator) in validators[..num_bad_leaders].iter() { validator .set_failpoint( From d01d497bfbccf96666f0ba7d586153e2ec24b5f0 Mon Sep 17 00:00:00 2001 From: danielxiangzl Date: Wed, 18 Dec 2024 15:14:45 -0800 Subject: [PATCH 05/10] tps --- testsuite/forge-cli/src/suites/realistic_environment.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testsuite/forge-cli/src/suites/realistic_environment.rs b/testsuite/forge-cli/src/suites/realistic_environment.rs index 92da302faca8f..ab38806a51841 100644 --- a/testsuite/forge-cli/src/suites/realistic_environment.rs +++ b/testsuite/forge-cli/src/suites/realistic_environment.rs @@ -83,7 +83,7 @@ pub(crate) fn realistic_env_sweep_wrap( pub(crate) fn realistic_env_load_sweep_test() -> ForgeConfig { realistic_env_sweep_wrap(10, 5, LoadVsPerfBenchmark { test: Box::new(PerformanceBenchmark), - workloads: Workloads::TPS(vec![1000, 2500, 5000, 7500, 10000]), + workloads: Workloads::TPS(vec![1000, 2000, 3000, 3500, 4000]), criteria: [ (95, 0.9, 1.1, 1.2, 0), (95, 0.9, 1.1, 1.2, 0), From 21875f41c20426eb52eb4c4ef23d8a6a88a926de Mon Sep 17 00:00:00 2001 From: danielxiangzl Date: Wed, 18 Dec 2024 15:47:12 -0800 Subject: [PATCH 06/10] tps --- testsuite/forge-cli/src/suites/realistic_environment.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testsuite/forge-cli/src/suites/realistic_environment.rs b/testsuite/forge-cli/src/suites/realistic_environment.rs index ab38806a51841..1c0b2c0a73e6d 100644 --- a/testsuite/forge-cli/src/suites/realistic_environment.rs +++ b/testsuite/forge-cli/src/suites/realistic_environment.rs @@ -83,7 +83,7 @@ pub(crate) fn realistic_env_sweep_wrap( pub(crate) fn realistic_env_load_sweep_test() -> ForgeConfig { realistic_env_sweep_wrap(10, 5, LoadVsPerfBenchmark { test: Box::new(PerformanceBenchmark), - workloads: Workloads::TPS(vec![1000, 2000, 3000, 3500, 4000]), + workloads: Workloads::TPS(vec![1000, 5000, 8000, 8500, 9000]), criteria: [ (95, 0.9, 1.1, 1.2, 0), (95, 0.9, 1.1, 1.2, 0), From 527e90caf71c4fd06e1da7e9da6acace8a1c51f5 Mon Sep 17 00:00:00 2001 From: danielxiangzl Date: Wed, 18 Dec 2024 16:33:27 -0800 Subject: [PATCH 07/10] 3 faults --- testsuite/forge-cli/src/suites/realistic_environment.rs | 2 +- testsuite/testcases/src/performance_test.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/testsuite/forge-cli/src/suites/realistic_environment.rs b/testsuite/forge-cli/src/suites/realistic_environment.rs index 1c0b2c0a73e6d..976e31e21f624 100644 --- a/testsuite/forge-cli/src/suites/realistic_environment.rs +++ b/testsuite/forge-cli/src/suites/realistic_environment.rs @@ -83,7 +83,7 @@ pub(crate) fn realistic_env_sweep_wrap( pub(crate) fn realistic_env_load_sweep_test() -> ForgeConfig { realistic_env_sweep_wrap(10, 5, LoadVsPerfBenchmark { test: Box::new(PerformanceBenchmark), - workloads: Workloads::TPS(vec![1000, 5000, 8000, 8500, 9000]), + workloads: Workloads::TPS(vec![1000, 2000, 2500, 3000, 3500]), criteria: [ (95, 0.9, 1.1, 1.2, 0), (95, 0.9, 1.1, 1.2, 0), diff --git a/testsuite/testcases/src/performance_test.rs b/testsuite/testcases/src/performance_test.rs index d7472c5b67c02..151d5e9bdb3fd 100644 --- a/testsuite/testcases/src/performance_test.rs +++ b/testsuite/testcases/src/performance_test.rs @@ -27,7 +27,7 @@ impl NetworkLoadTest for PerformanceBenchmark { ) -> Result<()> { let validators = { swarm.read().await.get_validator_clients_with_names() }; // 10 vals, test 1,2,3 failures - let num_bad_leaders = 1; + let num_bad_leaders = 3; for (name, validator) in validators[..num_bad_leaders].iter() { validator .set_failpoint( From ae2e0923d932792df99f5c51e8ac006d934aba08 Mon Sep 17 00:00:00 2001 From: danielxiangzl Date: Wed, 8 Jan 2025 21:31:13 -0800 Subject: [PATCH 08/10] hack co for baseline --- .../publisher/consensus_publisher.rs | 15 +++++++++++++++ consensus/src/pipeline/buffer_manager.rs | 6 +++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/consensus/src/consensus_observer/publisher/consensus_publisher.rs b/consensus/src/consensus_observer/publisher/consensus_publisher.rs index 899901593f7ed..85b3d5a26e144 100644 --- a/consensus/src/consensus_observer/publisher/consensus_publisher.rs +++ b/consensus/src/consensus_observer/publisher/consensus_publisher.rs @@ -17,9 +17,11 @@ use crate::consensus_observer::{ }; use aptos_channels::aptos_channel::Receiver; use aptos_config::{config::ConsensusObserverConfig, network_id::PeerNetworkId}; +use aptos_crypto::HashValue; use aptos_infallible::RwLock; use aptos_logger::{error, info, warn}; use aptos_network::application::interface::NetworkClient; +use dashmap::DashMap; use futures::StreamExt; use futures_channel::mpsc; use std::{collections::HashSet, sync::Arc, time::Duration}; @@ -41,6 +43,8 @@ pub struct ConsensusPublisher { // The sender for outbound network messages outbound_message_sender: mpsc::Sender<(PeerNetworkId, ConsensusObserverDirectSend)>, + + buffered_ordered_block: DashMap, } impl ConsensusPublisher { @@ -64,6 +68,7 @@ impl ConsensusPublisher { consensus_observer_config, active_subscribers: Arc::new(RwLock::new(HashSet::new())), outbound_message_sender, + buffered_ordered_block: DashMap::new(), }; // Return the publisher and the outbound message receiver @@ -231,6 +236,16 @@ impl ConsensusPublisher { } } + pub fn buffer_ordered_block(&self, block_id: HashValue, message: ConsensusObserverDirectSend) { + self.buffered_ordered_block.insert(block_id, message); + } + + pub fn flush_buffered_ordered_block(&self, block_id: HashValue) { + if let Some((_, message)) = self.buffered_ordered_block.remove(&block_id) { + self.publish_message(message); + } + } + /// Starts the consensus publisher pub async fn start( self, diff --git a/consensus/src/pipeline/buffer_manager.rs b/consensus/src/pipeline/buffer_manager.rs index 8325c64feaab9..4ba1d8ae3ee32 100644 --- a/consensus/src/pipeline/buffer_manager.rs +++ b/consensus/src/pipeline/buffer_manager.rs @@ -411,7 +411,9 @@ impl BufferManager { ordered_blocks.clone().into_iter().map(Arc::new).collect(), ordered_proof.clone(), ); - consensus_publisher.publish_message(message); + // consensus_publisher.publish_message(message); + let block_id = ordered_blocks.last().unwrap().id(); + consensus_publisher.buffer_ordered_block(block_id, message); } self.execution_schedule_phase_tx .send(request) @@ -526,6 +528,8 @@ impl BufferManager { self.reset().await; } if let Some(consensus_publisher) = &self.consensus_publisher { + let block_id = block.id(); + consensus_publisher.flush_buffered_ordered_block(block_id); let message = ConsensusObserverMessage::new_commit_decision_message(commit_proof.clone()); consensus_publisher.publish_message(message); From f06b5140cab056865479b2cda102f14a604c092c Mon Sep 17 00:00:00 2001 From: danielxiangzl Date: Wed, 8 Jan 2025 21:35:08 -0800 Subject: [PATCH 09/10] enable co --- config/src/config/consensus_observer_config.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/src/config/consensus_observer_config.rs b/config/src/config/consensus_observer_config.rs index 9351a5af94702..a4a1dc9068abf 100644 --- a/config/src/config/consensus_observer_config.rs +++ b/config/src/config/consensus_observer_config.rs @@ -9,9 +9,9 @@ use serde::{Deserialize, Serialize}; use serde_yaml::Value; // Useful constants for enabling consensus observer on different node types -const ENABLE_ON_VALIDATORS: bool = false; -const ENABLE_ON_VALIDATOR_FULLNODES: bool = false; -const ENABLE_ON_PUBLIC_FULLNODES: bool = false; +const ENABLE_ON_VALIDATORS: bool = true; +const ENABLE_ON_VALIDATOR_FULLNODES: bool = true; +const ENABLE_ON_PUBLIC_FULLNODES: bool = true; #[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize)] #[serde(default, deny_unknown_fields)] From 0c1d51b20a4d415cdce898f67bdd82750063202b Mon Sep 17 00:00:00 2001 From: danielxiangzl Date: Thu, 9 Jan 2025 10:28:42 -0800 Subject: [PATCH 10/10] 1 fault --- testsuite/forge-cli/src/suites/realistic_environment.rs | 2 +- testsuite/testcases/src/performance_test.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/testsuite/forge-cli/src/suites/realistic_environment.rs b/testsuite/forge-cli/src/suites/realistic_environment.rs index 976e31e21f624..1c0b2c0a73e6d 100644 --- a/testsuite/forge-cli/src/suites/realistic_environment.rs +++ b/testsuite/forge-cli/src/suites/realistic_environment.rs @@ -83,7 +83,7 @@ pub(crate) fn realistic_env_sweep_wrap( pub(crate) fn realistic_env_load_sweep_test() -> ForgeConfig { realistic_env_sweep_wrap(10, 5, LoadVsPerfBenchmark { test: Box::new(PerformanceBenchmark), - workloads: Workloads::TPS(vec![1000, 2000, 2500, 3000, 3500]), + workloads: Workloads::TPS(vec![1000, 5000, 8000, 8500, 9000]), criteria: [ (95, 0.9, 1.1, 1.2, 0), (95, 0.9, 1.1, 1.2, 0), diff --git a/testsuite/testcases/src/performance_test.rs b/testsuite/testcases/src/performance_test.rs index 151d5e9bdb3fd..d7472c5b67c02 100644 --- a/testsuite/testcases/src/performance_test.rs +++ b/testsuite/testcases/src/performance_test.rs @@ -27,7 +27,7 @@ impl NetworkLoadTest for PerformanceBenchmark { ) -> Result<()> { let validators = { swarm.read().await.get_validator_clients_with_names() }; // 10 vals, test 1,2,3 failures - let num_bad_leaders = 3; + let num_bad_leaders = 1; for (name, validator) in validators[..num_bad_leaders].iter() { validator .set_failpoint(