Skip to content

Commit

Permalink
feat(invariant): generate failed call sequence as solidity (#9827)
Browse files Browse the repository at this point in the history 
* feat(invariant): generate failed call sequence as solidity

* Fix test, format

* Tests nits
  • Loading branch information
@grandizzy
grandizzy authored Feb 6, 2025
1 parent 867484f commit c4ae688
Show file tree
Hide file tree
Showing 8 changed files with 148 additions and 18 deletions.
4 changes: 4 additions & 0 deletions crates/config/src/invariant.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ pub struct InvariantConfig {
pub show_metrics: bool,
/// Optional timeout (in seconds) for each invariant test.
pub timeout: Option<u32>,
/// Display counterexample as solidity calls.
pub show_solidity: bool,
}

impl Default for InvariantConfig {
Expand All @@ -48,6 +50,7 @@ impl Default for InvariantConfig {
failure_persist_dir: None,
show_metrics: false,
timeout: None,
show_solidity: false,
}
}
}
Expand All @@ -67,6 +70,7 @@ impl InvariantConfig {
failure_persist_dir: Some(cache_dir),
show_metrics: false,
timeout: None,
show_solidity: false,
}
}

Expand Down
4 changes: 4 additions & 0 deletions crates/evm/evm/src/executors/invariant/replay.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ pub fn replay_run(
coverage: &mut Option<HitMaps>,
deprecated_cheatcodes: &mut HashMap<&'static str, Option<&'static str>>,
inputs: &[BasicTxDetails],
show_solidity: bool,
) -> Result<Vec<BaseCounterExample>> {
// We want traces for a failed case.
if executor.inspector().tracer.is_none() {
Expand Down Expand Up @@ -64,6 +65,7 @@ pub fn replay_run(
&tx.call_details.calldata,
&ided_contracts,
call_result.traces,
show_solidity,
));
}

Expand Down Expand Up @@ -110,6 +112,7 @@ pub fn replay_error(
coverage: &mut Option<HitMaps>,
deprecated_cheatcodes: &mut HashMap<&'static str, Option<&'static str>>,
progress: Option<&ProgressBar>,
show_solidity: bool,
) -> Result<Vec<BaseCounterExample>> {
match failed_case.test_error {
// Don't use at the moment.
Expand Down Expand Up @@ -137,6 +140,7 @@ pub fn replay_error(
coverage,
deprecated_cheatcodes,
&calls,
show_solidity,
)
}
}
Expand Down
55 changes: 47 additions & 8 deletions crates/evm/fuzz/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,21 +41,28 @@ pub enum CounterExample {

#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct BaseCounterExample {
/// Address which makes the call
/// Address which makes the call.
pub sender: Option<Address>,
/// Address to which to call to
/// Address to which to call to.
pub addr: Option<Address>,
/// The data to provide
/// The data to provide.
pub calldata: Bytes,
/// Contract name if it exists
/// Contract name if it exists.
pub contract_name: Option<String>,
/// Function signature if it exists
/// Function name if it exists.
pub func_name: Option<String>,
/// Function signature if it exists.
pub signature: Option<String>,
/// Args used to call the function
/// Pretty formatted args used to call the function.
pub args: Option<String>,
/// Traces
/// Unformatted args used to call the function.
pub raw_args: Option<String>,
/// Counter example traces.
#[serde(skip)]
pub traces: Option<SparsedTraceArena>,
/// Whether to display sequence as solidity.
#[serde(skip)]
pub show_solidity: bool,
}

impl BaseCounterExample {
Expand All @@ -66,6 +73,7 @@ impl BaseCounterExample {
bytes: &Bytes,
contracts: &ContractsByAddress,
traces: Option<SparsedTraceArena>,
show_solidity: bool,
) -> Self {
if let Some((name, abi)) = &contracts.get(&addr) {
if let Some(func) = abi.functions().find(|f| f.selector() == bytes[..4]) {
Expand All @@ -76,11 +84,16 @@ impl BaseCounterExample {
addr: Some(addr),
calldata: bytes.clone(),
contract_name: Some(name.clone()),
func_name: Some(func.name.clone()),
signature: Some(func.signature()),
args: Some(
foundry_common::fmt::format_tokens(&args).format(", ").to_string(),
),
raw_args: Some(
foundry_common::fmt::format_tokens_raw(&args).format(", ").to_string(),
),
traces,
show_solidity,
};
}
}
Expand All @@ -91,9 +104,12 @@ impl BaseCounterExample {
addr: Some(addr),
calldata: bytes.clone(),
contract_name: None,
func_name: None,
signature: None,
args: None,
raw_args: None,
traces,
show_solidity: false,
}
}

Expand All @@ -108,17 +124,40 @@ impl BaseCounterExample {
addr: None,
calldata: bytes,
contract_name: None,
func_name: None,
signature: None,
args: Some(foundry_common::fmt::format_tokens(&args).format(", ").to_string()),
raw_args: Some(foundry_common::fmt::format_tokens_raw(&args).format(", ").to_string()),
traces,
show_solidity: false,
}
}
}

impl fmt::Display for BaseCounterExample {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
// Display counterexample as solidity.
if self.show_solidity {
if let (Some(sender), Some(contract), Some(address), Some(func_name), Some(args)) =
(&self.sender, &self.contract_name, &self.addr, &self.func_name, &self.raw_args)
{
writeln!(f, "\t\tvm.prank({sender});")?;
write!(
f,
"\t\t{}({}).{}({});",
contract.split_once(':').map_or(contract.as_str(), |(_, contract)| contract),
address,
func_name,
args
)?;

return Ok(())
}
}

// Regular counterexample display.
if let Some(sender) = self.sender {
write!(f, "sender={sender} addr=")?
write!(f, "\t\tsender={sender} addr=")?
}

if let Some(name) = &self.contract_name {
Expand Down
2 changes: 1 addition & 1 deletion crates/forge/src/result.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -455,7 +455,7 @@ impl fmt::Display for TestResult {
.as_str(),
);
for ex in sequence {
writeln!(s, "\t\t{ex}").unwrap();
writeln!(s, "{ex}").unwrap();
}
}
}
Expand Down
23 changes: 15 additions & 8 deletions crates/forge/src/runner.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -581,20 +581,24 @@ impl<'a> FunctionRunner<'a> {

let failure_dir = invariant_config.clone().failure_dir(self.cr.name);
let failure_file = failure_dir.join(&invariant_contract.invariant_function.name);
let show_solidity = invariant_config.clone().show_solidity;

// Try to replay recorded failure if any.
if let Ok(call_sequence) =
if let Ok(mut call_sequence) =
foundry_common::fs::read_json_file::<Vec<BaseCounterExample>>(failure_file.as_path())
{
// Create calls from failed sequence and check if invariant still broken.
let txes = call_sequence
.iter()
.map(|seq| BasicTxDetails {
sender: seq.sender.unwrap_or_default(),
call_details: CallDetails {
target: seq.addr.unwrap_or_default(),
calldata: seq.calldata.clone(),
},
.iter_mut()
.map(|seq| {
seq.show_solidity = show_solidity;
BasicTxDetails {
sender: seq.sender.unwrap_or_default(),
call_details: CallDetails {
target: seq.addr.unwrap_or_default(),
calldata: seq.calldata.clone(),
},
}
})
.collect::<Vec<BasicTxDetails>>();
if let Ok((success, replayed_entirely)) = check_sequence(
Expand Down Expand Up @@ -624,6 +628,7 @@ impl<'a> FunctionRunner<'a> {
&mut self.result.coverage,
&mut self.result.deprecated_cheatcodes,
&txes,
show_solidity,
);
self.result.invariant_replay_fail(
replayed_entirely,
Expand Down Expand Up @@ -674,6 +679,7 @@ impl<'a> FunctionRunner<'a> {
&mut self.result.coverage,
&mut self.result.deprecated_cheatcodes,
progress.as_ref(),
show_solidity,
) {
Ok(call_sequence) => {
if !call_sequence.is_empty() {
Expand Down Expand Up @@ -719,6 +725,7 @@ impl<'a> FunctionRunner<'a> {
&mut self.result.coverage,
&mut self.result.deprecated_cheatcodes,
&invariant_result.last_run_inputs,
show_solidity,
) {
error!(%err, "Failed to replay last invariant run");
}
Expand Down
4 changes: 3 additions & 1 deletion crates/forge/tests/cli/config.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1090,6 +1090,7 @@ max_assume_rejects = 65536
gas_report_samples = 256
failure_persist_dir = "cache/invariant"
show_metrics = false
show_solidity = false
[labels]
Expand Down Expand Up @@ -1193,7 +1194,8 @@ exclude = []
"gas_report_samples": 256,
"failure_persist_dir": "cache/invariant",
"show_metrics": false,
"timeout": null
"timeout": null,
"show_solidity": false
},
"ffi": false,
"allow_internal_expect_revert": false,
Expand Down
73 changes: 73 additions & 0 deletions crates/forge/tests/it/invariant.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1066,6 +1066,7 @@ contract InvariantSelectorsWeightTest is Test {
});

// Tests original and new counterexample lengths are displayed on failure.
// Tests switch from regular sequence output to solidity.
forgetest_init!(invariant_sequence_len, |prj, cmd| {
prj.update_config(|config| {
config.fuzz.seed = Some(U256::from(100u32));
Expand Down Expand Up @@ -1099,4 +1100,76 @@ contract InvariantSequenceLenTest is Test {
[Sequence] (original: 4, shrunk: 1)
...
"#]]);

// Check regular sequence output. Shrink disabled to show several lines.
cmd.forge_fuse().arg("clean").assert_success();
prj.update_config(|config| {
config.invariant.shrink_run_limit = 0;
});
cmd.forge_fuse().args(["test", "--mt", "invariant_increment"]).assert_failure().stdout_eq(
str![[r#"
...
Failing tests:
Encountered 1 failing test in test/InvariantSequenceLenTest.t.sol:InvariantSequenceLenTest
[FAIL: revert: invariant increment failure]
[Sequence] (original: 4, shrunk: 4)
sender=0x00000000000000000000000000000000000018dE addr=[src/Counter.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=setNumber(uint256) args=[1931387396117645594923 [1.931e21]]
sender=0x00000000000000000000000000000000000009d5 addr=[src/Counter.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=increment() args=[]
sender=0x0000000000000000000000000000000000000105 addr=[src/Counter.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=increment() args=[]
sender=0x00000000000000000000000000000000000009B2 addr=[src/Counter.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=setNumber(uint256) args=[996881781832960761274744263729582347 [9.968e35]]
invariant_increment() (runs: 0, calls: 0, reverts: 0)
Encountered a total of 1 failing tests, 0 tests succeeded
"#]],
);

// Check solidity sequence output on same failure.
cmd.forge_fuse().arg("clean").assert_success();
prj.update_config(|config| {
config.invariant.show_solidity = true;
});
cmd.forge_fuse().args(["test", "--mt", "invariant_increment"]).assert_failure().stdout_eq(
str![[r#"
...
Failing tests:
Encountered 1 failing test in test/InvariantSequenceLenTest.t.sol:InvariantSequenceLenTest
[FAIL: revert: invariant increment failure]
[Sequence] (original: 4, shrunk: 4)
vm.prank(0x00000000000000000000000000000000000018dE);
Counter(0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f).setNumber(1931387396117645594923);
vm.prank(0x00000000000000000000000000000000000009d5);
Counter(0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f).increment();
vm.prank(0x0000000000000000000000000000000000000105);
Counter(0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f).increment();
vm.prank(0x00000000000000000000000000000000000009B2);
Counter(0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f).setNumber(996881781832960761274744263729582347);
invariant_increment() (runs: 0, calls: 0, reverts: 0)
Encountered a total of 1 failing tests, 0 tests succeeded
"#]],
);

// Persisted failures should be able to switch output.
prj.update_config(|config| {
config.invariant.show_solidity = false;
});
cmd.forge_fuse().args(["test", "--mt", "invariant_increment"]).assert_failure().stdout_eq(
str![[r#"
...
Failing tests:
Encountered 1 failing test in test/InvariantSequenceLenTest.t.sol:InvariantSequenceLenTest
[FAIL: invariant_increment replay failure]
[Sequence] (original: 4, shrunk: 4)
sender=0x00000000000000000000000000000000000018dE addr=[src/Counter.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=setNumber(uint256) args=[1931387396117645594923 [1.931e21]]
sender=0x00000000000000000000000000000000000009d5 addr=[src/Counter.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=increment() args=[]
sender=0x0000000000000000000000000000000000000105 addr=[src/Counter.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=increment() args=[]
sender=0x00000000000000000000000000000000000009B2 addr=[src/Counter.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=setNumber(uint256) args=[996881781832960761274744263729582347 [9.968e35]]
invariant_increment() (runs: 1, calls: 1, reverts: 1)
Encountered a total of 1 failing tests, 0 tests succeeded
"#]],
);
});
1 change: 1 addition & 0 deletions crates/forge/tests/it/test_helpers.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ impl ForgeTestProfile {
),
show_metrics: false,
timeout: None,
show_solidity: false,
};

config.sanitized()
Expand Down

0 comments on commit c4ae688

Please sign in to comment.