Skip to content

Commit

Permalink
Add performance tests for bindings API (#1100)
Browse files Browse the repository at this point in the history
This PR adds performance tests to exercise the bindings API:

- `create_bindings` creates a blank Bindings object, useful as a
baseline and to later measure the cost of adding the built-ins
- `bindings` which construct the stack graphs and other related bindings
structures for the performance testing dataset files
- `resolve_references` attempts to resolve all the references found in
the dataset files

---------

Co-authored-by: OmarTawfik <[email protected]>
  • Loading branch information
ggiraldez and OmarTawfik authored Sep 26, 2024
1 parent 15c437c commit 828eff3
Show file tree
Hide file tree
Showing 19 changed files with 275 additions and 3,584 deletions.
3 changes: 3 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ updates:
update-types:
- "minor"
- "patch"
ignore:
# openzeppelin contracts used in perf tests. don't update/change, to prevent variations/failures in results:
- dependency-name: "@openzeppelin/contracts"

- package-ecosystem: "github-actions"
directory: "/"
Expand Down
2 changes: 2 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 5 additions & 1 deletion crates/solidity/testing/perf/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,12 @@ publish = false

[dev-dependencies]
iai-callgrind = { workspace = true }
infra_utils = { workspace = true }
semver = { workspace = true }
slang_solidity = { workspace = true }
metaslang_bindings = { workspace = true }
slang_solidity = { workspace = true, features = [
"__experimental_bindings_api",
] }

[[bench]]
name = "iai"
Expand Down
255 changes: 23 additions & 232 deletions crates/solidity/testing/perf/benches/iai/dataset.rs
Original file line number Diff line number Diff line change
@@ -1,245 +1,36 @@
use std::collections::BTreeSet;
use std::path::PathBuf;

use infra_utils::cargo::CargoWorkspace;
use infra_utils::paths::PathExtensions;
use semver::Version;
use slang_solidity::cst::Node;
use slang_solidity::kinds::{EdgeLabel, NonterminalKind};
use slang_solidity::language::Language;
use slang_solidity::query::Query;
use slang_solidity::text_index::TextIndex;

const SOLC_VERSION: Version = Version::new(0, 8, 20);
pub const SOLC_VERSION: Version = Version::new(0, 8, 20);

const SOURCES: &[&str] = &[
include_str!("./sources/EnumerableMap.sol"),
include_str!("./sources/ERC20.sol"),
include_str!("./sources/ERC721.sol"),
include_str!("./sources/Governor.sol"),
include_str!("./sources/SafeCast.sol"),
"node_modules/@openzeppelin/contracts/governance/Governor.sol",
"node_modules/@openzeppelin/contracts/token/ERC20/ERC20.sol",
"node_modules/@openzeppelin/contracts/token/ERC721/ERC721.sol",
"node_modules/@openzeppelin/contracts/utils/math/SafeCast.sol",
"node_modules/@openzeppelin/contracts/utils/structs/EnumerableMap.sol",
];

const FUNCTION_NAMES: &[&str] = &[
"CLOCK_MODE",
"_approve",
"_baseURI",
"_burn",
"_cancel",
"_castVote",
"_checkAuthorized",
"_checkGovernance",
"_checkOnERC721Received",
"_countVote",
"_defaultParams",
"_encodeStateBitmap",
"_executeOperations",
"_executor",
"_getApproved",
"_getVotes",
"_increaseBalance",
"_isAuthorized",
"_isValidDescriptionForProposer",
"_mint",
"_ownerOf",
"_propose",
"_queueOperations",
"_quorumReached",
"_requireOwned",
"_safeMint",
"_safeTransfer",
"_setApprovalForAll",
"_spendAllowance",
"_transfer",
"_tryHexToUint",
"_update",
"_validateStateBitmap",
"_voteSucceeded",
"allowance",
"approve",
"at",
"balanceOf",
"cancel",
"castVote",
"castVoteBySig",
"castVoteWithReason",
"castVoteWithReasonAndParams",
"castVoteWithReasonAndParamsBySig",
"clock",
"contains",
"decimals",
"execute",
"get",
"getApproved",
"getVotes",
"getVotesWithParams",
"hashProposal",
"isApprovedForAll",
"keys",
"length",
"name",
"onERC1155BatchReceived",
"onERC1155Received",
"onERC721Received",
"ownerOf",
"proposalDeadline",
"proposalEta",
"proposalNeedsQueuing",
"proposalProposer",
"proposalSnapshot",
"proposalThreshold",
"propose",
"queue",
"quorum",
"relay",
"remove",
"safeTransferFrom",
"set",
"setApprovalForAll",
"state",
"supportsInterface",
"symbol",
"toInt104",
"toInt112",
"toInt120",
"toInt128",
"toInt136",
"toInt144",
"toInt152",
"toInt16",
"toInt160",
"toInt168",
"toInt176",
"toInt184",
"toInt192",
"toInt200",
"toInt208",
"toInt216",
"toInt224",
"toInt232",
"toInt24",
"toInt240",
"toInt248",
"toInt256",
"toInt32",
"toInt40",
"toInt48",
"toInt56",
"toInt64",
"toInt72",
"toInt8",
"toInt80",
"toInt88",
"toInt96",
"toUint104",
"toUint112",
"toUint120",
"toUint128",
"toUint136",
"toUint144",
"toUint152",
"toUint16",
"toUint160",
"toUint168",
"toUint176",
"toUint184",
"toUint192",
"toUint200",
"toUint208",
"toUint216",
"toUint224",
"toUint232",
"toUint24",
"toUint240",
"toUint248",
"toUint256",
"toUint32",
"toUint40",
"toUint48",
"toUint56",
"toUint64",
"toUint72",
"toUint8",
"toUint80",
"toUint88",
"toUint96",
"tokenURI",
"totalSupply",
"transfer",
"transferFrom",
"tryGet",
"version",
"votingDelay",
"votingPeriod",
];

pub fn run_parser() -> Vec<Node> {
let language = Language::new(SOLC_VERSION).unwrap();

let mut trees = vec![];

for source in SOURCES {
let parse_output = language.parse(Language::ROOT_KIND, source);

assert!(
parse_output.is_valid(),
"Found parse errors:\n{0:#?}",
parse_output.errors(),
);

trees.push(parse_output.tree());
}

trees
pub struct SourceFile {
pub path: PathBuf,
pub contents: String,
}

pub fn run_cursor(trees: &Vec<Node>) {
let mut results = BTreeSet::new();
impl SourceFile {
pub fn load_all() -> Vec<Self> {
let crate_dir = CargoWorkspace::locate_source_crate("solidity_testing_perf").unwrap();

for tree in trees {
let mut cursor = tree.cursor_with_offset(TextIndex::ZERO);
SOURCES
.iter()
.map(|relative_path| {
let path = crate_dir.join(relative_path);
let contents = path.read_to_string().unwrap();

while cursor.go_to_next_nonterminal_with_kind(NonterminalKind::FunctionDefinition) {
results.extend(
cursor
.node()
.children()
.iter()
.filter(|edge| edge.label == Some(EdgeLabel::Name))
.map(|edge| edge.node.clone().unparse().trim().to_owned()),
);
}
Self { path, contents }
})
.collect()
}

assert!(
results.iter().eq(FUNCTION_NAMES.iter()),
"Function names don't match: {results:#?}"
);
}

pub fn run_query(trees: &Vec<Node>) {
let mut results = BTreeSet::new();

let queries = vec![Query::parse(
"[FunctionDefinition
@name name: [_]
]",
)
.unwrap()];

for tree in trees {
let cursor = tree.cursor_with_offset(TextIndex::ZERO);

for query_match in cursor.query(queries.clone()) {
assert_eq!(query_match.captures.len(), 1);

results.extend(
query_match.captures["name"]
.iter()
.map(|cursor| cursor.node().unparse().trim().to_owned()),
);
}
}

assert!(
results.iter().eq(FUNCTION_NAMES.iter()),
"Function names don't match: {results:#?}"
);
}
36 changes: 25 additions & 11 deletions crates/solidity/testing/perf/benches/iai/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,34 +8,48 @@
#![allow(clippy::unit_arg)]

mod dataset;
mod tests;

use std::hint::black_box;

use iai_callgrind::{
library_benchmark, library_benchmark_group, main, Direction, FlamegraphConfig,
LibraryBenchmarkConfig, Tool, ValgrindTool,
};
use slang_solidity::cst::Node;
use slang_solidity::bindings::Bindings;

#[library_benchmark]
fn parser() {
black_box(dataset::run_parser());
use crate::dataset::SourceFile;
use crate::tests::parser::ParsedFile;

#[library_benchmark(setup = tests::parser::setup)]
fn parser(files: Vec<SourceFile>) {
black_box(tests::parser::run(files));
}

#[library_benchmark(setup = tests::cursor::setup)]
fn cursor(files: Vec<ParsedFile>) {
black_box(tests::cursor::run(&files));
}

#[library_benchmark(setup = tests::query::setup)]
fn query(files: Vec<ParsedFile>) {
black_box(tests::query::run(&files));
}

#[library_benchmark(setup = dataset::run_parser)]
fn cursor(trees: Vec<Node>) {
black_box(dataset::run_cursor(&trees));
#[library_benchmark(setup = tests::definitions::setup)]
fn definitions(files: Vec<ParsedFile>) {
black_box(tests::definitions::run(&files));
}

#[library_benchmark(setup = dataset::run_parser)]
fn query(trees: Vec<Node>) {
black_box(dataset::run_query(&trees));
#[library_benchmark(setup = tests::references::setup)]
fn references(bindings: Bindings) {
black_box(tests::references::run(&bindings));
}

library_benchmark_group!(
name = benchmarks;

benchmarks = parser, cursor, query
benchmarks = parser, cursor, query, definitions, references
);

main!(
Expand Down
Loading

0 comments on commit 828eff3

Please sign in to comment.