fix benchmark dataset (#1155)

- allow upgrading the dataset dependencies through npm
- allow running benchmark tests along regular unit tests, so that they
are tested regularly in CI
- Fixed https://github.com/NomicFoundation/slang/security/dependabot/53
- Fixed https://github.com/NomicFoundation/slang/security/dependabot/54

.github/dependabot.yml

@@ -34,9 +34,6 @@ updates:
         update-types:
           - "minor"
           - "patch"
-    ignore:
-      # openzeppelin contracts used in perf tests. don't update/change, to prevent variations/failures in results:
-      - dependency-name: "@openzeppelin/contracts"
 
   - package-ecosystem: "github-actions"
     directory: "/"

crates/infra/cli/package.json

@@ -4,7 +4,7 @@
   "devDependencies": {
     "@actions/github": "6.0.0",
     "@changesets/changelog-github": "0.5.0",
-    "@changesets/cli": "2.27.9",
+    "@changesets/cli": "2.27.10",
     "@types/node": "22.8.6",
     "cspell": "8.15.5",
     "markdown-link-check": "3.12.2",

crates/solidity/testing/perf/Cargo.toml

@@ -5,15 +5,17 @@ rust-version.workspace = true
 edition.workspace = true
 publish = false
 
-[dev-dependencies]
-iai-callgrind = { workspace = true }
+[dependencies]
 infra_utils = { workspace = true }
 semver = { workspace = true }
 metaslang_bindings = { workspace = true }
 slang_solidity = { workspace = true, features = [
   "__experimental_bindings_api",
 ] }
 
+[dev-dependencies]
+iai-callgrind = { workspace = true }
+
 [[bench]]
 name = "iai"
 harness = false

crates/solidity/testing/perf/benches/iai/main.rs

@@ -1,14 +1,6 @@
-//! WARNING:
-//! The reported `iai` benchmark ID is constructed from: `{file_name}::{group_name}::{function_name}`
-//! For the function below: `iai::benchmarks::list_contracts`
-//! Changing any of the above would change the resulting benchmark ID, and disconnect it from previous results.
-
 #![allow(clippy::exit)]
 #![allow(clippy::needless_pass_by_value)]
 
-mod dataset;
-mod tests;
-
 use std::hint::black_box;
 
 use iai_callgrind::{
@@ -17,20 +9,30 @@ use iai_callgrind::{
 };
 use slang_solidity::bindings::Bindings;
 use slang_solidity::parser::ParseOutput;
+use solidity_testing_perf::dataset::SourceFile;
+use solidity_testing_perf::tests::definitions::Dependencies;
+use solidity_testing_perf::tests::parser::ParsedFile;
 
-use crate::dataset::SourceFile;
-use crate::tests::definitions::Dependencies;
-use crate::tests::parser::ParsedFile;
+mod __dependencies_used_in_lib__ {
+    use {infra_utils as _, metaslang_bindings as _, semver as _};
+}
 
 macro_rules! define_benchmark {
     ($name:ident, $payload:ty) => {
-        #[library_benchmark(setup = tests::$name::setup)]
+        #[library_benchmark(setup = solidity_testing_perf::tests::$name::setup)]
         fn $name(payload: $payload) {
-            black_box(tests::$name::run(payload));
+            black_box(solidity_testing_perf::tests::$name::run(payload));
         }
     };
 }
 
+/*
+ * WARNING:
+ * The reported `iai` benchmark ID is constructed from: `{file_name}::{group_name}::{function_name}`
+ * Changing any of the above would change the resulting benchmark ID, and disconnect it from previous results.
+ *
+ * __SLANG_INFRA_BENCHMARKS_LIST__ (keep in sync)
+ */
 define_benchmark!(parser, Vec<SourceFile>);
 define_benchmark!(cursor, Vec<ParsedFile>);
 define_benchmark!(query, Vec<ParsedFile>);
@@ -41,6 +43,7 @@ define_benchmark!(references, Bindings);
 library_benchmark_group!(
     name = benchmarks;
 
+    // __SLANG_INFRA_BENCHMARKS_LIST__ (keep in sync)
     benchmarks = parser, cursor, query, init_bindings, definitions, references
 );
 

crates/solidity/testing/perf/package.json

@@ -2,6 +2,6 @@
   "name": "@slang-private/solidity-testing-perf",
   "private": true,
   "devDependencies": {
-    "@openzeppelin/contracts": "5.0.0"
+    "@openzeppelin/contracts": "5.1.0"
   }
 }

crates/solidity/testing/perf/benches/iai/dataset.rs → crates/solidity/testing/perf/src/dataset.rs

@@ -1,6 +1,5 @@
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
-use infra_utils::cargo::CargoWorkspace;
 use infra_utils::paths::PathExtensions;
 use semver::Version;
 
@@ -21,12 +20,10 @@ pub struct SourceFile {
 
 impl SourceFile {
     pub fn load_all() -> Vec<Self> {
-        let crate_dir = CargoWorkspace::locate_source_crate("solidity_testing_perf").unwrap();
-
         SOURCES
             .iter()
             .map(|relative_path| {
-                let path = crate_dir.join(relative_path);
+                let path = Path::repo_path(relative_path);
                 let contents = path.read_to_string().unwrap();
 
                 Self { path, contents }

crates/solidity/testing/perf/src/lib.rs

@@ -0,0 +1,33 @@
+#![allow(clippy::exit)]
+#![allow(clippy::needless_pass_by_value)]
+
+pub mod dataset;
+pub mod tests;
+
+#[cfg(test)]
+mod __dependencies_used_in_benches__ {
+    use iai_callgrind as _;
+}
+
+#[cfg(test)]
+mod unit_tests {
+    macro_rules! define_test {
+        ($name:ident) => {
+            #[test]
+            fn $name() {
+                let payload = crate::tests::$name::setup();
+                crate::tests::$name::run(payload);
+            }
+        };
+    }
+
+    /*
+     * __SLANG_INFRA_BENCHMARKS_LIST__ (keep in sync)
+     */
+    define_test!(parser);
+    define_test!(cursor);
+    define_test!(query);
+    define_test!(init_bindings);
+    define_test!(definitions);
+    define_test!(references);
+}

crates/solidity/testing/perf/benches/iai/tests/cursor.rs → crates/solidity/testing/perf/src/tests/cursor.rs

@@ -20,7 +20,7 @@ pub fn run(files: Vec<ParsedFile>) {
     }
 
     assert_eq!(
-        functions_count, 200,
+        functions_count, 232,
         "Failed to fetch all function definitions"
     );
 }

crates/solidity/testing/perf/benches/iai/tests/definitions.rs → crates/solidity/testing/perf/src/tests/definitions.rs

@@ -35,11 +35,7 @@ pub fn run(dependencies: Dependencies) -> Bindings {
         definition_count += bindings.all_definitions().count();
     }
 
-    assert!(
-        // TODO(#1077): finalize the assertion counts once bindings are fully implemented:
-        definition_count >= 723,
-        "Only found {definition_count} definitions"
-    );
+    assert_eq!(definition_count, 2832, "Failed to fetch all definitions");
 
     bindings
 }

crates/solidity/testing/perf/benches/iai/tests/query.rs → crates/solidity/testing/perf/src/tests/query.rs

@@ -29,7 +29,7 @@ pub fn run(files: Vec<ParsedFile>) {
     }
 
     assert_eq!(
-        functions_count, 200,
+        functions_count, 232,
         "Failed to fetch all function definitions"
     );
 }

crates/solidity/testing/perf/benches/iai/tests/references.rs → crates/solidity/testing/perf/src/tests/references.rs

@@ -19,15 +19,10 @@ pub fn run(bindings: Bindings) {
         }
     }
 
-    assert!(
-        // TODO(#1077): finalize the assertion counts once bindings are fully implemented:
-        reference_count >= 1491,
-        "Only found {reference_count} references"
-    );
+    assert_eq!(reference_count, 1686, "Failed to fetch all references");
 
-    assert!(
-        // TODO(#1077): finalize the assertion counts once bindings are fully implemented:
-        resolved_references >= 1170,
-        "Only resolved {resolved_references} references"
+    assert_eq!(
+        resolved_references, 1353,
+        "Failed to resolve all references"
     );
 }